- broken - reimplementing z85 decoder, using hyphens again, doesn't work yet...

TODO

@@ -16,30 +16,7 @@ enable formats for secret key exports as well
 
 Add newlines to headers in define.h, so strlen() later catches the whole length.
 
-Z85 headers:
+Z85 Stream encode: add newline after last.
- - currently I use "----- BEGIN ... -----" and "----- END ... -----" as
-   header and footer for various z85 encoded outputs. The problem is, that
-   the "-" character is part of Z85 chars. An input of 0xc6,0x5a,0x0b,0x13 would
-   result z85 encoded as: "-----". So, I cannot be sure, when I find a header
-   delimiter, if it's really a delimiter or legitimate z85 encoded content.
-   Therefore, another delimiter must be used. "~~~~~ BEGIN .... ~~~~~" seems
-   to fit best and "~" is unused in Z85. 
-   Then the parser can be enhanced as well. Eg: on startup if a ~ occurs,
-   ignore input until the first non-~ appears. Then decode input until a
-   ~ or eof appears, ignore everything after. Comments would still be a
-   problem though. Currently I ignore lines containing whitespaces. But
-   if a file is read blockwise and the blocksize is very small, then a
-   comment line may span multiple blocks and isn't recognizable as a
-   "line" anymore. Maybe, comments shall start and end with a ~ as well, eg:
-   ~ BEGIN KEY ~
-   ~ Hash: 987298347 ~
-   [z85]
-   ~ END KEY ~
-   Here I use the same aproach for the headers, since there would also be
-   the problem how to recognize them properly if a header crosses boundaries
-   or something. By using this scheme, if a ~ is found everything following
-   is marked as to be ignored which could be saved as a state when using
-   blockmode.
 
 Check is_utf8 license.
   also found in https://gd.meizo.com/_files/lpc/ext/utf8.c

include/pcp.h

@@ -8,6 +8,7 @@ extern "C" {
 #include "pcp/config.h"
 #include "pcp/base85.h"
 #include "pcp/buffer.h"
+#include "pcp/config.h"
 #include "pcp/crypto.h"
 #include "pcp/defines.h"
 #include "pcp/digital_crc32.h"

include/pcp/defines.h

@@ -74,6 +74,15 @@ typedef unsigned int    qbyte;          /*   Quad byte = 32 bits */
 #define PCP_KEY_VERSION 6
 #define PCP_KEY_PRIMITIVE "CURVE25519-ED25519-SALSA20-POLY1305"
 
+typedef enum _ZBEGINS {
+  PCP_ENCRYPTED_FILE,
+  Z85_ENCODED_FILE,
+  ED25519_SIGNED_MESSAGE,
+  ED25519_SIGNATURE,
+  ED25519_CURVE29915_PUBLIC_KEY,
+  ED25519_CURVE29915_PRIVATE_KEY,
+} ZBEGINS;
+
 /** 
     \addtogroup KEYS
     @{

include/pcp/pcpstream.h

@@ -62,6 +62,7 @@ struct _pcp_stream_t {
   Buffer *b;         /**< The backend Buffer object */
   Buffer *cache;     /**< The caching Buffer object (for look ahead read) */
   Buffer *next;      /**< The caching Next-Buffer object (for look ahead read) */
+  Buffer *save;      /**< Temporary buffer to backup overflow data */
   uint8_t is_buffer; /**< Set to 1 if the backend is a Buffer */
   uint8_t eof;       /**< Set to 1 if EOF reached */
   uint8_t err;       /**< Set to 1 if an error occured */
@@ -71,9 +72,15 @@ struct _pcp_stream_t {
   size_t  linewr;    /**< Used for Z85 writing, number of chars written on last line */
   size_t  blocksize; /**< Blocksize used for z85, if requested */
   uint8_t is_output; /**< marks the stream as output stream */
+  uint8_t have_begin; /**< flag to indicate we already got the begin header, if any */
   size_t pos;        /**< remember i/o position */
 };
 
+typedef enum _PSVARS {
+  PSMAXLINE = 20000
+} PSVARS;
+
+
 /** The name used everywhere */
 typedef struct _pcp_stream_t Pcpstream;
 
@@ -268,7 +275,7 @@ void ps_unarmor(Pcpstream *stream);
 /* read from primary source, decode z85 and out into cache.
    if buf != NULL, consider it as the start of encoded data
    and remove headers and comments, then continue as normal. */
-size_t ps_read_decode(Pcpstream *stream, void *buf, size_t bufsize);
+size_t ps_read_decode(Pcpstream *stream);
 
 /* determine if primary source is z85 encoded, put the data
    read from it into the cache */
@@ -294,6 +301,16 @@ size_t ps_write_buf(Pcpstream *stream, Buffer *z);
 /* tell if we really reached eof, caching or not. 1=eof, 0=ok */
 int ps_left(Pcpstream *stream);
 
+/** Read a line from the stream.
+
+    \param[in] stream The stream object.
+    \param[out] line Linecontent will be written to this Buffer.
+
+    \return Returns the number of bytes read or -1 if PSMAXLINE have been
+            reached or the input doesn't have any newlines at all.
+ */
+int ps_readline(Pcpstream *stream, Buffer *line);
+
 #endif // HAVE_PCP_PCPSTEAM_H
 
 

include/pcp/util.h

@@ -74,7 +74,7 @@ char *_lc(char *in);
     \return Returns the offset or -1 of the offset were not found.
 
  */
-size_t _findoffset(byte *bin, size_t binlen, char *sigstart, size_t hlen);
+long int _findoffset(byte *bin, size_t binlen, char *sigstart, size_t hlen);
 
 /** XOR an input buffer with another buffer.
 

include/pcp/z85.h

@@ -156,6 +156,15 @@ size_t _buffer_is_binary(byte *buf, size_t len);
  */
 uint8_t _parse_zchar(Buffer *z, uint8_t c, uint8_t is_comment);
 
+
+long int z85_header_startswith(Buffer *buf, char *what);
+int z85_isheader(Buffer *buf);
+int z85_isend(Buffer *buf);
+int z85_isbegin(Buffer *buf);
+int z85_iscomment(Buffer *buf);
+int z85_isempty(Buffer *line);
+
+
 #endif /*  _HAVE_PCP_Z85_H */
 
 /**@}*/

libpcp/pcpstream.c

@@ -27,6 +27,7 @@ Pcpstream *ps_init(void) {
   stream->cache = NULL;
   stream->next = NULL;
   stream->fd = NULL;
+  stream->save =  buffer_new(32, "Pcpstreamsavebuf");
   stream->is_buffer = 0;
   stream->eof = 0;
   stream->err = 0;
@@ -79,15 +80,46 @@ void ps_unarmor(Pcpstream *stream) {
   stream->armor = 0;
 }
 
+void ps_rewind(Pcpstream *stream, void *buf, size_t bufsize) {
+  if(stream->is_buffer) {
+    stream->b->offset -= bufsize;
+  }
+  else {
+    buffer_clear(stream->save);
+    buffer_add(stream->save, buf, bufsize);
+  }
+  stream->pos -= bufsize;
+  stream->err = 0;
+  stream->eof = 0;
+}
+
 size_t ps_read_raw(Pcpstream *stream, void *buf, size_t readbytes) {
   size_t gotbytes = 0;
+  size_t idx = 0;
+
+  if(buffer_left(stream->save) > 0) {
+    /* something left from last rewind, first use this */
+    if(buffer_left(stream->save) >= readbytes) {
+      gotbytes = buffer_get_chunk(stream->save, buf, readbytes);
+      if(buffer_left(stream->save) == 0)
+	  buffer_clear(stream->save);
+      goto rawdone;
+    }
+    else {
+      /* fetch the remainder of the save buffer, remember how much
+	 to fetch from source next */
+      idx = buffer_get_chunk(stream->save, buf, buffer_left(stream->save));
+      buffer_clear(stream->save);
+      readbytes -= idx;
+    }
+  }
 
   if(stream->is_buffer) {
     /* check if there's enough space in our buffer */
     if(buffer_left(stream->b) < readbytes)
       readbytes = buffer_left(stream->b);
 
-    gotbytes = buffer_get_chunk(stream->b, buf, readbytes);
+    gotbytes += buffer_get_chunk(stream->b, buf+idx, readbytes);
     if(gotbytes == 0) {
       /* this should not happen with buffers */
       stream->eof = 1;
@@ -95,16 +127,15 @@ size_t ps_read_raw(Pcpstream *stream, void *buf, size_t readbytes) {
     }
   }
   else {
-    gotbytes = fread(buf, 1, readbytes, stream->fd);
+    size_t got = fread(buf+idx, 1, readbytes, stream->fd);
-    if(gotbytes == 0) {
+    gotbytes += got;
-      if(feof(stream->fd) != 0)
+    if(feof(stream->fd) != 0)
-	stream->eof = 1;
+      stream->eof = 1;
-      if(ferror(stream->fd) != 0)
+    if(ferror(stream->fd) != 0)
-	stream->err = 1;
+      stream->err = 1;
-    }
   }
 
-  stream->firstread = 1;
+ rawdone:
   return gotbytes;
 }
 
@@ -113,20 +144,23 @@ size_t ps_read_raw(Pcpstream *stream, void *buf, size_t readbytes) {
    that */
 
 size_t ps_read_cached(Pcpstream *stream, void *buf, size_t readbytes) {
-  /*
+  
   fprintf(stderr, "%ld <= %ld && %ld <= %ld\n",
 	  readbytes, buffer_left(stream->cache), readbytes, stream->blocksize) ;
 
   fprintf(stderr, "%d == 1 && %ld >= %ld\n", ps_left(stream), readbytes, buffer_left(stream->cache));
-  */
+  
   if(readbytes <= buffer_left(stream->cache) && readbytes <= stream->blocksize) {
     /* enough left in current cache */
+    fprintf(stderr, "  get all from cache\n");
     return buffer_get_chunk(stream->cache, buf, readbytes);
   }
-  else if(ps_left(stream) == 1 && readbytes >= buffer_left(stream->cache)) {
+  else if(ps_end(stream) == 1 && readbytes >= buffer_left(stream->cache) ) {
+    fprintf(stderr, "  get rest from cache\n");
     return buffer_get_chunk(stream->cache, buf, buffer_left(stream->cache));
   }
   else {
+    fprintf(stderr, "  fetch next\n");
      /* request for chunk larger than what we've got in the cache */
     Buffer *tmp = buffer_new(stream->blocksize, "Pcpreadchunktmp");
 
@@ -135,6 +169,8 @@ size_t ps_read_cached(Pcpstream *stream, void *buf, size_t readbytes) {
       buffer_get_chunk_tobuf(stream->cache, tmp, buffer_left(stream->cache)); 
     }
 
+#error EOF reached, cache empty, save filled, doesnt call ps_read_next()
+
     /* how much left to fetch */
     long int left = readbytes - buffer_size(tmp);
 
@@ -184,10 +220,10 @@ size_t ps_read_cached(Pcpstream *stream, void *buf, size_t readbytes) {
 
 /* read and decode the next chunk and put it into stream->next */
 size_t ps_read_next(Pcpstream *stream) {
-  if(ps_left(stream) == 0) {
+  if(ps_left(stream) == 0 || buffer_left(stream->save)) {
     if(stream->armor == 1) {
       /* fetch next chunk and decode it */
-      return ps_read_decode(stream, NULL, 0);
+      return ps_read_decode(stream);
     }
     else {
       /* unencoded source, fetch as is */
@@ -209,6 +245,7 @@ size_t ps_read(Pcpstream *stream, void *buf, size_t readbytes) {
   else if(buffer_size(stream->cache) > 0) {
     /* use cache */
     got = ps_read_cached(stream, buf, readbytes);
+    fprintf(stderr, "%ld = use cache directly\n", got);
   }
   else {
     /* no cache yet */
@@ -217,11 +254,13 @@ size_t ps_read(Pcpstream *stream, void *buf, size_t readbytes) {
          recursively call ps_read() again to return the apropriate data */
       ps_determine(stream);
       got = ps_read(stream, buf, readbytes);
+      fprintf(stderr, "%ld = ps_read(stream, buf, readbytes);\n", got);
     }
     else if(stream->armor == 1) {
       /* z85 encoding has already been determined, therefore the cache
 	 is now filled, use it then */
       got = ps_read_cached(stream, buf, readbytes);
+      fprintf(stderr, "%ld = ps_read_cached(stream, buf, readbytes);\n", got);
     }
     else {
       /* read directly from source */
@@ -230,10 +269,44 @@ size_t ps_read(Pcpstream *stream, void *buf, size_t readbytes) {
   }
 
   stream->pos += got;
-
+  fprintf(stderr, "  ps_read(): %ld\n", got);
   return got;
 }
 
+int ps_readline(Pcpstream *stream, Buffer *line) {
+  int c = 0, max = 1;
+  byte b[1];
+
+  while(c<PSMAXLINE) {
+    if(ps_read_raw(stream, b, 1) < 1) {
+      max = 0;
+      break; /* eof or err */
+    }
+    if(*b == '\r') {
+      continue;
+    }
+    else if(*b == '\n' || ps_left(stream) == 1) {
+      max = 0;
+      break;
+    }
+    else {
+      buffer_add8(line, *b);
+    } 
+    c++;
+  }
+
+  if(max) {
+    /* maxline reached without a newline.
+       backup the data we've got so far
+       for further processing */
+    buffer_add_buf(stream->save, line);
+    buffer_clear(line);
+    return -1;
+  }
+
+  return c;
+}
+
 void ps_determine(Pcpstream *stream) {
   /* read a raw chunk from source */
   void *buf = ucmalloc(stream->blocksize);
@@ -244,8 +317,11 @@ void ps_determine(Pcpstream *stream) {
     /* no, it's armored */
     stream->armor = 1;
 
+    /* put back raw data into read queue */
+    ps_rewind(stream, buf, got);
+
     /* decode the first chunk */
-    ps_read_decode(stream, buf, got);
+    ps_read_decode(stream);
     
     /* put it into the cache */
     buffer_add_buf(stream->cache, stream->next);
@@ -255,42 +331,92 @@ void ps_determine(Pcpstream *stream) {
     /* just put the raw stuff into the cache */
     buffer_add(stream->cache, buf, got);
   }
+
+  ucfree(buf, stream->blocksize);
+
+  stream->firstread = 1;
 }
 
-size_t ps_read_decode(Pcpstream *stream, void *buf, size_t bufsize) {
-  size_t i = 0;
-  uint8_t is_comment = 0;
-  uint8_t c;
-  Buffer *z = buffer_new(32, "ztemp");
-  byte *_buf = buf;
 
-  if(bufsize > 0) {
+size_t ps_read_decode(Pcpstream *stream) {
-    for(i=0; i<bufsize; ++i) {
+  Buffer *z = buffer_new(32, "ztemp");
-      c = _buf[i];
+  Buffer *line = buffer_new_str("line");
-      is_comment = _parse_zchar(z, c, is_comment);
+
-    }
+  buffer_info(stream->save);
+
+  if(buffer_left(stream->save) >= stream->blocksize && stream->firstread == 1) {
+    /* use the save buffer instead */
+    buffer_get_chunk_tobuf(stream->save, z, stream->blocksize);
   }
- 
+  else if(ps_left(stream) == 1 && buffer_left(stream->save) > 0) {
-  if(buffer_size(z) <  stream->blocksize) {
+    /* there's something left which doesn't end in a newline */
-    /* blocksize not full, continue with stream source */
+    buffer_get_chunk_tobuf(stream->save, z, stream->blocksize);
-    /* read in bytewise, ignore newlines and add until the block is full */
+  }
-    while (buffer_size(z) < stream->blocksize) {
+  else {
-      if (ps_read_raw(stream, &c, 1) == 1) {
+    /* continue reading linewise */
-	is_comment = _parse_zchar(z, c, is_comment);
+    while(buffer_size(z) <  stream->blocksize) {
+      buffer_clear(line);
+      if(ps_readline(stream, line) >= 0) {
+	fprintf(stderr, "got: <%s>\n", buffer_get_str(line));	
+	if(z85_isbegin(line) && stream->have_begin == 0) {
+	  /* begin header encountered */
+	  stream->have_begin = 1; /* otherwise ignore it */
+	  continue;
+	}
+	else if(z85_isend(line)) {
+	  /* end header encountered */
+	  break;
+	}
+	else if(z85_isempty(line)) {
+	  /* ignore empty lines */
+	  continue;
+	}
+	else {
+	  /* regular z85 encoded content */
+	  fprintf(stderr, "regular\n");
+	  if(buffer_size(z) + buffer_size(line) > stream->blocksize) {
+	    /* we've got more than needed.
+	       put what we need into z and the remainder
+	       into the save buffer for further reading. */
+	    fprintf(stderr, "overflow %ld + %ld > %ld\n",
+		    buffer_size(z), buffer_size(line), stream->blocksize);
+
+	    buffer_get_chunk_tobuf(line, z, stream->blocksize - buffer_size(z));
+	    buffer_get_chunk_tobuf(line, stream->save, buffer_left(line));
+	    buffer_add8(stream->save, '\n');
+	    break;
+	  }
+	  else {
+	    /* not enough yet, store it and go on */
+	    buffer_add_buf(z, line);
+	  }
+	}
       }
-      else
+      else {
+	/* eof or err */
 	break;
+      }
     }
   }
 
+  fprintf(stderr, "Z: <%s>\n", buffer_get_str(z));
+
   /* finally, decode it and put into next */
   size_t binlen, outlen;
   byte *bin = pcp_z85_decode(buffer_get_str(z), &binlen);
   if(bin == NULL) {
     /* it's not z85 encoded, so threat it as binary */
-    stream->armor = 0;
+    if(stream->firstread) {
-    buffer_add_buf(stream->next, z);
+      /* whoops, we're in the middle of z85 decoding and it failed */
-    outlen = buffer_size(stream->next);
+      stream->eof = 1;
+      stream->err = 1;
+      outlen = 0;
+    }
+    else {
+      stream->armor = 0;
+      buffer_add_buf(stream->next, z);
+      outlen = buffer_size(stream->next);
+    }
   }
   else {
     /* yes, successfully decoded it, put into cache */
@@ -299,12 +425,13 @@ size_t ps_read_decode(Pcpstream *stream, void *buf, size_t bufsize) {
   }
 
   buffer_free(z);
-
+  buffer_free(line);
 
 
   return outlen;
 }
 
+
 size_t ps_write(Pcpstream *stream, void *buf, size_t writebytes) {
   Buffer *z = buffer_new(32, "Pcpwritetemp");
 
@@ -484,6 +611,8 @@ void ps_close(Pcpstream *stream) {
   if(stream->next != NULL)
     buffer_free(stream->next);
 
+  buffer_free(stream->save);
+
   if(stream->is_buffer) {
     buffer_clear(stream->b);
     free(stream);
@@ -498,10 +627,17 @@ void ps_close(Pcpstream *stream) {
 
 int ps_end(Pcpstream *stream) {
   /* simulate open file if there's still something in the cache */
-  if(stream->cache != NULL)
+  if(stream->cache != NULL) {
     if(buffer_left(stream->cache) > 0) {
       return 0;
     }
+  }
+
+  /* if there's a lookahead buffer, do the same */
+  if(buffer_left(stream->save) > 0) {
+    return 0;
+  }
+
   return stream->eof;
 }
 

libpcp/util.c

@@ -31,9 +31,9 @@ char *_lc(char *in) {
 }
 
 /* find the offset of the beginning of a certain string within binary data */
-size_t _findoffset(byte *bin, size_t binlen, char *sigstart, size_t hlen) {
+long int _findoffset(byte *bin, size_t binlen, char *sigstart, size_t hlen) {
   size_t i;
-  size_t offset = 0;
+  long int offset = 0;
   int m = 0;
 
   for(i=0; i<binlen-hlen; ++i) {

libpcp/z85.c

@@ -22,6 +22,19 @@
 
 #include "z85.h"
 
+static char *begins[] = {
+  /* grep -r BEGIN * | egrep "\.h:" | awk -F '-----' '{print $2}' | sed -e 's/.*BEGIN /"/' -e 's/$/",/' */
+  "PCP ENCRYPTED FILE ",
+  "Z85 ENCODED FILE ",
+  "ED25519 SIGNED MESSAGE ",
+  "ED25519 SIGNATURE ",
+  "ED25519-CURVE29915 PUBLIC KEY",
+  "ED25519-CURVE29915 PRIVATE KEY",
+  NULL
+};
+
+
+
 uint8_t is_utf8(const byte *bytes) {
   if( (// non-overlong 2-byte
        (0xC2 <= bytes[0] && bytes[0] <= 0xDF) &&
@@ -293,3 +306,109 @@ char *pcp_readz85string(byte *input, size_t bufsize) {
 
   return NULL;
 }
+
+int z85_isheader(Buffer *buf) {
+  size_t len = buffer_size(buf);
+  byte *line = buffer_get(buf);
+
+  if(len < 15) {
+    /* minimum requirement: "----- END -----" */
+    return 0;
+  }
+  if(memcmp(line, "-----", 5)) {
+    /* doesn't start with hyphens */
+    return 0;
+  }
+
+  if(memcmp(line+(len-5), "-----", 5)) {
+    /* doesn't end with hyphens */
+    return 0;
+  }
+
+  /* true */
+  return 1;
+}
+
+long int z85_header_startswith(Buffer *buf, char *what) {
+  size_t len = buffer_size(buf);
+  byte *line = buffer_get(buf);
+  long int offset = 0;
+
+  if((offset = _findoffset(line+6, len-6, what, strlen(what))) >= 0) {
+    return offset;
+  }
+
+  /* nope */
+  return -1;
+}
+
+int z85_isend(Buffer *buf) {
+
+  if(! z85_isheader(buf))
+    return 0;
+  
+  if(z85_header_startswith(buf, "END") < 0)
+    return 0;
+
+  /* true */
+  return 1;
+}
+
+int z85_isbegin(Buffer *buf) {
+  size_t len;
+  size_t blen;
+  const char *begin;
+  long int offset;
+  int i;
+
+  if(! z85_isheader(buf))
+    return 0;
+
+  if((offset = z85_header_startswith(buf, "BEGIN")) < 0)
+    return 0;
+
+  /* determine type */
+  len = buffer_left(buf);
+  byte *line = ucmalloc(len); /* FIXME: maybe wrong, check it */
+  buffer_get_chunk(buf, line, offset);
+  for(i=0; (begin=begins[i]); i++ ) {
+    if(begin == NULL) break;
+    blen = strlen(begin);
+    if(blen <= len)
+      if(_findoffset(line+buf->offset, len, (char *)begin, blen) >= 0)
+	return i; /* i = ENUM ZBEGINS */
+  }
+
+  /* unknown but valid */
+  return -1;
+}
+
+int z85_iscomment(Buffer *buf) {
+  char *line = buffer_get_str(buf);
+ 
+  if(strchr(line, ' ') == NULL || strchr(line, '\t') == NULL)
+    return 0; /* non whitespace */
+  else
+    return 1; /* true */
+}
+
+int z85_isempty(Buffer *buf) {
+  byte *line = buffer_get(buf);
+  size_t len = buffer_size(buf);
+  size_t sp = 0;
+
+  if(len == 0)
+    return 1; /* true */
+
+  /* lines with whitespaces only are empty as well */
+  while(*line != '\0') {
+    if(*line == ' ' || *line == '\t')
+      sp++;
+    line++;
+  }
+
+  if(sp<len)
+    return 0; /* non-space chars found */
+  else
+    return 1; /* true */
+}

tests/pipetest.c

@@ -50,9 +50,11 @@ int main(int argc, char **argv) {
   void *buf = ucmalloc(rblocksize);
 
   while(!ps_end(in)) {
+    fprintf(stderr, "=== read:\n");
     got = ps_read(in, buf, rblocksize);
     if(got > 0)
       ps_write(out, buf, got);
+    fprintf(stderr, "======= got: %ld\n", got);
   }
 
   ps_finish(out);

tests/streamtest.c

@@ -4,8 +4,33 @@
 
 #include <pcp.h>
 
+int linetest() {
+  FILE *in;
+
+  if((in = fopen("x", "rb")) == NULL) {
+    fprintf(stderr, "oops, could not open file!\n");
+    return 1;
+  }
+
+  Pcpstream *pin = ps_new_file(in);
+  ps_setdetermine(pin, 8);
+  size_t got;
+  byte data[9] = {0};
+  while(!ps_end(pin)) {
+    if((got = ps_read(pin, data, 8)) > 0) {
+      fwrite(data, 1, got, stdout);
+    }
+  }
+
+  ps_close(pin);
+  return 0;
+}
+
 int main() {
   /* create a file with "encrypted" data */
+
+  return linetest();
+
   FILE *out, *in;
   unsigned char clear[8] = "ABCDEFGH";
   unsigned char key[8]   = "IxD8Lq1K";