From b8552af5e9eefb30fa57dc57045bf10bc29e425c Mon Sep 17 00:00:00 2001 From: "git@daemon.de" Date: Sat, 1 Mar 2014 11:58:10 +0100 Subject: [PATCH] - broken - reimplementing z85 decoder, using hyphens again, doesn't work yet... --- TODO | 25 +---- include/pcp.h | 1 + include/pcp/defines.h | 9 ++ include/pcp/pcpstream.h | 19 +++- include/pcp/util.h | 2 +- include/pcp/z85.h | 9 ++ libpcp/pcpstream.c | 216 ++++++++++++++++++++++++++++++++-------- libpcp/util.c | 4 +- libpcp/z85.c | 119 ++++++++++++++++++++++ tests/pipetest.c | 2 + tests/streamtest.c | 25 +++++ 11 files changed, 363 insertions(+), 68 deletions(-) diff --git a/TODO b/TODO index da6b7de..64845d4 100644 --- a/TODO +++ b/TODO @@ -16,30 +16,7 @@ enable formats for secret key exports as well Add newlines to headers in define.h, so strlen() later catches the whole length. -Z85 headers: - - currently I use "----- BEGIN ... -----" and "----- END ... -----" as - header and footer for various z85 encoded outputs. The problem is, that - the "-" character is part of Z85 chars. An input of 0xc6,0x5a,0x0b,0x13 would - result z85 encoded as: "-----". So, I cannot be sure, when I find a header - delimiter, if it's really a delimiter or legitimate z85 encoded content. - Therefore, another delimiter must be used. "~~~~~ BEGIN .... ~~~~~" seems - to fit best and "~" is unused in Z85. - Then the parser can be enhanced as well. Eg: on startup if a ~ occurs, - ignore input until the first non-~ appears. Then decode input until a - ~ or eof appears, ignore everything after. Comments would still be a - problem though. Currently I ignore lines containing whitespaces. But - if a file is read blockwise and the blocksize is very small, then a - comment line may span multiple blocks and isn't recognizable as a - "line" anymore. Maybe, comments shall start and end with a ~ as well, eg: - ~ BEGIN KEY ~ - ~ Hash: 987298347 ~ - [z85] - ~ END KEY ~ - Here I use the same aproach for the headers, since there would also be - the problem how to recognize them properly if a header crosses boundaries - or something. By using this scheme, if a ~ is found everything following - is marked as to be ignored which could be saved as a state when using - blockmode. +Z85 Stream encode: add newline after last. Check is_utf8 license. also found in https://gd.meizo.com/_files/lpc/ext/utf8.c diff --git a/include/pcp.h b/include/pcp.h index 8b16eae..fb0d6c6 100644 --- a/include/pcp.h +++ b/include/pcp.h @@ -8,6 +8,7 @@ extern "C" { #include "pcp/config.h" #include "pcp/base85.h" #include "pcp/buffer.h" +#include "pcp/config.h" #include "pcp/crypto.h" #include "pcp/defines.h" #include "pcp/digital_crc32.h" diff --git a/include/pcp/defines.h b/include/pcp/defines.h index c2f8e7f..be3e5b8 100644 --- a/include/pcp/defines.h +++ b/include/pcp/defines.h @@ -74,6 +74,15 @@ typedef unsigned int qbyte; /* Quad byte = 32 bits */ #define PCP_KEY_VERSION 6 #define PCP_KEY_PRIMITIVE "CURVE25519-ED25519-SALSA20-POLY1305" +typedef enum _ZBEGINS { + PCP_ENCRYPTED_FILE, + Z85_ENCODED_FILE, + ED25519_SIGNED_MESSAGE, + ED25519_SIGNATURE, + ED25519_CURVE29915_PUBLIC_KEY, + ED25519_CURVE29915_PRIVATE_KEY, +} ZBEGINS; + /** \addtogroup KEYS @{ diff --git a/include/pcp/pcpstream.h b/include/pcp/pcpstream.h index 1c03a9e..b944265 100644 --- a/include/pcp/pcpstream.h +++ b/include/pcp/pcpstream.h @@ -62,6 +62,7 @@ struct _pcp_stream_t { Buffer *b; /**< The backend Buffer object */ Buffer *cache; /**< The caching Buffer object (for look ahead read) */ Buffer *next; /**< The caching Next-Buffer object (for look ahead read) */ + Buffer *save; /**< Temporary buffer to backup overflow data */ uint8_t is_buffer; /**< Set to 1 if the backend is a Buffer */ uint8_t eof; /**< Set to 1 if EOF reached */ uint8_t err; /**< Set to 1 if an error occured */ @@ -71,9 +72,15 @@ struct _pcp_stream_t { size_t linewr; /**< Used for Z85 writing, number of chars written on last line */ size_t blocksize; /**< Blocksize used for z85, if requested */ uint8_t is_output; /**< marks the stream as output stream */ + uint8_t have_begin; /**< flag to indicate we already got the begin header, if any */ size_t pos; /**< remember i/o position */ }; +typedef enum _PSVARS { + PSMAXLINE = 20000 +} PSVARS; + + /** The name used everywhere */ typedef struct _pcp_stream_t Pcpstream; @@ -268,7 +275,7 @@ void ps_unarmor(Pcpstream *stream); /* read from primary source, decode z85 and out into cache. if buf != NULL, consider it as the start of encoded data and remove headers and comments, then continue as normal. */ -size_t ps_read_decode(Pcpstream *stream, void *buf, size_t bufsize); +size_t ps_read_decode(Pcpstream *stream); /* determine if primary source is z85 encoded, put the data read from it into the cache */ @@ -294,6 +301,16 @@ size_t ps_write_buf(Pcpstream *stream, Buffer *z); /* tell if we really reached eof, caching or not. 1=eof, 0=ok */ int ps_left(Pcpstream *stream); +/** Read a line from the stream. + + \param[in] stream The stream object. + \param[out] line Linecontent will be written to this Buffer. + + \return Returns the number of bytes read or -1 if PSMAXLINE have been + reached or the input doesn't have any newlines at all. + */ +int ps_readline(Pcpstream *stream, Buffer *line); + #endif // HAVE_PCP_PCPSTEAM_H diff --git a/include/pcp/util.h b/include/pcp/util.h index 42decb3..efe9f1e 100644 --- a/include/pcp/util.h +++ b/include/pcp/util.h @@ -74,7 +74,7 @@ char *_lc(char *in); \return Returns the offset or -1 of the offset were not found. */ -size_t _findoffset(byte *bin, size_t binlen, char *sigstart, size_t hlen); +long int _findoffset(byte *bin, size_t binlen, char *sigstart, size_t hlen); /** XOR an input buffer with another buffer. diff --git a/include/pcp/z85.h b/include/pcp/z85.h index 1a0eba9..de3d3b8 100644 --- a/include/pcp/z85.h +++ b/include/pcp/z85.h @@ -156,6 +156,15 @@ size_t _buffer_is_binary(byte *buf, size_t len); */ uint8_t _parse_zchar(Buffer *z, uint8_t c, uint8_t is_comment); + +long int z85_header_startswith(Buffer *buf, char *what); +int z85_isheader(Buffer *buf); +int z85_isend(Buffer *buf); +int z85_isbegin(Buffer *buf); +int z85_iscomment(Buffer *buf); +int z85_isempty(Buffer *line); + + #endif /* _HAVE_PCP_Z85_H */ /**@}*/ diff --git a/libpcp/pcpstream.c b/libpcp/pcpstream.c index 525ae02..b60057b 100644 --- a/libpcp/pcpstream.c +++ b/libpcp/pcpstream.c @@ -27,6 +27,7 @@ Pcpstream *ps_init(void) { stream->cache = NULL; stream->next = NULL; stream->fd = NULL; + stream->save = buffer_new(32, "Pcpstreamsavebuf"); stream->is_buffer = 0; stream->eof = 0; stream->err = 0; @@ -79,15 +80,46 @@ void ps_unarmor(Pcpstream *stream) { stream->armor = 0; } +void ps_rewind(Pcpstream *stream, void *buf, size_t bufsize) { + if(stream->is_buffer) { + stream->b->offset -= bufsize; + } + else { + buffer_clear(stream->save); + buffer_add(stream->save, buf, bufsize); + } + stream->pos -= bufsize; + stream->err = 0; + stream->eof = 0; +} + size_t ps_read_raw(Pcpstream *stream, void *buf, size_t readbytes) { size_t gotbytes = 0; + size_t idx = 0; + + if(buffer_left(stream->save) > 0) { + /* something left from last rewind, first use this */ + if(buffer_left(stream->save) >= readbytes) { + gotbytes = buffer_get_chunk(stream->save, buf, readbytes); + if(buffer_left(stream->save) == 0) + buffer_clear(stream->save); + goto rawdone; + } + else { + /* fetch the remainder of the save buffer, remember how much + to fetch from source next */ + idx = buffer_get_chunk(stream->save, buf, buffer_left(stream->save)); + buffer_clear(stream->save); + readbytes -= idx; + } + } if(stream->is_buffer) { /* check if there's enough space in our buffer */ if(buffer_left(stream->b) < readbytes) readbytes = buffer_left(stream->b); - gotbytes = buffer_get_chunk(stream->b, buf, readbytes); + gotbytes += buffer_get_chunk(stream->b, buf+idx, readbytes); if(gotbytes == 0) { /* this should not happen with buffers */ stream->eof = 1; @@ -95,16 +127,15 @@ size_t ps_read_raw(Pcpstream *stream, void *buf, size_t readbytes) { } } else { - gotbytes = fread(buf, 1, readbytes, stream->fd); - if(gotbytes == 0) { - if(feof(stream->fd) != 0) - stream->eof = 1; - if(ferror(stream->fd) != 0) - stream->err = 1; - } + size_t got = fread(buf+idx, 1, readbytes, stream->fd); + gotbytes += got; + if(feof(stream->fd) != 0) + stream->eof = 1; + if(ferror(stream->fd) != 0) + stream->err = 1; } - stream->firstread = 1; + rawdone: return gotbytes; } @@ -113,20 +144,23 @@ size_t ps_read_raw(Pcpstream *stream, void *buf, size_t readbytes) { that */ size_t ps_read_cached(Pcpstream *stream, void *buf, size_t readbytes) { - /* + fprintf(stderr, "%ld <= %ld && %ld <= %ld\n", readbytes, buffer_left(stream->cache), readbytes, stream->blocksize) ; fprintf(stderr, "%d == 1 && %ld >= %ld\n", ps_left(stream), readbytes, buffer_left(stream->cache)); - */ + if(readbytes <= buffer_left(stream->cache) && readbytes <= stream->blocksize) { /* enough left in current cache */ + fprintf(stderr, " get all from cache\n"); return buffer_get_chunk(stream->cache, buf, readbytes); } - else if(ps_left(stream) == 1 && readbytes >= buffer_left(stream->cache)) { + else if(ps_end(stream) == 1 && readbytes >= buffer_left(stream->cache) ) { + fprintf(stderr, " get rest from cache\n"); return buffer_get_chunk(stream->cache, buf, buffer_left(stream->cache)); } else { + fprintf(stderr, " fetch next\n"); /* request for chunk larger than what we've got in the cache */ Buffer *tmp = buffer_new(stream->blocksize, "Pcpreadchunktmp"); @@ -135,6 +169,8 @@ size_t ps_read_cached(Pcpstream *stream, void *buf, size_t readbytes) { buffer_get_chunk_tobuf(stream->cache, tmp, buffer_left(stream->cache)); } +#error EOF reached, cache empty, save filled, doesnt call ps_read_next() + /* how much left to fetch */ long int left = readbytes - buffer_size(tmp); @@ -184,10 +220,10 @@ size_t ps_read_cached(Pcpstream *stream, void *buf, size_t readbytes) { /* read and decode the next chunk and put it into stream->next */ size_t ps_read_next(Pcpstream *stream) { - if(ps_left(stream) == 0) { + if(ps_left(stream) == 0 || buffer_left(stream->save)) { if(stream->armor == 1) { /* fetch next chunk and decode it */ - return ps_read_decode(stream, NULL, 0); + return ps_read_decode(stream); } else { /* unencoded source, fetch as is */ @@ -209,6 +245,7 @@ size_t ps_read(Pcpstream *stream, void *buf, size_t readbytes) { else if(buffer_size(stream->cache) > 0) { /* use cache */ got = ps_read_cached(stream, buf, readbytes); + fprintf(stderr, "%ld = use cache directly\n", got); } else { /* no cache yet */ @@ -217,11 +254,13 @@ size_t ps_read(Pcpstream *stream, void *buf, size_t readbytes) { recursively call ps_read() again to return the apropriate data */ ps_determine(stream); got = ps_read(stream, buf, readbytes); + fprintf(stderr, "%ld = ps_read(stream, buf, readbytes);\n", got); } else if(stream->armor == 1) { /* z85 encoding has already been determined, therefore the cache is now filled, use it then */ got = ps_read_cached(stream, buf, readbytes); + fprintf(stderr, "%ld = ps_read_cached(stream, buf, readbytes);\n", got); } else { /* read directly from source */ @@ -230,10 +269,44 @@ size_t ps_read(Pcpstream *stream, void *buf, size_t readbytes) { } stream->pos += got; - + fprintf(stderr, " ps_read(): %ld\n", got); return got; } +int ps_readline(Pcpstream *stream, Buffer *line) { + int c = 0, max = 1; + byte b[1]; + + while(csave, line); + buffer_clear(line); + return -1; + } + + return c; +} + void ps_determine(Pcpstream *stream) { /* read a raw chunk from source */ void *buf = ucmalloc(stream->blocksize); @@ -244,8 +317,11 @@ void ps_determine(Pcpstream *stream) { /* no, it's armored */ stream->armor = 1; + /* put back raw data into read queue */ + ps_rewind(stream, buf, got); + /* decode the first chunk */ - ps_read_decode(stream, buf, got); + ps_read_decode(stream); /* put it into the cache */ buffer_add_buf(stream->cache, stream->next); @@ -255,42 +331,92 @@ void ps_determine(Pcpstream *stream) { /* just put the raw stuff into the cache */ buffer_add(stream->cache, buf, got); } + + ucfree(buf, stream->blocksize); + + stream->firstread = 1; } -size_t ps_read_decode(Pcpstream *stream, void *buf, size_t bufsize) { - size_t i = 0; - uint8_t is_comment = 0; - uint8_t c; - Buffer *z = buffer_new(32, "ztemp"); - byte *_buf = buf; - if(bufsize > 0) { - for(i=0; isave); + + if(buffer_left(stream->save) >= stream->blocksize && stream->firstread == 1) { + /* use the save buffer instead */ + buffer_get_chunk_tobuf(stream->save, z, stream->blocksize); } - - if(buffer_size(z) < stream->blocksize) { - /* blocksize not full, continue with stream source */ - /* read in bytewise, ignore newlines and add until the block is full */ - while (buffer_size(z) < stream->blocksize) { - if (ps_read_raw(stream, &c, 1) == 1) { - is_comment = _parse_zchar(z, c, is_comment); + else if(ps_left(stream) == 1 && buffer_left(stream->save) > 0) { + /* there's something left which doesn't end in a newline */ + buffer_get_chunk_tobuf(stream->save, z, stream->blocksize); + } + else { + /* continue reading linewise */ + while(buffer_size(z) < stream->blocksize) { + buffer_clear(line); + if(ps_readline(stream, line) >= 0) { + fprintf(stderr, "got: <%s>\n", buffer_get_str(line)); + if(z85_isbegin(line) && stream->have_begin == 0) { + /* begin header encountered */ + stream->have_begin = 1; /* otherwise ignore it */ + continue; + } + else if(z85_isend(line)) { + /* end header encountered */ + break; + } + else if(z85_isempty(line)) { + /* ignore empty lines */ + continue; + } + else { + /* regular z85 encoded content */ + fprintf(stderr, "regular\n"); + if(buffer_size(z) + buffer_size(line) > stream->blocksize) { + /* we've got more than needed. + put what we need into z and the remainder + into the save buffer for further reading. */ + fprintf(stderr, "overflow %ld + %ld > %ld\n", + buffer_size(z), buffer_size(line), stream->blocksize); + + buffer_get_chunk_tobuf(line, z, stream->blocksize - buffer_size(z)); + buffer_get_chunk_tobuf(line, stream->save, buffer_left(line)); + buffer_add8(stream->save, '\n'); + break; + } + else { + /* not enough yet, store it and go on */ + buffer_add_buf(z, line); + } + } } - else + else { + /* eof or err */ break; + } } } + fprintf(stderr, "Z: <%s>\n", buffer_get_str(z)); + /* finally, decode it and put into next */ size_t binlen, outlen; byte *bin = pcp_z85_decode(buffer_get_str(z), &binlen); if(bin == NULL) { /* it's not z85 encoded, so threat it as binary */ - stream->armor = 0; - buffer_add_buf(stream->next, z); - outlen = buffer_size(stream->next); + if(stream->firstread) { + /* whoops, we're in the middle of z85 decoding and it failed */ + stream->eof = 1; + stream->err = 1; + outlen = 0; + } + else { + stream->armor = 0; + buffer_add_buf(stream->next, z); + outlen = buffer_size(stream->next); + } } else { /* yes, successfully decoded it, put into cache */ @@ -299,12 +425,13 @@ size_t ps_read_decode(Pcpstream *stream, void *buf, size_t bufsize) { } buffer_free(z); - + buffer_free(line); return outlen; } + size_t ps_write(Pcpstream *stream, void *buf, size_t writebytes) { Buffer *z = buffer_new(32, "Pcpwritetemp"); @@ -484,6 +611,8 @@ void ps_close(Pcpstream *stream) { if(stream->next != NULL) buffer_free(stream->next); + buffer_free(stream->save); + if(stream->is_buffer) { buffer_clear(stream->b); free(stream); @@ -498,10 +627,17 @@ void ps_close(Pcpstream *stream) { int ps_end(Pcpstream *stream) { /* simulate open file if there's still something in the cache */ - if(stream->cache != NULL) + if(stream->cache != NULL) { if(buffer_left(stream->cache) > 0) { return 0; } + } + + /* if there's a lookahead buffer, do the same */ + if(buffer_left(stream->save) > 0) { + return 0; + } + return stream->eof; } diff --git a/libpcp/util.c b/libpcp/util.c index 45a18ce..7ba2908 100644 --- a/libpcp/util.c +++ b/libpcp/util.c @@ -31,9 +31,9 @@ char *_lc(char *in) { } /* find the offset of the beginning of a certain string within binary data */ -size_t _findoffset(byte *bin, size_t binlen, char *sigstart, size_t hlen) { +long int _findoffset(byte *bin, size_t binlen, char *sigstart, size_t hlen) { size_t i; - size_t offset = 0; + long int offset = 0; int m = 0; for(i=0; i= 0) { + return offset; + } + + /* nope */ + return -1; +} + +int z85_isend(Buffer *buf) { + + if(! z85_isheader(buf)) + return 0; + + if(z85_header_startswith(buf, "END") < 0) + return 0; + + /* true */ + return 1; +} + +int z85_isbegin(Buffer *buf) { + size_t len; + size_t blen; + const char *begin; + long int offset; + int i; + + if(! z85_isheader(buf)) + return 0; + + if((offset = z85_header_startswith(buf, "BEGIN")) < 0) + return 0; + + /* determine type */ + len = buffer_left(buf); + byte *line = ucmalloc(len); /* FIXME: maybe wrong, check it */ + buffer_get_chunk(buf, line, offset); + for(i=0; (begin=begins[i]); i++ ) { + if(begin == NULL) break; + blen = strlen(begin); + if(blen <= len) + if(_findoffset(line+buf->offset, len, (char *)begin, blen) >= 0) + return i; /* i = ENUM ZBEGINS */ + } + + /* unknown but valid */ + return -1; +} + +int z85_iscomment(Buffer *buf) { + char *line = buffer_get_str(buf); + + if(strchr(line, ' ') == NULL || strchr(line, '\t') == NULL) + return 0; /* non whitespace */ + else + return 1; /* true */ +} + +int z85_isempty(Buffer *buf) { + byte *line = buffer_get(buf); + size_t len = buffer_size(buf); + size_t sp = 0; + + if(len == 0) + return 1; /* true */ + + /* lines with whitespaces only are empty as well */ + while(*line != '\0') { + if(*line == ' ' || *line == '\t') + sp++; + line++; + } + + if(sp 0) ps_write(out, buf, got); + fprintf(stderr, "======= got: %ld\n", got); } ps_finish(out); diff --git a/tests/streamtest.c b/tests/streamtest.c index dd879d1..3c57b33 100644 --- a/tests/streamtest.c +++ b/tests/streamtest.c @@ -4,8 +4,33 @@ #include +int linetest() { + FILE *in; + + if((in = fopen("x", "rb")) == NULL) { + fprintf(stderr, "oops, could not open file!\n"); + return 1; + } + + Pcpstream *pin = ps_new_file(in); + ps_setdetermine(pin, 8); + size_t got; + byte data[9] = {0}; + while(!ps_end(pin)) { + if((got = ps_read(pin, data, 8)) > 0) { + fwrite(data, 1, got, stdout); + } + } + + ps_close(pin); + return 0; +} + int main() { /* create a file with "encrypted" data */ + + return linetest(); + FILE *out, *in; unsigned char clear[8] = "ABCDEFGH"; unsigned char key[8] = "IxD8Lq1K";