Add capn_init_(fp|mem) to read a stream from file/memory

Packed form still to be tested

Makefile

@@ -20,5 +20,5 @@ test: capn-test
 %-test.o: %-test.cpp *.h *.c *.inc
 	$(CXX) `gtest-config --cppflags --cxxflags` -o $@ -c $<
 
-capn-test: capn-test.o
+capn-test: capn-test.o capn-stream-test.o
 	$(CXX) `gtest-config --ldflags --libs` -o $@ $^

capn-stream-test.cpp

@@ -0,0 +1,72 @@
+#include "capn-stream.c"
+#include <gtest/gtest.h>
+
+template <int wordCount>
+union AlignedData {
+  uint8_t bytes[wordCount * 8];
+  uint64_t words[wordCount];
+};
+
+TEST(Stream, ReadEmptyStream_Even) {
+  AlignedData<2> data = {{
+    1, 0, 0, 0, // num of segs - 1
+    0, 0, 0, 0,
+    0, 0, 0, 0,
+    2, 3, 4, 0, // garbage that should be ignored
+  }};
+
+  struct capn ctx;
+  ASSERT_NE(0, capn_init_mem(&ctx, data.bytes, 12, 0));
+  ASSERT_EQ(0, capn_init_mem(&ctx, data.bytes, 16, 0));
+  EXPECT_EQ(2, ctx.segnum);
+  EXPECT_EQ(0, ctx.seglist->len);
+  EXPECT_EQ(0, ctx.seglist->next->len);
+  capn_free_mem(&ctx);
+}
+
+TEST(Stream, ReadEmptyStream_Odd) {
+  AlignedData<3> data = {{
+    2, 0, 0, 0, // num of segs - 1
+    0, 0, 0, 0,
+    0, 0, 0, 0,
+    0, 0, 0, 0,
+    2, 3, 4, 0, // garbage that should be ignored
+  }};
+
+  struct capn ctx;
+  ASSERT_NE(0, capn_init_mem(&ctx, data.bytes, 12, 0));
+
+  ASSERT_EQ(0, capn_init_mem(&ctx, data.bytes, 16, 0));
+  EXPECT_EQ(3, ctx.segnum);
+  EXPECT_EQ(0, ctx.seglist->len);
+  EXPECT_EQ(0, ctx.seglist->next->len);
+  capn_free_mem(&ctx);
+
+  ASSERT_EQ(0, capn_init_mem(&ctx, data.bytes, 20, 0));
+  EXPECT_EQ(3, ctx.segnum);
+  EXPECT_EQ(0, ctx.seglist->len);
+  EXPECT_EQ(0, ctx.seglist->next->len);
+  capn_free_mem(&ctx);
+}
+
+TEST(Stream, ReadStream_Even) {
+  AlignedData<5> data = {{
+     1, 0, 0, 0, // num of segs - 1
+     1, 0, 0, 0,
+     2, 0, 0, 0,
+     2, 3, 4, 0, // garbage that should be ignored
+     1, 2, 3, 4, 5, 6, 7, 8,
+     9,10,11,12,13,14,15,16,
+    17,18,19,20,21,22,23,24,
+  }};
+
+  struct capn ctx;
+  ASSERT_NE(0, capn_init_mem(&ctx, data.bytes, 36, 0));
+  ASSERT_EQ(0, capn_init_mem(&ctx, data.bytes, 40, 0));
+  EXPECT_EQ(2, ctx.segnum);
+  EXPECT_EQ(8, ctx.seglist->len);
+  EXPECT_EQ(1, ctx.seglist->data[0]);
+  EXPECT_EQ(16, ctx.seglist->next->len);
+  EXPECT_EQ(9, ctx.seglist->next->data[0]);
+  capn_free_mem(&ctx);
+}

capn-stream.c

@@ -1,19 +1,26 @@
 /* vim: set sw=8 ts=8 sts=8 noet: */
 #include "capn.h"
 #include <string.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <limits.h>
 
 #ifndef min
 static int min(int a, int b) { return (a < b) ? a : b; }
 #endif
 
 int capn_deflate(struct capn_stream* s) {
+	if (s->avail_in % 8) {
+		return CAPN_MISALIGNED;
+	}
+
 	while (s->avail_in) {
 		int i, sz = 0;
 		uint8_t hdr = 0;
 		uint8_t *p;
 
 		if (!s->avail_out)
-			return CAPN_NEED_MORE_OUT;
+			return CAPN_NEED_MORE;
 
 		if (s->raw > 0) {
 			sz = min(s->raw, min(s->avail_in, s->avail_out));
@@ -27,7 +34,7 @@ int capn_deflate(struct capn_stream* s) {
 		}
 
 		if (s->avail_in < 8)
-			return CAPN_NEED_MORE_IN;
+			return CAPN_NEED_MORE;
 
 		for (i = 0; i < 8; i++) {
 			if (s->next_in[i]) {
@@ -39,7 +46,7 @@ int capn_deflate(struct capn_stream* s) {
 		switch (sz) {
 		case 0:
 			if (s->avail_out < 2)
-				return CAPN_NEED_MORE_OUT;
+				return CAPN_NEED_MORE;
 
 			s->next_out[0] = hdr;
 			for (sz = 1; sz < min(s->avail_in/8, 256); sz++) {
@@ -57,7 +64,7 @@ int capn_deflate(struct capn_stream* s) {
 
 		case 8:
 			if (s->avail_out < 10)
-				return CAPN_NEED_MORE_OUT;
+				return CAPN_NEED_MORE;
 
 			s->next_out[0] = hdr;
 			memcpy(s->next_out+1, s->next_in, 8);
@@ -65,7 +72,7 @@ int capn_deflate(struct capn_stream* s) {
 			s->avail_in -= 8;
 
 			s->raw = min(s->avail_in, 256*8);
-			if ((p = memchr(s->next_in, 0, s->raw)) != NULL) {
+			if ((p = (uint8_t*) memchr(s->next_in, 0, s->raw)) != NULL) {
 				s->raw = (p - s->next_in) & ~7;
 			}
 
@@ -76,7 +83,7 @@ int capn_deflate(struct capn_stream* s) {
 
 		default:
 			if (s->avail_out < 1 + sz)
-				return CAPN_NEED_MORE_OUT;
+				return CAPN_NEED_MORE;
 
 			*(s->next_out++) = hdr;
 			for (i = 0; i < 8; i++) {
@@ -95,14 +102,15 @@ int capn_deflate(struct capn_stream* s) {
 }
 
 int capn_inflate(struct capn_stream* s) {
-	for (;;) {
+	if (s->avail_out % 8) {
+		return CAPN_MISALIGNED;
+	}
+
+	while (s->avail_out) {
 		int i, sz;
 		uint8_t hdr;
 
 		if (s->zeros > 0) {
-			if (s->avail_out == 0)
-				return CAPN_NEED_MORE_OUT;
-
 			sz = min(s->avail_out, s->zeros);
 			memset(s->next_out, 0, sz);
 			s->next_out += sz;
@@ -113,9 +121,7 @@ int capn_inflate(struct capn_stream* s) {
 
 		if (s->raw > 0) {
 			if (s->avail_in == 0)
-				return CAPN_NEED_MORE_IN;
+				return CAPN_NEED_MORE;
-			else if (s->avail_out == 0)
-				return CAPN_NEED_MORE_OUT;
 
 			sz = min(min(s->avail_out, s->raw), s->avail_in);
 			memcpy(s->next_out, s->next_in, sz);
@@ -129,17 +135,15 @@ int capn_inflate(struct capn_stream* s) {
 
 		if (s->avail_in == 0)
 			return 0;
-		else if (s->avail_out < 8)
-			return CAPN_NEED_MORE_OUT;
 		else if (s->avail_in < 2)
-			return CAPN_NEED_MORE_IN;
+			return CAPN_NEED_MORE;
 
 		switch (s->next_in[0]) {
 		case 0xFF:
 			/* 0xFF is followed by 8 bytes raw, followed by
 			 * a byte with length in words to read raw */
 			if (s->avail_in < 10)
-				return CAPN_NEED_MORE_IN;
+				return CAPN_NEED_MORE;
 
 			memcpy(s->next_out, s->next_in+1, 8);
 			s->raw = (int) s->next_in[9] * 8;
@@ -166,7 +170,7 @@ int capn_inflate(struct capn_stream* s) {
 					sz++;
 			}
 			if (s->avail_in < 2 + sz)
-				return CAPN_NEED_MORE_IN;
+				return CAPN_NEED_MORE;
 
 			s->next_in += 2;
 
@@ -183,5 +187,119 @@ int capn_inflate(struct capn_stream* s) {
 			continue;
 		}
 	}
+
+	return 0;
 }
 
+#define ZBUF_SZ 4096
+
+static int read_fp(void *p, size_t sz, FILE *f, struct capn_stream *z, uint8_t* zbuf, int packed) {
+	if (f && packed) {
+		z->next_out = (uint8_t*) p;
+		z->avail_out = sz;
+
+		while (z->avail_out && capn_inflate(z) == CAPN_NEED_MORE) {
+			int r;
+			memmove(zbuf, z->next_in, z->avail_in);
+			r = fread(zbuf+z->avail_in, 1, ZBUF_SZ - z->avail_in, f);
+			if (r <= 0)
+				return -1;
+			z->avail_in += r;
+		}
+		return 0;
+
+	} else if (f && !packed) {
+		return fread(p, sz, 1, f) != 1;
+
+	} else if (packed) {
+		z->next_out = (uint8_t*) p;
+		z->avail_out = sz;
+		return capn_inflate(z) != 0;
+
+	} else {
+		if (z->avail_in < sz)
+			return -1;
+		memcpy(p, z->next_in, sz);
+		z->next_in += sz;
+		z->avail_in -= sz;
+		return 0;
+	}
+}
+
+static int init_fp(struct capn *c, FILE *f, struct capn_stream *z, int packed) {
+	struct capn_segment *s = NULL;
+	uint32_t i, segnum, total = 0;
+	uint32_t hdr[1024];
+	uint8_t zbuf[ZBUF_SZ];
+	char *data = NULL;
+	memset(c, 0, sizeof(*c));
+
+	if (read_fp(&segnum, 4, f, z, zbuf, packed))
+		goto err;
+
+	segnum = capn_flip32(segnum);
+	if (segnum > 1023)
+		goto err;
+	segnum++;
+
+	s = (struct capn_segment*) calloc(segnum, sizeof(*s));
+	if (!s)
+		goto err;
+
+	if (read_fp(hdr, 8 * (segnum/2) + 4, f, z, zbuf, packed))
+		goto err;
+
+	for (i = 0; i < segnum; i++) {
+		uint32_t n = capn_flip32(hdr[i]);
+		if (n > INT_MAX/8 || n > UINT32_MAX/8 || UINT32_MAX - total < n*8)
+			goto err;
+		s[i].cap = s[i].len = n * 8;
+		total += s[i].len;
+	}
+
+	data = (char*) calloc(1, total);
+	if (!data)
+		goto err;
+
+	if (read_fp(data, total, f, z, zbuf, packed))
+		goto err;
+
+	for (i = 0; i < segnum; i++) {
+		s[i].data = data;
+		data += s[i].len;
+		capn_append_segment(c, &s[i]);
+	}
+
+	return 0;
+
+err:
+	memset(c, 0, sizeof(*c));
+	free(data);
+	free(s);
+	return -1;
+}
+
+int capn_init_fp(struct capn *c, FILE *f, int packed) {
+	struct capn_stream z;
+	memset(&z, 0, sizeof(z));
+	return init_fp(c, f, &z, packed);
+}
+
+int capn_init_mem(struct capn *c, const uint8_t *p, size_t sz, int packed) {
+	struct capn_stream z;
+	memset(&z, 0, sizeof(z));
+	z.next_in = p;
+	z.avail_in = sz;
+	return init_fp(c, NULL, &z, packed);
+}
+
+void capn_free_mem(struct capn *c) {
+	capn_free_fp(c);
+}
+
+void capn_free_fp(struct capn *c) {
+	if (c->seglist) {
+		free(c->seglist->data);
+		free(c->seglist);
+	}
+}

capn.h

@@ -4,6 +4,7 @@
 #define CAPN_H
 
 #include <stdint.h>
+#include <stdio.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -207,6 +208,12 @@ CAPN_INLINE int capn_write64(capn_ptr p, int off, uint64_t val);
 void capn_init_malloc(struct capn *c);
 void capn_free_malloc(struct capn *c);
 
+int capn_init_fp(struct capn *c, FILE *f, int packed);
+void capn_free_fp(struct capn *c);
+
+int capn_init_mem(struct capn *c, const uint8_t *p, size_t sz, int packed);
+void capn_free_mem(struct capn *c);
+
 /* capn_stream encapsulates the needed fields for capn_(deflate|inflate) in a
  * similar manner to z_stream from zlib
  *
@@ -216,27 +223,28 @@ void capn_free_malloc(struct capn *c);
  * Other fields should be zero initialized.
  */
 struct capn_stream {
-	uint8_t *next_in;
+	const uint8_t *next_in;
 	int avail_in;
 	uint8_t *next_out;
 	int avail_out;
 	int zeros, raw;
 };
 
-#define CAPN_NEED_MORE_IN -1
+#define CAPN_MISALIGNED -1
-#define CAPN_NEED_MORE_OUT -2
+#define CAPN_NEED_MORE -2
 
 /* capn_deflate deflates a stream to the packed format
  * capn_inflate inflates a stream from the packed format
  *
- * They will return CAPN_NEED_MORE_(IN|OUT) as appropriate or 0 if the entire
+ * Returns:
- * input has been processed.
+ * CAPN_MISALIGNED - if the unpacked data is not 8 byte aligned
+ * CAPN_NEED_MORE - more packed data/room is required (out for inflate, in for
+ * deflate)
+ * 0 - success, all output for inflate, all input for deflate processed
  */
 int capn_deflate(struct capn_stream*);
 int capn_inflate(struct capn_stream*);
 
-int capn_marshal_iptr(const union capn_iptr*, struct capn_ptr*, int off);
-
 /* Inline functions */