Merge branch 'baruch' into merge

Conflicts:
	Makefile		-- manually resolved
	compiler/schema.capnp	-- used from liamstask
	compiler/test.capnp	-- used from liamstask

.gitignore

@@ -2,3 +2,8 @@
 *.so
 /capn-test
 /capnpc-c
+/capn.a
+/gtest-all.a
+.cproject
+.project
+.settings

Makefile

@@ -1,7 +1,7 @@
 .PHONY: all clean test
 
-LDFLAGS=-O2 -Wall -Werror -fPIC
+LDFLAGS=-O2 -Wall -fPIC
-CFLAGS=-O2 -Wall -Werror -fPIC -I. -Wno-unused-function
+CFLAGS=-O2 -Wall -fPIC -I. -Wno-unused-function
 ifneq (,$(findstring gcc, $(CC)))
 	# gcc's maybe-unintialized is too imprecise, disable it.
 	# clang disbles this functionality by default and doesn't recognize the flag.
@@ -13,25 +13,36 @@ endif
 GTEST_CFLAGS=-std=c++11 `gtest-config --cppflags --cxxflags`
 GTEST_LDFLAGS=`gtest-config --ldflags --libs`
 
-all: capn.so capnpc-c test
+OBJS=capn-malloc.o capn-stream.o capn.o
+prefix = /usr
+
+all: capn.a capn.so capnpc-c test
 
 clean:
-	rm -f *.o *.so capnpc-c compiler/*.o
+	rm -f *.o *.so capnpc-c compiler/*.o *.a
 
 %.o: %.c *.h *.inc compiler/*.h
 	$(CC) $(CFLAGS) -c $< -o $@
 
-capn.so: capn-malloc.o capn-stream.o capn.o
+capn.so: $(OBJS)
 	$(CC) -shared $(LDFLAGS) $^ -o $@
 
-capnpc-c: compiler/capnpc-c.o compiler/schema.capnp.o compiler/str.o capn.so
+capn.a: $(OBJS)
+	$(AR) rcs $@ $^
+
+capnpc-c: compiler/capnpc-c.o compiler/schema.capnp.o compiler/str.o capn.a
 	$(CC) $(LDFLAGS) $^ -o $@
 
 test: capn-test
 	./capn-test
 
 %-test.o: %-test.cpp *.h *.c *.inc
-	$(CXX) -g -Wall -Werror -I. $(GTEST_CFLAGS) -o $@ -c $<
+	$(CXX) --std=c++11 -g -I. `gtest-config --cppflags --cxxflags` -o $@ -c $<
 
-capn-test: capn-test.o capn-stream-test.o compiler/test.capnp.o compiler/schema-test.o compiler/schema.capnp.o
+capn-test: capn-test.o capn-stream-test.o compiler/test.capnp.o compiler/schema-test.o compiler/schema.capnp.o capn.a
-	$(CXX) -g -Wall -Werror -I. $^ $(GTEST_LDFLAGS) -o $@
+	$(CXX) -std=c++11 -g -I. $^ `gtest-config --ldflags --libs` -o $@
+
+install:
+	install -c capnpc-c $(prefix)/bin/capnpc-c
+	install -c capn.so $(prefix)/lib/capn.so
+	install -c capn.a $(prefix)/lib/capn.a

README.md

@@ -0,0 +1,10 @@
+capnpc-c
+========
+
+This is a C plugin for cap'n'proto aka capnp.
+
+This is only the plugin, to properly make use of it you need to download, build
+and install capnpc and then build and install this project and then you can
+utilize it as:
+
+    capnpc compiler/test.capnp -oc

capn-malloc.c

@@ -3,6 +3,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include <limits.h>
+#include <errno.h>
 
 struct check_segment_alignment {
 	unsigned int foo : (sizeof(struct capn_segment)&7) ? -1 : 1;
@@ -109,14 +110,16 @@ static int init_fp(struct capn *c, FILE *f, struct capn_stream *z, int packed) {
 
 	capn_init_malloc(c);
 
+	/* Read the first four bytes to know how many headers we have */
 	if (read_fp(&segnum, 4, f, z, zbuf, packed))
 		goto err;
 
 	segnum = capn_flip32(segnum);
 	if (segnum > 1023)
 		goto err;
-	segnum++;
+	segnum++; /* The wire encoding was zero-based */
 
+	/* Read the header list */
 	if (read_fp(hdr, 8 * (segnum/2) + 4, f, z, zbuf, packed))
 		goto err;
 
@@ -128,10 +131,12 @@ static int init_fp(struct capn *c, FILE *f, struct capn_stream *z, int packed) {
 		total += hdr[i];
 	}
 
+	/* Allocate space for the data and the capn_segment structs */
 	s = (struct capn_segment*) calloc(1, total + (sizeof(*s) * segnum));
 	if (!s)
 		goto err;
 
+	/* Now read the data and setup the capn_segment structs */
 	data = (char*) (s+segnum);
 	if (read_fp(data, total, f, z, zbuf, packed))
 		goto err;
@@ -143,6 +148,7 @@ static int init_fp(struct capn *c, FILE *f, struct capn_stream *z, int packed) {
 		capn_append_segment(c, &s[i]);
 	}
 
+    /* Set the entire region to be freed on the last segment */
 	s[segnum-1].user = s;
 
 	return 0;
@@ -167,24 +173,8 @@ int capn_init_mem(struct capn *c, const uint8_t *p, size_t sz, int packed) {
 	return init_fp(c, NULL, &z, packed);
 }
 
-int
+static void header_calc(struct capn *c, uint32_t *headerlen, size_t *headersz)
-capn_write_mem(struct capn *c, uint8_t *p, size_t sz, int packed)
 {
-	struct capn_segment *seg;
-	struct capn_ptr root;
-	int i;
-	uint32_t headerlen;
-	size_t datasz;
-	uint32_t *header;
-
-	/* TODO support packing */
-	if (packed)
-		return -1;
-
-	if (c->segnum == 0)
-		return -1;
-
-	root = capn_root(c);
 	/* segnum == 1:
 	 *   [segnum][segsiz]
 	 * segnum == 2:
@@ -194,36 +184,201 @@ capn_write_mem(struct capn *c, uint8_t *p, size_t sz, int packed)
 	 * segnum == 4:
 	 *   [segnum][segsiz][segsiz][segsiz][segsiz][zeroes]
 	 */
-	headerlen = ((2 + c->segnum) / 2) * 2;
+	*headerlen = ((2 + c->segnum) / 2) * 2;
-	datasz = 4 * headerlen;
+	*headersz = 4 * *headerlen;
-	header = (uint32_t*) p;
+}
 
-	if (sz < datasz)
+static int header_render(struct capn *c, struct capn_segment *seg, uint32_t *header, uint32_t headerlen, size_t *datasz)
-		return -1;
+{
+	int i;
 
 	header[0] = capn_flip32(c->segnum - 1);
-	header[headerlen-1] = 0;
+	header[headerlen-1] = 0; /* Zero out the spare position in the header sizes */
-	for (i = 0, seg = root.seg; i < c->segnum; i++, seg = seg->next) {
+	for (i = 0; i < c->segnum; i++, seg = seg->next) {
 		if (0 == seg)
 			return -1;
-		datasz += seg->len;
+		*datasz += seg->len;
 		header[1 + i] = capn_flip32(seg->len / 8);
 	}
 	if (0 != seg)
 		return -1;
 
-	if (sz < datasz)
+	return 0;
+}
+
+int capn_write_mem_packed(struct capn *c, uint8_t *p, size_t sz)
+{
+	struct capn_segment *seg;
+	struct capn_ptr root;
+	uint32_t headerlen;
+	size_t headersz, datasz = 0;
+	uint32_t *header;
+	struct capn_stream z;
+	int ret;
+
+	root = capn_root(c);
+	header_calc(c, &headerlen, &headersz);
+	header = (uint32_t*) p + headersz + 2; /* must reserve two bytes for worst case expansion */
+
+	if (sz < headersz*2 + 2) /* We must have space for temporary writing of header to deflate */
 		return -1;
 
-	p += 4 * headerlen;
+	ret = header_render(c, root.seg, header, headerlen, &datasz);
-	sz -= 4 * headerlen;
+	if (ret != 0)
+		return -1;
+
+	memset(&z, 0, sizeof(z));
+	z.next_in = (uint8_t *)header;
+	z.avail_in = headersz;
+	z.next_out = p;
+	z.avail_out = sz;
+
+	// pack the headers
+	ret = capn_deflate(&z);
+	if (ret != 0 || z.avail_in != 0)
+		return -1;
 
 	for (seg = root.seg; seg; seg = seg->next) {
-		if (sz < seg->len)
+		z.next_in = (uint8_t *)seg->data;
+		z.avail_in = seg->len;
+		ret = capn_deflate(&z);
+		if (ret != 0 || z.avail_in != 0)
 			return -1;
+	}
+
+	return sz - z.avail_out;
+}
+
+int
+capn_write_mem(struct capn *c, uint8_t *p, size_t sz, int packed)
+{
+	struct capn_segment *seg;
+	struct capn_ptr root;
+	uint32_t headerlen;
+	size_t headersz, datasz = 0;
+	uint32_t *header;
+	int ret;
+
+	if (c->segnum == 0)
+		return -1;
+
+	if (packed)
+		return capn_write_mem_packed(c, p, sz);
+
+	root = capn_root(c);
+	header_calc(c, &headerlen, &headersz);
+	header = (uint32_t*) p;
+
+	if (sz < headersz)
+		return -1;
+
+	ret = header_render(c, root.seg, header, headerlen, &datasz);
+	if (ret != 0)
+		return -1;
+
+	if (sz < headersz + datasz)
+		return -1;
+
+	p += headersz;
+
+	for (seg = root.seg; seg; seg = seg->next) {
 		memcpy(p, seg->data, seg->len);
 		p += seg->len;
-		sz -= seg->len;
+	}
+
+	return headersz+datasz;
+}
+
+static int _write_fd(ssize_t (*write_fd)(int fd, void *p, size_t count), int fd, void *p, size_t count)
+{
+	int ret;
+	int sent = 0;
+
+	while (sent < count) {
+		ret = write_fd(fd, ((uint8_t*)p)+sent, count-sent);
+		if (ret < 0) {
+			if (errno == EAGAIN || errno == EINTR)
+				continue;
+			else
+				return -1;
+		}
+		sent += ret;
+	}
+
+	return 0;
+}
+
+int capn_write_fd(struct capn *c, ssize_t (*write_fd)(int fd, void *p, size_t count), int fd, int packed)
+{
+	unsigned char buf[4096];
+	struct capn_segment *seg;
+	struct capn_ptr root;
+	uint32_t headerlen;
+	size_t headersz, datasz = 0;
+	int ret;
+	struct capn_stream z;
+	unsigned char *p;
+
+	if (c->segnum == 0)
+		return -1;
+
+	root = capn_root(c);
+	header_calc(c, &headerlen, &headersz);
+
+	if (sizeof(buf) < headersz)
+		return -1;
+
+	ret = header_render(c, root.seg, (uint32_t*)buf, headerlen, &datasz);
+	if (ret != 0)
+		return -1;
+
+	if (packed) {
+		const int headerrem = sizeof(buf) - headersz;
+		const int maxpack = headersz + 2;
+		if (headerrem < maxpack)
+			return -1;
+
+		memset(&z, 0, sizeof(z));
+		z.next_in = buf;
+		z.avail_in = headersz;
+		z.next_out = buf + headersz;
+		z.avail_out = headerrem;
+		ret = capn_deflate(&z);
+		if (ret != 0)
+			return -1;
+
+		p = buf + headersz;
+		headersz = headerrem - z.avail_out;
+	} else {
+		p = buf;
+	}
+
+	ret = _write_fd(write_fd, fd, p, headersz);
+	if (ret < 0)
+		return -1;
+
+	datasz = headersz;
+	for (seg = root.seg; seg; seg = seg->next) {
+		size_t bufsz;
+		if (packed) {
+			memset(&z, 0, sizeof(z));
+			z.next_in = (uint8_t*)seg->data;
+			z.avail_in = seg->len;
+			z.next_out = buf;
+			z.avail_out = sizeof(buf);
+			ret = capn_deflate(&z);
+			if (ret != 0)
+				return -1;
+			p = buf;
+			bufsz = sizeof(buf) - z.avail_out;
+		} else {
+			p = (uint8_t*)seg->data;
+			bufsz = seg->len;
+		}
+		ret = _write_fd(write_fd, fd, p, bufsz);
+		if (ret < 0)
+			return -1;
+		datasz += bufsz;
 	}
 
 	return datasz;

capn-stream.c

@@ -12,7 +12,7 @@ int capn_deflate(struct capn_stream* s) {
 	}
 
 	while (s->avail_in) {
-		int i, sz = 0;
+		int i, sz;
 		uint8_t hdr = 0;
 		uint8_t *p;
 
@@ -33,6 +33,7 @@ int capn_deflate(struct capn_stream* s) {
 		if (s->avail_in < 8)
 			return CAPN_NEED_MORE;
 
+		sz = 0;
 		for (i = 0; i < 8; i++) {
 			if (s->next_in[i]) {
 				sz ++;
@@ -45,7 +46,7 @@ int capn_deflate(struct capn_stream* s) {
 			if (s->avail_out < 2)
 				return CAPN_NEED_MORE;
 
-			s->next_out[0] = hdr;
+			s->next_out[0] = 0;
 			for (sz = 1; sz < min(s->avail_in/8, 256); sz++) {
 				if (((uint64_t*) s->next_in)[sz] != 0) {
 					break;
@@ -63,7 +64,7 @@ int capn_deflate(struct capn_stream* s) {
 			if (s->avail_out < 10)
 				return CAPN_NEED_MORE;
 
-			s->next_out[0] = hdr;
+			s->next_out[0] = 0xFF;
 			memcpy(s->next_out+1, s->next_in, 8);
 			s->next_in += 8;
 			s->avail_in -= 8;
@@ -160,16 +161,16 @@ int capn_inflate(struct capn_stream* s) {
 			continue;
 
 		default:
+			hdr = s->next_in[0];
 			sz = 0;
-			hdr = s->next_in[1];
 			for (i = 0; i < 8; i++) {
 				if (hdr & (1 << i))
 					sz++;
 			}
-			if (s->avail_in < 2 + sz)
+			if (s->avail_in < 1 + sz)
 				return CAPN_NEED_MORE;
 
-			s->next_in += 2;
+			s->next_in += 1;
 
 			for (i = 0; i < 8; i++) {
 				if (hdr & (1 << i)) {
@@ -180,7 +181,7 @@ int capn_inflate(struct capn_stream* s) {
 			}
 
 			s->avail_out -= 8;
-			s->avail_in -= 2 + sz;
+			s->avail_in -= 1 + sz;
 			continue;
 		}
 	}

capn.c

@@ -186,8 +186,8 @@ end:
 }
 
 static struct capn_segment *lookup_segment(struct capn* c, struct capn_segment *s, uint32_t id) {
-	struct capn_tree **x;
+	struct capn_tree **x = &c->segtree;
-	struct capn_segment *y;
+	struct capn_segment *y = NULL;
 
 	if (s && s->id == id)
 		return s;
@@ -195,8 +195,6 @@ static struct capn_segment *lookup_segment(struct capn* c, struct capn_segment *
 		return NULL;
 
 	if (id < c->segnum) {
-		x = &c->segtree;
-		y = NULL;
 		while (*x) {
 			y = (struct capn_segment*) *x;
 			if (id == y->id) {

capn.h

@@ -246,6 +246,7 @@ int capn_init_mem(struct capn *c, const uint8_t *p, size_t sz, int packed);
  */
 /* TODO */
 /*int capn_write_fp(struct capn *c, FILE *f, int packed);*/
+int capn_write_fd(struct capn *c, ssize_t (*write_fd)(int fd, void *p, size_t count), int fd, int packed);
 int capn_write_mem(struct capn *c, uint8_t *p, size_t sz, int packed);
 
 void capn_free(struct capn *c);

compiler/test.capnp.c

@@ -14,7 +14,7 @@ static const uint8_t capn_buf[8168] = {
 	77,0,0,0,34,0,0,0,
 	77,0,0,0,26,0,0,0,
 	76,0,0,0,6,0,20,0,
-	205,207,62,235,24,0,0,0,
+	29,109,174,255,24,0,0,0,
 	33,1,0,0,41,0,0,0,
 	33,1,0,0,34,0,0,0,
 	33,1,0,0,35,0,0,0,
@@ -390,7 +390,7 @@ static const uint8_t capn_buf[8168] = {
 	77,0,0,0,34,0,0,0,
 	77,0,0,0,26,0,0,0,
 	76,0,0,0,6,0,20,0,
-	237,201,62,235,24,0,0,0,
+	61,103,174,255,24,0,0,0,
 	33,1,0,0,41,0,0,0,
 	33,1,0,0,34,0,0,0,
 	33,1,0,0,35,0,0,0,
@@ -685,7 +685,7 @@ static const uint8_t capn_buf[8168] = {
 	77,0,0,0,34,0,0,0,
 	77,0,0,0,26,0,0,0,
 	76,0,0,0,6,0,20,0,
-	81,197,62,235,24,0,0,0,
+	161,98,174,255,24,0,0,0,
 	33,1,0,0,41,0,0,0,
 	33,1,0,0,34,0,0,0,
 	33,1,0,0,35,0,0,0,