Merge from upstream

CHANGELOG.md

@@ -0,0 +1,2 @@
+## v0.1.9
+- Add `<maven.compiler.release>8</maven.compiler.release>` to pom.xml, to increase compability with Java 8.

Makefile

@@ -1,4 +1,5 @@
-CAPNP_CXX_FLAGS=$(shell pkg-config capnp --cflags --libs)
+PKG_CONFIG ?= pkg-config
+CAPNP_CXX_FLAGS=$(shell $(PKG_CONFIG) capnp --cflags --libs)
 
 ifeq ($(CAPNP_CXX_FLAGS),)
 $(warning "Warning: pkg-config failed to find compilation configuration for capnp.")

benchmark/pom.xml

@@ -5,7 +5,7 @@
     <artifactId>benchmark</artifactId>
     <packaging>jar</packaging>
     <description>capnproto-java benchmark</description>
-    <version>0.1.6-SNAPSHOT</version>
+    <version>0.1.10-SNAPSHOT</version>
     <name>capnproto-java benchmark</name>
     <organization>
         <name>org.capnproto</name>
@@ -33,12 +33,13 @@
       <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
       <maven.compiler.source>14</maven.compiler.source>
       <maven.compiler.target>14</maven.compiler.target>
+      <maven.compiler.release>14</maven.compiler.release>
     </properties>
     <dependencies>
         <dependency>
           <groupId>org.capnproto</groupId>
           <artifactId>runtime</artifactId>
-          <version>0.1.6-SNAPSHOT</version>
+          <version>0.1.10-SNAPSHOT</version>
         </dependency>
     </dependencies>
 

compiler/pom.xml

@@ -5,7 +5,7 @@
     <artifactId>compiler</artifactId>
     <packaging>jar</packaging>
     <description>schema compiler plugin for java</description>
-    <version>0.1.6-SNAPSHOT</version>
+    <version>0.1.10-SNAPSHOT</version>
     <name>capnpc-java</name>
     <organization>
         <name>org.capnproto</name>
@@ -31,6 +31,8 @@
     </developers>
     <properties>
       <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+      <maven.compiler.source>14</maven.compiler.source>
+      <maven.compiler.target>14</maven.compiler.target>
     </properties>
     <dependencies>
         <dependency>
@@ -42,7 +44,7 @@
         <dependency>
           <groupId>org.capnproto</groupId>
           <artifactId>runtime</artifactId>
-          <version>0.1.6-SNAPSHOT</version>
+          <version>0.1.10-SNAPSHOT</version>
         </dependency>
     </dependencies>
 

examples/pom.xml

@@ -5,7 +5,7 @@
     <artifactId>examples</artifactId>
     <packaging>jar</packaging>
     <description>capnproto-java examples</description>
-    <version>0.1.6-SNAPSHOT</version>
+    <version>0.1.10-SNAPSHOT</version>
     <name>capnproto-java examples</name>
     <organization>
         <name>org.capnproto</name>
@@ -38,7 +38,7 @@
         <dependency>
           <groupId>org.capnproto</groupId>
           <artifactId>runtime</artifactId>
-          <version>0.1.6-SNAPSHOT</version>
+          <version>0.1.10-SNAPSHOT</version>
         </dependency>
 
         <dependency>

pom.xml

@@ -4,7 +4,7 @@
     <groupId>org.capnproto</groupId>
     <artifactId>capnproto-java</artifactId>
     <packaging>pom</packaging>
-    <version>0.1.6-SNAPSHOT</version>
+    <version>0.1.10-SNAPSHOT</version>
     <name>Cap'n Proto for Java</name>
     <url>https://capnproto.org/</url>
     <modules>

runtime/pom.xml

@@ -5,7 +5,7 @@
     <artifactId>runtime</artifactId>
     <packaging>jar</packaging>
     <description>runtime</description>
-    <version>0.1.6-SNAPSHOT</version>
+    <version>0.1.10-SNAPSHOT</version>
     <name>Cap'n Proto runtime library</name>
     <organization>
         <name>org.capnproto</name>
@@ -64,7 +64,7 @@
         <plugin>
           <groupId>org.apache.maven.plugins</groupId>
           <artifactId>maven-compiler-plugin</artifactId>
-          <version>3.3</version>
+          <version>3.6.2</version>
           <configuration>
             <compilerArgument>-Xlint:unchecked</compilerArgument>
               <source>11</source>
@@ -75,6 +75,15 @@
     </build>
 
     <profiles>
+        <profile>
+            <id>jdk9FF</id>
+            <activation>
+                <jdk>(1.8,)</jdk>
+            </activation>
+            <properties>
+                <maven.compiler.release>8</maven.compiler.release>
+            </properties>
+        </profile>
       <profile>
         <id>release</id>
         <build>

runtime/src/main/java/org/capnproto/Arena.java

@@ -23,5 +23,5 @@ package org.capnproto;
 
 public interface Arena {
     public SegmentReader tryGetSegment(int id);
-    public void checkReadLimit(int numBytes);
+    public void checkReadLimit(int numWords);
 }

runtime/src/main/java/org/capnproto/BuilderArena.java

@@ -103,6 +103,30 @@ public final class BuilderArena implements Arena {
         return this.localCapTable;
     }
 
+    /**
+     * Constructs a BuilderArena from a ReaderArena and uses the size of the largest segment
+     * as the next allocation size.
+     */
+    BuilderArena(ReaderArena arena) {
+        this.segments = new ArrayList<SegmentBuilder>();
+        int largestSegment = SUGGESTED_FIRST_SEGMENT_WORDS*Constants.BYTES_PER_WORD;
+        for (int ii = 0; ii < arena.segments.size(); ++ii) {
+            SegmentReader segment = arena.segments.get(ii);
+            SegmentBuilder segmentBuilder = new SegmentBuilder(segment.buffer, this);
+            segmentBuilder.id = ii;
+            segmentBuilder.pos = segmentBuilder.capacity(); // buffer is pre-filled
+            segments.add(segmentBuilder);
+
+            // Find the largest segment for the allocation strategy.
+            largestSegment = Math.max(largestSegment, segment.buffer.capacity());
+        }
+        DefaultAllocator defaultAllocator = new DefaultAllocator(SUGGESTED_ALLOCATION_STRATEGY);
+
+        // Use largest segment as next size.
+        defaultAllocator.setNextAllocationSizeBytes(largestSegment);
+        this.allocator = defaultAllocator;
+    }
+
     @Override
     public final SegmentReader tryGetSegment(int id) {
         return this.segments.get(id);
@@ -126,6 +150,10 @@ public final class BuilderArena implements Arena {
         }
     }
 
+    /**
+     *  Allocates `amount` words in an existing segment or, if no suitable segment
+     *  exists, in a new segment.
+     */
     public AllocateResult allocate(int amount) {
         int len = this.segments.size();
 
@@ -136,6 +164,10 @@ public final class BuilderArena implements Arena {
                 return new AllocateResult(this.segments.get(len - 1), result);
             }
         }
+        if (amount >= 1 << 28) {
+            // Computing `amount * Constants.BYTES_PER_WORD` would overflow.
+            throw new RuntimeException("Too many words to allocate: " + amount);
+        }
         SegmentBuilder newSegment = new SegmentBuilder(
             this.allocator.allocateSegment(amount * Constants.BYTES_PER_WORD),
             this);

runtime/src/main/java/org/capnproto/DefaultAllocator.java

@@ -18,6 +18,17 @@ public class DefaultAllocator implements Allocator {
     public AllocationStrategy allocationStrategy =
         AllocationStrategy.GROW_HEURISTICALLY;
 
+    /**
+       The largest number of bytes to try allocating when using `GROW_HEURISTICALLY`.
+
+       Set this value smaller if you get the error:
+
+           java.lang.OutOfMemoryError: Requested array size exceeds VM limit
+
+       Experimentally, `Integer.MAX_VALUE - 2` seems to work on most systems.
+    */
+    public int maxSegmentBytes = Integer.MAX_VALUE - 2;
+
     public DefaultAllocator() {}
 
     public DefaultAllocator(AllocationStrategy allocationStrategy) {
@@ -52,13 +63,16 @@ public class DefaultAllocator implements Allocator {
 
         switch (this.allocationStrategy) {
             case GROW_HEURISTICALLY:
+                if (size < this.maxSegmentBytes - this.nextSize) {
                     this.nextSize += size;
+                } else {
+                    this.nextSize = maxSegmentBytes;
+                }
                 break;
             case FIXED_SIZE:
                 break;
         }
 
-        this.nextSize += size;
         return result;
     }
 }

runtime/src/main/java/org/capnproto/MessageBuilder.java

@@ -71,6 +71,22 @@ public final class MessageBuilder {
         this.arena = new BuilderArena(new DefaultAllocator(), firstSegment);
     }
 
+   /**
+    * Constructs a MessageBuilder from a MessageReader. This constructor is private
+    * because it is unsafe. To use it, you must call `unsafeConstructFromMessageReader()`.
+    */
+    private MessageBuilder(MessageReader messageReader) {
+        this.arena = new BuilderArena(messageReader.arena);
+    }
+
+    /**
+     * Constructs a MessageBuilder from a MessageReader without copying the segments.
+     * This method should only be used on trusted data. Otherwise you may observe infinite
+     * loops or large memory allocations or index-out-of-bounds errors.
+     */
+    public static MessageBuilder unsafeConstructFromMessageReader(MessageReader messageReader) {
+        return new MessageBuilder(messageReader);
+    }
 
     private AnyPointer.Builder getRootInternal() {
         if (this.arena.segments.isEmpty()) {

runtime/src/main/java/org/capnproto/ReaderArena.java

@@ -26,8 +26,8 @@ import java.nio.ByteBuffer;
 
 public final class ReaderArena implements Arena {
 
+    // Current limit. -1 means no limit.
     public long limit;
-    // current limit
 
     public final ArrayList<SegmentReader> segments;
 
@@ -45,11 +45,14 @@ public final class ReaderArena implements Arena {
     }
 
     @Override
-    public final void checkReadLimit(int numBytes) {
+    public final void checkReadLimit(int numWords) {
-        if (numBytes > limit) {
+        if (limit == -1) {
+            // No limit.
+            return;
+        } else if (numWords > limit) {
             throw new DecodeException("Read limit exceeded.");
         } else {
-            limit -= numBytes;
+            limit -= numWords;
         }
     }
 }

runtime/src/main/java/org/capnproto/ReaderOptions.java

@@ -22,7 +22,18 @@
 package org.capnproto;
 
 public final class ReaderOptions {
+    /**
+       How many words are allowed to be read before an exception is thrown,
+       to protect against denial of service attacks.
+
+       -1 means "no limit".
+    */
     public final long traversalLimitInWords;
+
+    /**
+       How many pointer indirections deep a message may be before an exception
+       is thrown.
+    */
     public final int nestingLimit;
 
     public ReaderOptions(long traversalLimitInWords, int nestingLimit) {

runtime/src/main/java/org/capnproto/SegmentBuilder.java

@@ -35,7 +35,7 @@ public final class SegmentBuilder extends SegmentReader {
     }
 
     // the total number of words the buffer can hold
-    private final int capacity() {
+    final int capacity() {
         this.buffer.rewind();
         return this.buffer.remaining() / 8;
     }

runtime/src/main/java/org/capnproto/Serialize.java

@@ -30,6 +30,10 @@ import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.TimeUnit;
 import java.util.function.Consumer;
 
+/**
+ * Serialization using the standard (unpacked) stream encoding:
+ * https://capnproto.org/encoding.html#serialization-over-a-stream
+ */
 public final class Serialize {
 
     static ByteBuffer makeByteBuffer(int bytes) {
@@ -39,6 +43,16 @@ public final class Serialize {
         return result;
     }
 
+    static int MAX_SEGMENT_WORDS = (1 << 28) - 1;
+
+    static ByteBuffer makeByteBufferForWords(int words) throws IOException {
+        if (words > MAX_SEGMENT_WORDS) {
+             // Trying to construct the segment would cause overflow.
+             throw new DecodeException("segment has too many words (" + words + ")");
+        }
+        return makeByteBuffer(words * Constants.BYTES_PER_WORD);
+    }
+
     public static void fillBuffer(ByteBuffer buffer, ReadableByteChannel bc) throws IOException {
         while(buffer.hasRemaining()) {
             int r = bc.read(buffer);
@@ -55,21 +69,22 @@ public final class Serialize {
     }
 
     public static MessageReader read(ReadableByteChannel bc, ReaderOptions options) throws IOException {
-        ByteBuffer firstWord = makeByteBuffer(Constants.BYTES_PER_WORD);
+        ByteBuffer firstWord = makeByteBufferForWords(1);
         fillBuffer(firstWord, bc);
 
-        int segmentCount = 1 + firstWord.getInt(0);
+        int rawSegmentCount = firstWord.getInt(0);
+        if (rawSegmentCount < 0 || rawSegmentCount > 511) {
+            throw new DecodeException("segment count must be between 0 and 512");
+        }
+
+        int segmentCount = 1 + rawSegmentCount;
 
         int segment0Size = 0;
         if (segmentCount > 0) {
             segment0Size = firstWord.getInt(4);
         }
 
-        int totalWords = segment0Size;
+        long totalWords = segment0Size;
-
-        if (segmentCount > 512) {
-            throw new IOException("too many segments");
-        }
 
         // in words
         ArrayList<Integer> moreSizes = new ArrayList<Integer>();
@@ -88,23 +103,17 @@ public final class Serialize {
             throw new DecodeException("Message size exceeds traversal limit.");
         }
 
-        ByteBuffer allSegments = makeByteBuffer(totalWords * Constants.BYTES_PER_WORD);
-        fillBuffer(allSegments, bc);
-
         ByteBuffer[] segmentSlices = new ByteBuffer[segmentCount];
 
-        allSegments.rewind();
+        segmentSlices[0] = makeByteBufferForWords(segment0Size);
-        segmentSlices[0] = allSegments.slice();
+        fillBuffer(segmentSlices[0], bc);
-        segmentSlices[0].limit(segment0Size * Constants.BYTES_PER_WORD);
+        segmentSlices[0].rewind();
-        segmentSlices[0].order(ByteOrder.LITTLE_ENDIAN);
 
         int offset = segment0Size;
         for (int ii = 1; ii < segmentCount; ++ii) {
-            allSegments.position(offset * Constants.BYTES_PER_WORD);
+            segmentSlices[ii] = makeByteBufferForWords(moreSizes.get(ii - 1));
-            segmentSlices[ii] = allSegments.slice();
+            fillBuffer(segmentSlices[ii], bc);
-            segmentSlices[ii].limit(moreSizes.get(ii - 1) * Constants.BYTES_PER_WORD);
+            segmentSlices[ii].rewind();
-            segmentSlices[ii].order(ByteOrder.LITTLE_ENDIAN);
-            offset += moreSizes.get(ii - 1);
         }
 
         return new MessageReader(segmentSlices, options);
@@ -114,15 +123,16 @@ public final class Serialize {
         return read(bb, ReaderOptions.DEFAULT_READER_OPTIONS);
     }
 
-    /*
+    /**
      * Upon return, `bb.position()` will be at the end of the message.
      */
     public static MessageReader read(ByteBuffer bb, ReaderOptions options) throws IOException {
         bb.order(ByteOrder.LITTLE_ENDIAN);
 
-        int segmentCount = 1 + bb.getInt();
+        int rawSegmentCount = bb.getInt();
-        if (segmentCount > 512) {
+        int segmentCount = 1 + rawSegmentCount;
-            throw new IOException("too many segments");
+        if (rawSegmentCount < 0 || rawSegmentCount > 511) {
+            throw new DecodeException("segment count must be between 0 and 512");
         }
 
         ByteBuffer[] segmentSlices = new ByteBuffer[segmentCount];
@@ -137,6 +147,10 @@ public final class Serialize {
 
         for (int ii = 0; ii < segmentCount; ++ii) {
             int segmentSize = bb.getInt(segmentSizesBase + ii * 4);
+            if (segmentSize > MAX_SEGMENT_WORDS -
+                (totalWords + segmentBase / Constants.BYTES_PER_WORD)) {
+                throw new DecodeException("segment size is too large");
+            }
 
             bb.position(segmentBase + totalWords * Constants.BYTES_PER_WORD);
             segmentSlices[ii] = bb.slice();
@@ -147,7 +161,7 @@ public final class Serialize {
         }
         bb.position(segmentBase + totalWords * Constants.BYTES_PER_WORD);
 
-        if (totalWords > options.traversalLimitInWords) {
+        if (options.traversalLimitInWords != -1 && totalWords > options.traversalLimitInWords) {
             throw new DecodeException("Message size exceeds traversal limit.");
         }
 
@@ -178,9 +192,8 @@ public final class Serialize {
         return bytes / Constants.BYTES_PER_WORD;
     }
 
-    public static void write(WritableByteChannel outputChannel,
+    private static void writeSegmentTable(WritableByteChannel outputChannel,
-                             MessageBuilder message) throws IOException {
+                              ByteBuffer[] segments) throws IOException {
-        ByteBuffer[] segments = message.getSegmentsForOutput();
         int tableSize = (segments.length + 2) & (~1);
 
         ByteBuffer table = ByteBuffer.allocate(4 * tableSize);
@@ -196,6 +209,34 @@ public final class Serialize {
         while (table.hasRemaining()) {
             outputChannel.write(table);
         }
+    }
+
+    /**
+     * Serializes a MessageBuilder to a WritableByteChannel.
+     */
+    public static void write(WritableByteChannel outputChannel,
+                             MessageBuilder message) throws IOException {
+        ByteBuffer[] segments = message.getSegmentsForOutput();
+        writeSegmentTable(outputChannel, segments);
+
+        for (ByteBuffer buffer : segments) {
+            while(buffer.hasRemaining()) {
+                outputChannel.write(buffer);
+            }
+        }
+    }
+
+    /**
+     * Serializes a MessageReader to a WritableByteChannel.
+     */
+    public static void write(WritableByteChannel outputChannel,
+                             MessageReader message) throws IOException {
+        ByteBuffer[] segments = new ByteBuffer[message.arena.segments.size()];
+        for (int ii = 0 ; ii < message.arena.segments.size(); ++ii) {
+            segments[ii] = message.arena.segments.get(ii).buffer.duplicate();
+        }
+
+        writeSegmentTable(outputChannel, segments);
 
         for (ByteBuffer buffer : segments) {
             while(buffer.hasRemaining()) {

runtime/src/main/java/org/capnproto/SerializePacked.java

@@ -21,6 +21,9 @@
 
 package org.capnproto;
 
+/**
+ * Serialization using the packed encoding: https://capnproto.org/encoding.html#packing
+ */
 public final class SerializePacked {
 
     public static MessageReader read(BufferedInputStream input) throws java.io.IOException {
@@ -42,16 +45,42 @@ public final class SerializePacked {
         return Serialize.read(packedInput, options);
     }
 
+    /**
+     * Serializes a MessageBuilder to a BufferedOutputStream.
+     */
     public static void write(BufferedOutputStream output,
                              MessageBuilder message) throws java.io.IOException {
         PackedOutputStream packedOutputStream = new PackedOutputStream(output);
         Serialize.write(packedOutputStream, message);
     }
 
+    /**
+     * Serializes a MessageReader to a BufferedOutputStream.
+     */
+    public static void write(BufferedOutputStream output,
+                             MessageReader message) throws java.io.IOException {
+        PackedOutputStream packedOutputStream = new PackedOutputStream(output);
+        Serialize.write(packedOutputStream, message);
+    }
+
+    /**
+     * Serializes a MessageBuilder to an unbuffered output stream.
+     */
     public static void writeToUnbuffered(java.nio.channels.WritableByteChannel output,
                                        MessageBuilder message) throws java.io.IOException {
         BufferedOutputStreamWrapper buffered = new BufferedOutputStreamWrapper(output);
         write(buffered, message);
         buffered.flush();
     }
+
+    /**
+     * Serializes a MessageReader to an unbuffered output stream.
+     */
+    public static void writeToUnbuffered(java.nio.channels.WritableByteChannel output,
+                                         MessageReader message) throws java.io.IOException {
+        BufferedOutputStreamWrapper buffered = new BufferedOutputStreamWrapper(output);
+        write(buffered, message);
+        buffered.flush();
+    }
+
 }

runtime/src/test/java/org/capnproto/DefaultAllocatorTest.java

@@ -0,0 +1,26 @@
+package org.capnproto;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class DefaultAllocatorTest {
+    @Test
+    public void maxSegmentBytes() {
+      DefaultAllocator allocator = new DefaultAllocator();
+      Assert.assertEquals(allocator.allocationStrategy,
+                          BuilderArena.AllocationStrategy.GROW_HEURISTICALLY);
+      allocator.maxSegmentBytes = (1 << 25) - 1;
+
+      int allocationSize = 1 << 24;
+      allocator.setNextAllocationSizeBytes(allocationSize);
+
+      Assert.assertEquals(allocationSize,
+                          allocator.allocateSegment(allocationSize).capacity());
+
+      Assert.assertEquals(allocator.maxSegmentBytes,
+                          allocator.allocateSegment(allocationSize).capacity());
+
+      Assert.assertEquals(allocator.maxSegmentBytes,
+                          allocator.allocateSegment(allocationSize).capacity());
+    }
+}

runtime/src/test/java/org/capnproto/SerializeTest.java

@@ -61,6 +61,10 @@ public class SerializeTest {
     {
       MessageReader messageReader = Serialize.read(new ArrayInputStream(ByteBuffer.wrap(exampleBytes)));
       checkSegmentContents(exampleSegmentCount, messageReader.arena);
+
+      byte[] outputBytes = new byte[exampleBytes.length];
+      Serialize.write(new ArrayOutputStream(ByteBuffer.wrap(outputBytes)), messageReader);
+      Assert.assertArrayEquals(exampleBytes, outputBytes);
     }
 
     // ------