[#14] Remove length calculation

Clojurescript doesn't have a method to compute the size of protobuf messages, and
we can get away without having one by using auto-expanding ByteArrayOutputStream
types.  We therefore remove support for all the size-XX serdes and (pb/length)
functions.  This is part of the breaking changes to support both Clojure
and Clojurescript

Partially addresses #14

Signed-off-by: Greg Haskins <[email protected]>

project.clj

@@ -1,4 +1,4 @@
-(defproject protojure "1.0.2-SNAPSHOT"
+(defproject protojure "1.1.0-SNAPSHOT"
   :description "Support library for protoc-gen-clojure, providing native Clojure support for Google Protocol Buffers and GRPC applications"
   :url "http://github.com/protojure/library"
   :license {:name "Apache License 2.0"

src/protojure/grpc/codec/lpm.clj

@@ -151,38 +151,34 @@ The value for the **content-coding** option must be one of
 ;;--------------------------------------------------------------------------------------
 ;; Encoder
 ;;--------------------------------------------------------------------------------------
-(defn- encode-header [os compressed? len]
+(defn- encode-buffer [buf len compressed? os]
   (.write os (int (if compressed? 1 0)))
-  (.write os (num->bytes len)))
-
-(defn- encode-uncompressed
-  ([msg os]
-   (encode-uncompressed msg (pb/length msg) os))
-  ([msg len os]
-   (encode-header os false len)
-   (pb/->pb msg os)))
-
-(defn- encode-compressed-buffer [buf len os]
-  (encode-header os true len)
+  (.write os (num->bytes len))
   (.write os buf))
 
-(defn- compress-msg [compressor msg]
+(defn- encode-uncompressed [msg os]
+  (let [buf (pb/->pb msg)
+        len (count buf)]
+    (encode-buffer buf len false os)))
+
+(defn- compress-buffer [compressor buf]
   (let [os (ByteArrayOutputStream.)
         cos (compressor os)]
-    (pb/->pb msg cos)
+    (.write cos buf)
     (.close cos)
     (.toByteArray os)))
 
 (defn- encode-maybe-compressed
   "This function will encode the message either with or without compression,
   depending on whichever results in the smaller message"
   [msg compressor os]
-  (let [buf (compress-msg compressor msg)
-        clen (count buf)
-        len (pb/length msg)]
+  (let [buf (pb/->pb msg)
+        len (count buf)
+        cbuf (compress-buffer compressor buf)
+        clen (count buf)]
     (if (< clen len)
-      (encode-compressed-buffer buf clen os)
-      (encode-uncompressed msg len os))))
+      (encode-buffer cbuf clen true os)
+      (encode-buffer buf len false os))))
 
 ;;--------------------------------------------------------------------------------------------
 (defn encode

src/protojure/protobuf.clj

@@ -4,20 +4,18 @@
 
 (ns protojure.protobuf
   "Main API entry point for protobuf applications"
-  (:import (com.google.protobuf
-            CodedOutputStream)))
+  (:import (com.google.protobuf CodedOutputStream)
+           (java.io ByteArrayOutputStream)))
 
 (defprotocol Writer
-  (serialize [this os])
-  (length [this]))
+  (serialize [this os]))
 
 (defn ->pb
   "Serialize a record implementing the [[Writer]] protocol into protobuf bytes."
   ([msg]
-   (let [len (length msg)
-         data (byte-array len)]
-     (->pb msg data)
-     data))
+   (let [os (ByteArrayOutputStream.)]
+     (->pb msg os)
+     (.toByteArray os)))
   ([msg output]
    (let [os (CodedOutputStream/newInstance output)]
      (serialize msg os)

src/protojure/protobuf/serdes.clj

@@ -80,23 +80,10 @@
         (when-not (and (get options# :optimize true) (~default? value#))
           (. os# ~sym tag# value#))))))
 
-(defmacro defsizefn [type default?]
-  (let [name (symbol (str "size-" type))
-        sym (symbol (str "compute" type "Size"))
-        doc (format "Compute length of serialized '%s' type" type)]
-    `(defn ~name ~doc
-       ([tag# value#]
-        (~name tag# {} value#))
-       ([tag# options# value#]
-        (if-not (and (get options# :optimize true) (~default? value#))
-          (. CodedOutputStream ~sym tag# value#)
-          0)))))
-
 (defmacro defserdes [type default?]
   `(do
      (defparsefn ~type)
-     (defwritefn ~type ~default?)
-     (defsizefn ~type ~default?)))
+     (defwritefn ~type ~default?)))
 
 (def default-scalar? #(or (nil? %) (zero? %)))
 (def default-string? empty?)
@@ -137,16 +124,6 @@
      (let [bytestring (ByteString/copyFrom value)]
        (.writeBytes os tag bytestring)))))
 
-(defn size-Bytes
-  "Compute length of serialized 'Bytes' type"
-  ([tag value]
-   (size-Bytes tag {} value))
-  ([tag {:keys [optimize] :or {optimize true} :as options} value]
-   (if-not (and optimize (empty? value))
-     (let [bytestring (ByteString/copyFrom value)]
-       (CodedOutputStream/computeBytesSize tag bytestring))
-     0)))
-
 (defn cis->undefined
   "Deserialize an unknown type, retaining its tag/type"
   [tag is]
@@ -208,11 +185,13 @@
 (defn write-embedded
   "Serialize an embedded type along with tag/length metadata"
   [tag item os]
-  (let [len (if (some? item) (pb/length item) 0)]
-    (when-not (zero? len)
-      (.writeTag os tag 2);; embedded messages are always type=2 (string)
-      (.writeUInt32NoTag os len)
-      (pb/serialize item os))))
+  (when (some? item)
+    (let [bytes (pb/->pb item)
+          len (count bytes)]
+      (when-not (zero? len)
+        (.writeTag os tag 2);; embedded messages are always type=2 (string)
+        (.writeUInt32NoTag os len)
+        (.writeRawBytes os bytes)))))
 
 ;; FIXME: Add support for optimizing packable types
 (defn write-repeated
@@ -224,26 +203,4 @@
 (defn write-map
   "Serialize user format [key val] using given map item constructor"
   [constructor tag items os]
-  (write-repeated write-embedded tag (map (fn [[key value]] (constructor {:key key :value value})) items) os))
-
-(defn size-embedded
-  "Compute length of serialized embedded type, including the metadata header"
-  [tag item]
-  (let [len (if (some? item) (pb/length item) 0)]
-    (if-not (zero? len)
-      (+
-       (size-UInt32 tag {:optimize false} len)  ;; This accounts for the tag+length preamble
-       len)                   ;; And this is the embedded item itself
-      0)))
-
-(defn size-repeated
-  "Compute length of serialized repeated type"
-  [f tag items]
-  (if-not (empty? items)
-    (reduce + (map (partial f tag) items))
-    0))
-
-(defn size-map
-  "Compute length of user format [key val] using given map item constructor"
-  [constructor tag item]
-  (size-repeated size-embedded tag (map (fn [[key value]] (constructor {:key key :value value})) item)))
+  (write-repeated write-embedded tag (map (fn [[key value]] (constructor {:key key :value value})) items) os))

test/example/hello.clj

@@ -40,10 +40,7 @@
   pb/Writer
 
   (serialize [this os]
-    (write-String 1  {:optimize true} (:name this) os))
-
-  (length [this]
-    (reduce + [(size-String 1  {:optimize true} (:name this))])))
+    (write-String 1  {:optimize true} (:name this) os)))
 
 (s/def :com.sttgts.omnia.hello.messages.HelloRequest/name string?)
 (s/def ::HelloRequest-spec (s/keys :opt-un [:com.sttgts.omnia.hello.messages.HelloRequest/name]))
@@ -90,11 +87,7 @@
 
   (serialize [this os]
     (write-String 1  {:optimize true} (:name this) os)
-    (write-Int32 2  {:optimize true} (:count this) os))
-
-  (length [this]
-    (reduce + [(size-String 1  {:optimize true} (:name this))
-               (size-Int32 2  {:optimize true} (:count this))])))
+    (write-Int32 2  {:optimize true} (:count this) os)))
 
 (s/def :com.sttgts.omnia.hello.messages.RepeatHelloRequest/name string?)
 (s/def :com.sttgts.omnia.hello.messages.RepeatHelloRequest/count int?)
@@ -142,10 +135,7 @@
   pb/Writer
 
   (serialize [this os]
-    (write-String 1  {:optimize true} (:message this) os))
-
-  (length [this]
-    (reduce + [(size-String 1  {:optimize true} (:message this))])))
+    (write-String 1  {:optimize true} (:message this) os)))
 
 (s/def :com.sttgts.omnia.hello.messages.HelloReply/message string?)
 (s/def ::HelloReply-spec (s/keys :opt-un [:com.sttgts.omnia.hello.messages.HelloReply/message]))

test/example/types.clj

@@ -18,13 +18,7 @@
   (serialize [this os]
     (write-String 1 (:currency_code this) os)
     (write-Int64 2 (:units this) os)
-    (write-Int32 3 (:nanos this) os))
-
-  (length [this]
-    (+
-     (size-String 1 (:currency_code this))
-     (size-Int64 2 (:units this))
-     (size-Int32 3 (:nanos this)))))
+    (write-Int32 3 (:nanos this) os)))
 
 (def Money-defaults {:currency_code "" :units 0 :nanos 0})
 
@@ -69,10 +63,7 @@
   pb/Writer
 
   (serialize [this os]
-    (write-repeated write-Int32 1 (:data this) os))
-
-  (length [this]
-    (size-repeated size-Int32 1 (:data this))))
+    (write-repeated write-Int32 1 (:data this) os)))
 
 (def SimpleRepeated-defaults {:data []})
 
@@ -116,10 +107,7 @@
   pb/Writer
 
   (serialize [this os]
-    (write-String 1  {:optimize true} (:s this) os))
-
-  (length [this]
-    (size-String 1  {:optimize true} (:s this))))
+    (write-String 1  {:optimize true} (:s this) os)))
 
 (def SimpleString-defaults {:s ""})
 
@@ -164,12 +152,7 @@
 
   (serialize [this os]
     (write-String 1  {:optimize true} (:key this) os)
-    (write-Int32 2  {:optimize true} (:value this) os))
-
-  (length [this]
-    (+
-     (size-String 1  {:optimize true} (:key this))
-     (size-Int32 2  {:optimize true} (:value this)))))
+    (write-Int32 2  {:optimize true} (:value this) os)))
 
 (def AllThingsMap-MSimpleEntry-defaults {:key "" :value 0})
 
@@ -215,12 +198,7 @@
 
   (serialize [this os]
     (write-String 1  {:optimize true} (:key this) os)
-    (write-embedded 2 (:value this) os))
-
-  (length [this]
-    (+
-     (size-String 1  {:optimize true} (:key this))
-     (size-embedded 2 (:value this)))))
+    (write-embedded 2 (:value this) os)))
 
 (def AllThingsMap-MComplexEntry-defaults {:key ""})
 
@@ -270,15 +248,7 @@
     (write-Int32 2  {:optimize true} (:i this) os)
     (write-map new-AllThingsMap-MSimpleEntry 3 (:mSimple this) os)
     (write-map new-AllThingsMap-MComplexEntry 4 (:mComplex this) os)
-    (write-embedded 5 (:sSimple this) os))
-
-  (length [this]
-    (+
-     (size-String 1  {:optimize true} (:s this))
-     (size-Int32 2  {:optimize true} (:i this))
-     (size-map new-AllThingsMap-MSimpleEntry 3 (:mSimple this))
-     (size-map new-AllThingsMap-MComplexEntry 4 (:mComplex this))
-     (size-embedded 5 (:sSimple this)))))
+    (write-embedded 5 (:sSimple this) os)))
 
 (def AllThingsMap-defaults {:s "" :i 0 :mSimple [] :mComplex []})