feat!: add streaming decode

packages/protons-runtime/src/codec.ts

@@ -1,13 +1,13 @@
 import type { Writer, Reader } from './index.ts'
 
 // https://developers.google.com/protocol-buffers/docs/encoding#structure
-export enum CODEC_TYPES {
-  VARINT = 0,
-  BIT64,
-  LENGTH_DELIMITED,
-  START_GROUP,
-  END_GROUP,
-  BIT32
+export const CODEC_TYPES = {
+  VARINT: 0,
+  BIT64: 1,
+  LENGTH_DELIMITED: 2,
+  START_GROUP: 3,
+  END_GROUP: 4,
+  BIT32: 5
 }
 
 export interface EncodeOptions {
@@ -59,18 +59,24 @@ export interface DecodeFunction<T> {
   (reader: Reader, length?: number, opts?: DecodeOptions<T>): T
 }
 
+export interface StreamFunction<T> {
+  (reader: Reader, length?: number, prefix?: string, opts?: DecodeOptions<T>): Generator<any>
+}
+
 export interface Codec<T> {
   name: string
-  type: CODEC_TYPES
+  type: number
   encode: EncodeFunction<T>
   decode: DecodeFunction<T>
+  stream: StreamFunction<T>
 }
 
-export function createCodec <T> (name: string, type: CODEC_TYPES, encode: EncodeFunction<T>, decode: DecodeFunction<T>): Codec<T> {
+export function createCodec <T> (name: string, type: number, encode: EncodeFunction<T>, decode: DecodeFunction<T>, stream: StreamFunction<T>): Codec<T> {
   return {
     name,
     type,
     encode,
-    decode
+    decode,
+    stream
   }
 }

packages/protons-runtime/src/codecs/enum.ts

@@ -1,8 +1,8 @@
 import { createCodec, CODEC_TYPES } from '../codec.ts'
-import type { DecodeFunction, EncodeFunction, Codec } from '../codec.ts'
+import type { DecodeFunction, EncodeFunction, Codec, StreamFunction } from '../codec.ts'
 
 export function enumeration <T> (v: any): Codec<T> {
-  function findValue (val: string | number): number {
+  function findValue (val: any): number {
     // Use the reverse mapping to look up the enum key for the stored value
     // https://www.typescriptlang.org/docs/handbook/enums.html#reverse-mappings
     if (v[val.toString()] == null) {
@@ -12,18 +12,23 @@ export function enumeration <T> (v: any): Codec<T> {
     return v[val]
   }
 
-  const encode: EncodeFunction<number | string> = function enumEncode (val, writer) {
+  const encode: EncodeFunction<T> = function enumEncode (val, writer) {
     const enumValue = findValue(val)
 
     writer.int32(enumValue)
   }
 
-  const decode: DecodeFunction<number | string> = function enumDecode (reader) {
+  const decode: DecodeFunction<any> = function enumDecode (reader) {
     const val = reader.int32()
 
     return findValue(val)
   }
 
-  // @ts-expect-error yeah yeah
-  return createCodec('enum', CODEC_TYPES.VARINT, encode, decode)
+  const stream: StreamFunction<T> = function * enumStream (reader) {
+    const val = reader.int32()
+
+    yield findValue(val)
+  }
+
+  return createCodec<T>('enum', CODEC_TYPES.VARINT, encode, decode, stream)
 }

packages/protons-runtime/src/codecs/message.ts

@@ -1,10 +1,10 @@
 import { createCodec, CODEC_TYPES } from '../codec.ts'
-import type { EncodeFunction, DecodeFunction, Codec } from '../codec.ts'
+import type { EncodeFunction, DecodeFunction, Codec, StreamFunction } from '../codec.ts'
 
 export interface Factory<A, T> {
   new (obj: A): T
 }
 
-export function message <T> (encode: EncodeFunction<T>, decode: DecodeFunction<T>): Codec<T> {
-  return createCodec('message', CODEC_TYPES.LENGTH_DELIMITED, encode, decode)
+export function message <T> (encode: EncodeFunction<T>, decode: DecodeFunction<T>, stream: StreamFunction<T>): Codec<T> {
+  return createCodec('message', CODEC_TYPES.LENGTH_DELIMITED, encode, decode, stream)
 }

packages/protons-runtime/src/index.ts

@@ -28,6 +28,10 @@ export {
   encodeMessage
 } from './encode.ts'
 
+export {
+  streamMessage
+} from './stream.ts'
+
 export { enumeration } from './codecs/enum.ts'
 export { message } from './codecs/message.ts'
 export { createReader as reader } from './utils/reader.ts'

packages/protons-runtime/src/stream.ts

@@ -0,0 +1,9 @@
+import { createReader } from './utils/reader.ts'
+import type { Codec } from './codec.ts'
+import type { Uint8ArrayList } from 'uint8arraylist'
+
+export function * streamMessage <T> (buf: Uint8Array | Uint8ArrayList, codec: Pick<Codec<T>, 'stream'>, opts?: any): Generator<any> {
+  const reader = createReader(buf)
+
+  yield * codec.stream(reader, undefined, opts)
+}

packages/protons/package.json

@@ -56,6 +56,7 @@
   },
   "devDependencies": {
     "aegir": "^47.0.5",
+    "it-all": "^3.0.9",
     "long": "^5.2.0",
     "pbjs": "^0.0.14",
     "protobufjs": "^7.0.0",

packages/protons/src/fields/array-field.ts

@@ -0,0 +1,109 @@
+import { Enum } from '../types/enum.ts'
+import { Message } from '../types/message.ts'
+import { codecTypes, Field } from './field.ts'
+import type { FieldDef } from './field.ts'
+import type { Parent, Type } from '../types/index.ts'
+
+export interface ArrayFieldDef extends FieldDef {
+  rule: 'repeated'
+}
+
+export function isArrayFieldDef (obj?: any): obj is ArrayFieldDef {
+  return obj?.rule === 'repeated'
+}
+
+export class ArrayField extends Field {
+  private lengthLimit?: number
+
+  constructor (name: string, def: ArrayFieldDef, parent: Parent) {
+    super(name, def, parent)
+
+    this.lengthLimit = def.options?.['(protons.options).limit']
+  }
+
+  getInterfaceField (parent: Parent, indent = ''): string {
+    return `${super.getInterfaceField(parent, indent)}[]`
+  }
+
+  getDefaultField (parent: Parent): string {
+    return `${this.name}: []`
+  }
+
+  getEncoder (parent: Parent): string {
+    const type = parent.findType(this.type)
+    let id = (this.id << 3) | codecTypes[this.type]
+
+    if (type instanceof Message) {
+      id = (this.id << 3) | codecTypes.message
+    }
+
+    if (type instanceof Enum) {
+      id = (this.id << 3) | codecTypes.enum
+    }
+
+    return `
+        if (obj.${this.name} != null) {
+          for (const value of obj.${this.name}) {
+            w.uint32(${id})
+            ${type.getEncoder(this, 'value')}
+          }
+        }`
+  }
+
+  getDecoder (parent: Parent): string {
+    parent.addImport('protons-runtime', 'MaxLengthError')
+
+    let limit = `
+              if (opts.limits?.${this.name} != null && obj.${this.name}.length === opts.limits.${this.name}) {
+                throw new MaxLengthError('Decode error - repeated field "${this.name}" had too many elements')
+              }
+`
+
+    if (this.lengthLimit != null) {
+      limit += `
+              if (obj.${this.name}.length === ${this.lengthLimit}) {
+                throw new MaxLengthError('Decode error - repeated field "${this.name}" had too many elements')
+              }
+`
+    }
+
+    const type: Type = parent.findType(this.type)
+
+    return `case ${this.id}: {${limit}
+              obj.${this.name}.push(${type.getDecoder(this)})
+              break
+            }`
+  }
+
+  getStreamingDecoder (parent: Parent): string {
+    parent.addImport('protons-runtime', 'MaxLengthError')
+
+    let limit = `
+              if (opts.limits?.${this.name} != null && obj.${this.name} === opts.limits.${this.name}) {
+                throw new MaxLengthError('Streaming decode error - repeated field "${this.name}" had too many elements')
+              }
+`
+
+    if (this.lengthLimit != null) {
+      limit += `
+              if (obj.${this.name} === ${this.lengthLimit}) {
+                throw new MaxLengthError('Streaming decode error - repeated field "${this.name}" had too many elements')
+              }
+`
+    }
+
+    const type: Type = parent.findType(this.type)
+
+    return `case ${this.id}: {${limit}
+              ${type.getStreamingDecoder(this, `\`\${prefix != null ? \`\${prefix}.\` : ''}${this.name}\``, '  ')}
+
+              obj.${this.name}++
+
+              break
+            }`
+  }
+
+  getLimitField (): string {
+    return `${this.name}: ${this.lengthLimit ?? 0}`
+  }
+}

packages/protons/src/fields/enum-field.ts

@@ -0,0 +1,53 @@
+import { Enum } from '../types/enum.ts'
+import { Field } from './field.ts'
+import type { FieldDef } from './field.ts'
+import type { Parent } from '../types/index.ts'
+
+export class EnumField extends Field {
+  private enum: Enum
+
+  constructor (name: string, def: FieldDef, parent: Parent) {
+    super(name, def, parent)
+
+    const type = parent.findType(def.type)
+
+    if (!(type instanceof Enum)) {
+      throw new Error(`Type "${def.type}" was not an Enum`)
+    }
+
+    this.enum = type
+  }
+
+  getDefaultField (parent: Parent): string {
+    if (this.optional) {
+      return ''
+    }
+
+    const type = parent.findType(this.type)
+
+    return `${this.name}: ${type.pbType}.${this.enum.lowestValueName}`
+  }
+/*
+  getValueTest (): string {
+    console.info(this.name, 'optional', this.optional, 'proto2required', this.proto2Required)
+
+    if (this.proto2Required) {
+      return 'true'
+    }
+
+    const valueTest = `obj.${this.name} != null`
+
+    // singular enums default to 0, but enums can be defined without a 0
+    // value which is against the proto3 spec but is tolerated
+    if (this.optional || this.proto2Required) {
+      return valueTest
+    }
+
+    if (this.enum.lowestValue === 0) {
+      return `${valueTest} && __${this.type}Values[obj.${this.name}] !== 0`
+    }
+
+    return valueTest
+  }
+    */
+}