diff --git a/package-lock.json b/package-lock.json index 1d57189ac..7c88b0186 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2615,76 +2615,6 @@ ], "peer": true }, - "node_modules/@msgpackr-extract/msgpackr-extract-darwin-x64": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.3.tgz", - "integrity": "sha512-mdzd3AVzYKuUmiWOQ8GNhl64/IoFGol569zNRdkLReh6LRLHOXxU4U8eq0JwaD8iFHdVGqSy4IjFL4reoWCDFw==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "peer": true - }, - "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.3.tgz", - "integrity": "sha512-fg0uy/dG/nZEXfYilKoRe7yALaNmHoYeIoJuJ7KJ+YyU2bvY8vPv27f7UKhGRpY6euFYqEVhxCFZgAUNQBM3nw==", - "cpu": [ - "arm" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "peer": true - }, - "node_modules/@msgpackr-extract/msgpackr-extract-linux-arm64": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.3.tgz", - "integrity": "sha512-YxQL+ax0XqBJDZiKimS2XQaf+2wDGVa1enVRGzEvLLVFeqa5kx2bWbtcSXgsxjQB7nRqqIGFIcLteF/sHeVtQg==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "peer": true - }, - "node_modules/@msgpackr-extract/msgpackr-extract-linux-x64": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.3.tgz", - "integrity": "sha512-cvwNfbP07pKUfq1uH+S6KJ7dT9K8WOE4ZiAcsrSes+UY55E/0jLYc+vq+DO7jlmqRb5zAggExKm0H7O/CBaesg==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "peer": true - }, - "node_modules/@msgpackr-extract/msgpackr-extract-win32-x64": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.3.tgz", - "integrity": "sha512-x0fWaQtYp4E6sktbsdAqnehxDgEc/VwM7uLsRCYWaiGu0ykYdZPiS8zCWdnjHwyiumousxfBm4SO31eXqwEZhQ==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "peer": true - }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -3359,6 +3289,32 @@ "tao": "index.js" } }, + "node_modules/@o1js/native": { + "version": "2.10.0-dev.6d3a3", + "resolved": "https://registry.npmjs.org/@o1js/native/-/native-2.10.0-dev.6d3a3.tgz", + "integrity": "sha512-kPhlySKeT7pBnJqTo2BlNJkgr3ey9aT4Vpo27aF1ZrQtk6W3STbY1bVNzh8fnDPn5blnS2bMpS3P80B+Slf+rQ==", + "peer": true, + "optionalDependencies": { + "@o1js/native-darwin-arm64": "2.10.0-dev.6d3a3", + "@o1js/native-darwin-x64": "2.10.0-dev.6d3a3", + "@o1js/native-linux-arm64": "2.10.0-dev.6d3a3", + "@o1js/native-linux-x64": "2.10.0-dev.6d3a3", + "@o1js/native-win32-x64": "2.10.0-dev.6d3a3" + } + }, + "node_modules/@o1js/native-darwin-arm64": { + "version": "2.10.0-dev.6d3a3", + "resolved": "https://registry.npmjs.org/@o1js/native-darwin-arm64/-/native-darwin-arm64-2.10.0-dev.6d3a3.tgz", + "integrity": "sha512-1qeaDPBO+E9obvaGa1TgGUmh3or9HsHJLQNZlNtlSZX21XlVWLdbAcXgVCA+GFDihcEItbJGjLvIleHzlz0hJQ==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "darwin" + ], + "peer": true + }, "node_modules/@octokit/action": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/@octokit/action/-/action-6.1.0.tgz", @@ -5290,26 +5246,6 @@ "url": "https://opencollective.com/parcel" } }, - "node_modules/@parcel/watcher-android-arm64": { - "version": "2.5.6", - "resolved": "https://registry.npmjs.org/@parcel/watcher-android-arm64/-/watcher-android-arm64-2.5.6.tgz", - "integrity": "sha512-YQxSS34tPF/6ZG7r/Ih9xy+kP/WwediEUsqmtf0cuCV5TPPKw/PQHRhueUo6JdeFJaqV3pyjm0GdYjZotbRt/A==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">= 10.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, "node_modules/@parcel/watcher-darwin-arm64": { "version": "2.5.6", "resolved": "https://registry.npmjs.org/@parcel/watcher-darwin-arm64/-/watcher-darwin-arm64-2.5.6.tgz", @@ -5330,226 +5266,6 @@ "url": "https://opencollective.com/parcel" } }, - "node_modules/@parcel/watcher-darwin-x64": { - "version": "2.5.6", - "resolved": "https://registry.npmjs.org/@parcel/watcher-darwin-x64/-/watcher-darwin-x64-2.5.6.tgz", - "integrity": "sha512-HgvOf3W9dhithcwOWX9uDZyn1lW9R+7tPZ4sug+NGrGIo4Rk1hAXLEbcH1TQSqxts0NYXXlOWqVpvS1SFS4fRg==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">= 10.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/@parcel/watcher-freebsd-x64": { - "version": "2.5.6", - "resolved": "https://registry.npmjs.org/@parcel/watcher-freebsd-x64/-/watcher-freebsd-x64-2.5.6.tgz", - "integrity": "sha512-vJVi8yd/qzJxEKHkeemh7w3YAn6RJCtYlE4HPMoVnCpIXEzSrxErBW5SJBgKLbXU3WdIpkjBTeUNtyBVn8TRng==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">= 10.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/@parcel/watcher-linux-arm-glibc": { - "version": "2.5.6", - "resolved": "https://registry.npmjs.org/@parcel/watcher-linux-arm-glibc/-/watcher-linux-arm-glibc-2.5.6.tgz", - "integrity": "sha512-9JiYfB6h6BgV50CCfasfLf/uvOcJskMSwcdH1PHH9rvS1IrNy8zad6IUVPVUfmXr+u+Km9IxcfMLzgdOudz9EQ==", - "cpu": [ - "arm" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/@parcel/watcher-linux-arm-musl": { - "version": "2.5.6", - "resolved": "https://registry.npmjs.org/@parcel/watcher-linux-arm-musl/-/watcher-linux-arm-musl-2.5.6.tgz", - "integrity": "sha512-Ve3gUCG57nuUUSyjBq/MAM0CzArtuIOxsBdQ+ftz6ho8n7s1i9E1Nmk/xmP323r2YL0SONs1EuwqBp2u1k5fxg==", - "cpu": [ - "arm" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/@parcel/watcher-linux-arm64-glibc": { - "version": "2.5.6", - "resolved": "https://registry.npmjs.org/@parcel/watcher-linux-arm64-glibc/-/watcher-linux-arm64-glibc-2.5.6.tgz", - "integrity": "sha512-f2g/DT3NhGPdBmMWYoxixqYr3v/UXcmLOYy16Bx0TM20Tchduwr4EaCbmxh1321TABqPGDpS8D/ggOTaljijOA==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/@parcel/watcher-linux-arm64-musl": { - "version": "2.5.6", - "resolved": "https://registry.npmjs.org/@parcel/watcher-linux-arm64-musl/-/watcher-linux-arm64-musl-2.5.6.tgz", - "integrity": "sha512-qb6naMDGlbCwdhLj6hgoVKJl2odL34z2sqkC7Z6kzir8b5W65WYDpLB6R06KabvZdgoHI/zxke4b3zR0wAbDTA==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/@parcel/watcher-linux-x64-glibc": { - "version": "2.5.6", - "resolved": "https://registry.npmjs.org/@parcel/watcher-linux-x64-glibc/-/watcher-linux-x64-glibc-2.5.6.tgz", - "integrity": "sha512-kbT5wvNQlx7NaGjzPFu8nVIW1rWqV780O7ZtkjuWaPUgpv2NMFpjYERVi0UYj1msZNyCzGlaCWEtzc+exjMGbQ==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/@parcel/watcher-linux-x64-musl": { - "version": "2.5.6", - "resolved": "https://registry.npmjs.org/@parcel/watcher-linux-x64-musl/-/watcher-linux-x64-musl-2.5.6.tgz", - "integrity": "sha512-1JRFeC+h7RdXwldHzTsmdtYR/Ku8SylLgTU/reMuqdVD7CtLwf0VR1FqeprZ0eHQkO0vqsbvFLXUmYm/uNKJBg==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/@parcel/watcher-win32-arm64": { - "version": "2.5.6", - "resolved": "https://registry.npmjs.org/@parcel/watcher-win32-arm64/-/watcher-win32-arm64-2.5.6.tgz", - "integrity": "sha512-3ukyebjc6eGlw9yRt678DxVF7rjXatWiHvTXqphZLvo7aC5NdEgFufVwjFfY51ijYEWpXbqF5jtrK275z52D4Q==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 10.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/@parcel/watcher-win32-ia32": { - "version": "2.5.6", - "resolved": "https://registry.npmjs.org/@parcel/watcher-win32-ia32/-/watcher-win32-ia32-2.5.6.tgz", - "integrity": "sha512-k35yLp1ZMwwee3Ez/pxBi5cf4AoBKYXj00CZ80jUz5h8prpiaQsiRPKQMxoLstNuqe2vR4RNPEAEcjEFzhEz/g==", - "cpu": [ - "ia32" - ], - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 10.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/@parcel/watcher-win32-x64": { - "version": "2.5.6", - "resolved": "https://registry.npmjs.org/@parcel/watcher-win32-x64/-/watcher-win32-x64-2.5.6.tgz", - "integrity": "sha512-hbQlYcCq5dlAX9Qx+kFb0FHue6vbjlf0FrNzSKdYK2APUf7tGfGxQCk2ihEREmbR6ZMc0MVAD5RIX/41gpUzTw==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 10.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, "node_modules/@phenomnomnominal/tsquery": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/@phenomnomnominal/tsquery/-/tsquery-5.0.1.tgz", @@ -6216,150 +5932,6 @@ "node": ">=10" } }, - "node_modules/@swc/core-darwin-x64": { - "version": "1.15.13", - "resolved": "https://registry.npmjs.org/@swc/core-darwin-x64/-/core-darwin-x64-1.15.13.tgz", - "integrity": "sha512-cVifxQUKhaE7qcO/y9Mq6PEhoyvN9tSLzCnnFZ4EIabFHBuLtDDO6a+vLveOy98hAs5Qu1+bb5Nv0oa1Pihe3Q==", - "cpu": [ - "x64" - ], - "license": "Apache-2.0 AND MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=10" - } - }, - "node_modules/@swc/core-linux-arm-gnueabihf": { - "version": "1.15.13", - "resolved": "https://registry.npmjs.org/@swc/core-linux-arm-gnueabihf/-/core-linux-arm-gnueabihf-1.15.13.tgz", - "integrity": "sha512-t+xxEzZ48enl/wGGy7SRYd7kImWQ/+wvVFD7g5JZo234g6/QnIgZ+YdfIyjHB+ZJI3F7a2IQHS7RNjxF29UkWw==", - "cpu": [ - "arm" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=10" - } - }, - "node_modules/@swc/core-linux-arm64-gnu": { - "version": "1.15.13", - "resolved": "https://registry.npmjs.org/@swc/core-linux-arm64-gnu/-/core-linux-arm64-gnu-1.15.13.tgz", - "integrity": "sha512-VndeGvKmTXFn6AGwjy0Kg8i7HccOCE7Jt/vmZwRxGtOfNZM1RLYRQ7MfDLo6T0h1Bq6eYzps3L5Ma4zBmjOnOg==", - "cpu": [ - "arm64" - ], - "license": "Apache-2.0 AND MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=10" - } - }, - "node_modules/@swc/core-linux-arm64-musl": { - "version": "1.15.13", - "resolved": "https://registry.npmjs.org/@swc/core-linux-arm64-musl/-/core-linux-arm64-musl-1.15.13.tgz", - "integrity": "sha512-SmZ9m+XqCB35NddHCctvHFLqPZDAs5j8IgD36GoutufDJmeq2VNfgk5rQoqNqKmAK3Y7iFdEmI76QoHIWiCLyw==", - "cpu": [ - "arm64" - ], - "license": "Apache-2.0 AND MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=10" - } - }, - "node_modules/@swc/core-linux-x64-gnu": { - "version": "1.15.13", - "resolved": "https://registry.npmjs.org/@swc/core-linux-x64-gnu/-/core-linux-x64-gnu-1.15.13.tgz", - "integrity": "sha512-5rij+vB9a29aNkHq72EXI2ihDZPszJb4zlApJY4aCC/q6utgqFA6CkrfTfIb+O8hxtG3zP5KERETz8mfFK6A0A==", - "cpu": [ - "x64" - ], - "license": "Apache-2.0 AND MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=10" - } - }, - "node_modules/@swc/core-linux-x64-musl": { - "version": "1.15.13", - "resolved": "https://registry.npmjs.org/@swc/core-linux-x64-musl/-/core-linux-x64-musl-1.15.13.tgz", - "integrity": "sha512-OlSlaOK9JplQ5qn07WiBLibkOw7iml2++ojEXhhR3rbWrNEKCD7sd8+6wSavsInyFdw4PhLA+Hy6YyDBIE23Yw==", - "cpu": [ - "x64" - ], - "license": "Apache-2.0 AND MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=10" - } - }, - "node_modules/@swc/core-win32-arm64-msvc": { - "version": "1.15.13", - "resolved": "https://registry.npmjs.org/@swc/core-win32-arm64-msvc/-/core-win32-arm64-msvc-1.15.13.tgz", - "integrity": "sha512-zwQii5YVdsfG8Ti9gIKgBKZg8qMkRZxl+OlYWUT5D93Jl4NuNBRausP20tfEkQdAPSRrMCSUZBM6FhW7izAZRg==", - "cpu": [ - "arm64" - ], - "license": "Apache-2.0 AND MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=10" - } - }, - "node_modules/@swc/core-win32-ia32-msvc": { - "version": "1.15.13", - "resolved": "https://registry.npmjs.org/@swc/core-win32-ia32-msvc/-/core-win32-ia32-msvc-1.15.13.tgz", - "integrity": "sha512-hYXvyVVntqRlYoAIDwNzkS3tL2ijP3rxyWQMNKaxcCxxkCDto/w3meOK/OB6rbQSkNw0qTUcBfU9k+T0ptYdfQ==", - "cpu": [ - "ia32" - ], - "license": "Apache-2.0 AND MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=10" - } - }, - "node_modules/@swc/core-win32-x64-msvc": { - "version": "1.15.13", - "resolved": "https://registry.npmjs.org/@swc/core-win32-x64-msvc/-/core-win32-x64-msvc-1.15.13.tgz", - "integrity": "sha512-XTzKs7c/vYCcjmcwawnQvlHHNS1naJEAzcBckMI5OJlnrcgW8UtcX9NHFYvNjGtXuKv0/9KvqL4fuahdvlNGKw==", - "cpu": [ - "x64" - ], - "license": "Apache-2.0 AND MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=10" - } - }, "node_modules/@swc/counter": { "version": "0.1.3", "resolved": "https://registry.npmjs.org/@swc/counter/-/counter-0.1.3.tgz", @@ -22411,9 +21983,9 @@ } }, "node_modules/o1js": { - "version": "2.13.0", - "resolved": "https://registry.npmjs.org/o1js/-/o1js-2.13.0.tgz", - "integrity": "sha512-kRmOfILkZkpfiTfVEqhfGJkQq8csUYzkFnSgSnH/DIEY3uFgNfXpcZQXIvk0ZEIuDyL98sH3hwxR/DCJnUFndA==", + "version": "2.10.0-dev.6d3a3", + "resolved": "https://registry.npmjs.org/o1js/-/o1js-2.10.0-dev.6d3a3.tgz", + "integrity": "sha512-QoTFmcgrWj/6GTBvKxh6DGyfhPsslXWkKfG0FZv3tBL0OunFeQKQr9SORgXbGK/c1s0KDSEqIKrP6OWLAsz1FA==", "license": "Apache-2.0", "peer": true, "dependencies": { @@ -22430,6 +22002,9 @@ }, "engines": { "node": ">=18.14.0" + }, + "optionalDependencies": { + "@o1js/native": "2.10.0-dev.6d3a3" } }, "node_modules/o1js/node_modules/cachedir": { @@ -28967,11 +28542,12 @@ "@types/ws": "^8.5.4" }, "peerDependencies": { + "@o1js/native": "2.10.0-dev.6d3a3", "@proto-kit/common": "*", "@proto-kit/module": "*", "@proto-kit/protocol": "*", "@proto-kit/sequencer": "*", - "o1js": "^2.10.0", + "o1js": "2.10.0-dev.6d3a3", "tsyringe": "^4.10.0" } }, @@ -28999,6 +28575,7 @@ "@types/yargs": "17.0.32" }, "peerDependencies": { + "@o1js/native": "2.10.0-dev.6d3a3", "@proto-kit/api": "*", "@proto-kit/common": "*", "@proto-kit/indexer": "*", @@ -29008,7 +28585,7 @@ "@proto-kit/sdk": "*", "@proto-kit/sequencer": "*", "@proto-kit/stack": "*", - "o1js": "^2.10.0", + "o1js": "2.10.0-dev.6d3a3", "tsyringe": "^4.10.0" } }, @@ -29145,7 +28722,8 @@ "@types/lodash": "^4.14.194" }, "peerDependencies": { - "o1js": "^2.10.0", + "@o1js/native": "2.10.0-dev.6d3a3", + "o1js": "2.10.0-dev.6d3a3", "tsyringe": "^4.10.0" } }, @@ -29176,12 +28754,13 @@ "cachedir": "^2.4.0" }, "peerDependencies": { + "@o1js/native": "2.10.0-dev.6d3a3", "@proto-kit/common": "*", "@proto-kit/persistance": "*", "@proto-kit/sdk": "*", "@proto-kit/sequencer": "*", "bullmq": "^4.18.3", - "o1js": "^2.10.0", + "o1js": "2.10.0-dev.6d3a3", "tsyringe": "^4.10.0" } }, @@ -29229,6 +28808,7 @@ "@types/ws": "^8.5.4" }, "peerDependencies": { + "@o1js/native": "2.10.0-dev.6d3a3", "@proto-kit/api": "*", "@proto-kit/common": "*", "@proto-kit/library": "*", @@ -29238,7 +28818,7 @@ "@proto-kit/sdk": "*", "@proto-kit/sequencer": "*", "koa": "^2.14.2", - "o1js": "^2.10.0", + "o1js": "2.10.0-dev.6d3a3", "tsyringe": "^4.10.0", "type-graphql": "2.0.0-rc.2", "typegraphql-prisma": "0.28" @@ -29259,11 +28839,12 @@ "@types/lodash": "^4.14.194" }, "peerDependencies": { + "@o1js/native": "2.10.0-dev.6d3a3", "@proto-kit/common": "*", "@proto-kit/module": "*", "@proto-kit/protocol": "*", "@proto-kit/sequencer": "*", - "o1js": "^2.10.0", + "o1js": "2.10.0-dev.6d3a3", "tsyringe": "^4.10.0" } }, @@ -29281,9 +28862,10 @@ "@types/lodash": "^4.14.194" }, "peerDependencies": { + "@o1js/native": "2.10.0-dev.6d3a3", "@proto-kit/common": "*", "@proto-kit/protocol": "*", - "o1js": "^2.10.0", + "o1js": "2.10.0-dev.6d3a3", "tsyringe": "^4.10.0" } }, @@ -29303,11 +28885,12 @@ "@types/lodash": "^4.14.194" }, "peerDependencies": { + "@o1js/native": "2.10.0-dev.6d3a3", "@proto-kit/common": "*", "@proto-kit/module": "*", "@proto-kit/protocol": "*", "@proto-kit/sequencer": "*", - "o1js": "^2.10.0", + "o1js": "2.10.0-dev.6d3a3", "tsyringe": "^4.10.0" } }, @@ -29342,6 +28925,7 @@ "@types/node": "^20.8.10" }, "peerDependencies": { + "@o1js/native": "2.10.0-dev.6d3a3", "@proto-kit/api": "*", "@proto-kit/common": "*", "@proto-kit/library": "*", @@ -29350,7 +28934,7 @@ "@proto-kit/protocol": "*", "@proto-kit/sdk": "*", "@proto-kit/sequencer": "*", - "o1js": "^2.10.0", + "o1js": "2.10.0-dev.6d3a3", "tsyringe": "^4.10.0", "type-graphql": "2.0.0-rc.2", "typegraphql-prisma": "^0.28" @@ -29371,8 +28955,9 @@ "@types/lodash": "^4.14.194" }, "peerDependencies": { + "@o1js/native": "2.10.0-dev.6d3a3", "@proto-kit/common": "*", - "o1js": "^2.10.0", + "o1js": "2.10.0-dev.6d3a3", "ts-pattern": "^4.3.0", "tsyringe": "^4.10.0" } @@ -29393,12 +28978,13 @@ "@types/lodash": "^4.14.194" }, "peerDependencies": { + "@o1js/native": "2.10.0-dev.6d3a3", "@proto-kit/common": "*", "@proto-kit/library": "*", "@proto-kit/module": "*", "@proto-kit/protocol": "*", "@proto-kit/sequencer": "*", - "o1js": "^2.10.0", + "o1js": "2.10.0-dev.6d3a3", "tsyringe": "^4.10.0" } }, @@ -29412,7 +28998,8 @@ "lodash-es": "^4.17.21", "mina-fungible-token": "^1.1.0", "reflect-metadata": "^0.1.13", - "ts-pattern": "^4.3.0" + "ts-pattern": "^4.3.0", + "typescript-memoize": "^1.1.1" }, "devDependencies": { "@jest/globals": "^29.5.0", @@ -29421,10 +29008,11 @@ "@types/node": "^20.2.5" }, "peerDependencies": { + "@o1js/native": "2.10.0-dev.6d3a3", "@proto-kit/common": "*", "@proto-kit/module": "*", "@proto-kit/protocol": "*", - "o1js": "^2.10.0", + "o1js": "2.10.0-dev.6d3a3", "tsyringe": "^4.10.0" } }, @@ -29492,6 +29080,7 @@ "cachedir": "^2.4.0" }, "peerDependencies": { + "@o1js/native": "2.10.0-dev.6d3a3", "@proto-kit/api": "*", "@proto-kit/common": "*", "@proto-kit/deployment": "*", @@ -29503,7 +29092,7 @@ "@proto-kit/protocol": "*", "@proto-kit/sdk": "*", "@proto-kit/sequencer": "*", - "o1js": "^2.10.0", + "o1js": "2.10.0-dev.6d3a3", "tsyringe": "^4.10.0" } }, diff --git a/packages/api/package.json b/packages/api/package.json index d91f5108c..91a97800b 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -49,7 +49,8 @@ "@proto-kit/module": "*", "@proto-kit/protocol": "*", "@proto-kit/sequencer": "*", - "o1js": "^2.10.0", + "o1js": "2.10.0-dev.6d3a3", + "@o1js/native": "2.10.0-dev.6d3a3", "tsyringe": "^4.10.0" }, "devDependencies": { diff --git a/packages/api/src/graphql/modules/LinkedMerkleWitnessResolver.ts b/packages/api/src/graphql/modules/LinkedMerkleWitnessResolver.ts index 58a9b29c7..8f47aaaa9 100644 --- a/packages/api/src/graphql/modules/LinkedMerkleWitnessResolver.ts +++ b/packages/api/src/graphql/modules/LinkedMerkleWitnessResolver.ts @@ -6,6 +6,7 @@ import { } from "@proto-kit/common"; import { AsyncLinkedLeafStore, + AsyncMerkleTreeStore, CachedLinkedLeafStore, } from "@proto-kit/sequencer"; @@ -42,7 +43,9 @@ export class LinkedMerkleWitnessDTO { export class LinkedMerkleWitnessResolver extends GraphqlModule { public constructor( @inject("AsyncLinkedLeafStore") - private readonly treeStore: AsyncLinkedLeafStore + private readonly leafStore: AsyncLinkedLeafStore, + @inject("AsyncTreeStore") + private readonly treeStore: AsyncMerkleTreeStore ) { super(); } @@ -52,7 +55,10 @@ export class LinkedMerkleWitnessResolver extends GraphqlModule { "Allows retrieval of merkle witnesses corresponding to a specific path in the appchain's state tree. These proves are generally retrieved from the current 'proven' state", }) public async witness(@Arg("path") path: string) { - const syncStore = await CachedLinkedLeafStore.new(this.treeStore); + const syncStore = await CachedLinkedLeafStore.new( + this.leafStore, + this.treeStore + ); const tree = new LinkedMerkleTree(syncStore.treeStore, syncStore); await syncStore.preloadKey(BigInt(path)); diff --git a/packages/api/src/graphql/modules/MerkleWitnessResolver.ts b/packages/api/src/graphql/modules/MerkleWitnessResolver.ts index a39324ab0..8464b9577 100644 --- a/packages/api/src/graphql/modules/MerkleWitnessResolver.ts +++ b/packages/api/src/graphql/modules/MerkleWitnessResolver.ts @@ -9,6 +9,7 @@ import { } from "@proto-kit/common"; import { AsyncLinkedLeafStore, + AsyncMerkleTreeStore, CachedLinkedLeafStore, } from "@proto-kit/sequencer"; @@ -92,7 +93,9 @@ export class LinkedTreeWitnessDTO { export class MerkleWitnessResolver extends GraphqlModule { public constructor( @inject("AsyncLinkedLeafStore") - private readonly treeStore: AsyncLinkedLeafStore + private readonly leafStore: AsyncLinkedLeafStore, + @inject("AsyncTreeStore") + private readonly treeStore: AsyncMerkleTreeStore ) { super(); } @@ -102,7 +105,10 @@ export class MerkleWitnessResolver extends GraphqlModule { "Allows retrieval of merkle witnesses corresponding to a specific path in the appchain's state tree. These proves are generally retrieved from the current 'proven' state", }) public async witness(@Arg("path") path: string) { - const syncStore = await CachedLinkedLeafStore.new(this.treeStore); + const syncStore = await CachedLinkedLeafStore.new( + this.leafStore, + this.treeStore + ); await syncStore.preloadKey(BigInt(path)); const tree = new LinkedMerkleTree(syncStore.treeStore, syncStore); diff --git a/packages/cli/package.json b/packages/cli/package.json index 3f30a7a27..1077d5767 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -41,7 +41,8 @@ "@proto-kit/sequencer": "*", "@proto-kit/stack": "*", "@proto-kit/indexer": "*", - "o1js": "^2.10.0", + "o1js": "2.10.0-dev.6d3a3", + "@o1js/native": "2.10.0-dev.6d3a3", "tsyringe": "^4.10.0" }, "devDependencies": { diff --git a/packages/cli/src/scripts/settlement/deploy.ts b/packages/cli/src/scripts/settlement/deploy.ts index 2d206fb16..14e49c581 100644 --- a/packages/cli/src/scripts/settlement/deploy.ts +++ b/packages/cli/src/scripts/settlement/deploy.ts @@ -1,6 +1,5 @@ import "reflect-metadata"; import { container } from "tsyringe"; -import type { Environment } from "@proto-kit/stack"; import { loadEnvironmentVariables, @@ -11,14 +10,28 @@ import { loadUserModules } from "../../utils/loadUserModules"; export default async function (options: LoadEnvOptions) { try { - const { Provable, PublicKey } = await import("o1js"); + loadEnvironmentVariables(options); + const { Provable, PublicKey, PrivateKey } = await import("o1js"); const { Runtime } = await import("@proto-kit/module"); const { Protocol } = await import("@proto-kit/protocol"); - const { AppChain, Sequencer, SettlementModule, InMemoryDatabase } = - await import("@proto-kit/sequencer"); + const { + AppChain, + Sequencer, + SettlementModule, + InMemoryDatabase, + BatchProducerModule, + BridgingModule, + ConstantFeeStrategy, + InMemoryMinaSigner, + MinaBaseLayer, + PrivateMempool, + LocalTaskQueue, + LocalTaskWorkerModule, + VanillaTaskWorkerModules, + SequencerStartupModule, + } = await import("@proto-kit/sequencer"); - const { DefaultModules, DefaultConfigs } = await import("@proto-kit/stack"); - loadEnvironmentVariables(options); + const { DefaultConfigs } = await import("@proto-kit/stack"); const { runtime, protocol } = await loadUserModules(); const appChain = AppChain.from({ Runtime: Runtime.from(runtime.modules), @@ -28,7 +41,18 @@ export default async function (options: LoadEnvOptions) { }), Sequencer: Sequencer.from({ Database: InMemoryDatabase, - ...DefaultModules.settlementScript(), + BaseLayer: MinaBaseLayer, + FeeStrategy: ConstantFeeStrategy, + BatchProducerModule, + SettlementModule, + SettlementSigner: InMemoryMinaSigner, + BridgingModule, + Mempool: PrivateMempool, + TaskQueue: LocalTaskQueue, + LocalTaskWorker: LocalTaskWorkerModule.from( + VanillaTaskWorkerModules.allTasks() + ), + SequencerStartupModule, }), }); @@ -40,16 +64,46 @@ export default async function (options: LoadEnvOptions) { }, Sequencer: { ...DefaultConfigs.inMemoryDatabase(), - ...DefaultConfigs.settlementScript({ - // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - preset: options.env as Environment, - }), + BaseLayer: { + network: { + // eslint-disable-next-line max-len + // eslint-disable-next-line @typescript-eslint/consistent-type-assertions,@typescript-eslint/no-unsafe-assignment + type: process.env.MINA_NETWORK as any, + graphql: process.env.MINA_NODE_GRAPHQL!, + archive: process.env.MINA_ARCHIVE_GRAPHQL!, + accountManager: process.env.MINA_ACCOUNT_MANAGER!, + }, + }, + SettlementSigner: { + feepayer: PrivateKey.fromBase58( + process.env.PROTOKIT_SEQUENCER_PRIVATE_KEY! + ), + contractKeys: [ + PrivateKey.fromBase58( + process.env.PROTOKIT_SETTLEMENT_CONTRACT_PRIVATE_KEY! + ), + PrivateKey.fromBase58( + process.env.PROTOKIT_DISPATCHER_CONTRACT_PRIVATE_KEY! + ), + PrivateKey.fromBase58( + process.env.PROTOKIT_MINA_BRIDGE_CONTRACT_PRIVATE_KEY! + ), + ], + }, + FeeStrategy: {}, + BatchProducerModule: {}, SettlementModule: { addresses: undefined, }, BridgingModule: { addresses: undefined, }, + SequencerStartupModule: {}, + TaskQueue: { + simulatedDuration: 0, + }, + LocalTaskWorker: VanillaTaskWorkerModules.defaultConfig(), + Mempool: {}, }, }); diff --git a/packages/common/package.json b/packages/common/package.json index 2aa0b697c..ab3af1498 100644 --- a/packages/common/package.json +++ b/packages/common/package.json @@ -26,7 +26,8 @@ "typescript-memoize": "^1.1.1" }, "peerDependencies": { - "o1js": "^2.10.0", + "o1js": "2.10.0-dev.6d3a3", + "@o1js/native": "2.10.0-dev.6d3a3", "tsyringe": "^4.10.0" }, "devDependencies": { diff --git a/packages/common/src/cache/ProxyCache.ts b/packages/common/src/cache/ProxyCache.ts index 6c5a97f43..dce19aef4 100644 --- a/packages/common/src/cache/ProxyCache.ts +++ b/packages/common/src/cache/ProxyCache.ts @@ -9,8 +9,6 @@ export class ProxyCache implements O1Cache { debug = false; - cacheDirectory = this.realCache.cacheDirectory; - public read(header: CacheHeader): Uint8Array | undefined { return this.realCache.read(header); } diff --git a/packages/common/src/compiling/services/ChildVerificationKeyService.ts b/packages/common/src/compiling/services/ChildVerificationKeyService.ts index 1e90738d1..526b522ef 100644 --- a/packages/common/src/compiling/services/ChildVerificationKeyService.ts +++ b/packages/common/src/compiling/services/ChildVerificationKeyService.ts @@ -23,4 +23,12 @@ export class ChildVerificationKeyService { } return artifact.verificationKey; } + + public getAsConstant(name: string) { + const vk = this.getVerificationKey(name); + if (!vk.hash.isConstant()) { + throw new Error("Sanity check - vk hash has to be constant"); + } + return vk; + } } diff --git a/packages/common/src/utils.ts b/packages/common/src/utils.ts index 27a0a036c..56c64ec3f 100644 --- a/packages/common/src/utils.ts +++ b/packages/common/src/utils.ts @@ -311,3 +311,10 @@ export function assertDefined( throw new Error(msg ?? "Value is undefined"); } } + +export function takeFirst(arr: T[]): T { + if (arr.length === 0) { + throw new Error("takeFirst called with empty array"); + } + return arr[0]; +} diff --git a/packages/common/src/zkProgrammable/FeatureFlagsExtension.ts b/packages/common/src/zkProgrammable/FeatureFlagsExtension.ts new file mode 100644 index 000000000..f0338ee9f --- /dev/null +++ b/packages/common/src/zkProgrammable/FeatureFlagsExtension.ts @@ -0,0 +1,27 @@ +import { FeatureFlags } from "o1js"; + +function combineFeatureFlag(a: boolean | undefined, b: boolean | undefined) { + if (a === true || b === true) { + return true; + } else if (a === undefined || b === undefined) { + return undefined; + } else { + return false; + } +} + +export function combineFeatureFlags( + a: FeatureFlags, + b: FeatureFlags +): FeatureFlags { + return { + xor: combineFeatureFlag(a.xor, b.xor), + rot: combineFeatureFlag(a.rot, b.rot), + lookup: combineFeatureFlag(a.lookup, b.lookup), + foreignFieldAdd: combineFeatureFlag(a.foreignFieldAdd, b.foreignFieldAdd), + foreignFieldMul: combineFeatureFlag(a.foreignFieldMul, b.foreignFieldMul), + rangeCheck0: combineFeatureFlag(a.rangeCheck0, b.rangeCheck0), + rangeCheck1: combineFeatureFlag(a.rangeCheck1, b.rangeCheck1), + runtimeTables: combineFeatureFlag(a.runtimeTables, b.runtimeTables), + }; +} diff --git a/packages/common/src/zkProgrammable/ZkProgrammable.ts b/packages/common/src/zkProgrammable/ZkProgrammable.ts index 1736af95f..4983cad40 100644 --- a/packages/common/src/zkProgrammable/ZkProgrammable.ts +++ b/packages/common/src/zkProgrammable/ZkProgrammable.ts @@ -5,15 +5,19 @@ import { Field, Provable, Cache as O1Cache, + DynamicProof, + FlexibleProvable, + FeatureFlags, } from "o1js"; import { Memoize } from "typescript-memoize"; import { log } from "../log"; import { dummyVerificationKey } from "../dummyVerificationKey"; -import { reduceSequential } from "../utils"; +import { mapSequential, reduceSequential } from "../utils"; import type { CompileRegistry } from "../compiling/CompileRegistry"; import { MOCK_PROOF } from "./provableMethod"; +import { combineFeatureFlags } from "./FeatureFlagsExtension"; const errors = { areProofsEnabledNotSet: (name: string) => @@ -52,6 +56,8 @@ export interface PlainZkProgram< PublicOutput = undefined, > { name: string; + publicInputType: FlexibleProvable; + publicOutputType: FlexibleProvable; compile: Compile; verify: Verify; Proof: ReturnType< @@ -75,8 +81,15 @@ export interface PlainZkProgram< }>) >; analyzeMethods: () => Promise< - Record>> + Record< + string, + Awaited> & { + // TODO Properly model ProofClass here + proofs: any[]; + } + > >; + maxProofsVerified: () => Promise<0 | 1 | 2>; } export function verifyToMockable( @@ -125,17 +138,18 @@ export abstract class ZkProgrammable< > { public abstract get areProofsEnabled(): AreProofsEnabled | undefined; - public abstract zkProgramFactory(): PlainZkProgram< - PublicInput, - PublicOutput - >[]; + public abstract zkProgramFactory(): Promise< + PlainZkProgram[] + >; private zkProgramSingleton?: PlainZkProgram[]; @Memoize() - public get zkProgram(): PlainZkProgram[] { + public async zkProgram(): Promise< + PlainZkProgram[] + > { if (this.zkProgramSingleton === undefined) { - this.zkProgramSingleton = this.zkProgramFactory(); + this.zkProgramSingleton = await this.zkProgramFactory(); } return this.zkProgramSingleton.map((bucket) => { @@ -150,9 +164,76 @@ export abstract class ZkProgrammable< }); } + @Memoize() + public async proofType(): Promise> { + const programs = await this.zkProgram(); + + const Template = programs[0].Proof; + const maxProofsVerifeds = await mapSequential(programs, (p) => + p.maxProofsVerified() + ); + // eslint-disable-next-line @typescript-eslint/consistent-type-assertions + const maxProofsVerified = Math.max(...maxProofsVerifeds) as 0 | 1 | 2; + + return class ZkProgrammableProofType extends Proof< + PublicInput, + PublicOutput + > { + static publicInputType = Template.publicInputType; + + static publicOutputType = Template.publicOutputType; + + static maxProofsVerified = maxProofsVerified; + }; + } + + @Memoize() + public async dynamicProofType(): Promise< + typeof DynamicProof + > { + const programs = await this.zkProgram(); + + let maxProofsVerified: 0 | 1 | 2; + let featureFlags: FeatureFlags; + + // We actually only need to compute maxProofsVerified and featuresflags if proofs + // are enabled, otherwise o1js will ignore it anyways. This way startup is a bit + // faster for non-proof environments + if (this.areProofsEnabled?.areProofsEnabled === true) { + const maxProofsVerifieds = await mapSequential( + programs, + async (zkProgram) => await zkProgram.maxProofsVerified() + ); + // eslint-disable-next-line @typescript-eslint/consistent-type-assertions + maxProofsVerified = Math.max(...maxProofsVerifieds) as 0 | 1 | 2; + const featureFlagsSet = await mapSequential( + programs, + async (zkProgram) => await FeatureFlags.fromZkProgram(zkProgram) + ); + featureFlags = featureFlagsSet.reduce(combineFeatureFlags); + } else { + featureFlags = FeatureFlags.allNone; + maxProofsVerified = 0; + } + + return class DynamicProofType extends DynamicProof< + PublicInput, + PublicOutput + > { + static publicInputType = programs[0].publicInputType; + + static publicOutputType = programs[0].publicOutputType; + + static maxProofsVerified = maxProofsVerified; + + static featureFlags = featureFlags; + }; + } + public async compile(registry: CompileRegistry) { + const programs = await this.zkProgram(); return await reduceSequential( - this.zkProgram, + programs, async (acc, program) => { const result = await registry.compile(program); return { diff --git a/packages/common/src/zkProgrammable/provableMethod.ts b/packages/common/src/zkProgrammable/provableMethod.ts index d01b1fef6..595a954a8 100644 --- a/packages/common/src/zkProgrammable/provableMethod.ts +++ b/packages/common/src/zkProgrammable/provableMethod.ts @@ -26,7 +26,7 @@ export function toProver( return async function prover(this: ZkProgrammable) { const { areProofsEnabled } = this.areProofsEnabled!; - const zkProgram = this.zkProgram.find((prog) => + const zkProgram = (await this.zkProgram()).find((prog) => Object.keys(prog.methods).includes(methodName) ); diff --git a/packages/common/test/zkProgrammable/ZkProgrammable.test.ts b/packages/common/test/zkProgrammable/ZkProgrammable.test.ts index e5bf7cb5c..500ecb2c7 100644 --- a/packages/common/test/zkProgrammable/ZkProgrammable.test.ts +++ b/packages/common/test/zkProgrammable/ZkProgrammable.test.ts @@ -11,6 +11,7 @@ import { MOCK_VERIFICATION_KEY, ZkProgrammable, ProvableMethodExecutionContext, + takeFirst, } from "../../src"; const appChainMock: AreProofsEnabled = { @@ -60,7 +61,7 @@ class TestProgrammable extends ZkProgrammable< }; } - public zkProgramFactory() { + public async zkProgramFactory() { const program = ZkProgram({ name: "testprogram", publicInput: TestPublicInput, @@ -89,9 +90,12 @@ class TestProgrammable extends ZkProgrammable< return [ { name: program.name, + publicInputType: program.publicInputType, + publicOutputType: program.publicOutputType, compile: program.compile.bind(program), verify: program.verify.bind(program), analyzeMethods: program.analyzeMethods.bind(program), + maxProofsVerified: program.maxProofsVerified.bind(program), Proof: SelfProof, methods, }, @@ -106,19 +110,21 @@ class OtherTestProgrammable extends ZkProgrammable { super(); } - proofType = this.testProgrammable.zkProgram[0].Proof; - @provableMethod() - public async bar(testProgrammableProof: InstanceType) { + public async bar( + testProgrammableProof: InstanceType< + Awaited> + > + ) { testProgrammableProof.verify(); } - public zkProgramFactory() { + public async zkProgramFactory() { const program = ZkProgram({ name: "testprogram2", methods: { bar: { - privateInputs: [this.testProgrammable.zkProgram[0].Proof], + privateInputs: [await this.testProgrammable.proofType()], method: this.bar.bind(this), }, }, @@ -133,9 +139,12 @@ class OtherTestProgrammable extends ZkProgrammable { return [ { name: program.name, + publicInputType: program.publicInputType, + publicOutputType: program.publicOutputType, compile: program.compile.bind(program), verify: program.verify.bind(program), analyzeMethods: program.analyzeMethods.bind(program), + maxProofsVerified: program.maxProofsVerified.bind(program), Proof: SelfProof, methods, }, @@ -189,7 +198,11 @@ describe("zkProgrammable", () => { testProgrammable = new TestProgrammable(); testProgrammable.areProofsEnabled.setProofsEnabled(areProofsEnabled); zkProgramFactorySpy = jest.spyOn(testProgrammable, "zkProgramFactory"); - artifact = await testProgrammable.zkProgram[0].compile(); + + artifact = await testProgrammable + .zkProgram() + .then((p) => takeFirst(p)) + .then((p) => p.compile()); }, 500_000); describe("zkProgramFactory", () => { @@ -216,7 +229,8 @@ describe("zkProgrammable", () => { it("if proofs are disabled, it should successfully verify mock proofs", async () => { expect.assertions(1); - const proof = new testProgrammable.zkProgram[0].Proof({ + const program = await testProgrammable.zkProgram().then(takeFirst); + const proof = new program.Proof({ proof: MOCK_PROOF, publicInput: new TestPublicInput({ @@ -230,7 +244,7 @@ describe("zkProgrammable", () => { maxProofsVerified: 0, }); - const verified = await testProgrammable.zkProgram[0].verify(proof); + const verified = await program.verify(proof); expect(verified).toBe(shouldVerifyMockProofs); @@ -254,7 +268,10 @@ describe("zkProgrammable", () => { describe("zkProgram interoperability", () => { beforeAll(async () => { otherTestProgrammable = new OtherTestProgrammable(testProgrammable); - await otherTestProgrammable.zkProgram[0].compile(); + await otherTestProgrammable + .zkProgram() + .then(takeFirst) + .then((p) => p.compile()); }, 500_000); it("should successfully pass proof of one zkProgram as input to another zkProgram", async () => { @@ -267,8 +284,10 @@ describe("zkProgrammable", () => { const testProof = await executionContext .current() .result.prove>(); - const testProofVerified = - await testProgrammable.zkProgram[0].verify(testProof); + const zkProgram = await testProgrammable + .zkProgram() + .then(takeFirst); + const testProofVerified = await zkProgram.verify(testProof); // execute bar await otherTestProgrammable.bar(testProof); @@ -277,8 +296,11 @@ describe("zkProgrammable", () => { const otherTestProof = await executionContext .current() .result.prove>(); + const otherZkProgram = await otherTestProgrammable + .zkProgram() + .then(takeFirst); const otherTestProofVerified = - await otherTestProgrammable.zkProgram[0].verify(otherTestProof); + await otherZkProgram.verify(otherTestProof); expect(testProof.publicOutput.bar.toString()).toBe( testPublicInput.foo.toString() diff --git a/packages/deployment/package.json b/packages/deployment/package.json index 56814874a..e96d79144 100644 --- a/packages/deployment/package.json +++ b/packages/deployment/package.json @@ -33,7 +33,8 @@ "@proto-kit/sdk": "*", "@proto-kit/sequencer": "*", "bullmq": "^4.18.3", - "o1js": "^2.10.0", + "o1js": "2.10.0-dev.6d3a3", + "@o1js/native": "2.10.0-dev.6d3a3", "tsyringe": "^4.10.0" }, "devDependencies": { diff --git a/packages/deployment/src/queue/BullQueue.ts b/packages/deployment/src/queue/BullQueue.ts index 2a4f4e640..8c037fa8d 100644 --- a/packages/deployment/src/queue/BullQueue.ts +++ b/packages/deployment/src/queue/BullQueue.ts @@ -94,6 +94,9 @@ export class BullQueue const queue = new Queue(queueName, { connection: redis, + defaultJobOptions: { + attempts: this.config.retryAttempts ?? 2, + }, }); const events = new QueueEvents(queueName, { connection: redis }); diff --git a/packages/indexer/package.json b/packages/indexer/package.json index a9a1819e4..2ec921fa8 100644 --- a/packages/indexer/package.json +++ b/packages/indexer/package.json @@ -43,7 +43,8 @@ "@proto-kit/sdk": "*", "@proto-kit/sequencer": "*", "koa": "^2.14.2", - "o1js": "^2.10.0", + "o1js": "2.10.0-dev.6d3a3", + "@o1js/native": "2.10.0-dev.6d3a3", "tsyringe": "^4.10.0", "type-graphql": "2.0.0-rc.2", "typegraphql-prisma": "0.28" diff --git a/packages/indexer/src/tasks/IndexBlockTask.ts b/packages/indexer/src/tasks/IndexBlockTask.ts index 96d92677c..6e7bdc5ba 100644 --- a/packages/indexer/src/tasks/IndexBlockTask.ts +++ b/packages/indexer/src/tasks/IndexBlockTask.ts @@ -1,5 +1,6 @@ import { BlockQueue, + task, Task, TaskSerializer, TaskWorkerModule, @@ -13,6 +14,7 @@ import { } from "./IndexBlockTaskParameters"; @injectable() +@task() export class IndexBlockTask extends TaskWorkerModule implements Task diff --git a/packages/library/package.json b/packages/library/package.json index 1ab3772eb..c8d8f6feb 100644 --- a/packages/library/package.json +++ b/packages/library/package.json @@ -28,7 +28,8 @@ "@proto-kit/module": "*", "@proto-kit/protocol": "*", "@proto-kit/sequencer": "*", - "o1js": "^2.10.0", + "o1js": "2.10.0-dev.6d3a3", + "@o1js/native": "2.10.0-dev.6d3a3", "tsyringe": "^4.10.0" }, "devDependencies": { diff --git a/packages/library/src/hooks/RuntimeFeeAnalyzerService.ts b/packages/library/src/hooks/RuntimeFeeAnalyzerService.ts index 1e8d2581c..6bb8e13a3 100644 --- a/packages/library/src/hooks/RuntimeFeeAnalyzerService.ts +++ b/packages/library/src/hooks/RuntimeFeeAnalyzerService.ts @@ -82,66 +82,66 @@ export class RuntimeFeeAnalyzerService extends ConfigurableModule - >( - async (accum, program) => { - const [valuesProg, indexesProg] = await accum; - const analyzedMethods = await program.analyzeMethods(); - const [valuesMeth, indexesMeth] = Object.keys(program.methods).reduce< - [FeeTreeValues, FeeIndexes] - >( - // eslint-disable-next-line @typescript-eslint/no-shadow - ([values, indexes], combinedMethodName) => { - const { rows } = analyzedMethods[combinedMethodName]; - // const rows = 1000; - const [moduleName, methodName] = combinedMethodName.split("."); - const methodId = this.runtime.methodIdResolver.getMethodId( - moduleName, - methodName - ); - - /** - * Determine the fee config for the given method id, and merge it with - * the default fee config. - */ - return [ - { - ...values, - - [methodId.toString()]: { - methodId, - - baseFee: - this.config.methods[combinedMethodName]?.baseFee ?? - this.config.baseFee, - - perWeightUnitFee: - this.config.methods[combinedMethodName] - ?.perWeightUnitFee ?? this.config.perWeightUnitFee, - - weight: - this.config.methods[combinedMethodName]?.weight ?? - BigInt(rows), - }, + const programs = await this.runtime.zkProgrammable.zkProgram(); + const [values, indexes] = await programs.reduce< + Promise<[FeeTreeValues, FeeIndexes]> + >( + async (accum, program) => { + const [valuesProg, indexesProg] = await accum; + const analyzedMethods = await program.analyzeMethods(); + const [valuesMeth, indexesMeth] = Object.keys(program.methods).reduce< + [FeeTreeValues, FeeIndexes] + >( + // eslint-disable-next-line @typescript-eslint/no-shadow + ([values, indexes], combinedMethodName) => { + const { rows } = analyzedMethods[combinedMethodName]; + // const rows = 1000; + const [moduleName, methodName] = combinedMethodName.split("."); + const methodId = this.runtime.methodIdResolver.getMethodId( + moduleName, + methodName + ); + + /** + * Determine the fee config for the given method id, and merge it with + * the default fee config. + */ + return [ + { + ...values, + + [methodId.toString()]: { + methodId, + + baseFee: + this.config.methods[combinedMethodName]?.baseFee ?? + this.config.baseFee, + + perWeightUnitFee: + this.config.methods[combinedMethodName]?.perWeightUnitFee ?? + this.config.perWeightUnitFee, + + weight: + this.config.methods[combinedMethodName]?.weight ?? + BigInt(rows), }, - { - ...indexes, - // eslint-disable-next-line no-plusplus - [methodId.toString()]: BigInt(methodCounter++), - }, - ]; - }, - [{}, {}] - ); - return [ - { ...valuesProg, ...valuesMeth }, - { ...indexesProg, ...indexesMeth }, - ]; - }, - Promise.resolve([{}, {}]) - ); + }, + { + ...indexes, + // eslint-disable-next-line no-plusplus + [methodId.toString()]: BigInt(methodCounter++), + }, + ]; + }, + [{}, {}] + ); + return [ + { ...valuesProg, ...valuesMeth }, + { ...indexesProg, ...indexesMeth }, + ]; + }, + Promise.resolve([{}, {}]) + ); const tree = new FeeTree(new InMemoryMerkleTreeStorage()); diff --git a/packages/module/package.json b/packages/module/package.json index 6c6437a0a..a0a793dbb 100644 --- a/packages/module/package.json +++ b/packages/module/package.json @@ -29,7 +29,8 @@ "peerDependencies": { "@proto-kit/common": "*", "@proto-kit/protocol": "*", - "o1js": "^2.10.0", + "o1js": "2.10.0-dev.6d3a3", + "@o1js/native": "2.10.0-dev.6d3a3", "tsyringe": "^4.10.0" }, "gitHead": "8a7eca319272a15162dc4ad04bdc134b1017716d" diff --git a/packages/module/src/runtime/Runtime.ts b/packages/module/src/runtime/Runtime.ts index 24a0f0a43..811fcf380 100644 --- a/packages/module/src/runtime/Runtime.ts +++ b/packages/module/src/runtime/Runtime.ts @@ -76,7 +76,9 @@ export class RuntimeZkProgrammable< return this.runtime.areProofsEnabled; } - public zkProgramFactory(): PlainZkProgram[] { + public async zkProgramFactory(): Promise< + PlainZkProgram[] + > { type Methods = Record< string, { @@ -253,9 +255,12 @@ export class RuntimeZkProgrammable< return { name, + publicInputType: program.publicInputType, + publicOutputType: program.publicOutputType, compile: program.compile.bind(program), verify: program.verify.bind(program), analyzeMethods: program.analyzeMethods.bind(program), + maxProofsVerified: program.maxProofsVerified.bind(program), Proof: SelfProof, methods, }; diff --git a/packages/module/test/modules/Balances.test.ts b/packages/module/test/modules/Balances.test.ts index d61f6a3d9..0217683e6 100644 --- a/packages/module/test/modules/Balances.test.ts +++ b/packages/module/test/modules/Balances.test.ts @@ -81,7 +81,8 @@ describe("balances", () => { "1439144406936083177718146178121957896974210157062549589517697792374542035761"; const expectedStatus = true; - await runtime.zkProgrammable.zkProgram[0].compile(); + const runtimeProgram = await runtime.zkProgrammable.zkProgram(); + await runtimeProgram[0].compile(); await balances.getTotalSupply(); @@ -89,7 +90,7 @@ describe("balances", () => { const proof = await result.prove>(); - const verified = await runtime.zkProgrammable.zkProgram[0].verify(proof); + const verified = await runtimeProgram[0].verify(proof); runtime.zkProgrammable.areProofsEnabled?.setProofsEnabled(false); diff --git a/packages/persistance/package.json b/packages/persistance/package.json index 67f600e6a..ced11f26c 100644 --- a/packages/persistance/package.json +++ b/packages/persistance/package.json @@ -32,7 +32,8 @@ "@proto-kit/module": "*", "@proto-kit/protocol": "*", "@proto-kit/sequencer": "*", - "o1js": "^2.10.0", + "o1js": "2.10.0-dev.6d3a3", + "@o1js/native": "2.10.0-dev.6d3a3", "tsyringe": "^4.10.0" }, "devDependencies": { diff --git a/packages/persistance/src/PrismaDatabaseConnection.ts b/packages/persistance/src/PrismaDatabaseConnection.ts index 7195a810e..c59585419 100644 --- a/packages/persistance/src/PrismaDatabaseConnection.ts +++ b/packages/persistance/src/PrismaDatabaseConnection.ts @@ -13,6 +13,7 @@ import { PrismaBlockStorage } from "./services/prisma/PrismaBlockStorage"; import { PrismaSettlementStorage } from "./services/prisma/PrismaSettlementStorage"; import { PrismaMessageStorage } from "./services/prisma/PrismaMessageStorage"; import { PrismaTransactionStorage } from "./services/prisma/PrismaTransactionStorage"; +import { PrismaLinkedLeafStore } from "./services/prisma/PrismaLinkedLeafStore"; export interface PrismaDatabaseConfig { // Either object-based config or connection string @@ -57,7 +58,7 @@ export class PrismaDatabaseConnection StorageDependencyMinimumDependencies<{ readonly prisma: PrismaDatabaseConnection; }>, - "blockTreeStore" | "asyncLinkedLeafStore" | "unprovenLinkedLeafStore" + "blockTreeStore" | "asyncTreeStore" | "unprovenTreeStore" > { return { asyncStateService: { @@ -94,6 +95,26 @@ export class PrismaDatabaseConnection transactionStorage: { useClass: PrismaTransactionStorage, }, + + asyncLinkedLeafStore: { + useGenerated: (module) => { + return new PrismaLinkedLeafStore( + module.prisma, + module.prisma.tracer, + "batch" + ); + }, + }, + + unprovenLinkedLeafStore: { + useGenerated: (module) => { + return new PrismaLinkedLeafStore( + module.prisma, + module.prisma.tracer, + "block" + ); + }, + }, }; } diff --git a/packages/persistance/src/PrismaRedisDatabase.ts b/packages/persistance/src/PrismaRedisDatabase.ts index 87ef9af46..fe39fa433 100644 --- a/packages/persistance/src/PrismaRedisDatabase.ts +++ b/packages/persistance/src/PrismaRedisDatabase.ts @@ -8,7 +8,6 @@ import { } from "@proto-kit/sequencer"; import { ChildContainerProvider, dependencyFactory } from "@proto-kit/common"; import { PrismaClient } from "@prisma/client"; -import { RedisClientType } from "redis"; import { inject } from "tsyringe"; import { @@ -17,12 +16,9 @@ import { PrismaDatabaseConnection, } from "./PrismaDatabaseConnection"; import { - RedisConnection, RedisConnectionConfig, RedisConnectionModule, - RedisTransaction, } from "./RedisConnection"; -import { PrismaLinkedLeafStore } from "./services/prisma/PrismaLinkedLeafStore"; export interface PrismaRedisCombinedConfig { prisma: PrismaDatabaseConfig; @@ -34,7 +30,7 @@ export interface PrismaRedisCombinedConfig { @dependencyFactory() export class PrismaRedisDatabase extends SequencerModule - implements PrismaConnection, RedisConnection, Database + implements PrismaConnection, Database { public prisma: PrismaDatabaseConnection; @@ -50,14 +46,6 @@ export class PrismaRedisDatabase return this.prisma.prismaClient; } - public get redisClient(): RedisClientType { - return this.redis.redisClient; - } - - public get currentMulti(): RedisTransaction { - return this.redis.currentMulti; - } - public create(childContainerProvider: ChildContainerProvider) { super.create(childContainerProvider); this.prisma.create(childContainerProvider); @@ -68,27 +56,8 @@ export class PrismaRedisDatabase return { ...PrismaDatabaseConnection.dependencies(), ...RedisConnectionModule.dependencies(), - - asyncLinkedLeafStore: { - useGenerated: (module) => { - return new PrismaLinkedLeafStore( - module.prisma, - module.redis, - module.tracer, - "batch" - ); - }, - }, - - unprovenLinkedLeafStore: { - useGenerated: (module) => { - return new PrismaLinkedLeafStore( - module.prisma, - module.redis, - module.tracer, - "block" - ); - }, + TreeDatabase: { + useGenerated: (dbModule) => dbModule.redis, }, }; } @@ -115,7 +84,7 @@ export class PrismaRedisDatabase // TODO Long-term we want to somehow make sure we can rollback one data source // if commiting the other one's transaction fails await this.prisma.executeInTransaction(async () => { - await this.redis.executeInTransaction(f); + await f(); }); } } diff --git a/packages/persistance/src/RedisConnection.ts b/packages/persistance/src/RedisConnection.ts index ace9d9268..bd2b4bd03 100644 --- a/packages/persistance/src/RedisConnection.ts +++ b/packages/persistance/src/RedisConnection.ts @@ -43,14 +43,14 @@ export class RedisConnectionModule public static dependencies(): Pick< StorageDependencyMinimumDependencies<{ redis: RedisConnectionModule }>, - "asyncMerkleStore" | "blockTreeStore" | "unprovenMerkleStore" + "blockTreeStore" | "asyncTreeStore" | "unprovenTreeStore" > { return { - asyncMerkleStore: { + asyncTreeStore: { useGenerated: ({ redis }) => new RedisMerkleTreeStore(redis, redis.tracer), }, - unprovenMerkleStore: { + unprovenTreeStore: { useGenerated: ({ redis }) => new RedisMerkleTreeStore(redis, redis.tracer, "unproven"), }, diff --git a/packages/persistance/src/services/prisma/PrismaLinkedLeafStore.ts b/packages/persistance/src/services/prisma/PrismaLinkedLeafStore.ts index 8b25f1624..a0e37af6a 100644 --- a/packages/persistance/src/services/prisma/PrismaLinkedLeafStore.ts +++ b/packages/persistance/src/services/prisma/PrismaLinkedLeafStore.ts @@ -1,11 +1,9 @@ -import { noop, StoredLeaf } from "@proto-kit/common"; -import { AsyncLinkedLeafStore, Tracer } from "@proto-kit/sequencer"; +import { StoredLeaf } from "@proto-kit/common"; +import { AsyncLinkedLeafStore, trace, Tracer } from "@proto-kit/sequencer"; import { injectable } from "tsyringe"; import { Prisma } from "@prisma/client"; -import { PrismaConnection } from "../../PrismaDatabaseConnection"; -import { RedisMerkleTreeStore } from "../redis/RedisMerkleTreeStore"; -import { RedisConnection } from "../../RedisConnection"; +import type { PrismaConnection } from "../../PrismaDatabaseConnection"; import { Decimal } from "./PrismaStateService"; @@ -13,24 +11,11 @@ import { Decimal } from "./PrismaStateService"; export class PrismaLinkedLeafStore implements AsyncLinkedLeafStore { private cache: StoredLeaf[] = []; - private readonly redisMerkleStore: RedisMerkleTreeStore; - public constructor( private readonly connection: PrismaConnection, - redisConnection: RedisConnection, - tracer: Tracer, + public readonly tracer: Tracer, private readonly mask: string = "base" - ) { - this.redisMerkleStore = new RedisMerkleTreeStore( - redisConnection, - tracer, - mask - ); - } - - public get treeStore() { - return this.redisMerkleStore; - } + ) {} private assertCacheEmpty() { if (this.cache.length > 0) { @@ -38,11 +23,8 @@ export class PrismaLinkedLeafStore implements AsyncLinkedLeafStore { } } - public async openTransaction(): Promise { - noop(); - } - - public async commit(): Promise { + @trace("LinkedLeafStore.commit") + public async flush(): Promise { if (this.cache.length > 0) { const data = this.cache.map((entry) => ({ path: entry.leaf.path.toString(), @@ -126,6 +108,7 @@ export class PrismaLinkedLeafStore implements AsyncLinkedLeafStore { : undefined; } + @trace("getPreviousLeaves", ([paths]) => ({ numPaths: paths.length })) public async getPreviousLeavesAsync(paths: bigint[]) { this.assertCacheEmpty(); diff --git a/packages/processor/package.json b/packages/processor/package.json index 30b9c4554..f3de2b126 100644 --- a/packages/processor/package.json +++ b/packages/processor/package.json @@ -45,7 +45,8 @@ "@proto-kit/protocol": "*", "@proto-kit/sdk": "*", "@proto-kit/sequencer": "*", - "o1js": "^2.10.0", + "o1js": "2.10.0-dev.6d3a3", + "@o1js/native": "2.10.0-dev.6d3a3", "tsyringe": "^4.10.0", "type-graphql": "2.0.0-rc.2", "typegraphql-prisma": "^0.28" diff --git a/packages/protocol/package.json b/packages/protocol/package.json index 70d43e21b..96501819e 100644 --- a/packages/protocol/package.json +++ b/packages/protocol/package.json @@ -25,7 +25,8 @@ }, "peerDependencies": { "@proto-kit/common": "*", - "o1js": "^2.10.0", + "o1js": "2.10.0-dev.6d3a3", + "@o1js/native": "2.10.0-dev.6d3a3", "ts-pattern": "^4.3.0", "tsyringe": "^4.10.0" }, diff --git a/packages/protocol/src/Constants.ts b/packages/protocol/src/Constants.ts index 30f2a732c..1e2de64dd 100644 --- a/packages/protocol/src/Constants.ts +++ b/packages/protocol/src/Constants.ts @@ -1,3 +1,5 @@ +import { log } from "@proto-kit/common"; + const constants = { STATE_TRANSITION_BATCH_SIZE: 4, BLOCK_ARGUMENT_BATCH_SIZE: 4, @@ -9,6 +11,7 @@ const prefix = "PROTOKIT"; export const Constants = { getConstant( name: Key, + // TODO Remove this pattern and delegate parsing to the called - this is bad imo transform: (arg: string) => (typeof constants)[Key] ): (typeof constants)[Key] { const env = process.env[name] ?? process.env[`${prefix}_${name}`]; @@ -18,4 +21,16 @@ export const Constants = { return constants[name]; } }, + + printAllConstants() { + const constantsString = + // eslint-disable-next-line @typescript-eslint/consistent-type-assertions + (Object.keys(constants) as (keyof typeof constants)[]) + .map((name) => { + const constant = Constants.getConstant(name, parseInt); + return `${name}=${constant}`; + }) + .join(", "); + log.info("Protocol constants: ", constantsString); + }, }; diff --git a/packages/protocol/src/prover/block/BlockProvable.ts b/packages/protocol/src/prover/block/BlockProvable.ts index 7ceef4e5a..af1742558 100644 --- a/packages/protocol/src/prover/block/BlockProvable.ts +++ b/packages/protocol/src/prover/block/BlockProvable.ts @@ -1,7 +1,18 @@ -import { Bool, Field, Poseidon, Proof, Provable, Struct } from "o1js"; +import { + Bool, + DynamicProof, + Field, + Poseidon, + Proof, + Provable, + Struct, +} from "o1js"; import { CompilableModule, WithZkProgrammable } from "@proto-kit/common"; -import { StateTransitionProof } from "../statetransition/StateTransitionProvable"; +import { + StateTransitionProverPublicInput, + StateTransitionProverPublicOutput, +} from "../statetransition/StateTransitionProvable"; import { NetworkState } from "../../model/network/NetworkState"; import { TransactionHashList } from "../accumulators/TransactionHashList"; import { MinaActionsHashList } from "../../utils/MinaPrefixedProvableHashList"; @@ -10,7 +21,10 @@ import { WitnessedRootHashList, WitnessedRootWitness, } from "../accumulators/WitnessedRootHashList"; -import { TransactionProof } from "../transaction/TransactionProvable"; +import { + TransactionProverPublicInput, + TransactionProverPublicOutput, +} from "../transaction/TransactionProvable"; import { BundleHashList, FieldTransition } from "../accumulators/BlockHashList"; import { NonMethods } from "../../utils/utils"; import { Constants } from "../../Constants"; @@ -345,6 +359,16 @@ export class BlockProverState { } } +export type DynamicSTProof = DynamicProof< + StateTransitionProverPublicInput, + StateTransitionProverPublicOutput +>; + +export type DynamicTransactionProof = DynamicProof< + TransactionProverPublicInput, + TransactionProverPublicOutput +>; + export type BlockProof = Proof; export interface BlockProvable @@ -368,8 +392,8 @@ export interface BlockProvable batch: BlockArgumentsBatch, deferSTProof: Bool, deferTransactionProof: Bool, - stateTransitionProof: StateTransitionProof, - transactionProof: TransactionProof + stateTransitionProof: DynamicSTProof, + transactionProof: DynamicTransactionProof ) => Promise; merge: ( diff --git a/packages/protocol/src/prover/block/BlockProver.ts b/packages/protocol/src/prover/block/BlockProver.ts index 6f600d896..561609365 100644 --- a/packages/protocol/src/prover/block/BlockProver.ts +++ b/packages/protocol/src/prover/block/BlockProver.ts @@ -2,6 +2,7 @@ import { Bool, Field, Provable, SelfProof, ZkProgram } from "o1js"; import { container, inject, injectable, injectAll } from "tsyringe"; import { AreProofsEnabled, + ChildVerificationKeyService, CompilableModule, CompileArtifact, CompileRegistry, @@ -16,7 +17,6 @@ import { import { ProtocolModule } from "../../protocol/ProtocolModule"; import { - StateTransitionProof, StateTransitionProvable, StateTransitionProverPublicInput, StateTransitionProverPublicOutput, @@ -35,7 +35,6 @@ import { assertEqualsIf } from "../../utils/utils"; import { StateServiceProvider } from "../../state/StateServiceProvider"; import { executeHooks } from "../utils"; import { - TransactionProof, TransactionProvable, TransactionProverPublicInput, TransactionProverPublicOutput, @@ -51,6 +50,8 @@ import { BlockProverPublicOutput, BlockProverState, BlockProverStateInput, + DynamicSTProof, + DynamicTransactionProof, } from "./BlockProvable"; import { BlockHashMerkleTreeWitness, @@ -83,7 +84,8 @@ export class BlockProverProgrammable extends ZkProgrammable< TransactionProverPublicOutput >, private readonly blockHooks: ProvableBlockHook[], - private readonly stateServiceProvider: StateServiceProvider + private readonly stateServiceProvider: StateServiceProvider, + private readonly childVerificationKeyService: ChildVerificationKeyService ) { super(); } @@ -141,7 +143,7 @@ export class BlockProverProgrammable extends ZkProgrammable< } public includeSTProof( - stateTransitionProof: StateTransitionProof, + stateTransitionProof: DynamicSTProof, apply: Bool, stateRoot: Field, pendingSTBatchesHash: Field, @@ -224,7 +226,7 @@ export class BlockProverProgrammable extends ZkProgrammable< private verifySTProof( state: BlockProverState, - stateTransitionProof: StateTransitionProof, + stateTransitionProof: DynamicSTProof, deferSTProof: Bool ) { // Verify ST Proof only if STs have been emitted, @@ -233,7 +235,12 @@ export class BlockProverProgrammable extends ZkProgrammable< const batchesEmpty = state.pendingSTBatches.commitment.equals(Field(0)); const verifyStProof = deferSTProof.not().and(batchesEmpty.not()); log.provable.debug("Verify STProof", verifyStProof); - stateTransitionProof.verifyIf(verifyStProof); + + // Brought in as a constant + const stProofVk = this.childVerificationKeyService.getAsConstant( + "StateTransitionProver" + ); + stateTransitionProof.verifyIf(stProofVk, verifyStProof); // Apply STProof if not deferred const stateProofResult = this.includeSTProof( @@ -250,7 +257,7 @@ export class BlockProverProgrammable extends ZkProgrammable< private verifyTransactionProof( state: BlockProverState, - transactionProof: TransactionProof, + transactionProof: DynamicTransactionProof, deferTransactionProof: Bool ) { // Verify Transaction proof if it has at least 1 tx and it isn't deferred @@ -259,7 +266,10 @@ export class BlockProverProgrammable extends ZkProgrammable< state.bundleList.isEmpty().not() ); - transactionProof.verifyIf(verifyTransactionProof); + // Brought in as a constant + const transactionProofVk = + this.childVerificationKeyService.getAsConstant("TransactionProver"); + transactionProof.verifyIf(transactionProofVk, verifyTransactionProof); // Fast-forward transaction trackers by the results of the aggregated transaction proof // Implicitly, the 'from' values here are asserted against the publicInput, since the hashlists @@ -402,8 +412,8 @@ export class BlockProverProgrammable extends ZkProgrammable< batch: BlockArgumentsBatch, deferSTProof: Bool, deferTransactionProof: Bool, - stateTransitionProof: StateTransitionProof, - transactionProof: TransactionProof + stateTransitionProof: DynamicSTProof, + transactionProof: DynamicTransactionProof ) { const finalize = deferTransactionProof.or(deferSTProof).not(); @@ -432,8 +442,9 @@ export class BlockProverProgrammable extends ZkProgrammable< deferSTProof: Bool, deferTransactionProof: Bool, finalize: Bool, - stateTransitionProof?: StateTransitionProof, - transactionProof?: TransactionProof + // TODO Add typing such that it is clear that either both are undefined or none is + stateTransitionProof?: DynamicSTProof, + transactionProof?: DynamicTransactionProof ): Promise { let state = this.parseState( publicInput, @@ -617,18 +628,19 @@ export class BlockProverProgrammable extends ZkProgrammable< * Recursive linking of proofs is done via the previously * injected StateTransitionProver and the required AppChainProof class */ - public zkProgramFactory(): PlainZkProgram< - BlockProverPublicInput, - BlockProverPublicOutput - >[] { - const { prover, stateTransitionProver, transactionProver } = this; - const StateTransitionProofClass = stateTransitionProver.zkProgram[0].Proof; - const TransactionProofClass = transactionProver.zkProgram[0].Proof; + public async zkProgramFactory(): Promise< + PlainZkProgram[] + > { + const { prover } = this; const proveBlockBatchWithProofs = prover.proveBlockBatchWithProofs.bind(prover); const proveBlockBatchNoProofs = prover.proveBlockBatchNoProofs.bind(prover); const merge = prover.merge.bind(prover); + const dynamicStProofType = + await this.stateTransitionProver.dynamicProofType(); + const dynamicTxProofType = await this.transactionProver.dynamicProofType(); + const program = ZkProgram({ name: "BlockProver", publicInput: BlockProverPublicInput, @@ -643,8 +655,8 @@ export class BlockProverProgrammable extends ZkProgrammable< BlockArgumentsBatch, Bool, Bool, - StateTransitionProofClass, - TransactionProofClass, + dynamicStProofType, + dynamicTxProofType, ], async method( publicInput: BlockProverPublicInput, @@ -654,8 +666,8 @@ export class BlockProverProgrammable extends ZkProgrammable< batch: BlockArgumentsBatch, deferSTProof: Bool, deferTransactionProof: Bool, - stateTransitionProof: StateTransitionProof, - transactionProof: TransactionProof + stateTransitionProof: DynamicSTProof, + transactionProof: DynamicTransactionProof ) { return { publicOutput: await proveBlockBatchWithProofs( @@ -730,11 +742,14 @@ export class BlockProverProgrammable extends ZkProgrammable< return [ { name: program.name, + publicInputType: program.publicInputType, + publicOutputType: program.publicOutputType, compile: program.compile.bind(program), verify: program.verify.bind(program), analyzeMethods: program.analyzeMethods.bind(program), Proof: SelfProofClass, methods, + maxProofsVerified: program.maxProofsVerified.bind(program), }, ]; } @@ -768,7 +783,8 @@ export class BlockProver @injectAll("ProvableBlockHook") blockHooks: ProvableBlockHook[], @inject("StateServiceProvider") - stateServiceProvider: StateServiceProvider + stateServiceProvider: StateServiceProvider, + childVerificationKeyService: ChildVerificationKeyService ) { super(); this.zkProgrammable = new BlockProverProgrammable( @@ -776,16 +792,19 @@ export class BlockProver stateTransitionProver.zkProgrammable, transactionProver.zkProgrammable, blockHooks, - stateServiceProvider + stateServiceProvider, + childVerificationKeyService ); } public async compile( registry: CompileRegistry ): Promise | undefined> { - return await registry.proverNeeded(async () => { + await registry.sideloaded(async () => { await this.stateTransitionProver.compile(registry); await this.transactionProver.compile(registry); + }); + return await registry.proverNeeded(async () => { return await this.zkProgrammable.compile(registry); }); } @@ -816,8 +835,8 @@ export class BlockProver batch: BlockArgumentsBatch, deferSTProof: Bool, deferTransactionProof: Bool, - stateTransitionProof: StateTransitionProof, - transactionProof: TransactionProof + stateTransitionProof: DynamicSTProof, + transactionProof: DynamicTransactionProof ): Promise { return this.zkProgrammable.proveBlockBatchWithProofs( publicInput, diff --git a/packages/protocol/src/prover/statetransition/StateTransitionProver.ts b/packages/protocol/src/prover/statetransition/StateTransitionProver.ts index e0f821cf0..ac7d1eda3 100644 --- a/packages/protocol/src/prover/statetransition/StateTransitionProver.ts +++ b/packages/protocol/src/prover/statetransition/StateTransitionProver.ts @@ -74,10 +74,12 @@ export class StateTransitionProverProgrammable extends ZkProgrammable< return this.stateTransitionProver.areProofsEnabled; } - public zkProgramFactory(): PlainZkProgram< - StateTransitionProverPublicInput, - StateTransitionProverPublicOutput - >[] { + public async zkProgramFactory(): Promise< + PlainZkProgram< + StateTransitionProverPublicInput, + StateTransitionProverPublicOutput + >[] + > { const instance = this; const program = ZkProgram({ @@ -144,6 +146,9 @@ export class StateTransitionProverProgrammable extends ZkProgrammable< analyzeMethods: program.analyzeMethods.bind(program), Proof: SelfProofClass, methods, + publicInputType: program.publicInputType, + publicOutputType: program.publicOutputType, + maxProofsVerified: program.maxProofsVerified.bind(program), }, ]; } diff --git a/packages/protocol/src/prover/transaction/TransactionProver.ts b/packages/protocol/src/prover/transaction/TransactionProver.ts index 1ef581beb..349e7b858 100644 --- a/packages/protocol/src/prover/transaction/TransactionProver.ts +++ b/packages/protocol/src/prover/transaction/TransactionProver.ts @@ -72,6 +72,7 @@ export class TransactionProverZkProgrammable extends ZkProgrammable< > { public constructor( private readonly prover: TransactionProver, + public readonly runtime: WithZkProgrammable, private readonly transactionHooks: ProvableTransactionHook[], private readonly stateServiceProvider: StateServiceProvider, private readonly verificationKeyService: MinimalVKTreeService @@ -366,16 +367,21 @@ export class TransactionProverZkProgrammable extends ZkProgrammable< * Recursive linking of proofs is done via the previously * injected StateTransitionProver and the required AppChainProof class */ - public zkProgramFactory(): PlainZkProgram< - TransactionProverPublicInput, - TransactionProverPublicOutput - >[] { + public async zkProgramFactory(): Promise< + PlainZkProgram< + TransactionProverPublicInput, + TransactionProverPublicOutput + >[] + > { const { prover } = this; const proveTransaction = prover.proveTransaction.bind(prover); const proveTransactions = prover.proveTransactions.bind(prover); const merge = prover.merge.bind(prover); const dummy = prover.dummy.bind(prover); + const runtimeProofType = + await this.runtime.zkProgrammable.dynamicProofType(); + const program = ZkProgram({ name: "TransactionProver", publicInput: TransactionProverPublicInput, @@ -383,7 +389,7 @@ export class TransactionProverZkProgrammable extends ZkProgrammable< methods: { proveTransaction: { - privateInputs: [DynamicRuntimeProof, TransactionProverExecutionData], + privateInputs: [runtimeProofType, TransactionProverExecutionData], async method( publicInput: TransactionProverPublicInput, @@ -469,9 +475,12 @@ export class TransactionProverZkProgrammable extends ZkProgrammable< return [ { name: program.name, + publicInputType: program.publicInputType, + publicOutputType: program.publicOutputType, compile: program.compile.bind(program), verify: program.verify.bind(program), analyzeMethods: program.analyzeMethods.bind(program), + maxProofsVerified: program.maxProofsVerified.bind(program), Proof: SelfProofClass, methods, }, @@ -504,6 +513,7 @@ export class TransactionProver super(); this.zkProgrammable = new TransactionProverZkProgrammable( this, + runtime, transactionHooks, stateServiceProvider, verificationKeyService diff --git a/packages/protocol/src/settlement/contracts/settlement/SettlementBase.ts b/packages/protocol/src/settlement/contracts/settlement/SettlementBase.ts index 050543b5f..c44857d1b 100644 --- a/packages/protocol/src/settlement/contracts/settlement/SettlementBase.ts +++ b/packages/protocol/src/settlement/contracts/settlement/SettlementBase.ts @@ -150,8 +150,8 @@ export abstract class SettlementBase throw new Error("Sanity check - vk hash has to be constant"); } // Verify the blockproof - blockProof.verify(blockProofVk); + // Get and assert on-chain values const stateRoot = this.stateRoot.getAndRequireEquals(); const networkStateHash = this.networkStateHash.getAndRequireEquals(); diff --git a/packages/protocol/test/BlockProver.test.ts b/packages/protocol/test/BlockProver.test.ts index f06ac25a1..8219e9a2a 100644 --- a/packages/protocol/test/BlockProver.test.ts +++ b/packages/protocol/test/BlockProver.test.ts @@ -4,13 +4,12 @@ import { PlainZkProgram, ZkProgrammable, } from "@proto-kit/common"; -import { Bool, Field, Proof, UInt64, ZkProgram } from "o1js"; +import { Field, Proof, UInt64, ZkProgram } from "o1js"; import "reflect-metadata"; import { MethodPublicOutput, NetworkState, - AuthorizedTransaction, StateTransitionProverPublicInput, StateTransitionProverPublicOutput, } from "../src"; @@ -38,7 +37,9 @@ class RuntimeZkProgrammable extends ZkProgrammable< return new MockAppChain(); } - zkProgramFactory(): PlainZkProgram[] { + public async zkProgramFactory(): Promise< + PlainZkProgram[] + > { const program = ZkProgram({ name: "BlockProverTestProgram", publicOutput: MethodPublicOutput, @@ -48,9 +49,12 @@ class RuntimeZkProgrammable extends ZkProgrammable< return [ { name: program.name, + publicInputType: program.publicInputType, + publicOutputType: program.publicOutputType, compile: program.compile.bind(program), verify: program.verify.bind(program), analyzeMethods: program.analyzeMethods.bind(program), + maxProofsVerified: program.maxProofsVerified.bind(program), methods: {}, Proof: ZkProgram.Proof(program), }, diff --git a/packages/sdk/package.json b/packages/sdk/package.json index e0f472c71..a20846755 100644 --- a/packages/sdk/package.json +++ b/packages/sdk/package.json @@ -30,7 +30,8 @@ "@proto-kit/module": "*", "@proto-kit/protocol": "*", "@proto-kit/sequencer": "*", - "o1js": "^2.10.0", + "o1js": "2.10.0-dev.6d3a3", + "@o1js/native": "2.10.0-dev.6d3a3", "tsyringe": "^4.10.0" }, "devDependencies": { diff --git a/packages/sdk/src/query/BlockStorageNetworkStateModule.ts b/packages/sdk/src/query/BlockStorageNetworkStateModule.ts index 274037c14..23cc3598b 100644 --- a/packages/sdk/src/query/BlockStorageNetworkStateModule.ts +++ b/packages/sdk/src/query/BlockStorageNetworkStateModule.ts @@ -38,7 +38,7 @@ export class BlockStorageNetworkStateModule } public async getUnprovenNetworkState(): Promise { - const latestBlock = await this.unprovenStorage.getLatestBlock(); + const latestBlock = await this.unprovenQueue.getLatestBlockAndResult(); return latestBlock?.block.networkState.during; } @@ -47,6 +47,8 @@ export class BlockStorageNetworkStateModule * with afterBundle() hooks executed */ public async getStagedNetworkState(): Promise { + // TODO Result could be null here, add method that specifically looks for the + // last block with a result const result = await this.unprovenStorage.getLatestBlock(); return result?.result.afterNetworkState; } diff --git a/packages/sdk/src/query/StateServiceQueryModule.ts b/packages/sdk/src/query/StateServiceQueryModule.ts index 399fdb617..bdcee959f 100644 --- a/packages/sdk/src/query/StateServiceQueryModule.ts +++ b/packages/sdk/src/query/StateServiceQueryModule.ts @@ -4,6 +4,7 @@ import { CachedLinkedLeafStore, AsyncLinkedLeafStore, AppChainModule, + AsyncMerkleTreeStore, } from "@proto-kit/sequencer"; import { Field } from "o1js"; import { inject, injectable } from "tsyringe"; @@ -30,8 +31,12 @@ export class StateServiceQueryModule ); } - public get treeStore(): AsyncLinkedLeafStore { - return this.sequencer.dependencyContainer.resolve("AsyncLinkedMerkleStore"); + public get leafStore(): AsyncLinkedLeafStore { + return this.sequencer.dependencyContainer.resolve("AsyncLinkedLeafStore"); + } + + public get treeStore(): AsyncMerkleTreeStore { + return this.sequencer.dependencyContainer.resolve("AsyncTreeStore"); } public get(key: Field) { @@ -41,7 +46,10 @@ export class StateServiceQueryModule public async merkleWitness( path: Field ): Promise { - const syncStore = await CachedLinkedLeafStore.new(this.treeStore); + const syncStore = await CachedLinkedLeafStore.new( + this.leafStore, + this.treeStore + ); await syncStore.preloadKey(path.toBigInt()); const tree = new LinkedMerkleTree(syncStore.treeStore, syncStore); diff --git a/packages/sdk/test/stprover-emit-sts.test.ts b/packages/sdk/test/stprover-emit-sts.test.ts index ef9170aed..5283feaf9 100644 --- a/packages/sdk/test/stprover-emit-sts.test.ts +++ b/packages/sdk/test/stprover-emit-sts.test.ts @@ -64,7 +64,7 @@ describe("StateTransition", () => { await appChain.start(); appChain.setSigner(senderKey); - }); + }, 30_000); afterEach(async () => { await appChain.close(); diff --git a/packages/sequencer/package.json b/packages/sequencer/package.json index 421370c2d..22d73a37d 100644 --- a/packages/sequencer/package.json +++ b/packages/sequencer/package.json @@ -23,7 +23,8 @@ "@proto-kit/common": "*", "@proto-kit/module": "*", "@proto-kit/protocol": "*", - "o1js": "^2.10.0", + "o1js": "2.10.0-dev.6d3a3", + "@o1js/native": "2.10.0-dev.6d3a3", "tsyringe": "^4.10.0" }, "devDependencies": { @@ -38,7 +39,8 @@ "lodash-es": "^4.17.21", "mina-fungible-token": "^1.1.0", "reflect-metadata": "^0.1.13", - "ts-pattern": "^4.3.0" + "ts-pattern": "^4.3.0", + "typescript-memoize": "^1.1.1" }, "gitHead": "8a7eca319272a15162dc4ad04bdc134b1017716d" } diff --git a/packages/sequencer/src/helpers/CircuitAnalysisModule.ts b/packages/sequencer/src/helpers/CircuitAnalysisModule.ts index c9b33572f..7b38e1e80 100644 --- a/packages/sequencer/src/helpers/CircuitAnalysisModule.ts +++ b/packages/sequencer/src/helpers/CircuitAnalysisModule.ts @@ -35,10 +35,10 @@ export class CircuitAnalysisModule { const zkProgrammablePromises = await mapSequential( zkProgrammables, - (withZkProgrammable) => - mapSequential( + async (withZkProgrammable) => + await mapSequential( // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - withZkProgrammable.zkProgrammable.zkProgramFactory() as PlainZkProgram< + (await withZkProgrammable.zkProgrammable.zkProgramFactory()) as PlainZkProgram< unknown, unknown >[], diff --git a/packages/sequencer/src/helpers/utils.ts b/packages/sequencer/src/helpers/utils.ts index 3e31f7013..ba8a55d41 100644 --- a/packages/sequencer/src/helpers/utils.ts +++ b/packages/sequencer/src/helpers/utils.ts @@ -1,6 +1,7 @@ import { Field, Proof, DynamicProof } from "o1js"; import { Subclass } from "@proto-kit/protocol"; -import { MOCK_PROOF, TypedClass } from "@proto-kit/common"; +import { mapSequential, MOCK_PROOF, TypedClass } from "@proto-kit/common"; +import { Memoize } from "typescript-memoize"; import { TaskSerializer } from "../worker/flow/Task"; @@ -30,28 +31,37 @@ export function distinctByString string }>( type JsonProof = ReturnType; -abstract class ProofTaskSerializerBase { +abstract class ProofTaskSerializerBase< + PublicInputType, + PublicOutputType, + Type extends + | (TypedClass> & + typeof Proof) + | (TypedClass> & + typeof DynamicProof), +> { protected constructor( - private readonly proofClassInternal: Subclass< - | typeof Proof - | typeof DynamicProof - > + private readonly proofClassInternalFun: () => Promise> ) {} - protected getDummy< - T extends - | Proof - | DynamicProof, - >(c: TypedClass, jsonProof: JsonProof): T { + @Memoize() + protected get proofClass() { + return this.proofClassInternalFun(); + } + + protected async getDummy( + c: Subclass, + jsonProof: JsonProof + ): Promise> { + const proofClass = await this.proofClass; // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment - const publicInput: PublicInputType = - this.proofClassInternal.publicInputType.fromFields( - jsonProof.publicInput.map(Field), - [] - ); + const publicInput: PublicInputType = proofClass.publicInputType.fromFields( + jsonProof.publicInput.map(Field), + [] + ); // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment const publicOutput: PublicOutputType = - this.proofClassInternal.publicOutputType.fromFields( + proofClass.publicOutputType.fromFields( jsonProof.publicOutput.map(Field), [] ); @@ -64,28 +74,29 @@ abstract class ProofTaskSerializerBase { }); } - public toJSON( + public async toJSON( proof: | Proof | DynamicProof - ): string { - return JSON.stringify(this.toJSONProof(proof)); + ): Promise { + return JSON.stringify(await this.toJSONProof(proof)); } - public toJSONProof( + public async toJSONProof( proof: | Proof | DynamicProof - ): JsonProof { + ): Promise { if (proof.proof === MOCK_PROOF) { + const proofClass = await this.proofClass; return { - publicInput: this.proofClassInternal.publicInputType + publicInput: proofClass.publicInputType // eslint-disable-next-line max-len // eslint-disable-next-line @typescript-eslint/consistent-type-assertions,@typescript-eslint/no-unsafe-argument .toFields(proof.publicInput as any) .map(String), - publicOutput: this.proofClassInternal.publicOutputType + publicOutput: proofClass.publicOutputType // eslint-disable-next-line max-len // eslint-disable-next-line @typescript-eslint/consistent-type-assertions,@typescript-eslint/no-unsafe-argument .toFields(proof.publicOutput as any) @@ -100,13 +111,15 @@ abstract class ProofTaskSerializerBase { } export class ProofTaskSerializer - extends ProofTaskSerializerBase + extends ProofTaskSerializerBase< + PublicInputType, + PublicOutputType, + typeof Proof + > implements TaskSerializer> { public constructor( - private readonly proofClass: Subclass< - typeof Proof - > + proofClass: () => Promise> ) { super(proofClass); } @@ -122,20 +135,24 @@ export class ProofTaskSerializer jsonProof: JsonProof ): Promise> { if (jsonProof.proof === MOCK_PROOF) { - return this.getDummy(this.proofClass, jsonProof); + return await this.getDummy(await this.proofClass, jsonProof); } - return await this.proofClass.fromJSON(jsonProof); + return await (await this.proofClass).fromJSON(jsonProof); } } export class DynamicProofTaskSerializer - extends ProofTaskSerializerBase + extends ProofTaskSerializerBase< + PublicInputType, + PublicOutputType, + typeof DynamicProof + > implements TaskSerializer> { public constructor( - private readonly proofClass: Subclass< - typeof DynamicProof + proofClass: () => Promise< + Subclass> > ) { super(proofClass); @@ -151,12 +168,12 @@ export class DynamicProofTaskSerializer public async fromJSONProof( jsonProof: JsonProof ): Promise> { + const proofClass = await this.proofClass; + if (jsonProof.proof === MOCK_PROOF) { - return this.getDummy(this.proofClass, jsonProof); + return await this.getDummy(proofClass, jsonProof); } - const { proofClass } = this; - return await proofClass.fromJSON(jsonProof); } } @@ -169,11 +186,13 @@ export class PairProofTaskSerializer< > implements TaskSerializer< PairTuple> > { - private readonly proofSerializer = new ProofTaskSerializer(this.proofClass); + private readonly proofSerializer = new ProofTaskSerializer( + this.proofClassFun + ); public constructor( - private readonly proofClass: Subclass< - typeof Proof + private readonly proofClassFun: () => Promise< + Subclass> > ) {} @@ -188,11 +207,13 @@ export class PairProofTaskSerializer< ]; } - public toJSON( + public async toJSON( input: PairTuple> - ): string { + ): Promise { return JSON.stringify( - input.map((element) => this.proofSerializer.toJSONProof(element)) + await mapSequential(input, (element) => + this.proofSerializer.toJSONProof(element) + ) ); } } diff --git a/packages/sequencer/src/index.ts b/packages/sequencer/src/index.ts index 7da10194e..2b79ee088 100644 --- a/packages/sequencer/src/index.ts +++ b/packages/sequencer/src/index.ts @@ -23,6 +23,8 @@ export * from "./worker/worker/FlowTaskWorker"; export * from "./worker/worker/LocalTaskWorkerModule"; export * from "./worker/worker/TaskWorkerModule"; export * from "./worker/worker/WorkerReadyModule"; +export * from "./worker/startup/WorkerRegistrationTask"; +export * from "./worker/startup/CloseWorkerError"; export * from "./protocol/baselayer/BaseLayer"; export * from "./protocol/baselayer/MinaBaseLayer"; export * from "./protocol/baselayer/NoopBaseLayer"; @@ -34,10 +36,12 @@ export * from "./protocol/baselayer/fees/ConstantFeeStrategy"; export * from "./protocol/production/helpers/UntypedOption"; export * from "./protocol/production/helpers/UntypedStateTransition"; export * from "./protocol/production/tasks/TransactionProvingTask"; +export * from "./protocol/production/tasks/TransactionReductionTask"; export * from "./protocol/production/tasks/RuntimeProvingTask"; export * from "./protocol/production/tasks/StateTransitionTask"; export * from "./protocol/production/tasks/StateTransitionReductionTask"; export * from "./protocol/production/tasks/NewBlockTask"; +export * from "./protocol/production/tasks/BlockReductionTask"; export * from "./protocol/production/tasks/serializers/ArtifactionRecordSerializer"; export * from "./protocol/production/tasks/serializers/BlockProofSerializer"; export * from "./protocol/production/tasks/serializers/DecodedStateSerializer"; @@ -47,6 +51,10 @@ export * from "./protocol/production/tasks/serializers/RuntimeVerificationKeyAtt export * from "./protocol/production/tasks/serializers/StateTransitionParametersSerializer"; export * from "./protocol/production/tasks/serializers/TransactionProvingTaskParameterSerializer"; export * from "./protocol/production/tasks/serializers/VerificationKeySerializer"; +export * from "./protocol/production/tasks/compile/CircuitCompileTask"; +export * from "./protocol/production/tasks/compile/ProtocolCompileTask"; +export * from "./protocol/production/tasks/compile/RuntimeCompileTask"; +export * from "./protocol/production/tasks/compile/SettlementCompileTask"; export * from "./protocol/production/trigger/BlockTrigger"; export * from "./protocol/production/trigger/ManualBlockTrigger"; export * from "./protocol/production/trigger/TimedBlockTrigger"; @@ -77,6 +85,7 @@ export * from "./storage/repositories/MessageStorage"; export * from "./storage/repositories/TransactionStorage"; export * from "./storage/inmemory/InMemoryDatabase"; export * from "./storage/inmemory/InMemoryAsyncMerkleTreeStore"; +export * from "./storage/inmemory/InMemoryAsyncLinkedLeafStore"; export * from "./storage/inmemory/InMemoryBlockStorage"; export * from "./storage/inmemory/InMemoryBatchStorage"; export * from "./storage/inmemory/InMemorySettlementStorage"; diff --git a/packages/sequencer/src/logging/ConsoleTracer.ts b/packages/sequencer/src/logging/ConsoleTracer.ts index 96fc4b5cb..3e419938d 100644 --- a/packages/sequencer/src/logging/ConsoleTracer.ts +++ b/packages/sequencer/src/logging/ConsoleTracer.ts @@ -14,7 +14,7 @@ type StoreType = Record; @closeable() export class ConsoleTracer implements Tracer { // Hard-code this for the moment. Needs to be configured. - timeInterval: number = 60000; + timeInterval: number = 180000; store: StoreType = {}; diff --git a/packages/sequencer/src/protocol/production/BatchProducerModule.ts b/packages/sequencer/src/protocol/production/BatchProducerModule.ts index 81ee673be..a1e4eec98 100644 --- a/packages/sequencer/src/protocol/production/BatchProducerModule.ts +++ b/packages/sequencer/src/protocol/production/BatchProducerModule.ts @@ -18,6 +18,7 @@ import type { Database } from "../../storage/Database"; import { AsyncLinkedLeafStore } from "../../state/async/AsyncLinkedLeafStore"; import { CachedLinkedLeafStore } from "../../state/lmt/CachedLinkedLeafStore"; import { ensureNotBusy } from "../../helpers/BusyGuard"; +import { AsyncMerkleTreeStore } from "../../state/async/AsyncMerkleTreeStore"; import { BlockProofSerializer } from "./tasks/serializers/BlockProofSerializer"; import { BatchTracingService } from "./tracing/BatchTracingService"; @@ -47,10 +48,14 @@ const errors = { export class BatchProducerModule extends SequencerModule { public constructor( @inject("AsyncLinkedLeafStore") - private readonly merkleStore: AsyncLinkedLeafStore, + private readonly leafStore: AsyncLinkedLeafStore, + @inject("AsyncTreeStore") + private readonly merkleStore: AsyncMerkleTreeStore, @inject("BatchStorage") private readonly batchStorage: BatchStorage, @inject("Database") private readonly database: Database, + @inject("TreeDatabase") + private readonly treeDatabase: Database, private readonly batchFlow: BatchFlow, private readonly blockProofSerializer: BlockProofSerializer, private readonly batchTraceService: BatchTracingService @@ -85,8 +90,11 @@ export class BatchProducerModule extends SequencerModule { // Apply state changes to current MerkleTreeStore await this.database.executeInTransaction(async () => { await this.batchStorage.pushBatch(batchWithStateDiff.batch); - await batchWithStateDiff.changes.mergeIntoParent(); }); + await batchWithStateDiff.changes.mergeIntoParent( + this.database, + this.treeDatabase + ); // TODO Add transition from unproven to proven state for stateservice // This needs proper DB-level masking @@ -106,7 +114,7 @@ export class BatchProducerModule extends SequencerModule { const blockHashes = blocks.map((bundle) => bundle.block.hash.toString()); - const jsonProof = this.blockProofSerializer + const jsonProof = await this.blockProofSerializer .getBlockProofSerializer() .toJSONProof(batch.proof); @@ -149,7 +157,10 @@ export class BatchProducerModule extends SequencerModule { throw errors.blockWithoutTxs(); } - const merkleTreeStore = await CachedLinkedLeafStore.new(this.merkleStore); + const merkleTreeStore = await CachedLinkedLeafStore.new( + this.leafStore, + this.merkleStore + ); const trace = await this.batchTraceService.traceBatch( blocks.map((block) => block), diff --git a/packages/sequencer/src/protocol/production/flow/BatchFlow.ts b/packages/sequencer/src/protocol/production/flow/BatchFlow.ts index e2267ec60..3cc2f9286 100644 --- a/packages/sequencer/src/protocol/production/flow/BatchFlow.ts +++ b/packages/sequencer/src/protocol/production/flow/BatchFlow.ts @@ -65,16 +65,20 @@ export class BatchFlow { } } - private dummySTProof() { - return this.protocol.stateTransitionProver.zkProgrammable.zkProgram[0].Proof.dummy( + private async dummySTProof() { + const program = + await this.protocol.stateTransitionProver.zkProgrammable.zkProgram(); + return await program[0].Proof.dummy( StateTransitionProverPublicInput.empty(), StateTransitionProverPublicOutput.empty(), 2 ); } - private dummyTransactionProof() { - return this.protocol.transactionProver.zkProgrammable.zkProgram[0].Proof.dummy( + private async dummyTransactionProof() { + const program = + await this.protocol.transactionProver.zkProgrammable.zkProgram(); + return await program[0].Proof.dummy( TransactionProverPublicInput.empty(), TransactionProverPublicInput.empty(), 2 diff --git a/packages/sequencer/src/protocol/production/flow/StateTransitionFlow.ts b/packages/sequencer/src/protocol/production/flow/StateTransitionFlow.ts index 6321920b0..16071cfea 100644 --- a/packages/sequencer/src/protocol/production/flow/StateTransitionFlow.ts +++ b/packages/sequencer/src/protocol/production/flow/StateTransitionFlow.ts @@ -37,11 +37,9 @@ export class StateTransitionFlow { witnessedRootsHash: Field(0), }; - return await this.protocol.stateTransitionProver.zkProgrammable.zkProgram[0].Proof.dummy( - emptyInputOutput, - emptyInputOutput, - 2 - ); + const program = + await this.protocol.stateTransitionProver.zkProgrammable.zkProgram(); + return await program[0].Proof.dummy(emptyInputOutput, emptyInputOutput, 2); } private createFlow(name: string, inputLength: number) { diff --git a/packages/sequencer/src/protocol/production/sequencing/BlockProducerModule.ts b/packages/sequencer/src/protocol/production/sequencing/BlockProducerModule.ts index 278de217a..9617048a2 100644 --- a/packages/sequencer/src/protocol/production/sequencing/BlockProducerModule.ts +++ b/packages/sequencer/src/protocol/production/sequencing/BlockProducerModule.ts @@ -44,6 +44,8 @@ export class BlockProducerModule extends SequencerModule { private readonly unprovenStateService: AsyncStateService, @inject("UnprovenLinkedLeafStore") private readonly unprovenLinkedLeafStore: AsyncLinkedLeafStore, + @inject("UnprovenTreeStore") + private readonly unprovenTreeStore: AsyncMerkleTreeStore, @inject("BlockQueue") private readonly blockQueue: BlockQueue, @inject("TransactionStorage") @@ -56,6 +58,7 @@ export class BlockProducerModule extends SequencerModule { private readonly methodIdResolver: MethodIdResolver, @inject("Runtime") private readonly runtime: Runtime, @inject("Database") private readonly database: Database, + @inject("TreeDatabase") private readonly treeDatabase: Database, @inject("Tracer") public readonly tracer: Tracer ) { super(); @@ -118,20 +121,24 @@ export class BlockProducerModule extends SequencerModule { await this.resultService.generateMetadataForNextBlock( block, this.unprovenLinkedLeafStore, + this.unprovenTreeStore, this.blockTreeStore, this.unprovenStateService ); await this.tracer.trace( "block.result.commit", - async () => + async () => { await this.database.executeInTransaction(async () => { - await blockHashTreeStore.mergeIntoParent(); - await treeStore.mergeIntoParent(); - await stateService.mergeIntoParent(); - await this.blockQueue.pushResult(result); - }), + await stateService.mergeIntoParent(); + await treeStore.mergeLeavesIntoParent(); + }); + await this.treeDatabase.executeInTransaction(async () => { + await blockHashTreeStore.mergeIntoParent(); + await treeStore.mergeTreeIntoParent(); + }); + }, traceMetadata ); @@ -140,6 +147,9 @@ export class BlockProducerModule extends SequencerModule { @ensureNotBusy() public async tryProduceBlock(): Promise { + // Check if previous result has been computed, and if not, compute it + await this.blockResultCompleteCheck(); + const block = await this.produceBlock(); if (block === undefined) { @@ -209,7 +219,6 @@ export class BlockProducerModule extends SequencerModule { async () => { // Push changes to the database atomically await this.database.executeInTransaction(async () => { - await stateChanges.mergeIntoParent(); await this.blockQueue.pushBlock(block); // Remove included or dropped txs, leave skipped ones alone @@ -228,6 +237,8 @@ export class BlockProducerModule extends SequencerModule { await this.transactionStorage.reportSkippedTransactions( orderingMetadata.skippedPaths ); + + await stateChanges.mergeIntoParent(); }); }, { @@ -239,19 +250,23 @@ export class BlockProducerModule extends SequencerModule { return blockResult?.block; } - public async blockResultCompleteCheck() { + public async blockResultCompleteCheck(): Promise< + "genesis" | "existent" | "generated" + > { // Check if metadata height is behind block production. // This can happen when the sequencer crashes after a block has been produced // but before the metadata generation has finished const latestBlock = await this.blockQueue.getLatestBlockAndResult(); - // eslint-disable-next-line sonarjs/no-collapsible-if if (latestBlock !== undefined) { if (latestBlock.result === undefined) { await this.generateMetadata(latestBlock.block); + return "generated"; } // Here, the metadata has been computed already + return "existent"; } // If we reach here, its a genesis startup, no blocks exist yet + return "genesis"; } public async start() { diff --git a/packages/sequencer/src/protocol/production/sequencing/BlockResultService.ts b/packages/sequencer/src/protocol/production/sequencing/BlockResultService.ts index 5419344bb..6afb098fa 100644 --- a/packages/sequencer/src/protocol/production/sequencing/BlockResultService.ts +++ b/packages/sequencer/src/protocol/production/sequencing/BlockResultService.ts @@ -188,7 +188,8 @@ export class BlockResultService { })) public async generateMetadataForNextBlock( block: Block, - merkleTreeStore: AsyncLinkedLeafStore, + leafStore: AsyncLinkedLeafStore, + treeStore: AsyncMerkleTreeStore, blockHashTreeStore: AsyncMerkleTreeStore, stateService: AsyncStateService ): Promise<{ @@ -202,7 +203,7 @@ export class BlockResultService { block.beforeBlockStateTransitions ); - const inMemoryStore = await CachedLinkedLeafStore.new(merkleTreeStore); + const inMemoryStore = await CachedLinkedLeafStore.new(leafStore, treeStore); const tree = await this.applyStateDiff(inMemoryStore, combinedDiff); diff --git a/packages/sequencer/src/protocol/production/tasks/BlockReductionTask.ts b/packages/sequencer/src/protocol/production/tasks/BlockReductionTask.ts index 24b5423a0..5ab06d909 100644 --- a/packages/sequencer/src/protocol/production/tasks/BlockReductionTask.ts +++ b/packages/sequencer/src/protocol/production/tasks/BlockReductionTask.ts @@ -8,10 +8,14 @@ import { } from "@proto-kit/protocol"; import { CompileRegistry, + dependencyFactory, ProvableMethodExecutionContext, } from "@proto-kit/common"; -import { TaskWorkerModule } from "../../../worker/worker/TaskWorkerModule"; +import { + task, + TaskWorkerModule, +} from "../../../worker/worker/TaskWorkerModule"; import { Task, TaskSerializer } from "../../../worker/flow/Task"; import { PairProofTaskSerializer, @@ -19,8 +23,12 @@ import { ProofTaskSerializer, } from "../../../helpers/utils"; +import { BlockProverCompileTask } from "./compile/ProtocolCompileTask"; + @injectable() @scoped(Lifecycle.ContainerScoped) +@task() +@dependencyFactory() export class BlockReductionTask extends TaskWorkerModule implements Task, BlockProof> @@ -41,15 +49,23 @@ export class BlockReductionTask this.blockProver = this.protocol.blockProver; } + public static dependencies() { + return { + BlockProverCompileTask: { + useClass: BlockProverCompileTask, + }, + }; + } + public inputSerializer(): TaskSerializer> { - return new PairProofTaskSerializer( - this.blockProver.zkProgrammable.zkProgram[0].Proof + return new PairProofTaskSerializer(() => + this.blockProver.zkProgrammable.proofType() ); } public resultSerializer(): TaskSerializer { - return new ProofTaskSerializer( - this.blockProver.zkProgrammable.zkProgram[0].Proof + return new ProofTaskSerializer(() => + this.blockProver.zkProgrammable.proofType() ); } diff --git a/packages/sequencer/src/protocol/production/tasks/CircuitCompilerTask.ts b/packages/sequencer/src/protocol/production/tasks/CircuitCompilerTask.ts deleted file mode 100644 index 2e14928d8..000000000 --- a/packages/sequencer/src/protocol/production/tasks/CircuitCompilerTask.ts +++ /dev/null @@ -1,196 +0,0 @@ -import { inject, injectable, Lifecycle, scoped } from "tsyringe"; -import { Runtime } from "@proto-kit/module"; -import { - log, - mapSequential, - StringKeyOf, - ArtifactRecord, - CompileRegistry, - CompilableModule, - safeParseJson, - reduceSequential, -} from "@proto-kit/common"; -import { - MandatorySettlementModulesRecord, - Protocol, - SettlementContractModule, - RuntimeVerificationKeyRootService, - MandatoryProtocolModulesRecord, - type SettlementModulesRecord, - BridgingSettlementContractArgs, - ContractArgsRegistry, -} from "@proto-kit/protocol"; - -import { TaskSerializer } from "../../../worker/flow/Task"; -import { UnpreparingTask } from "../../../worker/flow/UnpreparingTask"; -import { SignedSettlementPermissions } from "../../../settlement/permissions/SignedSettlementPermissions"; -import { ProvenSettlementPermissions } from "../../../settlement/permissions/ProvenSettlementPermissions"; - -import { - ArtifactRecordSerializer, - SerializedArtifactRecord, -} from "./serializers/ArtifactionRecordSerializer"; - -export type CompilerTaskParams = { - existingArtifacts: ArtifactRecord; - targets: string[]; - runtimeVKRoot?: string; - isSignedSettlement?: boolean; -}; - -@injectable() -@scoped(Lifecycle.ContainerScoped) -export class CircuitCompilerTask extends UnpreparingTask< - CompilerTaskParams, - ArtifactRecord -> { - public name = "compiledCircuit"; - - public constructor( - @inject("Runtime") protected readonly runtime: Runtime, - @inject("Protocol") - protected readonly protocol: Protocol, - private readonly compileRegistry: CompileRegistry, - private readonly contractArgsRegistry: ContractArgsRegistry - ) { - super(); - } - - public inputSerializer(): TaskSerializer { - type CompilerTaskParamsJSON = { - targets: string[]; - runtimeVKRoot?: string; - existingArtifacts: SerializedArtifactRecord; - isSignedSettlement?: boolean; - }; - - const serializer = new ArtifactRecordSerializer(); - return { - toJSON: (input) => - JSON.stringify({ - targets: input.targets, - runtimeVKRoot: input.runtimeVKRoot, - existingArtifacts: serializer.toJSON(input.existingArtifacts), - isSignedSettlement: input.isSignedSettlement, - } satisfies CompilerTaskParamsJSON), - fromJSON: (input) => { - const json = safeParseJson(input); - return { - targets: json.targets, - runtimeVKRoot: json.runtimeVKRoot, - existingArtifacts: serializer.fromJSON(json.existingArtifacts), - isSignedSettlement: json.isSignedSettlement, - }; - }, - }; - } - - public resultSerializer(): TaskSerializer { - const serializer = new ArtifactRecordSerializer(); - return { - toJSON: (input) => JSON.stringify(serializer.toJSON(input)), - fromJSON: (input) => - serializer.fromJSON(safeParseJson(input)), - }; - } - - public getSettlementTargets(): Record { - // We only care about the BridgeContract for now - later with caching, - // we might want to expand that to all protocol circuits - const container = this.protocol.dependencyContainer; - if (container.isRegistered("SettlementContractModule")) { - const settlementModule = container.resolve< - SettlementContractModule - >("SettlementContractModule"); - - // Needed so that all contractFactory functions are called, because - // they set static args on the contracts - settlementModule.getContractClasses(); - - const moduleNames = - // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - settlementModule.moduleNames as StringKeyOf[]; - - const modules = moduleNames.map<[string, CompilableModule]>((name) => [ - `Settlement.${name}`, - settlementModule.resolve(name), - ]); - - const sumModule = { - compile: async (registry: CompileRegistry) => { - await reduceSequential<[string, CompilableModule], ArtifactRecord>( - modules, - async (record, [moduleName, module]) => { - log.info(`Compiling ${moduleName}`); - const artifacts = await module.compile(registry); - return { - ...record, - ...artifacts, - }; - }, - {} - ); - }, - }; - - const combinedModules = [...modules, ["Settlement", sumModule]]; - - return Object.fromEntries(combinedModules); - } - return {}; - } - - public async compute(input: CompilerTaskParams): Promise { - log.info("Computing VKs"); - - this.compileRegistry.addArtifactsRaw(input.existingArtifacts); - - // We need to initialize the VK tree root if we have it, so that - // the BlockProver can bake in that root - if (input.runtimeVKRoot !== undefined) { - this.protocol.dependencyContainer - .resolve(RuntimeVerificationKeyRootService) - .setRoot(BigInt(input.runtimeVKRoot)); - } - - if (input.isSignedSettlement !== undefined) { - this.contractArgsRegistry.addArgs( - "SettlementContract", - { - signedSettlements: input.isSignedSettlement, - // TODO Add distinction between mina and custom tokens - BridgeContractPermissions: (input.isSignedSettlement - ? new SignedSettlementPermissions() - : new ProvenSettlementPermissions() - ).bridgeContractMina(), - } - ); - } - - // TODO make adaptive - const targets: Record = { - runtime: this.runtime, - protocol: this.protocol.blockProver, - ...this.getSettlementTargets(), - }; - - const msg = `Compiling targets [${input.targets}]`; - log.time(msg); - await mapSequential(input.targets, async (target) => { - if (target in targets) { - await targets[target].compile(this.compileRegistry); - } else { - log.info( - // TODO Is that right? Or should we check that the bridge exists on the sequencer side? - `Compile target ${target} not found, skipping` - ); - } - }); - log.timeEnd.info(msg); - - const newEntries = Object.entries( - this.compileRegistry.getAllArtifacts() - ).filter(([key]) => !(key in input.existingArtifacts)); - return Object.fromEntries(newEntries); - } -} diff --git a/packages/sequencer/src/protocol/production/tasks/NewBlockTask.ts b/packages/sequencer/src/protocol/production/tasks/NewBlockTask.ts index bac58cf3c..f40fbb1a6 100644 --- a/packages/sequencer/src/protocol/production/tasks/NewBlockTask.ts +++ b/packages/sequencer/src/protocol/production/tasks/NewBlockTask.ts @@ -20,16 +20,21 @@ import { Bool } from "o1js"; import { ProvableMethodExecutionContext, CompileRegistry, + dependencyFactory, } from "@proto-kit/common"; import { Task, TaskSerializer } from "../../../worker/flow/Task"; import { ProofTaskSerializer } from "../../../helpers/utils"; -import { TaskWorkerModule } from "../../../worker/worker/TaskWorkerModule"; +import { + task, + TaskWorkerModule, +} from "../../../worker/worker/TaskWorkerModule"; import { PairingDerivedInput } from "../flow/ReductionTaskFlow"; import type { TaskStateRecord } from "../tracing/BlockTracingService"; import { NewBlockProvingParametersSerializer } from "./serializers/NewBlockProvingParametersSerializer"; import { executeWithPrefilledStateService } from "./TransactionProvingTask"; +import { BlockProverCompileTask } from "./compile/ProtocolCompileTask"; export type NewBlockArguments = { args: BlockArguments; @@ -55,6 +60,8 @@ export type NewBlockProvingParameters = PairingDerivedInput< @injectable() @scoped(Lifecycle.ContainerScoped) +@task() +@dependencyFactory() export class NewBlockTask extends TaskWorkerModule implements Task @@ -79,13 +86,21 @@ export class NewBlockTask this.blockProver = protocol.blockProver; } + public static dependencies() { + return { + BlockProverCompileTask: { + useClass: BlockProverCompileTask, + }, + }; + } + public inputSerializer(): TaskSerializer { - const stProofSerializer = new ProofTaskSerializer( - this.stateTransitionProver.zkProgrammable.zkProgram[0].Proof + const stProofSerializer = new ProofTaskSerializer(() => + this.stateTransitionProver.zkProgrammable.proofType() ); - const transactionProofSerializer = new ProofTaskSerializer( - this.transactionProver.zkProgrammable.zkProgram[0].Proof + const transactionProofSerializer = new ProofTaskSerializer(() => + this.transactionProver.zkProgrammable.proofType() ); return new NewBlockProvingParametersSerializer( @@ -95,8 +110,8 @@ export class NewBlockTask } public resultSerializer(): TaskSerializer { - return new ProofTaskSerializer( - this.blockProver.zkProgrammable.zkProgram[0].Proof + return new ProofTaskSerializer(() => + this.blockProver.zkProgrammable.proofType() ); } @@ -140,9 +155,16 @@ export class NewBlockTask blockWitness, blockArgumentBatch, Bool(false) - // deferSTProof.or(deferTransactionProof) ); } else { + const DynamicSTProof = + await this.stateTransitionProver.zkProgrammable.dynamicProofType(); + const stProof = DynamicSTProof.fromProof(input1); + + const DynamicTransactionProof = + await this.transactionProver.zkProgrammable.dynamicProofType(); + const txProof = DynamicTransactionProof.fromProof(input2); + await this.blockProver.proveBlockBatchWithProofs( publicInput, stateWitness, @@ -151,8 +173,8 @@ export class NewBlockTask blockArgumentBatch, deferSTProof, deferTransactionProof, - input1, - input2 + stProof, + txProof ); } } diff --git a/packages/sequencer/src/protocol/production/tasks/RuntimeProvingTask.ts b/packages/sequencer/src/protocol/production/tasks/RuntimeProvingTask.ts index 3fe0756d3..6cfdb9126 100644 --- a/packages/sequencer/src/protocol/production/tasks/RuntimeProvingTask.ts +++ b/packages/sequencer/src/protocol/production/tasks/RuntimeProvingTask.ts @@ -10,16 +10,20 @@ import { RuntimeMethodExecutionContext, } from "@proto-kit/protocol"; import { Proof } from "o1js"; -import { CompileRegistry } from "@proto-kit/common"; +import { CompileRegistry, dependencyFactory } from "@proto-kit/common"; import { Task, TaskSerializer } from "../../../worker/flow/Task"; import { ProofTaskSerializer } from "../../../helpers/utils"; -import { TaskWorkerModule } from "../../../worker/worker/TaskWorkerModule"; +import { + task, + TaskWorkerModule, +} from "../../../worker/worker/TaskWorkerModule"; import { PreFilledStateService } from "../../../state/prefilled/PreFilledStateService"; import { PendingTransaction } from "../../../mempool/PendingTransaction"; import { TaskStateRecord } from "../tracing/BlockTracingService"; import { RuntimeProofParametersSerializer } from "./serializers/RuntimeProofParametersSerializer"; +import { RuntimeCompileTask } from "./compile/RuntimeCompileTask"; type RuntimeProof = Proof; @@ -31,6 +35,8 @@ export interface RuntimeProofParameters { @injectable() @scoped(Lifecycle.ContainerScoped) +@task() +@dependencyFactory() export class RuntimeProvingTask extends TaskWorkerModule implements Task @@ -48,12 +54,22 @@ export class RuntimeProvingTask super(); } + public static dependencies() { + return { + RuntimeCompileTask: { + useClass: RuntimeCompileTask, + }, + }; + } + public inputSerializer(): TaskSerializer { return new RuntimeProofParametersSerializer(); } public resultSerializer(): TaskSerializer { - return new ProofTaskSerializer(this.runtimeZkProgrammable[0].Proof); + return new ProofTaskSerializer(() => + this.runtime.zkProgrammable.proofType() + ); } public async compute(input: RuntimeProofParameters): Promise { diff --git a/packages/sequencer/src/protocol/production/tasks/StateTransitionReductionTask.ts b/packages/sequencer/src/protocol/production/tasks/StateTransitionReductionTask.ts index 940f1faaa..abd98dbf1 100644 --- a/packages/sequencer/src/protocol/production/tasks/StateTransitionReductionTask.ts +++ b/packages/sequencer/src/protocol/production/tasks/StateTransitionReductionTask.ts @@ -8,10 +8,14 @@ import { } from "@proto-kit/protocol"; import { CompileRegistry, + dependencyFactory, ProvableMethodExecutionContext, } from "@proto-kit/common"; -import { TaskWorkerModule } from "../../../worker/worker/TaskWorkerModule"; +import { + task, + TaskWorkerModule, +} from "../../../worker/worker/TaskWorkerModule"; import { Task, TaskSerializer } from "../../../worker/flow/Task"; import { PairProofTaskSerializer, @@ -19,8 +23,12 @@ import { ProofTaskSerializer, } from "../../../helpers/utils"; +import { STProverCompileTask } from "./compile/ProtocolCompileTask"; + @injectable() @scoped(Lifecycle.ContainerScoped) +@task() +@dependencyFactory() export class StateTransitionReductionTask extends TaskWorkerModule implements Task, StateTransitionProof> @@ -42,15 +50,23 @@ export class StateTransitionReductionTask this.stateTransitionProver = this.protocol.stateTransitionProver; } + public static dependencies() { + return { + STProverCompileTask: { + useClass: STProverCompileTask, + }, + }; + } + public inputSerializer(): TaskSerializer> { - return new PairProofTaskSerializer( - this.stateTransitionProver.zkProgrammable.zkProgram[0].Proof + return new PairProofTaskSerializer(() => + this.stateTransitionProver.zkProgrammable.proofType() ); } public resultSerializer(): TaskSerializer { - return new ProofTaskSerializer( - this.stateTransitionProver.zkProgrammable.zkProgram[0].Proof + return new ProofTaskSerializer(() => + this.stateTransitionProver.zkProgrammable.proofType() ); } diff --git a/packages/sequencer/src/protocol/production/tasks/StateTransitionTask.ts b/packages/sequencer/src/protocol/production/tasks/StateTransitionTask.ts index 532a0d5d4..a942a551c 100644 --- a/packages/sequencer/src/protocol/production/tasks/StateTransitionTask.ts +++ b/packages/sequencer/src/protocol/production/tasks/StateTransitionTask.ts @@ -14,13 +14,18 @@ import { ProvableMethodExecutionContext, CompileRegistry, LinkedMerkleTreeWitness, + dependencyFactory, } from "@proto-kit/common"; import { Task, TaskSerializer } from "../../../worker/flow/Task"; import { ProofTaskSerializer } from "../../../helpers/utils"; -import { TaskWorkerModule } from "../../../worker/worker/TaskWorkerModule"; +import { + task, + TaskWorkerModule, +} from "../../../worker/worker/TaskWorkerModule"; import { StateTransitionParametersSerializer } from "./serializers/StateTransitionParametersSerializer"; +import { STProverCompileTask } from "./compile/ProtocolCompileTask"; export interface StateTransitionProofParameters { publicInput: StateTransitionProverPublicInput; @@ -31,6 +36,8 @@ export interface StateTransitionProofParameters { @injectable() @scoped(Lifecycle.ContainerScoped) +@task() +@dependencyFactory() export class StateTransitionTask extends TaskWorkerModule implements Task @@ -56,11 +63,19 @@ export class StateTransitionTask } public resultSerializer(): TaskSerializer { - return new ProofTaskSerializer( - this.stateTransitionProver.zkProgrammable.zkProgram[0].Proof + return new ProofTaskSerializer(() => + this.stateTransitionProver.zkProgrammable.proofType() ); } + public static dependencies() { + return { + STProverCompileTask: { + useClass: STProverCompileTask, + }, + }; + } + public async compute( input: StateTransitionProofParameters ): Promise { diff --git a/packages/sequencer/src/protocol/production/tasks/TransactionProvingTask.ts b/packages/sequencer/src/protocol/production/tasks/TransactionProvingTask.ts index 5f6896b99..386a14b4d 100644 --- a/packages/sequencer/src/protocol/production/tasks/TransactionProvingTask.ts +++ b/packages/sequencer/src/protocol/production/tasks/TransactionProvingTask.ts @@ -13,16 +13,21 @@ import { inject, injectable, Lifecycle, scoped } from "tsyringe"; import { ProvableMethodExecutionContext, CompileRegistry, + dependencyFactory, } from "@proto-kit/common"; import { ProofTaskSerializer } from "../../../helpers/utils"; import { TaskSerializer, Task } from "../../../worker/flow/Task"; import { PreFilledStateService } from "../../../state/prefilled/PreFilledStateService"; -import { TaskWorkerModule } from "../../../worker/worker/TaskWorkerModule"; +import { + task, + TaskWorkerModule, +} from "../../../worker/worker/TaskWorkerModule"; import type { TaskStateRecord } from "../tracing/BlockTracingService"; import { TransactionProvingTaskParameterSerializer } from "./serializers/TransactionProvingTaskParameterSerializer"; import { TransactionProvingTaskParameters } from "./serializers/types/TransactionProvingTypes"; +import { TransactionProverCompileTask } from "./compile/ProtocolCompileTask"; export async function executeWithPrefilledStateService( stateServiceProvider: StateServiceProvider, @@ -49,15 +54,14 @@ export async function executeWithPrefilledStateService( @injectable() @scoped(Lifecycle.ContainerScoped) +@task() +@dependencyFactory() export class TransactionProvingTask extends TaskWorkerModule implements Task { private readonly transactionProver: TransactionProvable; - private readonly runtimeProofType = - this.runtime.zkProgrammable.zkProgram[0].Proof; - public name = "transaction"; public constructor( @@ -73,9 +77,21 @@ export class TransactionProvingTask this.transactionProver = protocol.transactionProver; } + public static dependencies() { + return { + TransactionProverCompileTask: { + useClass: TransactionProverCompileTask, + }, + }; + } + + private async runtimeProofType() { + return await this.runtime.zkProgrammable.proofType(); + } + public inputSerializer(): TaskSerializer { const runtimeProofSerializer = new ProofTaskSerializer( - this.runtimeProofType + this.runtimeProofType.bind(this) ); return new TransactionProvingTaskParameterSerializer( runtimeProofSerializer @@ -83,8 +99,8 @@ export class TransactionProvingTask } public resultSerializer(): TaskSerializer { - return new ProofTaskSerializer( - this.transactionProver.zkProgrammable.zkProgram[0].Proof + return new ProofTaskSerializer(() => + this.transactionProver.zkProgrammable.proofType() ); } diff --git a/packages/sequencer/src/protocol/production/tasks/TransactionReductionTask.ts b/packages/sequencer/src/protocol/production/tasks/TransactionReductionTask.ts index 46e5ddac9..7c7cb4d5a 100644 --- a/packages/sequencer/src/protocol/production/tasks/TransactionReductionTask.ts +++ b/packages/sequencer/src/protocol/production/tasks/TransactionReductionTask.ts @@ -8,10 +8,14 @@ import { } from "@proto-kit/protocol"; import { CompileRegistry, + dependencyFactory, ProvableMethodExecutionContext, } from "@proto-kit/common"; -import { TaskWorkerModule } from "../../../worker/worker/TaskWorkerModule"; +import { + task, + TaskWorkerModule, +} from "../../../worker/worker/TaskWorkerModule"; import { Task, TaskSerializer } from "../../../worker/flow/Task"; import { PairProofTaskSerializer, @@ -19,8 +23,12 @@ import { ProofTaskSerializer, } from "../../../helpers/utils"; +import { TransactionProverCompileTask } from "./compile/ProtocolCompileTask"; + @injectable() @scoped(Lifecycle.ContainerScoped) +@task() +@dependencyFactory() export class TransactionReductionTask extends TaskWorkerModule implements Task, TransactionProof> @@ -41,15 +49,23 @@ export class TransactionReductionTask this.transactionProver = this.protocol.transactionProver; } + public static dependencies() { + return { + TransactionProverCompileTask: { + useClass: TransactionProverCompileTask, + }, + }; + } + public inputSerializer(): TaskSerializer> { - return new PairProofTaskSerializer( - this.transactionProver.zkProgrammable.zkProgram[0].Proof + return new PairProofTaskSerializer(() => + this.transactionProver.zkProgrammable.proofType() ); } public resultSerializer(): TaskSerializer { - return new ProofTaskSerializer( - this.transactionProver.zkProgrammable.zkProgram[0].Proof + return new ProofTaskSerializer(() => + this.transactionProver.zkProgrammable.proofType() ); } diff --git a/packages/sequencer/src/protocol/production/tasks/compile/CircuitCompileTask.ts b/packages/sequencer/src/protocol/production/tasks/compile/CircuitCompileTask.ts new file mode 100644 index 000000000..3a87809df --- /dev/null +++ b/packages/sequencer/src/protocol/production/tasks/compile/CircuitCompileTask.ts @@ -0,0 +1,127 @@ +import { + log, + mapSequential, + ArtifactRecord, + CompileRegistry, + CompilableModule, + safeParseJson, + NoConfig, + ChildVerificationKeyService, +} from "@proto-kit/common"; +import { + Protocol, + RuntimeVerificationKeyRootService, + MandatoryProtocolModulesRecord, + BridgingSettlementContractArgs, + ContractArgsRegistry, +} from "@proto-kit/protocol"; + +import { TaskSerializer } from "../../../../worker/flow/Task"; +import { UnpreparingTask } from "../../../../worker/flow/UnpreparingTask"; +import { SignedSettlementPermissions } from "../../../../settlement/permissions/SignedSettlementPermissions"; +import { ProvenSettlementPermissions } from "../../../../settlement/permissions/ProvenSettlementPermissions"; +import { + ArtifactRecordSerializer, + SerializedArtifactRecord, +} from "../serializers/ArtifactionRecordSerializer"; + +export type CompilerTaskParams = { + existingArtifacts: ArtifactRecord; + runtimeVKRoot?: string; + isSignedSettlement?: boolean; +}; + +export abstract class CircuitCompileTask< + Config = NoConfig, +> extends UnpreparingTask { + protected constructor( + protected readonly protocol: Protocol, + protected readonly compileRegistry: CompileRegistry, + protected readonly contractArgsRegistry: ContractArgsRegistry + ) { + super(); + + this.protocol.dependencyContainer + .resolve(ChildVerificationKeyService) + .setCompileRegistry(this.compileRegistry); + } + + public inputSerializer(): TaskSerializer { + type CompilerTaskParamsJSON = { + runtimeVKRoot?: string; + existingArtifacts: SerializedArtifactRecord; + isSignedSettlement?: boolean; + }; + + const serializer = new ArtifactRecordSerializer(); + return { + toJSON: (input) => + JSON.stringify({ + runtimeVKRoot: input.runtimeVKRoot, + existingArtifacts: serializer.toJSON(input.existingArtifacts), + isSignedSettlement: input.isSignedSettlement, + } satisfies CompilerTaskParamsJSON), + fromJSON: (input) => { + const json = safeParseJson(input); + return { + runtimeVKRoot: json.runtimeVKRoot, + existingArtifacts: serializer.fromJSON(json.existingArtifacts), + isSignedSettlement: json.isSignedSettlement, + }; + }, + }; + } + + public resultSerializer(): TaskSerializer { + const serializer = new ArtifactRecordSerializer(); + return { + toJSON: (input) => JSON.stringify(serializer.toJSON(input)), + fromJSON: (input) => + serializer.fromJSON(safeParseJson(input)), + }; + } + + public abstract getTargets(): Promise; + + public async compute(input: CompilerTaskParams): Promise { + log.info("Computing VKs"); + + this.compileRegistry.addArtifactsRaw(input.existingArtifacts); + + // We need to initialize the VK tree root if we have it, so that + // the BlockProver can bake in that root + if (input.runtimeVKRoot !== undefined) { + this.protocol.dependencyContainer + .resolve(RuntimeVerificationKeyRootService) + .setRoot(BigInt(input.runtimeVKRoot)); + } + + if (input.isSignedSettlement !== undefined) { + this.contractArgsRegistry.addArgs( + "SettlementContract", + { + signedSettlements: input.isSignedSettlement, + // TODO Add distinction between mina and custom tokens + BridgeContractPermissions: (input.isSignedSettlement + ? new SignedSettlementPermissions() + : new ProvenSettlementPermissions() + ).bridgeContractMina(), + } + ); + } + + const targets = await this.getTargets(); + + const msg = `Compiling targets ${this.name}`; + log.time(msg); + await mapSequential(targets, async (target) => { + await target.compile(this.compileRegistry); + }); + log.timeEnd.info(msg); + + const newEntries = Object.entries( + this.compileRegistry.getAllArtifacts() + ).filter(([key]) => !(key in input.existingArtifacts)); + return Object.fromEntries(newEntries); + } +} diff --git a/packages/sequencer/src/protocol/production/tasks/compile/ProtocolCompileTask.ts b/packages/sequencer/src/protocol/production/tasks/compile/ProtocolCompileTask.ts new file mode 100644 index 000000000..592867ad5 --- /dev/null +++ b/packages/sequencer/src/protocol/production/tasks/compile/ProtocolCompileTask.ts @@ -0,0 +1,59 @@ +import { inject, injectable } from "tsyringe"; +import { CompilableModule, CompileRegistry } from "@proto-kit/common"; +import { + ContractArgsRegistry, + MandatoryProtocolModulesRecord, + Protocol, +} from "@proto-kit/protocol"; + +import { task } from "../../../../worker/worker/TaskWorkerModule"; + +import { CircuitCompileTask } from "./CircuitCompileTask"; + +@injectable() +export class ProtocolCompileTask extends CircuitCompileTask { + public name = "undefined"; + + public constructor( + @inject("Protocol") + protocol: Protocol, + compileRegistry: CompileRegistry, + contractArgsRegistry: ContractArgsRegistry + ) { + super(protocol, compileRegistry, contractArgsRegistry); + + this.name = `compile-${this.getTargetProtocolModule().toLowerCase()}`; + } + + public getTargetProtocolModule(): string { + throw new Error(""); + } + + public async getTargets(): Promise { + return [this.protocol.resolveOrFail(this.getTargetProtocolModule())]; + } +} + +@injectable() +@task() +export class BlockProverCompileTask extends ProtocolCompileTask { + public getTargetProtocolModule() { + return "BlockProver"; + } +} + +@injectable() +@task() +export class STProverCompileTask extends ProtocolCompileTask { + public getTargetProtocolModule() { + return "StateTransitionProver"; + } +} + +@injectable() +@task() +export class TransactionProverCompileTask extends ProtocolCompileTask { + public getTargetProtocolModule() { + return "TransactionProver"; + } +} diff --git a/packages/sequencer/src/protocol/production/tasks/compile/RuntimeCompileTask.ts b/packages/sequencer/src/protocol/production/tasks/compile/RuntimeCompileTask.ts new file mode 100644 index 000000000..5b6e919e1 --- /dev/null +++ b/packages/sequencer/src/protocol/production/tasks/compile/RuntimeCompileTask.ts @@ -0,0 +1,33 @@ +import { inject, injectable, Lifecycle, scoped } from "tsyringe"; +import { CompilableModule, CompileRegistry } from "@proto-kit/common"; +import { Runtime } from "@proto-kit/module"; +import { + ContractArgsRegistry, + MandatoryProtocolModulesRecord, + Protocol, +} from "@proto-kit/protocol"; + +import { task } from "../../../../worker/worker/TaskWorkerModule"; + +import { CircuitCompileTask } from "./CircuitCompileTask"; + +@injectable() +@scoped(Lifecycle.ContainerScoped) +@task() +export class RuntimeCompileTask extends CircuitCompileTask { + public name = "compile-runtime"; + + public constructor( + @inject("Runtime") protected readonly runtime: Runtime, + @inject("Protocol") + protocol: Protocol, + compileRegistry: CompileRegistry, + contractArgsRegistry: ContractArgsRegistry + ) { + super(protocol, compileRegistry, contractArgsRegistry); + } + + public async getTargets(): Promise { + return [this.runtime]; + } +} diff --git a/packages/sequencer/src/protocol/production/tasks/compile/SettlementCompileTask.ts b/packages/sequencer/src/protocol/production/tasks/compile/SettlementCompileTask.ts new file mode 100644 index 000000000..b139a95d1 --- /dev/null +++ b/packages/sequencer/src/protocol/production/tasks/compile/SettlementCompileTask.ts @@ -0,0 +1,96 @@ +import { inject, injectable, Lifecycle, scoped } from "tsyringe"; +import { + ArtifactRecord, + CompilableModule, + CompileRegistry, + log, + reduceSequential, + StringKeyOf, +} from "@proto-kit/common"; +import { + ContractArgsRegistry, + MandatoryProtocolModulesRecord, + MandatorySettlementModulesRecord, + Protocol, + SettlementContractModule, + SettlementModulesRecord, +} from "@proto-kit/protocol"; + +import { BatchProducerModule } from "../../BatchProducerModule"; +import { task } from "../../../../worker/worker/TaskWorkerModule"; + +import { CircuitCompileTask } from "./CircuitCompileTask"; + +@injectable() +@scoped(Lifecycle.ContainerScoped) +@task() +export class SettlementCompileTask extends CircuitCompileTask { + public name = "compile-settlement"; + + public constructor( + @inject("Protocol") + protocol: Protocol, + compileRegistry: CompileRegistry, + contractArgsRegistry: ContractArgsRegistry, + @inject("BatchProducerModule", { isOptional: true }) + batchProducerModule: BatchProducerModule | undefined + ) { + super(protocol, compileRegistry, contractArgsRegistry); + + const container = this.protocol.dependencyContainer; + if ( + !container.isRegistered("SettlementContractModule") && + // Disable this check for the sequencer + batchProducerModule === undefined + ) { + throw new Error( + "SettlementContractModule not configured but SettlementCompilerTask is - fix the configuration" + ); + } + } + + public getSettlementTargets(): CompilableModule[] { + // We only care about the BridgeContract for now - later with caching, + // we might want to expand that to all protocol circuits + const container = this.protocol.dependencyContainer; + const settlementModule = container.resolve< + SettlementContractModule + >("SettlementContractModule"); + + // Needed so that all contractFactory functions are called, because + // they set static args on the contracts + settlementModule.getContractClasses(); + + const moduleNames = + // eslint-disable-next-line @typescript-eslint/consistent-type-assertions + settlementModule.moduleNames as StringKeyOf[]; + + const modules = moduleNames.map<[string, CompilableModule]>((name) => [ + `Settlement.${name}`, + settlementModule.resolve(name), + ]); + + const sumModule = { + compile: async (registry: CompileRegistry) => { + await reduceSequential<[string, CompilableModule], ArtifactRecord>( + modules, + async (record, [moduleName, module]) => { + log.info(`Compiling ${moduleName}`); + const artifacts = await module.compile(registry); + return { + ...record, + ...artifacts, + }; + }, + {} + ); + }, + }; + + return [sumModule]; + } + + public async getTargets(): Promise { + return this.getSettlementTargets(); + } +} diff --git a/packages/sequencer/src/protocol/production/tasks/serializers/BlockProofSerializer.ts b/packages/sequencer/src/protocol/production/tasks/serializers/BlockProofSerializer.ts index 113db208a..650850729 100644 --- a/packages/sequencer/src/protocol/production/tasks/serializers/BlockProofSerializer.ts +++ b/packages/sequencer/src/protocol/production/tasks/serializers/BlockProofSerializer.ts @@ -24,8 +24,9 @@ export class BlockProofSerializer { public getBlockProofSerializer() { if (this.serializer === undefined) { const blockProver = this.protocol.resolve("BlockProver"); - const proofType = blockProver.zkProgrammable.zkProgram[0].Proof; - this.serializer = new ProofTaskSerializer(proofType); + this.serializer = new ProofTaskSerializer(() => + blockProver.zkProgrammable.proofType() + ); } return this.serializer; } diff --git a/packages/sequencer/src/protocol/production/tasks/serializers/NewBlockProvingParametersSerializer.ts b/packages/sequencer/src/protocol/production/tasks/serializers/NewBlockProvingParametersSerializer.ts index f54771bf6..3ddf5b040 100644 --- a/packages/sequencer/src/protocol/production/tasks/serializers/NewBlockProvingParametersSerializer.ts +++ b/packages/sequencer/src/protocol/production/tasks/serializers/NewBlockProvingParametersSerializer.ts @@ -60,10 +60,10 @@ export class NewBlockProvingParametersSerializer implements TaskSerializer ) {} - public toJSON(input: NewBlockPayload) { + public async toJSON(input: NewBlockPayload) { return JSON.stringify({ - input1: this.stProofSerializer.toJSON(input.input1), - input2: this.transactionProofSerializer.toJSON(input.input2), + input1: await this.stProofSerializer.toJSON(input.input1), + input2: await this.transactionProofSerializer.toJSON(input.input2), params: { publicInput: BlockProverPublicInput.toJSON(input.params.publicInput), diff --git a/packages/sequencer/src/protocol/production/tasks/serializers/TransactionProvingTaskParameterSerializer.ts b/packages/sequencer/src/protocol/production/tasks/serializers/TransactionProvingTaskParameterSerializer.ts index 24a7ce2cb..5c54735b2 100644 --- a/packages/sequencer/src/protocol/production/tasks/serializers/TransactionProvingTaskParameterSerializer.ts +++ b/packages/sequencer/src/protocol/production/tasks/serializers/TransactionProvingTaskParameterSerializer.ts @@ -81,17 +81,19 @@ export class TransactionProvingTaskParameterSerializer implements TaskSerializer }; } - public toJSON(inputs: TransactionProvingTaskParameters): string { + public async toJSON( + inputs: TransactionProvingTaskParameters + ): Promise { if (inputs === "dummy") { return "dummy"; } - const taskParamsJson: TransactionProvingTaskParametersJSON = inputs.map( - (input) => { + const taskParamsJson: TransactionProvingTaskParametersJSON = + await mapSequential(inputs, async (input) => { const { parameters, proof } = input; const { executionData } = parameters; - const proofJSON = this.runtimeProofSerializer.toJSONProof(proof); + const proofJSON = await this.runtimeProofSerializer.toJSONProof(proof); const parametersJSON: TransactionProverTaskParametersJSON = { publicInput: TransactionProverPublicInput.toJSON( @@ -112,8 +114,7 @@ export class TransactionProvingTaskParameterSerializer implements TaskSerializer }; return { parameters: parametersJSON, proof: proofJSON }; - } - ); + }); return JSON.stringify(taskParamsJson); } diff --git a/packages/sequencer/src/protocol/runtime/RuntimeVerificationKeyService.ts b/packages/sequencer/src/protocol/runtime/RuntimeVerificationKeyService.ts index 5445d6e63..2c62bd98e 100644 --- a/packages/sequencer/src/protocol/runtime/RuntimeVerificationKeyService.ts +++ b/packages/sequencer/src/protocol/runtime/RuntimeVerificationKeyService.ts @@ -64,7 +64,7 @@ export class VerificationKeyService extends ConfigurableModule<{}> { public async initializeVKTree(artifacts: Record) { const mappings = await mapSequential( - this.runtime.zkProgrammable.zkProgram, + await this.runtime.zkProgrammable.zkProgram(), async (program) => { const artifact = artifacts[program.name]; diff --git a/packages/sequencer/src/sequencer/SequencerStartupModule.ts b/packages/sequencer/src/sequencer/SequencerStartupModule.ts index 603835f7a..745bbab1e 100644 --- a/packages/sequencer/src/sequencer/SequencerStartupModule.ts +++ b/packages/sequencer/src/sequencer/SequencerStartupModule.ts @@ -4,6 +4,7 @@ import { ContractArgsRegistry, MandatoryProtocolModulesRecord, Protocol, + ProtocolConstants, RuntimeVerificationKeyRootService, } from "@proto-kit/protocol"; import { @@ -12,17 +13,28 @@ import { ChildVerificationKeyService, CompileRegistry, AreProofsEnabled, + CompileArtifact, + mapSequential, } from "@proto-kit/common"; import { Flow, FlowCreator } from "../worker/flow/Flow"; -import { WorkerRegistrationFlow } from "../worker/worker/startup/WorkerRegistrationFlow"; -import { - CircuitCompilerTask, - CompilerTaskParams, -} from "../protocol/production/tasks/CircuitCompilerTask"; +import { WorkerRegistrationFlow } from "../worker/startup/WorkerRegistrationFlow"; import { VerificationKeyService } from "../protocol/runtime/RuntimeVerificationKeyService"; import type { MinaBaseLayer } from "../protocol/baselayer/MinaBaseLayer"; import { NoopBaseLayer } from "../protocol/baselayer/NoopBaseLayer"; +import { RuntimeCompileTask } from "../protocol/production/tasks/compile/RuntimeCompileTask"; +import { SettlementCompileTask } from "../protocol/production/tasks/compile/SettlementCompileTask"; +import { + CircuitCompileTask, + CompilerTaskParams, +} from "../protocol/production/tasks/compile/CircuitCompileTask"; +import { Task } from "../worker/flow/Task"; +import { SettlementModule } from "../settlement/SettlementModule"; +import { + BlockProverCompileTask, + STProverCompileTask, + TransactionProverCompileTask, +} from "../protocol/production/tasks/compile/ProtocolCompileTask"; import { SequencerModule, sequencerModule } from "./builder/SequencerModule"; import { Closeable, closeable } from "./builder/Closeable"; @@ -37,7 +49,11 @@ export class SequencerStartupModule private readonly flowCreator: FlowCreator, @inject("Protocol") private readonly protocol: Protocol, - private readonly compileTask: CircuitCompilerTask, + private readonly runtimeCompileTask: RuntimeCompileTask, + private readonly stProverCompileTask: STProverCompileTask, + private readonly transactionProverCompileTask: TransactionProverCompileTask, + private readonly blockProverCompileTask: BlockProverCompileTask, + private readonly settlementCompileTask: SettlementCompileTask, private readonly verificationKeyService: VerificationKeyService, private readonly registrationFlow: WorkerRegistrationFlow, private readonly compileRegistry: CompileRegistry, @@ -45,28 +61,34 @@ export class SequencerStartupModule private readonly baseLayer: MinaBaseLayer | undefined, @inject("AreProofsEnabled") private readonly areProofsEnabled: AreProofsEnabled, - private readonly contractArgsRegistry: ContractArgsRegistry + private readonly contractArgsRegistry: ContractArgsRegistry, + @inject("SettlementModule", { isOptional: true }) + private readonly settlementModule: SettlementModule | undefined ) { super(); } private async pushCompileTask( flow: Flow<{}>, + task: Task, payload: CompilerTaskParams ): Promise { return await flow.withFlow(async (res, rej) => { - await flow.pushTask(this.compileTask, payload, async (result) => { + await flow.pushTask(task, payload, async (result) => { res(result); }); }); } public async compileRuntime(flow: Flow<{}>) { - const artifacts = await this.pushCompileTask(flow, { - existingArtifacts: {}, - targets: ["runtime"], - runtimeVKRoot: undefined, - }); + const artifacts = await this.pushCompileTask( + flow, + this.runtimeCompileTask, + { + existingArtifacts: {}, + runtimeVKRoot: undefined, + } + ); // Init runtime VK tree await this.verificationKeyService.initializeVKTree(artifacts); @@ -82,51 +104,42 @@ export class SequencerStartupModule return root; } - private async compileProtocolAndBridge( + private async compileBridge( flow: Flow<{}>, - runtimeVkTreeRoot: bigint, + runtimeVKRoot: bigint, isSignedSettlement?: boolean ) { - // Can happen in parallel - type ParallelResult = { - protocol?: ArtifactRecord; - bridge?: ArtifactRecord; - }; - const result = await flow.withFlow(async (res, rej) => { - const results: ParallelResult = {}; - - const resolveIfPossible = () => { - const { bridge, protocol } = results; - if (bridge !== undefined && protocol !== undefined) { - res({ ...protocol, ...bridge }); - } - }; - await flow.pushTask( - this.compileTask, + this.settlementCompileTask, { - existingArtifacts: {}, - targets: ["protocol"], - runtimeVKRoot: runtimeVkTreeRoot.toString(), + existingArtifacts: this.compileRegistry.getAllArtifacts(), + runtimeVKRoot: runtimeVKRoot.toString(), + isSignedSettlement, }, - async (protocolResult) => { - results.protocol = protocolResult; - resolveIfPossible(); + async (bridgeResult) => { + res(bridgeResult); } ); + }); + this.compileRegistry.addArtifactsRaw(result); + return result; + } + private async compileProtocol( + flow: Flow<{}>, + task: CircuitCompileTask, + runtimeVkTreeRoot: bigint + ) { + const result = await flow.withFlow(async (res, rej) => { await flow.pushTask( - this.compileTask, + task, { - existingArtifacts: {}, - targets: ["Settlement.BridgeContract"], - runtimeVKRoot: undefined, - isSignedSettlement, + existingArtifacts: this.compileRegistry.getAllArtifacts(), + runtimeVKRoot: runtimeVkTreeRoot.toString(), }, - async (bridgeResult) => { - results.bridge = bridgeResult; - resolveIfPossible(); + async (protocolResult) => { + res(protocolResult); } ); }); @@ -135,6 +148,8 @@ export class SequencerStartupModule } public async start() { + ProtocolConstants.printAllConstants(); + const flow = this.flowCreator.createFlow("compile-circuits", {}); this.protocol.dependencyContainer @@ -159,26 +174,40 @@ export class SequencerStartupModule const root = await this.compileRuntime(flow); - const protocolBridgeArtifacts = await this.compileProtocolAndBridge( - flow, - root, - isSignedSettlement - ); + const tasks = [ + this.stProverCompileTask, + this.transactionProverCompileTask, + this.blockProverCompileTask, + ]; + await mapSequential(tasks, async (task) => { + await this.compileProtocol(flow, task, root); + }); - log.info("Protocol circuits compiled"); + let bridgeVk: CompileArtifact | undefined = undefined; - // Init BridgeContract vk for settlement contract - const bridgeVk = protocolBridgeArtifacts.BridgeContract; - if (bridgeVk !== undefined) { - // TODO Inject CompileRegistry directly - this.contractArgsRegistry.addArgs( - "SettlementContract", - { - BridgeContractVerificationKey: bridgeVk.verificationKey, - } + if (this.settlementModule !== undefined) { + const bridgeArtifacts = await this.compileBridge( + flow, + root, + isSignedSettlement ); + + // TODO Why is this not in SettlementStartupModule? + // Init BridgeContract vk for settlement contract + bridgeVk = bridgeArtifacts.BridgeContract; + if (bridgeVk !== undefined) { + // TODO Inject CompileRegistry directly + this.contractArgsRegistry.addArgs( + "SettlementContract", + { + BridgeContractVerificationKey: bridgeVk.verificationKey, + } + ); + } } + log.info("Protocol circuits compiled"); + await this.registrationFlow.start({ runtimeVerificationKeyRoot: root, bridgeContractVerificationKey: bridgeVk?.verificationKey, diff --git a/packages/sequencer/src/sequencer/SettlementStartupModule.ts b/packages/sequencer/src/sequencer/SettlementStartupModule.ts index 1936fb3c1..bdc3bee6e 100644 --- a/packages/sequencer/src/sequencer/SettlementStartupModule.ts +++ b/packages/sequencer/src/sequencer/SettlementStartupModule.ts @@ -7,7 +7,7 @@ import { } from "@proto-kit/common"; import { FlowCreator } from "../worker/flow/Flow"; -import { CircuitCompilerTask } from "../protocol/production/tasks/CircuitCompilerTask"; +import { SettlementCompileTask } from "../protocol/production/tasks/compile/SettlementCompileTask"; @injectable() export class SettlementStartupModule { @@ -15,7 +15,7 @@ export class SettlementStartupModule { public constructor( private readonly compileRegistry: CompileRegistry, private readonly flowCreator: FlowCreator, - private readonly compileTask: CircuitCompilerTask + private readonly compileTask: SettlementCompileTask ) {} // TODO Compile only individual contracts - this however runs into the @@ -29,7 +29,6 @@ export class SettlementStartupModule { this.compileTask, { existingArtifacts: this.compileRegistry.getAllArtifacts(), - targets: ["Settlement"], runtimeVKRoot: undefined, }, async (result) => res(result) diff --git a/packages/sequencer/src/settlement/BridgingModule.ts b/packages/sequencer/src/settlement/BridgingModule.ts index 79133bc73..1a5e735a2 100644 --- a/packages/sequencer/src/settlement/BridgingModule.ts +++ b/packages/sequencer/src/settlement/BridgingModule.ts @@ -56,6 +56,7 @@ import { AsyncLinkedLeafStore } from "../state/async/AsyncLinkedLeafStore"; import { CachedLinkedLeafStore } from "../state/lmt/CachedLinkedLeafStore"; import { SettleableBatch } from "../storage/model/Batch"; import { SequencerModule } from "../sequencer/builder/SequencerModule"; +import { AsyncMerkleTreeStore } from "../state/async/AsyncMerkleTreeStore"; import type { SettlementModule } from "./SettlementModule"; import { SettlementUtils } from "./utils/SettlementUtils"; @@ -114,6 +115,8 @@ export class BridgingModule extends SequencerModule { private readonly outgoingMessageCollector: OutgoingMessageCollector, @inject("AsyncLinkedLeafStore") private readonly linkedLeafStore: AsyncLinkedLeafStore, + @inject("AsyncTreeStore") + private readonly treeStore: AsyncMerkleTreeStore, @inject("FeeStrategy") private readonly feeStrategy: FeeStrategy, @inject("BaseLayer") private readonly baseLayer: MinaBaseLayer, @@ -590,7 +593,10 @@ export class BridgingModule extends SequencerModule { const bridgeContract = this.createBridgeContract(bridgeAddress, tokenId); - const cachedStore = await CachedLinkedLeafStore.new(this.linkedLeafStore); + const cachedStore = await CachedLinkedLeafStore.new( + this.linkedLeafStore, + this.treeStore + ); const tree = new LinkedMerkleTree(cachedStore.treeStore, cachedStore); // Create withdrawal batches and send them as L1 transactions diff --git a/packages/sequencer/src/settlement/interactions/bridging/BridgingDeployInteraction.ts b/packages/sequencer/src/settlement/interactions/bridging/BridgingDeployInteraction.ts index c1c24d4a4..9fa2e6194 100644 --- a/packages/sequencer/src/settlement/interactions/bridging/BridgingDeployInteraction.ts +++ b/packages/sequencer/src/settlement/interactions/bridging/BridgingDeployInteraction.ts @@ -21,6 +21,8 @@ import { SettlementUtils } from "../../utils/SettlementUtils"; @injectable() export class BridgingDeployInteraction implements DeployInteraction { + private utils: SettlementUtils; + public constructor( @inject("AddressRegistry") private readonly addressRegistry: AddressRegistry, @@ -33,7 +35,9 @@ export class BridgingDeployInteraction implements DeployInteraction { private readonly feeStrategy: FeeStrategy, @inject("TransactionSender") private readonly transactionSender: MinaTransactionSender - ) {} + ) { + this.utils = new SettlementUtils(this.baseLayer, this.signer); + } protected settlementContractModule(): SettlementContractModule { return this.protocol.dependencyContainer.resolve( @@ -59,7 +63,8 @@ export class BridgingDeployInteraction implements DeployInteraction { const feepayer = this.signer.getFeepayerKey(); - const nonce = options?.nonce ?? 0; + const nonce = + options?.nonce ?? (await this.utils.fetchNonce(feepayer)) ?? 0; const sm = this.settlementContractModule(); const { diff --git a/packages/sequencer/src/settlement/interactions/vanilla/VanillaDeployInteraction.ts b/packages/sequencer/src/settlement/interactions/vanilla/VanillaDeployInteraction.ts index c5fb126fe..448630418 100644 --- a/packages/sequencer/src/settlement/interactions/vanilla/VanillaDeployInteraction.ts +++ b/packages/sequencer/src/settlement/interactions/vanilla/VanillaDeployInteraction.ts @@ -21,6 +21,8 @@ import { SettlementUtils } from "../../utils/SettlementUtils"; @injectable() export class VanillaDeployInteraction implements DeployInteraction { + private utils: SettlementUtils; + public constructor( @inject("AddressRegistry") private readonly addressRegistry: AddressRegistry, @@ -33,7 +35,9 @@ export class VanillaDeployInteraction implements DeployInteraction { private readonly feeStrategy: FeeStrategy, @inject("TransactionSender") private readonly transactionSender: MinaTransactionSender - ) {} + ) { + this.utils = new SettlementUtils(this.baseLayer, this.signer); + } protected settlementContractModule(): SettlementContractModule { return this.protocol.dependencyContainer.resolve( @@ -62,7 +66,8 @@ export class VanillaDeployInteraction implements DeployInteraction { const feepayer = this.signer.getFeepayerKey(); - const nonce = options?.nonce ?? 0; + const nonce = + options?.nonce ?? (await this.utils.fetchNonce(feepayer)) ?? 0; const sm = this.settlementContractModule(); const { SettlementContract: settlementContract } = sm.createContracts({ diff --git a/packages/sequencer/src/settlement/messages/IncomingMessagesService.ts b/packages/sequencer/src/settlement/messages/IncomingMessagesService.ts index 6311cb30a..52e266679 100644 --- a/packages/sequencer/src/settlement/messages/IncomingMessagesService.ts +++ b/packages/sequencer/src/settlement/messages/IncomingMessagesService.ts @@ -3,7 +3,7 @@ import { ACTIONS_EMPTY_HASH } from "@proto-kit/protocol"; import { SettlementStorage } from "../../storage/repositories/SettlementStorage"; import { MessageStorage } from "../../storage/repositories/MessageStorage"; -import { BlockStorage } from "../../storage/repositories/BlockStorage"; +import { BlockQueue } from "../../storage/repositories/BlockStorage"; import { PendingTransaction } from "../../mempool/PendingTransaction"; import type { BridgingModule } from "../BridgingModule"; @@ -18,8 +18,8 @@ export class IncomingMessagesService { private readonly messageStorage: MessageStorage, @inject("IncomingMessageAdapter") private readonly messagesAdapter: IncomingMessageAdapter, - @inject("BlockStorage") - private readonly blockStorage: BlockStorage, + @inject("BlockQueue") + private readonly blockStorage: BlockQueue, @inject("BridgingModule") private readonly bridgingModule: BridgingModule ) {} @@ -102,7 +102,7 @@ export class IncomingMessagesService { public async getPendingMessages() { const latestSettlement = await this.settlementStorage.getLatestSettlement(); - const latestBlock = await this.blockStorage.getLatestBlock(); + const latestBlock = await this.blockStorage.getLatestBlockAndResult(); const messagesHashCursor = latestBlock?.block?.toMessagesHash?.toString() ?? diff --git a/packages/sequencer/src/settlement/tasks/SettlementProvingTask.ts b/packages/sequencer/src/settlement/tasks/SettlementProvingTask.ts index c21d08d05..594f48799 100644 --- a/packages/sequencer/src/settlement/tasks/SettlementProvingTask.ts +++ b/packages/sequencer/src/settlement/tasks/SettlementProvingTask.ts @@ -5,6 +5,7 @@ import { CompileRegistry, mapSequential, safeParseJson, + dependencyFactory, } from "@proto-kit/common"; import { MandatoryProtocolModulesRecord, @@ -39,7 +40,8 @@ import { DynamicProofTaskSerializer, } from "../../helpers/utils"; import { Task, TaskSerializer } from "../../worker/flow/Task"; -import { TaskWorkerModule } from "../../worker/worker/TaskWorkerModule"; +import { task, TaskWorkerModule } from "../../worker/worker/TaskWorkerModule"; +import { SettlementCompileTask } from "../../protocol/production/tasks/compile/SettlementCompileTask"; import { ContractRegistry } from "./ContractRegistry"; @@ -74,6 +76,8 @@ export class SomeProofSubclass extends Proof { */ @injectable() @scoped(Lifecycle.ContainerScoped) +@task() +@dependencyFactory() export class SettlementProvingTask extends TaskWorkerModule implements Task @@ -103,6 +107,14 @@ export class SettlementProvingTask } } + public static dependencies() { + return { + SettlementCompileTask: { + useClass: SettlementCompileTask, + }, + }; + } + private async withCustomInstance( transaction: Transaction, state: ChainStateTaskArgs, @@ -170,11 +182,11 @@ export class SettlementProvingTask return proofType.prototype instanceof Proof ? new ProofTaskSerializer( // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - proofType as Subclass> + async () => proofType as Subclass> ) : new DynamicProofTaskSerializer( // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - proofType as Subclass> + async () => proofType as Subclass> ); } @@ -329,85 +341,85 @@ export class SettlementProvingTask }; }, - toJSON: (input: TransactionTaskArgs): string => { + toJSON: async (input: TransactionTaskArgs): Promise => { const transaction = input.transaction.toJSON(); - const lazyProofs = - input.transaction.transaction.accountUpdates.map( - (au) => { - if (au.lazyAuthorization?.kind === "lazy-proof") { - const lazyProof = au.lazyAuthorization; - - // eslint-disable-next-line no-underscore-dangle - const method = lazyProof.ZkappClass._methods?.find( - (methodInterface) => - methodInterface.methodName === lazyProof.methodName - ); - if (method === undefined) { - throw new Error("Method interface not found"); - } + const lazyProofs = await mapSequential( + input.transaction.transaction.accountUpdates, + async (au) => { + if (au.lazyAuthorization?.kind === "lazy-proof") { + const lazyProof = au.lazyAuthorization; + + // eslint-disable-next-line no-underscore-dangle + const method = lazyProof.ZkappClass._methods?.find( + (methodInterface) => + methodInterface.methodName === lazyProof.methodName + ); + if (method === undefined) { + throw new Error("Method interface not found"); + } + + // args are [public key, tokenId, ...args] + const args = method.args.slice(2); - // args are [public key, tokenId, ...args] - const args = method.args.slice(2); - - const encodedArgs = lazyProof.args - .map((arg, index) => { - const argType = args[index]; - const argTypeProvable = ProvableType.get(argType); - const argProofs = this.extractProofTypes(argType); - - if (argProofs.length === 0) { - // Special case for AUForest - if (arg instanceof AccountUpdateForest) { - const accountUpdates = AccountUpdateForest.toFlatArray( - arg - ).map((update) => AccountUpdate.toJSON(update)); - - return { - // eslint-disable-next-line @typescript-eslint/consistent-type-assertions - fields: [] as string[], - aux: [ - JSON.stringify({ - accountUpdates, - typeName: "AccountUpdateForest", - }), - ], - }; - } - - const fields = argTypeProvable - .toFields(arg) - .map((f) => f.toString()); - const aux = argTypeProvable - .toAuxiliary(arg) - .map((x) => JSON.stringify(x)); + const encodedArgs = ( + await mapSequential(lazyProof.args, async (arg, index) => { + const argType = args[index]; + const argTypeProvable = ProvableType.get(argType); + const argProofs = this.extractProofTypes(argType); + + if (argProofs.length === 0) { + // Special case for AUForest + if (arg instanceof AccountUpdateForest) { + const accountUpdates = AccountUpdateForest.toFlatArray( + arg + ).map((update) => AccountUpdate.toJSON(update)); return { - fields, - aux, + // eslint-disable-next-line @typescript-eslint/consistent-type-assertions + fields: [] as string[], + aux: [ + JSON.stringify({ + accountUpdates, + typeName: "AccountUpdateForest", + }), + ], }; - } else { - const serializer = this.getProofSerializer(argProofs[0]); - // eslint-disable-next-line @typescript-eslint/no-unsafe-argument - return serializer.toJSON(arg); } - }) - .filter(filterNonUndefined); - - return { - methodName: lazyProof.methodName, - zkappClassName: lazyProof.ZkappClass.name, - args: encodedArgs, - blindingValue: lazyProof.blindingValue.toString(), - memoized: lazyProof.memoized.map((value) => ({ - fields: value.fields.map((f) => f.toString()), - aux: value.aux, - })), - }; - } - return null; + + const fields = argTypeProvable + .toFields(arg) + .map((f) => f.toString()); + const aux = argTypeProvable + .toAuxiliary(arg) + .map((x) => JSON.stringify(x)); + + return { + fields, + aux, + }; + } else { + const serializer = this.getProofSerializer(argProofs[0]); + // eslint-disable-next-line @typescript-eslint/no-unsafe-argument + return await serializer.toJSON(arg); + } + }) + ).filter(filterNonUndefined); + + return { + methodName: lazyProof.methodName, + zkappClassName: lazyProof.ZkappClass.name, + args: encodedArgs, + blindingValue: lazyProof.blindingValue.toString(), + memoized: lazyProof.memoized.map((value) => ({ + fields: value.fields.map((f) => f.toString()), + aux: value.aux, + })), + }; } - ); + return null; + } + ); const jsonObject: JsonInputObject = { transaction, diff --git a/packages/sequencer/src/settlement/transactions/MinaTransactionSimulator.ts b/packages/sequencer/src/settlement/transactions/MinaTransactionSimulator.ts index b3d8713a2..0323312b0 100644 --- a/packages/sequencer/src/settlement/transactions/MinaTransactionSimulator.ts +++ b/packages/sequencer/src/settlement/transactions/MinaTransactionSimulator.ts @@ -18,7 +18,7 @@ import { } from "@proto-kit/protocol"; import { match } from "ts-pattern"; import { inject, injectable } from "tsyringe"; -import { hashWithPrefix, noop, range } from "@proto-kit/common"; +import { hashWithPrefix, log, noop, range } from "@proto-kit/common"; import { distinctByPredicate } from "../../helpers/utils"; import type { MinaBaseLayer } from "../../protocol/baselayer/MinaBaseLayer"; @@ -173,17 +173,26 @@ export class MinaTransactionSimulator { const getAccountSafe = () => { try { return Mina.getAccount(publicKey, tokenId); - } catch { + } catch (e) { + log.trace(e); return undefined; } }; const account = match(fetchedAccount) .with(undefined, () => getAccountSafe()) - .with({ account: undefined }, () => getAccountSafe()) + .with({ account: undefined }, (e) => { + // TODO Check if it's a "account not found" error, and if it's not then display the error + // (as it's probably networking related) + log.trace(e.error); + return getAccountSafe(); + }) .with({ error: undefined }, (v) => v.account) .exhaustive(); if (account !== undefined) { + log.trace( + `Reloaded account ${account.publicKey.toBase58()}, ${account.balance.toJSON()} MINA` + ); addCachedAccount(account); this.loaded[key] = account; } diff --git a/packages/sequencer/src/settlement/utils/SettlementUtils.ts b/packages/sequencer/src/settlement/utils/SettlementUtils.ts index 9c9f2bba8..f0824a6dc 100644 --- a/packages/sequencer/src/settlement/utils/SettlementUtils.ts +++ b/packages/sequencer/src/settlement/utils/SettlementUtils.ts @@ -2,12 +2,13 @@ import { Bool, fetchAccount, Field, + Mina, PrivateKey, PublicKey, Transaction, UInt32, } from "o1js"; -import { mapSequential } from "@proto-kit/common"; +import { log, mapSequential } from "@proto-kit/common"; import type { MinaBaseLayer } from "../../protocol/baselayer/MinaBaseLayer"; import { MinaSigner } from "../MinaSigner"; @@ -27,6 +28,29 @@ export class SettlementUtils { private readonly signer: MinaSigner ) {} + public async fetchNonce(publicKey: PublicKey): Promise { + const account = await this.safeFetchAccount(publicKey); + if (account !== undefined) { + return parseInt(account.nonce.toString(), 10); + } + return undefined; + } + + public async safeFetchAccount(publicKey: PublicKey, tokenId?: Field) { + const isLocal = this.baseLayer.isLocalBlockChain(); + if (isLocal && Mina.hasAccount(publicKey, tokenId)) { + return Mina.getAccount(publicKey, tokenId); + } else if (!isLocal) { + const fetchResult = await fetchAccount({ publicKey, tokenId }); + if (fetchResult.account !== undefined) { + return fetchResult.account; + } else { + log.info(fetchResult.error); + } + } + return undefined; + } + public signTransaction( tx: Transaction, options: SignTransactionOptions = {} diff --git a/packages/sequencer/src/state/async/AsyncLinkedLeafStore.ts b/packages/sequencer/src/state/async/AsyncLinkedLeafStore.ts index 5d9dcc0fb..3725a9986 100644 --- a/packages/sequencer/src/state/async/AsyncLinkedLeafStore.ts +++ b/packages/sequencer/src/state/async/AsyncLinkedLeafStore.ts @@ -1,13 +1,7 @@ import { StoredLeaf } from "@proto-kit/common"; -import { AsyncMerkleTreeStore } from "./AsyncMerkleTreeStore"; - export interface AsyncLinkedLeafStore { - treeStore: AsyncMerkleTreeStore; - - openTransaction: () => Promise; - - commit: () => Promise; + flush: () => Promise; writeLeaves: (leaves: StoredLeaf[]) => void; diff --git a/packages/sequencer/src/state/lmt/CachedLinkedLeafStore.ts b/packages/sequencer/src/state/lmt/CachedLinkedLeafStore.ts index b55c1b267..53c83203e 100644 --- a/packages/sequencer/src/state/lmt/CachedLinkedLeafStore.ts +++ b/packages/sequencer/src/state/lmt/CachedLinkedLeafStore.ts @@ -13,6 +13,8 @@ import groupBy from "lodash/groupBy"; import { AsyncLinkedLeafStore } from "../async/AsyncLinkedLeafStore"; import { CachedMerkleTreeStore } from "../merkle/CachedMerkleTreeStore"; +import { AsyncMerkleTreeStore } from "../async/AsyncMerkleTreeStore"; +import { Database } from "../../storage/Database"; export class CachedLinkedLeafStore implements LinkedLeafStore { private writeCache: { @@ -23,8 +25,11 @@ export class CachedLinkedLeafStore implements LinkedLeafStore { private readonly treeCache: CachedMerkleTreeStore; - private constructor(private readonly parent: AsyncLinkedLeafStore) { - this.treeCache = new CachedMerkleTreeStore(parent.treeStore); + private constructor( + private readonly parent: AsyncLinkedLeafStore, + parentTreeStore: AsyncMerkleTreeStore + ) { + this.treeCache = new CachedMerkleTreeStore(parentTreeStore); } public get treeStore() { @@ -32,9 +37,10 @@ export class CachedLinkedLeafStore implements LinkedLeafStore { } public static async new( - parent: AsyncLinkedLeafStore + parent: AsyncLinkedLeafStore, + parentTreeStore: AsyncMerkleTreeStore ): Promise { - const cachedInstance = new CachedLinkedLeafStore(parent); + const cachedInstance = new CachedLinkedLeafStore(parent, parentTreeStore); await cachedInstance.preloadMaximumIndex(); await cachedInstance.preloadZeroNode(); return cachedInstance; @@ -210,24 +216,37 @@ export class CachedLinkedLeafStore implements LinkedLeafStore { await this.preloadKeysInternal(paths); } - // This merges the cache into the parent tree and resets the cache, but not the - // in-memory merkle tree. - public async mergeIntoParent(): Promise { + public async mergeLeavesIntoParent() { const leaves = this.getWrittenLeaves(); // In case no state got set we can skip this step if (leaves.length === 0) { return; } - await this.parent.openTransaction(); - this.parent.writeLeaves(Object.values(leaves)); - await this.parent.commit(); + await this.parent.flush(); + + this.resetWrittenLeaves(); + } + public async mergeTreeIntoParent() { await this.treeCache.mergeIntoParent(); + } - this.resetWrittenLeaves(); + // This merges the cache into the parent tree and resets the cache, but not the + // in-memory merkle tree. + public async mergeIntoParent( + stateDb: Database, + treeDb: Database + ): Promise { + await stateDb.executeInTransaction(async () => { + await this.mergeLeavesIntoParent(); + }); + + await treeDb.executeInTransaction(async () => { + await this.mergeTreeIntoParent(); + }); } public getPreviousLeaf(path: bigint) { diff --git a/packages/sequencer/src/storage/StorageDependencyFactory.ts b/packages/sequencer/src/storage/StorageDependencyFactory.ts index dab77170f..51b95a35f 100644 --- a/packages/sequencer/src/storage/StorageDependencyFactory.ts +++ b/packages/sequencer/src/storage/StorageDependencyFactory.ts @@ -26,6 +26,9 @@ export interface StorageDependencyMinimumDependencies< messageStorage: DependencyDeclaration; settlementStorage: DependencyDeclaration; transactionStorage: DependencyDeclaration; + + asyncTreeStore: DependencyDeclaration; + unprovenTreeStore: DependencyDeclaration; } export interface DatabaseDependencyFactory { diff --git a/packages/sequencer/src/storage/inmemory/InMemoryAsyncLinkedLeafStore.ts b/packages/sequencer/src/storage/inmemory/InMemoryAsyncLinkedLeafStore.ts index 382e2ca7a..964d38a05 100644 --- a/packages/sequencer/src/storage/inmemory/InMemoryAsyncLinkedLeafStore.ts +++ b/packages/sequencer/src/storage/inmemory/InMemoryAsyncLinkedLeafStore.ts @@ -2,28 +2,10 @@ import { InMemoryLinkedLeafStore, LinkedLeaf, noop } from "@proto-kit/common"; import { AsyncLinkedLeafStore } from "../../state/async/AsyncLinkedLeafStore"; -import { InMemoryAsyncMerkleTreeStore } from "./InMemoryAsyncMerkleTreeStore"; - export class InMemoryAsyncLinkedLeafStore implements AsyncLinkedLeafStore { private readonly leafStore = new InMemoryLinkedLeafStore(); - private readonly nodeStore = new InMemoryAsyncMerkleTreeStore(); - - // public constructor() { - // const initialLeaf = initialLinkedLeaf(); - // this.leafStore.setLeaf(0n, initialLeaf); - // this.nodeStore.writeNodes([{ }]); - // } - - public get treeStore() { - return this.nodeStore; - } - - public async openTransaction(): Promise { - noop(); - } - - public async commit(): Promise { + public async flush(): Promise { noop(); } diff --git a/packages/sequencer/src/storage/inmemory/InMemoryDatabase.ts b/packages/sequencer/src/storage/inmemory/InMemoryDatabase.ts index 3a20eff9c..e1082b955 100644 --- a/packages/sequencer/src/storage/inmemory/InMemoryDatabase.ts +++ b/packages/sequencer/src/storage/inmemory/InMemoryDatabase.ts @@ -56,6 +56,15 @@ export class InMemoryDatabase extends SequencerModule implements Database { transactionStorage: { useClass: InMemoryTransactionStorage, }, + unprovenTreeStore: { + useClass: InMemoryAsyncMerkleTreeStore, + }, + asyncTreeStore: { + useClass: InMemoryAsyncMerkleTreeStore, + }, + treeDatabase: { + useToken: "Database", + }, }; } diff --git a/packages/sequencer/src/worker/flow/UnpreparingTask.ts b/packages/sequencer/src/worker/flow/UnpreparingTask.ts index 4ec71f076..c6eb3ff93 100644 --- a/packages/sequencer/src/worker/flow/UnpreparingTask.ts +++ b/packages/sequencer/src/worker/flow/UnpreparingTask.ts @@ -1,4 +1,4 @@ -import { noop } from "@proto-kit/common"; +import { NoConfig, noop } from "@proto-kit/common"; import { TaskWorkerModule } from "../worker/TaskWorkerModule"; @@ -8,8 +8,8 @@ import { Task, TaskSerializer } from "./Task"; * Contract: * Doesn't implement prepare() */ -export abstract class UnpreparingTask - extends TaskWorkerModule +export abstract class UnpreparingTask + extends TaskWorkerModule implements Task { abstract name: string; diff --git a/packages/sequencer/src/worker/queue/LocalTaskQueue.ts b/packages/sequencer/src/worker/queue/LocalTaskQueue.ts index 5ce6826f3..df03e90a3 100644 --- a/packages/sequencer/src/worker/queue/LocalTaskQueue.ts +++ b/packages/sequencer/src/worker/queue/LocalTaskQueue.ts @@ -149,6 +149,10 @@ export class LocalTaskQueue }); this.queuedTasks[queueName] = []; return functions; + } else if (tasks.length > 0) { + log.warn( + `Tasks found in queue ${queueName} but no worker registered` + ); } return []; @@ -161,11 +165,17 @@ export class LocalTaskQueue this.taskInProgress = false; // In case new tasks came up in the meantime, execute them as well - if (tasksToExecute.length > 0) { + if (this.hasTasksQueued()) { await this.workNextTasks(); } } + private hasTasksQueued() { + return Object.entries(this.queuedTasks).some( + ([, tasks]) => tasks.length > 0 + ); + } + public createWorker( queueName: string, executor: (data: TaskPayload) => Promise, diff --git a/packages/sequencer/src/worker/worker/startup/CloseWorkerError.ts b/packages/sequencer/src/worker/startup/CloseWorkerError.ts similarity index 100% rename from packages/sequencer/src/worker/worker/startup/CloseWorkerError.ts rename to packages/sequencer/src/worker/startup/CloseWorkerError.ts diff --git a/packages/sequencer/src/worker/worker/startup/WorkerRegistrationFlow.ts b/packages/sequencer/src/worker/startup/WorkerRegistrationFlow.ts similarity index 90% rename from packages/sequencer/src/worker/worker/startup/WorkerRegistrationFlow.ts rename to packages/sequencer/src/worker/startup/WorkerRegistrationFlow.ts index 17340300e..be6133a06 100644 --- a/packages/sequencer/src/worker/worker/startup/WorkerRegistrationFlow.ts +++ b/packages/sequencer/src/worker/startup/WorkerRegistrationFlow.ts @@ -1,8 +1,8 @@ import { injectable } from "tsyringe"; import { log } from "@proto-kit/common"; -import { Closeable } from "../../../sequencer/builder/Closeable"; -import { FlowCreator } from "../../flow/Flow"; +import { Closeable } from "../../sequencer/builder/Closeable"; +import { FlowCreator } from "../flow/Flow"; import { WorkerRegistrationTask, diff --git a/packages/sequencer/src/worker/worker/startup/WorkerRegistrationTask.ts b/packages/sequencer/src/worker/startup/WorkerRegistrationTask.ts similarity index 90% rename from packages/sequencer/src/worker/worker/startup/WorkerRegistrationTask.ts rename to packages/sequencer/src/worker/startup/WorkerRegistrationTask.ts index 3d4474717..d5343b15e 100644 --- a/packages/sequencer/src/worker/worker/startup/WorkerRegistrationTask.ts +++ b/packages/sequencer/src/worker/startup/WorkerRegistrationTask.ts @@ -15,18 +15,19 @@ import { } from "@proto-kit/protocol"; import { VerificationKey } from "o1js"; -import { Task } from "../../flow/Task"; -import { AbstractStartupTask } from "../../flow/AbstractStartupTask"; +import { Task } from "../flow/Task"; +import { AbstractStartupTask } from "../flow/AbstractStartupTask"; import { VerificationKeyJSON, VerificationKeySerializer, -} from "../../../protocol/production/tasks/serializers/VerificationKeySerializer"; +} from "../../protocol/production/tasks/serializers/VerificationKeySerializer"; import { ArtifactRecordSerializer, SerializedArtifactRecord, -} from "../../../protocol/production/tasks/serializers/ArtifactionRecordSerializer"; -import { SignedSettlementPermissions } from "../../../settlement/permissions/SignedSettlementPermissions"; -import { ProvenSettlementPermissions } from "../../../settlement/permissions/ProvenSettlementPermissions"; +} from "../../protocol/production/tasks/serializers/ArtifactionRecordSerializer"; +import { SignedSettlementPermissions } from "../../settlement/permissions/SignedSettlementPermissions"; +import { ProvenSettlementPermissions } from "../../settlement/permissions/ProvenSettlementPermissions"; +import { task } from "../worker/TaskWorkerModule"; import { CloseWorkerError } from "./CloseWorkerError"; @@ -39,6 +40,7 @@ export type WorkerStartupPayload = { }; @injectable() +@task() export class WorkerRegistrationTask extends AbstractStartupTask implements Task diff --git a/packages/sequencer/src/worker/worker/FlowTaskWorker.ts b/packages/sequencer/src/worker/worker/FlowTaskWorker.ts index 3c8fbdf2b..c86d47654 100644 --- a/packages/sequencer/src/worker/worker/FlowTaskWorker.ts +++ b/packages/sequencer/src/worker/worker/FlowTaskWorker.ts @@ -12,16 +12,14 @@ const errors = { }; // Had to use any here, because otherwise you couldn't assign any tasks to it -export class FlowTaskWorker< - Tasks extends Task[], -> implements Closeable { +export class FlowTaskWorker implements Closeable { private readonly queue: TaskQueue; private workers: Record = {}; public constructor( mq: TaskQueue, - private readonly tasks: Tasks + private readonly tasks: Task[] ) { this.queue = mq; } diff --git a/packages/sequencer/src/worker/worker/LocalTaskWorkerModule.ts b/packages/sequencer/src/worker/worker/LocalTaskWorkerModule.ts index 85fa05663..7b890b29e 100644 --- a/packages/sequencer/src/worker/worker/LocalTaskWorkerModule.ts +++ b/packages/sequencer/src/worker/worker/LocalTaskWorkerModule.ts @@ -7,8 +7,6 @@ import { ModulesRecord, NoConfig, Presets, - ResolvableModules, - StringKeyOf, TypedClass, } from "@proto-kit/common"; import { ReturnType } from "@proto-kit/protocol"; @@ -23,16 +21,15 @@ import { SettlementProvingTask } from "../../settlement/tasks/SettlementProvingT import { Task } from "../flow/Task"; import { TaskQueue } from "../queue/TaskQueue"; import { StateTransitionTask } from "../../protocol/production/tasks/StateTransitionTask"; -import { CircuitCompilerTask } from "../../protocol/production/tasks/CircuitCompilerTask"; import { closeable } from "../../sequencer/builder/Closeable"; import { StateTransitionReductionTask } from "../../protocol/production/tasks/StateTransitionReductionTask"; import { TransactionProvingTask } from "../../protocol/production/tasks/TransactionProvingTask"; import { BlockReductionTask } from "../../protocol/production/tasks/BlockReductionTask"; import { TransactionReductionTask } from "../../protocol/production/tasks/TransactionReductionTask"; +import { WorkerRegistrationTask } from "../startup/WorkerRegistrationTask"; import { FlowTaskWorker } from "./FlowTaskWorker"; import { TaskWorkerModule } from "./TaskWorkerModule"; -import { WorkerRegistrationTask } from "./startup/WorkerRegistrationTask"; // Temporary workaround against the compiler emitting // import("common/dist") inside the library artifacts @@ -41,7 +38,7 @@ export { TypedClass }; export type TaskWorkerModulesRecord = ModulesRecord< // TODO any -> unknown - TypedClass> + TypedClass & Task> >; type LocalTaskWorkerModuleEvents = { ready: [boolean] }; @@ -64,9 +61,7 @@ export class LocalTaskWorkerModule public containerEvents = new EventEmitter(); - private worker?: FlowTaskWorker< - InstanceType[StringKeyOf]>[] - > = undefined; + private worker?: FlowTaskWorker = undefined; public static from( modules: Tasks @@ -98,14 +93,17 @@ export class LocalTaskWorkerModule return this.container.resolve("TaskQueue"); } + public tasks() { + return this.container.resolveAll>("Task"); + } + public async start(): Promise { - const tasks = this.moduleNames.map((moduleName) => { - this.assertIsValidModuleName(moduleName); + const tasks = this.tasks(); - const task = this.resolve(moduleName); - log.debug(`Resolved task ${task.name}`); - return task; - }); + log.debug( + "Resolved tasks", + tasks.map((t) => t.name) + ); const worker = new FlowTaskWorker(this.taskQueue(), [...tasks]); this.worker = worker; @@ -139,7 +137,6 @@ export class VanillaTaskWorkerModules { TransactionReductionTask, BlockReductionTask, NewBlockTask, - CircuitCompilerTask, WorkerRegistrationTask, } satisfies TaskWorkerModulesRecord; } @@ -161,7 +158,6 @@ export class VanillaTaskWorkerModules { NewBlockTask: {}, StateTransitionReductionTask: {}, SettlementProvingTask: {}, - CircuitCompilerTask: {}, WorkerRegistrationTask: {}, } satisfies ModulesConfig< ReturnType diff --git a/packages/sequencer/src/worker/worker/TaskWorkerModule.ts b/packages/sequencer/src/worker/worker/TaskWorkerModule.ts index 35ca7b3f8..9fd8acdac 100644 --- a/packages/sequencer/src/worker/worker/TaskWorkerModule.ts +++ b/packages/sequencer/src/worker/worker/TaskWorkerModule.ts @@ -1,3 +1,11 @@ -import { ConfigurableModule, NoConfig } from "@proto-kit/common"; +import { ConfigurableModule, implement, NoConfig } from "@proto-kit/common"; -export abstract class TaskWorkerModule extends ConfigurableModule {} +import { Task } from "../flow/Task"; + +export abstract class TaskWorkerModule< + Config = NoConfig, +> extends ConfigurableModule {} + +export function task() { + return implement>("Task"); +} diff --git a/packages/sequencer/test-integration/workers/worker.ts b/packages/sequencer/test-integration/workers/worker.ts index c9210b74b..2772cdcb4 100644 --- a/packages/sequencer/test-integration/workers/worker.ts +++ b/packages/sequencer/test-integration/workers/worker.ts @@ -1,4 +1,6 @@ import "reflect-metadata"; + +import { setBackend } from "o1js"; // eslint-disable-next-line import/no-extraneous-dependencies import { BullQueue } from "@proto-kit/deployment"; import { container } from "tsyringe"; @@ -19,6 +21,8 @@ import { } from "./modules"; import { MinimumWorkerModules } from "./WorkerModules"; +setBackend("native"); + /* eslint-disable no-console */ async function main() { const proofsEnabled = process.env.PROOFS_ENABLED === "true"; diff --git a/packages/sequencer/test-integration/workers/workers-proven.test.ts b/packages/sequencer/test-integration/workers/workers-proven.test.ts index 047a92913..602baa16c 100644 --- a/packages/sequencer/test-integration/workers/workers-proven.test.ts +++ b/packages/sequencer/test-integration/workers/workers-proven.test.ts @@ -28,7 +28,7 @@ import { ChildProcessWorker } from "./ChildProcessWorker"; const timeout = 300000; // true -const proofsEnabled = false; +const proofsEnabled = true; const numWorkers = 1; diff --git a/packages/sequencer/test-proven/Proven.test.ts b/packages/sequencer/test-proven/Proven.test.ts index 14e41f9dd..1ae7939f5 100644 --- a/packages/sequencer/test-proven/Proven.test.ts +++ b/packages/sequencer/test-proven/Proven.test.ts @@ -18,7 +18,7 @@ import { } from "@proto-kit/protocol"; import { VanillaProtocolModules } from "@proto-kit/library"; import { container } from "tsyringe"; -import { PrivateKey, UInt64 } from "o1js"; +import { PrivateKey, UInt64, setBackend } from "o1js"; import { testingSequencerModules } from "../test/TestingSequencer"; import { @@ -35,6 +35,8 @@ import { ProvenBalance } from "../test/integration/mocks/ProvenBalance"; const timeout = 300000; +setBackend("native"); + describe("Proven", () => { let test: BlockTestService; @@ -221,6 +223,7 @@ describe("Proven", () => { }, timeout ); + it( "should produce large block", async () => { @@ -260,6 +263,45 @@ describe("Proven", () => { timeout * 10 ); + it( + "should produce empty + 1 tx", + async () => { + log.setLevel("INFO"); + + const privateKey = PrivateKey.random(); + + // await test.produceBlock(); + // await test.produceBlock(); + // await test.produceBlock(); + + await test.addTransaction({ + method: ["Balances", "addBalance"], + privateKey, + args: [PrivateKey.random().toPublicKey(), UInt64.from(100)], + }); + + // Produce 6 blocks, 5 txs each into 1 batch + const block = await test.produceBlock(); + await test.produceBlock(); + await test.produceBlock(); + await test.produceBlock(); + + expectDefined(block); + expect(block.transactions).toHaveLength(1); + expect(block.transactions[0].status.toBoolean()).toBe(true); + + const batch = await test.produceBatch(); + + expectDefined(batch); + + console.log(batch.proof); + + expect(batch.blockHashes).toHaveLength(4); + expect(batch.proof.proof.length).toBeGreaterThan(50); + }, + timeout * 10 + ); + afterAll(async () => { await appChain.close(); }); diff --git a/packages/sequencer/test/LinkedMerkleTreeIntegrity.ts b/packages/sequencer/test/LinkedMerkleTreeIntegrity.ts index c19a6f5d8..68e18ac33 100644 --- a/packages/sequencer/test/LinkedMerkleTreeIntegrity.ts +++ b/packages/sequencer/test/LinkedMerkleTreeIntegrity.ts @@ -1,10 +1,13 @@ import { Field } from "o1js"; import { LinkedLeafStruct, log } from "@proto-kit/common"; -import { AsyncLinkedLeafStore } from "../src/state/async/AsyncLinkedLeafStore"; +import { AsyncMerkleTreeStore, AsyncLinkedLeafStore } from "../src"; export namespace LinkedMerkleTreeIntegrity { - export async function checkIntegrity(store: AsyncLinkedLeafStore) { + export async function checkIntegrity( + store: AsyncLinkedLeafStore, + treeStore: AsyncMerkleTreeStore + ) { log.info("Checking tree integrity..."); let currentPath = 0n; @@ -18,7 +21,7 @@ export namespace LinkedMerkleTreeIntegrity { const leaf = leaves[0]!; - const treeValues = await store.treeStore.getNodesAsync([ + const treeValues = await treeStore.getNodesAsync([ { level: 0, key: leaf.index }, ]); if (treeValues.length === 0 || treeValues[0] === undefined) { diff --git a/packages/sequencer/test/integration/BlockProduction-test.ts b/packages/sequencer/test/integration/BlockProduction-test.ts index 8e2764a42..d98a1fecb 100644 --- a/packages/sequencer/test/integration/BlockProduction-test.ts +++ b/packages/sequencer/test/integration/BlockProduction-test.ts @@ -36,6 +36,7 @@ import { AppChain, BlockProducerModule, DatabaseDependencyFactory, + AsyncMerkleTreeStore, } from "../../src"; import { DefaultTestingSequencerModules, @@ -116,6 +117,7 @@ export function testBlockProduction< let test: BlockTestService; let linkedLeafStore: AsyncLinkedLeafStore; + let treeStore: AsyncMerkleTreeStore; beforeEach(async () => { const runtimeClass = Runtime.from({ @@ -189,7 +191,11 @@ export function testBlockProduction< app.sequencer.dependencyContainer.resolve( "UnprovenLinkedLeafStore" ); - }); + treeStore = + app.sequencer.dependencyContainer.resolve( + "UnprovenTreeStore" + ); + }, 30000); afterEach(async () => { await appChain.close(); @@ -302,7 +308,7 @@ export function testBlockProduction< expect(UInt64.fromFields(state2!)).toStrictEqual(UInt64.from(200)); await expect( - LinkedMerkleTreeIntegrity.checkIntegrity(linkedLeafStore) + LinkedMerkleTreeIntegrity.checkIntegrity(linkedLeafStore, treeStore) ).resolves.toBe(true); }, 60_000); @@ -337,7 +343,7 @@ export function testBlockProduction< expect(newState).toBeUndefined(); await expect( - LinkedMerkleTreeIntegrity.checkIntegrity(linkedLeafStore) + LinkedMerkleTreeIntegrity.checkIntegrity(linkedLeafStore, treeStore) ).resolves.toBe(true); }, 30_000); @@ -420,7 +426,7 @@ export function testBlockProduction< expect(block2!.transactions[0].statusMessage).toBeUndefined(); await expect( - LinkedMerkleTreeIntegrity.checkIntegrity(linkedLeafStore) + LinkedMerkleTreeIntegrity.checkIntegrity(linkedLeafStore, treeStore) ).resolves.toBe(true); }, 60_000); diff --git a/packages/sequencer/test/integration/MempoolTxRemoved.test.ts b/packages/sequencer/test/integration/MempoolTxRemoved.test.ts index 02fae71f5..ace902990 100644 --- a/packages/sequencer/test/integration/MempoolTxRemoved.test.ts +++ b/packages/sequencer/test/integration/MempoolTxRemoved.test.ts @@ -27,6 +27,7 @@ describe("mempool removal mechanism", () => { let trigger: ManualBlockTrigger; const createAppChain = async () => { + log.setLevel("TRACE"); const app = AppChain.from({ Sequencer: Sequencer.from(testingSequencerModules({})), Protocol: Protocol.from(Protocol.defaultModules()), diff --git a/packages/sequencer/test/merkle/CachedLinkedMerkleStore.test.ts b/packages/sequencer/test/merkle/CachedLinkedMerkleStore.test.ts index 5ceae2768..c95eaeab3 100644 --- a/packages/sequencer/test/merkle/CachedLinkedMerkleStore.test.ts +++ b/packages/sequencer/test/merkle/CachedLinkedMerkleStore.test.ts @@ -9,24 +9,31 @@ import { Field, Poseidon } from "o1js"; import { CachedLinkedLeafStore } from "../../src/state/lmt/CachedLinkedLeafStore"; import { InMemoryAsyncLinkedLeafStore } from "../../src/storage/inmemory/InMemoryAsyncLinkedLeafStore"; import { SyncCachedLinkedLeafStore } from "../../src/state/merkle/SyncCachedLinkedLeafStore"; +import { InMemoryAsyncMerkleTreeStore } from "../../src"; describe("cached linked merkle store", () => { let mainStore: InMemoryAsyncLinkedLeafStore; + let mainTreeStore: InMemoryAsyncMerkleTreeStore; let cache1: CachedLinkedLeafStore; let tree1: LinkedMerkleTree; beforeEach(async () => { mainStore = new InMemoryAsyncLinkedLeafStore(); + mainTreeStore = new InMemoryAsyncMerkleTreeStore(); - const cachedStore = await CachedLinkedLeafStore.new(mainStore); + const cachedStore = await CachedLinkedLeafStore.new( + mainStore, + mainTreeStore + ); const tmpTree = new LinkedMerkleTree(cachedStore.treeStore, cachedStore); tmpTree.setLeaf(5n, 10n); - await cachedStore.mergeIntoParent(); + await cachedStore.mergeLeavesIntoParent(); + await cachedStore.mergeTreeIntoParent(); - cache1 = await CachedLinkedLeafStore.new(mainStore); + cache1 = await CachedLinkedLeafStore.new(mainStore, mainTreeStore); tree1 = new LinkedMerkleTree(cache1.treeStore, cache1); }); @@ -78,7 +85,8 @@ describe("cached linked merkle store", () => { expectDefined(cache1.treeStore.getNode(1n, 0)); tree1.setLeaf(10n, 10n); - await cache1.mergeIntoParent(); + await cache1.mergeLeavesIntoParent(); + await cache1.mergeTreeIntoParent(); const leaf5 = tree1.getLeaf(5n); const leaf10 = tree1.getLeaf(10n); @@ -113,7 +121,8 @@ describe("cached linked merkle store", () => { tree1.setLeaf(11n, 11n); tree1.setLeaf(12n, 12n); tree1.setLeaf(13n, 13n); - await cache1.mergeIntoParent(); + await cache1.mergeLeavesIntoParent(); + await cache1.mergeTreeIntoParent(); const cache2 = new SyncCachedLinkedLeafStore(cache1); await cache2.preloadKeys([14n]); @@ -212,7 +221,7 @@ describe("cached linked merkle store", () => { leaf2.hash().toBigInt() ); expect(tree1.getRoot()).not.toEqual(tree2.getRoot()); - await cache2.mergeIntoParent(); + cache2.mergeIntoParent(); expect(tree1.getRoot()).toEqual(tree2.getRoot()); }); @@ -223,7 +232,8 @@ describe("cached linked merkle store", () => { await cache1.preloadKeys([10n, 20n]); treeCache1.setLeaf(10n, 10n); treeCache1.setLeaf(20n, 20n); - await cache1.mergeIntoParent(); + await cache1.mergeLeavesIntoParent(); + await cache1.mergeTreeIntoParent(); const cache2 = new SyncCachedLinkedLeafStore(cache1); const treeCache2 = new LinkedMerkleTree(cache2.treeStore, cache2); @@ -307,7 +317,7 @@ describe("cached linked merkle store", () => { expectDefined(leaf1); expectDefined(storedLeaf1); await expect( - mainStore.treeStore.getNodesAsync([{ key: storedLeaf1.index, level: 0 }]) + mainTreeStore.getNodesAsync([{ key: storedLeaf1.index, level: 0 }]) ).resolves.toStrictEqual([ Poseidon.hash([leaf1.value, leaf1.path, leaf1.nextPath]).toBigInt(), ]); @@ -379,9 +389,13 @@ describe("cached linked merkle store", () => { ); // Now the mainstore has the new 15n root. - await cache1.mergeIntoParent(); + await cache1.mergeLeavesIntoParent(); + await cache1.mergeTreeIntoParent(); - const cachedStore = await CachedLinkedLeafStore.new(mainStore); + const cachedStore = await CachedLinkedLeafStore.new( + mainStore, + mainTreeStore + ); await cachedStore.preloadKey(15n); expect( @@ -395,7 +409,8 @@ describe("cached linked merkle store", () => { expect.assertions(16); const mStore = new InMemoryAsyncLinkedLeafStore(); - const mCache = await CachedLinkedLeafStore.new(mStore); + const mTreeStore = new InMemoryAsyncMerkleTreeStore(); + const mCache = await CachedLinkedLeafStore.new(mStore, mTreeStore); const mCache2 = new SyncCachedLinkedLeafStore(mCache); const treeCache1 = new LinkedMerkleTree(mCache.treeStore, mCache); const treeCache2 = new LinkedMerkleTree(mCache2.treeStore, mCache2); diff --git a/packages/sequencer/test/production/tracing/StateTransitionTracingService.test.ts b/packages/sequencer/test/production/tracing/StateTransitionTracingService.test.ts index a670b41c6..d87974d51 100644 --- a/packages/sequencer/test/production/tracing/StateTransitionTracingService.test.ts +++ b/packages/sequencer/test/production/tracing/StateTransitionTracingService.test.ts @@ -19,8 +19,9 @@ import { StateTransitionProofParameters, ConsoleTracer, CachedLinkedLeafStore, + InMemoryAsyncMerkleTreeStore, + InMemoryAsyncLinkedLeafStore, } from "../../../src"; -import { InMemoryAsyncLinkedLeafStore } from "../../../src/storage/inmemory/InMemoryAsyncLinkedLeafStore"; function createST(obj: { path: string; @@ -115,7 +116,8 @@ async function applyBatchesToTree( // return sequencer; // } -const service = new StateTransitionTracingService(new ConsoleTracer()); +const tracer = new ConsoleTracer(); +const service = new StateTransitionTracingService(tracer); describe("StateTransitionTracingService", () => { const cases: { @@ -163,11 +165,12 @@ describe("StateTransitionTracingService", () => { describe.each(cases)("tracing two chunks of STs", ({ batch, numSTs }) => { const store = new InMemoryAsyncLinkedLeafStore(); + const treeStore = new InMemoryAsyncMerkleTreeStore(); let trace: StateTransitionProofParameters[]; beforeAll(async () => { - const cached = await CachedLinkedLeafStore.new(store); + const cached = await CachedLinkedLeafStore.new(store, treeStore); trace = await service.createMerkleTrace(cached, batch); }); @@ -187,7 +190,7 @@ describe("StateTransitionTracingService", () => { it("should set second publicInput correctly", async () => { const tree = await applyBatchesToTree( batch.slice(0, 4), - await CachedLinkedLeafStore.new(store) + await CachedLinkedLeafStore.new(store, treeStore) ); expect(trace[1].publicInput.root.toString()).toStrictEqual( @@ -222,7 +225,7 @@ describe("StateTransitionTracingService", () => { const witnessedRootsList = new WitnessedRootHashList(); const tempTree = await applyBatchesToTree( batch.slice(0, 2), - await CachedLinkedLeafStore.new(store) + await CachedLinkedLeafStore.new(store, treeStore) ); witnessedRootsList.push({ @@ -242,6 +245,7 @@ describe("StateTransitionTracingService", () => { describe("tracing two separate sequences", () => { const store = new InMemoryAsyncLinkedLeafStore(); + const treeStore = new InMemoryAsyncMerkleTreeStore(); let cached: CachedLinkedLeafStore; let trace1: StateTransitionProofParameters[]; @@ -271,10 +275,10 @@ describe("StateTransitionTracingService", () => { ]; beforeAll(async () => { - cached = await CachedLinkedLeafStore.new(store); + cached = await CachedLinkedLeafStore.new(store, treeStore); trace1 = await service.createMerkleTrace(cached, batches[0]); - const cached2 = await CachedLinkedLeafStore.new(store); + const cached2 = await CachedLinkedLeafStore.new(store, treeStore); tree1 = await applyBatchesToTree(batches[0], cached2); trace2 = await service.createMerkleTrace(cached, batches[1]); @@ -304,6 +308,7 @@ describe("StateTransitionTracingService", () => { describe("should trace correctly", () => { const store = new InMemoryAsyncLinkedLeafStore(); + const treeStore = new InMemoryAsyncMerkleTreeStore(); let cached: CachedLinkedLeafStore; const batches: TracingStateTransitionBatch[] = [ @@ -327,7 +332,7 @@ describe("StateTransitionTracingService", () => { let trace: StateTransitionProofParameters[]; beforeAll(async () => { - cached = await CachedLinkedLeafStore.new(store); + cached = await CachedLinkedLeafStore.new(store, treeStore); trace = await service.createMerkleTrace(cached, batches); }); @@ -371,7 +376,7 @@ describe("StateTransitionTracingService", () => { it("check that STs have been applied to the tree store", async () => { const tracedTree = new LinkedMerkleTree(cached.treeStore, cached); - const cached2 = await CachedLinkedLeafStore.new(store); + const cached2 = await CachedLinkedLeafStore.new(store, treeStore); const tree = await applyBatchesToTree(batches, cached2); expect(tracedTree.getRoot().toString()).toStrictEqual( @@ -379,4 +384,8 @@ describe("StateTransitionTracingService", () => { ); }); }); + + afterAll(async () => { + await tracer.close(); + }); }); diff --git a/packages/sequencer/test/protocol/production/sequencing/atomic-block-production.test.ts b/packages/sequencer/test/protocol/production/sequencing/atomic-block-production.test.ts index 79dccbcd4..dde755d41 100644 --- a/packages/sequencer/test/protocol/production/sequencing/atomic-block-production.test.ts +++ b/packages/sequencer/test/protocol/production/sequencing/atomic-block-production.test.ts @@ -89,7 +89,7 @@ describe("atomic block production", () => { * the second block production can succeed */ it("should recover from non-generated metadata", async () => { - expect.assertions(6); + expect.assertions(5); const module = appchain.sequencer.dependencyContainer.resolve(BlockResultService); @@ -102,9 +102,6 @@ describe("atomic block production", () => { await expect(() => trigger.produceBlock()).rejects.toThrow(); - // This checks that it correctly throws when producing a block with no previous result existing - await expect(() => trigger.produceBlock()).rejects.toThrow(); - await appchain.sequencer .resolve("BlockProducerModule") .blockResultCompleteCheck(); diff --git a/packages/sequencer/test/settlement/Settlement-only.ts b/packages/sequencer/test/settlement/Settlement-only.ts index c894bfbcf..ddae5a758 100644 --- a/packages/sequencer/test/settlement/Settlement-only.ts +++ b/packages/sequencer/test/settlement/Settlement-only.ts @@ -39,6 +39,7 @@ import { SettlementModule, SettlementProvingTask, VanillaTaskWorkerModules, + SettlementCompileTask, } from "../../src"; import { Withdrawals } from "./mocks/Withdrawals"; @@ -80,6 +81,7 @@ export const settlementOnlyTestFn = ( }, { SettlementProvingTask, + SettlementCompileTask, } ) ); diff --git a/packages/sequencer/test/settlement/Settlement.test.ts b/packages/sequencer/test/settlement/Settlement.test.ts index b1aa7044a..683fabe44 100644 --- a/packages/sequencer/test/settlement/Settlement.test.ts +++ b/packages/sequencer/test/settlement/Settlement.test.ts @@ -14,7 +14,7 @@ describe.each(["mock-proofs", "signed"] as const)( }, }; - describe("Default token", () => { + describe.only("Default token", () => { settlementTestFn(type, network); }); diff --git a/packages/stack/package.json b/packages/stack/package.json index 822633680..ac07a527e 100644 --- a/packages/stack/package.json +++ b/packages/stack/package.json @@ -32,7 +32,8 @@ "@proto-kit/protocol": "*", "@proto-kit/sdk": "*", "@proto-kit/sequencer": "*", - "o1js": "^2.10.0", + "o1js": "2.10.0-dev.6d3a3", + "@o1js/native": "2.10.0-dev.6d3a3", "tsyringe": "^4.10.0" }, "devDependencies": {