diff --git a/backend/src/database/migrations/U1770653666__add-automatic_projects_discovery-tables.sql b/backend/src/database/migrations/U1770653666__add-automatic_projects_discovery-tables.sql new file mode 100644 index 0000000000..a32dbe9a91 --- /dev/null +++ b/backend/src/database/migrations/U1770653666__add-automatic_projects_discovery-tables.sql @@ -0,0 +1,11 @@ +DROP INDEX IF EXISTS "ix_evaluatedProjects_onboarded"; +DROP INDEX IF EXISTS "ix_evaluatedProjects_evaluationScore"; +DROP INDEX IF EXISTS "ix_evaluatedProjects_evaluationStatus"; +DROP INDEX IF EXISTS "uix_evaluatedProjects_projectCatalogId"; +DROP TABLE IF EXISTS "evaluatedProjects"; + +DROP INDEX IF EXISTS "ix_projectCatalog_syncedAt"; +DROP INDEX IF EXISTS "ix_projectCatalog_lfCriticalityScore"; +DROP INDEX IF EXISTS "ix_projectCatalog_ossfCriticalityScore"; +DROP INDEX IF EXISTS "uix_projectCatalog_repoUrl"; +DROP TABLE IF EXISTS "projectCatalog"; diff --git a/backend/src/database/migrations/V1770653666__add-automatic_projects_discovery-tables.sql b/backend/src/database/migrations/V1770653666__add-automatic_projects_discovery-tables.sql new file mode 100644 index 0000000000..c2add79aae --- /dev/null +++ b/backend/src/database/migrations/V1770653666__add-automatic_projects_discovery-tables.sql @@ -0,0 +1,42 @@ +-- Project Catalog: candidate projects discovered from OSSF Criticality Score and other sources +CREATE TABLE IF NOT EXISTS "projectCatalog" ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + "projectSlug" VARCHAR(255) NOT NULL, + "repoName" VARCHAR(255) NOT NULL, + "repoUrl" VARCHAR(1024) NOT NULL, + "ossfCriticalityScore" DOUBLE PRECISION, + "lfCriticalityScore" DOUBLE PRECISION, + "syncedAt" TIMESTAMP WITH TIME ZONE DEFAULT NULL, + "createdAt" TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP +); + +CREATE UNIQUE INDEX "uix_projectCatalog_repoUrl" ON "projectCatalog" ("repoUrl"); +CREATE INDEX "ix_projectCatalog_ossfCriticalityScore" ON "projectCatalog" ("ossfCriticalityScore" DESC NULLS LAST); +CREATE INDEX "ix_projectCatalog_lfCriticalityScore" ON "projectCatalog" ("lfCriticalityScore" DESC NULLS LAST); +CREATE INDEX "ix_projectCatalog_syncedAt" ON "projectCatalog" ("syncedAt"); + +-- Evaluated Projects: AI evaluation results linked to catalog entries +CREATE TABLE IF NOT EXISTS "evaluatedProjects" ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + "projectCatalogId" UUID NOT NULL REFERENCES "projectCatalog"(id) ON DELETE CASCADE, + "evaluationStatus" VARCHAR(50) NOT NULL DEFAULT 'pending', + "evaluationScore" DOUBLE PRECISION, + "evaluation" JSONB, + "evaluationReason" TEXT, + "evaluatedAt" TIMESTAMP WITH TIME ZONE, + "starsCount" INTEGER, + "forksCount" INTEGER, + "commitsCount" INTEGER, + "pullRequestsCount" INTEGER, + "issuesCount" INTEGER, + "onboarded" BOOLEAN NOT NULL DEFAULT FALSE, + "onboardedAt" TIMESTAMP WITH TIME ZONE, + "createdAt" TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP +); + +CREATE UNIQUE INDEX "uix_evaluatedProjects_projectCatalogId" ON "evaluatedProjects" ("projectCatalogId"); +CREATE INDEX "ix_evaluatedProjects_evaluationStatus" ON "evaluatedProjects" ("evaluationStatus"); +CREATE INDEX "ix_evaluatedProjects_evaluationScore" ON "evaluatedProjects" ("evaluationScore" DESC NULLS LAST); +CREATE INDEX "ix_evaluatedProjects_onboarded" ON "evaluatedProjects" ("onboarded"); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6233274f15..1fce3b1f03 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -467,6 +467,110 @@ importers: specifier: ^3.3.3 version: 3.3.3 + services/apps/automatic_project_discovery_worker: + dependencies: + '@crowd/archetype-standard': + specifier: workspace:* + version: link:../../archetypes/standard + '@crowd/archetype-worker': + specifier: workspace:* + version: link:../../archetypes/worker + '@crowd/common': + specifier: workspace:* + version: link:../../libs/common + '@crowd/common_services': + specifier: workspace:* + version: link:../../libs/common_services + '@crowd/data-access-layer': + specifier: workspace:* + version: link:../../libs/data-access-layer + '@crowd/logging': + specifier: workspace:* + version: link:../../libs/logging + '@crowd/redis': + specifier: workspace:* + version: link:../../libs/redis + '@crowd/temporal': + specifier: workspace:* + version: link:../../libs/temporal + '@crowd/types': + specifier: workspace:* + version: link:../../libs/types + '@temporalio/activity': + specifier: ~1.11.8 + version: 1.11.8 + '@temporalio/client': + specifier: ~1.11.8 + version: 1.11.8 + '@temporalio/workflow': + specifier: ~1.11.8 + version: 1.11.8 + tsx: + specifier: ^4.7.1 + version: 4.7.3 + typescript: + specifier: ^5.6.3 + version: 5.6.3 + devDependencies: + '@types/node': + specifier: ^20.8.2 + version: 20.12.7 + nodemon: + specifier: ^3.0.1 + version: 3.1.0 + + services/apps/automatic_projects_discovery_worker: + dependencies: + '@crowd/archetype-standard': + specifier: workspace:* + version: link:../../archetypes/standard + '@crowd/archetype-worker': + specifier: workspace:* + version: link:../../archetypes/worker + '@crowd/common': + specifier: workspace:* + version: link:../../libs/common + '@crowd/common_services': + specifier: workspace:* + version: link:../../libs/common_services + '@crowd/data-access-layer': + specifier: workspace:* + version: link:../../libs/data-access-layer + '@crowd/logging': + specifier: workspace:* + version: link:../../libs/logging + '@crowd/redis': + specifier: workspace:* + version: link:../../libs/redis + '@crowd/temporal': + specifier: workspace:* + version: link:../../libs/temporal + '@crowd/types': + specifier: workspace:* + version: link:../../libs/types + '@temporalio/activity': + specifier: ~1.11.8 + version: 1.11.8 + '@temporalio/client': + specifier: ~1.11.8 + version: 1.11.8 + '@temporalio/workflow': + specifier: ~1.11.8 + version: 1.11.8 + tsx: + specifier: ^4.7.1 + version: 4.7.3 + typescript: + specifier: ^5.6.3 + version: 5.6.3 + devDependencies: + '@types/node': + specifier: ^20.8.2 + version: 20.12.7 + nodemon: + specifier: ^3.0.1 + version: 3.1.0 + services/apps/cache_worker: dependencies: '@crowd/archetype-standard': @@ -5426,10 +5530,6 @@ packages: brace-expansion@2.0.1: resolution: {integrity: sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==} - braces@3.0.2: - resolution: {integrity: sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==} - engines: {node: '>=8'} - braces@3.0.3: resolution: {integrity: sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==} engines: {node: '>=8'} @@ -6562,10 +6662,6 @@ packages: file-uri-to-path@1.0.0: resolution: {integrity: sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==} - fill-range@7.0.1: - resolution: {integrity: sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==} - engines: {node: '>=8'} - fill-range@7.1.1: resolution: {integrity: sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==} engines: {node: '>=8'} @@ -10047,8 +10143,8 @@ snapshots: dependencies: '@aws-crypto/sha256-browser': 3.0.0 '@aws-crypto/sha256-js': 3.0.0 - '@aws-sdk/client-sso-oidc': 3.572.0 - '@aws-sdk/client-sts': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0) + '@aws-sdk/client-sso-oidc': 3.572.0(@aws-sdk/client-sts@3.572.0) + '@aws-sdk/client-sts': 3.572.0 '@aws-sdk/core': 3.572.0 '@aws-sdk/credential-provider-node': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0)(@aws-sdk/client-sts@3.572.0) '@aws-sdk/middleware-host-header': 3.567.0 @@ -10245,11 +10341,11 @@ snapshots: transitivePeerDependencies: - aws-crt - '@aws-sdk/client-sso-oidc@3.572.0': + '@aws-sdk/client-sso-oidc@3.572.0(@aws-sdk/client-sts@3.572.0)': dependencies: '@aws-crypto/sha256-browser': 3.0.0 '@aws-crypto/sha256-js': 3.0.0 - '@aws-sdk/client-sts': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0) + '@aws-sdk/client-sts': 3.572.0 '@aws-sdk/core': 3.572.0 '@aws-sdk/credential-provider-node': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0)(@aws-sdk/client-sts@3.572.0) '@aws-sdk/middleware-host-header': 3.567.0 @@ -10288,6 +10384,7 @@ snapshots: '@smithy/util-utf8': 2.3.0 tslib: 2.6.2 transitivePeerDependencies: + - '@aws-sdk/client-sts' - aws-crt '@aws-sdk/client-sso-oidc@3.687.0(@aws-sdk/client-sts@3.687.0)': @@ -10508,11 +10605,11 @@ snapshots: transitivePeerDependencies: - aws-crt - '@aws-sdk/client-sts@3.572.0(@aws-sdk/client-sso-oidc@3.572.0)': + '@aws-sdk/client-sts@3.572.0': dependencies: '@aws-crypto/sha256-browser': 3.0.0 '@aws-crypto/sha256-js': 3.0.0 - '@aws-sdk/client-sso-oidc': 3.572.0 + '@aws-sdk/client-sso-oidc': 3.572.0(@aws-sdk/client-sts@3.572.0) '@aws-sdk/core': 3.572.0 '@aws-sdk/credential-provider-node': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0)(@aws-sdk/client-sts@3.572.0) '@aws-sdk/middleware-host-header': 3.567.0 @@ -10551,7 +10648,6 @@ snapshots: '@smithy/util-utf8': 2.3.0 tslib: 2.6.2 transitivePeerDependencies: - - '@aws-sdk/client-sso-oidc' - aws-crt '@aws-sdk/client-sts@3.687.0': @@ -10711,7 +10807,7 @@ snapshots: '@aws-sdk/credential-provider-ini@3.572.0(@aws-sdk/client-sso-oidc@3.572.0)(@aws-sdk/client-sts@3.572.0)': dependencies: - '@aws-sdk/client-sts': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0) + '@aws-sdk/client-sts': 3.572.0 '@aws-sdk/credential-provider-env': 3.568.0 '@aws-sdk/credential-provider-process': 3.572.0 '@aws-sdk/credential-provider-sso': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0) @@ -10878,7 +10974,7 @@ snapshots: '@aws-sdk/credential-provider-web-identity@3.568.0(@aws-sdk/client-sts@3.572.0)': dependencies: - '@aws-sdk/client-sts': 3.572.0(@aws-sdk/client-sso-oidc@3.572.0) + '@aws-sdk/client-sts': 3.572.0 '@aws-sdk/types': 3.567.0 '@smithy/property-provider': 2.2.0 '@smithy/types': 2.12.0 @@ -11138,7 +11234,7 @@ snapshots: '@aws-sdk/token-providers@3.572.0(@aws-sdk/client-sso-oidc@3.572.0)': dependencies: - '@aws-sdk/client-sso-oidc': 3.572.0 + '@aws-sdk/client-sso-oidc': 3.572.0(@aws-sdk/client-sts@3.572.0) '@aws-sdk/types': 3.567.0 '@smithy/property-provider': 2.2.0 '@smithy/shared-ini-file-loader': 2.4.0 @@ -11363,7 +11459,7 @@ snapshots: '@babel/traverse': 7.24.1 '@babel/types': 7.24.0 convert-source-map: 2.0.0 - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 gensync: 1.0.0-beta.2 json5: 2.2.3 semver: 6.3.1 @@ -11424,7 +11520,7 @@ snapshots: '@babel/core': 7.24.4 '@babel/helper-compilation-targets': 7.23.6 '@babel/helper-plugin-utils': 7.24.0 - debug: 4.3.7 + debug: 4.4.0(supports-color@5.5.0) lodash.debounce: 4.0.8 resolve: 1.22.8 transitivePeerDependencies: @@ -12091,7 +12187,7 @@ snapshots: '@babel/helper-split-export-declaration': 7.22.6 '@babel/parser': 7.24.4 '@babel/types': 7.24.0 - debug: 4.3.7 + debug: 4.4.0(supports-color@5.5.0) globals: 11.12.0 transitivePeerDependencies: - supports-color @@ -12383,7 +12479,7 @@ snapshots: '@eslint/eslintrc@2.1.4': dependencies: ajv: 6.12.6 - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 espree: 9.6.1 globals: 13.24.0 ignore: 5.3.1 @@ -12513,7 +12609,7 @@ snapshots: '@humanwhocodes/config-array@0.11.14': dependencies: '@humanwhocodes/object-schema': 2.0.3 - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 minimatch: 3.1.2 transitivePeerDependencies: - supports-color @@ -12915,7 +13011,7 @@ snapshots: '@opensearch-project/opensearch@2.11.0': dependencies: aws4: 1.12.0 - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 hpagent: 1.2.0 json11: 1.1.2 ms: 2.1.3 @@ -13766,7 +13862,7 @@ snapshots: '@superfaceai/parser': 1.2.0 abort-controller: 3.0.0 cross-fetch: 3.1.8(encoding@0.1.13) - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 isomorphic-form-data: 2.0.0 vm2: 3.9.19 transitivePeerDependencies: @@ -13777,7 +13873,7 @@ snapshots: dependencies: '@superfaceai/ast': 1.2.0 '@types/debug': 4.1.12 - debug: 4.3.7 + debug: 4.4.0(supports-color@5.5.0) typescript: 4.9.5 transitivePeerDependencies: - supports-color @@ -14131,7 +14227,7 @@ snapshots: '@typescript-eslint/scope-manager': 5.62.0 '@typescript-eslint/type-utils': 5.62.0(eslint@8.57.0)(typescript@5.6.3) '@typescript-eslint/utils': 5.62.0(eslint@8.57.0)(typescript@5.6.3) - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 eslint: 8.57.0 graphemer: 1.4.0 ignore: 5.3.1 @@ -14151,7 +14247,7 @@ snapshots: '@typescript-eslint/type-utils': 6.21.0(eslint@8.57.0)(typescript@5.6.3) '@typescript-eslint/utils': 6.21.0(eslint@8.57.0)(typescript@5.6.3) '@typescript-eslint/visitor-keys': 6.21.0 - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 eslint: 8.57.0 graphemer: 1.4.0 ignore: 5.3.1 @@ -14168,7 +14264,7 @@ snapshots: '@typescript-eslint/scope-manager': 5.62.0 '@typescript-eslint/types': 5.62.0 '@typescript-eslint/typescript-estree': 5.62.0(typescript@5.6.3) - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 eslint: 8.57.0 optionalDependencies: typescript: 5.6.3 @@ -14181,7 +14277,7 @@ snapshots: '@typescript-eslint/types': 6.21.0 '@typescript-eslint/typescript-estree': 6.21.0(typescript@5.6.3) '@typescript-eslint/visitor-keys': 6.21.0 - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 eslint: 8.57.0 optionalDependencies: typescript: 5.6.3 @@ -14202,7 +14298,7 @@ snapshots: dependencies: '@typescript-eslint/typescript-estree': 5.62.0(typescript@5.6.3) '@typescript-eslint/utils': 5.62.0(eslint@8.57.0)(typescript@5.6.3) - debug: 4.3.7 + debug: 4.4.0(supports-color@5.5.0) eslint: 8.57.0 tsutils: 3.21.0(typescript@5.6.3) optionalDependencies: @@ -14214,7 +14310,7 @@ snapshots: dependencies: '@typescript-eslint/typescript-estree': 6.21.0(typescript@5.6.3) '@typescript-eslint/utils': 6.21.0(eslint@8.57.0)(typescript@5.6.3) - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 eslint: 8.57.0 ts-api-utils: 1.3.0(typescript@5.6.3) optionalDependencies: @@ -14230,7 +14326,7 @@ snapshots: dependencies: '@typescript-eslint/types': 5.62.0 '@typescript-eslint/visitor-keys': 5.62.0 - debug: 4.3.7 + debug: 4.4.0(supports-color@5.5.0) globby: 11.1.0 is-glob: 4.0.3 semver: 7.6.0 @@ -14244,7 +14340,7 @@ snapshots: dependencies: '@typescript-eslint/types': 6.21.0 '@typescript-eslint/visitor-keys': 6.21.0 - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 globby: 11.1.0 is-glob: 4.0.3 minimatch: 9.0.3 @@ -14414,13 +14510,13 @@ snapshots: agent-base@6.0.2: dependencies: - debug: 4.4.0 + debug: 4.4.0(supports-color@5.5.0) transitivePeerDependencies: - supports-color agent-base@7.1.1: dependencies: - debug: 4.4.0 + debug: 4.4.0(supports-color@5.5.0) transitivePeerDependencies: - supports-color @@ -14862,10 +14958,6 @@ snapshots: dependencies: balanced-match: 1.0.2 - braces@3.0.2: - dependencies: - fill-range: 7.0.1 - braces@3.0.3: dependencies: fill-range: 7.1.1 @@ -14997,7 +15089,7 @@ snapshots: chokidar@3.6.0: dependencies: anymatch: 3.1.3 - braces: 3.0.2 + braces: 3.0.3 glob-parent: 5.1.2 is-binary-path: 2.1.0 is-glob: 4.0.3 @@ -15406,19 +15498,19 @@ snapshots: optionalDependencies: supports-color: 5.5.0 - debug@4.3.4(supports-color@5.5.0): + debug@4.3.4: dependencies: ms: 2.1.2 - optionalDependencies: - supports-color: 5.5.0 debug@4.3.7: dependencies: ms: 2.1.3 - debug@4.4.0: + debug@4.4.0(supports-color@5.5.0): dependencies: ms: 2.1.3 + optionalDependencies: + supports-color: 5.5.0 decamelize@1.2.0: {} @@ -15971,7 +16063,7 @@ snapshots: ajv: 6.12.6 chalk: 4.1.2 cross-spawn: 7.0.3 - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 doctrine: 3.0.0 escape-string-regexp: 4.0.0 eslint-scope: 7.2.2 @@ -16252,10 +16344,6 @@ snapshots: file-uri-to-path@1.0.0: {} - fill-range@7.0.1: - dependencies: - to-regex-range: 5.0.1 - fill-range@7.1.1: dependencies: to-regex-range: 5.0.1 @@ -16808,7 +16896,7 @@ snapshots: dependencies: '@tootallnate/once': 2.0.0 agent-base: 6.0.2 - debug: 4.4.0 + debug: 4.4.0(supports-color@5.5.0) transitivePeerDependencies: - supports-color @@ -16817,14 +16905,14 @@ snapshots: https-proxy-agent@5.0.1: dependencies: agent-base: 6.0.2 - debug: 4.3.7 + debug: 4.4.0(supports-color@5.5.0) transitivePeerDependencies: - supports-color https-proxy-agent@7.0.4: dependencies: agent-base: 7.1.1 - debug: 4.3.7 + debug: 4.4.0(supports-color@5.5.0) transitivePeerDependencies: - supports-color @@ -17218,7 +17306,7 @@ snapshots: dependencies: '@types/express': 4.17.21 '@types/jsonwebtoken': 9.0.6 - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 jose: 4.15.5 limiter: 1.1.5 lru-memoizer: 2.2.0 @@ -17278,7 +17366,7 @@ snapshots: dependencies: chalk: 5.4.1 commander: 13.1.0 - debug: 4.4.0 + debug: 4.4.0(supports-color@5.5.0) execa: 8.0.1 lilconfig: 3.1.3 listr2: 8.2.5 @@ -17744,7 +17832,7 @@ snapshots: nodemon@3.1.0: dependencies: chokidar: 3.6.0 - debug: 4.3.4(supports-color@5.5.0) + debug: 4.4.0(supports-color@5.5.0) ignore-by-default: 1.0.1 minimatch: 3.1.2 pstree.remy: 1.1.8 @@ -18449,7 +18537,7 @@ snapshots: command-line-usage: 6.1.3 config: 3.3.11 configstore: 5.0.1 - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 editor: 1.0.0 enquirer: 2.4.1 form-data: 4.0.0 @@ -18607,7 +18695,7 @@ snapshots: retry-request@4.2.2: dependencies: - debug: 4.3.7 + debug: 4.4.0(supports-color@5.5.0) extend: 3.0.2 transitivePeerDependencies: - supports-color @@ -18791,7 +18879,7 @@ snapshots: dependencies: '@types/debug': 4.1.12 '@types/validator': 13.11.9 - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 dottie: 2.0.6 inflection: 1.13.4 lodash: 4.17.21 @@ -19032,7 +19120,7 @@ snapshots: accepts: 1.3.8 base64id: 2.0.0 cors: 2.8.5 - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 engine.io: 6.5.4(bufferutil@4.0.8)(utf-8-validate@5.0.10) socket.io-adapter: 2.5.4(bufferutil@4.0.8)(utf-8-validate@5.0.10) socket.io-parser: 4.2.4 @@ -19188,7 +19276,7 @@ snapshots: dependencies: component-emitter: 1.3.1 cookiejar: 2.1.4 - debug: 4.3.4(supports-color@5.5.0) + debug: 4.3.4 fast-safe-stringify: 2.1.1 form-data: 4.0.0 formidable: 2.1.2 diff --git a/scripts/builders/automatic-projects-discovery-worker.env b/scripts/builders/automatic-projects-discovery-worker.env new file mode 100644 index 0000000000..8416386449 --- /dev/null +++ b/scripts/builders/automatic-projects-discovery-worker.env @@ -0,0 +1,4 @@ +DOCKERFILE="./services/docker/Dockerfile.automatic_projects_discovery_worker" +CONTEXT="../" +REPO="sjc.ocir.io/axbydjxa5zuh/automatic-projects-discovery-worker" +SERVICES="automatic-projects-discovery-worker" diff --git a/scripts/services/automatic-projects-discovery-worker.yaml b/scripts/services/automatic-projects-discovery-worker.yaml new file mode 100644 index 0000000000..5f3732b7c2 --- /dev/null +++ b/scripts/services/automatic-projects-discovery-worker.yaml @@ -0,0 +1,64 @@ +version: '3.1' + +x-env-args: &env-args + DOCKER_BUILDKIT: 1 + NODE_ENV: docker + SERVICE: automatic-projects-discovery-worker + CROWD_TEMPORAL_TASKQUEUE: automatic-projects-discovery + SHELL: /bin/sh + +services: + automatic-projects-discovery-worker: + build: + context: ../../ + dockerfile: ./scripts/services/docker/Dockerfile.automatic_projects_discovery_worker + command: 'pnpm run start' + working_dir: /usr/crowd/app/services/apps/automatic_projects_discovery_worker + env_file: + - ../../backend/.env.dist.local + - ../../backend/.env.dist.composed + - ../../backend/.env.override.local + - ../../backend/.env.override.composed + environment: + <<: *env-args + restart: always + networks: + - crowd-bridge + + automatic-projects-discovery-worker-dev: + build: + context: ../../ + dockerfile: ./scripts/services/docker/Dockerfile.automatic_projects_discovery_worker + command: 'pnpm run dev' + working_dir: /usr/crowd/app/services/apps/automatic_projects_discovery_worker + env_file: + - ../../backend/.env.dist.local + - ../../backend/.env.dist.composed + - ../../backend/.env.override.local + - ../../backend/.env.override.composed + environment: + <<: *env-args + hostname: automatic-projects-discovery-worker + networks: + - crowd-bridge + volumes: + - ../../services/libs/audit-logs/src:/usr/crowd/app/services/libs/audit-logs/src + - ../../services/libs/common/src:/usr/crowd/app/services/libs/common/src + - ../../services/libs/common_services/src:/usr/crowd/app/services/libs/common_services/src + - ../../services/libs/data-access-layer/src:/usr/crowd/app/services/libs/data-access-layer/src + - ../../services/libs/database/src:/usr/crowd/app/services/libs/database/src + - ../../services/libs/integrations/src:/usr/crowd/app/services/libs/integrations/src + - ../../services/libs/logging/src:/usr/crowd/app/services/libs/logging/src + - ../../services/libs/nango/src:/usr/crowd/app/services/libs/nango/src + - ../../services/libs/opensearch/src:/usr/crowd/app/services/libs/opensearch/src + - ../../services/libs/queue/src:/usr/crowd/app/services/libs/queue/src + - ../../services/libs/redis/src:/usr/crowd/app/services/libs/redis/src + - ../../services/libs/snowflake/src:/usr/crowd/app/services/libs/snowflake/src + - ../../services/libs/telemetry/src:/usr/crowd/app/services/libs/telemetry/src + - ../../services/libs/temporal/src:/usr/crowd/app/services/libs/temporal/src + - ../../services/libs/types/src:/usr/crowd/app/services/libs/types/src + - ../../services/apps/automatic_projects_discovery_worker/src:/usr/crowd/app/services/apps/automatic_projects_discovery_worker/src + +networks: + crowd-bridge: + external: true diff --git a/scripts/services/docker/Dockerfile.automatic_projects_discovery_worker b/scripts/services/docker/Dockerfile.automatic_projects_discovery_worker new file mode 100644 index 0000000000..860af6601e --- /dev/null +++ b/scripts/services/docker/Dockerfile.automatic_projects_discovery_worker @@ -0,0 +1,23 @@ +FROM node:20-alpine as builder + +RUN apk add --no-cache python3 make g++ + +WORKDIR /usr/crowd/app +RUN npm install -g corepack@latest && corepack enable pnpm && corepack prepare pnpm@9.15.0 --activate + +COPY ./pnpm-workspace.yaml ./pnpm-lock.yaml ./ +RUN pnpm fetch + +COPY ./services ./services +RUN pnpm i --frozen-lockfile + +FROM node:20-bookworm-slim as runner + +WORKDIR /usr/crowd/app +RUN npm install -g corepack@latest && corepack enable pnpm && corepack prepare pnpm@9.15.0 --activate && apt update && apt install -y ca-certificates --no-install-recommends && rm -rf /var/lib/apt/lists/* + +COPY --from=builder /usr/crowd/app/node_modules ./node_modules +COPY --from=builder /usr/crowd/app/services/base.tsconfig.json ./services/base.tsconfig.json +COPY --from=builder /usr/crowd/app/services/libs ./services/libs +COPY --from=builder /usr/crowd/app/services/archetypes/ ./services/archetypes +COPY --from=builder /usr/crowd/app/services/apps/automatic_projects_discovery_worker/ ./services/apps/automatic_projects_discovery_worker diff --git a/services/apps/automatic_projects_discovery_worker/package.json b/services/apps/automatic_projects_discovery_worker/package.json new file mode 100644 index 0000000000..1c79505f89 --- /dev/null +++ b/services/apps/automatic_projects_discovery_worker/package.json @@ -0,0 +1,34 @@ +{ + "name": "@crowd/automatic-projects-discovery-worker", + "scripts": { + "start": "CROWD_TEMPORAL_TASKQUEUE=automatic-projects-discovery SERVICE=automatic-projects-discovery-worker tsx src/main.ts", + "start:debug:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=automatic-projects-discovery SERVICE=automatic-projects-discovery-worker LOG_LEVEL=trace tsx --inspect=0.0.0.0:9232 src/main.ts", + "start:debug": "CROWD_TEMPORAL_TASKQUEUE=automatic-projects-discovery SERVICE=automatic-projects-discovery-worker LOG_LEVEL=trace tsx --inspect=0.0.0.0:9232 src/main.ts", + "dev:local": "nodemon --watch src --watch ../../libs --ext ts --exec pnpm run start:debug:local", + "dev": "nodemon --watch src --watch ../../libs --ext ts --exec pnpm run start:debug", + "lint": "npx eslint --ext .ts src --max-warnings=0", + "format": "npx prettier --write \"src/**/*.ts\"", + "format-check": "npx prettier --check .", + "tsc-check": "tsc --noEmit" + }, + "dependencies": { + "@crowd/archetype-standard": "workspace:*", + "@crowd/archetype-worker": "workspace:*", + "@crowd/common": "workspace:*", + "@crowd/common_services": "workspace:*", + "@crowd/data-access-layer": "workspace:*", + "@crowd/logging": "workspace:*", + "@crowd/redis": "workspace:*", + "@crowd/temporal": "workspace:*", + "@crowd/types": "workspace:*", + "@temporalio/activity": "~1.11.8", + "@temporalio/client": "~1.11.8", + "@temporalio/workflow": "~1.11.8", + "tsx": "^4.7.1", + "typescript": "^5.6.3" + }, + "devDependencies": { + "@types/node": "^20.8.2", + "nodemon": "^3.0.1" + } +} diff --git a/services/apps/automatic_projects_discovery_worker/src/activities.ts b/services/apps/automatic_projects_discovery_worker/src/activities.ts new file mode 100644 index 0000000000..3662234550 --- /dev/null +++ b/services/apps/automatic_projects_discovery_worker/src/activities.ts @@ -0,0 +1 @@ +export * from './activities/activities' diff --git a/services/apps/automatic_projects_discovery_worker/src/activities/activities.ts b/services/apps/automatic_projects_discovery_worker/src/activities/activities.ts new file mode 100644 index 0000000000..3aea7f8200 --- /dev/null +++ b/services/apps/automatic_projects_discovery_worker/src/activities/activities.ts @@ -0,0 +1,7 @@ +import { getServiceLogger } from '@crowd/logging' + +const log = getServiceLogger() + +export async function logDiscoveryRun(): Promise { + log.info('Automatic projects discovery workflow executed successfully.') +} diff --git a/services/apps/automatic_projects_discovery_worker/src/main.ts b/services/apps/automatic_projects_discovery_worker/src/main.ts new file mode 100644 index 0000000000..326c3a361a --- /dev/null +++ b/services/apps/automatic_projects_discovery_worker/src/main.ts @@ -0,0 +1,36 @@ +import { Config } from '@crowd/archetype-standard' +import { Options, ServiceWorker } from '@crowd/archetype-worker' + +import { scheduleProjectsDiscovery } from './schedules/scheduleProjectsDiscovery' + +const config: Config = { + envvars: [], + producer: { + enabled: false, + }, + temporal: { + enabled: true, + }, + redis: { + enabled: false, + }, +} + +const options: Options = { + postgres: { + enabled: false, + }, + opensearch: { + enabled: false, + }, +} + +export const svc = new ServiceWorker(config, options) + +setImmediate(async () => { + await svc.init() + + await scheduleProjectsDiscovery() + + await svc.start() +}) diff --git a/services/apps/automatic_projects_discovery_worker/src/schedules/scheduleProjectsDiscovery.ts b/services/apps/automatic_projects_discovery_worker/src/schedules/scheduleProjectsDiscovery.ts new file mode 100644 index 0000000000..847c2e4ce9 --- /dev/null +++ b/services/apps/automatic_projects_discovery_worker/src/schedules/scheduleProjectsDiscovery.ts @@ -0,0 +1,42 @@ +import { ScheduleAlreadyRunning, ScheduleOverlapPolicy } from '@temporalio/client' + +import { svc } from '../main' +import { discoverProjects } from '../workflows' + +const DEFAULT_CRON = '0 2 * * *' // Daily at 2:00 AM + +export const scheduleProjectsDiscovery = async () => { + const cronExpression = process.env.CROWD_AUTOMATIC_PROJECTS_DISCOVERY_CRON || DEFAULT_CRON + + svc.log.info(`Scheduling projects discovery with cron: ${cronExpression}`) + + try { + await svc.temporal.schedule.create({ + scheduleId: 'automaticProjectsDiscovery', + spec: { + cronExpressions: [cronExpression], + }, + policies: { + overlap: ScheduleOverlapPolicy.SKIP, + catchupWindow: '1 minute', + }, + action: { + type: 'startWorkflow', + workflowType: discoverProjects, + taskQueue: 'automatic-projects-discovery', + retry: { + initialInterval: '15 seconds', + backoffCoefficient: 2, + maximumAttempts: 3, + }, + }, + }) + } catch (err) { + if (err instanceof ScheduleAlreadyRunning) { + svc.log.info('Schedule already registered in Temporal.') + svc.log.info('Configuration may have changed since. Please make sure they are in sync.') + } else { + throw new Error(err) + } + } +} diff --git a/services/apps/automatic_projects_discovery_worker/src/workflows.ts b/services/apps/automatic_projects_discovery_worker/src/workflows.ts new file mode 100644 index 0000000000..07b00cee6f --- /dev/null +++ b/services/apps/automatic_projects_discovery_worker/src/workflows.ts @@ -0,0 +1,3 @@ +import { discoverProjects } from './workflows/discoverProjects' + +export { discoverProjects } diff --git a/services/apps/automatic_projects_discovery_worker/src/workflows/discoverProjects.ts b/services/apps/automatic_projects_discovery_worker/src/workflows/discoverProjects.ts new file mode 100644 index 0000000000..f43a9b5a12 --- /dev/null +++ b/services/apps/automatic_projects_discovery_worker/src/workflows/discoverProjects.ts @@ -0,0 +1,11 @@ +import { proxyActivities } from '@temporalio/workflow' + +import type * as activities from '../activities' + +const activity = proxyActivities({ + startToCloseTimeout: '1 minutes', +}) + +export async function discoverProjects(): Promise { + await activity.logDiscoveryRun() +} diff --git a/services/apps/automatic_projects_discovery_worker/tsconfig.json b/services/apps/automatic_projects_discovery_worker/tsconfig.json new file mode 100644 index 0000000000..bf7f183850 --- /dev/null +++ b/services/apps/automatic_projects_discovery_worker/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "../../base.tsconfig.json", + "include": ["src/**/*"] +}