diff --git a/backend/.env.docker b/backend/.env.docker index 01fe3a32..9f449895 100644 --- a/backend/.env.docker +++ b/backend/.env.docker @@ -29,4 +29,11 @@ LOKI_USERNAME="" LOKI_PASSWORD="" # Kafka / Redpanda configuration for node I/O, log, and event ingestion (Docker network) -LOG_KAFKA_BROKERS="redpanda:9092" \ No newline at end of file +LOG_KAFKA_BROKERS="redpanda:9092" + + +# GitHub template repository configuration +GITHUB_TEMPLATE_REPO=krishna9358/workflow-templates +GITHUB_TEMPLATE_BRANCH=main +# Optional: GitHub personal access token for higher rate limits (60/hr → 5000/hr) +GITHUB_TEMPLATE_TOKEN= \ No newline at end of file diff --git a/backend/drizzle/0020_create-templates.sql b/backend/drizzle/0020_create-templates.sql new file mode 100644 index 00000000..b2be64f5 --- /dev/null +++ b/backend/drizzle/0020_create-templates.sql @@ -0,0 +1,59 @@ +-- Create templates table +CREATE TABLE "templates" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "name" varchar(255) NOT NULL, + "description" text, + "category" varchar(100), + "tags" jsonb DEFAULT '[]'::jsonb NOT NULL, + "author" varchar(255), + "repository" varchar(255) NOT NULL, + "path" varchar(500) NOT NULL, + "branch" varchar(100) DEFAULT 'main' NOT NULL, + "version" varchar(50), + "commit_sha" varchar(100), + "manifest" jsonb NOT NULL, + "graph" jsonb, + "required_secrets" jsonb DEFAULT '[]'::jsonb NOT NULL, + "popularity" integer DEFAULT 0 NOT NULL, + "is_official" boolean DEFAULT false NOT NULL, + "is_verified" boolean DEFAULT false NOT NULL, + "is_active" boolean DEFAULT true NOT NULL, + "created_at" timestamp with time zone DEFAULT now() NOT NULL, + "updated_at" timestamp with time zone DEFAULT now() NOT NULL +); +--> statement-breakpoint +-- Create templates_submissions table +CREATE TABLE "templates_submissions" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL, + "template_name" varchar(255) NOT NULL, + "description" text, + "category" varchar(100), + "repository" varchar(255) NOT NULL, + "branch" varchar(100), + "path" varchar(500) NOT NULL, + "commit_sha" varchar(100), + "pr_number" integer, + "pr_url" varchar(500), + "status" varchar(50) DEFAULT 'pending' NOT NULL, + "submitted_by" varchar(191) NOT NULL, + "organization_id" varchar(191), + "manifest" jsonb, + "graph" jsonb, + "feedback" text, + "reviewed_by" varchar(191), + "reviewed_at" timestamp with time zone, + "created_at" timestamp with time zone DEFAULT now() NOT NULL, + "updated_at" timestamp with time zone DEFAULT now() NOT NULL +); +--> statement-breakpoint +-- Create indexes for templates +CREATE INDEX "templates_repository_path_idx" ON "templates" ("repository", "path"); +CREATE INDEX "templates_category_idx" ON "templates" ("category"); +CREATE INDEX "templates_is_active_idx" ON "templates" ("is_active"); +CREATE INDEX "templates_popularity_idx" ON "templates" ("popularity" DESC); +--> statement-breakpoint +-- Create indexes for templates_submissions +CREATE INDEX "templates_submissions_pr_number_idx" ON "templates_submissions" ("pr_number"); +CREATE INDEX "templates_submissions_submitted_by_idx" ON "templates_submissions" ("submitted_by"); +CREATE INDEX "templates_submissions_status_idx" ON "templates_submissions" ("status"); +CREATE INDEX "templates_submissions_organization_id_idx" ON "templates_submissions" ("organization_id"); diff --git a/backend/package.json b/backend/package.json index b6100e28..1956e287 100644 --- a/backend/package.json +++ b/backend/package.json @@ -45,6 +45,7 @@ "bcryptjs": "^3.0.3", "class-transformer": "^0.5.1", "class-validator": "^0.14.3", + "cookie-parser": "^1.4.7", "date-fns": "^4.1.0", "dotenv": "^17.2.3", "drizzle-orm": "^0.44.7", diff --git a/backend/src/app.module.ts b/backend/src/app.module.ts index 0c751013..f61e64f9 100644 --- a/backend/src/app.module.ts +++ b/backend/src/app.module.ts @@ -32,6 +32,7 @@ import { WebhooksModule } from './webhooks/webhooks.module'; import { HumanInputsModule } from './human-inputs/human-inputs.module'; import { McpServersModule } from './mcp-servers/mcp-servers.module'; import { McpGroupsModule } from './mcp-groups/mcp-groups.module'; +import { TemplatesModule } from './templates/templates.module'; const coreModules = [ AgentsModule, @@ -51,6 +52,7 @@ const coreModules = [ McpGroupsModule, McpModule, StudioMcpModule, + TemplatesModule, ]; const testingModules = process.env.NODE_ENV === 'production' ? [] : [TestingSupportModule]; diff --git a/backend/src/database/schema/index.ts b/backend/src/database/schema/index.ts index 6d985721..d8b1335e 100644 --- a/backend/src/database/schema/index.ts +++ b/backend/src/database/schema/index.ts @@ -20,3 +20,4 @@ export * from './mcp-servers'; export * from './node-io'; export * from './organization-settings'; +export * from './templates'; diff --git a/backend/src/database/schema/templates.ts b/backend/src/database/schema/templates.ts new file mode 100644 index 00000000..e3027cef --- /dev/null +++ b/backend/src/database/schema/templates.ts @@ -0,0 +1,99 @@ +import { + pgTable, + uuid, + varchar, + text, + timestamp, + jsonb, + boolean, + integer, +} from 'drizzle-orm/pg-core'; +import { z } from 'zod'; + +/** + * Templates table - stores workflow template metadata + * Templates are synced from GitHub repository + */ +export const templatesTable = pgTable('templates', { + id: uuid('id').primaryKey().defaultRandom(), + name: varchar('name', { length: 255 }).notNull(), + description: text('description'), + category: varchar('category', { length: 100 }), + tags: jsonb('tags').$type().default([]), + author: varchar('author', { length: 255 }), + // GitHub repository info + repository: varchar('repository', { length: 255 }).notNull(), // e.g., "org/templates" + path: varchar('path', { length: 500 }).notNull(), // Path to template in repo + branch: varchar('branch', { length: 100 }).default('main'), + version: varchar('version', { length: 50 }), // Optional version tag + commitSha: varchar('commit_sha', { length: 100 }), + // Template content + manifest: jsonb('manifest').$type().notNull(), + graph: jsonb('graph').$type>(), // Sanitized workflow graph + requiredSecrets: jsonb('required_secrets').$type().default([]), + // Stats and flags + popularity: integer('popularity').notNull().default(0), + isOfficial: boolean('is_official').notNull().default(false), + isVerified: boolean('is_verified').notNull().default(false), + isActive: boolean('is_active').notNull().default(true), + // Timestamps + createdAt: timestamp('created_at', { withTimezone: true }).defaultNow().notNull(), + updatedAt: timestamp('updated_at', { withTimezone: true }).defaultNow().notNull(), +}); + +/** + * Template submissions table - tracks PR-based template submissions + */ +export const templatesSubmissionsTable = pgTable('templates_submissions', { + id: uuid('id').primaryKey().defaultRandom(), + templateName: varchar('template_name', { length: 255 }).notNull(), + description: text('description'), + category: varchar('category', { length: 100 }), + repository: varchar('repository', { length: 255 }).notNull(), + branch: varchar('branch', { length: 100 }), + path: varchar('path', { length: 500 }).notNull(), + commitSha: varchar('commit_sha', { length: 100 }), + pullRequestNumber: integer('pr_number'), + pullRequestUrl: varchar('pr_url', { length: 500 }), + status: varchar('status', { length: 50 }).notNull().default('pending'), // pending, approved, rejected, merged + submittedBy: varchar('submitted_by', { length: 191 }).notNull(), + organizationId: varchar('organization_id', { length: 191 }), + manifest: jsonb('manifest').$type(), + graph: jsonb('graph').$type>(), + feedback: text('feedback'), + reviewedBy: varchar('reviewed_by', { length: 191 }), + reviewedAt: timestamp('reviewed_at', { withTimezone: true }), + createdAt: timestamp('created_at', { withTimezone: true }).defaultNow().notNull(), + updatedAt: timestamp('updated_at', { withTimezone: true }).defaultNow().notNull(), +}); + +// Zod schemas for validation +export const RequiredSecretSchema = z.object({ + name: z.string(), + type: z.string(), + description: z.string().optional(), + placeholder: z.string().optional(), +}); + +export const TemplateManifestSchema = z.object({ + name: z.string(), + description: z.string().optional(), + version: z.string().optional(), + author: z.string().optional(), + category: z.string().optional(), + tags: z.array(z.string()).optional(), + requiredSecrets: z.array(RequiredSecretSchema).optional(), + entryPoint: z.string().optional(), + screenshots: z.array(z.string()).optional(), + documentation: z.string().optional(), +}); + +// Type exports +export type TemplateManifest = z.infer; +export type RequiredSecret = z.infer; + +export type Template = typeof templatesTable.$inferSelect; +export type NewTemplate = typeof templatesTable.$inferInsert; + +export type TemplateSubmission = typeof templatesSubmissionsTable.$inferSelect; +export type NewTemplateSubmission = typeof templatesSubmissionsTable.$inferInsert; diff --git a/backend/src/templates/ARCHITECTURE.md b/backend/src/templates/ARCHITECTURE.md new file mode 100644 index 00000000..e559f35c --- /dev/null +++ b/backend/src/templates/ARCHITECTURE.md @@ -0,0 +1,790 @@ +# Template Library Architecture + +## Overview + +The Template Library provides a centralized repository of workflow templates that users can browse and use as starting points for their own workflows. Templates are stored in a **public** GitHub repository and synced to the application database. + +### Key Features + +- Browse workflow templates by category, tags, or search +- Templates stored in a public GitHub repository (no authentication needed) +- Automatic sync on backend startup +- Manual "Sync from GitHub" button for on-demand refresh +- Workflow sanitization to remove secrets before publishing +- Support for both community and official templates + +--- + +## Publishing Flow + +Templates are published via GitHub's web flow, removing the need for backend API authentication tokens. + +### Step-by-Step Process + +``` +User Frontend GitHub Backend + | | | | + | Publish Workflow | | | + |--------------------->| | | + | | | | + | | Open GitHub | | + | | New File Page | | + | |---------------->| | + | | | | + | | | User Creates PR | + | | |<-------------------| + | | | | + | | | PR Merged | + | |<----------------| | + | | | | + | | Admin clicks | | + | | "Sync from | | + | | GitHub" | | + | |---------------->| Sync Templates | + | | |<-------------------| + | | | | + | | Templates | | + | | Available | | + |<---------------------|<-----------------| | +``` + +### GitHub Web Flow + +1. User clicks "Publish as Template" in the UI +2. Frontend opens a pre-filled GitHub URL for creating a new file +3. User commits the file and creates a pull request +4. After review, PR is merged to the main branch +5. Admin clicks "Sync from GitHub" to pull in the new template (or waits for next backend restart) + +--- + +## Sync Flow + +Templates are fetched from a **public** GitHub repository using the GitHub API and stored in the database. No authentication token is needed. + +### Sync Triggers + +| Trigger | When | How | +| ---------------- | ------------------------------- | --------------------------------------- | +| **Startup sync** | Backend boots | `onModuleInit()` in `GitHubSyncService` | +| **Manual sync** | Admin clicks "Sync from GitHub" | `POST /templates/sync` | + +A **concurrent sync guard** prevents overlapping syncs from running simultaneously. + +### Sync Process + +```typescript +// Location: github-sync.service.ts + +async syncTemplates() { + // 0. Guard against concurrent syncs + if (this.isSyncing) return; + + // 1. Fetch repository config from environment + const { owner, repo, branch } = getRepoConfig(); + + // 2. List all files in /templates directory + const files = await fetchDirectory('templates'); + + // 3. For each JSON file: + for (const file of files) { + // a. Fetch file content + const content = await fetchFileContent(file.path); + + // b. Parse and validate template structure + const template = parseTemplateJson(content); + + // c. Upsert to database (by repository + path) + await templatesRepository.upsert({ + name: template._metadata.name, + manifest: template.manifest, + graph: template.graph, + // ... other fields + }); + } + + return { synced, failed, total }; +} +``` + +### Template JSON Structure + +Each template file in GitHub must follow this structure: + +```json +{ + "_metadata": { + "name": "My Workflow Template", + "description": "Does something useful", + "category": "automation", + "tags": ["api", "integration"], + "author": "username", + "version": "1.0.0" + }, + "manifest": { + "name": "My Workflow Template", + "version": "1.0.0", + "entryPoint": "trigger_1", + "nodeCount": 5, + "edgeCount": 4 + }, + "graph": { + "nodes": [...], + "edges": [...] + }, + "requiredSecrets": [ + { + "name": "api_key", + "type": "string", + "description": "API key for service" + } + ] +} +``` + +--- + +## Auto-Sync: Startup + +The backend automatically syncs templates on startup: + +1. **On startup** -- immediate sync when the backend boots + +### Implementation + +```typescript +// Location: github-sync.service.ts + +@Injectable() +export class GitHubSyncService implements OnModuleInit { + private isSyncing = false; + + async onModuleInit(): Promise { + // 1. Log repo config + const { owner, repo, branch } = this.getRepoConfig(); + this.logger.log(`Template repo: ${owner}/${repo} (branch: ${branch})`); + + // 2. Initial sync on startup + const result = await this.syncTemplates(); + } +} +``` + +### Behavior + +- Triggers when `TemplatesModule` is initialized +- Runs automatically on backend startup +- Logs results but doesn't fail the application if sync errors occur +- Concurrent sync guard prevents overlapping runs + +### How PR Merge -> Database Works + +This is the key flow for when a user creates a PR and an admin merges it: + +``` +1. User publishes template -> Frontend opens GitHub web flow +2. User creates PR on GitHub with template JSON file +3. Admin reviews and merges PR into main branch +4. Admin clicks "Sync from GitHub" in the dashboard + a. Fetches /templates directory listing from GitHub API + b. Finds the new .json file from the merged PR + c. Downloads and parses the template content + d. Upserts into the database (matched by repository + path) +5. Template appears in the dashboard on next page load +``` + +--- + +## Database Schema + +### templates Table + +Stores synced workflow templates. + +| Column | Type | Description | +| ------------------ | ------------ | -------------------------------------------- | +| `id` | uuid | Primary key | +| `name` | varchar(255) | Template name | +| `description` | text | Template description | +| `category` | varchar(100) | Category (e.g., "automation", "integration") | +| `tags` | jsonb | Array of tags | +| `author` | varchar(255) | Author username | +| `repository` | varchar(255) | GitHub repo (e.g., "org/templates") | +| `path` | varchar(500) | Path to template in repo | +| `branch` | varchar(100) | Git branch (default: "main") | +| `version` | varchar(50) | Template version | +| `commit_sha` | varchar(100) | Git commit SHA | +| `manifest` | jsonb | Template metadata | +| `graph` | jsonb | Sanitized workflow graph | +| `required_secrets` | jsonb | Required secrets for template | +| `popularity` | integer | Usage counter | +| `is_official` | boolean | Official template flag | +| `is_verified` | boolean | Verified template flag | +| `is_active` | boolean | Active status | +| `created_at` | timestamp | Creation timestamp | +| `updated_at` | timestamp | Last update timestamp | + +### templates_submissions Table + +Tracks PR-based template submissions (for future workflow). + +| Column | Type | Description | +| ----------------- | ------------ | ---------------------------------------- | +| `id` | uuid | Primary key | +| `template_name` | varchar(255) | Template name | +| `description` | text | Description | +| `category` | varchar(100) | Category | +| `repository` | varchar(255) | GitHub repo | +| `branch` | varchar(100) | Git branch | +| `path` | varchar(500) | Path to template | +| `commit_sha` | varchar(100) | Git commit SHA | +| `pr_number` | integer | Pull request number | +| `pr_url` | varchar(500) | Pull request URL | +| `status` | varchar(50) | Status: pending/approved/rejected/merged | +| `submitted_by` | varchar(191) | Submitter user ID | +| `organization_id` | varchar(191) | Organization ID | +| `manifest` | jsonb | Template metadata | +| `graph` | jsonb | Workflow graph | +| `feedback` | text | Review feedback | +| `reviewed_by` | varchar(191) | Reviewer user ID | +| `reviewed_at` | timestamp | Review timestamp | +| `created_at` | timestamp | Creation timestamp | +| `updated_at` | timestamp | Last update timestamp | + +--- + +## API Endpoints + +### Public Endpoints + +#### GET /templates + +List all templates with optional filters. + +**Query Parameters:** + +- `category` (optional) - Filter by category +- `search` (optional) - Search in name and description +- `tags` (optional) - Comma-separated list of tags + +**Response:** Array of template objects + +```bash +curl http://localhost:3211/templates?category=automation +``` + +#### GET /templates/categories + +List all categories with template counts. + +**Response:** Array of `{ category, count }` objects + +```bash +curl http://localhost:3211/templates/categories +``` + +#### GET /templates/tags + +List all available tags. + +**Response:** Array of tag strings + +```bash +curl http://localhost:3211/templates/tags +``` + +#### GET /templates/repo-info + +Get GitHub repository information. + +**Response:** `{ owner, repo, branch, url }` + +```bash +curl http://localhost:3211/templates/repo-info +``` + +#### GET /templates/:id + +Get template details by ID. + +**Response:** Template object with full details + +```bash +curl http://localhost:3211/templates/{id} +``` + +### Authenticated Endpoints + +#### GET /templates/my + +Get current user's submitted templates. + +**Response:** Array of submission objects + +```bash +curl -H "Authorization: Bearer {token}" http://localhost:3211/templates/my +``` + +#### GET /templates/submissions + +Get template submissions for current user. + +**Response:** Array of submission objects + +```bash +curl -H "Authorization: Bearer {token}" http://localhost:3211/templates/submissions +``` + +#### POST /templates/publish + +Validate a workflow for template submission. + +**Request Body:** + +```json +{ + "workflowId": "uuid", + "name": "Template Name", + "description": "Description", + "category": "automation", + "tags": ["tag1", "tag2"], + "author": "username" +} +``` + +**Note:** This endpoint currently validates but does not create PRs. Use GitHub web flow instead. + +#### POST /templates/:id/use + +Use a template to create a new workflow. + +**Request Body:** + +```json +{ + "workflowName": "My Workflow", + "secretMappings": { + "api_key": "secret_reference" + } +} +``` + +**Note:** Currently disabled. + +### Admin Endpoints + +#### POST /templates/sync + +Manually trigger template sync from GitHub. + +**Response:** Sync result with synced/failed counts + +```bash +curl -X POST -H "Authorization: Bearer {admin_token}" \ + http://localhost:3211/templates/sync +``` + +**Response Example:** + +```json +{ + "synced": ["template1", "template2"], + "failed": [ + { + "path": "templates/invalid.json", + "error": "Invalid template format" + } + ], + "total": 2 +} +``` + +--- + +## Environment Variables + +### Required Variables + +| Variable | Description | Example | +| ---------------------- | --------------------------------------------- | -------------------------------- | +| `GITHUB_TEMPLATE_REPO` | Public GitHub repository containing templates | `krishna9358/workflow-templates` | + +### Optional Variables + +| Variable | Description | Default | +| ------------------------ | ----------------------- | ------- | +| `GITHUB_TEMPLATE_BRANCH` | Git branch to sync from | `main` | + +### Example Configuration + +```bash +# .env or .env.docker + +# GitHub template repository (must be public) +GITHUB_TEMPLATE_REPO=krishna9358/workflow-templates +GITHUB_TEMPLATE_BRANCH=main +``` + +--- + +## Architecture Diagram + +``` ++-------------------------------------------------------------------------+ +| Template Library | ++-------------------------------------------------------------------------+ + + +--------------+ + | Frontend | + | | + | - Browse | + | - Search | + | - Publish | + +------+-------+ + | HTTP + v + +--------------------------------------------------------------------+ + | Backend API | + | | + | +-----------------+ +-----------------+ | + | | Templates | | GitHub Sync | | + | | Controller |<---| Service | | + | +--------+--------+ +--------+--------+ | + | | | | + | v v | + | +-----------------+ +-----------------+ | + | | Template | | Workflow | | + | | Service | | Sanitization | | + | +--------+--------+ +-----------------+ | + | | | + | v | + | +-----------------+ | + | | Templates | | + | | Repository | | + | +--------+--------+ | + +-----------+--------------------------------------------------------| + | Drizzle ORM + v + +---------------------------------------------------------------------+ + | PostgreSQL | + | | + | +---------------+ +-----------------------------+ | + | | templates | | templates_submissions | | + | | table | | table | | + | +---------------+ +-----------------------------+ | + +---------------------------------------------------------------------+ + + +---------------------------------------------------------------------+ + | GitHub Repository (public) | + | | + | /templates/ | + | +-- automation-template.json | + | +-- integration-template.json | + | +-- monitoring-template.json | + +---------------------------------------------------------------------+ + ^ + | GitHub API (unauthenticated, public repo) + | + +---------------+-----------------------------------------------------+ + | Sync Triggers | + | | + | 1. Startup sync --> onModuleInit() --> syncTemplates() | + | 2. Manual --> POST /templates/sync | + +---------------------------------------------------------------------+ +``` + +### Data Flow + +#### Template Sync Flow + +``` +Trigger (startup / manual) + | + +-> Check concurrent sync guard (skip if already syncing) + | + +-> Fetch templates/ directory from GitHub API + | + +-> For each JSON file: + | | + | +-> Fetch file content + | | + | +-> Parse and validate template structure + | | + | +-> Upsert to database (by repository + path) + | + +-> Return sync results { synced, failed, total } +``` + +#### Template Browse Flow + +``` +Frontend Request + | + +-> GET /templates?category=automation + | + +-> TemplatesController.listTemplates() + | + +-> TemplateService.listTemplates() + | + +-> TemplatesRepository.findAll() + | + +-> Database Query (Drizzle ORM) + | + +-> Return templates array +``` + +#### Publish Flow (GitHub Web Flow) + +``` +User Action + | + +-> Click "Publish as Template" in Workflow Builder + | + +-> Fill in metadata: name, description, category, tags, author + | + +-> Frontend sanitizes workflow graph (removes secrets) + | + +-> Frontend generates template JSON with metadata + sanitized graph + | + +-> Frontend opens GitHub URL in new tab: + | https://github.com/{repo}/new/{branch} + | ?filename=templates/{name}.json + | &value={template_content} + | &quick_pull=1 + | + +-> User reviews content on GitHub, clicks "Propose new file" + | + +-> User creates Pull Request for admin review + | + +-> Admin reviews and merges PR + | + +-> Admin clicks "Sync from GitHub" in the dashboard + | (or template is picked up on next backend restart) + | + +-> Template appears in dashboard +``` + +--- + +## Services + +### GitHubSyncService + +**Purpose:** Fetch templates from a public GitHub repo and sync to database + +**Key Methods:** + +- `syncTemplates()` - Main sync operation (with concurrent guard) +- `getRepoConfig()` - Read env vars (`GITHUB_TEMPLATE_REPO`, `GITHUB_TEMPLATE_BRANCH`) +- `fetchDirectory(path)` - List directory contents from GitHub API +- `fetchFileContent(path)` - Get file content from GitHub API +- `parseTemplateJson(content, path)` - Validate template structure +- `getRepositoryInfo()` - Get repo configuration + +**Lifecycle:** + +- `OnModuleInit` -- initial sync on startup + +### TemplateService + +**Purpose:** Business logic for template operations + +**Key Methods:** + +- `listTemplates(filters)` - Get filtered template list +- `getTemplateById(id)` - Get single template +- `getMyTemplates(userId)` - Get user's templates +- `getCategories()` - Get all categories +- `getTags()` - Get all tags + +### WorkflowSanitizationService + +**Purpose:** Remove secrets from workflows before publishing + +**Key Methods:** + +- `sanitizeWorkflow(graph)` - Remove secret references +- `validateSanitizedGraph(graph)` - Validate sanitized workflow +- `generateManifest(params)` - Generate template manifest + +**Secret Detection Patterns:** + +- `connectionType.kind === 'secret'` or `'primitive_secret'` +- Fields named `secretId`, `secret_name`, or `apiKey` +- String patterns: `{{secret:*}}` or `{{secrets.*}}` + +### TemplatesRepository + +**Purpose:** Database operations for templates + +**Key Methods:** + +- `findAll(filters)` - Find active templates +- `findById(id)` - Find by primary key +- `findByRepoAndPath(repository, path)` - Find by location +- `upsert(template)` - Create or update template +- `incrementPopularity(id)` - Track usage +- `getCategories()` - Get categories with counts +- `getTags()` - Get all tags + +--- + +## Security Considerations + +### Secret Sanitization + +All workflows are sanitized before being stored as templates: + +1. **Secret Detection:** Service scans graph for secret references +2. **Removal:** Secrets are replaced with placeholders +3. **Documentation:** Removed secrets are documented in `requiredSecrets` +4. **Validation:** Sanitized graph is validated for correctness + +### Public API Access + +Template browsing endpoints are public (no authentication required): + +- `GET /templates` +- `GET /templates/:id` +- `GET /templates/categories` +- `GET /templates/tags` +- `GET /templates/repo-info` + +This allows templates to be browsed without login, improving discoverability. + +--- + +## Error Handling + +### Sync Failures + +Template sync is resilient to failures: + +- Individual file failures don't stop the sync +- Failed templates are logged and returned in the `failed` array +- On startup, sync failures don't prevent application from starting + +### Invalid Templates + +Templates that fail validation are: + +- Logged with failure reason +- Included in sync results' `failed` array +- Not stored in the database + +Common validation failures: + +- Missing `_metadata.name` +- Missing `manifest` object +- Missing `graph` object +- Invalid JSON structure + +--- + +## Future Enhancements + +### Planned Features + +1. **Official Templates:** Mark templates from verified authors +2. **Template Ratings:** Allow users to rate and review templates +3. **Usage Analytics:** Track template usage patterns +4. **Versioning:** Support multiple versions per template +5. **Template Dependencies:** Allow templates to reference other templates +6. **Automated Testing:** Test templates before syncing + +### Scalability Considerations + +- **Caching:** Add Redis caching for frequently accessed templates +- **Pagination:** Add cursor-based pagination for template lists +- **Search:** Implement full-text search with PostgreSQL or OpenSearch +- **CDN:** Cache template files in CDN for faster access + +--- + +## Troubleshooting + +### Templates Not Showing + +1. **Check `GITHUB_TEMPLATE_REPO`** in `.env` -- must match your actual public repo (e.g., `krishna9358/workflow-templates`) +2. **Ensure the repo is public** -- the sync uses unauthenticated GitHub API calls +3. Check branch name in `GITHUB_TEMPLATE_BRANCH` +4. Review backend logs for sync errors (look for `GitHubSyncService` messages) +5. Check the startup log: `Template repo: owner/repo (branch: main)` +6. Manually trigger sync: `POST /templates/sync` (requires admin auth) +7. Click "Sync from GitHub" button in the UI + +### Common Errors + +| Error | Cause | Fix | +| -------------------------------- | ---------------------------------- | ----------------------------------------------------------- | +| `Directory not found: templates` | Wrong repo name or repo is private | Verify `GITHUB_TEMPLATE_REPO` and ensure the repo is public | +| `GitHub API error: 404` | Repo doesn't exist or is private | Check repo name and ensure it is public | +| `GitHub API error: 403` | Rate limit exceeded | Wait 1 hour (unauthenticated limit is 60 req/hr) | +| `Sync already in progress` | Concurrent sync guard triggered | Normal behavior -- wait for current sync to finish | + +### Sync Fails on Startup + +- Check network connectivity to GitHub API +- Ensure the repository is public +- Verify API rate limits (60 requests/hr for unauthenticated access) +- Review error logs in backend output + +--- + +## Related Files + +``` +backend/src/templates/ ++-- templates.module.ts # Module definition ++-- templates.controller.ts # HTTP endpoints ++-- templates.service.ts # Business logic ++-- github-sync.service.ts # GitHub API integration ++-- workflow-sanitization.service.ts # Secret removal ++-- templates.repository.ts # Database operations ++-- ARCHITECTURE.md # This file + +backend/src/database/schema/ ++-- templates.ts # Database schema definitions + +frontend/src/features/templates/ ++-- components/ # Template UI components ++-- hooks/ # Custom React hooks ++-- TemplateLibrary.tsx # Main page +``` + +--- + +## Version History + +- **v1.2.0** - Simplified to startup sync + manual sync + - Removed webhook controller and webhook-based sync + - Removed `GITHUB_TEMPLATE_TOKEN` and private repo support (repo must be public) + - Removed periodic sync (`setInterval`, `GITHUB_SYNC_INTERVAL_MS`, `OnModuleDestroy`) + - Removed `getHeaders()` method and `Authorization` header logic + - Architecture is now: startup sync + manual "Sync from GitHub" button + - Only 2 env vars: `GITHUB_TEMPLATE_REPO` and `GITHUB_TEMPLATE_BRANCH` + +- **v1.1.0** - Private repo support + periodic sync + - Added `GITHUB_TEMPLATE_TOKEN` auth for private repositories + - Added periodic sync (every 5 min) via `setInterval` + - Added concurrent sync guard to prevent overlapping syncs + - Fixed `findAll()` query composition (filters + sorting now work correctly) + - Fixed `.env` repo configuration (was pointing to wrong repo) + - Improved startup logging with repo config and auth status + - Configurable sync interval via `GITHUB_SYNC_INTERVAL_MS` + +- **v1.0.0** - Initial implementation with GitHub web flow + - Removed Octokit dependency + - Implemented startup sync via OnModuleInit + - Secret sanitization for workflow graphs + +--- + +## Support + +For issues or questions about the Template Library: + +1. Check this documentation first +2. Review backend logs for error messages +3. Verify environment variables are set correctly +4. Test GitHub repository access manually diff --git a/backend/src/templates/github-sync.service.ts b/backend/src/templates/github-sync.service.ts new file mode 100644 index 00000000..b712d033 --- /dev/null +++ b/backend/src/templates/github-sync.service.ts @@ -0,0 +1,411 @@ +import { Injectable, Logger, OnModuleInit } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import { TemplatesRepository } from './templates.repository'; +import { TemplateManifest } from '../database/schema/templates'; + +interface GitHubFile { + name: string; + path: string; + type: string; + url: string; +} + +interface GitHubContentResponse { + name: string; + path: string; + sha: string; + size: number; + url: string; + html_url: string; + git_url: string; + download_url: string | null; + type: string; + content?: string; + encoding?: string; +} + +interface TemplateJson { + _metadata: { + name: string; + description?: string; + category: string; + tags: string[]; + author: string; + version: string; + }; + manifest?: Record; + graph: Record; + requiredSecrets: { name: string; type: string; description?: string }[]; +} + +/** + * Cached response with ETag for conditional requests. + * When GitHub returns 304 Not Modified, we reuse the cached data + * without consuming a rate limit point. + */ +interface CachedResponse { + etag: string; + data: T; +} + +/** + * GitHub Sync Service + * Fetches templates from a public GitHub repository and stores them in the database. + * Syncs automatically on startup and on-demand via the admin "Sync from GitHub" button. + * + * Uses ETag-based conditional requests to minimize API usage: + * - First request: GitHub returns data + ETag header + * - Subsequent requests: We send If-None-Match with the stored ETag + * - If unchanged: GitHub returns 304 (no body, no rate limit hit) + * - If changed: GitHub returns 200 with new data + new ETag + */ +@Injectable() +export class GitHubSyncService implements OnModuleInit { + private readonly logger = new Logger(GitHubSyncService.name); + private isSyncing = false; + + /** In-memory ETag cache keyed by request URL */ + private readonly etagCache = new Map>(); + + constructor( + private readonly configService: ConfigService, + private readonly templatesRepository: TemplatesRepository, + ) {} + + /** + * Sync templates once on startup. + */ + async onModuleInit(): Promise { + const { owner, repo, branch } = this.getRepoConfig(); + const hasToken = !!this.getToken(); + this.logger.log(`Template repo: ${owner}/${repo} (branch: ${branch})`); + this.logger.log( + `GitHub API auth: ${hasToken ? 'token configured (5000 req/hr)' : 'unauthenticated (60 req/hr)'}`, + ); + this.logger.log('Starting automatic template sync...'); + try { + const result = await this.syncTemplates(); + this.logger.log( + `Startup sync complete: ${result.synced.length} synced, ${result.failed.length} failed`, + ); + } catch (err) { + this.logger.error('Startup sync failed', err); + // Don't throw - allow the application to start even if sync fails + } + } + + /** + * Get the GitHub token for authenticated API requests (optional). + * With a token: 5,000 requests/hour. Without: 60 requests/hour. + */ + private getToken(): string | undefined { + return this.configService.get('GITHUB_TEMPLATE_TOKEN'); + } + + /** + * Build common headers for GitHub API requests. + */ + private getHeaders(): Record { + const headers: Record = { + Accept: 'application/vnd.github.v3+json', + }; + const token = this.getToken(); + if (token) { + headers['Authorization'] = `Bearer ${token}`; + } + return headers; + } + + /** + * Get the GitHub repository configuration from environment variables. + */ + private getRepoConfig(): { owner: string; repo: string; branch: string } { + const repo = this.configService.get( + 'GITHUB_TEMPLATE_REPO', + 'krishna9358/workflow-templates', + ); + const branch = this.configService.get('GITHUB_TEMPLATE_BRANCH', 'main'); + const [owner, repoName] = repo.split('/'); + + if (!owner || !repoName) { + throw new Error('Invalid GITHUB_TEMPLATE_REPO format. Expected: owner/repo'); + } + + return { owner, repo: repoName, branch }; + } + + /** + * Fetch directory contents from GitHub's public API. + * Uses ETag conditional requests to avoid redundant data transfer. + */ + private async fetchDirectory(path: string): Promise<{ files: GitHubFile[]; cached: boolean }> { + const { owner, repo, branch } = this.getRepoConfig(); + const url = `https://api.github.com/repos/${owner}/${repo}/contents/${path}?ref=${branch}`; + + const headers = this.getHeaders(); + const cached = this.etagCache.get(url) as CachedResponse | undefined; + if (cached?.etag) { + headers['If-None-Match'] = cached.etag; + } + + const response = await fetch(url, { headers }); + + // 304 Not Modified — use cached data, zero rate limit cost + if (response.status === 304 && cached) { + this.logger.debug(`Directory ${path}: not modified (ETag hit)`); + return { files: cached.data, cached: true }; + } + + if (!response.ok) { + if (response.status === 404) { + this.logger.warn(`Directory not found: ${path}`); + return { files: [], cached: false }; + } + if (response.status === 403) { + const resetHeader = response.headers.get('x-ratelimit-reset'); + const resetIn = resetHeader + ? Math.ceil((Number(resetHeader) * 1000 - Date.now()) / 60000) + : '?'; + this.logger.warn( + `GitHub API rate limit exceeded. Resets in ~${resetIn} min. Skipping sync.`, + ); + return { files: [], cached: false }; + } + throw new Error(`GitHub API error: ${response.status} ${response.statusText}`); + } + + const data = (await response.json()) as GitHubFile[]; + const files = Array.isArray(data) ? data : []; + + // Cache the response with its ETag for future conditional requests + const etag = response.headers.get('etag'); + if (etag) { + this.etagCache.set(url, { etag, data: files }); + } + + return { files, cached: false }; + } + + /** + * Fetch a single file's content from GitHub. + * Uses ETag conditional requests to skip re-downloading unchanged files. + */ + private async fetchFileContent( + path: string, + ): Promise<{ content: string | null; cached: boolean }> { + const { owner, repo, branch } = this.getRepoConfig(); + const url = `https://api.github.com/repos/${owner}/${repo}/contents/${path}?ref=${branch}`; + + const headers = this.getHeaders(); + const cached = this.etagCache.get(url) as CachedResponse | undefined; + if (cached?.etag) { + headers['If-None-Match'] = cached.etag; + } + + const response = await fetch(url, { headers }); + + // 304 Not Modified — use cached content, zero rate limit cost + if (response.status === 304 && cached) { + this.logger.debug(`File ${path}: not modified (ETag hit)`); + return { content: cached.data, cached: true }; + } + + if (!response.ok) { + if (response.status === 404) { + this.logger.warn(`File not found: ${path}`); + return { content: null, cached: false }; + } + if (response.status === 403) { + this.logger.warn('GitHub API rate limit exceeded, skipping file fetch'); + return { content: null, cached: false }; + } + throw new Error(`GitHub API error: ${response.status} ${response.statusText}`); + } + + const data = (await response.json()) as GitHubContentResponse; + let content: string | null = null; + + if (data.content && data.encoding === 'base64') { + content = Buffer.from(data.content, 'base64').toString('utf-8'); + } else if (data.download_url) { + const dlResponse = await fetch(data.download_url); + content = await dlResponse.text(); + } + + // Cache the response with its ETag + const etag = response.headers.get('etag'); + if (etag && content) { + this.etagCache.set(url, { etag, data: content }); + } + + return { content, cached: false }; + } + + /** + * Parse and validate template JSON. + */ + private parseTemplateJson(content: string, path: string): TemplateJson | null { + try { + const template = JSON.parse(content) as TemplateJson; + + if (!template._metadata?.name) { + this.logger.warn(`Template missing _metadata.name: ${path}`); + return null; + } + + if (!template.graph) { + this.logger.warn(`Template missing graph: ${path}`); + return null; + } + + return template; + } catch (err) { + this.logger.error(`Failed to parse template JSON: ${path}`, err); + return null; + } + } + + /** + * Sync templates from GitHub to the database. + * Called on startup and when admin clicks "Sync from GitHub". + * + * Uses ETag conditional requests: if the directory listing hasn't changed, + * the entire sync is skipped with zero API cost. Individual file fetches + * also use ETags so unchanged files are not re-downloaded. + */ + async syncTemplates(): Promise<{ + synced: string[]; + failed: { path: string; error: string }[]; + unchanged: string[]; + total: number; + directoryCacheHit: boolean; + }> { + if (this.isSyncing) { + this.logger.warn('Sync already in progress, skipping'); + return { synced: [], failed: [], unchanged: [], total: 0, directoryCacheHit: false }; + } + this.isSyncing = true; + + const { owner, repo, branch } = this.getRepoConfig(); + this.logger.log(`Starting template sync from ${owner}/${repo}/${branch}`); + + const synced: string[] = []; + const failed: { path: string; error: string }[] = []; + const unchanged: string[] = []; + let dirCacheHit = false; + + try { + const { files, cached } = await this.fetchDirectory('templates'); + dirCacheHit = cached; + + if (files.length === 0) { + this.logger.warn('No files found in templates/ directory'); + return { synced, failed, unchanged, total: 0, directoryCacheHit: dirCacheHit }; + } + + if (dirCacheHit) { + this.logger.log( + `Directory listing unchanged (ETag cache hit). Checking ${files.length} files...`, + ); + } + + for (const file of files) { + if (file.type !== 'file') continue; + if (!file.name.endsWith('.json')) continue; + + try { + const { content, cached: fileCacheHit } = await this.fetchFileContent(file.path); + + if (!content) { + failed.push({ path: file.path, error: 'Failed to fetch content' }); + continue; + } + + // If the file content is unchanged (ETag hit), still upsert to keep + // the DB in sync but track it as unchanged for reporting + if (fileCacheHit) { + unchanged.push(file.path); + } + + const template = this.parseTemplateJson(content, file.path); + + if (!template) { + failed.push({ + path: file.path, + error: 'Invalid template format', + }); + continue; + } + + // Build manifest from _metadata if not provided separately + const manifest: TemplateManifest = (template.manifest as TemplateManifest) || { + name: template._metadata.name, + description: template._metadata.description, + version: template._metadata.version, + author: template._metadata.author, + category: template._metadata.category, + tags: template._metadata.tags, + }; + + await this.templatesRepository.upsert({ + name: template._metadata.name, + description: template._metadata.description, + category: template._metadata.category || 'other', + tags: template._metadata.tags || [], + author: template._metadata.author, + repository: `${owner}/${repo}`, + path: file.path, + branch, + version: template._metadata.version, + manifest, + graph: template.graph, + requiredSecrets: template.requiredSecrets, + }); + + synced.push(template._metadata.name); + this.logger.debug(`Synced template: ${template._metadata.name}`); + } catch (err) { + const error = err instanceof Error ? err.message : 'Unknown error'; + failed.push({ path: file.path, error }); + this.logger.error(`Failed to sync template: ${file.path}`, err); + } + } + + const cacheStats = unchanged.length > 0 ? `, ${unchanged.length} unchanged (ETag)` : ''; + this.logger.log( + `Sync complete: ${synced.length} synced, ${failed.length} failed${cacheStats}`, + ); + } catch (err) { + this.logger.error('Failed to sync templates from GitHub', err); + throw err; + } finally { + this.isSyncing = false; + } + + return { + synced, + failed, + unchanged, + total: synced.length, + directoryCacheHit: dirCacheHit, + }; + } + + /** + * Get repository information. + */ + async getRepositoryInfo(): Promise<{ + owner: string; + repo: string; + branch: string; + url: string; + }> { + const { owner, repo, branch } = this.getRepoConfig(); + return { + owner, + repo, + branch, + url: `https://github.com/${owner}/${repo}`, + }; + } +} diff --git a/backend/src/templates/templates.controller.ts b/backend/src/templates/templates.controller.ts new file mode 100644 index 00000000..934179a1 --- /dev/null +++ b/backend/src/templates/templates.controller.ts @@ -0,0 +1,175 @@ +import { + Controller, + Get, + Post, + Body, + Param, + Query, + UseGuards, + HttpCode, + HttpStatus, + HttpException, +} from '@nestjs/common'; +import { TemplateService } from './templates.service'; +import { GitHubSyncService } from './github-sync.service'; +import { CurrentAuth } from '../auth/auth-context.decorator'; +import { Roles } from '../auth/roles.decorator'; +import { RolesGuard } from '../auth/roles.guard'; +import { Public } from '../auth/public.decorator'; + +/** + * Templates Controller + * Handles template library API endpoints + */ +@Controller('templates') +export class TemplatesController { + constructor( + private readonly templateService: TemplateService, + private readonly githubSyncService: GitHubSyncService, + ) {} + + /** + * GET /templates - List all templates with optional filters (public) + */ + @Public() + @Get() + async listTemplates( + @Query('category') category?: string, + @Query('search') search?: string, + @Query('tags') tags?: string, + ) { + const filters: { + category?: string; + search?: string; + tags?: string[]; + } = {}; + + if (category) filters.category = category; + if (search) filters.search = search; + if (tags) filters.tags = tags.split(','); + + return await this.templateService.listTemplates(filters); + } + + /** + * GET /templates/categories - List available categories (public) + */ + @Public() + @Get('categories') + async getCategories() { + return await this.templateService.getCategories(); + } + + /** + * GET /templates/tags - List available tags (public) + */ + @Public() + @Get('tags') + async getTags() { + return await this.templateService.getTags(); + } + + /** + * GET /templates/my - Get user's submitted templates + */ + @Get('my') + async getMyTemplates(@CurrentAuth() auth: { userId?: string; organizationId?: string }) { + return await this.templateService.getMyTemplates(auth.userId || auth.organizationId); + } + + /** + * GET /templates/repo-info - Get GitHub repository information (public) + * IMPORTANT: Must come before :id route to avoid route conflict + */ + @Public() + @Get('repo-info') + async getRepoInfo() { + return await this.githubSyncService.getRepositoryInfo(); + } + + /** + * GET /templates/submissions - Get template submissions for current user + * IMPORTANT: Must come before :id route to avoid route conflict + */ + @Get('submissions') + async getSubmissions(@CurrentAuth() auth: { userId?: string; organizationId?: string }) { + return await this.templateService.getSubmissions(auth.userId || auth.organizationId || ''); + } + + /** + * GET /templates/:id - Get template details by ID (public) + * IMPORTANT: Must be last to avoid conflicting with specific routes + */ + @Public() + @Get(':id') + async getTemplate(@Param('id') id: string) { + const template = await this.templateService.getTemplateById(id); + if (!template) { + throw new HttpException('Template not found', HttpStatus.NOT_FOUND); + } + return template; + } + + /** + * POST /templates/publish - Validate a workflow for template submission + * + * Note: This endpoint now only validates templates. PR creation has been removed. + * Users should create PRs via GitHub web flow after validation. + */ + @Post('publish') + @UseGuards(RolesGuard) + @Roles('ADMIN') + @HttpCode(HttpStatus.ACCEPTED) + async publishTemplate( + @CurrentAuth() auth: { userId?: string; organizationId?: string }, + @Body() + dto: { + workflowId: string; + name: string; + description: string; + category: string; + tags: string[]; + author: string; + }, + ) { + return await this.templateService.publishTemplate({ + ...dto, + submittedBy: auth.userId || auth.organizationId || 'unknown', + organizationId: auth.organizationId, + }); + } + + /** + * POST /templates/:id/use - Use a template to create a new workflow + */ + @Post(':id/use') + @UseGuards(RolesGuard) + @Roles('ADMIN') + async useTemplate( + @Param('id') id: string, + @CurrentAuth() auth: { userId?: string; organizationId?: string }, + @Body() + dto: { + workflowName: string; + secretMappings?: Record; + }, + ) { + return await this.templateService.useTemplate(id, { + ...dto, + userId: auth.userId || auth.organizationId, + organizationId: auth.organizationId, + }); + } + + /** + * POST /templates/sync - Sync templates from GitHub (admin only) + * + * Fetches templates from the GitHub repository and stores them in the database. + */ + @Post('sync') + @UseGuards(RolesGuard) + @Roles('ADMIN') + async syncTemplates(@CurrentAuth() _auth: { organizationId?: string }) { + return await this.githubSyncService.syncTemplates(); + } +} diff --git a/backend/src/templates/templates.module.ts b/backend/src/templates/templates.module.ts new file mode 100644 index 00000000..58ebd7d0 --- /dev/null +++ b/backend/src/templates/templates.module.ts @@ -0,0 +1,24 @@ +import { Module } from '@nestjs/common'; +import { ConfigModule } from '@nestjs/config'; +import { DatabaseModule } from '../database/database.module'; +import { WorkflowsModule } from '../workflows/workflows.module'; +import { TemplatesController } from './templates.controller'; +import { TemplateService } from './templates.service'; +import { WorkflowSanitizationService } from './workflow-sanitization.service'; +import { TemplatesRepository } from './templates.repository'; +import { GitHubSyncService } from './github-sync.service'; + +/** + * Templates Module + * Handles template library operations. + * + * Uses GitHub web flow for publishing and GitHub API for syncing templates. + * Templates are synced on startup and via manual admin trigger. + */ +@Module({ + imports: [DatabaseModule, ConfigModule, WorkflowsModule], + controllers: [TemplatesController], + providers: [TemplateService, WorkflowSanitizationService, TemplatesRepository, GitHubSyncService], + exports: [TemplateService, GitHubSyncService], +}) +export class TemplatesModule {} diff --git a/backend/src/templates/templates.repository.ts b/backend/src/templates/templates.repository.ts new file mode 100644 index 00000000..0d92dc24 --- /dev/null +++ b/backend/src/templates/templates.repository.ts @@ -0,0 +1,247 @@ +import { Inject, Injectable } from '@nestjs/common'; +import { + templatesTable, + templatesSubmissionsTable, + type TemplateManifest, +} from '../database/schema/templates'; +import { eq, and, desc, sql } from 'drizzle-orm'; +import { type NodePgDatabase } from 'drizzle-orm/node-postgres'; +import { DRIZZLE_TOKEN } from '../database/database.module'; + +/** + * Templates Repository + * Handles database operations for templates + */ +@Injectable() +export class TemplatesRepository { + constructor(@Inject(DRIZZLE_TOKEN) private readonly db: NodePgDatabase) {} + + /** + * Find all active templates with optional filters. + */ + async findAll(filters?: { category?: string; search?: string; tags?: string[] }) { + const conditions = [eq(templatesTable.isActive, true)]; + + if (filters?.category) { + conditions.push(eq(templatesTable.category, filters.category)); + } + + if (filters?.search) { + const escaped = filters.search.replace(/%/g, '\\%').replace(/_/g, '\\_'); + conditions.push( + sql`(${templatesTable.name} ILIKE ${'%' + escaped + '%'} OR ${templatesTable.description} ILIKE ${'%' + escaped + '%'})`, + ); + } + + if (filters?.tags && filters.tags.length > 0) { + conditions.push(sql`${templatesTable.tags} @> ${JSON.stringify(filters.tags)}::jsonb`); + } + + return this.db + .select() + .from(templatesTable) + .where(and(...conditions)) + .orderBy(desc(templatesTable.popularity)) + .execute(); + } + + /** + * Find template by ID + */ + async findById(id: string) { + const results = await this.db + .select() + .from(templatesTable) + .where(eq(templatesTable.id, id)) + .limit(1) + .execute(); + + return results[0] || null; + } + + /** + * Find template by repository and path + */ + async findByRepoAndPath(repository: string, path: string) { + const results = await this.db + .select() + .from(templatesTable) + .where(and(eq(templatesTable.repository, repository), eq(templatesTable.path, path))) + .limit(1) + .execute(); + + return results[0] || null; + } + + /** + * Create or update a template + */ + async upsert(template: { + name: string; + description?: string; + category?: string; + tags?: string[]; + author?: string; + repository: string; + path: string; + branch?: string; + version?: string; + commitSha?: string; + manifest: TemplateManifest; + graph?: Record; + requiredSecrets?: { name: string; type: string; description?: string }[]; + isOfficial?: boolean; + isVerified?: boolean; + }) { + // Check if template already exists + const existing = await this.findByRepoAndPath(template.repository, template.path); + + if (existing) { + // Update existing template + const results = await this.db + .update(templatesTable) + .set({ + ...template, + updatedAt: new Date(), + }) + .where(eq(templatesTable.id, existing.id)) + .returning() + .execute(); + + return results[0]; + } else { + // Create new template + const results = await this.db.insert(templatesTable).values(template).returning().execute(); + + return results[0]; + } + } + + /** + * Increment popularity counter + */ + async incrementPopularity(id: string) { + await this.db + .update(templatesTable) + .set({ + popularity: sql`${templatesTable.popularity} + 1`, + }) + .where(eq(templatesTable.id, id)) + .execute(); + } + + /** + * Get all categories with counts + */ + async getCategories() { + const results = await this.db + .select({ + category: templatesTable.category, + count: sql`count(*)`.as('count'), + }) + .from(templatesTable) + .where(eq(templatesTable.isActive, true)) + .groupBy(templatesTable.category) + .execute(); + + return results; + } + + /** + * Get all tags + */ + async getTags() { + const templates = await this.db + .select({ + tags: templatesTable.tags, + }) + .from(templatesTable) + .where(eq(templatesTable.isActive, true)) + .execute(); + + const tagSet = new Set(); + for (const template of templates) { + if (Array.isArray(template.tags)) { + for (const tag of template.tags) { + tagSet.add(tag); + } + } + } + + return Array.from(tagSet).sort(); + } + + /** + * Create a template submission record + */ + async createSubmission(submission: { + templateName: string; + description?: string; + category?: string; + repository: string; + branch?: string; + path: string; + commitSha?: string; + pullRequestNumber?: number; + pullRequestUrl?: string; + submittedBy: string; + organizationId?: string; + manifest?: TemplateManifest; + graph?: Record; + }) { + const results = await this.db.insert(templatesSubmissionsTable).values(submission).returning(); + + return results[0]; + } + + /** + * Find submission by PR number + */ + async findSubmissionByPR(prNumber: number) { + const results = await this.db + .select() + .from(templatesSubmissionsTable) + .where(eq(templatesSubmissionsTable.pullRequestNumber, prNumber)) + .limit(1) + .execute(); + + return results[0] || null; + } + + /** + * Update submission status + */ + async updateSubmissionStatus( + id: string, + status: 'pending' | 'approved' | 'rejected' | 'merged', + reviewedBy?: string, + feedback?: string, + ) { + const results = await this.db + .update(templatesSubmissionsTable) + .set({ + status, + reviewedBy, + feedback, + reviewedAt: reviewedBy ? new Date() : undefined, + updatedAt: new Date(), + }) + .where(eq(templatesSubmissionsTable.id, id)) + .returning() + .execute(); + + return results[0]; + } + + /** + * Get submissions by user + */ + async findSubmissionsByUser(submittedBy: string) { + return await this.db + .select() + .from(templatesSubmissionsTable) + .where(eq(templatesSubmissionsTable.submittedBy, submittedBy)) + .orderBy(desc(templatesSubmissionsTable.createdAt)) + .execute(); + } +} diff --git a/backend/src/templates/templates.service.ts b/backend/src/templates/templates.service.ts new file mode 100644 index 00000000..fc67cfd4 --- /dev/null +++ b/backend/src/templates/templates.service.ts @@ -0,0 +1,169 @@ +import { Injectable, Logger, HttpException, HttpStatus, NotFoundException } from '@nestjs/common'; +import { WorkflowSanitizationService } from './workflow-sanitization.service'; +import { TemplatesRepository } from './templates.repository'; +import { WorkflowsService } from '../workflows/workflows.service'; +import { WorkflowGraphSchema } from '../workflows/dto/workflow-graph.dto'; +import type { AuthContext } from '../auth/types'; + +/** + * Templates Service + * Business logic for template operations + * + * Note: PR creation has been removed. The backend now serves templates + * for browsing only. Users will create PRs through GitHub web flow. + */ +@Injectable() +export class TemplateService { + private readonly logger = new Logger(TemplateService.name); + + constructor( + private readonly sanitizationService: WorkflowSanitizationService, + private readonly templatesRepository: TemplatesRepository, + private readonly workflowsService: WorkflowsService, + ) {} + + /** + * List all templates with optional filters + */ + async listTemplates(filters?: { category?: string; search?: string; tags?: string[] }) { + return await this.templatesRepository.findAll(filters); + } + + /** + * Get template by ID + */ + async getTemplateById(id: string) { + return await this.templatesRepository.findById(id); + } + + /** + * Get user's submitted templates + */ + async getMyTemplates(userId: string | undefined) { + if (!userId) return []; + return await this.templatesRepository.findSubmissionsByUser(userId); + } + + /** + * Get template categories + */ + async getCategories() { + return await this.templatesRepository.getCategories(); + } + + /** + * Get template tags + */ + async getTags() { + return await this.templatesRepository.getTags(); + } + + /** + * Publish a workflow as a template + * + * Note: With GitHub web flow, this is now disabled. Users should use + * the frontend modal which opens GitHub directly. + */ + async publishTemplate(_params: { + workflowId: string; + name: string; + description: string; + category: string; + tags: string[]; + author: string; + submittedBy: string; + organizationId?: string; + }) { + throw new HttpException( + 'Template publishing via API is disabled. Please use the GitHub web flow from the frontend.', + HttpStatus.NOT_IMPLEMENTED, + ); + } + + /** + * Use a template to create a new workflow + * + * Fetches the template by ID, creates a new workflow from its graph data, + * names it with the provided workflowName, and increments the template's + * popularity counter. + */ + async useTemplate( + templateId: string, + params: { + workflowName: string; + secretMappings?: Record; + userId?: string; + organizationId?: string; + }, + ) { + // 1. Find the template + const template = await this.templatesRepository.findById(templateId); + if (!template) { + throw new NotFoundException(`Template ${templateId} not found`); + } + + // 2. Validate that the template has graph data + if (!template.graph) { + throw new HttpException( + 'Template does not contain workflow graph data', + HttpStatus.UNPROCESSABLE_ENTITY, + ); + } + + // 3. Build the workflow graph from the template, overriding the name + // Templates may lack node positions (stripped during publish to reduce size) + // so we add default positions in a grid layout before schema validation. + const graphData: Record = { + ...template.graph, + name: params.workflowName, + }; + + if (Array.isArray(graphData.nodes)) { + graphData.nodes = (graphData.nodes as Record[]).map((node, idx) => { + if (!node.position || typeof node.position !== 'object') { + return { ...node, position: { x: 250, y: idx * 150 } }; + } + return node; + }); + } + + // Parse through the WorkflowGraphSchema to ensure it conforms to the + // expected shape (adds defaults for viewport, config, etc.) + const workflowGraph = WorkflowGraphSchema.parse(graphData); + + // 4. Create the workflow via WorkflowsService + const authContext: AuthContext = { + userId: params.userId ?? null, + organizationId: params.organizationId ?? null, + roles: ['ADMIN'], + isAuthenticated: true, + provider: 'template', + }; + + this.logger.log( + `Creating workflow "${params.workflowName}" from template "${template.name}" (${templateId})`, + ); + + const workflow = await this.workflowsService.create(workflowGraph, authContext); + + // 5. Increment the template's popularity counter + await this.templatesRepository.incrementPopularity(templateId); + + this.logger.log( + `Created workflow ${workflow.id} from template ${templateId}, popularity incremented`, + ); + + return { + workflow, + templateId, + templateName: template.name, + }; + } + + /** + * Get template submissions + */ + async getSubmissions(userId: string) { + return await this.templatesRepository.findSubmissionsByUser(userId); + } +} diff --git a/backend/src/templates/workflow-sanitization.service.ts b/backend/src/templates/workflow-sanitization.service.ts new file mode 100644 index 00000000..57a7f41e --- /dev/null +++ b/backend/src/templates/workflow-sanitization.service.ts @@ -0,0 +1,254 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { RequiredSecret } from '../database/schema/templates'; + +/** + * Workflow Sanitization Service + * Removes secrets from workflows before publishing as templates + */ +@Injectable() +export class WorkflowSanitizationService { + private readonly logger = new Logger(WorkflowSanitizationService.name); + + /** + * Sanitize a workflow graph by removing all secret references + * Returns the sanitized graph along with detected secrets + */ + sanitizeWorkflow(graph: Record): { + sanitizedGraph: Record; + requiredSecrets: RequiredSecret[]; + removedSecrets: string[]; + } { + const requiredSecrets: RequiredSecret[] = []; + const removedSecrets: string[] = []; + + // Deep clone to avoid mutating original + const sanitizedGraph = JSON.parse(JSON.stringify(graph)); + + // Traverse the graph to find and remove secret references + this.traverseAndSanitize(sanitizedGraph, requiredSecrets, removedSecrets); + + this.logger.log(`Sanitized workflow: removed ${removedSecrets.length} secrets`); + + return { + sanitizedGraph, + requiredSecrets, + removedSecrets, + }; + } + + /** + * Deep traverse the graph and sanitize secret references + */ + private traverseAndSanitize( + obj: unknown, + requiredSecrets: RequiredSecret[], + removedSecrets: string[], + parentPath = '', + ): void { + if (!obj || typeof obj !== 'object') { + return; + } + + if (Array.isArray(obj)) { + for (let i = 0; i < obj.length; i++) { + this.traverseAndSanitize(obj[i], requiredSecrets, removedSecrets, `${parentPath}[${i}]`); + } + return; + } + + for (const [key, value] of Object.entries(obj)) { + const currentPath = parentPath ? `${parentPath}.${key}` : key; + + // Check for secret reference pattern + if (this.isSecretReference(value, key)) { + const secretInfo = this.extractSecretInfo(value, key); + if (secretInfo) { + requiredSecrets.push(secretInfo); + removedSecrets.push(secretInfo.name); + + // Replace with placeholder + (obj as Record)[key] = this.createPlaceholder(secretInfo); + } + } else if (typeof value === 'object' && value !== null) { + this.traverseAndSanitize(value, requiredSecrets, removedSecrets, currentPath); + } + } + } + + /** + * Check if a value is a secret reference + */ + private isSecretReference(value: unknown, key: string): boolean { + // Check for connection type references + if (key === 'connectionType' && typeof value === 'object' && value !== null) { + const connection = value as Record; + return connection.kind === 'secret' || connection.kind === 'primitive_secret'; + } + + // Check for secret references in specific fields + if (key === 'secretId' || key === 'secret_name' || key === 'apiKey') { + return true; + } + + // Check for secret pattern in strings + if (typeof value === 'string') { + return value.startsWith('{{secret:') || value.startsWith('{{ secrets.'); + } + + return false; + } + + /** + * Extract secret information from a secret reference + */ + private extractSecretInfo(value: unknown, key: string): RequiredSecret | null { + if (typeof value === 'object' && value !== null) { + const connection = value as Record; + if (connection.kind === 'secret' || connection.kind === 'primitive_secret') { + return { + name: (connection.name as string) || `secret_${key}`, + type: (connection.type as string) || 'string', + description: connection.description as string | undefined, + placeholder: this.generatePlaceholder((connection.name as string) || key), + }; + } + } + + if (typeof value === 'string') { + const match = value.match(/{{secret:(.+?)}}/) || value.match(/{{secrets\.(.+?)}}/); + if (match) { + return { + name: match[1].trim(), + type: 'string', + placeholder: this.generatePlaceholder(match[1].trim()), + }; + } + } + + return { + name: `secret_${key}`, + type: 'string', + placeholder: this.generatePlaceholder(key), + }; + } + + /** + * Create a placeholder for a secret + */ + private createPlaceholder(secretInfo: RequiredSecret): string { + return secretInfo.placeholder || `{{REPLACE_WITH_${secretInfo.name.toUpperCase()}}`; + } + + /** + * Generate a placeholder string + */ + private generatePlaceholder(secretName: string): string { + return `REPLACE_WITH_${secretName.toUpperCase().replace(/[^A-Z0-9]/g, '_')}`; + } + + /** + * Validate that a sanitized workflow graph is still valid + */ + validateSanitizedGraph(graph: Record): { + valid: boolean; + errors: string[]; + } { + const errors: string[] = []; + + // Check if graph has required structure + if (!graph.nodes || !Array.isArray(graph.nodes)) { + errors.push('Graph must have a nodes array'); + } + + if (!graph.edges || !Array.isArray(graph.edges)) { + errors.push('Graph must have an edges array'); + } + + // Check if nodes have required properties + if (Array.isArray(graph.nodes) && graph.nodes.length > 0) { + for (const node of graph.nodes) { + if (typeof node !== 'object' || node === null) { + errors.push('All nodes must be objects'); + continue; + } + + if (!('id' in node)) { + errors.push(`Node missing required field: id`); + } + + if (!('componentId' in node)) { + errors.push(`Node ${node.id || 'unknown'} missing required field: componentId`); + } + } + } + + // Check for remaining secret references that shouldn't be there + const graphStr = JSON.stringify(graph); + const secretPatterns = ['{{secret:', '{{secrets.', 'connectionType.secret']; + for (const pattern of secretPatterns) { + if (graphStr.includes(pattern)) { + errors.push(`Graph still contains secret references: ${pattern}`); + } + } + + return { + valid: errors.length === 0, + errors, + }; + } + + /** + * Generate a template manifest from workflow and metadata + */ + generateManifest(params: { + name: string; + description: string; + category: string; + tags: string[]; + author: string; + graph: Record; + requiredSecrets: RequiredSecret[]; + }): Record { + const { name, description, category, tags, author, graph, requiredSecrets } = params; + + // Detect entry point (first trigger node) + const entryPoint = this.findEntryPoint(graph); + + return { + name, + description, + version: '1.0.0', + author, + category: category || 'other', + tags: tags || [], + requiredSecrets: requiredSecrets.map((s) => ({ + name: s.name, + type: s.type, + description: s.description || `Secret required for ${s.name}`, + })), + entryPoint, + nodeCount: Array.isArray(graph.nodes) ? graph.nodes.length : 0, + edgeCount: Array.isArray(graph.edges) ? graph.edges.length : 0, + createdAt: new Date().toISOString(), + }; + } + + /** + * Find the entry point node (first trigger node) + */ + private findEntryPoint(graph: Record): string | undefined { + if (!graph.nodes || !Array.isArray(graph.nodes)) { + return undefined; + } + + const triggerNode = graph.nodes.find((node: unknown) => { + if (typeof node === 'object' && node !== null) { + const n = node as Record; + return n.componentType === 'trigger'; + } + return false; + }); + + return triggerNode?.id as string | undefined; + } +} diff --git a/bun.lock b/bun.lock index 16b97dcf..52e1c2c3 100644 --- a/bun.lock +++ b/bun.lock @@ -60,6 +60,7 @@ "bcryptjs": "^3.0.3", "class-transformer": "^0.5.1", "class-validator": "^0.14.3", + "cookie-parser": "^1.4.7", "date-fns": "^4.1.0", "dotenv": "^17.2.3", "drizzle-orm": "^0.44.7", diff --git a/docs/TEMPLATE_LIBRARY.md b/docs/TEMPLATE_LIBRARY.md new file mode 100644 index 00000000..9dcb86e3 --- /dev/null +++ b/docs/TEMPLATE_LIBRARY.md @@ -0,0 +1,271 @@ +# Template Library Feature + +## Overview + +The Template Library feature allows users to share and discover workflow templates. Users can publish their workflows as templates, which are submitted via GitHub PR to a templates repository. Other users can browse and use these templates to quickly create new workflows. + +## Architecture + +### Backend Components + +1. **Templates Module** (`backend/src/templates/`) + - `templates.module.ts` - NestJS module configuration + - `templates.controller.ts` - API endpoints + - `templates.service.ts` - Business logic + - `templates.repository.ts` - Database operations + - `github-template.service.ts` - GitHub API integration + - `workflow-sanitization.service.ts` - Secret sanitization + +2. **Database Schema** (`backend/src/database/schema/templates.ts`) + - `templates` table - Stores template metadata (cached from GitHub) + - `templates_submissions` table - Tracks PR-based submissions + +### Frontend Components + +1. **Pages** + - `TemplateLibraryPage.tsx` - Main template library page with filtering + +2. **Features** + - `UseTemplateModal.tsx` - Modal for using a template + - `PublishTemplateModal.tsx` - Modal for publishing a workflow as template + +3. **Store** + - `templateStore.ts` - Zustand store for template state management + +4. **API** + - Extended `api.ts` with templates API client + +## API Endpoints + +### Public Endpoints + +- `GET /templates` - List all templates with optional filters + - Query params: `category`, `search`, `tags` +- `GET /templates/:id` - Get template details +- `GET /templates/categories` - Get available categories +- `GET /templates/tags` - Get available tags + +### Admin Endpoints + +- `POST /templates/publish` - Publish workflow as template (creates PR) +- `POST /templates/:id/use` - Use template to create new workflow +- `POST /templates/sync` - Sync templates from GitHub repository +- `GET /templates/my` - Get user's submitted templates +- `GET /templates/submissions` - Get template submissions + +## Environment Variables + +### Required + +```bash +# GitHub Configuration +GITHUB_TEMPLATE_REPO=org/templates-repo +GITHUB_TOKEN=ghp_xxx # GitHub PAT with repo permissions + +# GitHub OAuth (optional, for user authentication) +GITHUB_CLIENT_ID=xxx +GITHUB_CLIENT_SECRET=xxx +``` + +### GitHub Token Permissions + +The GitHub personal access token needs the following permissions: +- `repo` (full control of private repositories) +- `pull_requests` (to create PRs) + +## Workflow Sanitization + +When publishing a workflow as a template, the system: + +1. **Removes secret references** - All secret values are removed from the workflow graph +2. **Creates secret placeholders** - Each removed secret is documented as a required secret +3. **Validates the graph** - Ensures the sanitized graph is still valid +4. **Generates a manifest** - Creates metadata about the template + +### Required Secrets Schema + +```typescript +{ + name: string; // Secret name + type: string; // Secret type (e.g., "api_key", "token") + description?: string; // What this secret is for + placeholder?: string; // Example format +} +``` + +## Template Manifest + +Each template has a manifest with the following structure: + +```typescript +{ + name: string; // Template name + description?: string; // Template description + version?: string; // Version + author?: string; // Author name/org + category?: string; // Category + tags?: string[]; // Tags + requiredSecrets?: RequiredSecret[]; // Required secrets + entryPoint?: string; // Entry point reference +} +``` + +## GitHub PR Workflow + +### Publishing a Template + +1. User clicks "Publish as Template" in Workflow Builder +2. User fills in template metadata (name, description, category, tags, author) +3. Backend sanitizes the workflow graph (removes secrets) +4. Backend creates a new branch in the templates repository +5. Backend commits the template JSON files +6. Backend creates a pull request +7. User receives PR URL for tracking + +### Template File Structure + +``` +templates/ + ├── security-scanner.json + ├── incident-response.json + └── compliance-check.json +``` + +Each template file contains: + +```json +{ + "manifest": { ... }, + "graph": { ... }, + "requiredSecrets": [ ... ] +} +``` + +## Setup Instructions + +### 1. Create Templates Repository + +1. Create a new GitHub repository for templates +2. Configure repository settings (private/public based on your needs) +3. Add the repository URL to environment variables + +### 2. Configure GitHub App + +1. Create a GitHub Personal Access Token or GitHub App +2. Grant necessary permissions +3. Add credentials to environment variables + +### 3. Run Database Migration + +```bash +# The migration file is at: +backend/drizzle/0020_create-templates.sql +``` + +### 4. Add Templates Module + +The TemplatesModule is already imported in `backend/src/app.module.ts`. + +## Usage + +### For Users + +1. Browse templates in the Template Library +2. Filter by category, search, or tags +3. Click "Use Template" on a template +4. Configure required secrets +5. Create workflow from template + +### For Publishers + +1. Create a workflow in the Workflow Builder +2. Click "Publish as Template" in the top bar +3. Fill in template metadata +4. Submit - a PR will be created +5. Wait for PR review and merge +6. Template appears in library after sync + +## Template Types + +### Community Templates +- Submitted by users +- Reviewed before appearing in library +- Tagged with relevant categories + +### Official Templates +- Created and maintained by ShipSec team +- Verified and tested +- Marked with "Official" badge + +### Enterprise Templates +- Organization-specific templates +- Private to organization +- Custom workflows for internal use + +## Troubleshooting + +### Templates not appearing after PR merge + +1. Run the sync endpoint: `POST /templates/sync` +2. Check the GitHub repository configuration +3. Verify the backend has access to the repository + +### Secrets not being sanitized + +1. Check the workflow graph structure +2. Verify secret references follow the expected format +3. Check backend logs for sanitization errors + +### GitHub PR creation failing + +1. Verify `GITHUB_TOKEN` has correct permissions +2. Check `GITHUB_TEMPLATE_REPO` is correct +3. Ensure the repository exists and is accessible +4. Check GitHub rate limits + +## Development + +### Adding New Template Categories + +Edit `TEMPLATE_CATEGORIES` in `PublishTemplateModal.tsx`: + +```typescript +const TEMPLATE_CATEGORIES = [ + 'Security', + 'Monitoring', + 'Compliance', + 'Incident Response', + 'Data Processing', + 'Integration', + 'Automation', + 'Reporting', + 'Testing', + 'Other', + // Add your category here +]; +``` + +### Customizing Template Display + +Template cards are rendered in `TemplateCard` component within `TemplateLibraryPage.tsx`. + +### Modifying Sanitization Rules + +Edit `workflow-sanitization.service.ts` to customize how secrets are detected and removed. + +## Security Considerations + +1. **Secret Sanitization** - All secret values are removed before publishing +2. **PR Review** - Templates require review before being merged +3. **Access Control** - Only admins can publish templates +4. **Repository Permissions** - GitHub token should have minimal required permissions + +## Future Enhancements + +- [ ] Template versioning and updates +- [ ] Template ratings and reviews +- [ ] Template analytics (usage, popularity) +- [ ] Template preview screenshots +- [ ] Template documentation editor +- [ ] Bulk template operations +- [ ] Template marketplace integration diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 87fea899..0c7cd058 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -1,5 +1,6 @@ import { BrowserRouter, Routes, Route } from 'react-router-dom'; import { WorkflowList } from '@/pages/WorkflowList'; +import { TemplateLibraryPage } from '@/pages/TemplateLibraryPage'; import { WorkflowBuilder } from '@/features/workflow-builder/WorkflowBuilder'; import { SecretsManager } from '@/pages/SecretsManager'; import { ApiKeysManager } from '@/pages/ApiKeysManager'; @@ -52,6 +53,7 @@ function App() { } /> + } /> void; onSave: () => Promise | void; onImport?: (file: File) => Promise | void; onExport?: () => void; + onPublishTemplate?: () => void; canManageWorkflows?: boolean; onUndo?: () => void; onRedo?: () => void; canUndo?: boolean; canRedo?: boolean; + isInWorkflowBuilder?: boolean; hasAnalyticsSink?: boolean; } @@ -56,10 +59,12 @@ export function TopBar({ selectedRunId, selectedRunStatus, selectedRunOrgId, + isInWorkflowBuilder, onRun, onSave, onImport, onExport, + onPublishTemplate, canManageWorkflows = true, onUndo, onRedo, @@ -531,15 +536,38 @@ export function TopBar({ )} - + {/* Run button with dropdown for Publish */} +
+ + {onPublishTemplate && isInWorkflowBuilder && ( + + + + + + + + Publish as Template + + + + )} +
diff --git a/frontend/src/features/analytics/events.ts b/frontend/src/features/analytics/events.ts index 925e16a2..dee6dbe5 100644 --- a/frontend/src/features/analytics/events.ts +++ b/frontend/src/features/analytics/events.ts @@ -14,6 +14,8 @@ export const Events = { NodeAdded: 'ui_node_added', SecretCreated: 'ui_secret_created', SecretDeleted: 'ui_secret_deleted', + TemplateUseClicked: 'ui_template_use_clicked', + TemplatePublishClicked: 'ui_template_publish_clicked', } as const; type EventName = (typeof Events)[keyof typeof Events]; @@ -57,6 +59,15 @@ const payloadSchemas: Record> = { [Events.SecretDeleted]: z.object({ name_length: z.number().int().nonnegative().optional(), }), + [Events.TemplateUseClicked]: z.object({ + template_id: z.string().optional(), + template_name: z.string().optional(), + category: z.string().optional(), + }), + [Events.TemplatePublishClicked]: z.object({ + workflow_id: z.string().optional(), + template_name: z.string().optional(), + }), }; export function track(event: T, payload: unknown = {}): void { diff --git a/frontend/src/features/templates/PublishTemplateModal.tsx b/frontend/src/features/templates/PublishTemplateModal.tsx new file mode 100644 index 00000000..f540b561 --- /dev/null +++ b/frontend/src/features/templates/PublishTemplateModal.tsx @@ -0,0 +1,565 @@ +import { useState, useCallback } from 'react'; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from '@/components/ui/dialog'; +import { Button } from '@/components/ui/button'; +import { Input } from '@/components/ui/input'; +import { Label } from '@/components/ui/label'; +import { Textarea } from '@/components/ui/textarea'; +import { Badge } from '@/components/ui/badge'; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from '@/components/ui/select'; +import { Loader2, AlertCircle, CheckCircle2, GitPullRequest, X, ExternalLink } from 'lucide-react'; +import { API_BASE_URL, getApiAuthHeaders } from '@/services/api'; +import { cn } from '@/lib/utils'; + +interface PublishTemplateModalProps { + workflowId: string; + workflowName: string; + open: boolean; + onOpenChange: (open: boolean) => void; + onSuccess?: () => void; +} + +// GitHub repository configuration for templates +const GITHUB_TEMPLATE_REPO = 'krishna9358/workflow-templates'; // format: owner/repo +const GITHUB_BRANCH = 'main'; + +const TEMPLATE_CATEGORIES = [ + 'Security', + 'Monitoring', + 'Compliance', + 'Incident Response', + 'Data Processing', + 'Integration', + 'Automation', + 'Reporting', + 'Testing', + 'Other', +]; + +const COMMON_TAGS = [ + 'security', + 'monitoring', + 'automation', + 'integration', + 'api', + 'notification', + 'compliance', + 'scanning', + 'analysis', + 'reporting', + 'incident', + 'response', + 'forensics', + 'enrichment', + 'detection', +]; + +interface WorkflowResponse { + id: string; + name: string; + description?: string; + manifest: Record; + graph: Record; +} + +interface TemplateMetadata { + name: string; + description?: string; + category: string; + tags: string[]; + author: string; + version: string; +} + +interface TemplateJson { + _metadata: TemplateMetadata; + graph: Record; + requiredSecrets: { name: string; type: string; description?: string }[]; +} + +/** + * Sanitize secrets from the workflow graph by replacing secret references with placeholders + */ +function sanitizeGraphForTemplate(graph: Record): Record { + const sanitized = JSON.parse(JSON.stringify(graph)); // Deep clone + + // Helper to recursively sanitize secret references + const traverseAndSanitize = (obj: unknown): unknown => { + if (typeof obj === 'object' && obj !== null) { + if (Array.isArray(obj)) { + return obj.map(traverseAndSanitize); + } + + const result: Record = {}; + for (const [key, value] of Object.entries(obj)) { + // Check for secret reference patterns + if ( + key === 'secretId' || + key === 'secret_name' || + key === 'secretName' || + key === 'secret_ref' || + key === 'secretRef' + ) { + result[key] = '{{SECRET_PLACEHOLDER}}'; + } else if ( + typeof value === 'string' && + (value.includes('${secrets.') || + value.includes('${secret.') || + value.includes('{{secret.')) + ) { + // Replace secret interpolation expressions with placeholder + result[key] = value + .replace(/\$\{secrets\.[^}]+\}/g, '{{SECRET_PLACEHOLDER}}') + .replace(/\$\{secret\.[^}]+\}/g, '{{SECRET_PLACEHOLDER}}') + .replace(/\{\{secret\.[^}]+\}\}/g, '{{SECRET_PLACEHOLDER}}'); + } else { + result[key] = traverseAndSanitize(value); + } + } + return result; + } + return obj; + }; + + return traverseAndSanitize(sanitized) as Record; +} + +/** + * Extract secret requirements from the graph for documentation + */ +function extractRequiredSecrets( + graph: Record, +): { name: string; type: string; description?: string }[] { + const secrets = new Map(); + + const traverseAndExtract = (obj: unknown, path: string[] = []) => { + if (typeof obj === 'object' && obj !== null) { + if (Array.isArray(obj)) { + obj.forEach((item, idx) => traverseAndExtract(item, [...path, String(idx)])); + return; + } + + for (const [key, value] of Object.entries(obj)) { + if (key === 'secretId' || key === 'secret_name' || key === 'secretName') { + if (typeof value === 'string') { + // Infer type from context + const context = path[path.length - 2] || 'generic'; + const type = context.toLowerCase().includes('api') + ? 'api_key' + : context.toLowerCase().includes('token') + ? 'token' + : context.toLowerCase().includes('password') + ? 'password' + : 'generic'; + secrets.set(value, { type, description: `Secret for ${context}` }); + } + } else if (typeof value === 'object' && value !== null) { + traverseAndExtract(value, [...path, key]); + } + } + } + }; + + traverseAndExtract(graph); + return Array.from(secrets.entries()).map(([name, info]) => ({ + name, + type: info.type, + description: info.description, + })); +} + +/** + * Strip viewport from graph to reduce JSON size. + * Viewport is a UI layout hint and not needed for the template's functionality. + * Note: Node positions are preserved because WorkflowGraphSchema requires them. + */ +function stripLayoutData(graph: Record): Record { + const stripped = { ...graph }; + delete stripped.viewport; + return stripped; +} + +/** + * Generate the template JSON structure with metadata + */ +function generateTemplateJson(workflow: WorkflowResponse, metadata: TemplateMetadata): string { + const sanitizedGraph = sanitizeGraphForTemplate(workflow.graph); + const compactGraph = stripLayoutData(sanitizedGraph); + const requiredSecrets = extractRequiredSecrets(workflow.graph); + + const template: TemplateJson = { + _metadata: metadata, + graph: compactGraph, + requiredSecrets, + }; + + return JSON.stringify(template, null, 2); +} + +/** + * Generate GitHub URL for creating a new file with template content pre-filled. + * Uses minified JSON to keep the URL short enough for GitHub. + */ +function generateGitHubUrl( + owner: string, + repo: string, + branch: string, + filename: string, + templateName: string, + content: string, +): string { + const baseUrl = `https://github.com/${owner}/${repo}/new/${branch}`; + const params = new URLSearchParams(); + params.set('filename', filename); + params.set('value', content); + params.set('message', `Add template: ${templateName}`); + params.set('quick_pull', '1'); + + return `${baseUrl}?${params.toString()}`; +} + +/** + * Sanitize filename to be safe for use in URLs + */ +function sanitizeFilename(name: string): string { + return ( + name + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-+|-+$/g, '') + '.json' + ); +} + +export function PublishTemplateModal({ + workflowId, + workflowName, + open, + onOpenChange, + onSuccess, +}: PublishTemplateModalProps) { + const [name, setName] = useState(workflowName); + const [description, setDescription] = useState(''); + const [category, setCategory] = useState(''); + const [tags, setTags] = useState([]); + const [tagInput, setTagInput] = useState(''); + const [author, setAuthor] = useState(''); + const [error, setError] = useState(null); + const [isLoading, setIsLoading] = useState(false); + const [success, setSuccess] = useState(false); + + const handleSubmit = useCallback( + async (e: React.FormEvent) => { + e.preventDefault(); + setError(null); + setIsLoading(true); + + if (!name.trim()) { + setError('Please enter a template name'); + setIsLoading(false); + return; + } + + if (!category) { + setError('Please select a category'); + setIsLoading(false); + return; + } + + if (!author.trim()) { + setError('Please enter your name or organization'); + setIsLoading(false); + return; + } + + try { + // Fetch the workflow data from the backend + const headers = await getApiAuthHeaders(); + const response = await fetch(`${API_BASE_URL}/api/v1/workflows/${workflowId}`, { + headers, + }); + if (!response.ok) { + throw new Error('Failed to fetch workflow data'); + } + + const workflow: WorkflowResponse = await response.json(); + + // Generate the template JSON + const metadata: TemplateMetadata = { + name: name.trim(), + description: description.trim() || undefined, + category: category || '', + tags, + author: author.trim(), + version: '1.0.0', + }; + + const templateJson = generateTemplateJson(workflow, metadata); + const filename = `templates/${sanitizeFilename(name.trim())}`; + + // Parse the GitHub repo config + const [owner, repo] = GITHUB_TEMPLATE_REPO.split('/'); + + // Generate GitHub URL with template content pre-filled + const githubUrl = generateGitHubUrl( + owner, + repo, + GITHUB_BRANCH, + filename, + name.trim(), + templateJson, + ); + + // Open the GitHub URL in a new tab + window.open(githubUrl, '_blank', 'noopener,noreferrer'); + + // Show success state + setSuccess(true); + onSuccess?.(); + } catch (err) { + setError(err instanceof Error ? err.message : 'Failed to prepare template for publishing'); + } finally { + setIsLoading(false); + } + }, + [workflowId, name, description, category, tags, author, onSuccess], + ); + + const handleAddTag = () => { + const tag = tagInput.trim().toLowerCase(); + if (tag && !tags.includes(tag)) { + setTags([...tags, tag]); + } + setTagInput(''); + }; + + const handleRemoveTag = (tagToRemove: string) => { + setTags(tags.filter((tag) => tag !== tagToRemove)); + }; + + const handleAddCommonTag = (tag: string) => { + if (!tags.includes(tag)) { + setTags([...tags, tag]); + } + }; + + const handleClose = () => { + if (!isLoading) { + onOpenChange(false); + // Reset form after a delay to avoid visual glitch + setTimeout(() => { + setName(workflowName); + setDescription(''); + setCategory(''); + setTags([]); + setAuthor(''); + setError(null); + setSuccess(false); + }, 200); + } + }; + + return ( + + + + + + Publish as Template + + + Submit your workflow as a template to the GitHub repository. + + + + {success ? ( + // Success State +
+
+
+ +
+
+

Template Ready for Submission!

+

+ A new tab has opened with your template pre-filled on GitHub. +

+
+
+

+ Next steps: +

+
    +
  1. + Review the template content in the opened tab +
    2. +
  2. + Important: Click "Propose new file" (NOT "Commit + directly") +
    3. +
  3. + Create Pull Request to submit your template for review +
    4. +
+
+ Note: Creating a PR allows reviewers to check your template + before it's added to the library. +
+ + View Repository on GitHub + +
+

+ Your workflow will be reviewed before being added to the template library. + You'll be notified once it's approved. +

+
+
+ ) : ( + // Form +
+ {/* Template Name */} +
+ + setName(e.target.value)} + placeholder="My Security Template" + /> +
+ + {/* Description */} +
+ +