feat: env validation

.github/workflows/deploy.yml

@@ -167,6 +167,12 @@ jobs:
             cd "${APP_DIR}"
             git fetch --prune --tags origin
             git checkout --detach "${DEPLOY_REF}"
+            APP_COMMIT="$(git rev-parse HEAD)"
+
+            if [[ -z "$APP_COMMIT" ]]; then
+              echo "Unable to resolve APP_COMMIT from DEPLOY_REF='${DEPLOY_REF}'"
+              exit 1
+            fi
 
             cd infra
             umask 077
@@ -183,12 +189,13 @@ jobs:
             echo "$GHCR_TOKEN" | sudo docker login ghcr.io -u "$GHCR_USER" --password-stdin
 
             echo "IMAGE_TAG in shell: [$IMAGE_TAG]"
+            echo "APP_COMMIT in shell: [$APP_COMMIT]"
 
             # For some reason, not pulling explicitly makes the docker stack deploy throw an error that it can't find the package.
             sudo docker pull ghcr.io/knowledgefutures/pubpub:"$IMAGE_TAG"
 
             # deploy/update stack
-            sudo env IMAGE_TAG="$IMAGE_TAG" docker stack deploy -c stack.yml --with-registry-auth --resolve-image always --prune pubpub
+            sudo env IMAGE_TAG="$IMAGE_TAG" APP_COMMIT="$APP_COMMIT" docker stack deploy -c stack.yml --with-registry-auth --resolve-image always --prune pubpub
 
             # show progress and cleanup
             sudo docker stack services pubpub

.gitignore

@@ -40,6 +40,7 @@ yarn-error.log*
 .env.*
 !.env.enc
 !.env.dev.enc
+!infra/.env.test
 
 
 # vercel

.test/setup-env.js

@@ -4,40 +4,7 @@
  * handled by JSDom, but there are some edge cases.
  */
 
-process.env.DOI_SUBMISSION_URL = '';
-process.env.DOI_LOGIN_ID = '';
-process.env.DOI_LOGIN_PASSWORD = '';
-process.env.MATOMO_TOKEN_AUTH = '';
-process.env.MAILGUN_API_KEY = 'some-nonsense';
-process.env.MAILCHIMP_API_KEY = '';
-process.env.FIREBASE_SERVICE_ACCOUNT_BASE64 = '';
-process.env.CLOUDAMQP_APIKEY = '';
-process.env.CLOUDAMQP_URL = '';
-process.env.ALGOLIA_ID = 'ooo';
-process.env.ALGOLIA_KEY = 'ooo';
-process.env.ALGOLIA_SEARCH_KEY = 'ooo';
-process.env.JWT_SIGNING_SECRET = 'shhhhhh';
-process.env.BYPASS_CAPTCHA = 'true';
-process.env.FIREBASE_TEST_DB_URL = 'http://localhost:9875?ns=pubpub-v6';
-process.env.ZOTERO_CLIENT_KEY = 'abc';
-process.env.ZOTERO_CLIENT_SECRET = 'def';
 
-process.env.FASTLY_PURGE_TOKEN_PROD = 'token';
-process.env.FASTLY_SERVICE_ID_PROD = 'prod';
-
-process.env.FASTLY_PURGE_TOKEN_DUQDUQ = 'token_duqduq';
-process.env.FASTLY_SERVICE_ID_DUQDUQ = 'duqduq';
-
-if (process.env.INTEGRATION) {
-	try {
-		require('../config.js');
-	} catch (e) {
-		console.log('No config.js found');
-	}
-} else {
-	process.env.AWS_ACCESS_KEY_ID = '';
-	process.env.AWS_SECRET_ACCESS_KEY = '';
-}
 
 if (typeof document !== 'undefined') {
 	require('mutationobserver-shim');

infra/.env.test

@@ -0,0 +1,63 @@
+DOI_SUBMISSION_URL=https://fakeurl.org
+DOI_LOGIN_ID=xxx
+DOI_LOGIN_PASSWORD=xxx
+MATOMO_TOKEN_AUTH=xxx
+MAILGUN_API_KEY=some-nonsense
+MAILCHIMP_API_KEY=xxx
+FIREBASE_SERVICE_ACCOUNT_BASE64=xxx
+CLOUDAMQP_APIKEY=xxx
+CLOUDAMQP_URL=xxx
+ALGOLIA_ID=ooo
+ALGOLIA_KEY=ooo
+ALGOLIA_SEARCH_KEY=ooo
+JWT_SIGNING_SECRET=shhhhhh
+BYPASS_CAPTCHA=true
+FIREBASE_TEST_DB_URL=http://localhost:9875?ns=pubpub-v6
+ZOTERO_CLIENT_KEY=abc
+ZOTERO_CLIENT_SECRET=def
+
+FASTLY_PURGE_TOKEN=token
+FASTLY_SERVICE_ID=prod
+
+AWS_ACCESS_KEY_ID=xxx;
+AWS_SECRET_ACCESS_KEY=xxx
+JWT_SIGNING_SECRET=xxxxxx
+FIREBASE_SERVICE_ACCOUNT_BASE64=xxxxxxxxx
+
+AWS_BACKUP_ACCESS_KEY_ID=xxx
+AWS_BACKUP_SECRET_ACCESS_KEY=xxx
+S3_BACKUP_ENDPOINT=https://s3.aws.com
+S3_BACKUP_ACCESS_KEY=xxx
+S3_BACKUP_SECRET_KEY=xxx
+ALTCHA_HMAC_KEY=xxx
+AES_ENCRYPTION_KEY=xxx
+DATACITE_DEPOSIT_URL=https://deposit.com
+SLACK_WEBHOOK_URL=https://slack.com
+SENTRY_AUTH_TOKEN=xxx
+SENTRY_ORG=xxx
+METABASE_SECRET_KEY=xxx
+STITCH_WEBHOOK_URL=https://xxxxxxxxxxxxxxxxxxxx.com
+# AWS_ACCESS_KEY_ID: Required
+# AWS_SECRET_ACCESS_KEY: Required
+# AWS_BACKUP_ACCESS_KEY_ID: Required
+# AWS_BACKUP_SECRET_ACCESS_KEY: Required
+# MAILGUN_API_KEY: Required
+# ALGOLIA_ID: Required
+# ALGOLIA_KEY: Required
+# ALGOLIA_SEARCH_KEY: Required
+# ALTCHA_HMAC_KEY: Required
+# AES_ENCRYPTION_KEY: Required
+# DOI_LOGIN_ID: Required
+# DOI_LOGIN_PASSWORD: Required
+# DOI_SUBMISSION_URL: Required
+# DATACITE_DEPOSIT_URL: Required
+# CLOUDAMQP_URL: Required
+# ZOTERO_CLIENT_KEY: Required
+# ZOTERO_CLIENT_SECRET: Required
+# FASTLY_SERVICE_ID: Required
+# FASTLY_PURGE_TOKEN: Required
+# SLACK_WEBHOOK_URL: Required
+# SENTRY_AUTH_TOKEN: Required
+# SENTRY_ORG: Required
+# METABASE_SECRET_KEY: Required
+# STITCH_WEBHOOK_URL: Required

infra/ENV.md

@@ -0,0 +1,94 @@
+# Environment Variables
+
+All environment variables used by PubPub, with types, defaults, and descriptions.
+
+> Auto-generated from `server/env.ts` — do not edit manually.
+> Run `npx tsx tools/generateEnvDocs.ts` to regenerate.
+
+| Variable | Type | Required | Default | Description |
+|----------|------|----------|---------|-------------|
+| `NODE_ENV` | `"production"` \| `"development"` \| `"test"` | No | development | Node environment |
+| `PORT` | number | No | 9876 | HTTP server port |
+| `PUBPUB_PRODUCTION` | boolean | No | `false` | Treat this instance as the production PubPub deployment |
+| `IS_DUQDUQ` | boolean | No | `false` | Treat this instance as the DuqDuq staging deployment |
+| `IS_QUBQUB` | boolean | No | `false` | Treat this instance as the QubQub deployment |
+| `HEROKU_SLUG_COMMIT` | string | No | — | Git commit hash set by Heroku |
+| `PUBPUB_LOCAL_COMMUNITY` | string | No | — | Slug of the community to proxy in local dev (e.g. "stanford-jblp") |
+| `FORCE_BASE_PUBPUB` | boolean | No | `false` | Force the base PubPub site in development/QubQub mode |
+| `PUBPUB_READ_ONLY` | boolean | No | `false` | Enable read-only mode, disabling all mutations |
+| `DISABLE_SSL_REDIRECT` | boolean | No | `false` | Disable automatic HTTP → HTTPS redirect |
+| `DATABASE_URL` | string | **Yes** | — | Primary PostgreSQL connection URL |
+| `DATABASE_READ_REPLICA_1_URL` | string | No | — | PostgreSQL read-replica 1 URL |
+| `DATABASE_READ_REPLICA_2_URL` | string | No | — | PostgreSQL read-replica 2 URL |
+| `SEQUELIZE_MAX_CONNECTIONS` | number | No | — | Max DB pool connections (default: 20 for server, 5 for workers) |
+| `SEQUELIZE_IDLE_TIMEOUT` | number | No | — | DB pool idle timeout in ms (default: 60000) |
+| `SEQUELIZE_ACQUIRE_TIMEOUT` | number | No | — | DB pool acquire timeout in ms (default: 10000) |
+| `SEQUELIZE_MAX_USES` | number | No | — | Max times a DB connection may be reused (default: Infinity) |
+| `REQUEST_TIMEOUT_MS` | number | No | 30000 | Request abort timeout in ms |
+| `JWT_SIGNING_SECRET` | string | **Yes** | — | Secret used to sign JWT tokens |
+| `FIREBASE_SERVICE_ACCOUNT_BASE64` | string | **Yes** | — | Base64-encoded Firebase service-account JSON |
+| `FIREBASE_TEST_DB_URL` | string | No | — | Firebase Realtime Database URL for test env |
+| `AWS_ACCESS_KEY_ID` | string | **Yes** | — | AWS access key for S3 uploads |
+| `AWS_SECRET_ACCESS_KEY` | string | **Yes** | — | AWS secret key for S3 uploads |
+| `AWS_BACKUP_ACCESS_KEY_ID` | string | No | — | AWS access key for backup operations |
+| `AWS_BACKUP_SECRET_ACCESS_KEY` | string | No | — | AWS secret key for backup operations |
+| `S3_BACKUP_ENDPOINT` | string | No | — | S3-compatible endpoint for backups |
+| `S3_BACKUP_ACCESS_KEY` | string | No | — | S3 backup access key (if different from AWS) |
+| `S3_BACKUP_SECRET_KEY` | string | No | — | S3 backup secret key (if different from AWS) |
+| `S3_BACKUP_BUCKET` | string | No | — | S3 bucket name for backups |
+| `S3_BACKUP_KEY_PREFIX` | string | No | pg-backups | Key prefix for backup objects |
+| `MAILGUN_API_KEY` | string | **Yes** | — | Mailgun API key for transactional emails |
+| `MAILCHIMP_API_KEY` | string | No | — | Mailchimp API key for mailing lists |
+| `ALGOLIA_ID` | string | **Yes** | — | Algolia application ID |
+| `ALGOLIA_KEY` | string | **Yes** | — | Algolia admin API key |
+| `ALGOLIA_SEARCH_KEY` | string | **Yes** | — | Algolia public search-only key |
+| `ALTCHA_HMAC_KEY` | string | **Yes** | — | HMAC key for ALTCHA proof-of-work captcha |
+| `BYPASS_CAPTCHA` | boolean | No | `false` | Bypass captcha checks (dev/test only) |
+| `AES_ENCRYPTION_KEY` | string | **Yes** | — | AES-256 key for encrypting deposit credentials |
+| `DOI_LOGIN_ID` | string | No | — | CrossRef DOI deposit login ID |
+| `DOI_LOGIN_PASSWORD` | string | No | — | CrossRef DOI deposit login password |
+| `DOI_SUBMISSION_URL` | string | No | — | CrossRef DOI deposit endpoint URL |
+| `DATACITE_DEPOSIT_URL` | string | No | — | DataCite DOI deposit endpoint URL |
+| `CLOUDAMQP_URL` | string | No | — | CloudAMQP (RabbitMQ) connection URL |
+| `ZOTERO_CLIENT_KEY` | string | No | — | Zotero OAuth1 consumer key |
+| `ZOTERO_CLIENT_SECRET` | string | No | — | Zotero OAuth1 consumer secret |
+| `FASTLY_SERVICE_ID_PROD` | string | No | — | Fastly service ID for production |
+| `FASTLY_PURGE_TOKEN_PROD` | string | No | — | Fastly purge token for production |
+| `FASTLY_SERVICE_ID_DUQDUQ` | string | No | — | Fastly service ID for DuqDuq |
+| `FASTLY_PURGE_TOKEN_DUQDUQ` | string | No | — | Fastly purge token for DuqDuq |
+| `PURGE_TOKEN` | string | No | — | Legacy Fastly purge token |
+| `PUBSTASH_URL` | string | No | http://pubstash:8080 | PubStash service URL for paged exports |
+| `PUBSTASH_ACCESS_KEY` | string | No | — | PubStash access key |
+| `SLACK_WEBHOOK_URL` | string | No | — | Slack incoming webhook URL for notifications |
+| `STITCH_WEBHOOK_URL` | string | No | — | MongoDB Stitch webhook URL for analytics |
+| `METABASE_SECRET_KEY` | string | No | — | Metabase embedding secret key |
+| `SENTRY_AUTH_TOKEN` | string | No | — | Sentry auth token (build-time only) |
+| `SENTRY_ORG` | string | No | — | Sentry organization slug |
+| `BLOCKLIST_IP_ADDRESSES` | string | No | — | Comma-separated list of IP addresses to block |
+| `LARGE_COMMUNITY_SLUGS` | string | No | — | Comma-separated list of large community slugs for optimized queries |
+| `NEW_ACCOUNT_LINK_COMMENT_WINDOW_MINUTES` | number | No | — | Minutes after account creation during which link-comments are flagged |
+| `EXTRA_SUSPICIOUS_KEYWORDS` | string | No | — | Comma-separated extra keywords to flag uploads as suspicious |
+| `BACKUPS_SECRET` | string | No | — | GPG passphrase for encrypting backups |
+| `DEBUG_LOG` | string | No | — | Enable verbose debug logging |
+| `WORKER` | boolean | No | `false` | Set to true when running as a standalone worker process |
+| `DEFAULT_QUEUE_TASK_PRIORITY` | number | No | — | Default priority for worker queue tasks |
+| `PUBPUB_LOCAL_TASK_QUEUE` | string | No | — | Custom task queue name for local development |
+| `INTEGRATION_TESTING` | boolean | No | `false` | Signals that integration tests are running |
+| `TEST_FASTLY_PURGE` | boolean | No | `false` | Enable Fastly purge calls during tests |
+| `USE_LOCAL_DB` | boolean | No | `false` | Force use of local PostgreSQL in development |
+
+## Required Variables Checklist
+
+These must be set for the server to start:
+
+- [ ] `DATABASE_URL` — Primary PostgreSQL connection URL
+- [ ] `JWT_SIGNING_SECRET` — Secret used to sign JWT tokens
+- [ ] `FIREBASE_SERVICE_ACCOUNT_BASE64` — Base64-encoded Firebase service-account JSON
+- [ ] `AWS_ACCESS_KEY_ID` — AWS access key for S3 uploads
+- [ ] `AWS_SECRET_ACCESS_KEY` — AWS secret key for S3 uploads
+- [ ] `MAILGUN_API_KEY` — Mailgun API key for transactional emails
+- [ ] `ALGOLIA_ID` — Algolia application ID
+- [ ] `ALGOLIA_KEY` — Algolia admin API key
+- [ ] `ALGOLIA_SEARCH_KEY` — Algolia public search-only key
+- [ ] `ALTCHA_HMAC_KEY` — HMAC key for ALTCHA proof-of-work captcha
+- [ ] `AES_ENCRYPTION_KEY` — AES-256 key for encrypting deposit credentials

infra/stack.yml

@@ -27,6 +27,7 @@ services:
         environment:
             NODE_ENV: production
             PORT: '3000'
+            APP_COMMIT: '${APP_COMMIT:-}'
             # Reuse the existing env var name, but point it at the in-swarm broker:
             CLOUDAMQP_URL: 'amqp://appuser:apppassword@rabbitmq:5672/appvhost'
         networks: [appnet]
@@ -60,6 +61,7 @@ services:
         env_file: [.env]
         environment:
             NODE_ENV: production
+            APP_COMMIT: '${APP_COMMIT:-}'
             CLOUDAMQP_URL: 'amqp://appuser:apppassword@rabbitmq:5672/appvhost'
             CLOUDAMQP_APIKEY: ''
         command: ['pnpm', 'run', 'workers-prod']

localDatabase.js

@@ -2,21 +2,20 @@
 const path = require('path');
 
 const { createAndRunPostgresDatabase } = require('server/database');
+const { env } = require('server/env');
 const { isProd } = require('utils/environment');
 
 const setupLocalDatabase = async (definitely) => {
 	if (isProd()) {
 		throw new Error('Refusing to set up local database in production environment.');
 	} else if (definitely || process.env.USE_LOCAL_DB) {
-		process.env.DATABASE_URL = await createAndRunPostgresDatabase({
+		env.DATABASE_URL = await createAndRunPostgresDatabase({
 			username: 'pubpubdbadmin',
 			password: 'pubpub-db-password',
 			dbName: 'pubpub-localdb',
 			dbPath: path.join(process.cwd(), 'pubpub-localdb'),
 			drop: false,
 		});
-		process.env.DATABASE_READ_REPLICA_1_URL = process.env.DATABASE_URL;
-		process.env.DATABASE_READ_REPLICA_2_URL = process.env.DATABASE_URL;
 	}
 };
 

package.json

@@ -14,7 +14,7 @@
 		"dev": "./infra/dev.sh",
 		"dev:prod": "./infra/dev.sh prod",
 		"docker:build": "docker build -t pubpub:test-build .",
-		"api-dev": "concurrently \"pnpm run watch-server\" \"NODE_PATH=./dist/server/client:./dist/server:./dist node --enable-source-maps init.js --watch ./dist/server --watch ./node_modules/@pubpub\"",
+		"api-dev": "concurrently \"pnpm run watch-server\" \"NODE_PATH=./dist/server/client:./dist/server:./dist node --watch-preserve-output --enable-source-maps init.js --watch ./dist/server --watch ./node_modules/@pubpub\"",
 		"api-prod": "NODE_PATH=./dist/server/client:./dist/server:./dist node --enable-source-maps init.js",
 		"build-client-dev": "webpack --config ./client/webpack/webpackConfig.dev.js",
 		"build-dev": "webpack --config ./client/webpack/webpackConfig.dev.js --watch",
@@ -49,8 +49,7 @@
 		"workers-dev": "NODE_PATH=./dist/server/client:./dist/server WORKER=true nodemon dist/server/workers/init --watch dist/server -e js,ts",
 		"workers-prod": "NODE_PATH=./dist/server/client:./dist/server WORKER=true node --enable-source-maps dist/server/workers/init",
 		"pubstash-prod": "NODE_PATH=./dist/server/client:./dist/server:./dist node --enable-source-maps dist/server/pubstash/server.js",
-		"upload-sentry-sourcemaps": "sentry-cli sourcemaps inject -p pubpub-v6 dist/server && sentry-cli releases files -p pubpub-v6 $SOURCE_VERSION upload-sourcemaps --url-prefix /app --strip-common-prefix --wait dist/server",
-		"write-commit-version": "echo $SOURCE_VERSION > .app-commit"
+		"upload-sentry-sourcemaps": "sentry-cli sourcemaps inject -p pubpub-v6 dist/server && sentry-cli releases files -p pubpub-v6 $SOURCE_VERSION upload-sourcemaps --url-prefix /app --strip-common-prefix --wait dist/server"
 	},
 	"dependencies": {
 		"@analytics/core": "^0.12.7",

server/analytics/api.ts

@@ -6,19 +6,20 @@ import { initServer } from '@ts-rest/express';
 import { getCountryForTimezone } from 'countries-and-timezones';
 import express from 'express';
 
+import { env } from 'server/env';
 import { contract } from 'utils/api/contract';
 
 const s = initServer();
 
 const sendToStitch = async (
 	payload: AnalyticsEvent & { country: string | null; countryCode: string | null },
 ) => {
-	if (!process.env.STITCH_WEBHOOK_URL) {
+	if (!env.STITCH_WEBHOOK_URL) {
 		// throw new Error('Missing STITCH_WEBHOOK_URL');
 		return null;
 	}
 
-	const response = await fetch(process.env.STITCH_WEBHOOK_URL, {
+	const response = await fetch(env.STITCH_WEBHOOK_URL, {
 		method: 'POST',
 		body: JSON.stringify(payload),
 		headers: {

server/apiRoutes.ts

@@ -13,6 +13,7 @@ import { router as devApiRouter } from './dev/api';
 import { router as discussionRouter } from './discussion/api';
 import { router as doiRouter } from './doi/api';
 import { router as editorRouter } from './editor/api';
+import { env } from './env';
 import { router as integrationDataOAuth1Router } from './integrationDataOAuth1/api';
 import { router as landingPageFeatureRouter } from './landingPageFeature/api';
 import { router as layoutRouter } from './layout/api';
@@ -74,7 +75,7 @@ const apiRouter = Router()
 	.use(zoteroIntegrationRouter)
 	.use(apiDocsRouter);
 
-if (!isProd() && process.env.NODE_ENV !== 'test') {
+if (!isProd() && env.NODE_ENV !== 'test') {
 	apiRouter.use(devApiRouter);
 }
 

server/debug/api.ts

@@ -3,12 +3,14 @@
 
 import { type Request, type Response, Router } from 'express';
 
+import { env } from 'server/env';
+
 import { poolOptions, sequelize } from '../sequelize';
 
 export const router = Router(); // adjust path as needed
 
 export const poolStatsHandler = (req: Request, res: Response) => {
-	if (process.env.NODE_ENV === 'production') {
+	if (env.NODE_ENV === 'production') {
 		return res.status(404).json({ error: 'Not available in production' });
 	}
 

server/dev/api.ts

@@ -1,5 +1,6 @@
 import { Router } from 'express';
 
+import { env } from 'server/env';
 import { Community } from 'server/models';
 import { BadRequestError, ForbiddenError, NotFoundError } from 'server/utils/errors';
 import { wrap } from 'server/wrap';
@@ -8,14 +9,24 @@ import { canSelectCommunityForDevelopment } from 'utils/environment';
 export const router = Router();
 
 export const setSubdomain = async (subdomain: string | null) => {
-	process.env.FORCE_BASE_PUBPUB = subdomain === null ? 'true' : '';
-	if (subdomain) {
-		const exists = await Community.findOne({ where: { subdomain } });
-		if (!exists) {
-			throw new NotFoundError();
-		}
-		process.env.PUBPUB_LOCAL_COMMUNITY = subdomain;
+	const isBasePubPub = subdomain === null;
+	env.FORCE_BASE_PUBPUB = isBasePubPub;
+
+	if (isBasePubPub) {
+		env.PUBPUB_LOCAL_COMMUNITY = undefined;
+		return;
+	}
+
+	if (!subdomain) {
+		return;
 	}
+
+	const exists = await Community.findOne({ where: { subdomain } });
+	if (!exists) {
+		throw new NotFoundError();
+	}
+
+	env.PUBPUB_LOCAL_COMMUNITY = subdomain;
 };
 
 router.post(