Skip to content

MCP server for tech reports #95

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
max-ostapenko merged 6 commits into from
Apr 6, 2026
Merged

MCP server for tech reports #95

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
5b6d730
Added mcp server and refactored controllers to use reportService for …
max-ostapenko Mar 12, 2026
1fb9fed
feat(geoBreakdown): add geo breakdown data endpoint and query function
max-ostapenko Apr 4, 2026
bec8ec0
Merge remote-tracking branch 'origin/main' into clear-asp
max-ostapenko Apr 6, 2026
f61c5e8
refactor: streamline response handling by consolidating JSON response…
max-ostapenko Apr 6, 2026
04c3bb7
fix: correct Dependabot auto-merge condition placement in workflow
max-ostapenko Apr 6, 2026
cb43c8d
refactor: simplify controller imports and route handling logic
max-ostapenko Apr 6, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/workflows/dependabot.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
---
name: Dependabot Auto-Merge
name: Test

on:
pull_request:
Expand All @@ -8,7 +8,6 @@ on:

jobs:
test:
if: github.event.pull_request.user.login == 'dependabot[bot]' && github.repository == 'HTTPArchive/tech-report-apis'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
Expand All @@ -19,6 +18,7 @@ jobs:

dependabot:
name: Dependabot auto-merge
if: github.event.pull_request.user.login == 'dependabot[bot]' && github.repository == 'HTTPArchive/tech-report-apis'
runs-on: ubuntu-latest
needs: test

Expand Down
63 changes: 8 additions & 55 deletions src/controllers/categoriesController.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,60 +1,13 @@
import { firestore } from '../utils/db.js';
import { executeQuery, validateArrayParameter } from '../utils/controllerHelpers.js';
import { handleControllerError, sendJSONResponse } from '../utils/controllerHelpers.js';
import { queryCategories } from '../utils/reportService.js';

/**
* List categories with optional filtering and field selection
*/
const listCategories = async (req, res) => {
const queryBuilder = async (params) => {
/*
// Validate parameters
const supportedParams = ['category', 'onlyname', 'fields'];
const providedParams = Object.keys(params);
const unsupportedParams = providedParams.filter(param => !supportedParams.includes(param));

if (unsupportedParams.length > 0) {
const error = new Error(`Unsupported parameters: ${unsupportedParams.join(', ')}.`);
error.statusCode = 400;
throw error;
}
*/

const isOnlyNames = params.onlyname || typeof params.onlyname === 'string';
const hasCustomFields = params.fields && !isOnlyNames;

let query = firestore.collection('categories').orderBy('category', 'asc');

// Apply category filter with validation
const categoryParam = params.category || 'ALL';
if (categoryParam !== 'ALL') {
const categories = validateArrayParameter(categoryParam, 'category');
if (categories.length > 0) {
query = query.where('category', 'in', categories);
}
}

// Apply field selection
if (isOnlyNames) {
query = query.select('category');
} else if (hasCustomFields) {
const requestedFields = params.fields.split(',').map(f => f.trim());
query = query.select(...requestedFields);
}

return query;
};

const dataProcessor = (data, params) => {
const isOnlyNames = params.onlyname || typeof params.onlyname === 'string';

if (isOnlyNames) {
return data.map(item => item.category);
}

return data;
};

await executeQuery(req, res, 'categories', queryBuilder, dataProcessor);
try {
const data = await queryCategories(req.query);
sendJSONResponse(req, res, data);
} catch (error) {
handleControllerError(res, error, 'fetching categories');
}
};

export { listCategories };
22 changes: 9 additions & 13 deletions src/controllers/geosController.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,17 +1,13 @@
import { firestore } from '../utils/db.js';
import { executeQuery } from '../utils/controllerHelpers.js';
import { handleControllerError, sendJSONResponse } from '../utils/controllerHelpers.js';
import { queryGeos } from '../utils/reportService.js';

/**
* List all geographic locations from database
*/
const listGeos = async (req, res) => {
const queryBuilder = async () => {
return firestore.collection('geos').orderBy('mobile_origins', 'desc').select('geo');
};

await executeQuery(req, res, 'geos', queryBuilder);
try {
const data = await queryGeos();
sendJSONResponse(req, res, data);
} catch (error) {
handleControllerError(res, error, 'fetching geos');
}
};

export {
listGeos
};
export { listGeos };
22 changes: 9 additions & 13 deletions src/controllers/ranksController.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,17 +1,13 @@
import { firestore } from '../utils/db.js';
import { executeQuery } from '../utils/controllerHelpers.js';
import { handleControllerError, sendJSONResponse } from '../utils/controllerHelpers.js';
import { queryRanks } from '../utils/reportService.js';

/**
* List all rank options from database
*/
const listRanks = async (req, res) => {
const queryBuilder = async () => {
return firestore.collection('ranks').orderBy('mobile_origins', 'desc').select('rank');
};

await executeQuery(req, res, 'ranks', queryBuilder);
try {
const data = await queryRanks();
sendJSONResponse(req, res, data);
} catch (error) {
handleControllerError(res, error, 'fetching ranks');
}
};

export {
listRanks
};
export { listRanks };
142 changes: 5 additions & 137 deletions src/controllers/reportController.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,149 +1,17 @@
import { firestoreOld } from '../utils/db.js';
const firestore = firestoreOld;

import {
REQUIRED_PARAMS,
validateRequiredParams,
sendValidationError,
getLatestDate,
handleControllerError,
validateArrayParameter,
generateETag,
isModified
} from '../utils/controllerHelpers.js';

/**
* Configuration for different report types
*/
const REPORT_CONFIGS = {
adoption: {
table: 'adoption',
dataField: 'adoption'
},
pageWeight: {
table: 'page_weight',
dataField: 'pageWeight' // TODO: change to page_weight once migrated to new Firestore DB
},
lighthouse: {
table: 'lighthouse',
dataField: 'lighthouse'
},
cwv: {
table: 'core_web_vitals',
dataField: 'vitals'
},
audits: {
table: 'audits',
dataField: 'audits'
}
};

/**
* Generic report data controller factory
* Creates controllers for adoption, pageWeight, lighthouse, and cwv data.
* Pass { crossGeo: true } to get a cross-geography snapshot (omits geo filter,
* includes geo in projection, returns a single month of data).
*/
const createReportController = (reportType, { crossGeo = false } = {}) => {
const config = REPORT_CONFIGS[reportType];
if (!config) {
throw new Error(`Unknown report type: ${reportType}`);
}
import { handleControllerError, sendJSONResponse } from '../utils/controllerHelpers.js';
import { queryReport } from '../utils/reportService.js';

const createReportController = (reportType, defaults = {}) => {
return async (req, res) => {
try {
const params = req.query;

/*
// Validate supported parameters
const supportedParams = ['technology', 'geo', 'rank', 'start', 'end'];
const providedParams = Object.keys(params);
const unsupportedParams = providedParams.filter(param => !supportedParams.includes(param));

if (unsupportedParams.length > 0) {
const error = new Error(`Unsupported parameters: ${unsupportedParams.join(', ')}.`);
error.statusCode = 400;
throw error;
}
*/

// Validate required parameters using shared utility
const errors = validateRequiredParams(params, []);

if (errors) {
sendValidationError(res, errors);
return;
}

// Default technology, geo, and rank to 'ALL' if missing or empty
const technologyParam = params.technology || 'ALL';
const geoParam = params.geo || 'ALL';
const rankParam = params.rank || 'ALL';

// Validate and process technology array
const techArray = validateArrayParameter(technologyParam, 'technology');

// Build Firestore query
let query = firestore.collection(config.table);

query = query.where('rank', '==', rankParam);
query = query.where('technology', 'in', techArray);

// Apply version filter with special handling for 'ALL' case
if (params.version && techArray.length === 1) {
//query = query.where('version', '==', params.version); // TODO: Uncomment when migrating to a new data schema
} else {
//query = query.where('version', '==', 'ALL');
}

if (crossGeo) {
// Cross-geo: single-month snapshot, all geographies included.
// Use 'end' param if provided, otherwise default to latest available date.
const snapshotDate = params.end || await getLatestDate(firestore, config.table);
query = query.where('date', '==', snapshotDate);
query = query.select('date', 'technology', 'geo', config.dataField);
} else {
// Normal time-series: filter by geo, apply date range, no geo in projection.
query = query.where('geo', '==', geoParam);

// Handle 'latest' date substitution
let startDate = params.start;
if (startDate === 'latest') {
startDate = await getLatestDate(firestore, config.table);
}

if (startDate) query = query.where('date', '>=', startDate);
if (params.end) query = query.where('date', '<=', params.end);

query = query.select('date', 'technology', config.dataField);
}

// Execute query
const snapshot = await query.get();
const data = [];
snapshot.forEach(doc => {
data.push(doc.data());
});

// Send response with ETag support
const jsonData = JSON.stringify(data);
const etag = generateETag(jsonData);
res.setHeader('ETag', `"${etag}"`);
if (!isModified(req, etag)) {
res.statusCode = 304;
res.end();
return;
}
res.statusCode = 200;
res.end(jsonData);

const data = await queryReport(reportType, { ...defaults, ...req.query });
sendJSONResponse(req, res, data);
} catch (error) {
handleControllerError(res, error, `fetching ${reportType} data`);
}
};
};

// Export individual controller functions
export const listAuditsData = createReportController('audits');
export const listAdoptionData = createReportController('adoption');
export const listCWVTechData = createReportController('cwv');
Expand Down
80 changes: 9 additions & 71 deletions src/controllers/technologiesController.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,75 +1,13 @@
import { firestore } from '../utils/db.js';
import { executeQuery, validateTechnologyArray, validateArrayParameter, FIRESTORE_IN_LIMIT } from '../utils/controllerHelpers.js';
import { handleControllerError, sendJSONResponse } from '../utils/controllerHelpers.js';
import { queryTechnologies } from '../utils/reportService.js';

/**
* List technologies with optional filtering and field selection
*/
const listTechnologies = async (req, res) => {
const queryBuilder = async (params) => {
/*
// Validate parameters
const supportedParams = ['technology', 'category', 'onlyname', 'fields'];
const providedParams = Object.keys(params);
const unsupportedParams = providedParams.filter(param => !supportedParams.includes(param));

if (unsupportedParams.length > 0) {
const error = new Error(`Unsupported parameters: ${unsupportedParams.join(', ')}.`);
error.statusCode = 400;
throw error;
}
*/

const isOnlyNames = params.onlyname || typeof params.onlyname === 'string';
const hasCustomFields = params.fields && !isOnlyNames;

let query = firestore.collection('technologies').orderBy('technology', 'asc');

// Apply technology filter with validation
const technologyParam = params.technology || 'ALL';
if (technologyParam !== 'ALL') {
const technologies = validateTechnologyArray(technologyParam);
if (technologies === null) {
throw new Error(`Too many technologies specified. Maximum ${FIRESTORE_IN_LIMIT} allowed.`);
}
if (technologies.length > 0) {
query = query.where('technology', 'in', technologies);
}
}

// Apply category filter with validation
if (params.category) {
const categories = validateArrayParameter(params.category, 'category');
if (categories.length > 0) {
query = query.where('category_obj', 'array-contains-any', categories);
}
}

// Apply field selection
if (isOnlyNames) {
query = query.select('technology');
} else if (hasCustomFields) {
const requestedFields = params.fields.split(',').map(f => f.trim());
query = query.select(...requestedFields);
} else {
query = query.select('technology', 'category', 'description', 'icon', 'origins');
}

return query;
};

const dataProcessor = (data, params) => {
const isOnlyNames = params.onlyname || typeof params.onlyname === 'string';

if (isOnlyNames) {
return data.map(item => item.technology);
}

return data;
};

await executeQuery(req, res, 'technologies', queryBuilder, dataProcessor);
try {
const data = await queryTechnologies(req.query);
sendJSONResponse(req, res, data);
} catch (error) {
handleControllerError(res, error, 'fetching technologies');
}
};

export {
listTechnologies
};
export { listTechnologies };
Loading