Track events for markdown page views, llms.txt, RSS, MCP connections

.changeset/bold-points-flow.md

@@ -0,0 +1,5 @@
+---
+"gitbook": patch
+---
+
+Track new events for site analytics.

packages/gitbook/src/app/sites/static/[mode]/[siteURL]/[siteData]/llms.txt/route.ts

@@ -10,5 +10,6 @@ export async function GET(
     { params }: { params: Promise<RouteLayoutParams> }
 ) {
     const { context } = await getStaticSiteContext(await params);
+
     return serveLLMsTxt(context, { withMarkdownPages: true });
 }

packages/gitbook/src/app/sites/static/[mode]/[siteURL]/[siteData]/~gitbook/mcp/route.ts

@@ -10,13 +10,26 @@ import { createMcpHandler } from 'mcp-handler';
 import type { NextRequest } from 'next/server';
 import { z } from 'zod';
 
-async function handler(
-    nextRequest: NextRequest,
-    { params }: { params: Promise<RouteLayoutParams> }
-) {
+async function handler(request: NextRequest, { params }: { params: Promise<RouteLayoutParams> }) {
     const { context } = await getStaticSiteContext(await params);
     const { dataFetcher, linker, site } = context;
 
+    waitUntil(
+        trackServerInsightsEvents({
+            organizationId: context.organizationId,
+            siteId: context.site.id,
+            events: [
+                {
+                    type: 'mcp_request',
+                    location: {
+                        displayContext: SiteInsightsDisplayContext.Server,
+                    },
+                },
+            ],
+            request,
+        })
+    );
+
     const mcpHandler = createMcpHandler(
         (server) => {
             server.tool(
@@ -49,7 +62,7 @@ async function handler(
                                     },
                                 },
                             ],
-                            request: nextRequest,
+                            request,
                         })
                     );
 
@@ -118,10 +131,9 @@ async function handler(
     const requestURL = new URL(
         context.linker.toAbsoluteURL(context.linker.toPathInSite('~gitbook/mcp'))
     );
-    requestURL.search = nextRequest.nextUrl.search;
+    requestURL.search = request.nextUrl.search;
 
-    const request = new Request(requestURL, nextRequest);
-    return mcpHandler(request);
+    return mcpHandler(new Request(requestURL, request));
 }
 
 export { handler as GET, handler as POST };

packages/gitbook/src/lib/data/errors.ts

@@ -181,7 +181,7 @@ export function extractCacheControl(error: GitBookAPIError) {
  * Get a data fetcher exposable error from a JS error.
  * This function should never throw, even if the error is not in the expected format. In that case, it should return a generic error with code 500.
  */
-export function getExposableError(error: Error): DataFetcherErrorData {
+export function getExposableError(error: unknown): DataFetcherErrorData {
     if (error instanceof GitBookAPIError) {
         const cache = extractCacheControl(error);
 

packages/gitbook/src/lib/markdownPage.ts

@@ -1,6 +1,6 @@
 import type { GitBookSiteContext } from '@/lib/context';
-import type { DataFetcherResponse } from '@/lib/data';
-import { resolvePagePathDocumentOrGroup } from '@/lib/pages';
+import { DataFetcherError } from '@/lib/data';
+import type { ResolvedPagePath } from '@/lib/pages';
 import { getIndexablePages } from '@/lib/sitemap';
 import { getMarkdownForPagesTree } from '@/routes/llms';
 import { type RevisionPageDocument, type RevisionPageGroup, RevisionPageType } from '@gitbook/api';
@@ -14,37 +14,22 @@ import { gfm } from 'micromark-extension-gfm';
 import { remove } from 'unist-util-remove';
 import { type GitBookLinker, relativeToAbsoluteLinks } from './links';
 
-type MarkdownResult = DataFetcherResponse<string>;
-
 /**
  * Generate a markdown version of a page.
  * Handles both regular document pages and group pages (pages with child pages).
  */
 export async function getMarkdownForPage(
     context: GitBookSiteContext,
-    pagePath: string
-): Promise<MarkdownResult> {
-    const pageLookup = resolvePagePathDocumentOrGroup(context.revision.pages, pagePath);
-
-    if (!pageLookup) {
-        return {
-            error: {
-                message: `Page "${pagePath}" not found`,
-                code: 404,
-            },
-        };
-    }
-
+    pageLookup: ResolvedPagePath<RevisionPageDocument | RevisionPageGroup>
+): Promise<string> {
     const { page } = pageLookup;
 
     // Only handle documents and groups
     if (page.type !== RevisionPageType.Document && page.type !== RevisionPageType.Group) {
-        return {
-            error: {
-                message: `Page "${pagePath}" is not a document or group`,
-                code: 400,
-            },
-        };
+        throw new DataFetcherError(
+            `Page "${pageLookup.page.title}" is not a document or group`,
+            400
+        );
     }
 
     // Handle group pages
@@ -59,12 +44,7 @@ export async function getMarkdownForPage(
     });
 
     if (error) {
-        return {
-            error: {
-                message: 'An error occurred while fetching the markdown for this page',
-                code: 500,
-            },
-        };
+        throw error;
     }
 
     const tree = fromPageMarkdown({
@@ -78,7 +58,7 @@ export async function getMarkdownForPage(
         return servePageGroup(context, page);
     }
 
-    return { data: toPageMarkdown(tree) };
+    return toPageMarkdown(tree);
 }
 
 /**
@@ -150,15 +130,10 @@ function isEmptyMarkdownPage(tree: Root): boolean {
 async function servePageGroup(
     context: GitBookSiteContext,
     page: RevisionPageDocument | RevisionPageGroup
-): Promise<MarkdownResult> {
+): Promise<string> {
     const siteSpaceUrl = context.space.urls.published;
     if (!siteSpaceUrl) {
-        return {
-            error: {
-                message: `Page "${page.title}" is not published`,
-                code: 404,
-            },
-        };
+        throw new DataFetcherError(`Page "${page.title}" is not published`, 404);
     }
 
     const indexablePages = getIndexablePages(page.pages);
@@ -180,9 +155,7 @@ async function servePageGroup(
         ],
     };
 
-    return {
-        data: toMarkdown(markdownTree, {
-            bullet: '-',
-        }),
-    };
+    return toMarkdown(markdownTree, {
+        bullet: '-',
+    });
 }

packages/gitbook/src/lib/pages.ts

@@ -8,7 +8,7 @@ import {
 
 export type AncestorRevisionPage = RevisionPageDocument | RevisionPageGroup;
 
-type ResolvedPagePath<Page extends RevisionPageDocument | RevisionPageGroup> = {
+export type ResolvedPagePath<Page extends RevisionPageDocument | RevisionPageGroup> = {
     page: Page;
     ancestors: AncestorRevisionPage[];
 };

packages/gitbook/src/lib/tracking.ts

@@ -2,6 +2,7 @@ import type * as api from '@gitbook/api';
 import type { headers as nextHeaders } from 'next/headers';
 import { apiClient } from './data/api';
 import { GITBOOK_DISABLE_TRACKING } from './env';
+import { getLogger } from './logger';
 
 /**
  * Return true if events should be tracked on the site.
@@ -74,6 +75,12 @@ export async function trackServerInsightsEvents(args: {
     events: ServerInsightsEventInput[];
     request: Request;
 }) {
+    const logger = getLogger().subLogger('tracking');
+
+    logger.info(
+        `Tracking ${args.events.length} events for site ${args.siteId} (enabled=${!GITBOOK_DISABLE_TRACKING})`
+    );
+
     if (GITBOOK_DISABLE_TRACKING) {
         return;
     }

packages/gitbook/src/middleware.ts

@@ -1,4 +1,8 @@
-import { CustomizationThemeMode } from '@gitbook/api';
+import {
+    CustomizationThemeMode,
+    SiteInsightsDisplayContext,
+    SiteInsightsLLMSVariant,
+} from '@gitbook/api';
 import Negotiator from 'negotiator';
 import type { NextRequest } from 'next/server';
 import { NextResponse } from 'next/server';
@@ -36,9 +40,14 @@ import {
     normalizeVisitorURL,
     serveVisitorClaimsDataRequest,
 } from '@/lib/visitors';
+import { waitUntil } from '@/lib/waitUntil';
 import { serveResizedImage } from '@/routes/image';
 import { cookies } from 'next/headers';
-import { serveProxyAnalyticsEvent } from './lib/tracking';
+import {
+    type ServerInsightsEventInput,
+    serveProxyAnalyticsEvent,
+    trackServerInsightsEvents,
+} from './lib/tracking';
 export const config = {
     matcher: [
         '/((?!_next/static|_next/image|~gitbook/static|~gitbook/revalidate|~gitbook/monitoring|~scalar/proxy).*)',
@@ -411,12 +420,24 @@ async function serveSiteRoutes(requestURL: URL, request: NextRequest) {
         requestHeaders.set('origin', request.nextUrl.origin);
 
         const siteURLWithoutProtocol = `${siteCanonicalURL.host}${siteURLData.basePath}`;
-        const { pathname, routeType: routeTypeFromPathname } = encodePathInSiteContent(
-            siteURLData.pathname,
-            request
-        );
+        const {
+            pathname,
+            routeType: routeTypeFromPathname,
+            events,
+        } = encodePathInSiteContent(siteURLData.pathname, request);
         routeType = routeTypeFromPathname ?? routeType;
 
+        if (events && events.length > 0) {
+            waitUntil(
+                trackServerInsightsEvents({
+                    organizationId: siteURLData.organization,
+                    siteId: siteURLData.site,
+                    events,
+                    request,
+                })
+            );
+        }
+
         const route = [
             'sites',
             routeType,
@@ -633,6 +654,7 @@ function encodePathInSiteContent(
 ): {
     pathname: string;
     routeType?: 'static' | 'dynamic';
+    events?: ServerInsightsEventInput[] | undefined;
 } {
     const pathname = removeLeadingSlash(removeTrailingSlash(rawPathname));
 
@@ -646,12 +668,32 @@ function encodePathInSiteContent(
         return {
             pathname: `~gitbook/rss/${encodePagePath(rssMatch[2])}`,
             routeType: 'static',
+            events: [
+                {
+                    type: 'rss_request',
+                    location: {
+                        displayContext: SiteInsightsDisplayContext.Server,
+                    },
+                },
+            ],
         };
     }
 
     // We skip encoding for paginated llms-full.txt pages (i.e. llms-full.txt/100)
     if (pathname.match(LLMS_FULL_PATH_REGEX)) {
-        return { pathname, routeType: 'static' };
+        return {
+            pathname,
+            routeType: 'static',
+            events: [
+                {
+                    type: 'llms_request',
+                    llmsVariant: SiteInsightsLLMSVariant.Full,
+                    location: {
+                        displayContext: SiteInsightsDisplayContext.Server,
+                    },
+                },
+            ],
+        };
     }
 
     // If the pathname is an embedded page
@@ -667,16 +709,32 @@ function encodePathInSiteContent(
         case '~gitbook/embed/assistant':
         case '~gitbook/icon':
             return { pathname };
-        case '~gitbook/mcp':
+        // LLMs.txt, sitemap, sitemap-pages and robots.txt are always static
+        // as they only depend on the site structure / pages.
         case 'llms.txt':
         case 'llms-full.txt':
+            return {
+                pathname,
+                routeType: 'static',
+                events: [
+                    {
+                        type: 'llms_request',
+                        llmsVariant:
+                            pathname === 'llms.txt'
+                                ? SiteInsightsLLMSVariant.Standard
+                                : SiteInsightsLLMSVariant.Full,
+                        location: {
+                            displayContext: SiteInsightsDisplayContext.Server,
+                        },
+                    },
+                ],
+            };
+        case '~gitbook/mcp':
         case 'sitemap.xml':
         case 'sitemap-pages.xml':
         case 'robots.txt':
         case '~gitbook/embed/script.js':
         case '~gitbook/embed/demo':
-            // LLMs.txt, sitemap, sitemap-pages and robots.txt are always static
-            // as they only depend on the site structure / pages.
             return { pathname, routeType: 'static' };
         case '~gitbook/pdf':
         case '~gitbook/search':
@@ -693,6 +751,16 @@ function encodePathInSiteContent(
                     pathname: `~gitbook/markdown/${encodePagePath(pagePathWithoutMD)}`,
                     // The markdown content is always static and doesn't depend on the dynamic parameter (customization, theme, etc)
                     routeType: 'static',
+                    events: [
+                        {
+                            type: 'page_markdown_request',
+                            // TODO: track pageId / spaceId when possible
+                            // We don't do it at the moment as we can't easily extract it from the URL.
+                            location: {
+                                displayContext: SiteInsightsDisplayContext.Server,
+                            },
+                        },
+                    ],
                 };
             }
             return { pathname: encodePagePath(pathname) };