Add AI-friendly meta tags and enhanced structured data

.github/workflows/validate-build.yml

@@ -13,6 +13,7 @@ jobs:
         uses: actions/setup-go@v5
         with:
           go-version: stable
+          cache-dependency-path: blobl-editor/wasm/go.sum
       - name: Lint Go code
         uses: golangci/golangci-lint-action@v8
         with:
@@ -26,7 +27,18 @@ jobs:
       - uses: actions/setup-node@v4
         with:
           node-version: 18
+          cache: 'npm'
+      - name: Cache generated files
+        uses: actions/cache@v4
+        with:
+          path: |
+            src/js/vendor/prism/prism-bloblang.js
+            src/static/blobl.wasm
+          key: generated-${{ hashFiles('blobl-editor/wasm/go.sum') }}-${{ hashFiles('package-lock.json') }}
+          restore-keys: |
+            generated-${{ hashFiles('blobl-editor/wasm/go.sum') }}-
+            generated-
       - name: Bundle UI
         run: |
-          npm i
+          npm ci
           gulp bundle

gulp.d/tasks/generate-bloblang-grammar.js

@@ -13,8 +13,9 @@ function fetchText (url) {
   return new Promise((resolve, reject) => {
     const options = {
       headers: { 'User-Agent': 'docs-ui-build' },
+      timeout: 10000, // 10 second timeout
     }
-    https.get(url, options, (res) => {
+    const req = https.get(url, options, (res) => {
       if (res.statusCode !== 200) {
         reject(new Error(`HTTP ${res.statusCode}`))
         res.resume()
@@ -23,7 +24,12 @@ function fetchText (url) {
       let data = ''
       res.on('data', (chunk) => { data += chunk })
       res.on('end', () => resolve(data))
-    }).on('error', reject)
+    })
+    req.on('error', reject)
+    req.on('timeout', () => {
+      req.destroy()
+      reject(new Error(`Request timeout after 10s: ${url}`))
+    })
   })
 }
 

preview-src/index.adoc

@@ -5,6 +5,7 @@
 :!example-caption:
 :!table-caption:
 :page-pagination:
+:page-has-markdown:
 
 {description}
 

src/helpers/is-beta-feature.js

@@ -1,36 +1,26 @@
 'use strict'
 
-// Cache for beta status lookups (cleared between components)
-const cache = new Map()
-let currentComponent = null
+// Cache: Map<componentName, Map<url, isBeta>>
+let urlCache = null
+let cachedComponent = null
 
 module.exports = (navUrl, { data: { root } }) => {
   const { contentCatalog, page } = root
   if (page.layout === '404') return false
   if (!contentCatalog) return false
 
-  // Reset cache if component changed
-  if (currentComponent !== page.component.name) {
-    cache.clear()
-    currentComponent = page.component.name
-  }
-
-  // Check cache first
-  if (cache.has(navUrl)) {
-    return cache.get(navUrl)
-  }
-
-  // Query and cache result
-  const pages = contentCatalog.findBy({ component: page.component.name, family: 'page' })
+  // Build URL map once per component (O(n) once, then O(1) lookups)
+  if (cachedComponent !== page.component.name) {
+    urlCache = new Map()
+    cachedComponent = page.component.name
 
-  for (const navGroup of pages) {
-    if (navGroup.pub.url === navUrl) {
-      const isBeta = !!navGroup.asciidoc.attributes['page-beta']
-      cache.set(navUrl, isBeta)
-      return isBeta
+    const pages = contentCatalog.findBy({ component: page.component.name, family: 'page' })
+    for (const p of pages) {
+      if (p.pub?.url) {
+        urlCache.set(p.pub.url, !!p.asciidoc?.attributes?.['page-beta'])
+      }
     }
   }
 
-  cache.set(navUrl, false)
-  return false
+  return urlCache.get(navUrl) || false
 }

src/helpers/is-enterprise.js

@@ -1,47 +1,25 @@
 'use strict'
 
-/* Put this in nav-tree.hbs
-{{#if navigation.length}}
-  <ul class="nav-list">
-    {{#each navigation}}
-      <li class="nav-item{{#if (eq ./url @root.page.url)}} is-current-page{{/if}}" data-depth="{{or ../level 0}}">
-        {{#if ./content}}
-          {{#if ./url}}
-            <a class="nav-link
-              {{~#if (is-enterprise ./url)}} enterprise{{/if}}
-*/
-
-// Cache for component pages (cleared between render cycles)
-const cache = new Map()
-let currentComponent = null
+// Cache: Map<url, isEnterprise>
+let urlCache = null
+let cachedComponent = null
 
 module.exports = (navUrl, { data: { root } }) => {
   const { contentCatalog, page } = root
-
-  // Reset cache if component changed
-  if (currentComponent !== page.component.name) {
-    cache.clear()
-    currentComponent = page.component.name
-  }
-
-  // Check cache first
-  if (cache.has(navUrl)) {
-    return cache.get(navUrl)
-  }
-
-  // Query and cache result
-  const pages = contentCatalog.findBy({ component: page.component.name, family: 'page' })
-
-  for (let i = 0; i < pages.length; i++) {
-    const isEnterprise = pages[i].pub.url === navUrl &&
-      pages[i].asciidoc.attributes['page-enterprise'] === 'true'
-
-    if (pages[i].pub.url === navUrl) {
-      cache.set(navUrl, isEnterprise)
-      return isEnterprise
+  if (!contentCatalog) return false
+
+  // Build URL map once per component (O(n) once, then O(1) lookups)
+  if (cachedComponent !== page.component.name) {
+    urlCache = new Map()
+    cachedComponent = page.component.name
+
+    const pages = contentCatalog.findBy({ component: page.component.name, family: 'page' })
+    for (const p of pages) {
+      if (p.pub?.url) {
+        urlCache.set(p.pub.url, p.asciidoc?.attributes?.['page-enterprise'] === 'true')
+      }
     }
   }
 
-  cache.set(navUrl, false)
-  return false
+  return urlCache.get(navUrl) || false
 }

src/helpers/is-limited-availability-feature.js

@@ -1,38 +1,33 @@
 'use strict'
 
-let navGroupCache = null
-let componentCache = null
+// Cache: Map<url, isLimitedAvailability>
+let urlCache = null
+let cachedComponent = null
 
 module.exports = (navUrl, { data: { root } }) => {
   const { contentCatalog, page } = root
   if (page.layout === '404') return false
 
   // Preview mode: contentCatalog doesn't contain preview pages
-  // Fall back to checking if this nav item is the current page
   if (!contentCatalog) {
     const currentPageUrl = page.url
-    // Check if this navigation URL matches the current page
     const isCurrentPage = navUrl === currentPageUrl ||
                           (navUrl && page.src && navUrl.endsWith(page.src.basename.replace('.adoc', '.html')))
-    // Return the current page's limited availability status
     return isCurrentPage && page.attributes && page.attributes['limited-availability']
   }
 
-  // Only perform lookup if caches are invalid or stale
-  if (!navGroupCache || componentCache !== page.component.name) {
-    navGroupCache = contentCatalog.findBy({ component: page.component.name, family: 'page' })
-    componentCache = page.component.name
-  }
+  // Build URL map once per component (O(n) once, then O(1) lookups)
+  if (cachedComponent !== page.component.name) {
+    urlCache = new Map()
+    cachedComponent = page.component.name
 
-  // Iterate through cached pages and check for limited availability status
-  for (const navGroup of navGroupCache) {
-    // Guard against missing properties
-    if (navGroup.pub && navGroup.pub.url === navUrl &&
-        navGroup.asciidoc && navGroup.asciidoc.attributes &&
-        navGroup.asciidoc.attributes['page-limited-availability']) {
-      return true
+    const pages = contentCatalog.findBy({ component: page.component.name, family: 'page' })
+    for (const p of pages) {
+      if (p.pub?.url) {
+        urlCache.set(p.pub.url, !!p.asciidoc?.attributes?.['page-limited-availability'])
+      }
     }
   }
 
-  return false
+  return urlCache.get(navUrl) || false
 }

src/helpers/json-safe.js

@@ -0,0 +1,71 @@
+'use strict'
+
+/**
+ * Decodes HTML entities to their actual characters.
+ * Handles both named entities (e.g., &) and numeric entities (e.g., ’)
+ *
+ * @param {string} str - String with HTML entities
+ * @returns {string} - Decoded string
+ */
+function decodeHtmlEntities (str) {
+  // Named entities mapped to their Unicode code points
+  const namedEntities = {
+    '&': '\u0026', // &
+    '&lt;': '\u003C', // <
+    '&gt;': '\u003E', // >
+    '&quot;': '\u0022', // "
+    '&apos;': '\u0027', // '
+    '&#39;': '\u0027', // '
+    '&nbsp;': '\u00A0', // non-breaking space
+    '&ndash;': '\u2013', // en dash
+    '&mdash;': '\u2014', // em dash
+    '&lsquo;': '\u2018', // left single quote
+    '&rsquo;': '\u2019', // right single quote
+    '&ldquo;': '\u201C', // left double quote
+    '&rdquo;': '\u201D', // right double quote
+    '&hellip;': '\u2026', // ellipsis
+    '&copy;': '\u00A9', // copyright
+    '&reg;': '\u00AE', // registered
+    '&trade;': '\u2122', // trademark
+  }
+
+  // Replace named entities
+  let result = str.replace(/&[a-zA-Z]+;/g, (match) => namedEntities[match] || match)
+
+  // Replace numeric entities (decimal: &#8217; and hex: &#x2019;)
+  result = result.replace(/&#(\d+);/g, (_, code) => String.fromCharCode(parseInt(code, 10)))
+  result = result.replace(/&#x([0-9a-fA-F]+);/g, (_, code) => String.fromCharCode(parseInt(code, 16)))
+
+  return result
+}
+
+/**
+ * Escapes a string for safe inclusion in JSON-LD script blocks.
+ *
+ * This helper addresses several issues when embedding dynamic content in JSON:
+ * 1. HTML entities are decoded to actual characters (e.g., &#8217; becomes right single quote)
+ * 2. Backslashes must be escaped to avoid invalid JSON
+ * 3. Double quotes must be escaped to avoid breaking JSON strings
+ * 4. </script> substrings must be escaped to avoid breaking out of script tags
+ *
+ * @param {string} value - The string to make JSON-safe
+ * @returns {string} - The JSON-safe string
+ */
+module.exports = (value) => {
+  if (!value || typeof value !== 'string') return ''
+
+  // First decode HTML entities to their actual characters
+  const decoded = decodeHtmlEntities(value)
+
+  return decoded
+    // Escape backslashes first (must be done before escaping quotes)
+    .replace(/\\/g, '\\\\')
+    // Escape double quotes
+    .replace(/"/g, '\\"')
+    // Escape </script> to prevent breaking out of script block
+    .replace(/<\/script>/gi, '<\\/script>')
+    // Escape other control characters that could break JSON
+    .replace(/\n/g, '\\n')
+    .replace(/\r/g, '\\r')
+    .replace(/\t/g, '\\t')
+}

src/helpers/markdown-url.js

@@ -30,9 +30,14 @@ module.exports = ({ data: { root } }) => {
 
   const url = pageInfo.pub.url
 
-  // If URL ends with / (indexify format), append index.md
+  // Handle root path special case
+  if (url === '/') {
+    return '/index.md'
+  }
+
+  // If URL ends with / (indexify format), convert to .md (without index)
   if (url.endsWith('/')) {
-    const result = `${url}index.md`
+    const result = `${url.slice(0, -1)}.md`
     return result
   }
 

src/helpers/page-attribute.js

@@ -0,0 +1,47 @@
+'use strict'
+
+/**
+ * Get a page attribute from contentCatalog
+ *
+ * This helper queries contentCatalog to access page.asciidoc.attributes,
+ * which contains AsciiDoc document attributes set in the page or by extensions.
+ *
+ * Special handling for intrinsic attributes:
+ * - "description": Falls back to page.description (Antora intrinsic property)
+ * - "keywords": Falls back to page.keywords (Antora intrinsic property)
+ *
+ * Usage: {{page-attribute "description"}} or {{page-attribute "page-topic-type"}}
+ */
+
+module.exports = (attributeName, { data: { root } }) => {
+  const { contentCatalog, page } = root
+
+  // Safety checks
+  if (!contentCatalog || !page || !page.component || page.version === undefined) {
+    return null
+  }
+
+  // Query contentCatalog for full page object
+  const pageInfo = contentCatalog.getById({
+    component: page.component.name,
+    version: page.version,
+    module: page.module,
+    family: 'page',
+    relative: page.relativeSrcPath,
+  })
+
+  // First try asciidoc.attributes
+  const attrValue = pageInfo?.asciidoc?.attributes?.[attributeName]
+  if (attrValue) return attrValue
+
+  // Fall back to intrinsic page properties for special attributes
+  // Antora stores :description: and :keywords: as intrinsic properties
+  if (attributeName === 'description' && page.description) {
+    return page.description
+  }
+  if (attributeName === 'keywords' && page.keywords) {
+    return page.keywords
+  }
+
+  return null
+}

src/partials/head-meta.hbs

@@ -2,6 +2,9 @@
 {{!-- Prevent old versions from being indexed by search engines --}}
 {{#if (or (is-prerelease page) site.keys.preview)}}
 <meta name="robots" content="noindex">
+{{else}}
+{{!-- AI-friendly indexing directives for production pages --}}
+<meta name="robots" content="index, follow, max-snippet:-1, max-image-preview:large, max-video-preview:-1">
 {{/if}}
 {{#with site.components.ROOT}}
 <meta name="latest-redpanda-version" content="{{this.latest.version}}">