Update llms.adoc for proper markdown filenames and AI crawler permissions

.github/workflows/afdocs-checks.yml

@@ -27,7 +27,6 @@ jobs:
         with:
           site_name: redpanda-documentation
           max_timeout: 600
-          commit_ref: ${{ github.event.pull_request.head.sha }}
 
       - name: Run afdocs checks
         id: afdocs

antora-playbook.yml

@@ -2,7 +2,46 @@ site:
   title: Redpanda Docs
   start_page: home:ROOT:index.adoc
   url: https://docs.redpanda.com
-  robots: allow
+  robots: |
+    User-agent: *
+    Allow: /
+    Crawl-delay: 1
+
+    # OpenAI
+    User-agent: GPTBot
+    Allow: /
+
+    User-agent: ChatGPT-User
+    Allow: /
+
+    # Anthropic Claude (updated 2026)
+    User-agent: ClaudeBot
+    Allow: /
+
+    User-agent: Claude-User
+    Allow: /
+
+    User-agent: Claude-SearchBot
+    Allow: /
+
+    # Perplexity
+    User-agent: PerplexityBot
+    Allow: /
+
+    # Google AI
+    User-agent: Google-Extended
+    Allow: /
+
+    User-agent: GoogleOther
+    Allow: /
+
+    # Common Crawl
+    User-agent: CCBot
+    Allow: /
+
+    # Meta
+    User-agent: FacebookBot
+    Allow: /
   keys:
     announcement: true
     announcement-text: 'New! Redpanda Agentic Data Plane is here: a governed, high-performance foundation for AI agents.'
@@ -19,7 +58,8 @@ output:
   clean: true
 runtime:
   log:
-    level: info
+    level: warn
+  fetch: true
 content:
   sources:
   - url: .
@@ -50,8 +90,14 @@ asciidoc:
   - '@redpanda-data/docs-extensions-and-macros/asciidoc-extensions/add-line-numbers-highlights'
 antora:
   extensions:
+  - require: '@redpanda-data/docs-extensions-and-macros/extensions/git-full-clone'
+    # Configuration:
+    # unshallowTimeout: 120000  # Timeout in ms (default: 60000). Increase for very large repos
+    # skipUnshallow: false      # Set true for air-gapped environments
   - require: '@redpanda-data/docs-extensions-and-macros/extensions/generate-rp-connect-info'
   - require: '@redpanda-data/docs-extensions-and-macros/extensions/collect-bloblang-samples'
+  - require: '@redpanda-data/docs-extensions-and-macros/extensions/add-git-dates'
+  - require: '@redpanda-data/docs-extensions-and-macros/extensions/add-faq-structured-data'
   - require: '@redpanda-data/docs-extensions-and-macros/extensions/convert-to-markdown'
   - require: '@redpanda-data/docs-extensions-and-macros/extensions/generate-index-data'
     data:
@@ -104,6 +150,7 @@ antora:
             - '**/docker-compose.yaml'
             - '**/docker-compose.yml'
   - '@redpanda-data/docs-extensions-and-macros/extensions/convert-llms-to-txt'
+  - require: '@redpanda-data/docs-extensions-and-macros/extensions/convert-sitemap-to-markdown'
   - require: '@redpanda-data/docs-extensions-and-macros/extensions/validate-attributes'
   - require: '@redpanda-data/docs-extensions-and-macros/extensions/find-related-docs'
   - require: '@redpanda-data/docs-extensions-and-macros/extensions/find-related-labs'

home/modules/ROOT/pages/index.adoc

@@ -1,3 +1,12 @@
 = Welcome to Redpanda Documentation
 :page-role: home
-:description: Home page for the Redpanda docs site.
+:description: Home page for the Redpanda docs site.
+:page-faq-1-question: What is Redpanda?
+:page-faq-1-answer: Redpanda is a streaming data platform and Agentic Data Plane for developers. It's Kafka API-compatible but simpler to deploy and operate, designed for modern applications and AI agents requiring real-time data processing without the operational complexity of traditional platforms.
+:page-faq-2-question: How do I get started with Redpanda?
+:page-faq-2-answer: You can get started by installing Redpanda locally with Docker, deploying on Kubernetes, or signing up for Redpanda Cloud for a fully-managed service. Our quickstart guides walk you through creating your first topic and producing/consuming messages in minutes.
+:page-faq-2-anchor: get-started
+:page-faq-3-question: Is Redpanda compatible with Apache Kafka?
+:page-faq-3-answer: Yes, Redpanda is fully compatible with the Apache Kafka API. You can use existing Kafka clients, tools, and applications with Redpanda without any code changes, making migration seamless.
+:page-faq-4-question: What is an Agentic Data Plane?
+:page-faq-4-answer: An Agentic Data Plane is infrastructure that enables AI agents to access, process, and act on real-time data streams. Redpanda provides the high-performance streaming foundation that AI agents need to interact with live data, execute workflows, and coordinate actions across systems.

home/modules/ROOT/pages/llms.adoc

@@ -2,17 +2,55 @@
 
 > The Redpanda documentation site provides comprehensive resources for understanding, deploying, and managing Redpanda, a high-performance Kafka®-compatible streaming data platform. It includes guides, API references, tutorials, and best practices for self-managed and cloud deployments.
 
-## Access Markdown content
+## Discovery
 
-This site uses the indexify convention. All documentation pages are stored as `index.md` files in their respective directories. You can access markdown content in two ways:
+**For AI agents and LLMs:** This documentation is optimized for AI consumption. If you're an AI agent trying to help with Redpanda, start here.
 
-1. **Content negotiation**: Request any HTML URL with the `Accept: text/markdown` header to receive the markdown source.
-2. **Direct access**: Append `/index.md` to any directory URL (for example `/current/home/index.md`).
+### Find Redpanda documentation
 
-Common markdown URLs:
-- Redpanda Self-Managed: {site-url}/current/home/index.md[{site-url}/current/home/index.md]
-- Redpanda Cloud: {site-url}/redpanda-cloud/home/index.md[{site-url}/redpanda-cloud/home/index.md]
-- Redpanda Connect: {site-url}/redpanda-connect/home/index.md[{site-url}/redpanda-connect/home/index.md]
+**Primary discovery methods:**
+
+1. **This file**: `{site-url}/llms.txt` - You're reading it! This follows the llms.txt standard and provides a curated overview with links to comprehensive resources.
+
+2. **Sitemap**: `{site-url}/sitemap.md` - Contains all documentation pages including this file and all exports. The sitemap index includes `{site-url}/sitemap-llms.md` with all AI-optimized exports.
+
+3. **MCP server**: `https://docs.redpanda.com/mcp` - For Claude Code and MCP-compatible agents, this provides interactive documentation tools.
+
+### Interactive documentation tools (MCP Server)
+
+**https://docs.redpanda.com/mcp** - Model Context Protocol server
+
+Redpanda provides a remote Model Context Protocol (MCP) server that lets you access authoritative Redpanda documentation directly from your IDE or AI tool, such as Cursor, VS Code, ChatGPT, or Claude.
+
+The MCP server is hosted at: https://docs.redpanda.com/mcp. You can add this endpoint to any AI agent that supports MCP.
+
+For more details, see {site-url}/home/mcp-setup.md
+
+### Static documentation exports
+
+**For agents without MCP support**, we provide comprehensive static exports:
+
+- **{site-url}/llms-full.txt**: Complete documentation export with all components and versions (25MB+)
+- **Component-specific exports**: Focused documentation for individual products:
+  * **{site-url}/ROOT-full.txt**: Redpanda Self-Managed documentation
+  * **{site-url}/redpanda-cloud-full.txt**: Redpanda Cloud documentation
+  * **{site-url}/redpanda-connect-full.txt**: Redpanda Connect documentation
+  * **{site-url}/redpanda-labs-full.txt**: Labs and tutorials
+
+### Individual Markdown pages
+
+**Every HTML page has a markdown equivalent:**
+
+1. **Direct access**: Replace `.html` with `.md` in any URL
+   - Example: `{site-url}/current/home.html` → `{site-url}/current/home.md`
+
+2. **Content negotiation**: Request any URL with `Accept: text/markdown` header
+
+**Markdown features:**
+
+- YAML frontmatter with page metadata (title, description, categories, topic type, personas)
+- Git dates (accurate created/modified dates from commit history)
+- GitHub-Flavored Markdown format
 
 ## Overview
 
@@ -21,62 +59,57 @@ Redpanda docs serve both new and experienced users, offering detailed technical
 - **{site-url}/current.md[Redpanda Self-Managed]:** Deploy and manage Redpanda in your environment for ultimate control, high performance, and fault tolerance with Kafka compatibility and advanced enterprise features like Tiered Storage.
 - **{site-url}/redpanda-cloud.md[Redpanda Cloud]:** Simplify streaming with a fully-managed service: Serverless, Bring Your Own Cloud (BYOC), or Dedicated. Serverless is the fastest and easiest way to start data streaming. Choose BYOC for full data sovereignty.
 - **{site-url}/redpanda-connect.md[Redpanda Connect]:** Effortlessly integrate data across systems with hundreds of prebuilt connectors, change data capture (CDC) capabilities, and YAML-configurable workflows. Available as a standalone solution or as part of Redpanda Self-Managed, Redpanda Connect empowers organizations to build scalable, high-performance data pipelines with minimal complexity.
-- **{site-url}/redpanda-cloud/ai-agents/index.md[Redpanda Agentic Data Plane]:** Enterprise-grade infrastructure for building, deploying, and governing AI agents at scale. Provides Model Context Protocol (MCP) integration, declarative agent configuration, and compliance-grade audit trails for AI agents that need direct access to databases, queues, and business systems.
-- **{site-url}/current/console/index.md[Redpanda Console]:** Streamline operations with an intuitive Kafka UI for managing topics, partitions, and consumer groups, providing visibility and control over your data pipeline.
+- **{site-url}/redpanda-cloud/ai-agents.md[Redpanda Agentic Data Plane]:** Enterprise-grade infrastructure for building, deploying, and governing AI agents at scale. Provides Model Context Protocol (MCP) integration, declarative agent configuration, and compliance-grade audit trails for AI agents that need direct access to databases, queues, and business systems.
+- **{site-url}/current/console.md[Redpanda Console]:** Streamline operations with an intuitive Kafka UI for managing topics, partitions, and consumer groups, providing visibility and control over your data pipeline.
 
 ## Core documentation
 
-- {site-url}/current/get-started/quickstarts/index.md[Get started]: Fast-track your journey with Redpanda Self-Managed.
-- {site-url}/current/get-started/architecture/index.md[Redpanda concepts]: Deep-dive into key concepts like Tiered Storage, partitions, replication, and data streaming architecture.
+- {site-url}/current/get-started/quickstarts.md[Get started]: Fast-track your journey with Redpanda Self-Managed by deploying a complete example in Docker.
+- {site-url}/current/get-started/architecture.md[Redpanda concepts]: Deep-dive into key concepts like Tiered Storage, partitions, replication, and data streaming architecture.
 
 ## Deployment guides
 
-- {site-url}/current/deploy/index.md[Redpanda Self-Managed]: Install and configure Redpanda and Redpanda Console in self-hosted environments, including production best practices. Choose from Linux (bare-metal) or Kubernetes deployment instructions.
-- {site-url}/redpanda-cloud/get-started/cluster-types/index.md[Redpanda Cloud]: Guides for using Redpanda Cloud across Serverless, BYOC, and Dedicated options.
+- {site-url}/current/deploy.md[Redpanda Self-Managed]: Install and configure Redpanda and Redpanda Console in self-hosted environments, including production best practices. Choose from Linux (bare-metal) or Kubernetes deployment instructions.
+- {site-url}/redpanda-cloud/get-started/cluster-types.md[Redpanda Cloud]: Guides for using Redpanda Cloud across Serverless, BYOC, and Dedicated options.
 - {site-url}/redpanda-connect.md[Redpanda Connect]: Standalone documentation for deploying and managing Redpanda Connect, with or without Redpanda.
 
-## rpk CLI
+## Redpanda CLI
 
 Learn about `rpk`, the comprehensive command-line interface tool for managing Redpanda clusters.
 
-- {site-url}/current/reference/rpk/rpk-commands/index.md[rpk documentation for Redpanda Self-Managed]
-- {site-url}/redpanda-cloud/reference/rpk/rpk-commands/index.md[rpk documentation for Redpanda Cloud]
+- {site-url}/current/reference/rpk/rpk-commands.md[rpk documentation for Redpanda Self-Managed]
+- {site-url}/redpanda-cloud/reference/rpk/rpk-commands.md[rpk documentation for Redpanda Cloud]
 
 ## Redpanda Console
 
-- {site-url}/current/console/index.md[About Redpanda Console]: Learn about Redpanda's Kafka UI for managing topics, partitions, and consumer groups.
-- {site-url}/current/manage/console/index.md[Configure Redpanda Console]: Learn how to configure Redpanda Console for your needs.
+- {site-url}/current/console.md[About Redpanda Console]: Learn about Redpanda's Kafka UI for managing topics, partitions, and consumer groups.
+- {site-url}/current/manage/console.md[Configure Redpanda Console]: Learn how to configure Redpanda Console for your needs.
 
 ## AI agents and Agentic Data Plane
 
 Build and deploy production-ready AI agents with enterprise governance and compliance.
 
-- {site-url}/redpanda-cloud/ai-agents/adp-overview/index.md[Agentic Data Plane Overview]: Enterprise infrastructure for AI agents with built-in governance, observability, and compliance.
-- {site-url}/redpanda-cloud/ai-agents/mcp/index.md[Model Context Protocol (MCP)]: Give AI agents direct access to databases, queues, CRMs, and business systems without custom integration code.
-- {site-url}/redpanda-cloud/ai-agents/ai-gateway/index.md[AI Gateway]: Centralized gateway for AI agent interactions with intelligent routing, caching, and cost controls.
-- {site-url}/redpanda-cloud/ai-agents/agents/index.md[Building AI Agents]: Declarative agent configuration using Redpanda Connect connectors - no custom agent code required.
-- {site-url}/redpanda-cloud/ai-agents/observability/index.md[Agent Observability]: Track agent decisions, costs, and compliance with comprehensive audit trails.
+- {site-url}/redpanda-cloud/ai-agents/adp-overview.md[Agentic Data Plane Overview]: Enterprise infrastructure for AI agents with built-in governance, observability, and compliance.
+- {site-url}/redpanda-cloud/ai-agents/mcp.md[Model Context Protocol (MCP)]: Give AI agents direct access to databases, queues, CRMs, and business systems without custom integration code.
+- {site-url}/redpanda-cloud/ai-agents/ai-gateway.md[AI Gateway]: Centralized gateway for AI agent interactions with intelligent routing, caching, and cost controls.
+- {site-url}/redpanda-cloud/ai-agents/agents.md[Building AI Agents]: Declarative agent configuration using Redpanda Connect connectors - no custom agent code required.
+- {site-url}/redpanda-cloud/ai-agents/observability.md[Agent Observability]: Track agent decisions, costs, and compliance with comprehensive audit trails.
 
 ## API references
 
-- {site-url}/api/doc/admin.md[Admin API]: Manage and monitor Redpanda clusters using this REST API, including operations for configuration, partitions, and node health.
-- {site-url}/api/doc/schema-registry.md[Schema Registry API]: Use Redpanda's Kafka-compatible schema registry for managing and evolving schemas in your data streams.
-- {site-url}/api/doc/http-proxy.md[HTTP Proxy API]: Simplify interactions with Kafka topics and partitions with Redpanda's HTTP Proxy API, enabling easier data production and consumption over HTTP.
-- {site-url}/api/doc/cloud-controlplane.md[Cloud API]: Access and manage Redpanda Cloud resources programmatically, including provisioning clusters, monitoring usage, and configuring deployments.
+- {site-url}/api/admin.md[Admin API]: Manage and monitor Redpanda clusters using this REST API, including operations for configuration, partitions, and node health.
+- {site-url}/api/schema-registry.md[Schema Registry API]: Use Redpanda's Kafka-compatible schema registry for managing and evolving schemas in your data streams.
+- {site-url}/api/http-proxy.md[HTTP Proxy API]: Simplify interactions with Kafka topics and partitions with Redpanda's HTTP Proxy API, enabling easier data production and consumption over HTTP.
+- {site-url}/api/cloud-controlplane.md[Cloud API]: Access and manage Redpanda Cloud resources programmatically, including provisioning clusters, monitoring usage, and configuring deployments.
 
 ## Troubleshooting
 
-- {site-url}/current/troubleshoot/index.md[Troubleshooting guide]: Resolve common issues with installation, configuration, and performance.
-- {site-url}/current/troubleshoot/debug-bundle/index.md[Debug bundles]: Generate and analyze debug bundles for diagnosing cluster issues.
+- {site-url}/current/troubleshoot.md[Troubleshooting guide]: Resolve common issues with installation, configuration, and performance.
+- {site-url}/current/troubleshoot/debug-bundle.md[Debug bundles]: Generate and analyze debug bundles for diagnosing cluster issues.
 
 ## Self-Managed versioning
 
 The site includes versioned documentation for **Redpanda Self-Managed**:
 - **Latest version**: The most up-to-date documentation is available at {site-url}/current.md[{site-url}/current.md].
-- **Previous versions**: Previous versions can be accessed at `/{version_number}/home/index.md` (for example `{site-url}/24.1/home/index.md`).
-
-## Related resources
-
-- https://www.redpanda.com[Redpanda website]: Overview of Redpanda's platform and products.
-- {site-url}/sitemap.xml[Documentation sitemap]: Complete sitemap index for all documentation.
+- **Previous versions**: Previous versions can be accessed by replacing `current` with the version number in any URL (for example `{site-url}/<version>/home.md`).
 

local-antora-playbook.yml

@@ -42,8 +42,12 @@ asciidoc:
   - '@redpanda-data/docs-extensions-and-macros/asciidoc-extensions/add-line-numbers-highlights'
 antora:
   extensions:
+  - require: '@redpanda-data/docs-extensions-and-macros/extensions/git-full-clone'
+  - require: '@redpanda-data/docs-extensions-and-macros/extensions/generate-rp-connect-info'
   - require: '@redpanda-data/docs-extensions-and-macros/extensions/generate-rp-connect-categories'
   - require: '@redpanda-data/docs-extensions-and-macros/extensions/collect-bloblang-samples'
+  - require: '@redpanda-data/docs-extensions-and-macros/extensions/add-git-dates'
+  - require: '@redpanda-data/docs-extensions-and-macros/extensions/add-faq-structured-data'
   - require: '@redpanda-data/docs-extensions-and-macros/extensions/convert-to-markdown'
   - require: '@redpanda-data/docs-extensions-and-macros/extensions/process-context-switcher'
   - require: '@redpanda-data/docs-extensions-and-macros/extensions/generate-index-data'
@@ -78,7 +82,6 @@ antora:
       theme_variables:
         line_color: '#e2401b'
         font_family: Inter, sans-serif
-  - require: '@redpanda-data/docs-extensions-and-macros/extensions/generate-rp-connect-info'
   - require: '@redpanda-data/docs-extensions-and-macros/extensions/modify-redirects'
   - require: '@redpanda-data/docs-extensions-and-macros/extensions/unpublish-pages'
   - require: '@redpanda-data/docs-extensions-and-macros/extensions/unlisted-pages'
@@ -93,7 +96,8 @@ antora:
           file_patterns:
             - '**/docker-compose.yaml'
             - '**/docker-compose.yml'
-  - require: '@redpanda-data/docs-extensions-and-macros/extensions/convert-llms-to-txt.js'
+  - require: '@redpanda-data/docs-extensions-and-macros/extensions/convert-llms-to-txt'
+  - require: '@redpanda-data/docs-extensions-and-macros/extensions/convert-sitemap-to-markdown'
   - require: '@redpanda-data/docs-extensions-and-macros/extensions/validate-attributes'
   - require: '@redpanda-data/docs-extensions-and-macros/extensions/find-related-docs'
   - require: '@redpanda-data/docs-extensions-and-macros/extensions/find-related-labs'