{
  "version": "v7",
  "generated_utc": "2026-06-10T20:05:00Z",
  "public_safety": "Public-safe AI/LLM routing, technical SEO, schema, and claim-control maps. Internal lead lists, PPC spend/strategy, private competitor notes, prospect databases, and unpublished sales playbooks are excluded.",
  "purpose": "Crawler and answer-engine access policy for Depureco AI support files, balancing discoverability with thin-file indexing risk.",
  "official_sources": [
    {
      "authority": "OpenAI",
      "url": "https://developers.openai.com/api/docs/bots",
      "notes": "OpenAI documents OAI-SearchBot for ChatGPT search, GPTBot for model training crawls, and ChatGPT-User for user-triggered browsing."
    },
    {
      "authority": "Google Search Central",
      "url": "https://developers.google.com/search/docs/crawling-indexing/robots-meta-tag",
      "notes": "Google documents robots meta tags and X-Robots-Tag HTTP headers; non-HTML files should use X-Robots-Tag."
    },
    {
      "authority": "Google Search Central",
      "url": "https://developers.google.com/search/docs/crawling-indexing/sitemaps/overview",
      "notes": "Google says sitemaps help search engines discover important URLs and understand file relationships, but do not guarantee crawling or indexing."
    }
  ],
  "recommended_policy": {
    "business_goal": "Allow AI/search systems to discover Depureco public product/category/application context while preventing /ai/ support files from competing with money pages.",
    "root_files": [
      "/llms.txt",
      "/llms-full.txt",
      "/ai-sitemap.xml"
    ],
    "ai_support_files": [
      "/ai/*.md",
      "/ai/db/*.json"
    ],
    "headers": [
      {
        "path_pattern": "/ai/*.md",
        "header": "X-Robots-Tag",
        "value": "noindex, follow",
        "reason": "Markdown AI support docs should be crawlable but should not rank as landing pages."
      },
      {
        "path_pattern": "/ai/db/*.json",
        "header": "X-Robots-Tag",
        "value": "noindex, follow",
        "reason": "JSON database files should support retrieval and entity understanding, not rank directly."
      },
      {
        "path_pattern": "/llms.txt",
        "header": "Content-Type",
        "value": "text/plain; charset=utf-8",
        "reason": "Root LLM guide should resolve as plain text."
      },
      {
        "path_pattern": "/llms-full.txt",
        "header": "Content-Type",
        "value": "text/plain; charset=utf-8",
        "reason": "Full semantic guide should resolve as plain text."
      },
      {
        "path_pattern": "/ai-sitemap.xml",
        "header": "Content-Type",
        "value": "application/xml; charset=utf-8",
        "reason": "AI sitemap should parse as XML."
      }
    ],
    "robots_txt_recommendation": [
      {
        "line": "Sitemap: https://depurecousa.com/ai-sitemap.xml",
        "include_if": "ai-sitemap.xml is deployed and valid"
      },
      {
        "line": "User-agent: OAI-SearchBot\nAllow: /",
        "include_if": "Goal is visibility in ChatGPT search-style results"
      },
      {
        "line": "User-agent: GPTBot\nDisallow: /",
        "include_if": "Optional only if company chooses not to allow training crawls; do not use this to manage ChatGPT search visibility"
      },
      {
        "line": "User-agent: *\nAllow: /llms.txt\nAllow: /llms-full.txt\nAllow: /ai-sitemap.xml\nAllow: /ai/",
        "include_if": "Do not block /ai/ if you expect crawlers to read the support files; use X-Robots-Tag for noindex instead."
      }
    ]
  },
  "do_not_do": [
    "Do not Disallow /ai/ in robots.txt and also expect noindex headers to be seen; crawlers blocked by robots.txt may never request the file and therefore may never see X-Robots-Tag.",
    "Do not link AI database files in the main navigation.",
    "Do not place private prospecting/competitor/PPC files in /ai/."
  ]
}