# robots.txt for Cognitive Industries
#
# Policy, in one line: discovery is welcome, abuse is not.
#
# This file is a courtesy protocol for well-behaved crawlers. It is not a
# security control, and it is not treated as one. A client that ignores it is
# handled at the edge on behaviour and reputation, not on the string it puts in
# its User-Agent header. The reasoning is written up in full at
# https://cognitive-industries.org/docs/crawler-policy/
#
# If you operate a legitimate crawler and something here is in your way, email
# contact@cognitive-industries.org rather than working around it. We would rather adjust a rule.

# ── Default: everything public is open, including to AI crawlers ──────────────
User-agent: *
Allow: /
# Auth-gated surfaces. Nothing useful to index and nothing public behind them.
Disallow: /admin.html
Disallow: /hub
Disallow: /hub.html
# The one-time deterrence notice links. Fetching one burns it; they are not
# content and they are never meant to be indexed.
Disallow: /notice/
# Search and filter parameters produce infinite near-duplicate URLs. The
# canonical tag handles it too, but not crawling them at all is cheaper.
Disallow: /*?*q=
Disallow: /*?*search=
Disallow: /*?*utm_
Disallow: /*?*fbclid=
Disallow: /*?*gclid=
Disallow: /*?*ref=
Disallow: /*?*session=
Disallow: /*?*sid=
# Site-view override params. Same document either way, so they are duplicates.
Disallow: /*?*mode=
Disallow: /*?*choose
# Attack-toolkit paths that have never existed here. Listed so a scanner that
# reads robots.txt has no excuse, and so the edge log is unambiguous.
Disallow: /wp-admin/
Disallow: /wp-login.php
Disallow: /xmlrpc.php
Disallow: /.env
Disallow: /.git/
Disallow: /phpmyadmin/
Disallow: /cgi-bin/
# JS, CSS, images and fonts must stay crawlable. Google renders the page and
# blocking them breaks the rendered evaluation. Nothing below is disallowed.
Allow: /css/
Allow: /js/
Allow: /assets/
Allow: /og/
Crawl-delay: 1

# ── Search engines ── explicitly welcome, no crawl delay
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Googlebot-News
User-agent: Storebot-Google
User-agent: Bingbot
User-agent: Applebot
User-agent: DuckDuckBot
User-agent: YandexBot
User-agent: Baiduspider
User-agent: Slurp
User-agent: SeznamBot
User-agent: Qwantbot
User-agent: Neevabot
Allow: /
Disallow: /admin.html
Disallow: /hub
Disallow: /hub.html
Disallow: /notice/

# ── AI and AI-search ── explicitly welcome, no crawl delay
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: anthropic-ai
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: Amazonbot
User-agent: Bytespider
User-agent: cohere-ai
User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalFetcher
User-agent: facebookexternalhit
User-agent: DuckAssistBot
User-agent: MistralAI-User
User-agent: YouBot
User-agent: Diffbot
User-agent: omgili
User-agent: Timpibot
Allow: /
Disallow: /admin.html
Disallow: /hub
Disallow: /hub.html
Disallow: /notice/

# ── Archival and research ── explicitly welcome, no crawl delay
User-agent: CCBot
User-agent: ia_archiver
User-agent: archive.org_bot
User-agent: Internetarchive
User-agent: Wayback
User-agent: SemrushBot-SI
Allow: /
Disallow: /admin.html
Disallow: /hub
Disallow: /hub.html
Disallow: /notice/

# ── Social preview ── explicitly welcome, no crawl delay
User-agent: Twitterbot
User-agent: LinkedInBot
User-agent: Slackbot
User-agent: Slackbot-LinkExpanding
User-agent: Discordbot
User-agent: TelegramBot
User-agent: WhatsApp
User-agent: Pinterestbot
User-agent: redditbot
User-agent: SkypeUriPreview
User-agent: Embedly
User-agent: Iframely
User-agent: vkShare
Allow: /
Disallow: /admin.html
Disallow: /hub
Disallow: /hub.html
Disallow: /notice/

# ── Monitoring and accessibility ── explicitly welcome, no crawl delay
User-agent: UptimeRobot
User-agent: Better Uptime Bot
User-agent: Pingdom.com_bot
User-agent: StatusCake
User-agent: Site24x7
User-agent: Chrome-Lighthouse
User-agent: Google-InspectionTool
User-agent: AhrefsSiteAudit
Allow: /
Disallow: /admin.html
Disallow: /hub
Disallow: /hub.html
Disallow: /notice/

# ==== SEO backlink databases and bulk scrapers ====
# Not discovery. These crawl to resell an index of your site to third parties,
# or to feed a content farm. Excluding them costs no reach at all.
User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: MJ12bot
User-agent: DotBot
User-agent: BLEXBot
User-agent: DataForSeoBot
User-agent: PetalBot
User-agent: Sogou
User-agent: SeekportBot
User-agent: serpstatbot
User-agent: ZoominfoBot
User-agent: Barkrowler
User-agent: ImagesiftBot
User-agent: magpie-crawler
User-agent: Scrapy
User-agent: python-requests
User-agent: zgrab
User-agent: masscan
User-agent: Nuclei
User-agent: Nikto
User-agent: sqlmap
User-agent: WPScan
User-agent: Wappalyzer
Disallow: /

# ── Sitemaps ─────────────────────────────────────────────────────────────────
Sitemap: https://cognitive-industries.org/sitemap.xml

# ── Machine-readable extras ──────────────────────────────────────────────────
# https://cognitive-industries.org/llms.txt          LLM-oriented site index
# https://cognitive-industries.org/llms-full.txt     the same index plus full page prose
# https://cognitive-industries.org/openapi.json      public HTTP API description
# https://cognitive-industries.org/rss.xml           lab log feed
# https://cognitive-industries.org/.well-known/security.txt