From ac0d34437742ac4750c11c56b6b290caea6d155c Mon Sep 17 00:00:00 2001
From: aliferous3 <94698925+Aliferous3@users.noreply.github.com>
Date: Sun, 4 Oct 2026 15:52:34 +0400
Subject: [PATCH 1/2] Update crawler and AI bot policy

---
 public/robots.txt | 197 ++++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 183 insertions(+), 14 deletions(-)

diff --git a/public/robots.txt b/public/robots.txt
index 836c209..cf8035d 100644
--- a/public/robots.txt
+++ b/public/robots.txt
@@ -1,39 +1,208 @@
 # DEV LEDGER — crawler policy
-# Public product and documentation pages are intentionally discoverable.
-# Authenticated application surfaces, API routes, and telemetry relays are not.
-# Updated: 2026-09-28
+# Public product, documentation, security, legal, and journal pages are discoverable.
+# Authenticated application surfaces, APIs, and telemetry relays are not.
+# Search and answer-engine discovery is allowed.
+# Dedicated model-training/data-corpus crawlers are blocked.
+# Updated: 2026-10-04
+
+
+# ==================================================
+# DEFAULT POLICY
+# ==================================================
 
 User-agent: *
 Allow: /
+
 Disallow: /api/
 Disallow: /rly/
 Disallow: /overview
+Disallow: /projects
 Disallow: /activity
 Disallow: /code
+Disallow: /shipping
+
+
+# ==================================================
+# GOOGLE SEARCH
+# ==================================================
+
+User-agent: Googlebot
+User-agent: Googlebot-Image
+User-agent: Googlebot-Video
+User-agent: Googlebot-News
+Allow: /
+
+Disallow: /api/
+Disallow: /rly/
+Disallow: /overview
+Disallow: /projects
+Disallow: /activity
+Disallow: /code
+Disallow: /shipping
+
+
+# ==================================================
+# MICROSOFT BING
+# ==================================================
+
+User-agent: Bingbot
+Allow: /
+
+Disallow: /api/
+Disallow: /rly/
+Disallow: /overview
+Disallow: /projects
+Disallow: /activity
+Disallow: /code
+Disallow: /shipping
+
+
+# ==================================================
+# DUCKDUCKGO
+# ==================================================
+
+User-agent: DuckDuckBot
+Allow: /
+
+Disallow: /api/
+Disallow: /rly/
+Disallow: /overview
+Disallow: /projects
+Disallow: /activity
+Disallow: /code
+Disallow: /shipping
+
+
+# ==================================================
+# YANDEX
+# ==================================================
+
+User-agent: YandexBot
+Allow: /
+
+Disallow: /api/
+Disallow: /rly/
+Disallow: /overview
+Disallow: /projects
+Disallow: /activity
+Disallow: /code
+Disallow: /shipping
+
+
+# ==================================================
+# BAIDU
+# ==================================================
+
+User-agent: Baiduspider
+Allow: /
+
+Disallow: /api/
+Disallow: /rly/
+Disallow: /overview
+Disallow: /projects
+Disallow: /activity
+Disallow: /code
+Disallow: /shipping
+
+
+# ==================================================
+# OPENAI — SEARCH / RETRIEVAL
+# ==================================================
 
-# AI search, answer, retrieval, and model crawlers.
-# Discoverability-first policy: public pages are allowed; private/app surfaces remain blocked.
 User-agent: OAI-SearchBot
-User-agent: GPTBot
 User-agent: ChatGPT-User
-User-agent: ClaudeBot
+Allow: /
+
+Disallow: /api/
+Disallow: /rly/
+Disallow: /overview
+Disallow: /projects
+Disallow: /activity
+Disallow: /code
+Disallow: /shipping
+
+
+# ==================================================
+# ANTHROPIC — SEARCH / USER RETRIEVAL
+# ==================================================
+
 User-agent: Claude-SearchBot
 User-agent: Claude-User
+Allow: /
+
+Disallow: /api/
+Disallow: /rly/
+Disallow: /overview
+Disallow: /projects
+Disallow: /activity
+Disallow: /code
+Disallow: /shipping
+
+
+# ==================================================
+# PERPLEXITY — SEARCH / RETRIEVAL
+# ==================================================
+
 User-agent: PerplexityBot
 User-agent: Perplexity-User
-User-agent: Google-Extended
+Allow: /
+
+Disallow: /api/
+Disallow: /rly/
+Disallow: /overview
+Disallow: /projects
+Disallow: /activity
+Disallow: /code
+Disallow: /shipping
+
+
+# ==================================================
+# APPLE SEARCH
+# ==================================================
+
 User-agent: Applebot
-User-agent: Applebot-Extended
-User-agent: meta-webindexer
-User-agent: meta-externalagent
-User-agent: meta-externalfetcher
-User-agent: CCBot
-User-agent: Bytespider
 Allow: /
+
 Disallow: /api/
 Disallow: /rly/
 Disallow: /overview
+Disallow: /projects
 Disallow: /activity
 Disallow: /code
+Disallow: /shipping
+
+
+# ==================================================
+# MODEL TRAINING / DATA-CORPUS CRAWLERS
+# ==================================================
+
+User-agent: GPTBot
+Disallow: /
+
+User-agent: ClaudeBot
+Disallow: /
+
+User-agent: Google-Extended
+Disallow: /
+
+User-agent: Applebot-Extended
+Disallow: /
+
+User-agent: CCBot
+Disallow: /
+
+User-agent: Bytespider
+Disallow: /
+
+User-agent: meta-externalagent
+Disallow: /
+
+User-agent: meta-webindexer
+Disallow: /
+
+
+# ==================================================
+# SITEMAP
+# ==================================================
 
 Sitemap: https://devledger.site/sitemap.xml

From 5d4d7d2bc64cf45723332bbfd5db078050e62b4b Mon Sep 17 00:00:00 2001
From: aliferous3 <94698925+Aliferous3@users.noreply.github.com>
Date: Sun, 4 Oct 2026 15:53:54 +0400
Subject: [PATCH 2/2] Keep Meta user retrieval crawler explicit

---
 public/robots.txt | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/public/robots.txt b/public/robots.txt
index cf8035d..206832c 100644
--- a/public/robots.txt
+++ b/public/robots.txt
@@ -172,6 +172,24 @@ Disallow: /code
 Disallow: /shipping
 
 
+# ==================================================
+# META — USER RETRIEVAL
+# Allow user-requested fetching while keeping Meta's
+# dedicated training/indexing agents blocked below.
+# ==================================================
+
+User-agent: meta-externalfetcher
+Allow: /
+
+Disallow: /api/
+Disallow: /rly/
+Disallow: /overview
+Disallow: /projects
+Disallow: /activity
+Disallow: /code
+Disallow: /shipping
+
+
 # ==================================================
 # MODEL TRAINING / DATA-CORPUS CRAWLERS
 # ==================================================