# ============================================================
# robots.txt — KResearchLab 加密货币研究平台
# https://kresearchlab.com
# 最后更新: 2026-07-22
# 策略: 白名单优先 — 明确允许核心SEO页面，屏蔽敏感路径
# ============================================================

# ======================= 全局规则 =======================
User-agent: *
Allow: /
Allow: /stocks/
Allow: /us-stocks.html
Allow: /sitemap.xml

# 屏蔽敏感/无关目录
Disallow: /admin/
Disallow: /private/
Disallow: /temp/
Disallow: /backup/
Disallow: /_backup
Disallow: /*.py$
Disallow: /*.bat$
Disallow: /*.json$
Disallow: /*.log$
Disallow: /.git/
Disallow: /.env
Disallow: /.vscode/
Disallow: /*.failed.html$
Disallow: /*_TEST_MODIFIED*
Disallow: /test-

# ======================= 百度蜘蛛 =======================
User-agent: Baiduspider
Allow: /
Allow: /article/
Allow: /tools/
Allow: /reports/
Allow: /stocks/
Allow: /us-stocks.html
Allow: /images/
Allow: /css/
Allow: /js/
Allow: /sitemap.xml
Disallow: /admin/
Disallow: /private/
Disallow: /temp/
Crawl-delay: 0

User-agent: Baiduspider-mobile
Allow: /
Allow: /article/
Allow: /tools/
Allow: /reports/
Allow: /stocks/
Allow: /us-stocks.html
Allow: /images/
Allow: /css/
Allow: /js/
Disallow: /admin/
Crawl-delay: 0

User-agent: Baiduspider-image
Allow: /images/
Allow: /article/
Allow: /tools/
Allow: /reports/
Allow: /stocks/
Disallow: /admin/

# ======================= Googlebot =======================
User-agent: Googlebot
Allow: /
Allow: /css/
Allow: /js/
Allow: /article/
Allow: /tools/
Allow: /reports/
Allow: /stocks/
Allow: /us-stocks.html
Allow: /images/
Allow: /sitemap.xml
Disallow: /admin/
Disallow: /private/
Disallow: /temp/
Disallow: /*.py$
Disallow: /*.bat$
Crawl-delay: 0

User-agent: Googlebot-Image
Allow: /images/
Allow: /article/
Allow: /tools/
Allow: /stocks/
Disallow: /admin/

User-agent: Googlebot-Mobile
Allow: /
Allow: /article/
Allow: /tools/
Allow: /reports/
Allow: /stocks/
Allow: /us-stocks.html
Disallow: /admin/

# ======================= Bingbot =======================
User-agent: Bingbot
Allow: /
Allow: /css/
Allow: /js/
Allow: /article/
Allow: /tools/
Allow: /reports/
Allow: /stocks/
Allow: /us-stocks.html
Allow: /images/
Allow: /sitemap.xml
Disallow: /admin/
Disallow: /private/
Disallow: /temp/
Disallow: /*.py$
Crawl-delay: 0

# ======================= 搜狗蜘蛛 =======================
User-agent: Sogou web spider
Allow: /
Allow: /article/
Allow: /tools/
Allow: /reports/
Allow: /stocks/
Allow: /us-stocks.html
Disallow: /admin/
Crawl-delay: 1

# ======================= 360蜘蛛 =======================
User-agent: 360Spider
Allow: /
Allow: /article/
Allow: /tools/
Allow: /reports/
Allow: /stocks/
Allow: /us-stocks.html
Disallow: /admin/
Crawl-delay: 1

# ======================= 神马 (UC) =======================
User-agent: YisouSpider
Allow: /
Allow: /article/
Allow: /tools/
Allow: /reports/
Allow: /stocks/
Allow: /us-stocks.html
Disallow: /admin/
Crawl-delay: 1

# ======================= 头条蜘蛛 =======================
User-agent: Bytespider
Allow: /
Allow: /article/
Allow: /tools/
Allow: /reports/
Allow: /stocks/
Allow: /us-stocks.html
Disallow: /admin/
Crawl-delay: 1

# ======================= 其他海外搜索引擎 =======================
User-agent: DuckDuckBot
Allow: /
Allow: /article/
Allow: /tools/
Allow: /reports/
Allow: /stocks/
Allow: /us-stocks.html
Disallow: /admin/
Crawl-delay: 0

User-agent: YandexBot
Allow: /
Allow: /article/
Allow: /tools/
Allow: /reports/
Allow: /stocks/
Allow: /us-stocks.html
Disallow: /admin/
Crawl-delay: 1

User-agent: Applebot
Allow: /
Allow: /article/
Allow: /tools/
Allow: /reports/
Allow: /stocks/
Allow: /us-stocks.html
Allow: /images/
Disallow: /admin/

# ======================= 屏蔽垃圾爬虫 =======================
User-agent: SemrushBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: ia_archiver
Disallow: /

User-agent: Scrubby
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: CCBot
Disallow: /

# ======================= Sitemap =======================
Sitemap: https://kresearchlab.com/sitemap.xml