# 仅允许主要搜索引擎爬虫，禁止其他所有爬虫
User-agent: *
Disallow: /novel/
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Disallow: /search/
Disallow: /feed/
Disallow: /*/feed/
Disallow: /comments/feed/
Disallow: /*/comments/feed/
Disallow: /*?
Allow: /wp-content/uploads/
Allow: /wp-content/themes/*/assets/
Allow: /wp-content/themes/*/images/
Allow: /wp-content/themes/*/js/
Allow: /wp-content/themes/*/css/


User-agent: *
Disallow: /novel/
Disallow: /Novel/


# 允许谷歌爬虫
User-agent: Googlebot
Allow: /
Disallow: /novel/
Crawl-delay: 1

User-agent: Googlebot-Image
Allow: /wp-content/uploads/
Disallow: /novel/
Disallow: /wp-admin/
Crawl-delay: 2

User-agent: Googlebot-News
Allow: /
Disallow: /novel/

User-agent: Googlebot-Video
Allow: /
Disallow: /novel/

# 允许百度爬虫
User-agent: Baiduspider
Allow: /
Disallow: /novel/
Disallow: /wp-admin/
Crawl-delay: 2

User-agent: Baiduspider-image
Allow: /wp-content/uploads/
Disallow: /novel/
Disallow: /wp-admin/
Crawl-delay: 3

User-agent: Baiduspider-video
Allow: /
Disallow: /novel/

# 允许必应爬虫
User-agent: bingbot
Allow: /
Disallow: /novel/
Disallow: /wp-admin/
Crawl-delay: 1

User-agent: MSNBot
Allow: /
Disallow: /novel/
Crawl-delay: 2

User-agent: MSNBot-Media
Allow: /wp-content/uploads/
Disallow: /novel/
Crawl-delay: 3

# 允许Yandex爬虫
User-agent: YandexBot
Allow: /
Disallow: /novel/
Disallow: /wp-admin/
Crawl-delay: 2

User-agent: YandexImages
Allow: /wp-content/uploads/
Disallow: /novel/
Crawl-delay: 3

# 其他爬虫禁止访问
User-agent: *
Disallow: /

# 下面是针对已知SEO和采集爬虫的特别限制
User-agent: SemrushBot
Disallow: /
Crawl-delay: 30

User-agent: AhrefsBot
Disallow: /
Crawl-delay: 30

User-agent: DotBot
Disallow: /
Crawl-delay: 30

User-agent: MJ12bot
Disallow: /
Crawl-delay: 30

User-agent: Exabot
Disallow: /
Crawl-delay: 30

User-agent: CCBot
Disallow: /
Crawl-delay: 30

User-agent: rogerbot
Disallow: /
Crawl-delay: 30

User-agent: proximic
Disallow: /
Crawl-delay: 30

User-agent: GrapeshotCrawler
Disallow: /
Crawl-delay: 30

User-agent: PetalBot
Disallow: /
Crawl-delay: 30

User-agent: Bytespider
Disallow: /
Crawl-delay: 30

User-agent: Amazonbot
Disallow: /
Crawl-delay: 20

User-agent: Applebot
Allow: /
Disallow: /novel/
Crawl-delay: 5

# 社交媒体爬虫
User-agent: Twitterbot
Allow: /
Disallow: /novel/
Crawl-delay: 5

User-agent: FacebookExternalHit
Allow: /
Disallow: /novel/
Crawl-delay: 5

User-agent: facebookcatalog
Allow: /
Disallow: /novel/
Crawl-delay: 5

# Sitemap地址（WordPress自动生成）
Sitemap: https://bilitxt.com/wp-sitemap.xml