# Robots.txt file for MediaWiki sites to prevent unauthorized bot scraping
# Place this file at the root of your website: https://yoursite.com/robots.txt

# Allow major search engines
User-agent: Googlebot
Allow: /
Allow: /wiki/
Allow: /w/index.php?title=*&action=view
Allow: /w/index.php?title=Special:RecentChanges
Allow: /w/index.php?title=Special:Random

User-agent: Bingbot
Allow: /
Allow: /wiki/
Allow: /w/index.php?title=*&action=view
Allow: /w/index.php?title=Special:RecentChanges
Allow: /w/index.php?title=Special:Random

User-agent: Slurp
Allow: /
Allow: /wiki/
Allow: /w/index.php?title=*&action=view

User-agent: DuckDuckBot
Allow: /
Allow: /wiki/
Allow: /w/index.php?title=*&action=view

User-agent: Baiduspider
Allow: /
Allow: /wiki/

User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

# Block common scraping bots
User-agent: SemrushBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: YandexBot
Disallow: /

User-agent: SeznamBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: anthropic-ai
Disallow: /

# General bot restrictions with MediaWiki-specific rules
User-agent: *
Crawl-delay: 10

# Allow main wiki content
Allow: /wiki/
Allow: /w/index.php?title=*&action=view

# Block MediaWiki administrative and system directories
Disallow: /w/
Disallow: /mw-config/
Disallow: /includes/
Disallow: /maintenance/
Disallow: /languages/
Disallow: /serialized/
Disallow: /tests/
Disallow: /vendor/
Disallow: /cache/
Disallow: /images/deleted/
Disallow: /images/temp/

# Block API access
Disallow: /w/api.php
Disallow: /api/
Disallow: /w/api/

# Block specific directories
Disallow: /d20/
Disallow: /SRD/

# Block MediaWiki special pages and actions that shouldn't be indexed
Disallow: /w/index.php?title=Special:Search
Disallow: /w/index.php?title=Special:Random
Disallow: /w/index.php?title=Special:Watchlist
Disallow: /w/index.php?title=Special:Contributions
Disallow: /w/index.php?title=Special:Log
Disallow: /w/index.php?title=Special:BlockList
Disallow: /w/index.php?title=Special:ListUsers
Disallow: /w/index.php?title=Special:Statistics
Disallow: /w/index.php?title=Special:Version
Disallow: /w/index.php?title=Special:What*
Disallow: /w/index.php?title=Special:Upload
Disallow: /w/index.php?title=Special:UserLogin
Disallow: /w/index.php?title=Special:UserLogout
Disallow: /w/index.php?title=Special:Preferences

# Block problematic URL patterns
Disallow: /w/index.php?title=*&action=edit
Disallow: /w/index.php?title=*&action=history
Disallow: /w/index.php?title=*&action=delete
Disallow: /w/index.php?title=*&action=move
Disallow: /w/index.php?title=*&action=protect
Disallow: /w/index.php?title=*&action=watch
Disallow: /w/index.php?title=*&action=unwatch
Disallow: /w/index.php?title=*&action=rollback
Disallow: /w/index.php?title=*&printable=yes
Disallow: /w/index.php?title=*&oldid=*
Disallow: /w/index.php?title=*&diff=*
Disallow: /w/index.php?title=*&redirect=no

# Block talk pages and user pages (adjust based on your needs)
Disallow: /wiki/Talk:*
Disallow: /wiki/User:*
Disallow: /wiki/User_talk:*
Disallow: /wiki/MediaWiki:*
Disallow: /wiki/MediaWiki_talk:*
Disallow: /wiki/Template_talk:*
Disallow: /wiki/Help_talk:*
Disallow: /wiki/Category_talk:*

# Block file uploads and media
Disallow: /images/
Allow: /images/*.jpg
Allow: /images/*.jpeg
Allow: /images/*.png
Allow: /images/*.gif
Allow: /images/*.svg
Allow: /images/*.webp

# Block log files and backups
Disallow: /*.log$
Disallow: /*.sql$
Disallow: /*.bak$
Disallow: /*.tmp$

# Block query parameters that cause duplicate content
Disallow: /*?action=
Disallow: /*&action=
Disallow: /*?search=
Disallow: /*&search=
Disallow: /*?title=Special:
Disallow: /*&title=Special:

# Specify sitemap location (replace with your actual sitemap)
# Sitemap: https://yoursite.com/sitemap.xml