# robots.txt for stepaheadaba.com

# Allow all crawlers by default
User-agent: *
Disallow:

# Disallow specific unuseful crawlers
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: YandexBot
Disallow: /

# Block AI-specific crawlers
User-agent: OpenAI-GPT
Disallow: /

User-agent: GPT-3
Disallow: /

User-agent: Googlebot-Image
Disallow: /private/

# Disallow Googlebot from crawling private sections of the site
User-agent: Googlebot
Disallow: /private/

# Allow specific crawlers that are useful
User-agent: Bingbot
Disallow:

User-agent: Googlebot-News
Disallow:

# Allow Google's mobile crawler
User-agent: Googlebot-Mobile
Disallow:

# Sitemap URL
Sitemap: stepaheadaba.com/sitemap.xml

# Crawl-delay for bots that crawl too aggressively
User-agent: *
Crawl-delay: 10

Sitemap: https://www.stepaheadaba.com/sitemap.xml