# Wisdek Digital Marketing - Robots.txt
# Optimized for Google Search Console compliance
# Last updated: 2026-01-19
# Cache-busting update: 2026-01-19T21:09:58.722Z

# Sitemap location (Primary)
Sitemap: https://wisdek.com/sitemap.xml

# Major search engine crawlers - full access (no Crawl-delay to avoid GSC warnings)
User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: Googlebot-Mobile
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

# AI-powered search engines - EXPLICITLY ALLOWED for blog content and social sharing
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: CCBot
Allow: /

# SEO audit and analysis tools (rate limited)
User-agent: SemrushBot
Allow: /
Crawl-delay: 2

User-agent: AhrefsBot
Allow: /
Crawl-delay: 2

User-agent: MJ12bot
Allow: /
Crawl-delay: 2

User-agent: DotBot
Allow: /
Crawl-delay: 2

# Additional AI crawlers - Comment these out if you want to allow them
# User-agent: anthropic-ai
# Disallow: /
# 
# User-agent: Claude-Web
# Disallow: /
# 
# User-agent: cohere-ai
# Disallow: /
# 
# User-agent: Google-Extended
# Disallow: /
# 
# User-agent: PerplexityBot
# Disallow: /
# 
# User-agent: Omgilibot
# Disallow: /

# Block aggressive crawlers that don't respect crawl budgets
User-agent: DataForSeoBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: MegaIndex
Disallow: /

User-agent: SeznamBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: dotbot
Disallow: /

# Default rules for all other bots
User-agent: *
Allow: /
# IMPORTANT: Do NOT block /_next/static/ - Google needs access to CSS, JS, and static resources
# to properly render and evaluate pages for indexing and ranking
Allow: /_next/static/
Disallow: /api/
Disallow: /admin/
Disallow: /private/
Disallow: *.php$
Disallow: *.cgi$
Disallow: *.asp$
Disallow: *.aspx$
Disallow: /cgi-bin/