User-agent: *
Allow: /
Disallow: /admin/
Disallow: /private/
Disallow: /api/
Disallow: /cdn-cgi/
Disallow: /*.php$
Disallow: /*.json$
Disallow: /*?*
Disallow: /*/comments/
Disallow: /search/

# Explicitly allow important pages
Allow: /promptconverter.html
Allow: /howitworks.html
Allow: /login.html

# Block duplicate index pages
Disallow: /index.html
Disallow: /home.html
Disallow: /main.html

# Allow image crawling for Google Images
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.png$
Allow: /*.gif$
Allow: /*.webp$

# Google AdsBot
User-agent: AdsBot-Google
Allow: /

# Google Image Bot
User-agent: Googlebot-Image
Allow: /
Disallow: /api/

# Google News Bot
User-agent: Googlebot-News
Allow: /news/
Allow: /sitemap-news.xml
Crawl-delay: 1

# Regular Googlebot
User-agent: Googlebot
Allow: /
Disallow: /api/
Crawl-delay: 2

# Bing Bot
User-agent: Bingbot
Allow: /
Disallow: /api/
Crawl-delay: 2

# Block AI scrapers
User-agent: ChatGPT-User
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: Applebot
Disallow: /api/

# Sitemaps - Use HTTPS always for production
Sitemap: https://www.promptseen.co/sitemap.xml
Sitemap: https://www.promptseen.co/sitemap-posts.xml
Sitemap: https://www.promptseen.co/sitemap-news.xml
Sitemap: https://www.promptseen.co/sitemap-pages.xml

# Crawl delay for all other bots
User-agent: *
Crawl-delay: 3