# Robots.txt for Digital Garden Works
# https://digital-garden.works

# =============================================================================
# Welcome AI Crawlers! 🤖
# =============================================================================
# Digital Garden Works embraces AI and machine learning innovation.
# We welcome AI crawlers and allow training on our public content.
# Our mission is to contribute to the advancement of AI technology while
# maintaining privacy for user-specific and administrative areas.
# =============================================================================

# AI Crawlers - Explicitly Allowed
# These bots are used for AI training and model development

# OpenAI GPTBot
User-agent: GPTBot
Allow: /
Disallow: /_api/
Disallow: /admin
Disallow: /dashboard
Disallow: /login

# OpenAI ChatGPT User
User-agent: ChatGPT-User
Allow: /
Disallow: /_api/
Disallow: /admin
Disallow: /dashboard
Disallow: /login

# Google AI Training
User-agent: Google-Extended
Allow: /
Disallow: /_api/
Disallow: /admin
Disallow: /dashboard
Disallow: /login

# Anthropic AI
User-agent: Anthropic-ai
Allow: /
Disallow: /_api/
Disallow: /admin
Disallow: /dashboard
Disallow: /login

# Anthropic Claude Web
User-agent: Claude-Web
Allow: /
Disallow: /_api/
Disallow: /admin
Disallow: /dashboard
Disallow: /login

# Cohere AI
User-agent: Cohere-ai
Allow: /
Disallow: /_api/
Disallow: /admin
Disallow: /dashboard
Disallow: /login

# Perplexity AI
User-agent: PerplexityBot
Allow: /
Disallow: /_api/
Disallow: /admin
Disallow: /dashboard
Disallow: /login

# ByteDance/TikTok AI
User-agent: Bytespider
Allow: /
Disallow: /_api/
Disallow: /admin
Disallow: /dashboard
Disallow: /login

# Common Crawl (used for AI training datasets)
User-agent: CCBot
Allow: /
Disallow: /_api/
Disallow: /admin
Disallow: /dashboard
Disallow: /login

# Meta/Facebook AI
User-agent: FacebookBot
Allow: /
Disallow: /_api/
Disallow: /admin
Disallow: /dashboard
Disallow: /login

# Meta External Agent
User-agent: Meta-ExternalAgent
Allow: /
Disallow: /_api/
Disallow: /admin
Disallow: /dashboard
Disallow: /login

# Amazon AI
User-agent: Amazonbot
Allow: /
Disallow: /_api/
Disallow: /admin
Disallow: /dashboard
Disallow: /login

# Apple AI Extended
User-agent: Applebot-Extended
Allow: /
Disallow: /_api/
Disallow: /admin
Disallow: /dashboard
Disallow: /login

# Diffbot
User-agent: Diffbot
Allow: /
Disallow: /_api/
Disallow: /admin
Disallow: /dashboard
Disallow: /login

# OmgiliBot
User-agent: OmgiliBot
Allow: /
Disallow: /_api/
Disallow: /admin
Disallow: /dashboard
Disallow: /login

# You.com AI
User-agent: YouBot
Allow: /
Disallow: /_api/
Disallow: /admin
Disallow: /dashboard
Disallow: /login

# Allen Institute AI
User-agent: AI2Bot
Allow: /
Disallow: /_api/
Disallow: /admin
Disallow: /dashboard
Disallow: /login

# Timpibot
User-agent: Timpibot
Allow: /
Disallow: /_api/
Disallow: /admin
Disallow: /dashboard
Disallow: /login

# img2dataset (AI training data collection)
User-agent: img2dataset
Allow: /
Disallow: /_api/
Disallow: /admin
Disallow: /dashboard
Disallow: /login

# =============================================================================
# General Crawlers
# =============================================================================

# Allow all other crawlers
User-agent: *
Allow: /

# Disallow private and admin routes
Disallow: /_api/
Disallow: /admin
Disallow: /admin/*
Disallow: /dashboard
Disallow: /dashboard/*
Disallow: /login
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /reset-password/*

# Crawl delay for respectful crawling
Crawl-delay: 1

# Googlebot specific rules
User-agent: Googlebot
Allow: /
Disallow: /_api/
Disallow: /admin
Disallow: /admin/*
Disallow: /dashboard
Disallow: /dashboard/*
Disallow: /login
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /reset-password/*

# Bingbot specific rules
User-agent: Bingbot
Allow: /
Crawl-delay: 2

# Reference sitemap
Sitemap: https://digital-garden.works/sitemap.xml