User-agent: *
Disallow: /admin/
Disallow: /internal/
Disallow: /api/v1/private/
Disallow: /api/v2/private/
Disallow: /staging/
Disallow: /dev/
Disallow: /backup/
Disallow: /db-backups/
Disallow: /uploads/private/
Disallow: /config/
Disallow: /env/
Disallow: /.env
Disallow: /secrets/
Disallow: /internal-docs/
Disallow: /grafana/
Disallow: /kibana/
Disallow: /jupyter/
Disallow: /mlflow/
Disallow: /airflow/
Disallow: /vault/
Disallow: /consul/
Disallow: /prometheus/
Disallow: /node_exporter/
Disallow: /api/internal/model-registry/
Disallow: /api/internal/datasets/
Disallow: /api/internal/training-jobs/
Disallow: /api/internal/billing/
Disallow: /api/internal/users/export
Disallow: /api/internal/audit-logs/
Disallow: /static/internal/
Disallow: /reports/
Disallow: /analytics/raw/

User-agent: GPTBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: ClaudeBot
Disallow: /

Sitemap: https://clearlab.ai/sitemap.xml
Sitemap: https://clearlab.ai/sitemap-docs.xml