# ==================================================================== # ROBOTS.TXT FOR OVERFLOWW.IN # ==================================================================== # -------------------------------------------------------------------- # 1. GLOBAL RULES (Search Engines & General Crawlers) # -------------------------------------------------------------------- User-agent: * Allow: / # Block private, utility, and infrastructure endpoints Disallow: /login Disallow: /admin Disallow: /account Disallow: /cart Disallow: /checkout Disallow: /api/ # Prevent crawl budget waste from URL parameters (sorting filters, internal search) Disallow: /*?* # Explicitly ensure core public pages are crawlable Allow: /shop/ Allow: /gallery Allow: /workshop # -------------------------------------------------------------------- # 2. AI BOTS & LLM CRAWLERS (Throttling Aggressive Scrapers) # -------------------------------------------------------------------- User-agent: GPTBot User-agent: ChatGPT-User User-agent: OAI-SearchBot User-agent: ClaudeBot User-agent: Claude-User User-agent: Claude-SearchBot User-agent: Google-Extended User-agent: PerplexityBot Disallow: /login Disallow: /admin Disallow: /account Disallow: /cart Disallow: /checkout Disallow: /api/ Disallow: /*?* Allow: /shop/ Allow: /gallery Allow: /workshop # Enforce a delay to protect server speed (honored by Anthropic/OpenAI bots; ignored by Google) Crawl-delay: 10 # -------------------------------------------------------------------- # 3. SITEMAP INDEX # -------------------------------------------------------------------- Sitemap: https://www.overfloww.in/sitemap.xml