# ============================================================================== # robots.txt for cleaningsteamer.com # ============================================================================== User-agent: * Allow: / # ---------------------------------------------- # AI / Generative Engine crawlers -- explicitly allowed # ---------------------------------------------- # OpenAI User-agent: GPTBot Allow: / User-agent: ChatGPT-User Allow: / User-agent: OAI-SearchBot Allow: / # Anthropic User-agent: ClaudeBot Allow: / User-agent: Claude-Web Allow: / User-agent: anthropic-ai Allow: / # Perplexity User-agent: PerplexityBot Allow: / User-agent: Perplexity-User Allow: / # Google User-agent: Google-Extended Allow: / User-agent: Gemini Allow: / User-agent: Googlebot Allow: / User-agent: GoogleOther Allow: / User-agent: GoogleOther-Image Allow: / # Apple User-agent: Applebot Allow: / User-agent: Applebot-Extended Allow: / # Amazon / Alexa -- Crawl-delay: 3s, non-core search engine User-agent: Amazonbot Allow: / Crawl-delay: 3 # You.com -- Crawl-delay: 3s User-agent: YouBot Allow: / Crawl-delay: 3 # Cohere -- Crawl-delay: 3s User-agent: cohere-ai Allow: / Crawl-delay: 3 # Meta AI User-agent: Meta-ExternalAgent Allow: / User-agent: Meta-ExternalFetcher Allow: / # Microsoft / Bing -- Explicitly declare bingbot + msnbot User-agent: bingbot Allow: / User-agent: msnbot Allow: / User-agent: msnbot-media Allow: / User-agent: adidxbot Allow: / User-agent: BingPreview Allow: / # Mistral AI -- Crawl-delay: 3s User-agent: Mistralbot Allow: / Crawl-delay: 3 # DuckDuckGo -- Crawl-delay: 2s User-agent: DuckDuckBot Allow: / Crawl-delay: 2 # Yandex (Russia) -- Crawl-delay: 2s User-agent: YandexBot Allow: / Crawl-delay: 2 User-agent: YandexRender Allow: / Crawl-delay: 2 User-agent: YandexImages Allow: / Crawl-delay: 2 # Seznam (Czech Republic) -- Crawl-delay: 2s User-agent: SeznamBot Allow: / Crawl-delay: 2 # Naver (Korea) -- Crawl-delay: 2s User-agent: Yeti Allow: / Crawl-delay: 2 # Yahoo -- Crawl-delay: 2s User-agent: Slurp Allow: / Crawl-delay: 2 # ---------------------------------------------- # Chinese AI crawlers & search engines # ---------------------------------------------- # Baidu (ERNIE Bot / Baidu Search) User-agent: Baiduspider Allow: / User-agent: Baiduspider-render Allow: / # ByteDance (Doubao / Toutiao Search) User-agent: Bytespider Allow: / # Sogou Search User-agent: Sogou Spider Allow: / User-agent: Sogou web spider Allow: / # 360 Search User-agent: 360Spider Allow: / # Shenma Search (UC / Alibaba) User-agent: iaskspider Allow: / # Alibaba / Qwen (Tongyi Qianwen) User-agent: AlibabaBot Allow: / # Tencent (Hunyuan) User-agent: TencentBot Allow: / # DeepSeek User-agent: DeepSeekBot Allow: / # Zhipu AI (ChatGLM) User-agent: ChatGLM-Bot Allow: / # ---------------------------------------------- # Disallow admin and private areas (all bots) # ---------------------------------------------- User-agent: * Disallow: /admin/ Disallow: /private/ Disallow: /tmp/ Disallow: /node_modules/ Disallow: /scripts/ Disallow: /.git/ Disallow: /dist/ Disallow: /wp-admin/ Disallow: /wp-includes/ Disallow: /schema-demo/ # ---------------------------------------------- # Sitemaps -- Root domain # ---------------------------------------------- Sitemap: https://www.cleaningsteamer.com/sitemap-index.xml Sitemap: https://www.cleaningsteamer.com/sitemap.xml Sitemap: https://www.cleaningsteamer.com/sitemap_baidu.xml Sitemap: https://www.cleaningsteamer.com/sitemap-images.xml Sitemap: https://www.cleaningsteamer.com/sitemap-news.xml Sitemap: https://www.cleaningsteamer.com/sitemap.html Sitemap: https://www.cleaningsteamer.com/sitemap.txt Sitemap: https://www.cleaningsteamer.com/urllist.txt # --- Sitemaps: en language --- Sitemap: https://www.cleaningsteamer.com/lang/en/sitemap.xml # --- Sitemaps: zh language --- Sitemap: https://www.cleaningsteamer.com/lang/zh/sitemap.xml # --- Sitemaps: fr language (fr.cleaningsteamer.com) --- Sitemap: http://fr.cleaningsteamer.com/lang/fr/sitemap.xml # --- Sitemaps: es language (es.cleaningsteamer.com) --- Sitemap: http://es.cleaningsteamer.com/lang/es/sitemap.xml # --- Sitemaps: ar language (ar.cleaningsteamer.com) --- Sitemap: http://ar.cleaningsteamer.com/lang/ar/sitemap.xml # --- Sitemaps: de language (de.cleaningsteamer.com) --- Sitemap: http://de.cleaningsteamer.com/lang/de/sitemap.xml # --- Sitemaps: ru language (ru.cleaningsteamer.com) --- Sitemap: http://ru.cleaningsteamer.com/lang/ru/sitemap.xml # --- Sitemaps: pt language (pt.cleaningsteamer.com) --- Sitemap: http://pt.cleaningsteamer.com/lang/pt/sitemap.xml # --- Sitemaps: it language (it.cleaningsteamer.com) --- Sitemap: http://it.cleaningsteamer.com/lang/it/sitemap.xml # --- Sitemaps: ja language (ja.cleaningsteamer.com) --- Sitemap: http://ja.cleaningsteamer.com/lang/ja/sitemap.xml # ---------------------------------------------- # AI / LLM reference files -- AI-specific index # ---------------------------------------------- Sitemap: https://www.cleaningsteamer.com/llms.txt Sitemap: https://www.cleaningsteamer.com/llms-full.txt # ---------------------------------------------- # GEO compliance data -- Indexed via structured data # ---------------------------------------------- Sitemap: https://www.cleaningsteamer.com/data/geo/geo-data.json Sitemap: https://www.cleaningsteamer.com/data/geo/compliance.json # ---------------------------------------------- # AI crawler directives -- Standard protocol files # ---------------------------------------------- # https://www.cleaningsteamer.com/ai.txt # https://www.cleaningsteamer.com/.well-known/ai.txt # https://www.cleaningsteamer.com/.well-known/security.txt # ---------------------------------------------- # Structured data & site metadata # ---------------------------------------------- # https://www.cleaningsteamer.com/rss.xml # https://www.cleaningsteamer.com/lang/en/schema-localbusiness.json # https://www.cleaningsteamer.com/lang/zh/schema-localbusiness.json # http://fr.cleaningsteamer.com/lang/fr/schema-localbusiness.json # http://es.cleaningsteamer.com/lang/es/schema-localbusiness.json # http://ar.cleaningsteamer.com/lang/ar/schema-localbusiness.json # http://de.cleaningsteamer.com/lang/de/schema-localbusiness.json # http://ru.cleaningsteamer.com/lang/ru/schema-localbusiness.json # http://pt.cleaningsteamer.com/lang/pt/schema-localbusiness.json # http://it.cleaningsteamer.com/lang/it/schema-localbusiness.json # http://ja.cleaningsteamer.com/lang/ja/schema-localbusiness.json