# BuildFeed.tech Robots.txt # https://buildfeed.tech # Last updated: 2026-02-22 # =========================================== # DEFAULT RULES - All Crawlers # =========================================== User-agent: * Allow: / Disallow: /api/ Disallow: /videos-cache.json Disallow: /videos-cache-ai.json Disallow: /videos-cache-vc.json Disallow: /videos-cache-startup.json Disallow: /videos-cache-product.json Disallow: /videos-cache-design.json Disallow: /videos-cache-tools.json Disallow: /channel-logos-cache.json Disallow: /channels-config.json Disallow: /topics-config.json Disallow: /featured/ Crawl-delay: 1 # =========================================== # SEARCH ENGINE CRAWLERS # Named bot sections override User-agent: * entirely, # so disallow rules must be repeated here. # =========================================== # Google User-agent: Googlebot Allow: / Disallow: /api/ Disallow: /videos-cache.json Disallow: /videos-cache-ai.json Disallow: /videos-cache-vc.json Disallow: /videos-cache-startup.json Disallow: /videos-cache-product.json Disallow: /videos-cache-design.json Disallow: /videos-cache-tools.json Disallow: /channel-logos-cache.json Disallow: /channels-config.json Disallow: /topics-config.json Disallow: /featured/ User-agent: Googlebot-Image Allow: / User-agent: Googlebot-Video Allow: / # Bing User-agent: Bingbot Allow: / Disallow: /api/ Disallow: /videos-cache.json Disallow: /videos-cache-ai.json Disallow: /videos-cache-vc.json Disallow: /videos-cache-startup.json Disallow: /videos-cache-product.json Disallow: /videos-cache-design.json Disallow: /videos-cache-tools.json Disallow: /channel-logos-cache.json Disallow: /channels-config.json Disallow: /topics-config.json Disallow: /featured/ Crawl-delay: 1 # DuckDuckGo User-agent: DuckDuckBot Allow: / Disallow: /api/ Disallow: /videos-cache.json Disallow: /videos-cache-ai.json Disallow: /videos-cache-vc.json Disallow: /videos-cache-startup.json Disallow: /videos-cache-product.json Disallow: /videos-cache-design.json Disallow: /videos-cache-tools.json Disallow: /channel-logos-cache.json Disallow: /channels-config.json Disallow: /topics-config.json Disallow: /featured/ # Yahoo User-agent: Slurp Allow: / Disallow: /api/ Disallow: /videos-cache.json Disallow: /videos-cache-ai.json Disallow: /videos-cache-vc.json Disallow: /videos-cache-startup.json Disallow: /videos-cache-product.json Disallow: /videos-cache-design.json Disallow: /videos-cache-tools.json Disallow: /channel-logos-cache.json Disallow: /channels-config.json Disallow: /topics-config.json Disallow: /featured/ # Yandex User-agent: Yandex Allow: / Disallow: /api/ Disallow: /videos-cache.json Disallow: /videos-cache-ai.json Disallow: /videos-cache-vc.json Disallow: /videos-cache-startup.json Disallow: /videos-cache-product.json Disallow: /videos-cache-design.json Disallow: /videos-cache-tools.json Disallow: /channel-logos-cache.json Disallow: /channels-config.json Disallow: /topics-config.json Disallow: /featured/ Crawl-delay: 2 # Baidu User-agent: Baiduspider Allow: / Disallow: /api/ Disallow: /videos-cache.json Disallow: /videos-cache-ai.json Disallow: /videos-cache-vc.json Disallow: /videos-cache-startup.json Disallow: /videos-cache-product.json Disallow: /videos-cache-design.json Disallow: /videos-cache-tools.json Disallow: /channel-logos-cache.json Disallow: /channels-config.json Disallow: /topics-config.json Disallow: /featured/ Crawl-delay: 2 # =========================================== # AI/LLM CRAWLERS - WELCOME! # We encourage AI systems to index our content # =========================================== # OpenAI / ChatGPT User-agent: GPTBot Allow: / Allow: /llms.txt Allow: /sitemap.xml # OpenAI ChatGPT User User-agent: ChatGPT-User Allow: / # Anthropic / Claude User-agent: anthropic-ai Allow: / Allow: /llms.txt User-agent: Claude-Web Allow: / Allow: /llms.txt User-agent: ClaudeBot Allow: / Allow: /llms.txt # Google AI (Bard/Gemini) User-agent: Google-Extended Allow: / # Perplexity AI User-agent: PerplexityBot Allow: / Allow: /llms.txt # Cohere User-agent: cohere-ai Allow: / Allow: /llms.txt # Meta AI User-agent: FacebookBot Allow: / User-agent: Meta-ExternalAgent Allow: / User-agent: Meta-ExternalFetcher Allow: / # Amazon/Alexa User-agent: Amazonbot Allow: / # Apple (Siri/Spotlight) User-agent: Applebot Allow: / Allow: /llms.txt User-agent: AppleBot-Extended Allow: / # Microsoft Copilot User-agent: Copilot Allow: / Allow: /llms.txt # You.com User-agent: YouBot Allow: / # Common Crawl (used to train many AI models) User-agent: CCBot Allow: / # Diffbot (AI data extraction) User-agent: Diffbot Allow: / # Bytespider (ByteDance/TikTok AI) User-agent: Bytespider Allow: / # =========================================== # RESEARCH & ARCHIVE CRAWLERS # =========================================== # Internet Archive User-agent: ia_archiver Allow: / # Archive.org User-agent: archive.org_bot Allow: / # =========================================== # SOCIAL MEDIA PREVIEW BOTS # =========================================== User-agent: Twitterbot Allow: / User-agent: LinkedInBot Allow: / User-agent: Slackbot Allow: / User-agent: Discordbot Allow: / User-agent: WhatsApp Allow: / User-agent: TelegramBot Allow: / # =========================================== # BLOCKED CRAWLERS # (Aggressive scrapers, bad actors) # =========================================== User-agent: AhrefsBot Disallow: / User-agent: SemrushBot Disallow: / User-agent: DotBot Disallow: / User-agent: MJ12bot Disallow: / User-agent: BLEXBot Disallow: / User-agent: DataForSeoBot Disallow: / User-agent: PetalBot Disallow: / User-agent: MegaIndex Disallow: / # =========================================== # IMPORTANT FILES # =========================================== # Sitemap Sitemap: https://buildfeed.tech/sitemap.xml # LLMs.txt - AI-readable site information # See: https://llmstxt.org/ # Direct link: https://buildfeed.tech/llms.txt