Disallow: / User-agent: * Crawl-delay: 100.00 Disallow: /*.jpg$ Disallow: /*.Jpg$ Disallow: /*.JPG$ Disallow: /*.jpeg$ Disallow: /*.Jpeg$ Disallow: /*.JPEG$ Disallow: /*.pdf$ Disallow: /*.Pdf$ Disallow: /*.PDF$ Disallow: /*.webm$ Disallow: /*.webp$ Disallow: /*.zip$ Disallow: /*.Zip$ Disallow: /*.ZIP$ # Block all known AI crawlers and assistants # from using content for training AI models. # Source: https://robotstxt.com/ai User-Agent: AI2Bot User-Agent: AI2Bot-Dolma User-agent: Amazonbot User-agent: anthropic-ai User-agent: Applebot-Extended User-Agent: AwarioBot User-Agent: AwarioSmartBot User-Agent: AwarioRssBot User-agent: Bytespider User-agent: CCBot User-agent: ChatGPT-User User-agent: ClaudeBot User-agent: Claude-Web User-agent: cohere-ai User-Agent: cohere-training-data-crawler User-Agent: DataForSeoBot User-agent: Diffbot User-agent: FacebookBot User-agent: facebookexternalhit User-agent: GPTBot User-Agent: ICC-Crawler User-agent: img2dataset User-Agent: ImagesiftBot User-Agent: Kangaroo Bot User-Agent: Meltwater User-agent: Meta-ExternalAgent User-Agent: Meta-ExternalFetcher User-agent: MJ12bot User-agent: OAI-SearchBot User-agent: omgili User-agent: omgilibot User-Agent: PanguBot User-Agent: peer39_crawler User-agent: PerplexityBot User-agent: PetalBot User-agent: Scrapy User-Agent: Seekr User-Agent: Sentibot User-Agent: SemrushBot-OCOB User-agent: Timpibot User-Agent: TurnitinBot User-agent: VelenPublicWebCrawler User-Agent: webzio-extended User-agent: YandexBot User-agent: YouBot Disallow: / # Block any non-specified AI crawlers (e.g., new # or unknown bots) from using content for training # AI models. This directive is still experimental # and may not be supported by all AI crawlers. User-Agent: * DisallowAITraining: / # START YOAST BLOCK # --------------------------- User-agent: * Disallow: /?s= Disallow: /page/*/?s= Disallow: /search/ Disallow: /wp-admin/ Disallow: /fonts/ # --------------------------- # END YOAST BLOCK Sitemap: https://profferfish.com/sitemap_index.xml Sitemap: https://profferfish.com/post-sitemap.xml Sitemap: https://profferfish.com/page-sitemap.xml Sitemap: https://profferfish.com/thim_elementor_kit-sitemap.xml Sitemap: https://profferfish.com/category-sitemap.xml Sitemap: https://profferfish.com/post_tag-sitemap.xml