# START YOAST BLOCK
# ---------------------------
User-agent: *
Disallow:

Sitemap: https://www.agbi.com/sitemap_index.xml
# ---------------------------
# END YOAST BLOCK

# Custom robots.txt content
# -------------------------
User-agent: AI-SearchBot
User-agent: AliyunSecBot
User-agent: Amazonbot
User-agent: anthropic-ai
User-agent: Applebot-Extended
User-agent: archive.org_bot
User-agent: AudigentAdBot
User-agent: AwarioBot
User-agent: barkrowler
User-agent: BLEXBot
User-agent: Brightbot 1.0
User-agent: Bytespider
User-agent: CCBot
User-agent: Claudebot
User-agent: CriteoBot
User-agent: DataForSeoBot
User-agent: Diffbot
User-agent: dotbot
User-agent: FacebookBot
User-agent: Google-Extended
User-agent: GPTBot
User-agent: ias-ie
User-agent: ias-va
User-agent: LivelapBot
User-agent: Meta-ExternalAgent
User-agent: MJ12bot
User-agent: Moreover
User-agent: NoahBot
User-agent: omgili
User-agent: Paqlebot
User-agent: peer39_crawler
User-agent: PetalBot
User-agent: proximic
User-agent: Quantcastbot
User-agent: SentiBot
User-agent: SEOkicks
User-agent: SeznamBot
User-agent: SirdataBot
User-agent: TaraGroup Intelligent Bot V1
User-agent: TikTokSpider
User-agent: Timpibot
User-agent: trendictionbot
User-agent: TTD-Content
User-agent: TurnitinBot
User-agent: um-LN
User-agent: weborama-fetcher
User-agent: YaK
User-agent: YouBot
User-agent: ZoomBot
User-agent: ZoominfoBot
Disallow: /

User-agent: AhrefsBot
Crawl-delay: 2