User-agent: *
Crawl-delay: 5
Disallow: /admin/
Disallow: /Core/
Disallow: /tmp/
Disallow: /views/
Disallow: /Setup/
Disallow: /log/
#
Disallow: /newsletter/
Disallow: /en/newsletter/
Disallow: /index.php?cl=newsletter
#
Disallow: /AGB/
Disallow: /en/Terms-and-Conditions/
#
Disallow: /warenkorb/
Disallow: /en/cart/
Disallow: /index.php?cl=basket
#
Disallow: /mein-konto/
Disallow: /en/my-account/
Disallow: /index.php?cl=account
#
Disallow: /mein-merkzettel/
Disallow: /en/my-wishlist/
Disallow: /index.php?cl=account_noticelist
#
Disallow: /mein-wunschzettel/
Disallow: /en/my-gift-registry/
Disallow: /index.php?cl=account_wishlist
#
Disallow: /konto-eroeffnen/
Disallow: /en/open-account/
Disallow: /index.php?cl=register
#
Disallow: /passwort-vergessen/
Disallow: /en/forgot-password/
Disallow: /index.php?cl=forgotpwd
#
Disallow: /index.php?cl=moredetails
#
Disallow: /index.php?cl=review
#
Disallow: /index.php?cl=search
#
Disallow: /EXCEPTION_LOG.txt
#
# wildcards at the end, because of some crawlers see it as errors
Disallow: /*?cl=newsletter
Disallow: /*&cl=newsletter
#
Disallow: /*?cl=basket
Disallow: /*&cl=basket
#
Disallow: /*?cl=account
Disallow: /*&cl=account
#
Disallow: /*?cl=account_noticelist
Disallow: /*&cl=account_noticelist
#
Disallow: /*?cl=account_wishlist
Disallow: /*&cl=account_wishlist
#
Disallow: /*?cl=register
Disallow: /*&cl=register
#
Disallow: /*?cl=forgotpwd
Disallow: /*&cl=forgotpwd
#
Disallow: /*?cl=moredetails
Disallow: /*&cl=moredetails
#
Disallow: /*?cl=review
Disallow: /*&cl=review
#
Disallow: /*?cl=search
Disallow: /*&cl=search
#
Disallow: /*&fnc=tobasket
Disallow: /*&fnc=tocomparelist
Disallow: /*&addcompare=
#
Disallow: /*/sid/
Disallow: /*?sid=
Disallow: /*&sid=
#
Disallow: /*?cur=
Disallow: /*&cur


User-agent: emetriqContextualBot
Allow: /

User-agent: Mozilla/5.0 (compatible; OGDWCtxCrawler)
Allow: /

User-agent: cohere-ai
Disallow: /

User-agent: cohere-training-data-crawler
Disallow: /

User-agent: Webzio-Extended
Disallow: /

User-agent: YouBot
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: magpie-crawler
Disallow: /

User-agent: ia_archiver
Disallow: /

User-Agent: omgili
Disallow: /

User-Agent: omgilibot
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: Yeti
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: sentibot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: SirdataBot
Disallow: /

User-agent: LCC
Disallow: /

User-agent: TurnitinBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: dotbot
Disallow: /

User-Agent: ImagesiftBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Timpibot
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: Diffbot
Disallow: /


#Block bots
User-agent: A6-Indexer
Disallow: /

User-agent: AlphaSeoBot
Disallow: /

User-agent: AlphaSeoBot-SA
Disallow: /

# RDH, 08.19.19: I really don't want to block Applebot, but for now, I am. It is crawling us too much
User-agent: Applebot
Disallow: /

User-agent: AspiegelBot
Disallow: /

User-agent: barkrowler
Disallow: /

# RDH, 05.13.20: I really don't want to block bing, but for now, I am. It is also already in htaccess rules
User-agent: bingbot/2.0
Disallow: /

User-agent: Blackboard Safeassign
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: crawler4j
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: DotBot
Disallow: /

# RDH, 06.30.21: Very temporary to get some relief.
# User-Agent: Googlebot
# Disallow: /

User-agent: Gigabot
Disallow: /

User-agent: LieBaoFast
Disallow: /

User-agent: MauiBot
Disallow:/

User-agent: MauiBot (crawler.feedback+wc@gmail.com)
Disallow: /

User-agent: MegaIndex.ru/2.0
Disallow: /

User-agent: MqqBrowser
Disallow:/

User-agent: Nimbostratus-Bot/v1.3.2
Disallow: /

User-agent: Qwant-news
Disallow: /

User-agent: Qwantify
Disallow: /

User-agent: Seekport Crawler
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: SemrushBot-SA
Disallow: /

User-agent: SeznamBot
Disallow: /

User-agent: SputnikBot/2.3
Disallow:/

User-agent: The Knowledge AI
Disallow:/

User-agent: Timpibot/0.8
Disallow: /

User-agent: TinyTestBot
Disallow: /

# RDH, 03/11/22:
# Comment this out for JOT, who applied for a Crossref Similiarty Check account with TurnitIn;
# User-agent: TurnitinBot
# Disallow: /

User-agent: UCBrowser
Disallow: /

User-agent: yacybot
Disallow: /

User-agent: YandexBot
Disallow: /

User-agent: YandexBot/3.0
Disallow: /

User-agent: Yeti
Disallow: /

User-agent: YisouSpider
Disallow: /

User-agent: meta-webindexer
Disallow: /

Sitemap: https://www.suess-versand.de/export/sitemap.xml
Sitemap: https://www.suess-versand.de/export/sitemap_en.xml