## 2025-05-23

#  Blocked by WAF: BLEXBot, SemrushBot, Geedo, PetalBot, Mojeek, Seznam, Qwant, Sogou, AhrefsBot, turnitinbot, Implisensebot, AwarioBot, DataForSeoBot, SEBot-WA, BotPoke, Amazonbot, barkrowler, GoogleDocs

#  AI Bots (https://github.com/ai-robots-txt/ai.robots.txt/blob/main/robots.txt)
User-agent: Brightbot 1.0
User-agent: Bytespider
User-agent: CCBot
User-agent: cohere-training-data-crawler
User-agent: Crawlspace
User-agent: Diffbot
User-agent: DuckAssistBot
User-agent: iaskspider/2.0
User-agent: ICC-Crawler
User-agent: ImagesiftBot
User-agent: img2dataset
User-agent: ISSCyberRiskCrawler
User-agent: Kangaroo Bot
User-agent: omgili
User-agent: omgilibot
User-agent: PanguBot
User-agent: Sidetrade indexer bot
User-agent: Scrapy
User-agent: Timpibot
User-agent: VelenPublicWebCrawler
User-agent: Webzio-Extended
User-agent: YouBot
User-agent: Amazonbot
Disallow: /

#  MetaJob Bot (https://www.metajob.de/the/crawler)
User-agent: metajobbot
Allow: /de/jobs/
Disallow: /


User-agent: *
#  Suche
Disallow: /de/suche/
Disallow: /en/search/
# parameters
Disallow: *?bi=
Disallow: *&bi=
Disallow: *?dc=
Disallow: *&dc=
Disallow: *?kf*
Disallow: *&kf*
Disallow: *?f=*
Disallow: *&f=*
Disallow: *?s=*
Disallow: *&s=*
Disallow: *?aa=*
Disallow: *&aa=*
Disallow: *?action=*
Disallow: *&action=*
Disallow: *buy_now*
Disallow: *?shop=*
Disallow: *&shop=*
# directories
Disallow: /media/
Disallow: /download/
Disallow: /wug/
# files DE
Disallow: /download-file*
Disallow: /meine-fahrzeuge*
Disallow: /anmelden*
Disallow: /abmelden*
Disallow: /auftrag*
Disallow: /kundenkonto*
Disallow: /passwort*
# files EN
Disallow: /download-file*
Disallow: /my-cars*
Disallow: /login*
Disallow: /logoff*
Disallow: /order*
Disallow: /*-account*
Disallow: /*-passwort*
# cleanpath DE
Disallow: /de/suche/
Disallow: /de/download-file/
Disallow: /de/fahrzeug-identifikation/*
Disallow: /de/meine-fahrzeuge/
Disallow: /de/fahrzeug-auswahl/*action*
Disallow: /de/fahrzeug-auswahl/*hid*
Disallow: /de/fahrzeug-auswahl/*fin*
Disallow: /de/anmelden/
Disallow: /de/abmelden/
Disallow: /de/auftrag*
Disallow: /de/kundenkonto*
Disallow: /de/passwort*
# cleanpath EN
Disallow: /en/search/
Disallow: /en/download-file/
Disallow: /en/my-cars/
Disallow: /en/vehicle-identification/*
Disallow: /en/vehicle-selection/*action*
Disallow: /en/vehicle-selection/*hid*
Disallow: /en/vehicle-selection/*fin*
Disallow: /en/login/
Disallow: /en/logoff/
Disallow: /en/order*
Disallow: /en/*-account*
Disallow: /en/*-passwort*

Crawl-delay: 10