# 29/12/2020 paroles.cc/www/robots.txt: text/plain; charset=ütf-8
# 07/01/2021

User-agent: ia_archiver
Disallow: /

User-agent: dnslookup.fr
Disallow: /

# "Mozilla/5.0 (compatible; AhrefsBot/7.0; +http://ahrefs.com/robot/)"

User-agent: OFF_AhrefsBot
Disallow: /

# "Mozilla/5.0 (compatible; BLEXBot/1.0; +http://webmeup-crawler.com/)"

User-agent: BLEXBot
Crawl-delay: 10

# "Mozilla/5.0 (compatible; DotBot/1.1; http://www.opensiteexplorer.org/dotbot, help@moz.com)"
# https://moz.com/help/moz-procedures/crawlers/dotbot

User-agent: OFF_dotbot
Disallow: /

# "Mozilla/5.0 (compatible; MJ12bot/v1.4.8; http://mj12bot.com/)"

User-agent: MJ12bot
Disallow: /

# "Mozilla/5.0 (compatible; SemrushBot/7~bl; +http://www.semrush.com/bot.html)"

User-agent: OFF_SemrushBot
Disallow: /

# "Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)"

User-agent: OFF_YandexBot
User-agent: OFF_Yandex
Disallow: /

User-agent: HTTrack
User-agent: Java
User-agent: libwww
User-agent: LookSmart
User-agent: MSIECrawler
User-agent: newspaper/0.2.8
User-agent: Offline Explorer
User-agent: Python-urllib
User-agent: SiteSucker/2.0
User-agent: Teleport
User-agent: TeleportPro
User-agent: WebCopier
User-agent: WebZip
User-agent: Wget
User-agent: WinHTTrack
User-agent: ZyBorg
Disallow: /

# others

User-agent: *
Disallow: /adform/
Disallow: /admin
Disallow: /blog/
Disallow: /cache/
Disallow: /cgi-bin/
Disallow: /drupal/
Disallow: /forum/
Disallow: /honeypot
Disallow: /joomla/
Disallow: /login
Disallow: /openmr/
Disallow: /plus/
Disallow: /trackback/
Disallow: /wp-admin/
Disallow: /wp-content/
Disallow: /wp-login/
Disallow: /noc/
Disallow: /rtm/

#Disallow: /fr/
#Disallow: /*,tous

Sitemap: https://www.paroles.cc/sitemap.xml