settings.py

import re
useragent_bots = [
    r"^(AdsBot|Mediapartners)-Google",
    r"^Googlebot",
    r"^Mozilla/5.0 \(compatible; Googlebot/2.1;",
    r".+\(compatible; Googlebot-Mobile/",
    r"gsa-crawler \(Enterprise; S4-E9LJ2B82FJJAA; me@mycompany.com\)",
    "^Fast Crawler v X",
    r"^Mozilla/4.0 (compatible; MSIE 5.0; Windows NT)$",
    r"^Mozilla/4.0$",
    "Mozilla/4.05 [en]",
    "Mozilla/5.0 (Slurp/cat; slurp@inktomi.com; http://www.inktomi.com/slurp.html)",
    r"^Mozilla/5.0 \(compatible; Yahoo!",
    "Scooter/3.3",
    "Y!J-BSC/1.0 (http://help.yahoo.co.jp/help/jp/blog-search/)",
    "Yahoo! Mindset",
    r"^Yahoo-",
    r"^YahooFeedSeeker",
    r"^YahooSeeker",
    "YahooVideoSearch www.yahoo.com/",
    r"^YahooYSMcm",
    r"^slurp",
    r"^Mozilla/5.0 \(compatible; bingbot/",
    r"^Mozilla/5.0 \(compatible; YandexBot",
    r"^Mozilla/5.0 \(compatible; AhrefsBot/",
    r"^Mozilla/5.0 \(compatible; Baiduspider/",
    r"^Mozilla/5.0 \(compatible; IstellaBot/",
    r"^Mozilla/5.0 \(compatible; MJ12bot/v",
    r"^Mozilla/5.0 \(compatible; ProCogSEOBot/",
    r"^Mozilla/5.0 \(SemrushBot/",
    r"^Mozilla/5.0 \(special_archiver/",
    r"^ia_archiver",
    r"^QuerySeekerSpider",
    r"^python-requests/",
    r"^Python-urllib/",
    r"^libcurl/",
    r"^Wget/",
    r"^Java/"
]