User-agent: 008
Disallow: /

User-agent: Amazonbot               # Amazon's user agent
Disallow: /do-not-crawl/            # disallow this directory

User-agent: Bytespider                 
Disallow: /            # disallow this directory

User-agent: spider-feedback@bytedance.com 
Disallow: /            # disallow this directory

User-agent: Googlebot-News
Disallow: /
Disallow: /live_new/
Allow: /news/

User-agent: YandexVideoParser
Disallow: /

User-agent: veveobot
Disallow: /

User-agent: dotbot
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: SemrushBot
Disallow: /

User-Agent: trendictionbot
Disallow: /

User-agent: MauiBot
Disallow: /

User-agent: Yandex
Disallow: /

User-agent: *
Disallow: /forum/
Disallow: /new/
Disallow: /mob/
Disallow: /live_new/
Disallow: /livwm/
Disallow: /mobile/
Disallow: /interactive/movies/
Disallow: /movies/custom-comparisons-extended/
Disallow: /movies/keywords-extended/

User-agent: BingBot 
Crawl-delay: 10
Disallow: /forum/
Disallow: /new/
Disallow: /mob/
Disallow: /live_new/
Disallow: /livwm/
Disallow: /mobile/
Disallow: /interactive/movies/
Disallow: /movies/custom-comparisons-extended/
Disallow: /movies/keywords-extended/

User-agent: Applebot 
Crawl-delay: 10
Disallow: /forum/
Disallow: /new/
Disallow: /mob/
Disallow: /live_new/
Disallow: /livwm/
Disallow: /mobile/
Disallow: /interactive/movies/
Disallow: /movies/custom-comparisons-extended/
Disallow: /movies/keywords-extended/

User-agent: ahrefsbot
Crawl-delay: 10
Disallow: /forum/
Disallow: /new/
Disallow: /mob/
Disallow: /live_new/
Disallow: /livwm/
Disallow: /mobile/
Disallow: /interactive/movies/
Disallow: /movies/custom-comparisons-extended/
Disallow: /movies/keywords-extended/

User-agent: yeti
Crawl-delay: 10
Disallow: /forum/
Disallow: /new/
Disallow: /mob/
Disallow: /live_new/
Disallow: /livwm/
Disallow: /mobile/
Disallow: /interactive/movies/
Disallow: /movies/custom-comparisons-extended/
Disallow: /movies/keywords-extended/

User-agent: rogerbot
Crawl-delay: 10
Disallow: /forum/
Disallow: /new/
Disallow: /mob/
Disallow: /live_new/
Disallow: /livwm/
Disallow: /mobile/
Disallow: /interactive/movies/
Disallow: /movies/custom-comparisons-extended/
Disallow: /movies/keywords-extended/

User-agent: BLEXBot
Crawl-delay: 60
Disallow: /forum/
Disallow: /new/
Disallow: /mob/
Disallow: /live_new/
Disallow: /livwm/
Disallow: /mobile/
Disallow: /interactive/movies/
Disallow: /movies/custom-comparisons-extended/
Disallow: /movies/keywords-extended/

User-agent: proximic
Crawl-delay: 60
Disallow: /forum/
Disallow: /new/
Disallow: /mob/
Disallow: /live_new/
Disallow: /livwm/
Disallow: /mobile/
Disallow: /interactive/movies/
Disallow: /movies/custom-comparisons-extended/
Disallow: /movies/keywords-extended/

Sitemap: http://www.the-numbers.com/sitemap.xml