User-agent:* Disallow: /cgi-bin/ Disallow: /cdn-cgi/* Disallow: /config/ Disallow: */analysis-logger/* Disallow: /todays-paper/*article*.ece* Disallow: /migration_catalog/ Disallow: */getapplink* Disallow: /summercamp/ Disallow: */wf.fragment/* Disallow: /poll/vote.do Disallow: /photo/10000/ Disallow: */photo/10000/ #Disallow: /newsletter/* Disallow: /news-service/* #Disallow: /ebooks/ Disallow: /search/ Disallow: /SEARCH/ Disallow: /Search/ Disallow: /22390678/ Disallow: /walkathon Disallow: /heelsonwheels Disallow: /archive/print/ #Disallow: /todays-paper/ #Disallow: /static/ Disallow: /todayspaper/ Disallow: /today-paper/ Disallow: /thseweekend Disallow: /ywsummercamp Disallow: /frnf2014 Disallow: /summermusic Disallow: /summercamp Disallow: /yw25 Disallow: /thischess Disallow: /ywquiz Disallow: /mumbaiedition Disallow: /marathon Disallow: /flndrmk-spl Disallow: /system/ Disallow: /sachin Disallow: /supermom2016 Disallow: /ywq2015 Disallow: /fooddirectory Disallow: /chennaicoastalcleanup Disallow: /onam-1 Disallow: /onam-2 Disallow: /hohkidscarnival Disallow: /HOW2014 Disallow: /how2014 Disallow: /cookingcontest Disallow: /youngworld Disallow: /impulse2014 Disallow: /mssaward2015 Disallow: /mssaward2016 Disallow: /wow2015 Disallow: /wow2017/chennai/ Disallow: /wow2017/bangalore/ Disallow: /tickets2016/ Disallow: /tickets2017/ Disallow: /tickets2018/ Disallow: /ticket2018/ Disallow: /ywsummercamp/ Disallow: /tags/TH category/ Disallow: /tags/TopicRoot_TH/ Disallow: /thehindu/ Disallow: /200* Disallow: /201* Disallow: */http:* Disallow: */https:* Disallow: */mailto:* Disallow: *.ecehttp* Disallow: *.ece1http* Disallow: *.ece2http* # Blocked _ptid too many 5xx errors over Google search console (in any case # this is internal tracking so blocking doesn't affect) #Disallow: */?_ptid=* # Block picture sitemap because urls are noindex therein Disallow: /sitemap/archive/picture/ # Block all space or %20 suffixed with http or https protocols Disallow: * http:* Disallow: * https:* Disallow: *;http:* Disallow: *;https:* Disallow: *%20http:* Disallow: *%20https:* Disallow: */couponRedirect Disallow: *?redirect= Disallow: *?store= Disallow: /tag/*/*/ # Blocked until duplicate profile bylines fixed Disallow: /profile/ # profile articles allowed Allow: /profile/*article*.ece # profile exceptions allowed for pages with page=1 for rediects Allow: /profile/contributor/? Allow: /profile/photographers/? Allow: /profile/author/? Allow: /profile/contributor/$ Allow: /profile/photographers/$ Allow: /profile/author/$ Allow: /profile/$ Disallow: /sitemap/archive* Disallow: */fragment/* Disallow: /sitemap/ Allow: /sitemap/$ #Disallow Wayback machine which is a problem crawler User-Agent: ia_archiver Disallow: / #Disallow ChatGPT from extracting or interpreting our content User-agent: GPTBot Disallow: / Sitemap: https://www.thehindu.com/feeder/default.rss Sitemap: https://www.thehindu.com/coupons/sitemap.xml