# Domains that disallow PanguBot in robots.txt
# agent: pangubot
# generated: 2026-08-24T08:30:07.000Z
# count: 581
#
# What this count is, so it reconciles with the rate published elsewhere:
#   581 of 46105 domains with a known policy for this agent disallow it (1.3%).
#   Of those that also served a page, 504 of 29823 disallow it (1.7%).
#   https://crawlcensus.com/operator pages quote the second figure, because a site that refuses every
#   programmatic client is not evidence about AI policy. This file lists the first,
#   because a deny list must include a site whose robots.txt disallows you even when
#   its homepage did not answer our probe. Dividing this file's length by the corpus
#   size gives neither figure.
# measured by direct request, not inferred. Method: https://crawlcensus.com/methodology
# cursor: 1787467177.1494
# resume exactly here, no gap and no replay:
#   https://crawlcensus.com/agents/pangubot/changes.json?cursor=1787467177.1494
# this list is robots.txt only. Some domains additionally refuse you at the edge:
#   https://crawlcensus.com/api/v1/preflight
# licence: CC BY 4.0, attribute as "Source: Crawl Census (https://crawlcensus.com)"
01net.com
11freunde.de
15min.lt
20minutos.es
4players.de
588ku.com
aamulehti.fi
abendblatt.de
ad-magazin.de
adweek.com
aftenbladet.no
aftenposten.no
aftonbladet.se
agrarheute.com
al.com
alisweb.org
allgaeuer-zeitung.de
allure.com
amazon.ae
amazon.ca
amazon.co.jp
amazon.co.uk
amazon.co.za
amazon.com
amazon.com.au
amazon.com.br
amazon.com.mx
amazon.com.tr
amazon.de
amazon.eg
amazon.es
amazon.fr
amazon.ie
amazon.in
amazon.it
amazon.jp
amazon.nl
amazon.pl
amazon.sa
amazon.se
amazon.sg
amazonvideo.com
amny.com
amzn.com
anyflip.com
app.com
archinect.com
architecturaldigest.com
architecturaldigest.in
ard.de
ardaudiothek.de
ardmediathek.de
ardsounds.de
argusleader.com
arkansasonline.com
arstechnica.com
askmen.com
augsburger-allgemeine.de
auto-tests.com
autobild.de
autozeitung.de
azcentral.com
babycenter.com
babycentre.co.uk
baltimoresun.com
barcelona.cat
beaconjournal.com
beatstars.com
beatstars.net
bengo4.com
bfmtv.com
bild.de
blipfoto.com
bnn.de
bntu.by
bonappetit.com
bonn.de
boston.com
bostonglobe.com
bostonherald.com
bournemouthecho.co.uk
br.de
braunschweiger-zeitung.de
brigitte.de
bsc.es
bt.no
burlingtonfreepress.com
businessinsider.de
businessinsider.es
butenunbinnen.de
bz-berlin.de
castro.fm
chicagotribune.com
chugoku-np.co.jp
chunichi.co.jp
cincinnati.com
cio.com
citizen-times.com
cjonline.com
clarionledger.com
cleveland.com
cnet.com
cnn.com
cntraveler.com
cntraveller.com
cntraveller.in
commercialappeal.com
computerbild.de
computerhope.com
computerwoche.de
computerworld.com
congress.gov
conjur.com.br
contentmarketinginstitute.com
courier-journal.com
courrierinternational.com
coursehero.com
cruisecritic.com
crunchbase.com
csoonline.com
dailybreeze.com
dailycamera.com
dailyecho.co.uk
dailymail.co.uk
dailymail.com
dailynews.com
dailypress.com
darkreading.com
datacenterknowledge.com
dazeddigital.com
dealereprocess.com
defcon.org
delawareonline.com
delfi.ee
delfi.lt
delfi.lv
democratandchronicle.com
denverpost.com
derstandard.at
derstandard.de
derwesten.de
desertsun.com
desmoinesregister.com
detroitnews.com
dispatch.com
dobrzemieszkaj.pl
dr.dk
duden.de
dvdprime.com
dwds.de
e24.no
eadt.co.uk
eastbaytimes.com
echo-news.co.uk
edp24.co.uk
elgaronline.com
elpasotimes.com
eltern.de
empireonline.com
epicurious.com
equidia.fr
essen-und-trinken.de
estantevirtual.com.br
eumetsat.int
eurogamer.de
eurogamer.net
everydayhealth.com
expresso.pt
extremetech.com
f95zone.to
farmprogress.com
faz.net
flipboard.com
fliphtml5.com
floridatoday.com
forth.gr
france24.com
francebleu.fr
franceculture.fr
franceinter.fr
freep.com
furaffinity.net
futurezone.de
gainesville.com
gala.de
gamasutra.com
gamedeveloper.com
gamesindustry.biz
gds.it
geek.com
gentoo.org
glamour.com
glamourmagazine.co.uk
glasgowtimes.co.uk
gofeminin.de
golf.com
goop.com
gp.se
gq-magazin.de
gq-magazine.co.uk
gq.com
gqindia.com
gqitalia.it
gqmagazine.fr
greenme.it
greenvilleonline.com
grindosaur.com
gunbark.dev
heraldo.es
heraldscotland.com
heraldtribune.com
hessenschau.de
hinative.com
hoerzu.de
hoopshype.com
houseandgarden.co.uk
housepricesintheuk.co.uk
houstonpublicmedia.org
howlongtobeat.com
hs.fi
huffingtonpost.fr
huffingtonpost.it
iacr.org
ici.fr
iex.nl
ign.com
igt.com
ilex.by
ilmanifesto.it
iltalehti.fi
imageshack.com
imageshack.us
ina.fr
indystar.com
inews.co.uk
informationweek.com
inforum.com
infoworld.com
iqiyi.com
is.fi
ithome.com.tw
iza.ne.jp
jacksonville.com
jamanetwork.com
jfklibrary.org
journaldugeek.com
jsonline.com
jtbc.co.kr
jwz.org
kauppalehti.fi
kentonline.co.uk
klingai.com
knoxnews.com
kob.com
kotaku.com
kstp.com
lacucinaitaliana.it
lancashiretelegraph.co.uk
lansingstatejournal.com
larazon.es
lasillavacia.com
lastampa.it
launchpad.net
law.com
lci.fr
lecker.de
leckerschmecker.me
lehighvalleylive.com
lemonde.fr
lesechos.fr
letterboxd.com
letudiant.fr
levels.fyi
lifehacker.com
lightreading.com
livestrong.com
lohud.com
lyricstranslate.com
macworld.com
mailonsunday.co.uk
mailplus.co.uk
main-echo.de
mainpost.de
manager-magazin.de
mania.kr
mapgenie.io
mapquest.com
marinij.com
mashable.com
masslive.com
maville.com
maxroll.gg
mbta.com
mcall.com
mdr.de
mediamarkt.de
mediamarkt.nl
mediamarkt.pl
mediaworld.it
medpagetoday.com
mercurynews.com
metro.co.uk
miricanvas.com
mlive.com
monoskop.org
montgomeryadvertiser.com
mora.jp
morgenpost.de
msn.com
mtv.fi
myanimelist.net
myporntube.com
myspreadshop.co.uk
myspreadshop.com
myspreadshop.de
myspreadshop.net
nachrichten.at
naplesnews.com
nationalgeographic.de
nd-aktuell.de
ndr.de
nefisyemektarifleri.com
nettavisen.no
news-journalonline.com
news-leader.com
news-postseven.com
news-press.com
newsandstar.co.uk
newscientist.com
newspapers.com
newyorker.com
nicematin.com
niche.com
nintendo.co.jp
nintendo.com
nj.com
nn.de
nordbayern.de
northjersey.com
noticiasaominuto.com
nrc.nl
nrk.no
nrn.com
nrz.de
nybooks.com
nydailynews.com
nzherald.co.nz
observador.pt
occrp.org
ocregister.com
oefb.at
oekotest.de
ohtuleht.ee
oklahoman.com
omroepwest.nl
opovo.com.br
oregonlive.com
otz.de
oxfordmail.co.uk
palmbeachpost.com
pandora.com
parismatch.com
pcmag.com
pcworld.com
penguin.de
pennlive.com
piliapp.com
pilotonline.com
pimpbunny.com
pitchfork.com
pjstar.com
planet-wissen.de
pnj.com
politico.eu
portableapps.com
portalsamorzadowy.pl
posfie.com
posteo.de
postimees.ee
premiumoutlets.com
pressdemocrat.com
presse-citron.net
presstelegram.com
primevideo.com
providencejournal.com
pubhtml5.com
radiofrance.fr
rbb-online.de
rbb24.de
realcommercial.com.au
realestate.com.au
refinery29.com
registerguard.com
repubblica.it
revistagq.com
revistavanityfair.es
rfi.fr
rgj.com
road.cc
rockpapershotgun.com
rsi.ch
rts.ch
rtvdrenthe.nl
rtvnoord.nl
rtvutrecht.nl
ruhrnachrichten.de
rynekzdrowia.pl
salzburg24.at
sandiegouniontribune.com
sankei.com
sanspo.com
santacruzsentinel.com
saturn.de
savannahnow.com
schulferien.org
scoop.co.nz
scotusblog.com
seacoastonline.com
seekingalpha.com
self.com
sfstandard.com
sicnoticias.pt
silive.com
simon.com
sj-r.com
slate.com
sme.sk
sn.at
snapchat.com
southwalesargus.co.uk
spiegel.de
sportler.com
sportschau.de
spreadshirt.com
sr.de
srf.ch
statesman.com
statesmanjournal.com
statnews.com
stimme.de
storyblocks.com
suedkurier.de
supermarketnews.com
svd.se
svt.se
svtplay.se
swr.de
swr3.de
syracuse.com
t3n.de
tagesschau.de
tagesspiegel.de
tallahassee.com
tatler.com
tcpalm.com
techadvisor.com
techbook.de
ted.com
teenvogue.com
telegram.com
telegraph.co.uk
tennessean.com
tf1.fr
tf1info.fr
theanalyst.com
theargus.co.uk
thebulletin.org
theconversation.com
theepochtimes.com
theglobeandmail.com
theledger.com
themoviedb.org
thenational.scot
thenorthernecho.co.uk
thespec.com
thestar.com
thetelegraphandargus.co.uk
thisismoney.co.uk
thueringer-allgemeine.de
timesfreepress.com
tmdb.org
togetter.com
tokyo-np.co.jp
topekalibraries.info
travelbook.de
tripadvisor.at
tripadvisor.be
tripadvisor.ca
tripadvisor.ch
tripadvisor.co
tripadvisor.co.id
tripadvisor.co.nz
tripadvisor.co.uk
tripadvisor.co.za
tripadvisor.com
tripadvisor.com.au
tripadvisor.com.br
tripadvisor.com.gr
tripadvisor.com.hk
tripadvisor.com.mx
tripadvisor.com.my
tripadvisor.com.ph
tripadvisor.com.tr
tripadvisor.de
tripadvisor.es
tripadvisor.fr
tripadvisor.ie
tripadvisor.in
tripadvisor.jp
tripadvisor.nl
tripadvisor.pt
tripadvisor.ru
tu.berlin
tv2.dk
tv4.se
tv4play.se
tvmovie.de
tvnet.lv
twincities.com
u-cursos.cl
uesp.net
ultimahora.es
unil.ch
uol.com
uol.com.br
usatoday.com
usmagazine.com
utopia.de
vanityfair.com
vanityfair.fr
vanityfair.it
vcstar.com
veikkaus.fi
vermont.gov
vg.no
vg247.com
vivino.com
vndb.org
vogue.co.jp
vogue.co.uk
vogue.com
vogue.com.tw
vogue.de
vogue.es
vogue.fr
vogue.in
vogue.it
vogue.mx
vozpopuli.com
waz.de
wbur.org
wdr.de
wealthmanagement.com
welt.de
werstreamt.es
whattoexpect.com
wickedlocal.com
wired.co.uk
wired.com
wired.it
wired.jp
wnp.pl
womansworld.com
worcesternews.co.uk
worldsurfleague.com
wp.de
wpr.org
wral.com
wunderweib.de
xe.gr
yle.fi
ylilauta.org
yorkpress.co.uk
ytmnd.com
yugipedia.com
zakzak.co.jp
zdnet.com
zeit.de
zepto.com
zeptonow.com
zin.ru