{
  "verdicts": [
    {
      "id": "allow",
      "means": "robots.txt permits this agent and a live request carrying its user agent was served.",
      "action": "Fetch it. Nothing measured here stands in your way.",
      "fetch_would_work": true,
      "permitted": true,
      "may_change": true
    },
    {
      "id": "disallow",
      "means": "robots.txt disallows this agent at the site root.",
      "action": "Do not fetch it. This is the site's stated policy, and it holds whatever the network would do and whatever the origin would charge.",
      "fetch_would_work": null,
      "permitted": false,
      "may_change": true
    },
    {
      "id": "refuse",
      "means": "robots.txt permits this agent, and a live request carrying its user agent was refused at the network edge anyway.",
      "action": "Skip it. The permission is not real and the request will cost you a round trip for nothing. Worth telling the site owner if you care about the content: the refusal usually comes from a WAF rule the person who wrote the robots.txt did not intend.",
      "fetch_would_work": false,
      "permitted": true,
      "may_change": true
    },
    {
      "id": "pay",
      "means": "the origin answered this agent with HTTP 402 Payment Required.",
      "action": "Treat it as priced, not blocked. Routing around a 402 takes content the operator is selling. Where robots.txt also disallows you the verdict is disallow instead, because a price is not permission.",
      "fetch_would_work": false,
      "permitted": true,
      "may_change": true
    },
    {
      "id": "unknown",
      "means": "this census has not measured the domain for this agent.",
      "action": "Submit it to /api/v1/scan and ask again, unless the row says measurable is false - that means the site disallows CrawlCensusBot, so no amount of asking will produce an answer and you should read its robots.txt yourself.",
      "fetch_would_work": null,
      "permitted": null,
      "may_change": true
    }
  ],
  "do_not_fetch": [
    "disallow",
    "refuse",
    "pay"
  ],
  "note": "do_not_fetch is derived from the table rather than listed separately, so a client that reads it cannot hold a stale copy of which verdicts mean stop.",
  "license": "CC BY 4.0",
  "attribution": "Source: Crawl Census (https://crawlcensus.com)"
}