{"id":"7200f9a2-46b0-4c9d-9aea-f94557aa384e","shortId":"uFpTGZ","kind":"x402","title":"Fetch and parse robots.txt for a domain","tagline":"Fetch and parse robots.txt for a domain. Returns parsed allow/disallow rules per user-agent, sitemap URLs, crawl-delay d","description":"Fetch and parse robots.txt for a domain. Returns parsed allow/disallow rules per user-agent, sitemap URLs, crawl-delay directives, and raw content. Built for AI agents planning ethical web crawling and SEO audit. Use ?domain=example.com.","tags":[],"capabilities":["x402-base","http-get"],"categories":[],"synonyms":[],"warnings":[],"endpointUrl":"https://api.x402node.dev/web/robots","protocol":"x402","transport":"http","auth":{"type":"x402","details":[{"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","extra":{"name":"USD Coin","version":"2"},"payTo":"0x4466d4A84b7c49a6A094ec6eef4a0712D6dd125e","scheme":"exact","network":"base","mimeType":"application/json","resource":"https://api.x402node.dev/web/robots","description":"Fetch and parse robots.txt for a domain. Returns parsed allow/disallow rules per user-agent, sitemap URLs, crawl-delay directives, and raw content. Built for AI agents planning ethical web crawling and SEO audit. Use ?domain=example.com.","outputSchema":{"input":{"type":"http","method":"GET","discoverable":true}},"maxAmountRequired":"5000","maxTimeoutSeconds":60}]},"qualityScore":"0.000","qualityRationale":null,"verified":false,"liveness":"unknown","lastLivenessCheck":null,"agentReviews":{"count":0,"score_avg":null,"cost_usd_avg":null,"success_rate":null,"latency_p50_ms":null,"narrative_summary":null,"summary_updated_at":null},"enrichmentModel":null,"enrichmentVersion":1,"enrichedAt":null,"embedding":null,"createdAt":"2026-05-28T18:36:04.849Z","updatedAt":"2026-06-22T00:31:27.047Z","lastSeenAt":"2026-06-22T00:31:27.047Z","tsv":"'agent':22,43,56 'ai':55 'allow/disallow':17,38 'audit':63 'built':53 'content':52 'crawl':26,47,60 'crawl-delay':25,46 'd':28 'delay':27,48 'direct':49 'domain':7,14,35,65 'ethic':58 'example.com':66 'fetch':1,8,29 'http-get' 'pars':3,10,16,31,37 'per':19,40 'plan':57 'raw':51 'return':15,36 'robots.txt':4,11,32 'rule':18,39 'seo':62 'sitemap':23,44 'url':24,45 'use':64 'user':21,42 'user-ag':20,41 'web':59 'x402-base'","prices":[{"id":"c9fcded3-8ebb-480b-91f7-4e9286ced45a","listingId":"7200f9a2-46b0-4c9d-9aea-f94557aa384e","amountUsd":null,"unit":"per_call","nativeCurrency":"USD COIN","nativeAmount":"5000","chain":"base","payTo":"0x4466d4A84b7c49a6A094ec6eef4a0712D6dd125e","paymentMethod":"x402-usd coin","isPrimary":true,"details":{"scheme":"exact","mime_type":"application/json","accepts_all":[{"asset":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","extra":{"name":"USD Coin","version":"2"},"payTo":"0x4466d4A84b7c49a6A094ec6eef4a0712D6dd125e","scheme":"exact","network":"base","mimeType":"application/json","resource":"https://api.x402node.dev/web/robots","description":"Fetch and parse robots.txt for a domain. Returns parsed allow/disallow rules per user-agent, sitemap URLs, crawl-delay directives, and raw content. Built for AI agents planning ethical web crawling and SEO audit. Use ?domain=example.com.","outputSchema":{"input":{"type":"http","method":"GET","queryParams":{},"discoverable":true},"output":{"type":"object","properties":{}}},"maxAmountRequired":"5000","maxTimeoutSeconds":60}],"asset_contract":"0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913","max_timeout_seconds":60},"createdAt":"2026-05-28T18:36:04.849Z"}],"sources":[{"listingId":"7200f9a2-46b0-4c9d-9aea-f94557aa384e","source":"x402_bazaar","sourceId":"https://api.x402node.dev/web/robots","sourceUrl":"https://api.cdp.coinbase.com/platform/v2/x402/discovery/resources","isPrimary":true,"firstSeenAt":"2026-05-28T18:36:04.849Z","lastSeenAt":"2026-06-22T00:31:27.047Z"}],"details":{"listingId":"7200f9a2-46b0-4c9d-9aea-f94557aa384e","quickStartSnippet":null,"exampleRequest":null,"exampleResponse":null,"schema":{"input":{"type":"http","method":"GET","discoverable":true}},"openapiUrl":null,"agentsTxtUrl":null,"citations":[],"useCases":[],"bestFor":[],"notFor":[],"kindDetails":{"x402_version":1,"resource_type":"http","bazaar_last_updated":"2026-06-12T02:43:14.565Z"},"updatedAt":"2026-06-22T00:31:27.047Z"}}