{"id":"0e82f0db-739a-4ec6-bd35-894677e85467","shortId":"jtKtda","kind":"skill","title":"Common Crawl Index Query Agent","tagline":"Queries the Common Crawl Index API for large-scale web archive research and data extraction. Uses the CDX Server API, WARC record parsing with warcio, and the Common Crawl S3 bucket for bulk data access.","description":"# Common Crawl Index Query Agent\n\nQueries the Common Crawl Index API for large-scale web archive research and data extraction. Uses the CDX Server API, WARC record parsing with warcio, and the Common Crawl S3 bucket for bulk data access.\n\n## Installation\n\nBasic usage or getting-started notes:\n- Common Crawl data is stored on Amazon Web Services' Public Data Sets . All data and index files are free to download. Feel free to run your own index server, or analyze the index offline.\n- More about the URL index in the original announcement . For help, visit the Common Crawl user forum or Discord server . See also Getting Started .\n\n- Source: https://index.commoncrawl.org/\n\n## Documentation\n\n- https://index.commoncrawl.org/\n\n## Source\n\n- [Agent Skill Exchange](https://agentskillexchange.com/skills/common-crawl-index-query-agent/)","tags":["common","crawl","index","query","agent","skills","agentskillexchange","agent-skills","ai-agents","ai-tools","awesome-list","claude-code"],"capabilities":["skill","source-agentskillexchange","skill-common-crawl-index-query-agent","topic-agent-skills","topic-ai-agents","topic-ai-tools","topic-awesome-list","topic-claude-code","topic-codex","topic-cursor","topic-llm","topic-mcp","topic-npx-skills","topic-openclaw","topic-skills-catalog"],"categories":["skills"],"synonyms":[],"warnings":[],"endpointUrl":"https://skills.sh/agentskillexchange/skills/common-crawl-index-query-agent","protocol":"skill","transport":"skills-sh","auth":{"type":"none","details":{"cli":"npx skills add agentskillexchange/skills","source_repo":"https://github.com/agentskillexchange/skills","install_from":"skills.sh"}},"qualityScore":"0.454","qualityRationale":"deterministic score 0.45 from registry signals: · indexed on github topic:agent-skills · 8 github stars · SKILL.md body (834 chars)","verified":false,"liveness":"unknown","lastLivenessCheck":null,"agentReviews":{"count":0,"score_avg":null,"cost_usd_avg":null,"success_rate":null,"latency_p50_ms":null,"narrative_summary":null,"summary_updated_at":null},"enrichmentModel":"deterministic:skill-github:v1","enrichmentVersion":1,"enrichedAt":"2026-05-18T19:09:53.848Z","embedding":null,"createdAt":"2026-05-18T13:15:47.386Z","updatedAt":"2026-05-18T19:09:53.848Z","lastSeenAt":"2026-05-18T19:09:53.848Z","tsv":"'/skills/common-crawl-index-query-agent/)':159 'access':41,82 'agent':5,46,154 'agentskillexchange.com':158 'agentskillexchange.com/skills/common-crawl-index-query-agent/)':157 'also':146 'amazon':97 'analyz':121 'announc':133 'api':11,26,52,67 'archiv':17,58 'basic':84 'bucket':37,78 'bulk':39,80 'cdx':24,65 'common':1,8,34,42,49,75,91,138 'crawl':2,9,35,43,50,76,92,139 'data':20,40,61,81,93,101,104 'discord':143 'document':151 'download':111 'exchang':156 'extract':21,62 'feel':112 'file':107 'forum':141 'free':109,113 'get':88,147 'getting-start':87 'help':135 'index':3,10,44,51,106,118,123,129 'index.commoncrawl.org':150,152 'instal':83 'larg':14,55 'large-scal':13,54 'note':90 'offlin':124 'origin':132 'pars':29,70 'public':100 'queri':4,6,45,47 'record':28,69 'research':18,59 'run':115 's3':36,77 'scale':15,56 'see':145 'server':25,66,119,144 'servic':99 'set':102 'skill':155 'skill-common-crawl-index-query-agent' 'sourc':149,153 'source-agentskillexchange' 'start':89,148 'store':95 'topic-agent-skills' 'topic-ai-agents' 'topic-ai-tools' 'topic-awesome-list' 'topic-claude-code' 'topic-codex' 'topic-cursor' 'topic-llm' 'topic-mcp' 'topic-npx-skills' 'topic-openclaw' 'topic-skills-catalog' 'url':128 'usag':85 'use':22,63 'user':140 'visit':136 'warc':27,68 'warcio':31,72 'web':16,57,98","prices":[{"id":"0c0c86cb-6d39-4e39-909f-c51f681599e3","listingId":"0e82f0db-739a-4ec6-bd35-894677e85467","amountUsd":"0","unit":"free","nativeCurrency":null,"nativeAmount":null,"chain":null,"payTo":null,"paymentMethod":"skill-free","isPrimary":true,"details":{"org":"agentskillexchange","category":"skills","install_from":"skills.sh"},"createdAt":"2026-05-18T13:15:47.386Z"}],"sources":[{"listingId":"0e82f0db-739a-4ec6-bd35-894677e85467","source":"github","sourceId":"agentskillexchange/skills/common-crawl-index-query-agent","sourceUrl":"https://github.com/agentskillexchange/skills/tree/main/skills/common-crawl-index-query-agent","isPrimary":false,"firstSeenAt":"2026-05-18T13:15:47.386Z","lastSeenAt":"2026-05-18T19:09:53.848Z"}],"details":{"listingId":"0e82f0db-739a-4ec6-bd35-894677e85467","quickStartSnippet":null,"exampleRequest":null,"exampleResponse":null,"schema":null,"openapiUrl":null,"agentsTxtUrl":null,"citations":[],"useCases":[],"bestFor":[],"notFor":[],"kindDetails":{"org":"agentskillexchange","slug":"common-crawl-index-query-agent","github":{"repo":"agentskillexchange/skills","stars":8,"topics":["agent-skills","ai-agents","ai-tools","awesome-list","claude-code","codex","cursor","llm","mcp","npx-skills","openclaw","skills-catalog"],"license":"mit","html_url":"https://github.com/agentskillexchange/skills","pushed_at":"2026-05-18T19:02:17Z","description":"The open catalog of AI agent skills — 2,000+ security-scanned skills for Claude Code, Cursor, Codex, and more.","skill_md_sha":"6acf41bfc659b1c4d8b25f5138d99e11e2846c99","skill_md_path":"skills/common-crawl-index-query-agent/SKILL.md","default_branch":"main","skill_tree_url":"https://github.com/agentskillexchange/skills/tree/main/skills/common-crawl-index-query-agent"},"layout":"multi","source":"github","category":"skills","frontmatter":{"name":"Common Crawl Index Query Agent","description":"Queries the Common Crawl Index API for large-scale web archive research and data extraction. Uses the CDX Server API, WARC record parsing with warcio, and the Common Crawl S3 bucket for bulk data access."},"skills_sh_url":"https://skills.sh/agentskillexchange/skills/common-crawl-index-query-agent"},"updatedAt":"2026-05-18T19:09:53.848Z"}}