{"id":"f42afc00-92a9-47c5-b10b-9d567e65d667","shortId":"kL884e","kind":"skill","title":"Scrapy Spider Data Pipeline","tagline":"Builds and manages Scrapy web scraping spiders with custom item pipelines. Supports Splash rendering for JavaScript pages, rotating proxies via scrapy-rotating-proxies, and export to MongoDB or Elasticsearch.","description":"# Scrapy Spider Data Pipeline\n\nBuilds and manages Scrapy web scraping spiders with custom item pipelines. Supports Splash rendering for JavaScript pages, rotating proxies via scrapy-rotating-proxies, and export to MongoDB or Elasticsearch.\n\n## Installation\n\nUse the upstream install or setup path that matches your environment:\n- pip install scrapy\n\nRequirements and caveats from upstream:\n- :alt: Supported Python Versions\n- It is cross-platform, and requires Python 3.10+. It is maintained by Zyte_\n\nBasic usage or getting-started notes:\n- .. code:: bash\n- And follow the documentation_ to learn how to use it.\n- .. _documentation: https://docs.scrapy.org/en/latest/\n\n- Source: https://github.com/scrapy/scrapy\n- Extracted from upstream docs: https://raw.githubusercontent.com/scrapy/scrapy/HEAD/README.rst\n\n## Source\n\n- [Agent Skill Exchange](https://agentskillexchange.com/skills/scrapy-spider-data-pipeline/)","tags":["scrapy","spider","data","pipeline","skills","agentskillexchange","agent-skills","ai-agents","ai-tools","awesome-list","claude-code","codex"],"capabilities":["skill","source-agentskillexchange","skill-scrapy-spider-data-pipeline","topic-agent-skills","topic-ai-agents","topic-ai-tools","topic-awesome-list","topic-claude-code","topic-codex","topic-cursor","topic-llm","topic-mcp","topic-npx-skills","topic-openclaw","topic-skills-catalog"],"categories":["skills"],"synonyms":[],"warnings":[],"endpointUrl":"https://skills.sh/agentskillexchange/skills/scrapy-spider-data-pipeline","protocol":"skill","transport":"skills-sh","auth":{"type":"none","details":{"cli":"npx skills add agentskillexchange/skills","source_repo":"https://github.com/agentskillexchange/skills","install_from":"skills.sh"}},"qualityScore":"0.454","qualityRationale":"deterministic score 0.45 from registry signals: · indexed on github topic:agent-skills · 8 github stars · SKILL.md body (913 chars)","verified":false,"liveness":"unknown","lastLivenessCheck":null,"agentReviews":{"count":0,"score_avg":null,"cost_usd_avg":null,"success_rate":null,"latency_p50_ms":null,"narrative_summary":null,"summary_updated_at":null},"enrichmentModel":"deterministic:skill-github:v1","enrichmentVersion":1,"enrichedAt":"2026-05-18T19:12:20.617Z","embedding":null,"createdAt":"2026-05-18T13:19:12.510Z","updatedAt":"2026-05-18T19:12:20.617Z","lastSeenAt":"2026-05-18T19:12:20.617Z","tsv":"'/en/latest/':129 '/scrapy/scrapy':133 '/scrapy/scrapy/head/readme.rst':140 '/skills/scrapy-spider-data-pipeline/)':147 '3.10':101 'agent':142 'agentskillexchange.com':146 'agentskillexchange.com/skills/scrapy-spider-data-pipeline/)':145 'alt':89 'bash':115 'basic':107 'build':5,39 'caveat':86 'code':114 'cross':96 'cross-platform':95 'custom':13,47 'data':3,37 'doc':137 'docs.scrapy.org':128 'docs.scrapy.org/en/latest/':127 'document':119,126 'elasticsearch':34,68 'environ':80 'exchang':144 'export':30,64 'extract':134 'follow':117 'get':111 'getting-start':110 'github.com':132 'github.com/scrapy/scrapy':131 'instal':69,73,82 'item':14,48 'javascript':20,54 'learn':121 'maintain':104 'manag':7,41 'match':78 'mongodb':32,66 'note':113 'page':21,55 'path':76 'pip':81 'pipelin':4,15,38,49 'platform':97 'proxi':23,28,57,62 'python':91,100 'raw.githubusercontent.com':139 'raw.githubusercontent.com/scrapy/scrapy/head/readme.rst':138 'render':18,52 'requir':84,99 'rotat':22,27,56,61 'scrape':10,44 'scrapi':1,8,26,35,42,60,83 'scrapy-rotating-proxi':25,59 'setup':75 'skill':143 'skill-scrapy-spider-data-pipeline' 'sourc':130,141 'source-agentskillexchange' 'spider':2,11,36,45 'splash':17,51 'start':112 'support':16,50,90 'topic-agent-skills' 'topic-ai-agents' 'topic-ai-tools' 'topic-awesome-list' 'topic-claude-code' 'topic-codex' 'topic-cursor' 'topic-llm' 'topic-mcp' 'topic-npx-skills' 'topic-openclaw' 'topic-skills-catalog' 'upstream':72,88,136 'usag':108 'use':70,124 'version':92 'via':24,58 'web':9,43 'zyte':106","prices":[{"id":"e5345971-200c-4a6d-964c-9f67d15590e4","listingId":"f42afc00-92a9-47c5-b10b-9d567e65d667","amountUsd":"0","unit":"free","nativeCurrency":null,"nativeAmount":null,"chain":null,"payTo":null,"paymentMethod":"skill-free","isPrimary":true,"details":{"org":"agentskillexchange","category":"skills","install_from":"skills.sh"},"createdAt":"2026-05-18T13:19:12.510Z"}],"sources":[{"listingId":"f42afc00-92a9-47c5-b10b-9d567e65d667","source":"github","sourceId":"agentskillexchange/skills/scrapy-spider-data-pipeline","sourceUrl":"https://github.com/agentskillexchange/skills/tree/main/skills/scrapy-spider-data-pipeline","isPrimary":false,"firstSeenAt":"2026-05-18T13:19:12.510Z","lastSeenAt":"2026-05-18T19:12:20.617Z"}],"details":{"listingId":"f42afc00-92a9-47c5-b10b-9d567e65d667","quickStartSnippet":null,"exampleRequest":null,"exampleResponse":null,"schema":null,"openapiUrl":null,"agentsTxtUrl":null,"citations":[],"useCases":[],"bestFor":[],"notFor":[],"kindDetails":{"org":"agentskillexchange","slug":"scrapy-spider-data-pipeline","github":{"repo":"agentskillexchange/skills","stars":8,"topics":["agent-skills","ai-agents","ai-tools","awesome-list","claude-code","codex","cursor","llm","mcp","npx-skills","openclaw","skills-catalog"],"license":"mit","html_url":"https://github.com/agentskillexchange/skills","pushed_at":"2026-05-18T19:02:17Z","description":"The open catalog of AI agent skills — 2,000+ security-scanned skills for Claude Code, Cursor, Codex, and more.","skill_md_sha":"beaa48791d9e830278b1cbdc0f7e8fa2be9d1284","skill_md_path":"skills/scrapy-spider-data-pipeline/SKILL.md","default_branch":"main","skill_tree_url":"https://github.com/agentskillexchange/skills/tree/main/skills/scrapy-spider-data-pipeline"},"layout":"multi","source":"github","category":"skills","frontmatter":{"name":"Scrapy Spider Data Pipeline","description":"Builds and manages Scrapy web scraping spiders with custom item pipelines. Supports Splash rendering for JavaScript pages, rotating proxies via scrapy-rotating-proxies, and export to MongoDB or Elasticsearch."},"skills_sh_url":"https://skills.sh/agentskillexchange/skills/scrapy-spider-data-pipeline"},"updatedAt":"2026-05-18T19:12:20.617Z"}}