{"id":"72372821-2cca-4bf9-8150-9088d2b28444","shortId":"S2hHwt","kind":"skill","title":"Scrapy Pipeline Manager","tagline":"Manages Scrapy spider deployments via Scrapyd API with custom item pipelines for MongoDB ingestion, deduplication via MinHash LSH, and rotating proxy middleware configuration.","description":"# Scrapy Pipeline Manager\n\nManages Scrapy spider deployments via Scrapyd API with custom item pipelines for MongoDB ingestion, deduplication via MinHash LSH, and rotating proxy middleware configuration.\n\n## Installation\n\nUse the upstream install or setup path that matches your environment:\n- pip install scrapy\n\nRequirements and caveats from upstream:\n- :alt: Supported Python Versions\n- It is cross-platform, and requires Python 3.10+. It is maintained by Zyte_\n\nBasic usage or getting-started notes:\n- .. code:: bash\n- And follow the documentation_ to learn how to use it.\n- .. _documentation: https://docs.scrapy.org/en/latest/\n\n- Source: https://github.com/scrapy/scrapy\n- Extracted from upstream docs: https://raw.githubusercontent.com/scrapy/scrapy/HEAD/README.rst\n\n## Source\n\n- [Agent Skill Exchange](https://agentskillexchange.com/skills/scrapy-pipeline-manager/)","tags":["scrapy","pipeline","manager","skills","agentskillexchange","agent-skills","ai-agents","ai-tools","awesome-list","claude-code","codex","cursor"],"capabilities":["skill","source-agentskillexchange","skill-scrapy-pipeline-manager","topic-agent-skills","topic-ai-agents","topic-ai-tools","topic-awesome-list","topic-claude-code","topic-codex","topic-cursor","topic-llm","topic-mcp","topic-npx-skills","topic-openclaw","topic-skills-catalog"],"categories":["skills"],"synonyms":[],"warnings":[],"endpointUrl":"https://skills.sh/agentskillexchange/skills/scrapy-pipeline-manager","protocol":"skill","transport":"skills-sh","auth":{"type":"none","details":{"cli":"npx skills add agentskillexchange/skills","source_repo":"https://github.com/agentskillexchange/skills","install_from":"skills.sh"}},"qualityScore":"0.454","qualityRationale":"deterministic score 0.45 from registry signals: · indexed on github topic:agent-skills · 8 github stars · SKILL.md body (872 chars)","verified":false,"liveness":"unknown","lastLivenessCheck":null,"agentReviews":{"count":0,"score_avg":null,"cost_usd_avg":null,"success_rate":null,"latency_p50_ms":null,"narrative_summary":null,"summary_updated_at":null},"enrichmentModel":"deterministic:skill-github:v1","enrichmentVersion":1,"enrichedAt":"2026-05-18T19:12:20.335Z","embedding":null,"createdAt":"2026-05-18T13:19:12.118Z","updatedAt":"2026-05-18T19:12:20.335Z","lastSeenAt":"2026-05-18T19:12:20.335Z","tsv":"'/en/latest/':113 '/scrapy/scrapy':117 '/scrapy/scrapy/head/readme.rst':124 '/skills/scrapy-pipeline-manager/)':131 '3.10':85 'agent':126 'agentskillexchange.com':130 'agentskillexchange.com/skills/scrapy-pipeline-manager/)':129 'alt':73 'api':10,36 'bash':99 'basic':91 'caveat':70 'code':98 'configur':26,52 'cross':80 'cross-platform':79 'custom':12,38 'dedupl':18,44 'deploy':7,33 'doc':121 'docs.scrapy.org':112 'docs.scrapy.org/en/latest/':111 'document':103,110 'environ':64 'exchang':128 'extract':118 'follow':101 'get':95 'getting-start':94 'github.com':116 'github.com/scrapy/scrapy':115 'ingest':17,43 'instal':53,57,66 'item':13,39 'learn':105 'lsh':21,47 'maintain':88 'manag':3,4,29,30 'match':62 'middlewar':25,51 'minhash':20,46 'mongodb':16,42 'note':97 'path':60 'pip':65 'pipelin':2,14,28,40 'platform':81 'proxi':24,50 'python':75,84 'raw.githubusercontent.com':123 'raw.githubusercontent.com/scrapy/scrapy/head/readme.rst':122 'requir':68,83 'rotat':23,49 'scrapi':1,5,27,31,67 'scrapyd':9,35 'setup':59 'skill':127 'skill-scrapy-pipeline-manager' 'sourc':114,125 'source-agentskillexchange' 'spider':6,32 'start':96 'support':74 'topic-agent-skills' 'topic-ai-agents' 'topic-ai-tools' 'topic-awesome-list' 'topic-claude-code' 'topic-codex' 'topic-cursor' 'topic-llm' 'topic-mcp' 'topic-npx-skills' 'topic-openclaw' 'topic-skills-catalog' 'upstream':56,72,120 'usag':92 'use':54,108 'version':76 'via':8,19,34,45 'zyte':90","prices":[{"id":"c1a2aa2e-74ca-4636-b478-6b3357f715d3","listingId":"72372821-2cca-4bf9-8150-9088d2b28444","amountUsd":"0","unit":"free","nativeCurrency":null,"nativeAmount":null,"chain":null,"payTo":null,"paymentMethod":"skill-free","isPrimary":true,"details":{"org":"agentskillexchange","category":"skills","install_from":"skills.sh"},"createdAt":"2026-05-18T13:19:12.118Z"}],"sources":[{"listingId":"72372821-2cca-4bf9-8150-9088d2b28444","source":"github","sourceId":"agentskillexchange/skills/scrapy-pipeline-manager","sourceUrl":"https://github.com/agentskillexchange/skills/tree/main/skills/scrapy-pipeline-manager","isPrimary":false,"firstSeenAt":"2026-05-18T13:19:12.118Z","lastSeenAt":"2026-05-18T19:12:20.335Z"}],"details":{"listingId":"72372821-2cca-4bf9-8150-9088d2b28444","quickStartSnippet":null,"exampleRequest":null,"exampleResponse":null,"schema":null,"openapiUrl":null,"agentsTxtUrl":null,"citations":[],"useCases":[],"bestFor":[],"notFor":[],"kindDetails":{"org":"agentskillexchange","slug":"scrapy-pipeline-manager","github":{"repo":"agentskillexchange/skills","stars":8,"topics":["agent-skills","ai-agents","ai-tools","awesome-list","claude-code","codex","cursor","llm","mcp","npx-skills","openclaw","skills-catalog"],"license":"mit","html_url":"https://github.com/agentskillexchange/skills","pushed_at":"2026-05-18T19:02:17Z","description":"The open catalog of AI agent skills — 2,000+ security-scanned skills for Claude Code, Cursor, Codex, and more.","skill_md_sha":"6e3e152658ec3a8b252a6d66430f5c6bf4645046","skill_md_path":"skills/scrapy-pipeline-manager/SKILL.md","default_branch":"main","skill_tree_url":"https://github.com/agentskillexchange/skills/tree/main/skills/scrapy-pipeline-manager"},"layout":"multi","source":"github","category":"skills","frontmatter":{"name":"Scrapy Pipeline Manager","description":"Manages Scrapy spider deployments via Scrapyd API with custom item pipelines for MongoDB ingestion, deduplication via MinHash LSH, and rotating proxy middleware configuration."},"skills_sh_url":"https://skills.sh/agentskillexchange/skills/scrapy-pipeline-manager"},"updatedAt":"2026-05-18T19:12:20.335Z"}}