Scrapy performance improvements and memory consumtion
<p>Server</p>

<ul>
<li>6 GB RAM</li>
<li>4 Cores Intel Xeon 2.60GHz</li>
<li>32 CONCURRENT_REQUESTS</li>
<li>1m URLs in CSV</li>
<li>700 Mbit/s downstream</li>
<li>96% Memory Consumtion</li>
</ul>

<p>With debug mode on, the scrape stops after around 400 000 urls, most likely because the server runs out of memory. 
Without debug mode it takes up to 5 days which is pretty slow imo and
it takes way to much memory (96%)</p>

<p>any hints are highly welcome :)</p>

<pre><code>import scrapy
import csv

def get_urls_from_csv():
    with open('data.csv', newline='') as csv_file:
        data = csv.reader(csv_file, delimiter=',')
        scrapurls = []
        for row in data:
            scrapurls.append("http://"+row[2])
        return scrapurls

class rssitem(scrapy.Item):
    sourceurl = scrapy.Field()
    rssurl = scrapy.Field()


class RssparserSpider(scrapy.Spider):
    name = "rssspider"
    allowed_domains = ["*"]
    start_urls = ()

    def start_requests(self):
        return [scrapy.http.Request(url=start_url) for start_url in get_urls_from_csv()]

    def parse(self, response):
        res = response.xpath('//link[@type="application/rss+xml"]/@href')
        for sel in res:
            item = rssitem()
            item['sourceurl']=response.url
            item['rssurl']=sel.extract()
            yield item

        pass
</code></pre>

<pre><code>import csv
from collections import namedtuple

import scrapy


def get_urls_from_csv():
    with open('data.csv', newline='') as csv_file:
        data = csv.reader(csv_file, delimiter=',')
        for row in data:
            yield row[2]


# if you can use something else than scrapy
rssitem = namedtuple('rssitem', 'sourceurl rssurl')


class RssparserSpider(scrapy.Spider):
    name = "rssspider"
    allowed_domains = ["*"]
    start_urls = ()

    def start_requests(self): # remember that it returns generator
        for start_url in get_urls_from_csv():
            yield scrapy.http.Request(url="http://{}".format(start_url))

    def parse(self, response):
        res = response.xpath('//link[@type="application/rss+xml"]/@href')
        for sel in res:
            yield rssitem(response.url, sel.extract())
        pass
</code></pre>

python scrapy scrapy-spider