Export multiple scraped files in python from beautiful soup to a cvs file
<p>I have a csv list of urls that I need to scrape and organize into a csv file. I want the data from each url to be a row in the csv file. I have about 19000 urls to scrape, but am trying to figure this out using only handful. I am able to scrape the files and view them in the terminal, but when I export them to the csv file only the last file appears. </p>

<p>The urls appear in the csv file as:</p>

<p><a href="http://www.gpo.gov/fdsys/pkg/CREC-2005-01-26/html/CREC-2005-01-26-pt1-PgH199-6.htm" rel="nofollow">http://www.gpo.gov/fdsys/pkg/CREC-2005-01-26/html/CREC-2005-01-26-pt1-PgH199-6.htm</a></p>

<p><a href="http://www.gpo.gov/fdsys/pkg/CREC-2005-01-26/html/CREC-2005-01-26-pt1-PgH200-3.htm" rel="nofollow">http://www.gpo.gov/fdsys/pkg/CREC-2005-01-26/html/CREC-2005-01-26-pt1-PgH200-3.htm</a></p>

<p>I have a feeling I am doing something wrong with my loop, but can't seem to figure out where. Any help would be greatly appreciated!</p>

<p>Here is what I'm working with so far: </p>

<pre><code>import urllib
from bs4 import BeautifulSoup
import csv
import re
import pandas as pd
import requests

with open('/Users/test/Dropbox/one_minute_json/Extracting Data/a_2005_test.csv') as f:
reader = csv.reader(f)

for row in reader:
    html = urllib.urlopen(row[0])
    r = requests.get(html)
    soup = BeautifulSoup(r, "lxml")

for item in soup:

volume = int(re.findall(r"Volume (\d{1,3})", soup.title.text)[0])
print(volume)

issue = int(re.findall(r"Issue (\d{1,3})", soup.title.text)[0])
print(issue)



date = re.findall(r"\((.*?)\)", soup.title.text)[0]
print(date)

page = re.findall(r"\[Page (.*?)]", soup.pre.text.split('\n')[3])[0]
print(page)

title = soup.pre.text.split('\n\n  ')[1].strip()
print(title)

name = soup.pre.text.split('\n ')[2]
print(name)

text = soup.pre.text.split(')')[2]
print(text)

df = pd.DataFrame()
df['volume'] = [volume]
df['issue'] = [issue]
df['date'] = [date]
df['page'] = [page]
df['title'] = [title]
df['name'] = [name]
df['text'] = [text]

df.to_csv('test_scrape.csv', index=False)
</code></pre>

<p>Thanks!</p>

<p>Your indenting is completely off, try the following:</p>

<pre><code>from bs4 import BeautifulSoup
import csv
import re
import pandas as pd
import requests

with open('/Users/test/Dropbox/one_minute_json/Extracting Data/a_2005_test.csv') as f:
    reader = csv.reader(f)

    index = 0    
    df = pd.DataFrame(columns=["Volume", "issue", "date", "page", "title", "name", "text"])

    for row in reader:
        r = requests.get(row[0])
        soup = BeautifulSoup(r.text, "lxml")

        for item in soup:
            volume = int(re.findall(r"Volume (\d{1,3})", soup.title.text)[0])
            issue = int(re.findall(r"Issue (\d{1,3})", soup.title.text)[0])
            date = re.findall(r"\((.*?)\)", soup.title.text)[0]
            page = re.findall(r"\[Page (.*?)]", soup.pre.text.split('\n')[3])[0]
            title = soup.pre.text.split('\n\n  ')[1].strip()
            name = soup.pre.text.split('\n ')[2]
            text = soup.pre.text.split(')')[2]
            row = [volume, issue, date, page, title, name, text]

            df.loc[index] = row
            index += 1

    df.to_csv('test_scrape.csv', index=False)    
</code></pre>

python csv beautifulsoup