Python multiprocessing pool freezes without reason
<p>I am pretty new to python and I hope someone here can help me.</p>

<p>I started learning python some weeks ago and I tried to build a webcrawler.</p>

<p>The idea is the following: The first part crawls the domains from a website (for each letter). The second part checks if the domain is valid (reachable and not parked) and persists it in a database.</p>

<p>Everything is doing well till the crawler reaches 'r'. After some minutes the program freezes without any error message etc.
Also the letters after 'r' don't make any problems... The domain where the program freezes isn't the same.</p>

<p>Here is my code:</p>

<pre><code>import requests
import re
import logging
import time

from bs4 import BeautifulSoup

from multiprocessing.pool import Pool

""" Extract only the plain text of element
"""
def visible(element):
    if element.parent.name in ['style', 'script', '[document]', 'head', 'title']:
        return False
    elif re.match('.*&lt;!--.*--&gt;.*', str(element), re.DOTALL):
        return False
    elif re.fullmatch(r"[\s\r\n]", str(element)):
        return False
    return True


logging.basicConfig(format='%(asctime)s %(name)s - %(levelname)s: %(message)s', level=logging.ERROR)
logger = logging.getLogger('crawler')
hdlr = logging.FileHandler('crawler.log')
formatter = logging.Formatter('%(asctime)s %(name)s - %(levelname)s: %(message)s')
hdlr.setFormatter(formatter)
logger.addHandler(hdlr)
logger.setLevel(logging.DEBUG)

""" Checks if a domain is parked.
    Returns true if a domain is not parked, otherwise false
    """
def check_if_valid(website):
    try:
        resp = requests.get("http://www." + website, timeout=10, verify=False)

        soup = BeautifulSoup(resp.text, 'html.parser')

        if len(soup.find_all('script')) == 0:
            # check for very small web pages
            if len(resp.text) &lt; 700:
                return None
            # check for 'park' pattern
            text = filter(visible, soup.find_all(text=True))
            for elem in text:
                if 'park' in elem:
                    return None

        return "http://www." + website + "/"

    except requests.exceptions.RequestException as e:
        # no logging -&gt; too many exceptions
        return None
    except Exception as ex:
        logger.exception("Error during domain validation")


def persist_domains(nonParkedDomains):
    logger.info("Inserting domains into database")
    dbConn = mysqlDB.connect()

    for d in nonParkedDomains:
        mysqlDB.insert_company_domain(dbConn, d)

    mysqlDB.close_connection(dbConn)


if __name__ =="__main__":
    dryrun = True

    if dryrun:
        logger.warning("Testrun! Data does not get persisted!")

    url = "http://www.safedomain.at/"

#    chars = ['0-9', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't','u', 'v', 'w', 'x', 'y', 'z']
    chars = ['r','s', 't','u', 'v', 'w', 'x', 'y', 'z']
    payload = {'sub': 'domains', 'char': '', 'page': '1'}

    domains = list()
    cntValidDomains = 0


    logger.info("Start collecting domains from \"http://www.safedomain.at\"....")
    try:
        for c in chars:
            payload['char'] = c
            payload['page'] = '1'

            response = requests.get(url, params=payload, verify=False)
            soup = BeautifulSoup(response.text, 'html.parser')

            while not soup.find_all('a', {'data-pagenumber': True}):
                time.sleep(5)
                response = requests.get(url, params=payload, verify=False)
                soup = BeautifulSoup(response.text, 'html.parser')

            maxPage = int(soup.find_all('a', {'data-pagenumber': True})[-1].getText())

            domains = list()
            for page in range(1, maxPage + 1):
                payload['page'] = page

                logger.debug("Start crawling with following payload: char=%s page=%s", payload['char'], payload['page'])

                response = requests.get(url, params=payload)
                soup = BeautifulSoup(response.text, 'html.parser')

                for elem in soup.find_all('ul', {'class': 'arrow-list'}):
                    for link in elem.find_all('a'):
                        domains.append(link.getText())

            logger.info("Finished! Collected domains for %s: %s",c, len(domains))
            logger.info("Checking if domains are valid...")

            with Pool(48) as p:
                nonParkedDomains = p.map(check_if_valid, domains)

            p.close()
            p.join()

            nonParkedDomains = list(filter(None.__ne__, nonParkedDomains))

            cntTemp = cntTemp + len(nonParkedDomains)

            # check if domains should get persisted

            if dryrun:
                logger.info("Valid domains for %s in domains", c)
                for elem in nonParkedDomains:
                    logger.info(elem)
            else:
                persist_domains(nonParkedDomains)

            logger.info("Finished domain validation for %s!", c)
            cntValidDomains = cntTemp + cntValidDomains

        logger.info("Valid domains: %s", cntTemp)
        logger.info("Program finished!")

    except Exception as e:
        logger.exception("Domain collection stopped unexpectedly")
</code></pre>

<p>EDIT: After some hours debugging and testing I have an idea. Could it be that the requests module, which is used in the thread, causes troubles?</p>

<p>After several hours debugging and testing I could fix the problem.</p>

<p>Instead of the multiprocessing pool I've used the ThreadPoolExecutor (which is better for network applications)</p>

<p>I've figured out that the requests.get() in the threaded function caused some troubles. I changed the timeout to 1.</p>

<p>After these changes the program worked.</p>

<p>I don't know the exact reason but I would be very interested in it. If someone knows it I would appreciate if he/she could post it.</p>

python web-crawler python-multiprocessing