selenium python webscrape fails after first iteration
<p>Im iterating through tripadvisor to save comments(non-translated, original) and translated comments (from portuguese to english).
So the scraper first selects portuguese comments to be displayed , then as usual it converts them into english one by one and saves the translated comments in com_, whereas the expanded non-translated comments in expanded_comments.</p>

<p>The code works fine with first page but from second page onward it fails to save translated comments. Strangely it just translates only first comment from each of the pages and doesnt even save them.</p>

<pre><code>from selenium import webdriver
from selenium.webdriver.common.by import By
import time
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
com_=[]
expanded_comments=[]
date_=[]
driver = webdriver.Chrome("C:\Users\shalini\Downloads\chromedriver_win32\chromedriver.exe")
driver.maximize_window()
from bs4 import BeautifulSoup

def expand_reviews(driver):
    # TRYING TO EXPAND REVIEWS (&amp; CLOSE A POPUP)    
    try:
        driver.find_element_by_class_name("moreLink").click()
    except:
        print "err"
    try:
        driver.find_element_by_class_name("ui_close_x").click()
    except:
        print "err2"
    try:
        driver.find_element_by_class_name("moreLink").click()
    except:
        print "err3"




def save_comments(driver):
    expand_reviews(driver)
    # SELECTING ALL EXPANDED COMMENTS
    #xpanded_com_elements=driver.find_elements_by_class_name("entry")
    time.sleep(3)
    #or i in expanded_com_elements:
    #   expanded_comments.append(i.text)
    spi=driver.page_source
    sp=BeautifulSoup(spi)
    for t in sp.findAll("div",{"class":"entry"}):
        if not t.findAll("p",{"class":"partial_entry"}):
            #print t
            expanded_comments.append(t.getText())
    # Saving review date
    for d in sp.findAll("span",{"class":"recommend-titleInline"}) :
        date=d.text
        date_.append(date_)


    # SELECTING ALL GOOGLE-TRANSLATOR links
    gt= driver.find_elements(By.CSS_SELECTOR,".googleTranslation&gt;.link")

    # NOW PRINTING TRANSLATED COMMENTS
    for i in gt:
        try:
            driver.execute_script("arguments[0].click()",i)

            #com=driver.find_element_by_class_name("ui_overlay").text
            com= driver.find_element_by_xpath(".//span[@class = 'ui_overlay ui_modal ']//div[@class='entry']")
            com_.append(com.text)
            time.sleep(5)
            driver.find_element_by_class_name("ui_close_x").click().perform()
            time.sleep(5)
        except Exception as e:
            pass

# ITERATING THROIGH ALL 200 tripadvisor webpages and saving comments &amp; translated comments             
for i in range(200):
    page=i*10
    url="https://www.tripadvisor.com/Airline_Review-d8729164-Reviews-Cheap-Flights-or"+str(page)+"-TAP-Portugal#REVIEWS"
    driver.get(url)
    wait = WebDriverWait(driver, 10)
    if i==0:
        # SELECTING PORTUGUESE COMMENTS ONLY # Run for one time then iterate over pages
        try:
            langselction = wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "span.sprite-date_picker-triangle")))
            langselction.click()
            driver.find_element_by_xpath("//div[@class='languageList']//li[normalize-space(.)='Portuguese first']").click()
            time.sleep(5)
        except Exception as e:
            print e

    save_comments(driver)
</code></pre>

<p>There are 3 problems in your code</p>

<ol>
<li>Inside method <code>save_comments()</code>, at the <code>driver.find_element_by_class_name("ui_close_x").click().perform()</code>, the method <code>click()</code> of a webelement is not an ActionChain so you cannot call <code>perform()</code>. Therefore, that line should be like this:</li>
</ol>



<pre><code>driver.find_element_by_class_name("ui_close_x").click()
</code></pre>

<ol start="2">
<li>Inside method <code>save_comments()</code>, at the <code>com= driver.find_element_by_xpath(".//span[@class = 'ui_overlay ui_modal ']//div[@class='entry']")</code>, you find the element when it doesn't appear yet. So you have to add wait before this line. Your code should be like this:</li>
</ol>



<pre><code>wait = WebDriverWait(driver, 10)
wait.until(EC.element_to_be_clickable((By.XPATH, ".//span[@class = 'ui_overlay ui_modal ']//div[@class='entry']")))
com= driver.find_element_by_xpath(".//span[@class = 'ui_overlay ui_modal ']//div[@class='entry']")
</code></pre>

<ol start="3">
<li>There are 2 buttons which can open the review, one is displayed and one is hidden. So you have to skip the hidden button.</li>
</ol>



<pre><code>if not i.is_displayed():
    continue
driver.execute_script("arguments[0].click()",i)
</code></pre>

python selenium web-scraping