Python scraping of javascript web pages fails for https pages only
<p>I'm using PyQt5 to scrape web pages, which works great for http:// URLs, but not at all for https:// URLs.</p>

<p>The relevant part of my script is below:</p>

<pre><code>class WebPage(QWebPage):
    def __init__(self):
        super(WebPage, self).__init__()

        self.timerScreen = QTimer()
        self.timerScreen.setInterval(2000)
        self.timerScreen.setSingleShot(True)
        self.timerScreen.timeout.connect(self.handleLoadFinished)

        self.loadFinished.connect(self.timerScreen.start)


    def start(self, urls):
        self._urls = iter(urls)
        self.fetchNext()

    def fetchNext(self):
        try:
            url = next(self._urls)
        except StopIteration:
            return False
        else:
            self.mainFrame().load(QUrl(url))
        return True

    def processCurrentPage(self):
        url = self.mainFrame().url().toString()
        html = self.mainFrame().toHtml()

        #Do stuff with html
        print('loaded: [%d bytes] %s' % (self.bytesReceived(), url))

    def handleLoadFinished(self):
        self.processCurrentPage()
        if not self.fetchNext():
            qApp.quit()
</code></pre>

<p>For secure pages, the script returns a blank page. The only html coming back is <code>&lt;html&gt;&lt;head&gt;&lt;/head&gt;&lt;body&gt;&lt;/body&gt;&lt;/html&gt;</code>.</p>

<p>I'm at a bit of a loss. Is there a setting that I'm missing related to handling secure URLs?</p>

<p>tested with PyQt4 and normally opened pages with HTTPS</p>

<pre><code>import sys
from PyQt4.QtGui import QApplication
from PyQt4.QtCore import QUrl
from PyQt4.QtWebKit import QWebView

class Browser(QWebView):
    def __init__(self):
        QWebView.__init__(self)
        self.loadFinished.connect(self._result_available)

    def _result_available(self, ok):
        frame = self.page().mainFrame()
        print(frame.toHtml())

if __name__ == '__main__':
    app = QApplication(sys.argv)
    view = Browser()
    view.load(QUrl('https://www.google.com'))
    app.exec_()
</code></pre>

python https pyqt