WebDriver.Wait() not working inside a While loop used for infinite page scroll

35 Views Asked by At

I want to use WebDriver.Wait() function inside a while loop. While loop is used for scrolling an infinite scroll website. I want to have a wait between last_scroll and new_scroll to get page content loaded. Static time.sleep(10) is working fine but I don't want to use it.

try:
    driver.get(url)
    WebDriverWait(driver, 10).until(EC.presence_of_all_elements_located((By.CLASS_NAME, 'pb-xl')))
    self.writeToFile('test.txt','Page is loaded\n')
            # Get scroll height
    last_height = driver.execute_script("return document.body.scrollHeight")

    while True:
        print("Scrolling down...")
        # Scroll down to bottom
        driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
        print("Wait for new content to load...")
        # Wait to load page
        WebDriverWait(driver, 100).until(EC.presence_of_all_elements_located((By.CLASS_NAME, 'pb-xl')))
        # time.sleep(10)
        # Calculate new scroll height and compare with last scroll height
        new_height = driver.execute_script("return document.body.scrollHeight")
        if new_height == last_height:
            break
        print("Reached the end of the page.")
        last_height = new_height

        print("Loading next page")
    # print(driver.page_source)
    self.writeToFile('page_source.txt',driver.page_source)
except TimeoutException:
        self.writeToFile('test.txt','Timed out waiting for the element to appear\n')
except Exception as e:
        self.writeToFile('test.txt',f'Error: {str(e)}\n')

0

There are 0 best solutions below