I want to use WebDriver.Wait() function inside a while loop. While loop is used for scrolling an infinite scroll website. I want to have a wait between last_scroll and new_scroll to get page content loaded. Static time.sleep(10) is working fine but I don't want to use it.
try:
driver.get(url)
WebDriverWait(driver, 10).until(EC.presence_of_all_elements_located((By.CLASS_NAME, 'pb-xl')))
self.writeToFile('test.txt','Page is loaded\n')
# Get scroll height
last_height = driver.execute_script("return document.body.scrollHeight")
while True:
print("Scrolling down...")
# Scroll down to bottom
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
print("Wait for new content to load...")
# Wait to load page
WebDriverWait(driver, 100).until(EC.presence_of_all_elements_located((By.CLASS_NAME, 'pb-xl')))
# time.sleep(10)
# Calculate new scroll height and compare with last scroll height
new_height = driver.execute_script("return document.body.scrollHeight")
if new_height == last_height:
break
print("Reached the end of the page.")
last_height = new_height
print("Loading next page")
# print(driver.page_source)
self.writeToFile('page_source.txt',driver.page_source)
except TimeoutException:
self.writeToFile('test.txt','Timed out waiting for the element to appear\n')
except Exception as e:
self.writeToFile('test.txt',f'Error: {str(e)}\n')