Code: Select all
url = "https://www.falstaff.com/de/listings/die-besten-restaurants-in-rheinland-pfalz"
def get_linklist(url):
try:
driver
except:
open_page(url, False)
else:
navigate(url)
try:
time.sleep(2)
#click_cookie_msg()
driver.find_element(By.XPATH, "//p[contains(text(), 'Consent')]").click()
except Exception as e:
print(repr(e))
try:
# this loop scrolls down and clicks the 'load more' function
loop_count = 0
while loop_count < 36:
time.sleep(2)
more_xpath = "//button[contains(text(), 'Weitere anzeigen')]"
more_button = driver.find_element(By.XPATH, more_xpath)
#scroll down
js_code = "arguments[0].scrollIntoView();"
driver.execute_script(js_code, more_button)
time.sleep(2)
driver.execute_script("arguments[0].click();", more_button)
loop_count += 1
except Exception as e:
print(repr(e))
try:
soup = BeautifulSoup(driver.page_source, 'html.parser')
list_items = soup.find_all('div', {'class': 'list-item'})
counter = 0
for item in list_items:
counter += 1
print('Beautifulsoup results: ' + str(counter))
except Exception as e:
pass
try:
time.sleep(2)
search_list_xpath = '//*[@id="main"]/section[4]/div/div/div/div[2]/div[3]/div[1]/div/div/div[3]'
search_list = driver.find_element(By.XPATH, search_list_xpath)
list_items = search_list.find_elements(By.CLASS_NAME, 'list-item')
counter = 0
for item in list_items:
counter += 1
print('Selenium xpath results: ' + str(counter))
except Exception as e:
pass
try:
time.sleep(2)
linklist = []
wait = WebDriverWait(driver, 20)
list_items = wait.until(EC.visibility_of_all_elements_located((By.CLASS_NAME, 'list-item')))
for list_item in list_items:
href = list_item.find_element(By.TAG_NAME, 'a').get_attribute('href')
linklist.append(href)
except Exception as e:
print(repr(e))
pass
print('Selenium class_name results: ')
print(len(linklist))
Code: Select all
Beautifulsoup results: 100
Selenium xpath results: 100
Selenium class_name results:
100