I am trying to scrape data from a url using Selenium Python.
Code
from selenium import webdriver
from selenium.common.exceptions import WebDriverException
from selenium.webdriver.chrome.service import Service
from bs4 import BeautifulSoup
service = Service("/home/ubuntu/selenium_drivers/chromedriver")
options = webdriver.ChromeOptions()
options.add_argument("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.3")
options.add_argument("--headless")
options.add_argument('--ignore-certificate-errors')
options.add_argument("--enable-javascript")
options.add_argument('--incognito')
url_list = "https://memphissymphony.secure.force.com/ticket/#/events/a0S3o00001J8qqiEAB"
try:
driver = webdriver.Chrome(service = service, options = options)
driver.get(url_list)
driver.implicitly_wait(2)
html_content = driver.page_source
driver.quit()
except WebDriverException:
driver.quit()
soup = BeautifulSoup(html_content, 'html.parser')
print(soup)
After scraping I am getting the data with the content
Content
.
.
.
.
</div>
<noscript>
<h1>Please enable Javascript in your browser. Our site won't work without it.</h1>
</noscript>
</div>
<div class="pts-footer w-100 p-2 bg-dark text-light" style="position: absolute; bottom: 0;">
<p class="text-center m-0">
<small>
<span>Copyright ©</span> 2021 Memphis Symphony Orchestra
</small>
</p>
.
.
.
Also in the chrome browser javascript is enabled.
I added the options for enabling JavaScript but still its not working. How can I scrape the whole data from this url