How to use selenium with firefox to scrape websites?
echo "deb http://packages.linuxmint.com debian import" >> /etc/apt/sources.list && apt-get update
apt-get install firefox xvfb python-dev python-pip
pip install pyvirtualdisplay selenium
from pyvirtualdisplay import Display
import time
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException
display = Display(visible=0, size=(800, 600))
display.start()
def init_driver():
driver = webdriver.Firefox()
driver.wait = WebDriverWait(driver, 5)
return driver
def lookup(driver, query):
driver.get("http://www.google.com")
try:
box = driver.wait.until(EC.presence_of_element_located(
(By.NAME, "q")))
button = driver.wait.until(EC.element_to_be_clickable(
(By.NAME, "btnK")))
box.send_keys(query)
button.click()
except TimeoutException:
print("Box or Button not found in google.com")
if __name__ == "__main__":
driver = init_driver()
lookup(driver, "Selenium")
time.sleep(5)
driver.quit()
display.stop()
File "selenium_scrape.py", line 20
box = driver.wait.until(EC.presence_of_element_located(
^
IndentationError: expected an indented block
The difference is that you cannot use a packaged Chrome browser; you need a special driver... chromedriver.
Get the current latest version here: Chromedriver
Now you have 2 options, either to move the downloaded chromedriver so it is always accessible (option 1), or to define in your script how to access it.
Then move it so it is accessible when you use webdriver.Chrome():
sudo mv /path/to/download/chromedriver /usr/bin
Also set it to be allowed to be executed:
chmod a+x /usr/binchromedriver
Or you can define a path
import os
chr = "/Users/you/Downloads/chromedriver"
os.environ["webdriver.chrome.driver"] = chr
driver = webdriver.Chrome(chromedriver)
(Note: the original question was about Chrome, so my answer is about Chrome, not Firefox).
For me it works if I just extract the chromedriver into the same folder where the script is.
Then I run it as this
Xvfb :99 -ac -screen 0 1280x1024x16 &
echo 'Starting the test'
PATH=$PATH:. python selenimum_scrape.py
This will start the Xvfb and include the cromedriver into PATH.
And the modified version of your which works for me:
import os
import time
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException
# comment this out to run on the real display
os.environ['DISPLAY'] = ':99'
def init_driver():
driver = webdriver.Chrome()
driver.wait = WebDriverWait(driver, 5)
return driver
def lookup(driver, query):
driver.get("http://www.google.com")
try:
box = driver.wait.until(EC.presence_of_element_located(
(By.NAME, "q")))
# once we type the query, this button disappears
# button = driver.wait.until(EC.element_to_be_clickable(
# (By.NAME, "btnK")))
box.send_keys(query)
button = driver.wait.until(EC.element_to_be_clickable(
(By.NAME, "btnG")))
button.click()
except TimeoutException:
print("Box or Button not found in google.com")
if __name__ == "__main__":
driver = init_driver()
lookup(driver, "Selenium")
time.sleep(5)
driver.quit()
The question is (at the moment) about an indentation error. This can be easily fixed:
def lookup(driver, query):
driver.get("http://www.google.com")
try:
box = driver.wait.until(EC.presence_of_element_located(
(By.NAME, "q")))
button = driver.wait.until(EC.element_to_be_clickable(
(By.NAME, "btnK")))
box.send_keys(query)
button.click()
except TimeoutException:
print("Box or Button not found in google.com")