Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Auto-formatting using isort and autopep8 #43

Open
wants to merge 1 commit into
base: develop
Choose a base branch
from
Open

Auto-formatting using isort and autopep8 #43

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 9 additions & 6 deletions scrapy_selenium/middlewares.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,11 @@

from importlib import import_module

from selenium.webdriver.support.ui import WebDriverWait

from scrapy import signals
from scrapy.exceptions import NotConfigured
from scrapy.http import HtmlResponse
from selenium.webdriver.support.ui import WebDriverWait

from .http import SeleniumRequest

Expand All @@ -14,7 +15,7 @@ class SeleniumMiddleware:
"""Scrapy middleware handling the requests using selenium"""

def __init__(self, driver_name, driver_executable_path, driver_arguments,
browser_executable_path):
browser_executable_path):
"""Initialize the selenium webdriver

Parameters
Expand Down Expand Up @@ -55,8 +56,10 @@ def from_crawler(cls, crawler):
"""Initialize the middleware with the crawler settings"""

driver_name = crawler.settings.get('SELENIUM_DRIVER_NAME')
driver_executable_path = crawler.settings.get('SELENIUM_DRIVER_EXECUTABLE_PATH')
browser_executable_path = crawler.settings.get('SELENIUM_BROWSER_EXECUTABLE_PATH')
driver_executable_path = crawler.settings.get(
'SELENIUM_DRIVER_EXECUTABLE_PATH')
browser_executable_path = crawler.settings.get(
'SELENIUM_BROWSER_EXECUTABLE_PATH')
driver_arguments = crawler.settings.get('SELENIUM_DRIVER_ARGUMENTS')

if not driver_name or not driver_executable_path:
Expand All @@ -71,7 +74,8 @@ def from_crawler(cls, crawler):
browser_executable_path=browser_executable_path
)

crawler.signals.connect(middleware.spider_closed, signals.spider_closed)
crawler.signals.connect(
middleware.spider_closed, signals.spider_closed)

return middleware

Expand Down Expand Up @@ -118,4 +122,3 @@ def spider_closed(self):
"""Shutdown the driver when spider is closed"""

self.driver.quit()

5 changes: 2 additions & 3 deletions setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
"""This module contains the packaging routine for the pybook package"""

from setuptools import setup, find_packages
from setuptools import find_packages, setup

try:
from pip.download import PipSession
from pip.req import parse_requirements
Expand Down Expand Up @@ -30,5 +31,3 @@ def get_requirements(source):
packages=find_packages(),
install_requires=get_requirements('requirements/requirements.txt')
)


1 change: 0 additions & 1 deletion tests/test_middlewares.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@

from scrapy import Request
from scrapy.crawler import Crawler

from scrapy_selenium.http import SeleniumRequest
from scrapy_selenium.middlewares import SeleniumMiddleware

Expand Down