diff --git a/steam/spiders/product_spider.py b/steam/spiders/product_spider.py index 0d6ee6f..1f856d5 100644 --- a/steam/spiders/product_spider.py +++ b/steam/spiders/product_spider.py @@ -22,7 +22,7 @@ def load_product(response): found_id = re.findall('/app/(.*?)/', response.url) if found_id: id = found_id[0] - reviews_url = f"http://steamcommunity.com/app/{id}/reviews/?browsefilter=mostrecent&p=1" + reviews_url = f'http://steamcommunity.com/app/{id}/reviews/?browsefilter=mostrecent&p=1' loader.add_value('reviews_url', reviews_url) loader.add_value('id', id) @@ -44,7 +44,7 @@ def load_product(response): if prop in line: item = line.replace(prop, '').strip() loader.add_value(name, item) - except: + except: # noqa E722 pass loader.add_css('app_name', '.apphub_AppName ::text') @@ -77,24 +77,24 @@ def load_product(response): class ProductSpider(CrawlSpider): name = 'products' - start_urls = ["http://store.steampowered.com/search/?sort_by=Released_DESC"] + start_urls = ['http://store.steampowered.com/search/?sort_by=Released_DESC'] - allowed_domains=["steampowered.com"] + allowed_domains = ['steampowered.com'] rules = [ Rule(LinkExtractor( - allow='/app/(.+)/', - restrict_css='#search_result_container'), - callback='parse_product'), + allow='/app/(.+)/', + restrict_css='#search_result_container'), + callback='parse_product'), Rule(LinkExtractor( - allow='page=(\d+)', - restrict_css='.search_pagination_right')) + allow='page=(\d+)', + restrict_css='.search_pagination_right')) ] def parse_product(self, response): # Circumvent age selection form. if '/agecheck/app' in response.url: - logger.debug(f"Form-type age check triggered for {response.url}.") + logger.debug(f'Form-type age check triggered for {response.url}.') form = response.css('#agegate_box form') diff --git a/steam/spiders/review_spider.py b/steam/spiders/review_spider.py index a16ca91..f744897 100644 --- a/steam/spiders/review_spider.py +++ b/steam/spiders/review_spider.py @@ -63,7 +63,7 @@ def get_product_id(response): if not product_id: try: return re.findall("app/(.+?)/", response.url)[0] - except: + except: # noqa E722 return None else: return product_id @@ -72,9 +72,12 @@ def get_product_id(response): class ReviewSpider(scrapy.Spider): name = 'reviews' test_urls = [ - "http://steamcommunity.com/app/316790/reviews/?browsefilter=mostrecent&p=1", # Grim Fandango - "http://steamcommunity.com/app/207610/reviews/?browsefilter=mostrecent&p=1", # The Walking Dead - "http://steamcommunity.com/app/414700/reviews/?browsefilter=mostrecent&p=1" # Outlast 2 + # Grim Fandango + 'http://steamcommunity.com/app/316790/reviews/?browsefilter=mostrecent&p=1', + # The Walking Dead + 'http://steamcommunity.com/app/207610/reviews/?browsefilter=mostrecent&p=1', + # Outlast 2 + 'http://steamcommunity.com/app/414700/reviews/?browsefilter=mostrecent&p=1', ] def __init__(self, url_file=None, *args, **kwargs):