Skip to content

Commit

Permalink
Add option to scrape reviews for a particular Steam id
Browse files Browse the repository at this point in the history
  • Loading branch information
prncc committed Feb 2, 2018
1 parent 2806dc5 commit 97ac4a5
Showing 1 changed file with 9 additions and 2 deletions.
11 changes: 9 additions & 2 deletions steam/spiders/review_spider.py
Original file line number Diff line number Diff line change
Expand Up @@ -76,9 +76,10 @@ class ReviewSpider(scrapy.Spider):
'http://steamcommunity.com/app/416600/reviews/?browsefilter=mostrecent&p=1',
]

def __init__(self, url_file=None, *args, **kwargs):
def __init__(self, url_file=None, steam_id=None, *args, **kwargs):
super().__init__(*args, **kwargs)
self.url_file = url_file
self.steam_id = steam_id

def read_urls(self):
with open(self.url_file, 'r') as f:
Expand All @@ -88,7 +89,13 @@ def read_urls(self):
yield scrapy.Request(url, callback=self.parse)

def start_requests(self):
if self.url_file:
if self.steam_id:
url = (
f'http://steamcommunity.com/app/{self.steam_id}/reviews/'
'?browsefilter=mostrecent&p=1'
)
yield Request(url, callback=self.parse)
elif self.url_file:
yield from self.read_urls()
else:
for url in self.test_urls:
Expand Down

0 comments on commit 97ac4a5

Please sign in to comment.