add DirbotItem to items.py so that genspider creates a valid spider

dirbot/items.py

@@ -1,10 +1,15 @@
 from scrapy.item import Item, Field
 
-class Website(Item):
+
+class DirbotItem(Item):
 
     name = Field()
-    url = Field()
     description = Field()
 
+
+class Website(DirbotItem):
+
+    url = Field()
+
     def __str__(self):
-        return "Website: name=%s url=%s" % (self['name'], self['url'])
+        return "Website: name=%s url=%s" % (self.get('name'), self.get('url'))

dirbot/pipelines.py

@@ -1,5 +1,6 @@
 from scrapy.exceptions import DropItem
 
+
 class FilterWordsPipeline(object):
     """A pipeline for filtering out items which contain certain words in their
     description"""

dirbot/settings.py

@@ -9,4 +9,3 @@
 USER_AGENT = '%s/%s' % (BOT_NAME, BOT_VERSION)
 
 ITEM_PIPELINES = ['dirbot.pipelines.FilterWordsPipeline']
-

dirbot/spiders/dmoz.py

@@ -3,22 +3,25 @@
 
 from dirbot.items import Website
 
+
 class DmozSpider(BaseSpider):
-   name = "dmoz"
-   allowed_domains = ["dmoz.org"]
-   start_urls = [
-       "http://www.dmoz.org/Computers/Programming/Languages/Python/Books/",
-       "http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/"
-   ]
+    name = "dmoz"
+    allowed_domains = ["dmoz.org"]
+    start_urls = [
+        "http://www.dmoz.org/Computers/Programming/Languages/Python/Books/",
+        "http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/",
+    ]
+
+    def parse(self, response):
+        hxs = HtmlXPathSelector(response)
+        sites = hxs.select('//ul/li')
+        items = []
+
+        for site in sites:
+            item = Website()
+            item['name'] = site.select('a/text()').extract()
+            item['url'] = site.select('a/@href').extract()
+            item['description'] = site.select('text()').extract()
+            items.append(item)
 
-   def parse(self, response):
-       hxs = HtmlXPathSelector(response)
-       sites = hxs.select('//ul/li')
-       items = []
-       for site in sites:
-           item = Website()
-           item['name'] = site.select('a/text()').extract()
-           item['url'] = site.select('a/@href').extract()
-           item['description'] = site.select('text()').extract()
-           items.append(item)
-       return items
+        return items