Crawling categories works

1 files changed, 17 insertions, 10 deletions

diff --git a/israblog/israblog/spiders/blogs.py b/israblog/israblog/spiders/blogs.py
index b7aadc1..03f03fe 100644
--- a/israblog/israblog/spiders/blogs.py
+++ b/israblog/israblog/spiders/blogs.py
@@ -1,3 +1,5 @@
+import re
+
 from datetime import datetime
 from scrapy import log
 from scrapy.contrib.spiders import CrawlSpider, Rule
@@ -7,10 +9,10 @@ from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
 class IsrablogSpider(CrawlSpider):
     name = 'israblog'
     allowed_domains = ['israblog.nana10.co.il']
-    rules = [Rule(SgmlLinkExtractor(allow=[
-        '/blogread.asp?blog=\d+',
-        '/blogread.asp?blog=\d+&year=\d+&month=\d+'
-    ]), 'parse_blog')]
+    rules = (
+        Rule(SgmlLinkExtractor(allow=['blogread.asp?blog=\d+(&year=\d+&month=\d+)?']), callback='parse_blog'),
+        Rule(SgmlLinkExtractor(allow=['/Category/\d+/.*',]), follow=True),
+    )
     start_urls = [
         'http://israblog.nana10.co.il/',
         'http://israblog.nana10.co.il/lastblogs.aspx',
@@ -18,12 +20,17 @@ class IsrablogSpider(CrawlSpider):
         'http://israblog.nana10.co.il/Categories',
     ]
 
+    def parse_start_url(self, response):
+        yield self.parse_blog(response)
+
     def parse_blog(self, response):
         self.log(response.url)
-        blog, year, month = re.match(r'/blogread.asp\?blog=(\d+)(?:&year=(\d+)&month=(\d+))?', response.url)
-        if not year and not month:
-            now = datetime.now()
-            year = now.year()
-            month = now.month()
-        self.log(blog, year, month)
+        match = re.match(r'/blogread.asp\?blog=(\d+)(?:&year=(\d+)&month=(\d+))?', response.url)
+        if match:
+            blog, year, month = match.groups()
+            if not year and not month:
+                now = datetime.now()
+                year = now.year()
+                month = now.month()
+            self.log(blog, year, month)
         return None