diff options
| -rw-r--r-- | israblog/israblog/spiders/blogs.py | 27 |
1 files changed, 17 insertions, 10 deletions
diff --git a/israblog/israblog/spiders/blogs.py b/israblog/israblog/spiders/blogs.py index b7aadc1..03f03fe 100644 --- a/israblog/israblog/spiders/blogs.py +++ b/israblog/israblog/spiders/blogs.py @@ -1,3 +1,5 @@ +import re + from datetime import datetime from scrapy import log from scrapy.contrib.spiders import CrawlSpider, Rule @@ -7,10 +9,10 @@ from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor class IsrablogSpider(CrawlSpider): name = 'israblog' allowed_domains = ['israblog.nana10.co.il'] - rules = [Rule(SgmlLinkExtractor(allow=[ - '/blogread.asp?blog=\d+', - '/blogread.asp?blog=\d+&year=\d+&month=\d+' - ]), 'parse_blog')] + rules = ( + Rule(SgmlLinkExtractor(allow=['blogread.asp?blog=\d+(&year=\d+&month=\d+)?']), callback='parse_blog'), + Rule(SgmlLinkExtractor(allow=['/Category/\d+/.*',]), follow=True), + ) start_urls = [ 'http://israblog.nana10.co.il/', 'http://israblog.nana10.co.il/lastblogs.aspx', @@ -18,12 +20,17 @@ class IsrablogSpider(CrawlSpider): 'http://israblog.nana10.co.il/Categories', ] + def parse_start_url(self, response): + yield self.parse_blog(response) + def parse_blog(self, response): self.log(response.url) - blog, year, month = re.match(r'/blogread.asp\?blog=(\d+)(?:&year=(\d+)&month=(\d+))?', response.url) - if not year and not month: - now = datetime.now() - year = now.year() - month = now.month() - self.log(blog, year, month) + match = re.match(r'/blogread.asp\?blog=(\d+)(?:&year=(\d+)&month=(\d+))?', response.url) + if match: + blog, year, month = match.groups() + if not year and not month: + now = datetime.now() + year = now.year() + month = now.month() + self.log(blog, year, month) return None |