From 917e5380cd000db7cf1d2f3f9eb594dbf86a97dc Mon Sep 17 00:00:00 2001 From: Yuval Adam Date: Sat, 29 Mar 2014 02:15:54 +0300 Subject: Initial spider to test --- israblog/israblog/settings.py | 2 +- israblog/israblog/spiders/blogs.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 israblog/israblog/spiders/blogs.py (limited to 'israblog') diff --git a/israblog/israblog/settings.py b/israblog/israblog/settings.py index ce238f2..2699683 100644 --- a/israblog/israblog/settings.py +++ b/israblog/israblog/settings.py @@ -12,4 +12,4 @@ SPIDER_MODULES = ['israblog.spiders'] NEWSPIDER_MODULE = 'israblog.spiders' # Crawl responsibly by identifying yourself (and your website) on the user-agent -#USER_AGENT = 'israblog (+http://www.yourdomain.com)' +USER_AGENT = 'Israblog Archive Spider (+https://github.com/yuvadm/israblog-archive)' diff --git a/israblog/israblog/spiders/blogs.py b/israblog/israblog/spiders/blogs.py new file mode 100644 index 0000000..b7aadc1 --- /dev/null +++ b/israblog/israblog/spiders/blogs.py @@ -0,0 +1,29 @@ +from datetime import datetime +from scrapy import log +from scrapy.contrib.spiders import CrawlSpider, Rule +from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor + + +class IsrablogSpider(CrawlSpider): + name = 'israblog' + allowed_domains = ['israblog.nana10.co.il'] + rules = [Rule(SgmlLinkExtractor(allow=[ + '/blogread.asp?blog=\d+', + '/blogread.asp?blog=\d+&year=\d+&month=\d+' + ]), 'parse_blog')] + start_urls = [ + 'http://israblog.nana10.co.il/', + 'http://israblog.nana10.co.il/lastblogs.aspx', + 'http://israblog.nana10.co.il/activeblogs.aspx', + 'http://israblog.nana10.co.il/Categories', + ] + + def parse_blog(self, response): + self.log(response.url) + blog, year, month = re.match(r'/blogread.asp\?blog=(\d+)(?:&year=(\d+)&month=(\d+))?', response.url) + if not year and not month: + now = datetime.now() + year = now.year() + month = now.month() + self.log(blog, year, month) + return None -- cgit v1.3.1