diff options
| -rw-r--r-- | israblog/israblog/items.py | 10 | ||||
| -rw-r--r-- | israblog/israblog/settings.py | 11 | ||||
| -rw-r--r-- | israblog/israblog/spiders/blogs.py | 20 |
3 files changed, 11 insertions, 30 deletions
diff --git a/israblog/israblog/items.py b/israblog/israblog/items.py index 706a41b..5784d7c 100644 --- a/israblog/israblog/items.py +++ b/israblog/israblog/items.py @@ -1,11 +1,5 @@ -# Define here the models for your scraped items -# -# See documentation in: -# http://doc.scrapy.org/en/latest/topics/items.html - from scrapy.item import Item, Field + class IsrablogItem(Item): - # define the fields for your item here like: - # name = Field() - pass + page = Field() diff --git a/israblog/israblog/settings.py b/israblog/israblog/settings.py index 2699683..7c4faa0 100644 --- a/israblog/israblog/settings.py +++ b/israblog/israblog/settings.py @@ -1,10 +1,4 @@ -# Scrapy settings for israblog project -# -# For simplicity, this file contains only the most important settings by -# default. All the other settings are documented here: -# -# http://doc.scrapy.org/en/latest/topics/settings.html -# +from os import environ BOT_NAME = 'israblog' @@ -13,3 +7,6 @@ NEWSPIDER_MODULE = 'israblog.spiders' # Crawl responsibly by identifying yourself (and your website) on the user-agent USER_AGENT = 'Israblog Archive Spider (+https://github.com/yuvadm/israblog-archive)' + +AWS_ACCESS_KEY_ID = environ.get('AWS_ACCESS_KEY_ID') +AWS_SECRET_ACCESS_KEY = environ.get('AWS_SECRET_ACCESS_KEY') diff --git a/israblog/israblog/spiders/blogs.py b/israblog/israblog/spiders/blogs.py index 03f03fe..7447694 100644 --- a/israblog/israblog/spiders/blogs.py +++ b/israblog/israblog/spiders/blogs.py @@ -1,16 +1,16 @@ import re -from datetime import datetime from scrapy import log from scrapy.contrib.spiders import CrawlSpider, Rule from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor +from scrapy.settings import Settings class IsrablogSpider(CrawlSpider): name = 'israblog' allowed_domains = ['israblog.nana10.co.il'] rules = ( - Rule(SgmlLinkExtractor(allow=['blogread.asp?blog=\d+(&year=\d+&month=\d+)?']), callback='parse_blog'), + Rule(SgmlLinkExtractor(allow=[r'blogread\.asp']), callback='parse_blog', follow=True), Rule(SgmlLinkExtractor(allow=['/Category/\d+/.*',]), follow=True), ) start_urls = [ @@ -20,17 +20,7 @@ class IsrablogSpider(CrawlSpider): 'http://israblog.nana10.co.il/Categories', ] - def parse_start_url(self, response): - yield self.parse_blog(response) - def parse_blog(self, response): - self.log(response.url) - match = re.match(r'/blogread.asp\?blog=(\d+)(?:&year=(\d+)&month=(\d+))?', response.url) - if match: - blog, year, month = match.groups() - if not year and not month: - now = datetime.now() - year = now.year() - month = now.month() - self.log(blog, year, month) - return None + with open(response.url.split('/')[-1], 'wb') as f: + f.write(response.body) + |