3 files changed, 11 insertions, 30 deletions
diff --git a/israblog/israblog/items.py b/israblog/israblog/items.py
index 706a41b..5784d7c 100644
--- a/israblog/israblog/items.py
+++ b/israblog/israblog/items.py
@@ -1,11 +1,5 @@
-# Define here the models for your scraped items
-#
-# See documentation in:
-# http://doc.scrapy.org/en/latest/topics/items.html
-
 from scrapy.item import Item, Field
 
+
 class IsrablogItem(Item):
-    # define the fields for your item here like:
-    # name = Field()
-    pass
+    page = Field()
diff --git a/israblog/israblog/settings.py b/israblog/israblog/settings.py
index 2699683..7c4faa0 100644
--- a/israblog/israblog/settings.py
+++ b/israblog/israblog/settings.py
@@ -1,10 +1,4 @@
-# Scrapy settings for israblog project
-#
-# For simplicity, this file contains only the most important settings by
-# default. All the other settings are documented here:
-#
-#     http://doc.scrapy.org/en/latest/topics/settings.html
-#
+from os import environ
 
 BOT_NAME = 'israblog'
 
@@ -13,3 +7,6 @@ NEWSPIDER_MODULE = 'israblog.spiders'
 
 # Crawl responsibly by identifying yourself (and your website) on the user-agent
 USER_AGENT = 'Israblog Archive Spider (+https://github.com/yuvadm/israblog-archive)'
+
+AWS_ACCESS_KEY_ID = environ.get('AWS_ACCESS_KEY_ID')
+AWS_SECRET_ACCESS_KEY = environ.get('AWS_SECRET_ACCESS_KEY')
diff --git a/israblog/israblog/spiders/blogs.py b/israblog/israblog/spiders/blogs.py
index 03f03fe..7447694 100644
--- a/israblog/israblog/spiders/blogs.py
+++ b/israblog/israblog/spiders/blogs.py
@@ -1,16 +1,16 @@
 import re
 
-from datetime import datetime
 from scrapy import log
 from scrapy.contrib.spiders import CrawlSpider, Rule
 from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
+from scrapy.settings import Settings
 
 
 class IsrablogSpider(CrawlSpider):
     name = 'israblog'
     allowed_domains = ['israblog.nana10.co.il']
     rules = (
-        Rule(SgmlLinkExtractor(allow=['blogread.asp?blog=\d+(&year=\d+&month=\d+)?']), callback='parse_blog'),
+        Rule(SgmlLinkExtractor(allow=[r'blogread\.asp']), callback='parse_blog', follow=True),
         Rule(SgmlLinkExtractor(allow=['/Category/\d+/.*',]), follow=True),
     )
     start_urls = [
@@ -20,17 +20,7 @@ class IsrablogSpider(CrawlSpider):
         'http://israblog.nana10.co.il/Categories',
     ]
 
-    def parse_start_url(self, response):
-        yield self.parse_blog(response)
-
     def parse_blog(self, response):
-        self.log(response.url)
-        match = re.match(r'/blogread.asp\?blog=(\d+)(?:&year=(\d+)&month=(\d+))?', response.url)
-        if match:
-            blog, year, month = match.groups()
-            if not year and not month:
-                now = datetime.now()
-                year = now.year()
-                month = now.month()
-            self.log(blog, year, month)
-        return None
+        with open(response.url.split('/')[-1], 'wb') as f:
+            f.write(response.body)
+