Initial spider to test

2 files changed, 30 insertions, 1 deletions

diff --git a/israblog/israblog/settings.py b/israblog/israblog/settings.py
index ce238f2..2699683 100644
--- a/israblog/israblog/settings.py
+++ b/israblog/israblog/settings.py
@@ -12,4 +12,4 @@ SPIDER_MODULES = ['israblog.spiders']
 NEWSPIDER_MODULE = 'israblog.spiders'
 
 # Crawl responsibly by identifying yourself (and your website) on the user-agent
-#USER_AGENT = 'israblog (+http://www.yourdomain.com)'
+USER_AGENT = 'Israblog Archive Spider (+https://github.com/yuvadm/israblog-archive)'
diff --git a/israblog/israblog/spiders/blogs.py b/israblog/israblog/spiders/blogs.py
new file mode 100644
index 0000000..b7aadc1
--- /dev/null
+++ b/israblog/israblog/spiders/blogs.py
@@ -0,0 +1,29 @@
+from datetime import datetime
+from scrapy import log
+from scrapy.contrib.spiders import CrawlSpider, Rule
+from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
+
+
+class IsrablogSpider(CrawlSpider):
+    name = 'israblog'
+    allowed_domains = ['israblog.nana10.co.il']
+    rules = [Rule(SgmlLinkExtractor(allow=[
+        '/blogread.asp?blog=\d+',
+        '/blogread.asp?blog=\d+&year=\d+&month=\d+'
+    ]), 'parse_blog')]
+    start_urls = [
+        'http://israblog.nana10.co.il/',
+        'http://israblog.nana10.co.il/lastblogs.aspx',
+        'http://israblog.nana10.co.il/activeblogs.aspx',
+        'http://israblog.nana10.co.il/Categories',
+    ]
+
+    def parse_blog(self, response):
+        self.log(response.url)
+        blog, year, month = re.match(r'/blogread.asp\?blog=(\d+)(?:&year=(\d+)&month=(\d+))?', response.url)
+        if not year and not month:
+            now = datetime.now()
+            year = now.year()
+            month = now.month()
+        self.log(blog, year, month)
+        return None