Add parsing tasks

3 files changed, 20 insertions, 2 deletions

diff --git a/newsdiff/core/parsers/base.py b/newsdiff/core/parsers/base.py
index 7270ed9..dae2949 100644
--- a/newsdiff/core/parsers/base.py
+++ b/newsdiff/core/parsers/base.py
@@ -20,8 +20,7 @@ class HtmlSoupParser(object):
     def process_homepage(self):
         soup = self.get_page(self.HOMEPAGE_URL)
         articles = self.parse_homepage(soup)
-        for article_url in articles:
-            self.process_article(article_url)
+        return articles
 
     def process_article(self, url):
         soup = self.get_page(url)
diff --git a/newsdiff/core/tasks.py b/newsdiff/core/tasks.py
index 223aea5..34e12ac 100644
--- a/newsdiff/core/tasks.py
+++ b/newsdiff/core/tasks.py
@@ -1 +1,16 @@
+import logging
+
 from newsdiff.celery import app
+from newsdiff.core.parsers.haaretz import HaaretzParser
+
+@app.task()
+def process_haaretz_homepage():
+    hp = HaaretzParser()
+    articles = hp.process_homepage()
+    for article_url in articles:
+        process_haaretz_article.delay(article_url)
+
+@app.task(rate_limit='12/m')
+def process_haaretz_article(url):
+    hp = HaaretzParser()
+    hp.process_article(url)
diff --git a/newsdiff/settings/base.py b/newsdiff/settings/base.py
index 493384f..6c3f7ad 100644
--- a/newsdiff/settings/base.py
+++ b/newsdiff/settings/base.py
@@ -132,4 +132,8 @@ CELERYBEAT_SCHEDULE = {
         'task': 'newsdiff.rain.tasks.import_rain_radar_image',
         'schedule': timedelta(minutes=10)
     },
+    'haaretz_homepage': {
+        'task': 'newsdiff.core.tasks.process_haaretz_homepage',
+        'schedule': timedelta(minutes=15)
+    }
 }