From 920b3d9d6731ac66bc8abb682cf932cd7568eb5d Mon Sep 17 00:00:00 2001 From: Yuval Adam Date: Fri, 24 Jan 2014 15:41:34 +0100 Subject: Add parsing tasks --- newsdiff/core/parsers/base.py | 3 +-- newsdiff/core/tasks.py | 15 +++++++++++++++ newsdiff/settings/base.py | 4 ++++ 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/newsdiff/core/parsers/base.py b/newsdiff/core/parsers/base.py index 7270ed9..dae2949 100644 --- a/newsdiff/core/parsers/base.py +++ b/newsdiff/core/parsers/base.py @@ -20,8 +20,7 @@ class HtmlSoupParser(object): def process_homepage(self): soup = self.get_page(self.HOMEPAGE_URL) articles = self.parse_homepage(soup) - for article_url in articles: - self.process_article(article_url) + return articles def process_article(self, url): soup = self.get_page(url) diff --git a/newsdiff/core/tasks.py b/newsdiff/core/tasks.py index 223aea5..34e12ac 100644 --- a/newsdiff/core/tasks.py +++ b/newsdiff/core/tasks.py @@ -1 +1,16 @@ +import logging + from newsdiff.celery import app +from newsdiff.core.parsers.haaretz import HaaretzParser + +@app.task() +def process_haaretz_homepage(): + hp = HaaretzParser() + articles = hp.process_homepage() + for article_url in articles: + process_haaretz_article.delay(article_url) + +@app.task(rate_limit='12/m') +def process_haaretz_article(url): + hp = HaaretzParser() + hp.process_article(url) diff --git a/newsdiff/settings/base.py b/newsdiff/settings/base.py index 493384f..6c3f7ad 100644 --- a/newsdiff/settings/base.py +++ b/newsdiff/settings/base.py @@ -132,4 +132,8 @@ CELERYBEAT_SCHEDULE = { 'task': 'newsdiff.rain.tasks.import_rain_radar_image', 'schedule': timedelta(minutes=10) }, + 'haaretz_homepage': { + 'task': 'newsdiff.core.tasks.process_haaretz_homepage', + 'schedule': timedelta(minutes=15) + } } -- cgit v1.3.1