diff options
| -rw-r--r-- | newsdiff/core/parsers/base.py | 3 | ||||
| -rw-r--r-- | newsdiff/core/tasks.py | 15 | ||||
| -rw-r--r-- | newsdiff/settings/base.py | 4 |
3 files changed, 20 insertions, 2 deletions
diff --git a/newsdiff/core/parsers/base.py b/newsdiff/core/parsers/base.py index 7270ed9..dae2949 100644 --- a/newsdiff/core/parsers/base.py +++ b/newsdiff/core/parsers/base.py @@ -20,8 +20,7 @@ class HtmlSoupParser(object): def process_homepage(self): soup = self.get_page(self.HOMEPAGE_URL) articles = self.parse_homepage(soup) - for article_url in articles: - self.process_article(article_url) + return articles def process_article(self, url): soup = self.get_page(url) diff --git a/newsdiff/core/tasks.py b/newsdiff/core/tasks.py index 223aea5..34e12ac 100644 --- a/newsdiff/core/tasks.py +++ b/newsdiff/core/tasks.py @@ -1 +1,16 @@ +import logging + from newsdiff.celery import app +from newsdiff.core.parsers.haaretz import HaaretzParser + +@app.task() +def process_haaretz_homepage(): + hp = HaaretzParser() + articles = hp.process_homepage() + for article_url in articles: + process_haaretz_article.delay(article_url) + +@app.task(rate_limit='12/m') +def process_haaretz_article(url): + hp = HaaretzParser() + hp.process_article(url) diff --git a/newsdiff/settings/base.py b/newsdiff/settings/base.py index 493384f..6c3f7ad 100644 --- a/newsdiff/settings/base.py +++ b/newsdiff/settings/base.py @@ -132,4 +132,8 @@ CELERYBEAT_SCHEDULE = { 'task': 'newsdiff.rain.tasks.import_rain_radar_image', 'schedule': timedelta(minutes=10) }, + 'haaretz_homepage': { + 'task': 'newsdiff.core.tasks.process_haaretz_homepage', + 'schedule': timedelta(minutes=15) + } } |