From d8d555b1fbc6a641c33265ee60ee872748437c4b Mon Sep 17 00:00:00 2001 From: Yuval Adam Date: Fri, 24 Jan 2014 23:38:03 +0100 Subject: Normalize APIs and tests --- newsdiff/core/parsers/base.py | 4 ++-- newsdiff/core/parsers/haaretz.py | 4 ++-- newsdiff/core/tests/tests.py | 5 +++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/newsdiff/core/parsers/base.py b/newsdiff/core/parsers/base.py index dae2949..532568a 100644 --- a/newsdiff/core/parsers/base.py +++ b/newsdiff/core/parsers/base.py @@ -26,10 +26,10 @@ class HtmlSoupParser(object): soup = self.get_page(url) self.parse_article(url, soup) - def parse_homepage(soup): + def parse_homepage(self, soup): raise NotImplementedError - def parse_article(self, url): + def parse_article(self, url, soup): raise NotImplementedError def clean_article_href(self, href): diff --git a/newsdiff/core/parsers/haaretz.py b/newsdiff/core/parsers/haaretz.py index d154fb9..8928663 100644 --- a/newsdiff/core/parsers/haaretz.py +++ b/newsdiff/core/parsers/haaretz.py @@ -15,8 +15,8 @@ class HaaretzParser(HtmlSoupParser): BASE_URL = 'http://www.haaretz.co.il' HOMEPAGE_URL = BASE_URL + '/' - ARTICLE_HREF_PATTERN = re.compile(r'''^(http:\/\/www\.haaretz\.co\.il)?/((news|opinions|magazine|captain)/[a-zA-Z0-9\-\/]*(\.premium-)?)?\d\.\d+(#article_comments)?$''') - ARTICLE_ID_PATTERN = re.compile(r'1.[0-9]+') + ARTICLE_HREF_PATTERN = re.compile(r'''^(http:\/\/www\.haaretz\.co\.il)?/((news|opinions|magazine|captain)/[a-zA-Z0-9\-\/]*(\.premium-)?)?1\.\d+(#article_comments)?$''') + ARTICLE_ID_PATTERN = re.compile(r'1\.\d+') ARTICLE_MODEL = HaaretzArticle IMAGE_MODEL = HaaretzImage diff --git a/newsdiff/core/tests/tests.py b/newsdiff/core/tests/tests.py index 6f91aad..b30b143 100644 --- a/newsdiff/core/tests/tests.py +++ b/newsdiff/core/tests/tests.py @@ -26,10 +26,11 @@ class HaaretzParsingTestCase(TestCase): def test_parse_article(self): article_id = '1.2198200' + article_url = '{}/{}'.format(self.parser.BASE_URL, article_id) soup = self._get_soup('{}.html'.format(article_id)) - self.parser.parse_article(article_id, soup) + self.parser.parse_article(article_url, soup) article = HaaretzArticle.objects.get(haaretz_id=article_id) - self.assertEqual(article.url, 'http://www.haaretz.co.il/{}'.format(article_id)) + self.assertEqual(article.url, article_url) images = article.images self.assertEqual(images.count(), 2) -- cgit v1.3.1