Lots of parser and test refactoring

5 files changed, 28 insertions, 12 deletions

diff --git a/newsdiff/core/parsers/__init__.py b/newsdiff/core/parsers/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/newsdiff/core/parsers/__init__.py
diff --git a/newsdiff/core/parsers/base.py b/newsdiff/core/parsers/base.py
index 938fe86..e103527 100644
--- a/newsdiff/core/parsers/base.py
+++ b/newsdiff/core/parsers/base.py
@@ -14,11 +14,18 @@ class HtmlSoupParser(object):
 
     def get_page(self, url):
         req = requests.get(url, headers=HTTP_HEADERS)
-        if req.ok:
-            return BeautifulSoup(req.text, 'lxml')
+        return BeautifulSoup(req.text, 'lxml')
 
-    def get_homepage(self):
-        return get_page(self.HOMEPAGE_URL)
+    def process_homepage(self):
+        soup = self.get_page(self.HOMEPAGE_URL)
+        self.parse_homepage(soup)
+
+    def process_article(self, url):
+        soup = self.get_page(url)
+        self.parse_article(soup)
+
+    def parse_homepage(soup):
+        raise NotImplementedError
 
     def parse_article(self, url):
         raise NotImplementedError
diff --git a/newsdiff/core/parsers/haaretz.py b/newsdiff/core/parsers/haaretz.py
index e44988f..c43379f 100644
--- a/newsdiff/core/parsers/haaretz.py
+++ b/newsdiff/core/parsers/haaretz.py
@@ -16,14 +16,12 @@ class HaaretzParser(HtmlSoupParser):
     ARTICLE_MODEL = HaaretzArticle
     IMAGE_MODEL = HaaretzImage
 
-    def parse_haaretz_homepage(self):
-        soup = self.get_homepage()
-        articles = soup.find_all('a', href=ARTICLE_HREF_PATTERN)
+    def parse_homepage(self, soup):
+        articles = soup.find_all('a', href=self.ARTICLE_HREF_PATTERN)
         hrefs = [article['href'] for article in articles]
-        hrefs = list(set(map(clean_haaretz_href, hrefs)))
+        return list(set(map(self.clean_article_href, hrefs)))
 
-    def parse_article(self, url):
-        soup = self.get_page(url)
+    def parse_article(self, soup):
         title = soup.find('h1', class_='mainTitle').text.strip()
         subtitle = soup.find('h2', class_='subtitle').text.strip()
         author_bar = soup.find('ul', class_='author-bar')
diff --git a/newsdiff/core/tests/haaretz.html b/newsdiff/core/tests/content/haaretz.html
index ea4d761..ea4d761 100644
--- a/newsdiff/core/tests/haaretz.html
+++ b/newsdiff/core/tests/content/haaretz.html
diff --git a/newsdiff/core/tests/tests.py b/newsdiff/core/tests/tests.py
index 2146b73..35566e1 100644
--- a/newsdiff/core/tests/tests.py
+++ b/newsdiff/core/tests/tests.py
@@ -1,10 +1,21 @@
+from bs4 import BeautifulSoup
 from django.test import TestCase
+from unipath import FSPath as Path
 
+from newsdiff.core.parsers.haaretz import HaaretzParser
 
 class HaaretzParsingTestCase(TestCase):
 
+    CONTENT_DIR = Path(__file__).absolute().parent.child('content')
+
     def setUp(self):
-        pass
+        self.parser = HaaretzParser()
+
+    def _get_content(self, filename):
+        with open(self.CONTENT_DIR.child(filename), 'r') as f:
+            return f.read()
 
     def test_parse_homepage(self):
-        self.assertTrue(True)
+        soup = BeautifulSoup(self._get_content('haaretz.html'))
+        articles = self.parser.parse_homepage(soup)
+        self.assertTrue(articles)