Lots of Haaretz fixes, ready to go

5 files changed, 183 insertions, 13 deletions

diff --git a/newsdiff/core/migrations/0004_auto__add_unique_haaretzimage_origin_url.py b/newsdiff/core/migrations/0004_auto__add_unique_haaretzimage_origin_url.py
new file mode 100644
index 0000000..dad824a
--- /dev/null
+++ b/newsdiff/core/migrations/0004_auto__add_unique_haaretzimage_origin_url.py
@@ -0,0 +1,77 @@
+# -*- coding: utf-8 -*-
+from south.utils import datetime_utils as datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding unique constraint on 'HaaretzImage', fields ['origin_url']
+        db.create_unique(u'core_haaretzimage', ['origin_url'])
+
+
+    def backwards(self, orm):
+        # Removing unique constraint on 'HaaretzImage', fields ['origin_url']
+        db.delete_unique(u'core_haaretzimage', ['origin_url'])
+
+
+    models = {
+        u'auth.group': {
+            'Meta': {'object_name': 'Group'},
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+        },
+        u'auth.permission': {
+            'Meta': {'ordering': "(u'content_type__app_label', u'content_type__model', u'codename')", 'unique_together': "((u'content_type', u'codename'),)", 'object_name': 'Permission'},
+            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+        },
+        u'contenttypes.contenttype': {
+            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        },
+        u'core.haaretzarticle': {
+            'Meta': {'object_name': 'HaaretzArticle'},
+            'date': ('django.db.models.fields.DateTimeField', [], {}),
+            'haaretz_id': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '12', 'db_index': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'subtitle': ('django.db.models.fields.CharField', [], {'max_length': '300', 'null': 'True', 'blank': 'True'}),
+            'text': ('django.db.models.fields.TextField', [], {}),
+            'title': ('django.db.models.fields.CharField', [], {'max_length': '140'}),
+            'url': ('django.db.models.fields.CharField', [], {'max_length': '200'})
+        },
+        u'core.haaretzimage': {
+            'Meta': {'object_name': 'HaaretzImage'},
+            'article': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'images'", 'to': u"orm['core.HaaretzArticle']"}),
+            'caption': ('django.db.models.fields.CharField', [], {'max_length': '140', 'null': 'True', 'blank': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'image': ('django.db.models.fields.files.ImageField', [], {'max_length': '100'}),
+            'origin_url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '200'})
+        },
+        u'core.newsdiffuser': {
+            'Meta': {'object_name': 'NewsDiffUser'},
+            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
+            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Group']"}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Permission']"}),
+            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+        }
+    }
+
+    complete_apps = ['core']
\ No newline at end of file
diff --git a/newsdiff/core/migrations/0005_auto__add_field_haaretzarticle_author__chg_field_haaretzarticle_subtit.py b/newsdiff/core/migrations/0005_auto__add_field_haaretzarticle_author__chg_field_haaretzarticle_subtit.py
new file mode 100644
index 0000000..fdb605c
--- /dev/null
+++ b/newsdiff/core/migrations/0005_auto__add_field_haaretzarticle_author__chg_field_haaretzarticle_subtit.py
@@ -0,0 +1,86 @@
+# -*- coding: utf-8 -*-
+from south.utils import datetime_utils as datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding field 'HaaretzArticle.author'
+        db.add_column(u'core_haaretzarticle', 'author',
+                      self.gf('django.db.models.fields.CharField')(default='(unknown)', max_length=30),
+                      keep_default=False)
+
+
+        # Changing field 'HaaretzArticle.subtitle'
+        db.alter_column(u'core_haaretzarticle', 'subtitle', self.gf('django.db.models.fields.CharField')(max_length=500, null=True))
+
+    def backwards(self, orm):
+        # Deleting field 'HaaretzArticle.author'
+        db.delete_column(u'core_haaretzarticle', 'author')
+
+
+        # Changing field 'HaaretzArticle.subtitle'
+        db.alter_column(u'core_haaretzarticle', 'subtitle', self.gf('django.db.models.fields.CharField')(max_length=300, null=True))
+
+    models = {
+        u'auth.group': {
+            'Meta': {'object_name': 'Group'},
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+        },
+        u'auth.permission': {
+            'Meta': {'ordering': "(u'content_type__app_label', u'content_type__model', u'codename')", 'unique_together': "((u'content_type', u'codename'),)", 'object_name': 'Permission'},
+            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+        },
+        u'contenttypes.contenttype': {
+            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        },
+        u'core.haaretzarticle': {
+            'Meta': {'object_name': 'HaaretzArticle'},
+            'author': ('django.db.models.fields.CharField', [], {'max_length': '30'}),
+            'date': ('django.db.models.fields.DateTimeField', [], {}),
+            'haaretz_id': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '12', 'db_index': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'subtitle': ('django.db.models.fields.CharField', [], {'max_length': '500', 'null': 'True', 'blank': 'True'}),
+            'text': ('django.db.models.fields.TextField', [], {}),
+            'title': ('django.db.models.fields.CharField', [], {'max_length': '140'}),
+            'url': ('django.db.models.fields.CharField', [], {'max_length': '200'})
+        },
+        u'core.haaretzimage': {
+            'Meta': {'object_name': 'HaaretzImage'},
+            'article': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'images'", 'to': u"orm['core.HaaretzArticle']"}),
+            'caption': ('django.db.models.fields.CharField', [], {'max_length': '140', 'null': 'True', 'blank': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'image': ('django.db.models.fields.files.ImageField', [], {'max_length': '100'}),
+            'origin_url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '200'})
+        },
+        u'core.newsdiffuser': {
+            'Meta': {'object_name': 'NewsDiffUser'},
+            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
+            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Group']"}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Permission']"}),
+            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+        }
+    }
+
+    complete_apps = ['core']
\ No newline at end of file
diff --git a/newsdiff/core/models.py b/newsdiff/core/models.py
index ef58b0a..39938fb 100644
--- a/newsdiff/core/models.py
+++ b/newsdiff/core/models.py
@@ -11,7 +11,8 @@ class HaaretzArticle(models.Model):
     url = models.CharField(max_length=200)
     haaretz_id = models.CharField(max_length=12, unique=True, db_index=True)
     title = models.CharField(max_length=140)
-    subtitle = models.CharField(max_length=300, blank=True, null=True)
+    subtitle = models.CharField(max_length=500, blank=True, null=True)
+    author = models.CharField(max_length=30)
     text = models.TextField()
     date = models.DateTimeField()
 
@@ -21,7 +22,7 @@ class HaaretzArticle(models.Model):
 
 class HaaretzImage(models.Model):
     article = models.ForeignKey(HaaretzArticle, related_name='images')
-    origin_url = models.CharField(max_length=200)
+    origin_url = models.CharField(max_length=200, unique=True)
     image = models.ImageField(upload_to='images/haaretz')
     caption = models.CharField(max_length=140, blank=True, null=True)
 
diff --git a/newsdiff/core/parsers/base.py b/newsdiff/core/parsers/base.py
index e103527..7270ed9 100644
--- a/newsdiff/core/parsers/base.py
+++ b/newsdiff/core/parsers/base.py
@@ -1,6 +1,7 @@
 import re
 import requests
 
+from bs4 import BeautifulSoup
 from pytz import timezone
 
 
@@ -13,16 +14,18 @@ class HtmlSoupParser(object):
     TIMEZONE = timezone('Asia/Jerusalem')
 
     def get_page(self, url):
-        req = requests.get(url, headers=HTTP_HEADERS)
+        req = requests.get(url, headers=self.HTTP_HEADERS)
         return BeautifulSoup(req.text, 'lxml')
 
     def process_homepage(self):
         soup = self.get_page(self.HOMEPAGE_URL)
-        self.parse_homepage(soup)
+        articles = self.parse_homepage(soup)
+        for article_url in articles:
+            self.process_article(article_url)
 
     def process_article(self, url):
         soup = self.get_page(url)
-        self.parse_article(soup)
+        self.parse_article(url, soup)
 
     def parse_homepage(soup):
         raise NotImplementedError
@@ -31,4 +34,6 @@ class HtmlSoupParser(object):
         raise NotImplementedError
 
     def clean_article_href(self, href):
+        if not href.startswith('http'):
+            href = '{}{}'.format(self.BASE_URL, href)
         return href
diff --git a/newsdiff/core/parsers/haaretz.py b/newsdiff/core/parsers/haaretz.py
index 58f7dd5..eac912a 100644
--- a/newsdiff/core/parsers/haaretz.py
+++ b/newsdiff/core/parsers/haaretz.py
@@ -11,8 +11,10 @@ from ..utils import get_file_from_url
 
 class HaaretzParser(HtmlSoupParser):
 
-    HOMEPAGE_URL = 'http://www.haaretz.co.il/'
+    BASE_URL = 'http://www.haaretz.co.il'
+    HOMEPAGE_URL = BASE_URL + '/'
     ARTICLE_HREF_PATTERN = re.compile(r'''^(http:\/\/www\.haaretz\.co\.il)?/((news|opinions|magazine|captain)/[a-zA-Z0-9\-\/]*(\.premium-)?)?\d\.\d+(#article_comments)?$''')
+    ARTICLE_ID_PATTERN = re.compile(r'1.[0-9]+')
     ARTICLE_MODEL = HaaretzArticle
     IMAGE_MODEL = HaaretzImage
 
@@ -22,12 +24,12 @@ class HaaretzParser(HtmlSoupParser):
         article_urls = list(set(map(self.clean_article_href, hrefs)))
         return article_urls
 
-    def parse_article(self, haaretz_id, soup, url=None):
-        if not url:
-            url = 'http://www.haaretz.co.il/{}'.format(haaretz_id)
+    def parse_article(self, url, soup):
+        haaretz_id = re.findall(self.ARTICLE_ID_PATTERN, url)[0]
         title = soup.find('h1', class_='mainTitle').text.strip()
         subtitle = soup.find('h2', class_='subtitle').text.strip()
         author_bar = soup.find('ul', class_='author-bar')
+        author = author_bar.find(class_=re.compile('autorBar(Anchor|Writers)')).text.strip()
         date = author_bar.find_all('li')[1].text.strip()
         time = author_bar.find_all('li')[2].text.strip()
         article_date = self.TIMEZONE.localize(datetime.strptime(' '.join([date, time]), '%d.%m.%Y %H:%M'))
@@ -50,11 +52,10 @@ class HaaretzParser(HtmlSoupParser):
                 existing_article.save()
         except self.ARTICLE_MODEL.DoesNotExist:
             article = self.ARTICLE_MODEL(url=url, haaretz_id=haaretz_id, title=title,
-                subtitle=subtitle, text=article_text, date=article_date)
+                subtitle=subtitle, author=author, text=article_text, date=article_date)
             article.save()
 
             images = soup.find('div', id='article-box').find_all('div', class_='inArticleHoldImage')
-            print images
             for image in images:
                 img = image.find('img')
                 img_url = 'http://www.haaretz.co.il{}'.format(img['src'].split('_gen')[0])
@@ -62,8 +63,8 @@ class HaaretzParser(HtmlSoupParser):
 
                 name, image_file = get_file_from_url(img_url)
 
-                article_image, created = self.IMAGE_MODEL.objects.get_or_create(article=article,
-                    origin_url=img_url, defaults={'caption': caption})
+                article_image, created = self.IMAGE_MODEL.objects.get_or_create(origin_url=img_url,
+                    defaults={'article': article, 'caption': caption})
 
                 if created:
                     article_image.image.save(name, image_file)