diff options
5 files changed, 183 insertions, 13 deletions
diff --git a/newsdiff/core/migrations/0004_auto__add_unique_haaretzimage_origin_url.py b/newsdiff/core/migrations/0004_auto__add_unique_haaretzimage_origin_url.py new file mode 100644 index 0000000..dad824a --- /dev/null +++ b/newsdiff/core/migrations/0004_auto__add_unique_haaretzimage_origin_url.py @@ -0,0 +1,77 @@ +# -*- coding: utf-8 -*- +from south.utils import datetime_utils as datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding unique constraint on 'HaaretzImage', fields ['origin_url'] + db.create_unique(u'core_haaretzimage', ['origin_url']) + + + def backwards(self, orm): + # Removing unique constraint on 'HaaretzImage', fields ['origin_url'] + db.delete_unique(u'core_haaretzimage', ['origin_url']) + + + models = { + u'auth.group': { + 'Meta': {'object_name': 'Group'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + u'auth.permission': { + 'Meta': {'ordering': "(u'content_type__app_label', u'content_type__model', u'codename')", 'unique_together': "((u'content_type', u'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + u'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + u'core.haaretzarticle': { + 'Meta': {'object_name': 'HaaretzArticle'}, + 'date': ('django.db.models.fields.DateTimeField', [], {}), + 'haaretz_id': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '12', 'db_index': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'subtitle': ('django.db.models.fields.CharField', [], {'max_length': '300', 'null': 'True', 'blank': 'True'}), + 'text': ('django.db.models.fields.TextField', [], {}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '140'}), + 'url': ('django.db.models.fields.CharField', [], {'max_length': '200'}) + }, + u'core.haaretzimage': { + 'Meta': {'object_name': 'HaaretzImage'}, + 'article': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'images'", 'to': u"orm['core.HaaretzArticle']"}), + 'caption': ('django.db.models.fields.CharField', [], {'max_length': '140', 'null': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'image': ('django.db.models.fields.files.ImageField', [], {'max_length': '100'}), + 'origin_url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '200'}) + }, + u'core.newsdiffuser': { + 'Meta': {'object_name': 'NewsDiffUser'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Group']"}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Permission']"}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + } + } + + complete_apps = ['core']
\ No newline at end of file diff --git a/newsdiff/core/migrations/0005_auto__add_field_haaretzarticle_author__chg_field_haaretzarticle_subtit.py b/newsdiff/core/migrations/0005_auto__add_field_haaretzarticle_author__chg_field_haaretzarticle_subtit.py new file mode 100644 index 0000000..fdb605c --- /dev/null +++ b/newsdiff/core/migrations/0005_auto__add_field_haaretzarticle_author__chg_field_haaretzarticle_subtit.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- +from south.utils import datetime_utils as datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding field 'HaaretzArticle.author' + db.add_column(u'core_haaretzarticle', 'author', + self.gf('django.db.models.fields.CharField')(default='(unknown)', max_length=30), + keep_default=False) + + + # Changing field 'HaaretzArticle.subtitle' + db.alter_column(u'core_haaretzarticle', 'subtitle', self.gf('django.db.models.fields.CharField')(max_length=500, null=True)) + + def backwards(self, orm): + # Deleting field 'HaaretzArticle.author' + db.delete_column(u'core_haaretzarticle', 'author') + + + # Changing field 'HaaretzArticle.subtitle' + db.alter_column(u'core_haaretzarticle', 'subtitle', self.gf('django.db.models.fields.CharField')(max_length=300, null=True)) + + models = { + u'auth.group': { + 'Meta': {'object_name': 'Group'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + u'auth.permission': { + 'Meta': {'ordering': "(u'content_type__app_label', u'content_type__model', u'codename')", 'unique_together': "((u'content_type', u'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + u'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + u'core.haaretzarticle': { + 'Meta': {'object_name': 'HaaretzArticle'}, + 'author': ('django.db.models.fields.CharField', [], {'max_length': '30'}), + 'date': ('django.db.models.fields.DateTimeField', [], {}), + 'haaretz_id': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '12', 'db_index': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'subtitle': ('django.db.models.fields.CharField', [], {'max_length': '500', 'null': 'True', 'blank': 'True'}), + 'text': ('django.db.models.fields.TextField', [], {}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '140'}), + 'url': ('django.db.models.fields.CharField', [], {'max_length': '200'}) + }, + u'core.haaretzimage': { + 'Meta': {'object_name': 'HaaretzImage'}, + 'article': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'images'", 'to': u"orm['core.HaaretzArticle']"}), + 'caption': ('django.db.models.fields.CharField', [], {'max_length': '140', 'null': 'True', 'blank': 'True'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'image': ('django.db.models.fields.files.ImageField', [], {'max_length': '100'}), + 'origin_url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '200'}) + }, + u'core.newsdiffuser': { + 'Meta': {'object_name': 'NewsDiffUser'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Group']"}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Permission']"}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + } + } + + complete_apps = ['core']
\ No newline at end of file diff --git a/newsdiff/core/models.py b/newsdiff/core/models.py index ef58b0a..39938fb 100644 --- a/newsdiff/core/models.py +++ b/newsdiff/core/models.py @@ -11,7 +11,8 @@ class HaaretzArticle(models.Model): url = models.CharField(max_length=200) haaretz_id = models.CharField(max_length=12, unique=True, db_index=True) title = models.CharField(max_length=140) - subtitle = models.CharField(max_length=300, blank=True, null=True) + subtitle = models.CharField(max_length=500, blank=True, null=True) + author = models.CharField(max_length=30) text = models.TextField() date = models.DateTimeField() @@ -21,7 +22,7 @@ class HaaretzArticle(models.Model): class HaaretzImage(models.Model): article = models.ForeignKey(HaaretzArticle, related_name='images') - origin_url = models.CharField(max_length=200) + origin_url = models.CharField(max_length=200, unique=True) image = models.ImageField(upload_to='images/haaretz') caption = models.CharField(max_length=140, blank=True, null=True) diff --git a/newsdiff/core/parsers/base.py b/newsdiff/core/parsers/base.py index e103527..7270ed9 100644 --- a/newsdiff/core/parsers/base.py +++ b/newsdiff/core/parsers/base.py @@ -1,6 +1,7 @@ import re import requests +from bs4 import BeautifulSoup from pytz import timezone @@ -13,16 +14,18 @@ class HtmlSoupParser(object): TIMEZONE = timezone('Asia/Jerusalem') def get_page(self, url): - req = requests.get(url, headers=HTTP_HEADERS) + req = requests.get(url, headers=self.HTTP_HEADERS) return BeautifulSoup(req.text, 'lxml') def process_homepage(self): soup = self.get_page(self.HOMEPAGE_URL) - self.parse_homepage(soup) + articles = self.parse_homepage(soup) + for article_url in articles: + self.process_article(article_url) def process_article(self, url): soup = self.get_page(url) - self.parse_article(soup) + self.parse_article(url, soup) def parse_homepage(soup): raise NotImplementedError @@ -31,4 +34,6 @@ class HtmlSoupParser(object): raise NotImplementedError def clean_article_href(self, href): + if not href.startswith('http'): + href = '{}{}'.format(self.BASE_URL, href) return href diff --git a/newsdiff/core/parsers/haaretz.py b/newsdiff/core/parsers/haaretz.py index 58f7dd5..eac912a 100644 --- a/newsdiff/core/parsers/haaretz.py +++ b/newsdiff/core/parsers/haaretz.py @@ -11,8 +11,10 @@ from ..utils import get_file_from_url class HaaretzParser(HtmlSoupParser): - HOMEPAGE_URL = 'http://www.haaretz.co.il/' + BASE_URL = 'http://www.haaretz.co.il' + HOMEPAGE_URL = BASE_URL + '/' ARTICLE_HREF_PATTERN = re.compile(r'''^(http:\/\/www\.haaretz\.co\.il)?/((news|opinions|magazine|captain)/[a-zA-Z0-9\-\/]*(\.premium-)?)?\d\.\d+(#article_comments)?$''') + ARTICLE_ID_PATTERN = re.compile(r'1.[0-9]+') ARTICLE_MODEL = HaaretzArticle IMAGE_MODEL = HaaretzImage @@ -22,12 +24,12 @@ class HaaretzParser(HtmlSoupParser): article_urls = list(set(map(self.clean_article_href, hrefs))) return article_urls - def parse_article(self, haaretz_id, soup, url=None): - if not url: - url = 'http://www.haaretz.co.il/{}'.format(haaretz_id) + def parse_article(self, url, soup): + haaretz_id = re.findall(self.ARTICLE_ID_PATTERN, url)[0] title = soup.find('h1', class_='mainTitle').text.strip() subtitle = soup.find('h2', class_='subtitle').text.strip() author_bar = soup.find('ul', class_='author-bar') + author = author_bar.find(class_=re.compile('autorBar(Anchor|Writers)')).text.strip() date = author_bar.find_all('li')[1].text.strip() time = author_bar.find_all('li')[2].text.strip() article_date = self.TIMEZONE.localize(datetime.strptime(' '.join([date, time]), '%d.%m.%Y %H:%M')) @@ -50,11 +52,10 @@ class HaaretzParser(HtmlSoupParser): existing_article.save() except self.ARTICLE_MODEL.DoesNotExist: article = self.ARTICLE_MODEL(url=url, haaretz_id=haaretz_id, title=title, - subtitle=subtitle, text=article_text, date=article_date) + subtitle=subtitle, author=author, text=article_text, date=article_date) article.save() images = soup.find('div', id='article-box').find_all('div', class_='inArticleHoldImage') - print images for image in images: img = image.find('img') img_url = 'http://www.haaretz.co.il{}'.format(img['src'].split('_gen')[0]) @@ -62,8 +63,8 @@ class HaaretzParser(HtmlSoupParser): name, image_file = get_file_from_url(img_url) - article_image, created = self.IMAGE_MODEL.objects.get_or_create(article=article, - origin_url=img_url, defaults={'caption': caption}) + article_image, created = self.IMAGE_MODEL.objects.get_or_create(origin_url=img_url, + defaults={'article': article, 'caption': caption}) if created: article_image.image.save(name, image_file) |