From 762474eecd03b8068228660c904bfaf53f87ed65 Mon Sep 17 00:00:00 2001 From: Yuval Adam Date: Sun, 8 Dec 2013 21:48:10 +0200 Subject: DB cleanup, working Haaretz tasks --- .gitignore | 2 + newsdiff/__init__.py | 3 + newsdiff/celery.py | 12 +++ newsdiff/core/admin.py | 2 + newsdiff/core/migrations/0001_initial.py | 85 +++++++++++++++- .../core/migrations/0002_auto__add_newsdiffuser.py | 112 --------------------- newsdiff/core/models.py | 3 +- newsdiff/core/tasks.py | 41 ++++++++ newsdiff/core/utils.py | 13 +++ newsdiff/settings/base.py | 1 + requirements.txt | 3 + 11 files changed, 162 insertions(+), 115 deletions(-) create mode 100644 newsdiff/celery.py delete mode 100644 newsdiff/core/migrations/0002_auto__add_newsdiffuser.py create mode 100644 newsdiff/core/tasks.py create mode 100644 newsdiff/core/utils.py diff --git a/.gitignore b/.gitignore index 0d20b64..641b4c4 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ *.pyc + +newsdiff/media/ diff --git a/newsdiff/__init__.py b/newsdiff/__init__.py index e69de29..1afb35e 100644 --- a/newsdiff/__init__.py +++ b/newsdiff/__init__.py @@ -0,0 +1,3 @@ +from __future__ import absolute_import + +from .celery import app diff --git a/newsdiff/celery.py b/newsdiff/celery.py new file mode 100644 index 0000000..c1e2111 --- /dev/null +++ b/newsdiff/celery.py @@ -0,0 +1,12 @@ +from __future__ import absolute_import + +import os + +from celery import Celery +from django.conf import settings + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'segmanta.settings') + +app = Celery('newsdiff') +app.config_from_object('django.conf:settings') +app.autodiscover_tasks(lambda: settings.INSTALLED_APPS) diff --git a/newsdiff/core/admin.py b/newsdiff/core/admin.py index 481afd5..ab4b576 100644 --- a/newsdiff/core/admin.py +++ b/newsdiff/core/admin.py @@ -2,3 +2,5 @@ from django.contrib import admin from .models import * admin.site.register(NewsDiffUser) +admin.site.register(HaaretzArticle) +admin.site.register(HaaretzImage) diff --git a/newsdiff/core/migrations/0001_initial.py b/newsdiff/core/migrations/0001_initial.py index b32e267..a42a829 100644 --- a/newsdiff/core/migrations/0001_initial.py +++ b/newsdiff/core/migrations/0001_initial.py @@ -8,12 +8,46 @@ from django.db import models class Migration(SchemaMigration): def forwards(self, orm): + # Adding model 'NewsDiffUser' + db.create_table(u'core_newsdiffuser', ( + (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('password', self.gf('django.db.models.fields.CharField')(max_length=128)), + ('last_login', self.gf('django.db.models.fields.DateTimeField')(default=datetime.datetime.now)), + ('is_superuser', self.gf('django.db.models.fields.BooleanField')(default=False)), + ('username', self.gf('django.db.models.fields.CharField')(unique=True, max_length=30)), + ('first_name', self.gf('django.db.models.fields.CharField')(max_length=30, blank=True)), + ('last_name', self.gf('django.db.models.fields.CharField')(max_length=30, blank=True)), + ('email', self.gf('django.db.models.fields.EmailField')(max_length=75, blank=True)), + ('is_staff', self.gf('django.db.models.fields.BooleanField')(default=False)), + ('is_active', self.gf('django.db.models.fields.BooleanField')(default=True)), + ('date_joined', self.gf('django.db.models.fields.DateTimeField')(default=datetime.datetime.now)), + )) + db.send_create_signal(u'core', ['NewsDiffUser']) + + # Adding M2M table for field groups on 'NewsDiffUser' + m2m_table_name = db.shorten_name(u'core_newsdiffuser_groups') + db.create_table(m2m_table_name, ( + ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)), + ('newsdiffuser', models.ForeignKey(orm[u'core.newsdiffuser'], null=False)), + ('group', models.ForeignKey(orm[u'auth.group'], null=False)) + )) + db.create_unique(m2m_table_name, ['newsdiffuser_id', 'group_id']) + + # Adding M2M table for field user_permissions on 'NewsDiffUser' + m2m_table_name = db.shorten_name(u'core_newsdiffuser_user_permissions') + db.create_table(m2m_table_name, ( + ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)), + ('newsdiffuser', models.ForeignKey(orm[u'core.newsdiffuser'], null=False)), + ('permission', models.ForeignKey(orm[u'auth.permission'], null=False)) + )) + db.create_unique(m2m_table_name, ['newsdiffuser_id', 'permission_id']) + # Adding model 'HaaretzArticle' db.create_table(u'core_haaretzarticle', ( (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), ('url', self.gf('django.db.models.fields.CharField')(max_length=200)), ('title', self.gf('django.db.models.fields.CharField')(max_length=140)), - ('subtitle', self.gf('django.db.models.fields.CharField')(max_length=300)), + ('subtitle', self.gf('django.db.models.fields.CharField')(max_length=300, null=True, blank=True)), ('text', self.gf('django.db.models.fields.TextField')()), ('date', self.gf('django.db.models.fields.DateTimeField')()), )) @@ -25,11 +59,21 @@ class Migration(SchemaMigration): ('article', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['core.HaaretzArticle'])), ('origin_url', self.gf('django.db.models.fields.CharField')(max_length=200)), ('image', self.gf('django.db.models.fields.files.ImageField')(max_length=100)), + ('caption', self.gf('django.db.models.fields.CharField')(max_length=140, null=True, blank=True)), )) db.send_create_signal(u'core', ['HaaretzImage']) def backwards(self, orm): + # Deleting model 'NewsDiffUser' + db.delete_table(u'core_newsdiffuser') + + # Removing M2M table for field groups on 'NewsDiffUser' + db.delete_table(db.shorten_name(u'core_newsdiffuser_groups')) + + # Removing M2M table for field user_permissions on 'NewsDiffUser' + db.delete_table(db.shorten_name(u'core_newsdiffuser_user_permissions')) + # Deleting model 'HaaretzArticle' db.delete_table(u'core_haaretzarticle') @@ -38,11 +82,31 @@ class Migration(SchemaMigration): models = { + u'auth.group': { + 'Meta': {'object_name': 'Group'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) + }, + u'auth.permission': { + 'Meta': {'ordering': "(u'content_type__app_label', u'content_type__model', u'codename')", 'unique_together': "((u'content_type', u'codename'),)", 'object_name': 'Permission'}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + u'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, u'core.haaretzarticle': { 'Meta': {'object_name': 'HaaretzArticle'}, 'date': ('django.db.models.fields.DateTimeField', [], {}), u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'subtitle': ('django.db.models.fields.CharField', [], {'max_length': '300'}), + 'subtitle': ('django.db.models.fields.CharField', [], {'max_length': '300', 'null': 'True', 'blank': 'True'}), 'text': ('django.db.models.fields.TextField', [], {}), 'title': ('django.db.models.fields.CharField', [], {'max_length': '140'}), 'url': ('django.db.models.fields.CharField', [], {'max_length': '200'}) @@ -50,9 +114,26 @@ class Migration(SchemaMigration): u'core.haaretzimage': { 'Meta': {'object_name': 'HaaretzImage'}, 'article': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['core.HaaretzArticle']"}), + 'caption': ('django.db.models.fields.CharField', [], {'max_length': '140', 'null': 'True', 'blank': 'True'}), u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), 'image': ('django.db.models.fields.files.ImageField', [], {'max_length': '100'}), 'origin_url': ('django.db.models.fields.CharField', [], {'max_length': '200'}) + }, + u'core.newsdiffuser': { + 'Meta': {'object_name': 'NewsDiffUser'}, + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Group']"}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Permission']"}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) } } diff --git a/newsdiff/core/migrations/0002_auto__add_newsdiffuser.py b/newsdiff/core/migrations/0002_auto__add_newsdiffuser.py deleted file mode 100644 index 0f9bfc3..0000000 --- a/newsdiff/core/migrations/0002_auto__add_newsdiffuser.py +++ /dev/null @@ -1,112 +0,0 @@ -# -*- coding: utf-8 -*- -from south.utils import datetime_utils as datetime -from south.db import db -from south.v2 import SchemaMigration -from django.db import models - - -class Migration(SchemaMigration): - - def forwards(self, orm): - # Adding model 'NewsDiffUser' - db.create_table(u'core_newsdiffuser', ( - (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), - ('password', self.gf('django.db.models.fields.CharField')(max_length=128)), - ('last_login', self.gf('django.db.models.fields.DateTimeField')(default=datetime.datetime.now)), - ('is_superuser', self.gf('django.db.models.fields.BooleanField')(default=False)), - ('username', self.gf('django.db.models.fields.CharField')(unique=True, max_length=30)), - ('first_name', self.gf('django.db.models.fields.CharField')(max_length=30, blank=True)), - ('last_name', self.gf('django.db.models.fields.CharField')(max_length=30, blank=True)), - ('email', self.gf('django.db.models.fields.EmailField')(max_length=75, blank=True)), - ('is_staff', self.gf('django.db.models.fields.BooleanField')(default=False)), - ('is_active', self.gf('django.db.models.fields.BooleanField')(default=True)), - ('date_joined', self.gf('django.db.models.fields.DateTimeField')(default=datetime.datetime.now)), - )) - db.send_create_signal(u'core', ['NewsDiffUser']) - - # Adding M2M table for field groups on 'NewsDiffUser' - m2m_table_name = db.shorten_name(u'core_newsdiffuser_groups') - db.create_table(m2m_table_name, ( - ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)), - ('newsdiffuser', models.ForeignKey(orm[u'core.newsdiffuser'], null=False)), - ('group', models.ForeignKey(orm[u'auth.group'], null=False)) - )) - db.create_unique(m2m_table_name, ['newsdiffuser_id', 'group_id']) - - # Adding M2M table for field user_permissions on 'NewsDiffUser' - m2m_table_name = db.shorten_name(u'core_newsdiffuser_user_permissions') - db.create_table(m2m_table_name, ( - ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)), - ('newsdiffuser', models.ForeignKey(orm[u'core.newsdiffuser'], null=False)), - ('permission', models.ForeignKey(orm[u'auth.permission'], null=False)) - )) - db.create_unique(m2m_table_name, ['newsdiffuser_id', 'permission_id']) - - - def backwards(self, orm): - # Deleting model 'NewsDiffUser' - db.delete_table(u'core_newsdiffuser') - - # Removing M2M table for field groups on 'NewsDiffUser' - db.delete_table(db.shorten_name(u'core_newsdiffuser_groups')) - - # Removing M2M table for field user_permissions on 'NewsDiffUser' - db.delete_table(db.shorten_name(u'core_newsdiffuser_user_permissions')) - - - models = { - u'auth.group': { - 'Meta': {'object_name': 'Group'}, - u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), - 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) - }, - u'auth.permission': { - 'Meta': {'ordering': "(u'content_type__app_label', u'content_type__model', u'codename')", 'unique_together': "((u'content_type', u'codename'),)", 'object_name': 'Permission'}, - 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), - 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}), - u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) - }, - u'contenttypes.contenttype': { - 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, - 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), - u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), - 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) - }, - u'core.haaretzarticle': { - 'Meta': {'object_name': 'HaaretzArticle'}, - 'date': ('django.db.models.fields.DateTimeField', [], {}), - u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'subtitle': ('django.db.models.fields.CharField', [], {'max_length': '300'}), - 'text': ('django.db.models.fields.TextField', [], {}), - 'title': ('django.db.models.fields.CharField', [], {'max_length': '140'}), - 'url': ('django.db.models.fields.CharField', [], {'max_length': '200'}) - }, - u'core.haaretzimage': { - 'Meta': {'object_name': 'HaaretzImage'}, - 'article': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['core.HaaretzArticle']"}), - u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'image': ('django.db.models.fields.files.ImageField', [], {'max_length': '100'}), - 'origin_url': ('django.db.models.fields.CharField', [], {'max_length': '200'}) - }, - u'core.newsdiffuser': { - 'Meta': {'object_name': 'NewsDiffUser'}, - 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), - 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), - 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), - 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Group']"}), - u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), - 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), - 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), - 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), - 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), - 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), - 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Permission']"}), - 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) - } - } - - complete_apps = ['core'] \ No newline at end of file diff --git a/newsdiff/core/models.py b/newsdiff/core/models.py index 546b1cd..3db2314 100644 --- a/newsdiff/core/models.py +++ b/newsdiff/core/models.py @@ -10,7 +10,7 @@ class NewsDiffUser(AbstractUser): class HaaretzArticle(models.Model): url = models.CharField(max_length=200) title = models.CharField(max_length=140) - subtitle = models.CharField(max_length=300) + subtitle = models.CharField(max_length=300, blank=True, null=True) text = models.TextField() date = models.DateTimeField() @@ -19,3 +19,4 @@ class HaaretzImage(models.Model): article = models.ForeignKey(HaaretzArticle) origin_url = models.CharField(max_length=200) image = models.ImageField(upload_to='images/haaretz') + caption = models.CharField(max_length=140, blank=True, null=True) diff --git a/newsdiff/core/tasks.py b/newsdiff/core/tasks.py new file mode 100644 index 0000000..1a3cf1c --- /dev/null +++ b/newsdiff/core/tasks.py @@ -0,0 +1,41 @@ +import requests + +from bs4 import BeautifulSoup +from datetime import datetime +from newsdiff.celery import app +from pytz import timezone + +from .models import HaaretzArticle, HaaretzImage +from .utils import get_image_from_url + +ISRAEL_TIMEZONE = timezone('Asia/Jerusalem') + +@app.task() +def get_haaretz_article(url): + req = requests.get(url, headers={ + 'User-Agent': 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' + }) + if req.ok: + soup = BeautifulSoup(req.text, 'lxml') + title = soup.find('h1', class_='mainTitle').text.strip() + subtitle = soup.find('h2', class_='subtitle').text.strip() + author_bar = soup.find('ul', class_='author-bar') + date = author_bar.find_all('li')[1].text.strip() + time = author_bar.find_all('li')[2].text.strip() + article_date = ISRAEL_TIMEZONE.localize(datetime.strptime(' '.join([date, time]), '%d.%m.%Y %H:%M')) + article_body = soup.find('div', id='article-box').find_all('p') + article_text = '\n\n'.join([p.text.strip() for p in article_body]) + + article = HaaretzArticle(url=url, title=title, subtitle=subtitle, text=article_text, date=article_date) + article.save() + + images = soup.find('div', id='article-box').find_all('div', class_='inArticleHoldImage') + for image in images: + img = image.find('img') + img_url = 'http://www.haaretz.co.il{}'.format(img['src'].split('_gen')[0]) + caption = img.title + + name, image_file = get_image_from_url(img_url) + article_image = HaaretzImage(article=article, origin_url=url, caption=caption) + article_image.image.save(name, image_file) + article_image.save() diff --git a/newsdiff/core/utils.py b/newsdiff/core/utils.py new file mode 100644 index 0000000..2a5f4fc --- /dev/null +++ b/newsdiff/core/utils.py @@ -0,0 +1,13 @@ +import urllib2 + +from django.core.files import File +from django.core.files.temp import NamedTemporaryFile +from urlparse import urlparse + +def get_image_from_url(url): + tmp = NamedTemporaryFile(delete=True) + u = urllib2.urlopen(url) + tmp.write(u.read()) + tmp.flush() + name = urlparse(u.geturl()).path.split('/')[-1] + return name, File(tmp) diff --git a/newsdiff/settings/base.py b/newsdiff/settings/base.py index c0e3e39..75ab332 100644 --- a/newsdiff/settings/base.py +++ b/newsdiff/settings/base.py @@ -82,6 +82,7 @@ INSTALLED_APPS = ( 'south', 'storages', + 'celery', 'newsdiff.core' ) diff --git a/requirements.txt b/requirements.txt index ec9b7a3..6133a59 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,12 +4,15 @@ South==0.8.4 Unipath==1.0 amqp==1.3.3 anyjson==0.3.3 +beautifulsoup4==4.3.2 billiard==3.3.0.11 celery==3.1.6 django-reversion==1.8.0 django-storages==1.1.8 honcho==0.5.0 kombu==3.0.7 +lxml==3.2.4 psycopg2==2.5.1 pytz==2013.8 +requests==2.1.0 wsgiref==0.1.2 -- cgit v1.3.1