DB cleanup, working Haaretz tasks

11 files changed, 162 insertions, 115 deletions

diff --git a/.gitignore b/.gitignore
index 0d20b64..641b4c4 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1 +1,3 @@
 *.pyc
+
+newsdiff/media/
diff --git a/newsdiff/__init__.py b/newsdiff/__init__.py
index e69de29..1afb35e 100644
--- a/newsdiff/__init__.py
+++ b/newsdiff/__init__.py
@@ -0,0 +1,3 @@
+from __future__ import absolute_import
+
+from .celery import app
diff --git a/newsdiff/celery.py b/newsdiff/celery.py
new file mode 100644
index 0000000..c1e2111
--- /dev/null
+++ b/newsdiff/celery.py
@@ -0,0 +1,12 @@
+from __future__ import absolute_import
+
+import os
+
+from celery import Celery
+from django.conf import settings
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'segmanta.settings')
+
+app = Celery('newsdiff')
+app.config_from_object('django.conf:settings')
+app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
diff --git a/newsdiff/core/admin.py b/newsdiff/core/admin.py
index 481afd5..ab4b576 100644
--- a/newsdiff/core/admin.py
+++ b/newsdiff/core/admin.py
@@ -2,3 +2,5 @@ from django.contrib import admin
 from .models import *
 
 admin.site.register(NewsDiffUser)
+admin.site.register(HaaretzArticle)
+admin.site.register(HaaretzImage)
diff --git a/newsdiff/core/migrations/0001_initial.py b/newsdiff/core/migrations/0001_initial.py
index b32e267..a42a829 100644
--- a/newsdiff/core/migrations/0001_initial.py
+++ b/newsdiff/core/migrations/0001_initial.py
@@ -8,12 +8,46 @@ from django.db import models
 class Migration(SchemaMigration):
 
     def forwards(self, orm):
+        # Adding model 'NewsDiffUser'
+        db.create_table(u'core_newsdiffuser', (
+            (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('password', self.gf('django.db.models.fields.CharField')(max_length=128)),
+            ('last_login', self.gf('django.db.models.fields.DateTimeField')(default=datetime.datetime.now)),
+            ('is_superuser', self.gf('django.db.models.fields.BooleanField')(default=False)),
+            ('username', self.gf('django.db.models.fields.CharField')(unique=True, max_length=30)),
+            ('first_name', self.gf('django.db.models.fields.CharField')(max_length=30, blank=True)),
+            ('last_name', self.gf('django.db.models.fields.CharField')(max_length=30, blank=True)),
+            ('email', self.gf('django.db.models.fields.EmailField')(max_length=75, blank=True)),
+            ('is_staff', self.gf('django.db.models.fields.BooleanField')(default=False)),
+            ('is_active', self.gf('django.db.models.fields.BooleanField')(default=True)),
+            ('date_joined', self.gf('django.db.models.fields.DateTimeField')(default=datetime.datetime.now)),
+        ))
+        db.send_create_signal(u'core', ['NewsDiffUser'])
+
+        # Adding M2M table for field groups on 'NewsDiffUser'
+        m2m_table_name = db.shorten_name(u'core_newsdiffuser_groups')
+        db.create_table(m2m_table_name, (
+            ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)),
+            ('newsdiffuser', models.ForeignKey(orm[u'core.newsdiffuser'], null=False)),
+            ('group', models.ForeignKey(orm[u'auth.group'], null=False))
+        ))
+        db.create_unique(m2m_table_name, ['newsdiffuser_id', 'group_id'])
+
+        # Adding M2M table for field user_permissions on 'NewsDiffUser'
+        m2m_table_name = db.shorten_name(u'core_newsdiffuser_user_permissions')
+        db.create_table(m2m_table_name, (
+            ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)),
+            ('newsdiffuser', models.ForeignKey(orm[u'core.newsdiffuser'], null=False)),
+            ('permission', models.ForeignKey(orm[u'auth.permission'], null=False))
+        ))
+        db.create_unique(m2m_table_name, ['newsdiffuser_id', 'permission_id'])
+
         # Adding model 'HaaretzArticle'
         db.create_table(u'core_haaretzarticle', (
             (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
             ('url', self.gf('django.db.models.fields.CharField')(max_length=200)),
             ('title', self.gf('django.db.models.fields.CharField')(max_length=140)),
-            ('subtitle', self.gf('django.db.models.fields.CharField')(max_length=300)),
+            ('subtitle', self.gf('django.db.models.fields.CharField')(max_length=300, null=True, blank=True)),
             ('text', self.gf('django.db.models.fields.TextField')()),
             ('date', self.gf('django.db.models.fields.DateTimeField')()),
         ))
@@ -25,11 +59,21 @@ class Migration(SchemaMigration):
             ('article', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['core.HaaretzArticle'])),
             ('origin_url', self.gf('django.db.models.fields.CharField')(max_length=200)),
             ('image', self.gf('django.db.models.fields.files.ImageField')(max_length=100)),
+            ('caption', self.gf('django.db.models.fields.CharField')(max_length=140, null=True, blank=True)),
         ))
         db.send_create_signal(u'core', ['HaaretzImage'])
 
 
     def backwards(self, orm):
+        # Deleting model 'NewsDiffUser'
+        db.delete_table(u'core_newsdiffuser')
+
+        # Removing M2M table for field groups on 'NewsDiffUser'
+        db.delete_table(db.shorten_name(u'core_newsdiffuser_groups'))
+
+        # Removing M2M table for field user_permissions on 'NewsDiffUser'
+        db.delete_table(db.shorten_name(u'core_newsdiffuser_user_permissions'))
+
         # Deleting model 'HaaretzArticle'
         db.delete_table(u'core_haaretzarticle')
 
@@ -38,11 +82,31 @@ class Migration(SchemaMigration):
 
 
     models = {
+        u'auth.group': {
+            'Meta': {'object_name': 'Group'},
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+        },
+        u'auth.permission': {
+            'Meta': {'ordering': "(u'content_type__app_label', u'content_type__model', u'codename')", 'unique_together': "((u'content_type', u'codename'),)", 'object_name': 'Permission'},
+            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+        },
+        u'contenttypes.contenttype': {
+            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        },
         u'core.haaretzarticle': {
             'Meta': {'object_name': 'HaaretzArticle'},
             'date': ('django.db.models.fields.DateTimeField', [], {}),
             u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
-            'subtitle': ('django.db.models.fields.CharField', [], {'max_length': '300'}),
+            'subtitle': ('django.db.models.fields.CharField', [], {'max_length': '300', 'null': 'True', 'blank': 'True'}),
             'text': ('django.db.models.fields.TextField', [], {}),
             'title': ('django.db.models.fields.CharField', [], {'max_length': '140'}),
             'url': ('django.db.models.fields.CharField', [], {'max_length': '200'})
@@ -50,9 +114,26 @@ class Migration(SchemaMigration):
         u'core.haaretzimage': {
             'Meta': {'object_name': 'HaaretzImage'},
             'article': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['core.HaaretzArticle']"}),
+            'caption': ('django.db.models.fields.CharField', [], {'max_length': '140', 'null': 'True', 'blank': 'True'}),
             u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
             'image': ('django.db.models.fields.files.ImageField', [], {'max_length': '100'}),
             'origin_url': ('django.db.models.fields.CharField', [], {'max_length': '200'})
+        },
+        u'core.newsdiffuser': {
+            'Meta': {'object_name': 'NewsDiffUser'},
+            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
+            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Group']"}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Permission']"}),
+            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
         }
     }
 
diff --git a/newsdiff/core/migrations/0002_auto__add_newsdiffuser.py b/newsdiff/core/migrations/0002_auto__add_newsdiffuser.py
deleted file mode 100644
index 0f9bfc3..0000000
--- a/newsdiff/core/migrations/0002_auto__add_newsdiffuser.py
+++ /dev/null
@@ -1,112 +0,0 @@
-# -*- coding: utf-8 -*-
-from south.utils import datetime_utils as datetime
-from south.db import db
-from south.v2 import SchemaMigration
-from django.db import models
-
-
-class Migration(SchemaMigration):
-
-    def forwards(self, orm):
-        # Adding model 'NewsDiffUser'
-        db.create_table(u'core_newsdiffuser', (
-            (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
-            ('password', self.gf('django.db.models.fields.CharField')(max_length=128)),
-            ('last_login', self.gf('django.db.models.fields.DateTimeField')(default=datetime.datetime.now)),
-            ('is_superuser', self.gf('django.db.models.fields.BooleanField')(default=False)),
-            ('username', self.gf('django.db.models.fields.CharField')(unique=True, max_length=30)),
-            ('first_name', self.gf('django.db.models.fields.CharField')(max_length=30, blank=True)),
-            ('last_name', self.gf('django.db.models.fields.CharField')(max_length=30, blank=True)),
-            ('email', self.gf('django.db.models.fields.EmailField')(max_length=75, blank=True)),
-            ('is_staff', self.gf('django.db.models.fields.BooleanField')(default=False)),
-            ('is_active', self.gf('django.db.models.fields.BooleanField')(default=True)),
-            ('date_joined', self.gf('django.db.models.fields.DateTimeField')(default=datetime.datetime.now)),
-        ))
-        db.send_create_signal(u'core', ['NewsDiffUser'])
-
-        # Adding M2M table for field groups on 'NewsDiffUser'
-        m2m_table_name = db.shorten_name(u'core_newsdiffuser_groups')
-        db.create_table(m2m_table_name, (
-            ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)),
-            ('newsdiffuser', models.ForeignKey(orm[u'core.newsdiffuser'], null=False)),
-            ('group', models.ForeignKey(orm[u'auth.group'], null=False))
-        ))
-        db.create_unique(m2m_table_name, ['newsdiffuser_id', 'group_id'])
-
-        # Adding M2M table for field user_permissions on 'NewsDiffUser'
-        m2m_table_name = db.shorten_name(u'core_newsdiffuser_user_permissions')
-        db.create_table(m2m_table_name, (
-            ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)),
-            ('newsdiffuser', models.ForeignKey(orm[u'core.newsdiffuser'], null=False)),
-            ('permission', models.ForeignKey(orm[u'auth.permission'], null=False))
-        ))
-        db.create_unique(m2m_table_name, ['newsdiffuser_id', 'permission_id'])
-
-
-    def backwards(self, orm):
-        # Deleting model 'NewsDiffUser'
-        db.delete_table(u'core_newsdiffuser')
-
-        # Removing M2M table for field groups on 'NewsDiffUser'
-        db.delete_table(db.shorten_name(u'core_newsdiffuser_groups'))
-
-        # Removing M2M table for field user_permissions on 'NewsDiffUser'
-        db.delete_table(db.shorten_name(u'core_newsdiffuser_user_permissions'))
-
-
-    models = {
-        u'auth.group': {
-            'Meta': {'object_name': 'Group'},
-            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
-            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
-            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
-        },
-        u'auth.permission': {
-            'Meta': {'ordering': "(u'content_type__app_label', u'content_type__model', u'codename')", 'unique_together': "((u'content_type', u'codename'),)", 'object_name': 'Permission'},
-            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
-            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}),
-            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
-            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
-        },
-        u'contenttypes.contenttype': {
-            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
-            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
-            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
-            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
-            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
-        },
-        u'core.haaretzarticle': {
-            'Meta': {'object_name': 'HaaretzArticle'},
-            'date': ('django.db.models.fields.DateTimeField', [], {}),
-            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
-            'subtitle': ('django.db.models.fields.CharField', [], {'max_length': '300'}),
-            'text': ('django.db.models.fields.TextField', [], {}),
-            'title': ('django.db.models.fields.CharField', [], {'max_length': '140'}),
-            'url': ('django.db.models.fields.CharField', [], {'max_length': '200'})
-        },
-        u'core.haaretzimage': {
-            'Meta': {'object_name': 'HaaretzImage'},
-            'article': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['core.HaaretzArticle']"}),
-            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
-            'image': ('django.db.models.fields.files.ImageField', [], {'max_length': '100'}),
-            'origin_url': ('django.db.models.fields.CharField', [], {'max_length': '200'})
-        },
-        u'core.newsdiffuser': {
-            'Meta': {'object_name': 'NewsDiffUser'},
-            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
-            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
-            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
-            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Group']"}),
-            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
-            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
-            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
-            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
-            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
-            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
-            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
-            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Permission']"}),
-            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
-        }
-    }
-
-    complete_apps = ['core']
\ No newline at end of file
diff --git a/newsdiff/core/models.py b/newsdiff/core/models.py
index 546b1cd..3db2314 100644
--- a/newsdiff/core/models.py
+++ b/newsdiff/core/models.py
@@ -10,7 +10,7 @@ class NewsDiffUser(AbstractUser):
 class HaaretzArticle(models.Model):
     url = models.CharField(max_length=200)
     title = models.CharField(max_length=140)
-    subtitle = models.CharField(max_length=300)
+    subtitle = models.CharField(max_length=300, blank=True, null=True)
     text = models.TextField()
     date = models.DateTimeField()
 
@@ -19,3 +19,4 @@ class HaaretzImage(models.Model):
     article = models.ForeignKey(HaaretzArticle)
     origin_url = models.CharField(max_length=200)
     image = models.ImageField(upload_to='images/haaretz')
+    caption = models.CharField(max_length=140, blank=True, null=True)
diff --git a/newsdiff/core/tasks.py b/newsdiff/core/tasks.py
new file mode 100644
index 0000000..1a3cf1c
--- /dev/null
+++ b/newsdiff/core/tasks.py
@@ -0,0 +1,41 @@
+import requests
+
+from bs4 import BeautifulSoup
+from datetime import datetime
+from newsdiff.celery import app
+from pytz import timezone
+
+from .models import HaaretzArticle, HaaretzImage
+from .utils import get_image_from_url
+
+ISRAEL_TIMEZONE = timezone('Asia/Jerusalem')
+
+@app.task()
+def get_haaretz_article(url):
+    req = requests.get(url, headers={
+        'User-Agent': 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)'
+    })
+    if req.ok:
+        soup = BeautifulSoup(req.text, 'lxml')
+        title = soup.find('h1', class_='mainTitle').text.strip()
+        subtitle = soup.find('h2', class_='subtitle').text.strip()
+        author_bar = soup.find('ul', class_='author-bar')
+        date = author_bar.find_all('li')[1].text.strip()
+        time = author_bar.find_all('li')[2].text.strip()
+        article_date = ISRAEL_TIMEZONE.localize(datetime.strptime(' '.join([date, time]), '%d.%m.%Y %H:%M'))
+        article_body = soup.find('div', id='article-box').find_all('p')
+        article_text = '\n\n'.join([p.text.strip() for p in article_body])
+
+        article = HaaretzArticle(url=url, title=title, subtitle=subtitle, text=article_text, date=article_date)
+        article.save()
+
+        images = soup.find('div', id='article-box').find_all('div', class_='inArticleHoldImage')
+        for image in images:
+            img = image.find('img')
+            img_url = 'http://www.haaretz.co.il{}'.format(img['src'].split('_gen')[0])
+            caption = img.title
+
+            name, image_file = get_image_from_url(img_url)
+            article_image = HaaretzImage(article=article, origin_url=url, caption=caption)
+            article_image.image.save(name, image_file)
+            article_image.save()
diff --git a/newsdiff/core/utils.py b/newsdiff/core/utils.py
new file mode 100644
index 0000000..2a5f4fc
--- /dev/null
+++ b/newsdiff/core/utils.py
@@ -0,0 +1,13 @@
+import urllib2
+
+from django.core.files import File
+from django.core.files.temp import NamedTemporaryFile
+from urlparse import urlparse
+
+def get_image_from_url(url):
+    tmp = NamedTemporaryFile(delete=True)
+    u = urllib2.urlopen(url)
+    tmp.write(u.read())
+    tmp.flush()
+    name = urlparse(u.geturl()).path.split('/')[-1]
+    return name, File(tmp)
diff --git a/newsdiff/settings/base.py b/newsdiff/settings/base.py
index c0e3e39..75ab332 100644
--- a/newsdiff/settings/base.py
+++ b/newsdiff/settings/base.py
@@ -82,6 +82,7 @@ INSTALLED_APPS = (
 
     'south',
     'storages',
+    'celery',
 
     'newsdiff.core'
 )
diff --git a/requirements.txt b/requirements.txt
index ec9b7a3..6133a59 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -4,12 +4,15 @@ South==0.8.4
 Unipath==1.0
 amqp==1.3.3
 anyjson==0.3.3
+beautifulsoup4==4.3.2
 billiard==3.3.0.11
 celery==3.1.6
 django-reversion==1.8.0
 django-storages==1.1.8
 honcho==0.5.0
 kombu==3.0.7
+lxml==3.2.4
 psycopg2==2.5.1
 pytz==2013.8
+requests==2.1.0
 wsgiref==0.1.2