Merge branch 'master' of github.com:yuvadm/newsdiff

23 files changed, 204 insertions, 10 deletions

diff --git a/.gitignore b/.gitignore
index 641b4c4..0c722fd 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,6 @@
 *.pyc
+*.rdb
+
+celerybeat-schedule
 
 newsdiff/media/
diff --git a/Procfile.dev b/Procfile.dev
index 848adfd..30c1b97 100644
--- a/Procfile.dev
+++ b/Procfile.dev
@@ -1 +1,3 @@
 web: python manage.py runserver
+#worker: celery -A newsdiff worker -B --loglevel=info
+#redis: redis-server
diff --git a/newsdiff/celery.py b/newsdiff/celery.py
index c1e2111..cfb4ab7 100644
--- a/newsdiff/celery.py
+++ b/newsdiff/celery.py
@@ -5,7 +5,7 @@ import os
 from celery import Celery
 from django.conf import settings
 
-os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'segmanta.settings')
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'newsdiff.settings')
 
 app = Celery('newsdiff')
 app.config_from_object('django.conf:settings')
diff --git a/newsdiff/core/parsers/haaretz.py b/newsdiff/core/parsers/haaretz.py
index 360cf25..58f7dd5 100644
--- a/newsdiff/core/parsers/haaretz.py
+++ b/newsdiff/core/parsers/haaretz.py
@@ -6,7 +6,7 @@ from datetime import datetime
 
 from .base import HtmlSoupParser
 from ..models import HaaretzArticle, HaaretzImage
-from ..utils import get_image_from_url
+from ..utils import get_file_from_url
 
 
 class HaaretzParser(HtmlSoupParser):
@@ -60,7 +60,7 @@ class HaaretzParser(HtmlSoupParser):
                 img_url = 'http://www.haaretz.co.il{}'.format(img['src'].split('_gen')[0])
                 caption = img.title
 
-                name, image_file = get_image_from_url(img_url)
+                name, image_file = get_file_from_url(img_url)
 
                 article_image, created = self.IMAGE_MODEL.objects.get_or_create(article=article,
                     origin_url=img_url, defaults={'caption': caption})
diff --git a/newsdiff/core/utils.py b/newsdiff/core/utils.py
index 2a5f4fc..d104a81 100644
--- a/newsdiff/core/utils.py
+++ b/newsdiff/core/utils.py
@@ -1,13 +1,18 @@
-import urllib2
+import logging
+import requests
 
 from django.core.files import File
 from django.core.files.temp import NamedTemporaryFile
 from urlparse import urlparse
 
-def get_image_from_url(url):
+
+def get_file_from_url(url):
     tmp = NamedTemporaryFile(delete=True)
-    u = urllib2.urlopen(url)
-    tmp.write(u.read())
+    req = requests.get(url)
+    if not req.ok:
+        logging.error('Failed to import image from {}'.format(url))
+        return None
+    tmp.write(req.content)
     tmp.flush()
-    name = urlparse(u.geturl()).path.split('/')[-1]
+    name = urlparse(req.url).path.split('/')[-1]
     return name, File(tmp)
diff --git a/newsdiff/docs/__init__.py b/newsdiff/docs/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/newsdiff/docs/__init__.py
diff --git a/newsdiff/docs/admin.py b/newsdiff/docs/admin.py
new file mode 100644
index 0000000..115e8a0
--- /dev/null
+++ b/newsdiff/docs/admin.py
@@ -0,0 +1,5 @@
+from django.contrib import admin
+
+from .models import *
+
+admin.site.register(Document)
diff --git a/newsdiff/docs/migrations/0001_initial.py b/newsdiff/docs/migrations/0001_initial.py
new file mode 100644
index 0000000..3fa5fef
--- /dev/null
+++ b/newsdiff/docs/migrations/0001_initial.py
@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+from south.utils import datetime_utils as datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding model 'Document'
+        db.create_table(u'docs_document', (
+            (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('name', self.gf('django.db.models.fields.CharField')(default='(untitled)', max_length=100)),
+            ('url', self.gf('django.db.models.fields.URLField')(max_length=200)),
+            ('created', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, blank=True)),
+            ('document', self.gf('django.db.models.fields.files.FileField')(max_length=100)),
+        ))
+        db.send_create_signal(u'docs', ['Document'])
+
+
+    def backwards(self, orm):
+        # Deleting model 'Document'
+        db.delete_table(u'docs_document')
+
+
+    models = {
+        u'docs.document': {
+            'Meta': {'object_name': 'Document'},
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'document': ('django.db.models.fields.files.FileField', [], {'max_length': '100'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'default': "'(untitled)'", 'max_length': '100'}),
+            'url': ('django.db.models.fields.URLField', [], {'max_length': '200'})
+        }
+    }
+
+    complete_apps = ['docs']
\ No newline at end of file
diff --git a/newsdiff/docs/migrations/__init__.py b/newsdiff/docs/migrations/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/newsdiff/docs/migrations/__init__.py
diff --git a/newsdiff/docs/models.py b/newsdiff/docs/models.py
new file mode 100644
index 0000000..717eb3d
--- /dev/null
+++ b/newsdiff/docs/models.py
@@ -0,0 +1,11 @@
+from django.db import models
+
+
+class Document(models.Model):
+    name = models.CharField(max_length=100, default='(untitled)')
+    url = models.URLField()
+    created = models.DateTimeField(auto_now_add=True, editable=False)
+    document = models.FileField(upload_to='documents')
+
+    def __unicode__(self):
+        return '{} ({})'.format(self.name, self.url)
diff --git a/newsdiff/docs/tasks.py b/newsdiff/docs/tasks.py
new file mode 100644
index 0000000..c40e9ff
--- /dev/null
+++ b/newsdiff/docs/tasks.py
@@ -0,0 +1,14 @@
+import logging
+
+from newsdiff.celery import app
+from newsdiff.core.utils import get_file_from_url
+
+from .models import Document
+
+
+@app.task()
+def import_document(url):
+    name, doc = get_file_from_url(url)
+    document = Document(url=url)
+    document.document.save(name, doc)
+    document.save()
diff --git a/newsdiff/docs/tests.py b/newsdiff/docs/tests.py
new file mode 100644
index 0000000..dee311c
--- /dev/null
+++ b/newsdiff/docs/tests.py
@@ -0,0 +1,15 @@
+from django.test import TestCase
+
+from .models import Document
+from .tasks import import_document
+
+
+class DocumentTestCase(TestCase):
+    def setUp(self):
+        pass
+
+    def test_import_document(self):
+        import_document('http://www.google.com/webmasters/docs/search-engine-optimization-starter-guide.pdf')
+        document = Document.objects.all()[0]
+        self.assertTrue(document.created)
+        self.assertTrue(document.document.url)
diff --git a/newsdiff/docs/views.py b/newsdiff/docs/views.py
new file mode 100644
index 0000000..91ea44a
--- /dev/null
+++ b/newsdiff/docs/views.py
@@ -0,0 +1,3 @@
+from django.shortcuts import render
+
+# Create your views here.
diff --git a/newsdiff/rain/__init__.py b/newsdiff/rain/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/newsdiff/rain/__init__.py
diff --git a/newsdiff/rain/admin.py b/newsdiff/rain/admin.py
new file mode 100644
index 0000000..1421afd
--- /dev/null
+++ b/newsdiff/rain/admin.py
@@ -0,0 +1,4 @@
+from django.contrib import admin
+from .models import *
+
+admin.site.register(RainRadarImage)
diff --git a/newsdiff/rain/migrations/0001_initial.py b/newsdiff/rain/migrations/0001_initial.py
new file mode 100644
index 0000000..e60bdb0
--- /dev/null
+++ b/newsdiff/rain/migrations/0001_initial.py
@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+from south.utils import datetime_utils as datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding model 'RainRadarImage'
+        db.create_table(u'rain_rainradarimage', (
+            (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('created', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, blank=True)),
+            ('image', self.gf('django.db.models.fields.files.ImageField')(max_length=100)),
+        ))
+        db.send_create_signal(u'rain', ['RainRadarImage'])
+
+
+    def backwards(self, orm):
+        # Deleting model 'RainRadarImage'
+        db.delete_table(u'rain_rainradarimage')
+
+
+    models = {
+        u'rain.rainradarimage': {
+            'Meta': {'object_name': 'RainRadarImage'},
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'image': ('django.db.models.fields.files.ImageField', [], {'max_length': '100'})
+        }
+    }
+
+    complete_apps = ['rain']
\ No newline at end of file
diff --git a/newsdiff/rain/migrations/__init__.py b/newsdiff/rain/migrations/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/newsdiff/rain/migrations/__init__.py
diff --git a/newsdiff/rain/models.py b/newsdiff/rain/models.py
new file mode 100644
index 0000000..1ff8915
--- /dev/null
+++ b/newsdiff/rain/models.py
@@ -0,0 +1,9 @@
+from django.db import models
+
+
+class RainRadarImage(models.Model):
+    created = models.DateTimeField(auto_now_add=True, editable=False)
+    image = models.ImageField(upload_to='rain/%Y/%m/%d')
+
+    def __unicode__(self):
+        return self.created.strftime('%Y/%m/%d %H:%M')
diff --git a/newsdiff/rain/tasks.py b/newsdiff/rain/tasks.py
new file mode 100644
index 0000000..3ae0eb2
--- /dev/null
+++ b/newsdiff/rain/tasks.py
@@ -0,0 +1,19 @@
+import logging
+
+from datetime import datetime
+from newsdiff.celery import app
+from newsdiff.core.utils import get_file_from_url
+
+from .models import RainRadarImage
+
+RAIN_RADAR_IMAGE_URL = 'http://www.ims.gov.il/Ims/Pages/RadarImage.aspx'
+
+
+@app.task()
+def import_rain_radar_image():
+    logging.info('run')
+    _name, image = get_file_from_url(RAIN_RADAR_IMAGE_URL)
+    name = datetime.now().strftime('%H%M.gif')
+    image_obj = RainRadarImage()
+    image_obj.image.save(name, image)
+    image_obj.save()
diff --git a/newsdiff/rain/tests.py b/newsdiff/rain/tests.py
new file mode 100644
index 0000000..3ff7dfb
--- /dev/null
+++ b/newsdiff/rain/tests.py
@@ -0,0 +1,15 @@
+from django.test import TestCase
+
+from .models import RainRadarImage
+from .tasks import import_rain_radar_image
+
+
+class RainRadarTestCase(TestCase):
+    def setUp(self):
+        pass
+
+    def test_import_rain_radar_image(self):
+        import_rain_radar_image()
+        image = RainRadarImage.objects.all()[0]
+        self.assertTrue(image.created)
+        self.assertTrue(image.image.url)
diff --git a/newsdiff/rain/views.py b/newsdiff/rain/views.py
new file mode 100644
index 0000000..91ea44a
--- /dev/null
+++ b/newsdiff/rain/views.py
@@ -0,0 +1,3 @@
+from django.shortcuts import render
+
+# Create your views here.
diff --git a/newsdiff/settings/base.py b/newsdiff/settings/base.py
index 234cf69..3fb6569 100644
--- a/newsdiff/settings/base.py
+++ b/newsdiff/settings/base.py
@@ -1,5 +1,7 @@
+from datetime import timedelta
 from unipath import FSPath as Path
 
+
 PROJECT_DIR = Path(__file__).absolute().ancestor(2)
 
 DEBUG = False
@@ -13,7 +15,7 @@ MANAGERS = ADMINS
 
 ALLOWED_HOSTS = []
 
-TIME_ZONE = 'Etc/UTC'
+TIME_ZONE = 'Asia/Jerusalem'
 LANGUAGE_CODE = 'en-us'
 SITE_ID = 1
 USE_I18N = True
@@ -86,7 +88,9 @@ INSTALLED_APPS = (
     'celery',
     'reversion',
 
-    'newsdiff.core'
+    'newsdiff.core',
+    'newsdiff.docs',
+    'newsdiff.rain'
 )
 
 AUTH_USER_MODEL = 'core.NewsDiffUser'
@@ -120,3 +124,11 @@ LOGGING = {
 }
 
 TEST_RUNNER = 'django_nose.NoseTestSuiteRunner'
+
+CELERY_ACCEPT_CONTENT = ['json']
+CELERYBEAT_SCHEDULE = {
+    'rain': {
+        'task': 'newsdiff.rain.tasks.import_rain_radar_image',
+        'schedule': timedelta(minutes=10)
+    },
+}
diff --git a/newsdiff/settings/local.py b/newsdiff/settings/local.py
index 95b64c1..cf76879 100644
--- a/newsdiff/settings/local.py
+++ b/newsdiff/settings/local.py
@@ -16,9 +16,11 @@ DATABASES = {
     }
 }
 
+
 MEDIA_ROOT = PROJECT_DIR.child('media')
 MEDIA_URL = '/media/'
 
 EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'
 
 CELERY_ALWAYS_EAGER = True  # Makes all celery tasks run synchronously
+BROKER_URL = 'redis://localhost:6379/0'