Initial git fork of django-storagesHEADmaster

1 files changed, 501 insertions, 0 deletions

diff --git a/storages/backends/s3boto.py b/storages/backends/s3boto.py
new file mode 100644
index 0000000..354cd0c
--- /dev/null
+++ b/storages/backends/s3boto.py
@@ -0,0 +1,501 @@
+import os
+import posixpath
+import mimetypes
+from gzip import GzipFile
+import datetime
+from tempfile import SpooledTemporaryFile
+
+try:
+    from cStringIO import StringIO
+except ImportError:
+    from StringIO import StringIO  # noqa
+
+from django.core.files.base import File
+from django.core.files.storage import Storage
+from django.core.exceptions import ImproperlyConfigured, SuspiciousOperation
+from django.utils.encoding import force_unicode, smart_str, filepath_to_uri
+
+try:
+    from boto import __version__ as boto_version
+    from boto.s3.connection import S3Connection, SubdomainCallingFormat
+    from boto.exception import S3ResponseError
+    from boto.s3.key import Key as S3Key
+    from boto.utils import parse_ts
+except ImportError:
+    raise ImproperlyConfigured("Could not load Boto's S3 bindings.\n"
+                               "See https://github.com/boto/boto")
+
+from storages.utils import setting
+
+boto_version_info = tuple([int(i) for i in boto_version.split('-')[0].split('.')])
+
+if boto_version_info[:2] < (2, 4):
+    raise ImproperlyConfigured("The installed Boto library must be 2.4 or "
+                               "higher.\nSee https://github.com/boto/boto")
+
+
+def parse_ts_extended(ts):
+    RFC1123 = '%a, %d %b %Y %H:%M:%S %Z'
+    rv = None
+    try:
+        rv = parse_ts(ts)
+    except ValueError:
+        rv = datetime.datetime.strptime(ts, RFC1123)
+    return rv
+
+
+def safe_join(base, *paths):
+    """
+    A version of django.utils._os.safe_join for S3 paths.
+
+    Joins one or more path components to the base path component
+    intelligently. Returns a normalized version of the final path.
+
+    The final path must be located inside of the base path component
+    (otherwise a ValueError is raised).
+
+    Paths outside the base path indicate a possible security
+    sensitive operation.
+    """
+    from urlparse import urljoin
+    base_path = force_unicode(base)
+    base_path = base_path.rstrip('/')
+    paths = [force_unicode(p) for p in paths]
+
+    final_path = base_path
+    for path in paths:
+        final_path = urljoin(final_path.rstrip('/') + "/", path)
+
+    # Ensure final_path starts with base_path and that the next character after
+    # the final path is '/' (or nothing, in which case final_path must be
+    # equal to base_path).
+    base_path_len = len(base_path)
+    if (not final_path.startswith(base_path) or
+            final_path[base_path_len:base_path_len + 1] not in ('', '/')):
+        raise ValueError('the joined path is located outside of the base path'
+                         ' component')
+
+    return final_path.lstrip('/')
+
+
+class S3BotoStorageFile(File):
+    """
+    The default file object used by the S3BotoStorage backend.
+
+    This file implements file streaming using boto's multipart
+    uploading functionality. The file can be opened in read or
+    write mode.
+
+    This class extends Django's File class. However, the contained
+    data is only the data contained in the current buffer. So you
+    should not access the contained file object directly. You should
+    access the data via this class.
+
+    Warning: This file *must* be closed using the close() method in
+    order to properly write the file to S3. Be sure to close the file
+    in your application.
+    """
+    # TODO: Read/Write (rw) mode may be a bit undefined at the moment. Needs testing.
+    # TODO: When Django drops support for Python 2.5, rewrite to use the
+    #       BufferedIO streams in the Python 2.6 io module.
+    buffer_size = setting('AWS_S3_FILE_BUFFER_SIZE', 5242880)
+
+    def __init__(self, name, mode, storage, buffer_size=None):
+        self._storage = storage
+        self.name = name[len(self._storage.location):].lstrip('/')
+        self._mode = mode
+        self.key = storage.bucket.get_key(self._storage._encode_name(name))
+        if not self.key and 'w' in mode:
+            self.key = storage.bucket.new_key(storage._encode_name(name))
+        self._is_dirty = False
+        self._file = None
+        self._multipart = None
+        # 5 MB is the minimum part size (if there is more than one part).
+        # Amazon allows up to 10,000 parts.  The default supports uploads
+        # up to roughly 50 GB.  Increase the part size to accommodate
+        # for files larger than this.
+        if buffer_size is not None:
+            self.buffer_size = buffer_size
+        self._write_counter = 0
+
+    @property
+    def size(self):
+        return self.key.size
+
+    def _get_file(self):
+        if self._file is None:
+            self._file = SpooledTemporaryFile(
+                max_size=self._storage.max_memory_size,
+                suffix=".S3BotoStorageFile",
+                dir=setting("FILE_UPLOAD_TEMP_DIR", None)
+            )
+            if 'r' in self._mode:
+                self._is_dirty = False
+                self.key.get_contents_to_file(self._file)
+                self._file.seek(0)
+            if self._storage.gzip and self.key.content_encoding == 'gzip':
+                self._file = GzipFile(mode=self._mode, fileobj=self._file)
+        return self._file
+
+    def _set_file(self, value):
+        self._file = value
+
+    file = property(_get_file, _set_file)
+
+    def read(self, *args, **kwargs):
+        if 'r' not in self._mode:
+            raise AttributeError("File was not opened in read mode.")
+        return super(S3BotoStorageFile, self).read(*args, **kwargs)
+
+    def write(self, *args, **kwargs):
+        if 'w' not in self._mode:
+            raise AttributeError("File was not opened in write mode.")
+        self._is_dirty = True
+        if self._multipart is None:
+            provider = self.key.bucket.connection.provider
+            upload_headers = {
+                provider.acl_header: self._storage.default_acl
+            }
+            upload_headers.update({'Content-Type': mimetypes.guess_type(self.key.name)[0] or self._storage.key_class.DefaultContentType})
+            upload_headers.update(self._storage.headers)
+            self._multipart = self._storage.bucket.initiate_multipart_upload(
+                self.key.name,
+                headers=upload_headers,
+                reduced_redundancy=self._storage.reduced_redundancy
+            )
+        if self.buffer_size <= self._buffer_file_size:
+            self._flush_write_buffer()
+        return super(S3BotoStorageFile, self).write(*args, **kwargs)
+
+    @property
+    def _buffer_file_size(self):
+        pos = self.file.tell()
+        self.file.seek(0, os.SEEK_END)
+        length = self.file.tell()
+        self.file.seek(pos)
+        return length
+
+    def _flush_write_buffer(self):
+        """
+        Flushes the write buffer.
+        """
+        if self._buffer_file_size:
+            self._write_counter += 1
+            self.file.seek(0)
+            headers = self._storage.headers.copy()
+            self._multipart.upload_part_from_file(
+                self.file, self._write_counter, headers=headers)
+            self.file.close()
+            self._file = None
+
+    def close(self):
+        if self._is_dirty:
+            self._flush_write_buffer()
+            self._multipart.complete_upload()
+        else:
+            if not self._multipart is None:
+                self._multipart.cancel_upload()
+        self.key.close()
+
+
+class S3BotoStorage(Storage):
+    """
+    Amazon Simple Storage Service using Boto
+
+    This storage backend supports opening files in read or write
+    mode and supports streaming(buffering) data in chunks to S3
+    when writing.
+    """
+    connection_class = S3Connection
+    connection_response_error = S3ResponseError
+    file_class = S3BotoStorageFile
+    key_class = S3Key
+
+    # used for looking up the access and secret key from env vars
+    access_key_names = ['AWS_S3_ACCESS_KEY_ID', 'AWS_ACCESS_KEY_ID']
+    secret_key_names = ['AWS_S3_SECRET_ACCESS_KEY', 'AWS_SECRET_ACCESS_KEY']
+
+    access_key = setting('AWS_S3_ACCESS_KEY_ID', setting('AWS_ACCESS_KEY_ID'))
+    secret_key = setting('AWS_S3_SECRET_ACCESS_KEY', setting('AWS_SECRET_ACCESS_KEY'))
+    file_overwrite = setting('AWS_S3_FILE_OVERWRITE', True)
+    headers = setting('AWS_HEADERS', {})
+    bucket_name = setting('AWS_STORAGE_BUCKET_NAME')
+    auto_create_bucket = setting('AWS_AUTO_CREATE_BUCKET', False)
+    default_acl = setting('AWS_DEFAULT_ACL', 'public-read')
+    bucket_acl = setting('AWS_BUCKET_ACL', default_acl)
+    querystring_auth = setting('AWS_QUERYSTRING_AUTH', True)
+    querystring_expire = setting('AWS_QUERYSTRING_EXPIRE', 3600)
+    reduced_redundancy = setting('AWS_REDUCED_REDUNDANCY', False)
+    location = setting('AWS_LOCATION', '')
+    encryption = setting('AWS_S3_ENCRYPTION', False)
+    custom_domain = setting('AWS_S3_CUSTOM_DOMAIN')
+    calling_format = setting('AWS_S3_CALLING_FORMAT', SubdomainCallingFormat())
+    secure_urls = setting('AWS_S3_SECURE_URLS', True)
+    file_name_charset = setting('AWS_S3_FILE_NAME_CHARSET', 'utf-8')
+    gzip = setting('AWS_IS_GZIPPED', False)
+    preload_metadata = setting('AWS_PRELOAD_METADATA', False)
+    gzip_content_types = setting('GZIP_CONTENT_TYPES', (
+        'text/css',
+        'application/javascript',
+        'application/x-javascript',
+    ))
+    url_protocol = setting('AWS_S3_URL_PROTOCOL', 'http:')
+    host = setting('AWS_S3_HOST', S3Connection.DefaultHost)
+    use_ssl = setting('AWS_S3_USE_SSL', True)
+    port = setting('AWS_S3_PORT', None)
+
+    # The max amount of memory a returned file can take up before being
+    # rolled over into a temporary file on disk. Default is 0: Do not roll over.
+    max_memory_size = setting('AWS_S3_MAX_MEMORY_SIZE', 0)
+
+    def __init__(self, acl=None, bucket=None, **settings):
+        # check if some of the settings we've provided as class attributes
+        # need to be overwritten with values passed in here
+        for name, value in settings.items():
+            if hasattr(self, name):
+                setattr(self, name, value)
+
+        # For backward-compatibility of old differing parameter names
+        if acl is not None:
+            self.default_acl = acl
+        if bucket is not None:
+            self.bucket_name = bucket
+
+        self.location = (self.location or '').lstrip('/')
+        # Backward-compatibility: given the anteriority of the SECURE_URL setting
+        # we fall back to https if specified in order to avoid the construction
+        # of unsecure urls.
+        if self.secure_urls:
+            self.url_protocol = 'https:'
+
+        self._entries = {}
+        self._bucket = None
+        self._connection = None
+
+        if not self.access_key and not self.secret_key:
+            self.access_key, self.secret_key = self._get_access_keys()
+
+    @property
+    def connection(self):
+        if self._connection is None:
+            self._connection = self.connection_class(
+                self.access_key,
+                self.secret_key,
+                is_secure=self.use_ssl,
+                calling_format=self.calling_format,
+                host=self.host,
+                port=self.port,
+            )
+        return self._connection
+
+    @property
+    def bucket(self):
+        """
+        Get the current bucket. If there is no current bucket object
+        create it.
+        """
+        if self._bucket is None:
+            self._bucket = self._get_or_create_bucket(self.bucket_name)
+        return self._bucket
+
+    @property
+    def entries(self):
+        """
+        Get the locally cached files for the bucket.
+        """
+        if self.preload_metadata and not self._entries:
+            self._entries = dict((self._decode_name(entry.key), entry)
+                                for entry in self.bucket.list(prefix=self.location))
+        return self._entries
+
+    def _get_access_keys(self):
+        """
+        Gets the access keys to use when accessing S3. If none
+        are provided to the class in the constructor or in the
+        settings then get them from the environment variables.
+        """
+        def lookup_env(names):
+            for name in names:
+                value = os.environ.get(name)
+                if value:
+                    return value
+        access_key = self.access_key or lookup_env(self.access_key_names)
+        secret_key = self.secret_key or lookup_env(self.secret_key_names)
+        return access_key, secret_key
+
+    def _get_or_create_bucket(self, name):
+        """
+        Retrieves a bucket if it exists, otherwise creates it.
+        """
+        try:
+            return self.connection.get_bucket(name,
+                validate=self.auto_create_bucket)
+        except self.connection_response_error:
+            if self.auto_create_bucket:
+                bucket = self.connection.create_bucket(name)
+                bucket.set_acl(self.bucket_acl)
+                return bucket
+            raise ImproperlyConfigured("Bucket %s does not exist. Buckets "
+                                       "can be automatically created by "
+                                       "setting AWS_AUTO_CREATE_BUCKET to "
+                                       "``True``." % name)
+
+    def _clean_name(self, name):
+        """
+        Cleans the name so that Windows style paths work
+        """
+        # Normalize Windows style paths
+        clean_name = posixpath.normpath(name).replace('\\', '/')
+
+        # os.path.normpath() can strip trailing slashes so we implement
+        # a workaround here.
+        if name.endswith('/') and not clean_name.endswith('/'):
+            # Add a trailing slash as it was stripped.
+            return clean_name + '/'
+        else:
+            return clean_name
+
+    def _normalize_name(self, name):
+        """
+        Normalizes the name so that paths like /path/to/ignored/../something.txt
+        work. We check to make sure that the path pointed to is not outside
+        the directory specified by the LOCATION setting.
+        """
+        try:
+            return safe_join(self.location, name)
+        except ValueError:
+            raise SuspiciousOperation("Attempted access to '%s' denied." %
+                                      name)
+
+    def _encode_name(self, name):
+        return smart_str(name, encoding=self.file_name_charset)
+
+    def _decode_name(self, name):
+        return force_unicode(name, encoding=self.file_name_charset)
+
+    def _compress_content(self, content):
+        """Gzip a given string content."""
+        zbuf = StringIO()
+        zfile = GzipFile(mode='wb', compresslevel=6, fileobj=zbuf)
+        try:
+            zfile.write(content.read())
+        finally:
+            zfile.close()
+        zbuf.seek(0)
+        content.file = zbuf
+        content.seek(0)
+        return content
+
+    def _open(self, name, mode='rb'):
+        name = self._normalize_name(self._clean_name(name))
+        f = self.file_class(name, mode, self)
+        if not f.key:
+            raise IOError('File does not exist: %s' % name)
+        return f
+
+    def _save(self, name, content):
+        cleaned_name = self._clean_name(name)
+        name = self._normalize_name(cleaned_name)
+        headers = self.headers.copy()
+        content_type = getattr(content, 'content_type',
+            mimetypes.guess_type(name)[0] or self.key_class.DefaultContentType)
+
+        # setting the content_type in the key object is not enough.
+        headers.update({'Content-Type': content_type})
+
+        if self.gzip and content_type in self.gzip_content_types:
+            content = self._compress_content(content)
+            headers.update({'Content-Encoding': 'gzip'})
+
+        content.name = cleaned_name
+        encoded_name = self._encode_name(name)
+        key = self.bucket.get_key(encoded_name)
+        if not key:
+            key = self.bucket.new_key(encoded_name)
+        if self.preload_metadata:
+            self._entries[encoded_name] = key
+
+        key.set_metadata('Content-Type', content_type)
+        self._save_content(key, content, headers=headers)
+        return cleaned_name
+
+    def _save_content(self, key, content, headers):
+        # only pass backwards incompatible arguments if they vary from the default
+        kwargs = {}
+        if self.encryption:
+            kwargs['encrypt_key'] = self.encryption
+        key.set_contents_from_file(content, headers=headers,
+                                   policy=self.default_acl,
+                                   reduced_redundancy=self.reduced_redundancy,
+                                   rewind=True, **kwargs)
+
+    def delete(self, name):
+        name = self._normalize_name(self._clean_name(name))
+        self.bucket.delete_key(self._encode_name(name))
+
+    def exists(self, name):
+        name = self._normalize_name(self._clean_name(name))
+        if self.entries:
+            return name in self.entries
+        k = self.bucket.new_key(self._encode_name(name))
+        return k.exists()
+
+    def listdir(self, name):
+        name = self._normalize_name(self._clean_name(name))
+        # for the bucket.list and logic below name needs to end in /
+        # But for the root path "" we leave it as an empty string
+        if name and not name.endswith('/'):
+            name += '/'
+
+        dirlist = self.bucket.list(self._encode_name(name))
+        files = []
+        dirs = set()
+        base_parts = name.split("/")[:-1]
+        for item in dirlist:
+            parts = item.name.split("/")
+            parts = parts[len(base_parts):]
+            if len(parts) == 1:
+                # File
+                files.append(parts[0])
+            elif len(parts) > 1:
+                # Directory
+                dirs.add(parts[0])
+        return list(dirs), files
+
+    def size(self, name):
+        name = self._normalize_name(self._clean_name(name))
+        if self.entries:
+            entry = self.entries.get(name)
+            if entry:
+                return entry.size
+            return 0
+        return self.bucket.get_key(self._encode_name(name)).size
+
+    def modified_time(self, name):
+        name = self._normalize_name(self._clean_name(name))
+        entry = self.entries.get(name)
+        # only call self.bucket.get_key() if the key is not found
+        # in the preloaded metadata.
+        if entry is None:
+            entry = self.bucket.get_key(self._encode_name(name))
+        # Parse the last_modified string to a local datetime object.
+        return parse_ts_extended(entry.last_modified)
+
+    def url(self, name, headers=None, response_headers=None):
+        # Preserve the trailing slash after normalizing the path.
+        name = self._normalize_name(self._clean_name(name))
+        if self.custom_domain:
+            return "%s//%s/%s" % (self.url_protocol,
+                                  self.custom_domain, filepath_to_uri(name))
+        return self.connection.generate_url(self.querystring_expire,
+            method='GET', bucket=self.bucket.name, key=self._encode_name(name),
+            headers=headers,
+            query_auth=self.querystring_auth, force_http=not self.secure_urls,
+            response_headers=response_headers)
+
+    def get_available_name(self, name):
+        """ Overwrite existing file with the same name. """
+        if self.file_overwrite:
+            name = self._clean_name(name)
+            return name
+        return super(S3BotoStorage, self).get_available_name(name)