1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
|
# Django storage using libcloud providers
# Aymeric Barantal (mric at chamal.fr) 2011
#
import os
from django.conf import settings
from django.core.files.storage import Storage
from django.core.files.base import File
from django.core.exceptions import ImproperlyConfigured
try:
from cStringIO import StringIO
except ImportError:
from StringIO import StringIO
try:
from libcloud.storage.providers import get_driver
from libcloud.storage.types import ObjectDoesNotExistError, Provider
except ImportError:
raise ImproperlyConfigured("Could not load libcloud")
class LibCloudStorage(Storage):
"""Django storage derived class using apache libcloud to operate
on supported providers"""
def __init__(self, provider_name=None, option=None):
if provider_name is None:
provider_name = getattr(settings, 'DEFAULT_LIBCLOUD_PROVIDER', 'default')
self.provider = settings.LIBCLOUD_PROVIDERS.get(provider_name)
if not self.provider:
raise ImproperlyConfigured(
'LIBCLOUD_PROVIDERS %s not defined or invalid' % provider_name)
extra_kwargs = {}
if 'region' in self.provider:
extra_kwargs['region'] = self.provider['region']
try:
provider_type = self.provider['type']
if isinstance(provider_type, basestring):
module_path, tag = provider_type.rsplit('.', 1)
if module_path != 'libcloud.storage.types.Provider':
raise ValueError("Invalid module path")
provider_type = getattr(Provider, tag)
Driver = get_driver(provider_type)
self.driver = Driver(
self.provider['user'],
self.provider['key'],
**extra_kwargs
)
except Exception as e:
raise ImproperlyConfigured(
"Unable to create libcloud driver type %s: %s" % \
(self.provider.get('type'), e))
self.bucket = self.provider['bucket'] # Limit to one container
def _get_bucket(self):
"""Helper to get bucket object (libcloud container)"""
return self.driver.get_container(self.bucket)
def _clean_name(self, name):
"""Clean name (windows directories)"""
return os.path.normpath(name).replace('\\', '/')
def _get_object(self, name):
"""Get object by its name. Return None if object not found"""
clean_name = self._clean_name(name)
try:
return self.driver.get_object(self.bucket, clean_name)
except ObjectDoesNotExistError:
return None
def delete(self, name):
"""Delete object on remote"""
obj = self._get_object(name)
if obj:
return self.driver.delete_object(obj)
else:
raise Exception('Object to delete does not exists')
def exists(self, name):
obj = self._get_object(name)
return True if obj else False
def listdir(self, path='/'):
"""Lists the contents of the specified path,
returning a 2-tuple of lists; the first item being
directories, the second item being files.
"""
container = self._get_bucket()
objects = self.driver.list_container_objects(container)
path = self._clean_name(path)
if not path.endswith('/'):
path = "%s/" % path
files = []
dirs = []
# TOFIX: better algorithm to filter correctly
# (and not depend on google-storage empty folder naming)
for o in objects:
if path == '/':
if o.name.count('/') == 0:
files.append(o.name)
elif o.name.count('/') == 1:
dir_name = o.name[:o.name.index('/')]
if not dir_name in dirs:
dirs.append(dir_name)
elif o.name.startswith(path):
if o.name.count('/') <= path.count('/'):
# TOFIX : special case for google storage with empty dir
if o.name.endswith('_$folder$'):
name = o.name[:-9]
name = name[len(path):]
dirs.append(name)
else:
name = o.name[len(path):]
files.append(name)
return (dirs, files)
def size(self, name):
obj = self._get_object(name)
if obj:
return obj.size
else:
return -1
def url(self, name):
obj = self._get_object(name)
return self.driver.get_object_cdn_url(obj)
def _open(self, name, mode='rb'):
remote_file = LibCloudFile(name, self, mode=mode)
return remote_file
def _read(self, name, start_range=None, end_range=None):
obj = self._get_object(name)
# TOFIX : we should be able to read chunk by chunk
return next(self.driver.download_object_as_stream(obj, obj.size))
def _save(self, name, file):
self.driver.upload_object_via_stream(iter(file), self._get_bucket(), name)
return name
class LibCloudFile(File):
"""File inherited class for libcloud storage objects read and write"""
def __init__(self, name, storage, mode):
self._name = name
self._storage = storage
self._mode = mode
self._is_dirty = False
self.file = StringIO()
self.start_range = 0
@property
def size(self):
if not hasattr(self, '_size'):
self._size = self._storage.size(self._name)
return self._size
def read(self, num_bytes=None):
if num_bytes is None:
args = []
self.start_range = 0
else:
args = [self.start_range, self.start_range + num_bytes - 1]
data = self._storage._read(self._name, *args)
self.file = StringIO(data)
return self.file.getvalue()
def write(self, content):
if 'w' not in self._mode:
raise AttributeError("File was opened for read-only access.")
self.file = StringIO(content)
self._is_dirty = True
def close(self):
if self._is_dirty:
self._storage._save(self._name, self.file)
self.file.close()
|
