diff options
Diffstat (limited to 'requests/models.py')
| -rw-r--r-- | requests/models.py | 621 |
1 files changed, 0 insertions, 621 deletions
diff --git a/requests/models.py b/requests/models.py deleted file mode 100644 index b3a60c2..0000000 --- a/requests/models.py +++ /dev/null @@ -1,621 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -requests.models -~~~~~~~~~~~~~~~ - -""" - -import urllib -import urllib2 -import socket -import zlib - -from urllib2 import HTTPError -from urlparse import urlparse, urlunparse, urljoin -from datetime import datetime - -from .config import settings -from .monkeys import Request as _Request, HTTPBasicAuthHandler, HTTPForcedBasicAuthHandler, HTTPDigestAuthHandler, HTTPRedirectHandler -from .structures import CaseInsensitiveDict -from .packages.poster.encode import multipart_encode -from .packages.poster.streaminghttp import register_openers, get_handlers -from .utils import dict_from_cookiejar -from .exceptions import RequestException, AuthenticationError, Timeout, URLRequired, InvalidMethod, TooManyRedirects -from .status_codes import codes - - -REDIRECT_STATI = (codes.moved, codes.found, codes.other, codes.temporary_moved) - - - -class Request(object): - """The :class:`Request <models.Request>` object. It carries out all functionality of - Requests. Recommended interface is with the Requests functions. - """ - - def __init__(self, - url=None, headers=dict(), files=None, method=None, data=dict(), - params=dict(), auth=None, cookiejar=None, timeout=None, redirect=False, - allow_redirects=False, proxies=None): - - #: Float describ the timeout of the request. - # (Use socket.setdefaulttimeout() as fallback) - self.timeout = timeout - - #: Request URL. - self.url = url - - #: Dictonary of HTTP Headers to attach to the :class:`Request <models.Request>`. - self.headers = headers - - #: Dictionary of files to multipart upload (``{filename: content}``). - self.files = files - - #: HTTP Method to use. Available: GET, HEAD, PUT, POST, DELETE. - self.method = method - - #: Dictionary or byte of request body data to attach to the - #: :class:`Request <models.Request>`. - self.data = None - - #: Dictionary or byte of querystring data to attach to the - #: :class:`Request <models.Request>`. - self.params = None - - #: True if :class:`Request <models.Request>` is part of a redirect chain (disables history - #: and HTTPError storage). - self.redirect = redirect - - #: Set to True if full redirects are allowed (e.g. re-POST-ing of data at new ``Location``) - self.allow_redirects = allow_redirects - - # Dictionary mapping protocol to the URL of the proxy (e.g. {'http': 'foo.bar:3128'}) - self.proxies = proxies - - self.data, self._enc_data = self._encode_params(data) - self.params, self._enc_params = self._encode_params(params) - - #: :class:`Response <models.Response>` instance, containing - #: content and metadata of HTTP Response, once :attr:`sent <send>`. - self.response = Response() - - if isinstance(auth, (list, tuple)): - auth = AuthObject(*auth) - if not auth: - auth = auth_manager.get_auth(self.url) - - #: :class:`AuthObject` to attach to :class:`Request <models.Request>`. - self.auth = auth - - #: CookieJar to attach to :class:`Request <models.Request>`. - self.cookiejar = cookiejar - - #: True if Request has been sent. - self.sent = False - - - # Header manipulation and defaults. - - if settings.accept_gzip: - settings.base_headers.update({'Accept-Encoding': 'gzip'}) - - if headers: - headers = CaseInsensitiveDict(self.headers) - else: - headers = CaseInsensitiveDict() - - for (k, v) in settings.base_headers.items(): - if k not in headers: - headers[k] = v - - self.headers = headers - - - def __repr__(self): - return '<Request [%s]>' % (self.method) - - - def _checks(self): - """Deterministic checks for consistency.""" - - if not self.url: - raise URLRequired - - - def _get_opener(self): - """Creates appropriate opener object for urllib2.""" - - _handlers = [] - - if self.cookiejar is not None: - _handlers.append(urllib2.HTTPCookieProcessor(self.cookiejar)) - - if self.auth: - if not isinstance(self.auth.handler, (urllib2.AbstractBasicAuthHandler, urllib2.AbstractDigestAuthHandler)): - # TODO: REMOVE THIS COMPLETELY - auth_manager.add_password(self.auth.realm, self.url, self.auth.username, self.auth.password) - self.auth.handler = self.auth.handler(auth_manager) - auth_manager.add_auth(self.url, self.auth) - - _handlers.append(self.auth.handler) - - if self.proxies: - _handlers.append(urllib2.ProxyHandler(self.proxies)) - - _handlers.append(HTTPRedirectHandler) - - if not _handlers: - return urllib2.urlopen - - if self.data or self.files: - _handlers.extend(get_handlers()) - - opener = urllib2.build_opener(*_handlers) - - if self.headers: - # Allow default headers in the opener to be overloaded - normal_keys = [k.capitalize() for k in self.headers] - for key, val in opener.addheaders[:]: - if key not in normal_keys: - continue - # Remove it, we have a value to take its place - opener.addheaders.remove((key, val)) - - return opener.open - - - def _build_response(self, resp, is_error=False): - """Build internal :class:`Response <models.Response>` object from given response.""" - - def build(resp): - - response = Response() - response.status_code = getattr(resp, 'code', None) - - try: - response.headers = CaseInsensitiveDict(getattr(resp.info(), 'dict', None)) - response.read = resp.read - response._resp = resp - response._close = resp.close - - if self.cookiejar: - - response.cookies = dict_from_cookiejar(self.cookiejar) - - - except AttributeError: - pass - - if is_error: - response.error = resp - - response.url = getattr(resp, 'url', None) - - return response - - - history = [] - - r = build(resp) - - if r.status_code in REDIRECT_STATI and not self.redirect: - - while ( - ('location' in r.headers) and - ((r.status_code is codes.see_other) or (self.allow_redirects)) - ): - - r.close() - - if not len(history) < settings.max_redirects: - raise TooManyRedirects() - - history.append(r) - - url = r.headers['location'] - - # Handle redirection without scheme (see: RFC 1808 Section 4) - if url.startswith('//'): - parsed_rurl = urlparse(r.url) - url = '%s:%s' % (parsed_rurl.scheme, url) - - # Facilitate non-RFC2616-compliant 'location' headers - # (e.g. '/path/to/resource' instead of 'http://domain.tld/path/to/resource') - if not urlparse(url).netloc: - url = urljoin(r.url, urllib.quote(urllib.unquote(url))) - - # http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.4 - if r.status_code is codes.see_other: - method = 'GET' - else: - method = self.method - - request = Request( - url, self.headers, self.files, method, - self.data, self.params, self.auth, self.cookiejar, - redirect=True - ) - request.send() - r = request.response - - r.history = history - - self.response = r - self.response.request = self - - - @staticmethod - def _encode_params(data): - """Encode parameters in a piece of data. - - If the data supplied is a dictionary, encodes each parameter in it, and - returns a list of tuples containing the encoded parameters, and a urlencoded - version of that. - - Otherwise, assumes the data is already encoded appropriately, and - returns it twice. - - """ - if hasattr(data, 'items'): - result = [] - for k, vs in data.items(): - for v in isinstance(vs, list) and vs or [vs]: - result.append((k.encode('utf-8') if isinstance(k, unicode) else k, - v.encode('utf-8') if isinstance(v, unicode) else v)) - return result, urllib.urlencode(result, doseq=True) - else: - return data, data - - - def _build_url(self): - """Build the actual URL to use.""" - - # Support for unicode domain names and paths. - scheme, netloc, path, params, query, fragment = urlparse(self.url) - netloc = netloc.encode('idna') - if isinstance(path, unicode): - path = path.encode('utf-8') - path = urllib.quote(urllib.unquote(path)) - self.url = str(urlunparse([ scheme, netloc, path, params, query, fragment ])) - - if self._enc_params: - if urlparse(self.url).query: - return '%s&%s' % (self.url, self._enc_params) - else: - return '%s?%s' % (self.url, self._enc_params) - else: - return self.url - - - def send(self, anyway=False): - """Sends the request. Returns True of successful, false if not. - If there was an HTTPError during transmission, - self.response.status_code will contain the HTTPError code. - - Once a request is successfully sent, `sent` will equal True. - - :param anyway: If True, request will be sent, even if it has - already been sent. - """ - - self._checks() - success = False - - # Logging - if settings.verbose: - settings.verbose.write('%s %s %s\n' % ( - datetime.now().isoformat(), self.method, self.url - )) - - - url = self._build_url() - if self.method in ('GET', 'HEAD', 'DELETE'): - req = _Request(url, method=self.method) - else: - - if self.files: - register_openers() - - if self.data: - self.files.update(self.data) - - datagen, headers = multipart_encode(self.files) - req = _Request(url, data=datagen, headers=headers, method=self.method) - - else: - req = _Request(url, data=self._enc_data, method=self.method) - - if self.headers: - for k,v in self.headers.iteritems(): - req.add_header(k, v) - - if not self.sent or anyway: - - try: - opener = self._get_opener() - try: - - resp = opener(req, timeout=self.timeout) - - except TypeError, err: - # timeout argument is new since Python v2.6 - if not 'timeout' in str(err): - raise - - if settings.timeout_fallback: - # fall-back and use global socket timeout (This is not thread-safe!) - old_timeout = socket.getdefaulttimeout() - socket.setdefaulttimeout(self.timeout) - - resp = opener(req) - - if settings.timeout_fallback: - # restore gobal timeout - socket.setdefaulttimeout(old_timeout) - - if self.cookiejar is not None: - self.cookiejar.extract_cookies(resp, req) - - except (urllib2.HTTPError, urllib2.URLError), why: - if hasattr(why, 'reason'): - if isinstance(why.reason, socket.timeout): - why = Timeout(why) - - self._build_response(why, is_error=True) - - - else: - self._build_response(resp) - self.response.ok = True - - - self.sent = self.response.ok - - return self.sent - - - -class Response(object): - """The core :class:`Response <models.Response>` object. All - :class:`Request <models.Request>` objects contain a - :class:`response <models.Response>` attribute, which is an instance - of this class. - """ - - def __init__(self): - #: Raw content of the response, in bytes. - #: If ``content-encoding`` of response was set to ``gzip``, the - #: response data will be automatically deflated. - self._content = None - #: Integer Code of responded HTTP Status. - self.status_code = None - #: Case-insensitive Dictionary of Response Headers. - #: For example, ``headers['content-encoding']`` will return the - #: value of a ``'Content-Encoding'`` response header. - self.headers = CaseInsensitiveDict() - #: Final URL location of Response. - self.url = None - #: True if no :attr:`error` occured. - self.ok = False - #: Resulting :class:`HTTPError` of request, if one occured. - self.error = None - #: A list of :class:`Response <models.Response>` objects from - #: the history of the Request. Any redirect responses will end - #: up here. - self.history = [] - #: The Request that created the Response. - self.request = None - #: A dictionary of Cookies the server sent back. - self.cookies = None - - - def __repr__(self): - return '<Response [%s]>' % (self.status_code) - - - def __nonzero__(self): - """Returns true if :attr:`status_code` is 'OK'.""" - return not self.error - - - def __getattr__(self, name): - """Read and returns the full stream when accessing to :attr: `content`""" - if name == 'content': - if self._content is not None: - return self._content - self._content = self.read() - if self.headers.get('content-encoding', '') == 'gzip': - try: - self._content = zlib.decompress(self._content, 16+zlib.MAX_WBITS) - except zlib.error: - pass - return self._content - else: - raise AttributeError - - def raise_for_status(self): - """Raises stored :class:`HTTPError` or :class:`URLError`, if one occured.""" - if self.error: - raise self.error - - - def close(self): - if self._resp.fp is not None and hasattr(self._resp.fp, '_sock'): - self._resp.fp._sock.recv = None - self._close() - -class AuthManager(object): - """Requests Authentication Manager.""" - - def __new__(cls): - singleton = cls.__dict__.get('__singleton__') - if singleton is not None: - return singleton - - cls.__singleton__ = singleton = object.__new__(cls) - - return singleton - - - def __init__(self): - self.passwd = {} - self._auth = {} - - - def __repr__(self): - return '<AuthManager [%s]>' % (self.method) - - - def add_auth(self, uri, auth): - """Registers AuthObject to AuthManager.""" - - uri = self.reduce_uri(uri, False) - - # try to make it an AuthObject - if not isinstance(auth, AuthObject): - try: - auth = AuthObject(*auth) - except TypeError: - pass - - self._auth[uri] = auth - - - def add_password(self, realm, uri, user, passwd): - """Adds password to AuthManager.""" - # uri could be a single URI or a sequence - if isinstance(uri, basestring): - uri = [uri] - - reduced_uri = tuple([self.reduce_uri(u, False) for u in uri]) - - if reduced_uri not in self.passwd: - self.passwd[reduced_uri] = {} - self.passwd[reduced_uri] = (user, passwd) - - - def find_user_password(self, realm, authuri): - for uris, authinfo in self.passwd.iteritems(): - reduced_authuri = self.reduce_uri(authuri, False) - for uri in uris: - if self.is_suburi(uri, reduced_authuri): - return authinfo - - return (None, None) - - - def get_auth(self, uri): - (in_domain, in_path) = self.reduce_uri(uri, False) - - for domain, path, authority in ( - (i[0][0], i[0][1], i[1]) for i in self._auth.iteritems() - ): - if in_domain == domain: - if path in in_path: - return authority - - - def reduce_uri(self, uri, default_port=True): - """Accept authority or URI and extract only the authority and path.""" - - # note HTTP URLs do not have a userinfo component - parts = urllib2.urlparse.urlsplit(uri) - - if parts[1]: - # URI - scheme = parts[0] - authority = parts[1] - path = parts[2] or '/' - else: - # host or host:port - scheme = None - authority = uri - path = '/' - - host, port = urllib2.splitport(authority) - - if default_port and port is None and scheme is not None: - dport = {"http": 80, - "https": 443, - }.get(scheme) - if dport is not None: - authority = "%s:%d" % (host, dport) - - return authority, path - - - def is_suburi(self, base, test): - """Check if test is below base in a URI tree - - Both args must be URIs in reduced form. - """ - if base == test: - return True - if base[0] != test[0]: - return False - common = urllib2.posixpath.commonprefix((base[1], test[1])) - if len(common) == len(base[1]): - return True - return False - - - def empty(self): - self.passwd = {} - - - def remove(self, uri, realm=None): - # uri could be a single URI or a sequence - if isinstance(uri, basestring): - uri = [uri] - - for default_port in True, False: - reduced_uri = tuple([self.reduce_uri(u, default_port) for u in uri]) - del self.passwd[reduced_uri][realm] - - - def __contains__(self, uri): - # uri could be a single URI or a sequence - if isinstance(uri, basestring): - uri = [uri] - - uri = tuple([self.reduce_uri(u, False) for u in uri]) - - if uri in self.passwd: - return True - - return False - -auth_manager = AuthManager() - - - -class AuthObject(object): - """The :class:`AuthObject` is a simple HTTP Authentication token. When - given to a Requests function, it enables Basic HTTP Authentication for that - Request. You can also enable Authorization for domain realms with AutoAuth. - See AutoAuth for more details. - - :param username: Username to authenticate with. - :param password: Password for given username. - :param realm: (optional) the realm this auth applies to - :param handler: (optional) basic || digest || proxy_basic || proxy_digest - """ - - _handlers = { - 'basic': HTTPBasicAuthHandler, - 'forced_basic': HTTPForcedBasicAuthHandler, - 'digest': HTTPDigestAuthHandler, - 'proxy_basic': urllib2.ProxyBasicAuthHandler, - 'proxy_digest': urllib2.ProxyDigestAuthHandler - } - - def __init__(self, username, password, handler='forced_basic', realm=None): - self.username = username - self.password = password - self.realm = realm - - if isinstance(handler, basestring): - self.handler = self._handlers.get(handler.lower(), HTTPForcedBasicAuthHandler) - else: - self.handler = handler |