diff options
| author | Joshua Gourneau <josh@gourneau.com> | 2012-03-15 00:57:29 -0700 |
|---|---|---|
| committer | Joshua Gourneau <josh@gourneau.com> | 2012-03-15 00:57:29 -0700 |
| commit | 1a5a8ae4950da0139a5081d41d9fb00ab73d91fd (patch) | |
| tree | 34cdbfe82b5ffeefa7645f48bda86df5dc66d37f /requests | |
| parent | 3b17438a4227f105ba0b719904509def94481ff8 (diff) | |
old version of requests
Diffstat (limited to 'requests')
36 files changed, 2157 insertions, 0 deletions
diff --git a/requests/__init__.py b/requests/__init__.py new file mode 100644 index 0000000..15a5050 --- /dev/null +++ b/requests/__init__.py @@ -0,0 +1,4 @@ +# -*- coding: utf-8 -*- + +from core import * +from core import __version__ diff --git a/requests/__init__.pyc b/requests/__init__.pyc Binary files differnew file mode 100644 index 0000000..87b03e9 --- /dev/null +++ b/requests/__init__.pyc diff --git a/requests/api.py b/requests/api.py new file mode 100644 index 0000000..e22ba42 --- /dev/null +++ b/requests/api.py @@ -0,0 +1,188 @@ +# -*- coding: utf-8 -*- + +""" +requests.api +~~~~~~~~~~~~ + +This module impliments the Requests API. + +:copyright: (c) 2011 by Kenneth Reitz. +:license: ISC, see LICENSE for more details. + +""" + +import config +from .models import Request, Response, AuthObject +from .status_codes import codes +from .hooks import dispatch_hook +from .utils import cookiejar_from_dict + +from urlparse import urlparse + +__all__ = ('request', 'get', 'head', 'post', 'patch', 'put', 'delete') + +def request(method, url, + params=None, data=None, headers=None, cookies=None, files=None, auth=None, + timeout=None, allow_redirects=False, proxies=None, hooks=None): + + """Constructs and sends a :class:`Request <models.Request>`. + Returns :class:`Response <models.Response>` object. + + :param method: method for the new :class:`Request` object. + :param url: URL for the new :class:`Request` object. + :param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. + :param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`. + :param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`. + :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. + :param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload. + :param auth: (optional) AuthObject to enable Basic HTTP Auth. + :param timeout: (optional) Float describing the timeout of the request. + :param allow_redirects: (optional) Boolean. Set to True if POST/PUT/DELETE redirect following is allowed. + :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. + """ + + if cookies is None: + cookies = {} + + cookies = cookiejar_from_dict(cookies) + + args = dict( + method = method, + url = url, + data = data, + params = params, + headers = headers, + cookiejar = cookies, + files = files, + auth = auth, + timeout = timeout or config.settings.timeout, + allow_redirects = allow_redirects, + proxies = proxies or config.settings.proxies, + ) + + # Arguments manipulation hook. + args = dispatch_hook('args', hooks, args) + + r = Request(**args) + + # Pre-request hook. + r = dispatch_hook('pre_request', hooks, r) + + # Send the HTTP Request. + r.send() + + # Post-request hook. + r = dispatch_hook('post_request', hooks, r) + + # Response manipulation hook. + r.response = dispatch_hook('response', hooks, r.response) + + return r.response + + +def get(url, **kwargs): + + """Sends a GET request. Returns :class:`Response` object. + + :param url: URL for the new :class:`Request` object. + :param params: (optional) Dictionary of parameters, or bytes, to be sent in the query string for the :class:`Request`. + :param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`. + :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. + :param auth: (optional) AuthObject to enable Basic HTTP Auth. + :param timeout: (optional) Float describing the timeout of the request. + :param allow_redirects: (optional) Boolean. Set to False to disable redirect following. + :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. + """ + + kwargs.setdefault('allow_redirects', True) + return request('GET', url, **kwargs) + + +def head(url, **kwargs): + + """Sends a HEAD request. Returns :class:`Response` object. + + :param url: URL for the new :class:`Request` object. + :param params: (optional) Dictionary of parameters, or bytes, to be sent in the query string for the :class:`Request`. + :param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`. + :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. + :param auth: (optional) AuthObject to enable Basic HTTP Auth. + :param timeout: (optional) Float describing the timeout of the request. + :param allow_redirects: (optional) Boolean. Set to False to disable redirect following. + :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. + """ + + kwargs.setdefault('allow_redirects', True) + return request('HEAD', url, **kwargs) + + +def post(url, data='', **kwargs): + + """Sends a POST request. Returns :class:`Response` object. + + :param url: URL for the new :class:`Request` object. + :param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`. + :param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`. + :param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload. + :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. + :param auth: (optional) AuthObject to enable Basic HTTP Auth. + :param timeout: (optional) Float describing the timeout of the request. + :param allow_redirects: (optional) Boolean. Set to True if redirect following is allowed. + :param params: (optional) Dictionary of parameters, or bytes, to be sent in the query string for the :class:`Request`. + :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. + """ + + return request('POST', url, data=data, **kwargs) + + +def put(url, data='', **kwargs): + """Sends a PUT request. Returns :class:`Response` object. + + :param url: URL for the new :class:`Request` object. + :param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`. + :param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`. + :param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload. + :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. + :param auth: (optional) AuthObject to enable Basic HTTP Auth. + :param timeout: (optional) Float describing the timeout of the request. + :param allow_redirects: (optional) Boolean. Set to True if redirect following is allowed. + :param params: (optional) Dictionary of parameters, or bytes, to be sent in the query string for the :class:`Request`. + :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. + """ + + return request('PUT', url, data=data, **kwargs) + + +def patch(url, data='', **kwargs): + """Sends a PATCH request. Returns :class:`Response` object. + + :param url: URL for the new :class:`Request` object. + :param data: (optional) Dictionary or bytes to send in the body of the :class:`Request`. + :param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`. + :param files: (optional) Dictionary of 'filename': file-like-objects for multipart encoding upload. + :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. + :param auth: (optional) AuthObject to enable Basic HTTP Auth. + :param timeout: (optional) Float describing the timeout of the request. + :param allow_redirects: (optional) Boolean. Set to True if redirect following is allowed. + :param params: (optional) Dictionary of parameters, or bytes, to be sent in the query string for the :class:`Request`. + :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. + """ + + return request('PATCH', url, **kwargs) + + +def delete(url, **kwargs): + + """Sends a DELETE request. Returns :class:`Response` object. + + :param url: URL for the new :class:`Request` object. + :param params: (optional) Dictionary of parameters, or bytes, to be sent in the query string for the :class:`Request`. + :param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`. + :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. + :param auth: (optional) AuthObject to enable Basic HTTP Auth. + :param timeout: (optional) Float describing the timeout of the request. + :param allow_redirects: (optional) Boolean. Set to True if redirect following is allowed. + :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. + """ + + return request('DELETE', url, **kwargs) diff --git a/requests/api.pyc b/requests/api.pyc Binary files differnew file mode 100644 index 0000000..48cde31 --- /dev/null +++ b/requests/api.pyc diff --git a/requests/async.py b/requests/async.py new file mode 100644 index 0000000..ab04084 --- /dev/null +++ b/requests/async.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- + +""" + requests.async + ~~~~~~~~~~~~~~ + + This module implements the main Requests system, after monkey-patching + the urllib2 module with eventlet or gevent.. + + :copyright: (c) 2011 by Kenneth Reitz. + :license: ISC, see LICENSE for more details. +""" + + +from __future__ import absolute_import + +import urllib +import urllib2 + +from urllib2 import HTTPError + + +try: + import eventlet + eventlet.monkey_patch() +except ImportError: + pass + +if not 'eventlet' in locals(): + try: + from gevent import monkey + monkey.patch_all() + except ImportError: + pass + + +if not 'eventlet' in locals(): + raise ImportError('No Async adaptations of urllib2 found!') + + +from .core import * diff --git a/requests/config.py b/requests/config.py new file mode 100644 index 0000000..39be2ed --- /dev/null +++ b/requests/config.py @@ -0,0 +1,67 @@ +# -*- coding: utf-8 -*- + +""" +requests.config +~~~~~~~~~~~~~~~ + +This module provides the Requests settings feature set. + +""" + +class Settings(object): + _singleton = {} + + # attributes with defaults + __attrs__ = [] + + def __init__(self, **kwargs): + super(Settings, self).__init__() + + self.__dict__ = self._singleton + + + def __call__(self, *args, **kwargs): + # new instance of class to call + r = self.__class__() + + # cache previous settings for __exit__ + r.__cache = self.__dict__.copy() + map(self.__cache.setdefault, self.__attrs__) + + # set new settings + self.__dict__.update(*args, **kwargs) + + return r + + + def __enter__(self): + pass + + + def __exit__(self, *args): + + # restore cached copy + self.__dict__.update(self.__cache.copy()) + del self.__cache + + + def __getattribute__(self, key): + if key in object.__getattribute__(self, '__attrs__'): + try: + return object.__getattribute__(self, key) + except AttributeError: + return None + return object.__getattribute__(self, key) + + +settings = Settings() + +settings.base_headers = {'User-Agent': 'python-requests.org'} +settings.accept_gzip = True +settings.proxies = None +settings.verbose = None +settings.timeout = None +settings.max_redirects = 30 + +#: Use socket.setdefaulttimeout() as fallback? +settings.timeout_fallback = True diff --git a/requests/config.pyc b/requests/config.pyc Binary files differnew file mode 100644 index 0000000..dc58c95 --- /dev/null +++ b/requests/config.pyc diff --git a/requests/core.py b/requests/core.py new file mode 100644 index 0000000..505f8a2 --- /dev/null +++ b/requests/core.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- + +""" +requests.core +~~~~~~~~~~~~~ + +This module implements the main Requests system. + +:copyright: (c) 2011 by Kenneth Reitz. +:license: ISC, see LICENSE for more details. + +""" + +__title__ = 'requests' +__version__ = '0.6.2' +__build__ = 0x000602 +__author__ = 'Kenneth Reitz' +__license__ = 'ISC' +__copyright__ = 'Copyright 2011 Kenneth Reitz' + + +from models import HTTPError +from api import * +from exceptions import * +from sessions import session +from status_codes import codes +from config import settings
\ No newline at end of file diff --git a/requests/core.pyc b/requests/core.pyc Binary files differnew file mode 100644 index 0000000..7fbdea3 --- /dev/null +++ b/requests/core.pyc diff --git a/requests/exceptions.py b/requests/exceptions.py new file mode 100644 index 0000000..c08c614 --- /dev/null +++ b/requests/exceptions.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- + +""" +requests.exceptions +~~~~~~~~~~~~~~~ + +""" + +class RequestException(Exception): + """There was an ambiguous exception that occured while handling your + request.""" + +class AuthenticationError(RequestException): + """The authentication credentials provided were invalid.""" + +class Timeout(RequestException): + """The request timed out.""" + +class URLRequired(RequestException): + """A valid URL is required to make a request.""" + +class InvalidMethod(RequestException): + """An inappropriate method was attempted.""" + +class TooManyRedirects(RequestException): + """Too many redirects.""" diff --git a/requests/exceptions.pyc b/requests/exceptions.pyc Binary files differnew file mode 100644 index 0000000..3eae134 --- /dev/null +++ b/requests/exceptions.pyc diff --git a/requests/hooks.py b/requests/hooks.py new file mode 100644 index 0000000..2938029 --- /dev/null +++ b/requests/hooks.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- + +""" +requests.hooks +~~~~~~~~~~~~~~ + +This module provides the capabilities for the Requests hooks system. + +Available hooks: + +``args``: + A dictionary of the arguments being sent to Request(). + +``pre_request``: + The Request object, directly before being sent. + +``post_request``: + The Request object, directly after being sent. + +``response``: + The response generated from a Request. + +""" + +import warnings + + +def dispatch_hook(key, hooks, hook_data): + """Dipatches a hook dictionary on a given peice of data.""" + + hooks = hooks or dict() + + if key in hooks: + try: + return hooks.get(key).__call__(hook_data) or hook_data + + except Exception, why: + warnings.warn(str(why)) + + return hook_data diff --git a/requests/hooks.pyc b/requests/hooks.pyc Binary files differnew file mode 100644 index 0000000..8d4faeb --- /dev/null +++ b/requests/hooks.pyc diff --git a/requests/models.py b/requests/models.py new file mode 100644 index 0000000..b3a60c2 --- /dev/null +++ b/requests/models.py @@ -0,0 +1,621 @@ +# -*- coding: utf-8 -*- + +""" +requests.models +~~~~~~~~~~~~~~~ + +""" + +import urllib +import urllib2 +import socket +import zlib + +from urllib2 import HTTPError +from urlparse import urlparse, urlunparse, urljoin +from datetime import datetime + +from .config import settings +from .monkeys import Request as _Request, HTTPBasicAuthHandler, HTTPForcedBasicAuthHandler, HTTPDigestAuthHandler, HTTPRedirectHandler +from .structures import CaseInsensitiveDict +from .packages.poster.encode import multipart_encode +from .packages.poster.streaminghttp import register_openers, get_handlers +from .utils import dict_from_cookiejar +from .exceptions import RequestException, AuthenticationError, Timeout, URLRequired, InvalidMethod, TooManyRedirects +from .status_codes import codes + + +REDIRECT_STATI = (codes.moved, codes.found, codes.other, codes.temporary_moved) + + + +class Request(object): + """The :class:`Request <models.Request>` object. It carries out all functionality of + Requests. Recommended interface is with the Requests functions. + """ + + def __init__(self, + url=None, headers=dict(), files=None, method=None, data=dict(), + params=dict(), auth=None, cookiejar=None, timeout=None, redirect=False, + allow_redirects=False, proxies=None): + + #: Float describ the timeout of the request. + # (Use socket.setdefaulttimeout() as fallback) + self.timeout = timeout + + #: Request URL. + self.url = url + + #: Dictonary of HTTP Headers to attach to the :class:`Request <models.Request>`. + self.headers = headers + + #: Dictionary of files to multipart upload (``{filename: content}``). + self.files = files + + #: HTTP Method to use. Available: GET, HEAD, PUT, POST, DELETE. + self.method = method + + #: Dictionary or byte of request body data to attach to the + #: :class:`Request <models.Request>`. + self.data = None + + #: Dictionary or byte of querystring data to attach to the + #: :class:`Request <models.Request>`. + self.params = None + + #: True if :class:`Request <models.Request>` is part of a redirect chain (disables history + #: and HTTPError storage). + self.redirect = redirect + + #: Set to True if full redirects are allowed (e.g. re-POST-ing of data at new ``Location``) + self.allow_redirects = allow_redirects + + # Dictionary mapping protocol to the URL of the proxy (e.g. {'http': 'foo.bar:3128'}) + self.proxies = proxies + + self.data, self._enc_data = self._encode_params(data) + self.params, self._enc_params = self._encode_params(params) + + #: :class:`Response <models.Response>` instance, containing + #: content and metadata of HTTP Response, once :attr:`sent <send>`. + self.response = Response() + + if isinstance(auth, (list, tuple)): + auth = AuthObject(*auth) + if not auth: + auth = auth_manager.get_auth(self.url) + + #: :class:`AuthObject` to attach to :class:`Request <models.Request>`. + self.auth = auth + + #: CookieJar to attach to :class:`Request <models.Request>`. + self.cookiejar = cookiejar + + #: True if Request has been sent. + self.sent = False + + + # Header manipulation and defaults. + + if settings.accept_gzip: + settings.base_headers.update({'Accept-Encoding': 'gzip'}) + + if headers: + headers = CaseInsensitiveDict(self.headers) + else: + headers = CaseInsensitiveDict() + + for (k, v) in settings.base_headers.items(): + if k not in headers: + headers[k] = v + + self.headers = headers + + + def __repr__(self): + return '<Request [%s]>' % (self.method) + + + def _checks(self): + """Deterministic checks for consistency.""" + + if not self.url: + raise URLRequired + + + def _get_opener(self): + """Creates appropriate opener object for urllib2.""" + + _handlers = [] + + if self.cookiejar is not None: + _handlers.append(urllib2.HTTPCookieProcessor(self.cookiejar)) + + if self.auth: + if not isinstance(self.auth.handler, (urllib2.AbstractBasicAuthHandler, urllib2.AbstractDigestAuthHandler)): + # TODO: REMOVE THIS COMPLETELY + auth_manager.add_password(self.auth.realm, self.url, self.auth.username, self.auth.password) + self.auth.handler = self.auth.handler(auth_manager) + auth_manager.add_auth(self.url, self.auth) + + _handlers.append(self.auth.handler) + + if self.proxies: + _handlers.append(urllib2.ProxyHandler(self.proxies)) + + _handlers.append(HTTPRedirectHandler) + + if not _handlers: + return urllib2.urlopen + + if self.data or self.files: + _handlers.extend(get_handlers()) + + opener = urllib2.build_opener(*_handlers) + + if self.headers: + # Allow default headers in the opener to be overloaded + normal_keys = [k.capitalize() for k in self.headers] + for key, val in opener.addheaders[:]: + if key not in normal_keys: + continue + # Remove it, we have a value to take its place + opener.addheaders.remove((key, val)) + + return opener.open + + + def _build_response(self, resp, is_error=False): + """Build internal :class:`Response <models.Response>` object from given response.""" + + def build(resp): + + response = Response() + response.status_code = getattr(resp, 'code', None) + + try: + response.headers = CaseInsensitiveDict(getattr(resp.info(), 'dict', None)) + response.read = resp.read + response._resp = resp + response._close = resp.close + + if self.cookiejar: + + response.cookies = dict_from_cookiejar(self.cookiejar) + + + except AttributeError: + pass + + if is_error: + response.error = resp + + response.url = getattr(resp, 'url', None) + + return response + + + history = [] + + r = build(resp) + + if r.status_code in REDIRECT_STATI and not self.redirect: + + while ( + ('location' in r.headers) and + ((r.status_code is codes.see_other) or (self.allow_redirects)) + ): + + r.close() + + if not len(history) < settings.max_redirects: + raise TooManyRedirects() + + history.append(r) + + url = r.headers['location'] + + # Handle redirection without scheme (see: RFC 1808 Section 4) + if url.startswith('//'): + parsed_rurl = urlparse(r.url) + url = '%s:%s' % (parsed_rurl.scheme, url) + + # Facilitate non-RFC2616-compliant 'location' headers + # (e.g. '/path/to/resource' instead of 'http://domain.tld/path/to/resource') + if not urlparse(url).netloc: + url = urljoin(r.url, urllib.quote(urllib.unquote(url))) + + # http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.4 + if r.status_code is codes.see_other: + method = 'GET' + else: + method = self.method + + request = Request( + url, self.headers, self.files, method, + self.data, self.params, self.auth, self.cookiejar, + redirect=True + ) + request.send() + r = request.response + + r.history = history + + self.response = r + self.response.request = self + + + @staticmethod + def _encode_params(data): + """Encode parameters in a piece of data. + + If the data supplied is a dictionary, encodes each parameter in it, and + returns a list of tuples containing the encoded parameters, and a urlencoded + version of that. + + Otherwise, assumes the data is already encoded appropriately, and + returns it twice. + + """ + if hasattr(data, 'items'): + result = [] + for k, vs in data.items(): + for v in isinstance(vs, list) and vs or [vs]: + result.append((k.encode('utf-8') if isinstance(k, unicode) else k, + v.encode('utf-8') if isinstance(v, unicode) else v)) + return result, urllib.urlencode(result, doseq=True) + else: + return data, data + + + def _build_url(self): + """Build the actual URL to use.""" + + # Support for unicode domain names and paths. + scheme, netloc, path, params, query, fragment = urlparse(self.url) + netloc = netloc.encode('idna') + if isinstance(path, unicode): + path = path.encode('utf-8') + path = urllib.quote(urllib.unquote(path)) + self.url = str(urlunparse([ scheme, netloc, path, params, query, fragment ])) + + if self._enc_params: + if urlparse(self.url).query: + return '%s&%s' % (self.url, self._enc_params) + else: + return '%s?%s' % (self.url, self._enc_params) + else: + return self.url + + + def send(self, anyway=False): + """Sends the request. Returns True of successful, false if not. + If there was an HTTPError during transmission, + self.response.status_code will contain the HTTPError code. + + Once a request is successfully sent, `sent` will equal True. + + :param anyway: If True, request will be sent, even if it has + already been sent. + """ + + self._checks() + success = False + + # Logging + if settings.verbose: + settings.verbose.write('%s %s %s\n' % ( + datetime.now().isoformat(), self.method, self.url + )) + + + url = self._build_url() + if self.method in ('GET', 'HEAD', 'DELETE'): + req = _Request(url, method=self.method) + else: + + if self.files: + register_openers() + + if self.data: + self.files.update(self.data) + + datagen, headers = multipart_encode(self.files) + req = _Request(url, data=datagen, headers=headers, method=self.method) + + else: + req = _Request(url, data=self._enc_data, method=self.method) + + if self.headers: + for k,v in self.headers.iteritems(): + req.add_header(k, v) + + if not self.sent or anyway: + + try: + opener = self._get_opener() + try: + + resp = opener(req, timeout=self.timeout) + + except TypeError, err: + # timeout argument is new since Python v2.6 + if not 'timeout' in str(err): + raise + + if settings.timeout_fallback: + # fall-back and use global socket timeout (This is not thread-safe!) + old_timeout = socket.getdefaulttimeout() + socket.setdefaulttimeout(self.timeout) + + resp = opener(req) + + if settings.timeout_fallback: + # restore gobal timeout + socket.setdefaulttimeout(old_timeout) + + if self.cookiejar is not None: + self.cookiejar.extract_cookies(resp, req) + + except (urllib2.HTTPError, urllib2.URLError), why: + if hasattr(why, 'reason'): + if isinstance(why.reason, socket.timeout): + why = Timeout(why) + + self._build_response(why, is_error=True) + + + else: + self._build_response(resp) + self.response.ok = True + + + self.sent = self.response.ok + + return self.sent + + + +class Response(object): + """The core :class:`Response <models.Response>` object. All + :class:`Request <models.Request>` objects contain a + :class:`response <models.Response>` attribute, which is an instance + of this class. + """ + + def __init__(self): + #: Raw content of the response, in bytes. + #: If ``content-encoding`` of response was set to ``gzip``, the + #: response data will be automatically deflated. + self._content = None + #: Integer Code of responded HTTP Status. + self.status_code = None + #: Case-insensitive Dictionary of Response Headers. + #: For example, ``headers['content-encoding']`` will return the + #: value of a ``'Content-Encoding'`` response header. + self.headers = CaseInsensitiveDict() + #: Final URL location of Response. + self.url = None + #: True if no :attr:`error` occured. + self.ok = False + #: Resulting :class:`HTTPError` of request, if one occured. + self.error = None + #: A list of :class:`Response <models.Response>` objects from + #: the history of the Request. Any redirect responses will end + #: up here. + self.history = [] + #: The Request that created the Response. + self.request = None + #: A dictionary of Cookies the server sent back. + self.cookies = None + + + def __repr__(self): + return '<Response [%s]>' % (self.status_code) + + + def __nonzero__(self): + """Returns true if :attr:`status_code` is 'OK'.""" + return not self.error + + + def __getattr__(self, name): + """Read and returns the full stream when accessing to :attr: `content`""" + if name == 'content': + if self._content is not None: + return self._content + self._content = self.read() + if self.headers.get('content-encoding', '') == 'gzip': + try: + self._content = zlib.decompress(self._content, 16+zlib.MAX_WBITS) + except zlib.error: + pass + return self._content + else: + raise AttributeError + + def raise_for_status(self): + """Raises stored :class:`HTTPError` or :class:`URLError`, if one occured.""" + if self.error: + raise self.error + + + def close(self): + if self._resp.fp is not None and hasattr(self._resp.fp, '_sock'): + self._resp.fp._sock.recv = None + self._close() + +class AuthManager(object): + """Requests Authentication Manager.""" + + def __new__(cls): + singleton = cls.__dict__.get('__singleton__') + if singleton is not None: + return singleton + + cls.__singleton__ = singleton = object.__new__(cls) + + return singleton + + + def __init__(self): + self.passwd = {} + self._auth = {} + + + def __repr__(self): + return '<AuthManager [%s]>' % (self.method) + + + def add_auth(self, uri, auth): + """Registers AuthObject to AuthManager.""" + + uri = self.reduce_uri(uri, False) + + # try to make it an AuthObject + if not isinstance(auth, AuthObject): + try: + auth = AuthObject(*auth) + except TypeError: + pass + + self._auth[uri] = auth + + + def add_password(self, realm, uri, user, passwd): + """Adds password to AuthManager.""" + # uri could be a single URI or a sequence + if isinstance(uri, basestring): + uri = [uri] + + reduced_uri = tuple([self.reduce_uri(u, False) for u in uri]) + + if reduced_uri not in self.passwd: + self.passwd[reduced_uri] = {} + self.passwd[reduced_uri] = (user, passwd) + + + def find_user_password(self, realm, authuri): + for uris, authinfo in self.passwd.iteritems(): + reduced_authuri = self.reduce_uri(authuri, False) + for uri in uris: + if self.is_suburi(uri, reduced_authuri): + return authinfo + + return (None, None) + + + def get_auth(self, uri): + (in_domain, in_path) = self.reduce_uri(uri, False) + + for domain, path, authority in ( + (i[0][0], i[0][1], i[1]) for i in self._auth.iteritems() + ): + if in_domain == domain: + if path in in_path: + return authority + + + def reduce_uri(self, uri, default_port=True): + """Accept authority or URI and extract only the authority and path.""" + + # note HTTP URLs do not have a userinfo component + parts = urllib2.urlparse.urlsplit(uri) + + if parts[1]: + # URI + scheme = parts[0] + authority = parts[1] + path = parts[2] or '/' + else: + # host or host:port + scheme = None + authority = uri + path = '/' + + host, port = urllib2.splitport(authority) + + if default_port and port is None and scheme is not None: + dport = {"http": 80, + "https": 443, + }.get(scheme) + if dport is not None: + authority = "%s:%d" % (host, dport) + + return authority, path + + + def is_suburi(self, base, test): + """Check if test is below base in a URI tree + + Both args must be URIs in reduced form. + """ + if base == test: + return True + if base[0] != test[0]: + return False + common = urllib2.posixpath.commonprefix((base[1], test[1])) + if len(common) == len(base[1]): + return True + return False + + + def empty(self): + self.passwd = {} + + + def remove(self, uri, realm=None): + # uri could be a single URI or a sequence + if isinstance(uri, basestring): + uri = [uri] + + for default_port in True, False: + reduced_uri = tuple([self.reduce_uri(u, default_port) for u in uri]) + del self.passwd[reduced_uri][realm] + + + def __contains__(self, uri): + # uri could be a single URI or a sequence + if isinstance(uri, basestring): + uri = [uri] + + uri = tuple([self.reduce_uri(u, False) for u in uri]) + + if uri in self.passwd: + return True + + return False + +auth_manager = AuthManager() + + + +class AuthObject(object): + """The :class:`AuthObject` is a simple HTTP Authentication token. When + given to a Requests function, it enables Basic HTTP Authentication for that + Request. You can also enable Authorization for domain realms with AutoAuth. + See AutoAuth for more details. + + :param username: Username to authenticate with. + :param password: Password for given username. + :param realm: (optional) the realm this auth applies to + :param handler: (optional) basic || digest || proxy_basic || proxy_digest + """ + + _handlers = { + 'basic': HTTPBasicAuthHandler, + 'forced_basic': HTTPForcedBasicAuthHandler, + 'digest': HTTPDigestAuthHandler, + 'proxy_basic': urllib2.ProxyBasicAuthHandler, + 'proxy_digest': urllib2.ProxyDigestAuthHandler + } + + def __init__(self, username, password, handler='forced_basic', realm=None): + self.username = username + self.password = password + self.realm = realm + + if isinstance(handler, basestring): + self.handler = self._handlers.get(handler.lower(), HTTPForcedBasicAuthHandler) + else: + self.handler = handler diff --git a/requests/models.pyc b/requests/models.pyc Binary files differnew file mode 100644 index 0000000..4050ac8 --- /dev/null +++ b/requests/models.pyc diff --git a/requests/monkeys.py b/requests/monkeys.py new file mode 100644 index 0000000..c838071 --- /dev/null +++ b/requests/monkeys.py @@ -0,0 +1,148 @@ +#-*- coding: utf-8 -*- + +""" +requests.monkeys +~~~~~~~~~~~~~~~~ + +Urllib2 Monkey patches. + +""" + +import urllib2 +import re + +class Request(urllib2.Request): + """Hidden wrapper around the urllib2.Request object. Allows for manual + setting of HTTP methods. + """ + + def __init__(self, url, data=None, headers={}, origin_req_host=None, unverifiable=False, method=None): + urllib2.Request.__init__(self, url, data, headers, origin_req_host, unverifiable) + self.method = method + + def get_method(self): + if self.method: + return self.method + + return urllib2.Request.get_method(self) + + +class HTTPRedirectHandler(urllib2.HTTPRedirectHandler): + """HTTP Redirect handler.""" + def http_error_301(self, req, fp, code, msg, headers): + pass + + http_error_302 = http_error_303 = http_error_307 = http_error_301 + + + +class HTTPBasicAuthHandler(urllib2.HTTPBasicAuthHandler): + """HTTP Basic Auth Handler with authentication loop fixes.""" + + def __init__(self, *args, **kwargs): + urllib2.HTTPBasicAuthHandler.__init__(self, *args, **kwargs) + self.retried_req = None + self.retried = 0 + + + def reset_retry_count(self): + # Python 2.6.5 will call this on 401 or 407 errors and thus loop + # forever. We disable reset_retry_count completely and reset in + # http_error_auth_reqed instead. + pass + + + def http_error_auth_reqed(self, auth_header, host, req, headers): + # Reset the retry counter once for each request. + if req is not self.retried_req: + self.retried_req = req + self.retried = 0 + + return urllib2.HTTPBasicAuthHandler.http_error_auth_reqed( + self, auth_header, host, req, headers + ) + + + +class HTTPForcedBasicAuthHandler(HTTPBasicAuthHandler): + """HTTP Basic Auth Handler with forced Authentication.""" + + auth_header = 'Authorization' + rx = re.compile('(?:.*,)*[ \t]*([^ \t]+)[ \t]+' + 'realm=(["\'])(.*?)\\2', re.I) + + def __init__(self, *args, **kwargs): + HTTPBasicAuthHandler.__init__(self, *args, **kwargs) + + + def http_error_401(self, req, fp, code, msg, headers): + url = req.get_full_url() + response = self._http_error_auth_reqed('www-authenticate', url, req, headers) + self.reset_retry_count() + return response + + http_error_404 = http_error_401 + + + def _http_error_auth_reqed(self, authreq, host, req, headers): + + authreq = headers.get(authreq, None) + + if self.retried > 5: + # retry sending the username:password 5 times before failing. + raise urllib2.HTTPError(req.get_full_url(), 401, "basic auth failed", + headers, None) + else: + self.retried += 1 + + if authreq: + + mo = self.rx.search(authreq) + + if mo: + scheme, quote, realm = mo.groups() + + if scheme.lower() == 'basic': + response = self.retry_http_basic_auth(host, req, realm) + + if response and response.code not in (401, 404): + self.retried = 0 + return response + else: + response = self.retry_http_basic_auth(host, req, 'Realm') + + if response and response.code not in (401, 404): + self.retried = 0 + return response + + + +class HTTPDigestAuthHandler(urllib2.HTTPDigestAuthHandler): + + def __init__(self, *args, **kwargs): + urllib2.HTTPDigestAuthHandler.__init__(self, *args, **kwargs) + self.retried_req = None + + def reset_retry_count(self): + # Python 2.6.5 will call this on 401 or 407 errors and thus loop + # forever. We disable reset_retry_count completely and reset in + # http_error_auth_reqed instead. + pass + + def http_error_auth_reqed(self, auth_header, host, req, headers): + # Reset the retry counter once for each request. + if req is not self.retried_req: + self.retried_req = req + self.retried = 0 + # In python < 2.5 AbstractDigestAuthHandler raises a ValueError if + # it doesn't know about the auth type requested. This can happen if + # somebody is using BasicAuth and types a bad password. + + try: + return urllib2.HTTPDigestAuthHandler.http_error_auth_reqed( + self, auth_header, host, req, headers) + except ValueError, inst: + arg = inst.args[0] + if arg.startswith("AbstractDigestAuthHandler doesn't know "): + return + raise
\ No newline at end of file diff --git a/requests/monkeys.pyc b/requests/monkeys.pyc Binary files differnew file mode 100644 index 0000000..6003249 --- /dev/null +++ b/requests/monkeys.pyc diff --git a/requests/packages/__init__.py b/requests/packages/__init__.py new file mode 100644 index 0000000..ab2669e --- /dev/null +++ b/requests/packages/__init__.py @@ -0,0 +1,3 @@ +from __future__ import absolute_import + +from . import poster diff --git a/requests/packages/__init__.pyc b/requests/packages/__init__.pyc Binary files differnew file mode 100644 index 0000000..6669b7c --- /dev/null +++ b/requests/packages/__init__.pyc diff --git a/requests/packages/fuck.py b/requests/packages/fuck.py new file mode 100644 index 0000000..2e1b17e --- /dev/null +++ b/requests/packages/fuck.py @@ -0,0 +1,11 @@ +from urllib3.connectionpool import connection_from_url + +url = 'https://twitter.com' + +http_pool = connection_from_url(url, strict=False) +print http_pool.__dict__ + + +r = http_pool.urlopen('GET', url) +# print r.data +# print r.data diff --git a/requests/packages/poster/__init__.py b/requests/packages/poster/__init__.py new file mode 100644 index 0000000..6e216fc --- /dev/null +++ b/requests/packages/poster/__init__.py @@ -0,0 +1,34 @@ +# Copyright (c) 2010 Chris AtLee +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +"""poster module + +Support for streaming HTTP uploads, and multipart/form-data encoding + +```poster.version``` is a 3-tuple of integers representing the version number. +New releases of poster will always have a version number that compares greater +than an older version of poster. +New in version 0.6.""" + +from __future__ import absolute_import + +from . import streaminghttp +from . import encode + +version = (0, 8, 0) # Thanks JP! diff --git a/requests/packages/poster/__init__.pyc b/requests/packages/poster/__init__.pyc Binary files differnew file mode 100644 index 0000000..78a492b --- /dev/null +++ b/requests/packages/poster/__init__.pyc diff --git a/requests/packages/poster/encode.py b/requests/packages/poster/encode.py new file mode 100644 index 0000000..cf2298d --- /dev/null +++ b/requests/packages/poster/encode.py @@ -0,0 +1,414 @@ +"""multipart/form-data encoding module + +This module provides functions that faciliate encoding name/value pairs +as multipart/form-data suitable for a HTTP POST or PUT request. + +multipart/form-data is the standard way to upload files over HTTP""" + +__all__ = ['gen_boundary', 'encode_and_quote', 'MultipartParam', + 'encode_string', 'encode_file_header', 'get_body_size', 'get_headers', + 'multipart_encode'] + +try: + import uuid + def gen_boundary(): + """Returns a random string to use as the boundary for a message""" + return uuid.uuid4().hex +except ImportError: + import random, sha + def gen_boundary(): + """Returns a random string to use as the boundary for a message""" + bits = random.getrandbits(160) + return sha.new(str(bits)).hexdigest() + +import urllib, re, os, mimetypes +try: + from email.header import Header +except ImportError: + # Python 2.4 + from email.Header import Header + +def encode_and_quote(data): + """If ``data`` is unicode, return urllib.quote_plus(data.encode("utf-8")) + otherwise return urllib.quote_plus(data)""" + if data is None: + return None + + if isinstance(data, unicode): + data = data.encode("utf-8") + return urllib.quote_plus(data) + +def _strify(s): + """If s is a unicode string, encode it to UTF-8 and return the results, + otherwise return str(s), or None if s is None""" + if s is None: + return None + if isinstance(s, unicode): + return s.encode("utf-8") + return str(s) + +class MultipartParam(object): + """Represents a single parameter in a multipart/form-data request + + ``name`` is the name of this parameter. + + If ``value`` is set, it must be a string or unicode object to use as the + data for this parameter. + + If ``filename`` is set, it is what to say that this parameter's filename + is. Note that this does not have to be the actual filename any local file. + + If ``filetype`` is set, it is used as the Content-Type for this parameter. + If unset it defaults to "text/plain; charset=utf8" + + If ``filesize`` is set, it specifies the length of the file ``fileobj`` + + If ``fileobj`` is set, it must be a file-like object that supports + .read(). + + Both ``value`` and ``fileobj`` must not be set, doing so will + raise a ValueError assertion. + + If ``fileobj`` is set, and ``filesize`` is not specified, then + the file's size will be determined first by stat'ing ``fileobj``'s + file descriptor, and if that fails, by seeking to the end of the file, + recording the current position as the size, and then by seeking back to the + beginning of the file. + + ``cb`` is a callable which will be called from iter_encode with (self, + current, total), representing the current parameter, current amount + transferred, and the total size. + """ + def __init__(self, name, value=None, filename=None, filetype=None, + filesize=None, fileobj=None, cb=None): + self.name = Header(name).encode() + self.value = _strify(value) + if filename is None: + self.filename = None + else: + if isinstance(filename, unicode): + # Encode with XML entities + self.filename = filename.encode("ascii", "xmlcharrefreplace") + else: + self.filename = str(filename) + self.filename = self.filename.encode("string_escape").\ + replace('"', '\\"') + self.filetype = _strify(filetype) + + self.filesize = filesize + self.fileobj = fileobj + self.cb = cb + + if self.value is not None and self.fileobj is not None: + raise ValueError("Only one of value or fileobj may be specified") + + if fileobj is not None and filesize is None: + # Try and determine the file size + try: + self.filesize = os.fstat(fileobj.fileno()).st_size + except (OSError, AttributeError): + try: + fileobj.seek(0, 2) + self.filesize = fileobj.tell() + fileobj.seek(0) + except: + raise ValueError("Could not determine filesize") + + def __cmp__(self, other): + attrs = ['name', 'value', 'filename', 'filetype', 'filesize', 'fileobj'] + myattrs = [getattr(self, a) for a in attrs] + oattrs = [getattr(other, a) for a in attrs] + return cmp(myattrs, oattrs) + + def reset(self): + if self.fileobj is not None: + self.fileobj.seek(0) + elif self.value is None: + raise ValueError("Don't know how to reset this parameter") + + @classmethod + def from_file(cls, paramname, filename): + """Returns a new MultipartParam object constructed from the local + file at ``filename``. + + ``filesize`` is determined by os.path.getsize(``filename``) + + ``filetype`` is determined by mimetypes.guess_type(``filename``)[0] + + ``filename`` is set to os.path.basename(``filename``) + """ + + return cls(paramname, filename=os.path.basename(filename), + filetype=mimetypes.guess_type(filename)[0], + filesize=os.path.getsize(filename), + fileobj=open(filename, "rb")) + + @classmethod + def from_params(cls, params): + """Returns a list of MultipartParam objects from a sequence of + name, value pairs, MultipartParam instances, + or from a mapping of names to values + + The values may be strings or file objects, or MultipartParam objects. + MultipartParam object names must match the given names in the + name,value pairs or mapping, if applicable.""" + if hasattr(params, 'items'): + params = params.items() + + retval = [] + for item in params: + if isinstance(item, cls): + retval.append(item) + continue + name, value = item + if isinstance(value, cls): + assert value.name == name + retval.append(value) + continue + if hasattr(value, 'read'): + # Looks like a file object + filename = getattr(value, 'name', None) + if filename is not None: + filetype = mimetypes.guess_type(filename)[0] + else: + filetype = None + + retval.append(cls(name=name, filename=filename, + filetype=filetype, fileobj=value)) + else: + retval.append(cls(name, value)) + return retval + + def encode_hdr(self, boundary): + """Returns the header of the encoding of this parameter""" + boundary = encode_and_quote(boundary) + + headers = ["--%s" % boundary] + + if self.filename: + disposition = 'form-data; name="%s"; filename="%s"' % (self.name, + self.filename) + else: + disposition = 'form-data; name="%s"' % self.name + + headers.append("Content-Disposition: %s" % disposition) + + if self.filetype: + filetype = self.filetype + else: + filetype = "text/plain; charset=utf-8" + + headers.append("Content-Type: %s" % filetype) + + headers.append("") + headers.append("") + + return "\r\n".join(headers) + + def encode(self, boundary): + """Returns the string encoding of this parameter""" + if self.value is None: + value = self.fileobj.read() + else: + value = self.value + + if re.search("^--%s$" % re.escape(boundary), value, re.M): + raise ValueError("boundary found in encoded string") + + return "%s%s\r\n" % (self.encode_hdr(boundary), value) + + def iter_encode(self, boundary, blocksize=4096): + """Yields the encoding of this parameter + If self.fileobj is set, then blocks of ``blocksize`` bytes are read and + yielded.""" + total = self.get_size(boundary) + current = 0 + if self.value is not None: + block = self.encode(boundary) + current += len(block) + yield block + if self.cb: + self.cb(self, current, total) + else: + block = self.encode_hdr(boundary) + current += len(block) + yield block + if self.cb: + self.cb(self, current, total) + last_block = "" + encoded_boundary = "--%s" % encode_and_quote(boundary) + boundary_exp = re.compile("^%s$" % re.escape(encoded_boundary), + re.M) + while True: + block = self.fileobj.read(blocksize) + if not block: + current += 2 + yield "\r\n" + if self.cb: + self.cb(self, current, total) + break + last_block += block + if boundary_exp.search(last_block): + raise ValueError("boundary found in file data") + last_block = last_block[-len(encoded_boundary)-2:] + current += len(block) + yield block + if self.cb: + self.cb(self, current, total) + + def get_size(self, boundary): + """Returns the size in bytes that this param will be when encoded + with the given boundary.""" + if self.filesize is not None: + valuesize = self.filesize + else: + valuesize = len(self.value) + + return len(self.encode_hdr(boundary)) + 2 + valuesize + +def encode_string(boundary, name, value): + """Returns ``name`` and ``value`` encoded as a multipart/form-data + variable. ``boundary`` is the boundary string used throughout + a single request to separate variables.""" + + return MultipartParam(name, value).encode(boundary) + +def encode_file_header(boundary, paramname, filesize, filename=None, + filetype=None): + """Returns the leading data for a multipart/form-data field that contains + file data. + + ``boundary`` is the boundary string used throughout a single request to + separate variables. + + ``paramname`` is the name of the variable in this request. + + ``filesize`` is the size of the file data. + + ``filename`` if specified is the filename to give to this field. This + field is only useful to the server for determining the original filename. + + ``filetype`` if specified is the MIME type of this file. + + The actual file data should be sent after this header has been sent. + """ + + return MultipartParam(paramname, filesize=filesize, filename=filename, + filetype=filetype).encode_hdr(boundary) + +def get_body_size(params, boundary): + """Returns the number of bytes that the multipart/form-data encoding + of ``params`` will be.""" + size = sum(p.get_size(boundary) for p in MultipartParam.from_params(params)) + return size + len(boundary) + 6 + +def get_headers(params, boundary): + """Returns a dictionary with Content-Type and Content-Length headers + for the multipart/form-data encoding of ``params``.""" + headers = {} + boundary = urllib.quote_plus(boundary) + headers['Content-Type'] = "multipart/form-data; boundary=%s" % boundary + headers['Content-Length'] = str(get_body_size(params, boundary)) + return headers + +class multipart_yielder: + def __init__(self, params, boundary, cb): + self.params = params + self.boundary = boundary + self.cb = cb + + self.i = 0 + self.p = None + self.param_iter = None + self.current = 0 + self.total = get_body_size(params, boundary) + + def __iter__(self): + return self + + def next(self): + """generator function to yield multipart/form-data representation + of parameters""" + if self.param_iter is not None: + try: + block = self.param_iter.next() + self.current += len(block) + if self.cb: + self.cb(self.p, self.current, self.total) + return block + except StopIteration: + self.p = None + self.param_iter = None + + if self.i is None: + raise StopIteration + elif self.i >= len(self.params): + self.param_iter = None + self.p = None + self.i = None + block = "--%s--\r\n" % self.boundary + self.current += len(block) + if self.cb: + self.cb(self.p, self.current, self.total) + return block + + self.p = self.params[self.i] + self.param_iter = self.p.iter_encode(self.boundary) + self.i += 1 + return self.next() + + def reset(self): + self.i = 0 + self.current = 0 + for param in self.params: + param.reset() + +def multipart_encode(params, boundary=None, cb=None): + """Encode ``params`` as multipart/form-data. + + ``params`` should be a sequence of (name, value) pairs or MultipartParam + objects, or a mapping of names to values. + Values are either strings parameter values, or file-like objects to use as + the parameter value. The file-like objects must support .read() and either + .fileno() or both .seek() and .tell(). + + If ``boundary`` is set, then it as used as the MIME boundary. Otherwise + a randomly generated boundary will be used. In either case, if the + boundary string appears in the parameter values a ValueError will be + raised. + + If ``cb`` is set, it should be a callback which will get called as blocks + of data are encoded. It will be called with (param, current, total), + indicating the current parameter being encoded, the current amount encoded, + and the total amount to encode. + + Returns a tuple of `datagen`, `headers`, where `datagen` is a + generator that will yield blocks of data that make up the encoded + parameters, and `headers` is a dictionary with the assoicated + Content-Type and Content-Length headers. + + Examples: + + >>> datagen, headers = multipart_encode( [("key", "value1"), ("key", "value2")] ) + >>> s = "".join(datagen) + >>> assert "value2" in s and "value1" in s + + >>> p = MultipartParam("key", "value2") + >>> datagen, headers = multipart_encode( [("key", "value1"), p] ) + >>> s = "".join(datagen) + >>> assert "value2" in s and "value1" in s + + >>> datagen, headers = multipart_encode( {"key": "value1"} ) + >>> s = "".join(datagen) + >>> assert "value2" not in s and "value1" in s + + """ + if boundary is None: + boundary = gen_boundary() + else: + boundary = urllib.quote_plus(boundary) + + headers = get_headers(params, boundary) + params = MultipartParam.from_params(params) + + return multipart_yielder(params, boundary, cb), headers diff --git a/requests/packages/poster/encode.pyc b/requests/packages/poster/encode.pyc Binary files differnew file mode 100644 index 0000000..8641346 --- /dev/null +++ b/requests/packages/poster/encode.pyc diff --git a/requests/packages/poster/streaminghttp.py b/requests/packages/poster/streaminghttp.py new file mode 100644 index 0000000..1b591d4 --- /dev/null +++ b/requests/packages/poster/streaminghttp.py @@ -0,0 +1,199 @@ +"""Streaming HTTP uploads module. + +This module extends the standard httplib and urllib2 objects so that +iterable objects can be used in the body of HTTP requests. + +In most cases all one should have to do is call :func:`register_openers()` +to register the new streaming http handlers which will take priority over +the default handlers, and then you can use iterable objects in the body +of HTTP requests. + +**N.B.** You must specify a Content-Length header if using an iterable object +since there is no way to determine in advance the total size that will be +yielded, and there is no way to reset an interator. + +Example usage: + +>>> from StringIO import StringIO +>>> import urllib2, poster.streaminghttp + +>>> opener = poster.streaminghttp.register_openers() + +>>> s = "Test file data" +>>> f = StringIO(s) + +>>> req = urllib2.Request("http://localhost:5000", f, +... {'Content-Length': str(len(s))}) +""" + +import httplib, urllib2, socket +from httplib import NotConnected + +__all__ = ['StreamingHTTPConnection', 'StreamingHTTPRedirectHandler', + 'StreamingHTTPHandler', 'register_openers'] + +if hasattr(httplib, 'HTTPS'): + __all__.extend(['StreamingHTTPSHandler', 'StreamingHTTPSConnection']) + +class _StreamingHTTPMixin: + """Mixin class for HTTP and HTTPS connections that implements a streaming + send method.""" + def send(self, value): + """Send ``value`` to the server. + + ``value`` can be a string object, a file-like object that supports + a .read() method, or an iterable object that supports a .next() + method. + """ + # Based on python 2.6's httplib.HTTPConnection.send() + if self.sock is None: + if self.auto_open: + self.connect() + else: + raise NotConnected() + + # send the data to the server. if we get a broken pipe, then close + # the socket. we want to reconnect when somebody tries to send again. + # + # NOTE: we DO propagate the error, though, because we cannot simply + # ignore the error... the caller will know if they can retry. + if self.debuglevel > 0: + print "send:", repr(value) + try: + blocksize = 8192 + if hasattr(value, 'read') : + if hasattr(value, 'seek'): + value.seek(0) + if self.debuglevel > 0: + print "sendIng a read()able" + data = value.read(blocksize) + while data: + self.sock.sendall(data) + data = value.read(blocksize) + elif hasattr(value, 'next'): + if hasattr(value, 'reset'): + value.reset() + if self.debuglevel > 0: + print "sendIng an iterable" + for data in value: + self.sock.sendall(data) + else: + self.sock.sendall(value) + except socket.error, v: + if v[0] == 32: # Broken pipe + self.close() + raise + +class StreamingHTTPConnection(_StreamingHTTPMixin, httplib.HTTPConnection): + """Subclass of `httplib.HTTPConnection` that overrides the `send()` method + to support iterable body objects""" + +class StreamingHTTPRedirectHandler(urllib2.HTTPRedirectHandler): + """Subclass of `urllib2.HTTPRedirectHandler` that overrides the + `redirect_request` method to properly handle redirected POST requests + + This class is required because python 2.5's HTTPRedirectHandler does + not remove the Content-Type or Content-Length headers when requesting + the new resource, but the body of the original request is not preserved. + """ + + handler_order = urllib2.HTTPRedirectHandler.handler_order - 1 + + # From python2.6 urllib2's HTTPRedirectHandler + def redirect_request(self, req, fp, code, msg, headers, newurl): + """Return a Request or None in response to a redirect. + + This is called by the http_error_30x methods when a + redirection response is received. If a redirection should + take place, return a new Request to allow http_error_30x to + perform the redirect. Otherwise, raise HTTPError if no-one + else should try to handle this url. Return None if you can't + but another Handler might. + """ + m = req.get_method() + if (code in (301, 302, 303, 307) and m in ("GET", "HEAD") + or code in (301, 302, 303) and m == "POST"): + # Strictly (according to RFC 2616), 301 or 302 in response + # to a POST MUST NOT cause a redirection without confirmation + # from the user (of urllib2, in this case). In practice, + # essentially all clients do redirect in this case, so we + # do the same. + # be conciliant with URIs containing a space + newurl = newurl.replace(' ', '%20') + newheaders = dict((k, v) for k, v in req.headers.items() + if k.lower() not in ( + "content-length", "content-type") + ) + return urllib2.Request(newurl, + headers=newheaders, + origin_req_host=req.get_origin_req_host(), + unverifiable=True) + else: + raise urllib2.HTTPError(req.get_full_url(), code, msg, headers, fp) + +class StreamingHTTPHandler(urllib2.HTTPHandler): + """Subclass of `urllib2.HTTPHandler` that uses + StreamingHTTPConnection as its http connection class.""" + + handler_order = urllib2.HTTPHandler.handler_order - 1 + + def http_open(self, req): + """Open a StreamingHTTPConnection for the given request""" + return self.do_open(StreamingHTTPConnection, req) + + def http_request(self, req): + """Handle a HTTP request. Make sure that Content-Length is specified + if we're using an interable value""" + # Make sure that if we're using an iterable object as the request + # body, that we've also specified Content-Length + if req.has_data(): + data = req.get_data() + if hasattr(data, 'read') or hasattr(data, 'next'): + if not req.has_header('Content-length'): + raise ValueError( + "No Content-Length specified for iterable body") + return urllib2.HTTPHandler.do_request_(self, req) + +if hasattr(httplib, 'HTTPS'): + class StreamingHTTPSConnection(_StreamingHTTPMixin, + httplib.HTTPSConnection): + """Subclass of `httplib.HTTSConnection` that overrides the `send()` + method to support iterable body objects""" + + class StreamingHTTPSHandler(urllib2.HTTPSHandler): + """Subclass of `urllib2.HTTPSHandler` that uses + StreamingHTTPSConnection as its http connection class.""" + + handler_order = urllib2.HTTPSHandler.handler_order - 1 + + def https_open(self, req): + return self.do_open(StreamingHTTPSConnection, req) + + def https_request(self, req): + # Make sure that if we're using an iterable object as the request + # body, that we've also specified Content-Length + if req.has_data(): + data = req.get_data() + if hasattr(data, 'read') or hasattr(data, 'next'): + if not req.has_header('Content-length'): + raise ValueError( + "No Content-Length specified for iterable body") + return urllib2.HTTPSHandler.do_request_(self, req) + + +def get_handlers(): + handlers = [StreamingHTTPHandler, StreamingHTTPRedirectHandler] + if hasattr(httplib, "HTTPS"): + handlers.append(StreamingHTTPSHandler) + return handlers + +def register_openers(): + """Register the streaming http handlers in the global urllib2 default + opener object. + + Returns the created OpenerDirector object.""" + opener = urllib2.build_opener(*get_handlers()) + + urllib2.install_opener(opener) + + return opener diff --git a/requests/packages/poster/streaminghttp.pyc b/requests/packages/poster/streaminghttp.pyc Binary files differnew file mode 100644 index 0000000..e2ac5e8 --- /dev/null +++ b/requests/packages/poster/streaminghttp.pyc diff --git a/requests/packages/toy.py b/requests/packages/toy.py new file mode 100644 index 0000000..516098e --- /dev/null +++ b/requests/packages/toy.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" +urllib3-bug.py +~~~~~~~~~~~~~~ + +This module shows some odd behavior from urllib3's PoolManager. +""" + +from urllib3.poolmanager import PoolManager + +import logging +logging.basicConfig(level=logging.INFO) + +url = 'http://httpbin.org:80/get' + +pool = PoolManager(num_pools=10, maxsize=1, block=False) + +for i in range(14): + # c = pool.connection_from_url(url + '/' + str(i)) + # print pool.__dict__ + # print pool.pools.__dict__ + pool.urlopen('GET', url) + print '.'
\ No newline at end of file diff --git a/requests/patches.py b/requests/patches.py new file mode 100644 index 0000000..43a3b4c --- /dev/null +++ b/requests/patches.py @@ -0,0 +1,5 @@ +# -*- coding: utf-8 -*- + +""" +requests.monkeys +""" diff --git a/requests/sessions.py b/requests/sessions.py new file mode 100644 index 0000000..50b09f6 --- /dev/null +++ b/requests/sessions.py @@ -0,0 +1,84 @@ +# -*- coding: utf-8 -*- + +""" +requests.session +~~~~~~~~~~~~~~~ + +This module provides a Session object to manage and persist settings across +requests (cookies, auth, proxies). + +""" + +import cookielib + +from . import api +from .utils import add_dict_to_cookiejar + + + +class Session(object): + """A Requests session.""" + + __attrs__ = ['headers', 'cookies', 'auth', 'timeout', 'proxies', 'hooks'] + + + def __init__(self, **kwargs): + + # Set up a CookieJar to be used by default + self.cookies = cookielib.FileCookieJar() + + # Map args from kwargs to instance-local variables + map(lambda k, v: (k in self.__attrs__) and setattr(self, k, v), + kwargs.iterkeys(), kwargs.itervalues()) + + # Map and wrap requests.api methods + self._map_api_methods() + + + def __repr__(self): + return '<requests-client at 0x%x>' % (id(self)) + + def __enter__(self): + return self + + def __exit__(self, *args): + # print args + pass + + + def _map_api_methods(self): + """Reads each available method from requests.api and decorates + them with a wrapper, which inserts any instance-local attributes + (from __attrs__) that have been set, combining them with **kwargs. + """ + + def pass_args(func): + def wrapper_func(*args, **kwargs): + inst_attrs = dict((k, v) for k, v in self.__dict__.iteritems() + if k in self.__attrs__) + # Combine instance-local values with kwargs values, with + # priority to values in kwargs + kwargs = dict(inst_attrs.items() + kwargs.items()) + + # If a session request has a cookie_dict, inject the + # values into the existing CookieJar instead. + if isinstance(kwargs.get('cookies', None), dict): + kwargs['cookies'] = add_dict_to_cookiejar( + inst_attrs['cookies'], kwargs['cookies'] + ) + + if kwargs.get('headers', None) and inst_attrs.get('headers', None): + kwargs['headers'].update(inst_attrs['headers']) + + return func(*args, **kwargs) + return wrapper_func + + # Map and decorate each function available in requests.api + map(lambda fn: setattr(self, fn, pass_args(getattr(api, fn))), + api.__all__) + + +def session(**kwargs): + """Returns a :class:`Session` for context-managment.""" + + return Session(**kwargs)
\ No newline at end of file diff --git a/requests/sessions.pyc b/requests/sessions.pyc Binary files differnew file mode 100644 index 0000000..24ac4af --- /dev/null +++ b/requests/sessions.pyc diff --git a/requests/status_codes.py b/requests/status_codes.py new file mode 100644 index 0000000..a809de6 --- /dev/null +++ b/requests/status_codes.py @@ -0,0 +1,83 @@ +# -*- coding: utf-8 -*- + +from .structures import LookupDict + +_codes = { + + # Informational. + 100: ('continue',), + 101: ('switching_protocols',), + 102: ('processing',), + 103: ('checkpoint',), + 122: ('uri_too_long', 'request_uri_too_long'), + 200: ('ok', 'okay', 'all_ok', 'all_okay', 'all_good', '\\o/'), + 201: ('created',), + 202: ('accepted',), + 203: ('non_authoritative_info', 'non_authoritative_information'), + 204: ('no_content',), + 205: ('reset_content', 'reset'), + 206: ('partial_content', 'partial'), + 207: ('multi_status', 'multiple_status', 'multi_stati', 'multiple_stati'), + 208: ('im_used',), + + # Redirection. + 300: ('multiple_choices',), + 301: ('moved_permanently', 'moved', '\\o-'), + 302: ('found',), + 303: ('see_other', 'other'), + 304: ('not_modified',), + 305: ('use_proxy',), + 306: ('switch_proxy',), + 307: ('temporary_redirect', 'temporary_moved', 'temporary'), + 308: ('resume_incomplete', 'resume'), + + # Client Error. + 400: ('bad_request', 'bad'), + 401: ('unauthorized',), + 402: ('payment_required', 'payment'), + 403: ('forbidden',), + 404: ('not_found', '-o-'), + 405: ('method_not_allowed', 'not_allowed'), + 406: ('not_acceptable',), + 407: ('proxy_authentication_required', 'proxy_auth', 'proxy_authentication'), + 408: ('request_timeout', 'timeout'), + 409: ('conflict',), + 410: ('gone',), + 411: ('length_required',), + 412: ('precondition_failed', 'precondition'), + 413: ('request_entity_too_large',), + 414: ('request_uri_too_large',), + 415: ('unspported_media_type', 'unspported_media', 'media_type'), + 416: ('requested_range_not_satisfiable', 'requested_range', 'range_not_satisfiable'), + 417: ('expectation_failed',), + 418: ('im_a_teapot', 'teapot', 'i_am_a_teapot'), + 422: ('unprocessable_entity', 'unprocessable'), + 423: ('locked',), + 424: ('failed_depdendency', 'depdendency'), + 425: ('unordered_collection', 'unordered'), + 426: ('upgrade_required', 'upgrade'), + 444: ('no_response', 'none'), + 449: ('retry_with', 'retry'), + 450: ('blocked_by_windows_parental_controls', 'parental_controls'), + 499: ('client_closed_request',), + + # Server Error. + 500: ('internal_server_error', 'server_error', '/o\\'), + 501: ('not_implemented',), + 502: ('bad_gateway',), + 503: ('service_unavailable', 'unavailable'), + 504: ('gateway_timeout',), + 505: ('http_version_not_supported', 'http_version'), + 506: ('variant_also_negotiates',), + 507: ('insufficient_storage',), + 509: ('bandwidth_limit_exceeded', 'bandwidth'), + 510: ('not_extended',), +} + +codes = LookupDict(name='status_codes') + +for (code, titles) in _codes.items(): + for title in titles: + setattr(codes, title, code) + if not title.startswith('\\'): + setattr(codes, title.upper(), code)
\ No newline at end of file diff --git a/requests/status_codes.pyc b/requests/status_codes.pyc Binary files differnew file mode 100644 index 0000000..8bfc49d --- /dev/null +++ b/requests/status_codes.pyc diff --git a/requests/structures.py b/requests/structures.py new file mode 100644 index 0000000..d068bf9 --- /dev/null +++ b/requests/structures.py @@ -0,0 +1,65 @@ +# -*- coding: utf-8 -*- + +""" +requests.structures +~~~~~~~~~~~~~~~~~~~ + +Datastructures that power Requests. + +""" + +class CaseInsensitiveDict(dict): + """Case-insensitive Dictionary + + For example, ``headers['content-encoding']`` will return the + value of a ``'Content-Encoding'`` response header.""" + + @property + def lower_keys(self): + if not hasattr(self, '_lower_keys') or not self._lower_keys: + self._lower_keys = dict((k.lower(), k) for k in self.iterkeys()) + return self._lower_keys + + def _clear_lower_keys(self): + if hasattr(self, '_lower_keys'): + self._lower_keys.clear() + + def __setitem__(self, key, value): + dict.__setitem__(self, key, value) + self._clear_lower_keys() + + def __delitem__(self, key): + dict.__delitem__(self, key) + self._lower_keys.clear() + + def __contains__(self, key): + return key.lower() in self.lower_keys + + def __getitem__(self, key): + # We allow fall-through here, so values default to None + if key in self: + return dict.__getitem__(self, self.lower_keys[key.lower()]) + + def get(self, key, default=None): + if key in self: + return self[key] + else: + return default + +class LookupDict(dict): + """Dictionary lookup object.""" + + def __init__(self, name=None): + self.name = name + super(LookupDict, self).__init__() + + def __repr__(self): + return '<lookup \'%s\'>' % (self.name) + + def __getitem__(self, key): + # We allow fall-through here, so values default to None + + return self.__dict__.get(key, None) + + def get(self, key, default=None): + return self.__dict__.get(key, default)
\ No newline at end of file diff --git a/requests/structures.pyc b/requests/structures.pyc Binary files differnew file mode 100644 index 0000000..01f60fb --- /dev/null +++ b/requests/structures.pyc diff --git a/requests/utils.py b/requests/utils.py new file mode 100644 index 0000000..8ac78b4 --- /dev/null +++ b/requests/utils.py @@ -0,0 +1,72 @@ +# -*- coding: utf-8 -*- + +""" +requests.utils +~~~~~~~~~~~~~~ + +This module provides utlity functions that are used within Requests +that are also useful for external consumption. + +""" + +import cookielib + + +def dict_from_cookiejar(cookiejar): + """Returns a key/value dictionary from a CookieJar.""" + + cookie_dict = {} + + for _, cookies in cookiejar._cookies.items(): + for _, cookies in cookies.items(): + for cookie in cookies.values(): + # print cookie + cookie_dict[cookie.name] = cookie.value + + return cookie_dict + + +def cookiejar_from_dict(cookie_dict): + """Returns a CookieJar from a key/value dictionary.""" + + # return cookiejar if one was passed in + if isinstance(cookie_dict, cookielib.CookieJar): + return cookie_dict + + # create cookiejar + cj = cookielib.CookieJar() + + cj = add_dict_to_cookiejar(cj, cookie_dict) + + return cj + + +def add_dict_to_cookiejar(cj, cookie_dict): + """Returns a CookieJar from a key/value dictionary.""" + + for k, v in cookie_dict.items(): + + cookie = cookielib.Cookie( + version=0, + name=k, + value=v, + port=None, + port_specified=False, + domain='', + domain_specified=False, + domain_initial_dot=False, + path='/', + path_specified=True, + secure=False, + expires=None, + discard=True, + comment=None, + comment_url=None, + rest={'HttpOnly': None}, + rfc2109=False + ) + + # add cookie to cookiejar + cj.set_cookie(cookie) + + return cj diff --git a/requests/utils.pyc b/requests/utils.pyc Binary files differnew file mode 100644 index 0000000..2688a19 --- /dev/null +++ b/requests/utils.pyc |
