开源阅读_02requestsXlsx

urllib和urllib2是独立的模块，并没有直接的关系，两者相互结合实现复杂的功能
urllib和urllib2在python2中才可以使用
requests库中使用了urllib3(多次请求重复使用一个socket)

Feature Support
---------------
Requests is ready for today's web.

-   International Domains and URLs #国际化域名和URLS
-   Keep-Alive & Connection Pooling #keep—Alive&连接池
-   Sessions with Cookie Persistence #持久性cookie的会话
-   Browser-style SSL Verification #浏览器式SSL认证
-   Basic/Digest Authentication #基本/摘要认证
-   Elegant Key/Value Cookies #简明的key/value cookies
-   Automatic Decompression #自动解压缩
-   Automatic Content Decoding #自动内容解码
-   Unicode Response Bodies #Unicode响应体
-   Multipart File Uploads #文件分块上传
-   HTTP(S) Proxy Support #HTTP(S)代理支持
-   Connection Timeouts #连接超时
-   Streaming Downloads #数据流下载
-   `.netrc` Support #'.netrc'支持
-   Chunked Requests #Chunked请求

def dispatch_hook(key, hooks, hook_data, **kwargs):
    """Dispatches a hook dictionary on a given piece of data."""
    hooks = hooks or {}
    hooks = hooks.get(key)
    if hooks:
        if hasattr(hooks, '__call__'):
            hooks = [hooks]
        for hook in hooks:
            _hook_data = hook(hook_data, **kwargs)
            if _hook_data is not None:
                hook_data = _hook_data
    return hook_data

def dispatch_hook(key, hooks, hook_data, **kwargs):
    """Dispatches a hook dictionary on a given piece of data."""
    hooks = hooks or {}
    hooks = hooks.get(key)
    if hooks:
        if hasattr(hooks, '__call__'):
            hooks = [hooks]
        for hook in hooks:
            _hook_data = hook(hook_data, **kwargs)
            if _hook_data is not None:
                hook_data = _hook_data
    return hook_data

class CaseInsensitiveDict(MutableMapping):
    def __init__(self, data=None, **kwargs):
        self._store = OrderedDict()#有序key的map
        if data is None:
            data = {}
        self.update(data, **kwargs)

    def __setitem__(self, key, value):
        # Use the lowercased key for lookups, but store the actual
        # key alongside the value.
        self._store[key.lower()] = (key, value)#转小写

    def __getitem__(self, key):
        return self._store[key.lower()][1]#? 为何[1]

    def __iter__(self):
        return (casedkey for casedkey, mappedvalue in self._store.values())#生成器

    def lower_items(self):
        """Like iteritems(), but with all lowercase keys."""
        return (
            (lowerkey, keyval[1]) #? 为何只取了[1]，既是第一个也应该是[0]，为何在eq中需要使用
            for (lowerkey, keyval)
            in self._store.items()
        )

    def __eq__(self, other):
        if isinstance(other, Mapping):
            other = CaseInsensitiveDict(other) #兼容普通map
        else:
            return NotImplemented
        # Compare insensitively
        return dict(self.lower_items()) == dict(other.lower_items())　#间接调用普通dict的__eq__

    # Copy is required
    def copy(self):
        return CaseInsensitiveDict(self._store.values())

class LookupDict(dict):
    """Dictionary lookup object."""

    def __init__(self, name=None):
        self.name = name
        super(LookupDict, self).__init__()

_codes = {

    # Informational.
    100: ('continue',),
    101: ('switching_protocols',),
    102: ('processing',),
    103: ('checkpoint',),
    122: ('uri_too_long', 'request_uri_too_long'),
    200: ('ok', 'okay', 'all_ok', 'all_okay', 'all_good', '\\o/', '✓'),
    }
=>键值对互换
codes.continue=100  
codes.uri_too_long=122  
codes.request_uri_too_long=122

基本认证
Base64(user:pwd)后，放在Http头的Authorization中发送给服务端来作认证.
用Base64纯只是防君子不防小人的做法。所以只适合用在一些不那么要求安全性的场合。

摘要认证
digest authentication（HTTP1.1提出的基本认证的替代方法）
这个认证可以看做是基本认证的增强版本，不包含密码的明文传递。

def send(self, request, stream=False, timeout=None, verify=True, cert=None, proxies=None):
    conn = self.get_connection(request.url, proxies)

    self.cert_verify(conn, request.url, verify, cert)
    url = self.request_url(request, proxies)
    self.add_headers(request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies)

	resp = conn.urlopen(
		method=request.method,
		url=url,
		body=request.body,
		headers=request.headers,
		redirect=False,
		assert_same_host=False,
		preload_content=False,
		decode_content=False,
		retries=self.max_retries,
		timeout=timeout
	)
    return self.build_response(request, resp)

保留参数信息 和 cookie
利用 urllib3 的连接池
可以为 request 对象提供默认数据

def request(self, method, url,
    # Create the Request.
    req = Request(
        method=method.upper(),
        url=url,
        headers=headers,
        files=files,
        data=data or {},
        json=json,
        params=params or {},
        auth=auth,
        cookies=cookies,
        hooks=hooks,
    )
    prep = self.prepare_request(req)
    settings = self.merge_environment_settings(
        prep.url, proxies, stream, verify, cert
    )

    # Send the request.
    send_kwargs = {
        'timeout': timeout,
        'allow_redirects': allow_redirects,
    }
    send_kwargs.update(settings)
    resp = self.send(prep, **send_kwargs)

    return resp

p = PreparedRequest()
        p.prepare(
            method=request.method.upper(),
            url=request.url,
            files=request.files,
            data=request.data,
            json=request.json,
            headers=merge_setting(request.headers, self.headers, dict_class=CaseInsensitiveDict),#改进
            params=merge_setting(request.params, self.params),#改进
            auth=merge_setting(auth, self.auth),#改进
            cookies=merged_cookies,#改进
            hooks=merge_hooks(request.hooks, self.hooks),#改进
        )

def send(self, request, **kwargs):
    kwargs.setdefault('stream', self.stream)
    kwargs.setdefault('verify', self.verify)
    kwargs.setdefault('cert', self.cert)
    kwargs.setdefault('proxies', self.proxies)

    hooks = request.hooks
    adapter = self.get_adapter(url=request.url)
    r = adapter.send(request, **kwargs)
    r = dispatch_hook('response', hooks, r, **kwargs)
    extract_cookies_to_jar(self.cookies, request, r.raw)

    return r

def get_adapter(self, url):
    """
    Returns the appropriate connection adapter for the given URL.

    :rtype: requests.adapters.BaseAdapter
    """
    for (prefix, adapter) in self.adapters.items():

        if url.lower().startswith(prefix.lower()):
            return adapter

self.adapters = OrderedDict()
self.mount('https://', HTTPAdapter())
self.mount('http://', HTTPAdapter())

warnings.warn(
           "Non-string usernames will no longer be supported in Requests "
           "3.0.0. Please convert the object you've passed in ({!r}) to "
           "a string or bytes object in the near future to avoid "
           "problems.".format(username),
           category=DeprecationWarning,
       )

def __eq__(self, other):
    return all([
        self.username == getattr(other, 'username', None),
        self.password == getattr(other, 'password', None)
    ])

all(iterable)

if value[:1] == value[-1:] == '"':

get_proxy = lambda k: os.environ.get(k) or os.environ.get(k.upper())
if no_proxy is None:
    no_proxy = get_proxy('no_proxy')

@contextlib.contextmanager
def set_environ(env_name, value):
    """Set the environment variable 'env_name' to 'value'

    Save previous value, yield, and then restore the previous value stored in
    the environment variable 'env_name'.

    If 'value' is None, do nothing"""