зеркало из https://github.com/mozilla/treeherder.git
Bug 1016117 - Added/updated vendor libs to support publishing to pulse
This commit is contained in:
Jonathan Eads
mdoglio
Родитель
c370f477e2
Коммит
94bbbfb183
|
|
@ -2,10 +2,10 @@
|
|||
MySQL-python
|
||||
gunicorn==17.5
|
||||
django>=1.5,<1.6
|
||||
Celery==3.0.17
|
||||
Celery==3.1.15
|
||||
django-celery==3.0.17
|
||||
celerymon==1.0.3
|
||||
kombu==2.4.7
|
||||
kombu==3.0.22
|
||||
simplejson==3.3.0
|
||||
Cython==0.19.2
|
||||
gevent==1.0
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ python-memcached==1.48
|
|||
# requirements for mozillapulse
|
||||
mozillapulse==0.61
|
||||
carrot==0.10.7
|
||||
jsonschema==2.4.0
|
||||
|
||||
djangorestframework==2.3.12
|
||||
Unipath==1.0
|
||||
|
|
|
|||
|
|
@ -1,98 +0,0 @@
|
|||
from treeherder.etl.pulse import PulseDataAdapter, TreeherderPulseDataAdapter
|
||||
|
||||
|
||||
def test_process_data(sample_data):
|
||||
"""
|
||||
Test the ability of PulseDataAdapter.process_data() to process the
|
||||
raw data available in sample_data without missing any attributes.
|
||||
"""
|
||||
|
||||
pda = PulseDataAdapter(
|
||||
durable=False,
|
||||
logdir='logs',
|
||||
rawdata=False,
|
||||
outfile=None
|
||||
)
|
||||
|
||||
msg = Message()
|
||||
|
||||
for data in sample_data.raw_pulse_data:
|
||||
|
||||
data = pda.process_data(data, msg)
|
||||
|
||||
missing_attributes = pda.required_attributes.difference(
|
||||
set(data.keys())
|
||||
)
|
||||
|
||||
assert set() == missing_attributes
|
||||
|
||||
|
||||
class Message(object):
|
||||
"""Class that mimics the message object interface from
|
||||
mozilla pulse"""
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def ack(self):
|
||||
pass
|
||||
|
||||
|
||||
def test_load_data(sample_data, jm, mock_post_json_data,
|
||||
initial_data, mock_get_resultset):
|
||||
"""
|
||||
Test the ability of TreeherderPulseDataAdapter to load its transformed
|
||||
data through the restful api
|
||||
"""
|
||||
tpda = TreeherderPulseDataAdapter(
|
||||
loaddata=True,
|
||||
durable=False,
|
||||
logdir='logs',
|
||||
rawdata=False,
|
||||
outfile=None
|
||||
)
|
||||
|
||||
msg = Message()
|
||||
|
||||
for data in sample_data.raw_pulse_data[:1]:
|
||||
# change the branch (aka project) name on the raw data,
|
||||
# so that we can use the dataset created by jm
|
||||
data['payload']['build']['properties'][1][1] = jm.project
|
||||
data = tpda.process_data(data, msg)
|
||||
|
||||
stored_obj = jm.get_os_dhub().execute(
|
||||
proc="objectstore_test.selects.all")
|
||||
|
||||
jm.disconnect()
|
||||
|
||||
assert len(stored_obj) == 1
|
||||
|
||||
|
||||
def test_load_data_missing_attribute(sample_data, jm, mock_post_json_data, initial_data):
|
||||
"""
|
||||
Test that no objects is inserted in the object store if there is a missing attribute
|
||||
"""
|
||||
tpda = TreeherderPulseDataAdapter(
|
||||
loaddata=True,
|
||||
durable=False,
|
||||
logdir='logs',
|
||||
rawdata=False,
|
||||
outfile=None
|
||||
)
|
||||
|
||||
msg = Message()
|
||||
|
||||
for data in sample_data.raw_pulse_data[:1]:
|
||||
# change the branch (aka project) name on the raw data,
|
||||
# so that we can use the dataset created by jm
|
||||
|
||||
# delete the buildid attribute
|
||||
data['payload']['build']['properties'][4][0] = ""
|
||||
data['payload']['build']['properties'][1][1] = jm.project
|
||||
tpda.process_data(data, msg)
|
||||
|
||||
stored_obj = jm.get_os_dhub().execute(
|
||||
proc="objectstore_test.selects.all")
|
||||
|
||||
jm.disconnect()
|
||||
|
||||
assert len(stored_obj) == 0
|
||||
|
|
@ -16,7 +16,14 @@ from treeherder.etl import common
|
|||
from treeherder.etl import buildbot
|
||||
from treeherder.etl.mixins import OAuthLoaderMixin
|
||||
|
||||
|
||||
"""
|
||||
NOTE: This python library fails to load due to import conflicts with
|
||||
the new version of kombu that mozillapulse depends on. Leaving it
|
||||
here for now, the adaptation of the pulse data stream to the treeherder
|
||||
data structure might be useful in the future for ingesting data from pulse.
|
||||
At some point we should update mozillapulse or re-write this to work directly
|
||||
with rabbitmq or remove it entirely.
|
||||
"""
|
||||
|
||||
|
||||
class PulseDataAdapter(object):
|
||||
|
|
|
|||
|
|
@ -0,0 +1,130 @@
|
|||
# Makefile for Sphinx documentation
|
||||
#
|
||||
|
||||
# You can set these variables from the command line.
|
||||
SPHINXOPTS =
|
||||
SPHINXBUILD = sphinx-build
|
||||
PAPER =
|
||||
BUILDDIR = _build
|
||||
|
||||
# Internal variables.
|
||||
PAPEROPT_a4 = -D latex_paper_size=a4
|
||||
PAPEROPT_letter = -D latex_paper_size=letter
|
||||
ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
|
||||
|
||||
.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest
|
||||
|
||||
help:
|
||||
@echo "Please use \`make <target>' where <target> is one of"
|
||||
@echo " html to make standalone HTML files"
|
||||
@echo " dirhtml to make HTML files named index.html in directories"
|
||||
@echo " singlehtml to make a single large HTML file"
|
||||
@echo " pickle to make pickle files"
|
||||
@echo " json to make JSON files"
|
||||
@echo " htmlhelp to make HTML files and a HTML help project"
|
||||
@echo " qthelp to make HTML files and a qthelp project"
|
||||
@echo " devhelp to make HTML files and a Devhelp project"
|
||||
@echo " epub to make an epub"
|
||||
@echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
|
||||
@echo " latexpdf to make LaTeX files and run them through pdflatex"
|
||||
@echo " text to make text files"
|
||||
@echo " man to make manual pages"
|
||||
@echo " changes to make an overview of all changed/added/deprecated items"
|
||||
@echo " linkcheck to check all external links for integrity"
|
||||
@echo " doctest to run all doctests embedded in the documentation (if enabled)"
|
||||
|
||||
clean:
|
||||
-rm -rf $(BUILDDIR)/*
|
||||
|
||||
html:
|
||||
$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
|
||||
@echo
|
||||
@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
|
||||
|
||||
dirhtml:
|
||||
$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
|
||||
@echo
|
||||
@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
|
||||
|
||||
singlehtml:
|
||||
$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
|
||||
@echo
|
||||
@echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
|
||||
|
||||
pickle:
|
||||
$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
|
||||
@echo
|
||||
@echo "Build finished; now you can process the pickle files."
|
||||
|
||||
json:
|
||||
$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
|
||||
@echo
|
||||
@echo "Build finished; now you can process the JSON files."
|
||||
|
||||
htmlhelp:
|
||||
$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
|
||||
@echo
|
||||
@echo "Build finished; now you can run HTML Help Workshop with the" \
|
||||
".hhp project file in $(BUILDDIR)/htmlhelp."
|
||||
|
||||
qthelp:
|
||||
$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
|
||||
@echo
|
||||
@echo "Build finished; now you can run "qcollectiongenerator" with the" \
|
||||
".qhcp project file in $(BUILDDIR)/qthelp, like this:"
|
||||
@echo "# qcollectiongenerator $(BUILDDIR)/qthelp/django-browserid.qhcp"
|
||||
@echo "To view the help file:"
|
||||
@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/django-browserid.qhc"
|
||||
|
||||
devhelp:
|
||||
$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
|
||||
@echo
|
||||
@echo "Build finished."
|
||||
@echo "To view the help file:"
|
||||
@echo "# mkdir -p $$HOME/.local/share/devhelp/django-browserid"
|
||||
@echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/django-browserid"
|
||||
@echo "# devhelp"
|
||||
|
||||
epub:
|
||||
$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
|
||||
@echo
|
||||
@echo "Build finished. The epub file is in $(BUILDDIR)/epub."
|
||||
|
||||
latex:
|
||||
$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
|
||||
@echo
|
||||
@echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
|
||||
@echo "Run \`make' in that directory to run these through (pdf)latex" \
|
||||
"(use \`make latexpdf' here to do that automatically)."
|
||||
|
||||
latexpdf:
|
||||
$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
|
||||
@echo "Running LaTeX files through pdflatex..."
|
||||
make -C $(BUILDDIR)/latex all-pdf
|
||||
@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
|
||||
|
||||
text:
|
||||
$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
|
||||
@echo
|
||||
@echo "Build finished. The text files are in $(BUILDDIR)/text."
|
||||
|
||||
man:
|
||||
$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
|
||||
@echo
|
||||
@echo "Build finished. The manual pages are in $(BUILDDIR)/man."
|
||||
|
||||
changes:
|
||||
$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
|
||||
@echo
|
||||
@echo "The overview file is in $(BUILDDIR)/changes."
|
||||
|
||||
linkcheck:
|
||||
$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
|
||||
@echo
|
||||
@echo "Link check complete; look for any errors in the above output " \
|
||||
"or in $(BUILDDIR)/linkcheck/output.txt."
|
||||
|
||||
doctest:
|
||||
$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
|
||||
@echo "Testing of doctests in the sources finished, look at the " \
|
||||
"results in $(BUILDDIR)/doctest/output.txt."
|
||||
|
|
@ -0,0 +1,228 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# django-browserid documentation build configuration file, created by
|
||||
# sphinx-quickstart on Wed Mar 14 00:04:52 2012.
|
||||
#
|
||||
# This file is execfile()d with the current directory set to its containing dir.
|
||||
#
|
||||
# Note that not all possible configuration values are present in this
|
||||
# autogenerated file.
|
||||
#
|
||||
# All configuration values have a default; values that are commented out
|
||||
# serve to show the default.
|
||||
|
||||
import sys, os
|
||||
|
||||
# If extensions (or modules to document with autodoc) are in another directory,
|
||||
# add these directories to sys.path here. If the directory is relative to the
|
||||
# documentation root, use os.path.abspath to make it absolute, like shown here.
|
||||
#sys.path.insert(0, os.path.abspath('.'))
|
||||
|
||||
# Add project root where setup.py lives:
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))
|
||||
|
||||
# Set up Django so we can import django_browserid.
|
||||
os.environ['DJANGO_SETTINGS_MODULE'] = 'docs.settings'
|
||||
|
||||
# Now we can import django_browserid
|
||||
from django_browserid import __version__
|
||||
|
||||
# -- General configuration -----------------------------------------------------
|
||||
|
||||
# If your documentation needs a minimal Sphinx version, state it here.
|
||||
#needs_sphinx = '1.0'
|
||||
|
||||
# Add any Sphinx extension module names here, as strings. They can be extensions
|
||||
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
|
||||
extensions = ['sphinx.ext.autodoc']
|
||||
|
||||
# Add any paths that contain templates here, relative to this directory.
|
||||
templates_path = ['_templates']
|
||||
|
||||
# The suffix of source filenames.
|
||||
source_suffix = '.rst'
|
||||
|
||||
# The encoding of source files.
|
||||
#source_encoding = 'utf-8-sig'
|
||||
|
||||
# The master toctree document.
|
||||
master_doc = 'index'
|
||||
|
||||
# General information about the project.
|
||||
project = u'django-browserid'
|
||||
copyright = u'2012, Paul Osman, Michael Kelly'
|
||||
|
||||
# The version info for the project you're documenting, acts as replacement for
|
||||
# |version| and |release|, also used in various other places throughout the
|
||||
# built documents.
|
||||
#
|
||||
# The short X.Y version.
|
||||
version = __version__
|
||||
# The full version, including alpha/beta/rc tags.
|
||||
release = version # keeping things simple
|
||||
|
||||
# The language for content autogenerated by Sphinx. Refer to documentation
|
||||
# for a list of supported languages.
|
||||
#language = None
|
||||
|
||||
# There are two options for replacing |today|: either, you set today to some
|
||||
# non-false value, then it is used:
|
||||
#today = ''
|
||||
# Else, today_fmt is used as the format for a strftime call.
|
||||
#today_fmt = '%B %d, %Y'
|
||||
|
||||
# List of patterns, relative to source directory, that match files and
|
||||
# directories to ignore when looking for source files.
|
||||
exclude_patterns = ['_build']
|
||||
|
||||
# The reST default role (used for this markup: `text`) to use for all documents.
|
||||
#default_role = None
|
||||
|
||||
# If true, '()' will be appended to :func: etc. cross-reference text.
|
||||
#add_function_parentheses = True
|
||||
|
||||
# If true, the current module name will be prepended to all description
|
||||
# unit titles (such as .. function::).
|
||||
#add_module_names = True
|
||||
|
||||
# If true, sectionauthor and moduleauthor directives will be shown in the
|
||||
# output. They are ignored by default.
|
||||
#show_authors = False
|
||||
|
||||
# The name of the Pygments (syntax highlighting) style to use.
|
||||
pygments_style = 'sphinx'
|
||||
|
||||
# A list of ignored prefixes for module index sorting.
|
||||
#modindex_common_prefix = []
|
||||
|
||||
# Set the sort order of automatically documented members.
|
||||
autodoc_member_order = 'bysource'
|
||||
|
||||
|
||||
# -- Options for HTML output ---------------------------------------------------
|
||||
|
||||
# The theme to use for HTML and HTML Help pages. See the documentation for
|
||||
# a list of builtin themes.
|
||||
html_theme = 'default'
|
||||
|
||||
# Theme options are theme-specific and customize the look and feel of a theme
|
||||
# further. For a list of options available for each theme, see the
|
||||
# documentation.
|
||||
#html_theme_options = {}
|
||||
|
||||
# Add any paths that contain custom themes here, relative to this directory.
|
||||
#html_theme_path = []
|
||||
|
||||
# The name for this set of Sphinx documents. If None, it defaults to
|
||||
# "<project> v<release> documentation".
|
||||
#html_title = None
|
||||
|
||||
# A shorter title for the navigation bar. Default is the same as html_title.
|
||||
#html_short_title = None
|
||||
|
||||
# The name of an image file (relative to this directory) to place at the top
|
||||
# of the sidebar.
|
||||
#html_logo = None
|
||||
|
||||
# The name of an image file (within the static path) to use as favicon of the
|
||||
# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32
|
||||
# pixels large.
|
||||
#html_favicon = None
|
||||
|
||||
# Add any paths that contain custom static files (such as style sheets) here,
|
||||
# relative to this directory. They are copied after the builtin static files,
|
||||
# so a file named "default.css" will overwrite the builtin "default.css".
|
||||
html_static_path = []
|
||||
|
||||
# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
|
||||
# using the given strftime format.
|
||||
#html_last_updated_fmt = '%b %d, %Y'
|
||||
|
||||
# If true, SmartyPants will be used to convert quotes and dashes to
|
||||
# typographically correct entities.
|
||||
#html_use_smartypants = True
|
||||
|
||||
# Custom sidebar templates, maps document names to template names.
|
||||
#html_sidebars = {}
|
||||
|
||||
# Additional templates that should be rendered to pages, maps page names to
|
||||
# template names.
|
||||
#html_additional_pages = {}
|
||||
|
||||
# If false, no module index is generated.
|
||||
#html_domain_indices = True
|
||||
|
||||
# If false, no index is generated.
|
||||
#html_use_index = True
|
||||
|
||||
# If true, the index is split into individual pages for each letter.
|
||||
#html_split_index = False
|
||||
|
||||
# If true, links to the reST sources are added to the pages.
|
||||
#html_show_sourcelink = True
|
||||
|
||||
# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
|
||||
#html_show_sphinx = True
|
||||
|
||||
# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
|
||||
#html_show_copyright = True
|
||||
|
||||
# If true, an OpenSearch description file will be output, and all pages will
|
||||
# contain a <link> tag referring to it. The value of this option must be the
|
||||
# base URL from which the finished HTML is served.
|
||||
#html_use_opensearch = ''
|
||||
|
||||
# This is the file name suffix for HTML files (e.g. ".xhtml").
|
||||
#html_file_suffix = None
|
||||
|
||||
# Output file base name for HTML help builder.
|
||||
htmlhelp_basename = 'django-browseriddoc'
|
||||
|
||||
|
||||
# -- Options for LaTeX output --------------------------------------------------
|
||||
|
||||
# The paper size ('letter' or 'a4').
|
||||
#latex_paper_size = 'letter'
|
||||
|
||||
# The font size ('10pt', '11pt' or '12pt').
|
||||
#latex_font_size = '10pt'
|
||||
|
||||
# Grouping the document tree into LaTeX files. List of tuples
|
||||
# (source start file, target name, title, author, documentclass [howto/manual]).
|
||||
latex_documents = [
|
||||
('index', 'django-browserid.tex', u'django-browserid Documentation',
|
||||
u'Paul Osman, Michael Kelly', 'manual'),
|
||||
]
|
||||
|
||||
# The name of an image file (relative to this directory) to place at the top of
|
||||
# the title page.
|
||||
#latex_logo = None
|
||||
|
||||
# For "manual" documents, if this is true, then toplevel headings are parts,
|
||||
# not chapters.
|
||||
#latex_use_parts = False
|
||||
|
||||
# If true, show page references after internal links.
|
||||
#latex_show_pagerefs = False
|
||||
|
||||
# If true, show URL addresses after external links.
|
||||
#latex_show_urls = False
|
||||
|
||||
# Additional stuff for the LaTeX preamble.
|
||||
#latex_preamble = ''
|
||||
|
||||
# Documents to append as an appendix to all manuals.
|
||||
#latex_appendices = []
|
||||
|
||||
# If false, no module index is generated.
|
||||
#latex_domain_indices = True
|
||||
|
||||
|
||||
# -- Options for manual page output --------------------------------------------
|
||||
|
||||
# One entry per manual page. List of tuples
|
||||
# (source start file, name, description, authors, manual section).
|
||||
man_pages = [
|
||||
('index', 'django-browserid', u'django-browserid Documentation',
|
||||
[u'Paul Osman, Michael Kelly'], 1)
|
||||
]
|
||||
|
|
@ -0,0 +1,116 @@
|
|||
Advanced Usage
|
||||
==============
|
||||
|
||||
.. _auto-user:
|
||||
|
||||
Automatic Account Creation
|
||||
--------------------------
|
||||
|
||||
``django-browserid`` will automatically create a user account for new
|
||||
users. The user account will be created with the verified
|
||||
email returned from the BrowserID verification service, and a URL safe
|
||||
base64 encoded SHA1 of the email with the padding removed as the
|
||||
username.
|
||||
|
||||
To provide a customized username, you can provide a different
|
||||
algorithm via your settings.py::
|
||||
|
||||
# settings.py
|
||||
BROWSERID_CREATE_USER = True
|
||||
def username(email):
|
||||
return email.rsplit('@', 1)[0]
|
||||
BROWSERID_USERNAME_ALGO = username
|
||||
|
||||
You can can provide your own function to create users by setting
|
||||
``BROWSERID_CREATE_USER`` to a string path pointing to a function::
|
||||
|
||||
# module/util.py
|
||||
def create_user(email):
|
||||
return User.objects.create_user(email, email)
|
||||
|
||||
# settings.py
|
||||
BROWSERID_CREATE_USER = 'module.util.create_user'
|
||||
|
||||
You can disable account creation, but continue to use the
|
||||
``browserid_verify`` view to authenticate existing users with the
|
||||
following::
|
||||
|
||||
BROWSERID_CREATE_USER = False
|
||||
|
||||
|
||||
Custom Verification
|
||||
-------------------
|
||||
|
||||
If you want to customize the verification view, you can do so by subclassing
|
||||
:class:`django_browserid.views.Verify` and overriding the methods to insert your
|
||||
custom logic.
|
||||
|
||||
If you want complete control over account verification, you should create your
|
||||
own view and use :func:`django_browserid.RemoteVerifier` to manually verify a
|
||||
BrowserID assertion with something like the following:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
from django_browserid import get_audience, RemoteVerifier
|
||||
from django_browserid.forms import BrowserIDForm
|
||||
|
||||
|
||||
def myview(request):
|
||||
# ...
|
||||
if request.method == 'POST':
|
||||
form = BrowserIDForm(data=request.POST)
|
||||
if form.is_valid():
|
||||
verifier = RemoteVerifier()
|
||||
result = verifier.verify(form.cleaned_data['assertion'], get_audience(request))
|
||||
if result:
|
||||
# check for user account, create account for new users, etc
|
||||
user = my_get_or_create_user(result['email'])
|
||||
|
||||
See :func:`django_browserid.RemoteVerifier` for more info on how to use the
|
||||
verifier object.
|
||||
|
||||
|
||||
Custom User Model
|
||||
-----------------
|
||||
|
||||
Django 1.5 allows you to specify a custom model to use in place of the built-in
|
||||
User model with the ``AUTH_USER_MODEL`` setting. ``django-browserid`` supports
|
||||
custom User models, but you will most likely need to add a few extra
|
||||
customizations to make things work properly:
|
||||
|
||||
* ``django_browserid.BrowserIDBackend`` has three methods that deal with User
|
||||
objects: ``create_user``, ``get_user``, and ``filter_users_by_email``. You may
|
||||
have to subclass ``BrowserIDBackend`` and override these methods to work with
|
||||
your custom User class.
|
||||
|
||||
* ``browserid_login`` assumes that your custom User class has an attribute
|
||||
called ``email`` that contains the user's email address. You can either add
|
||||
an email field to your model, or add a `property`_ to the model that returns
|
||||
the user's email address.
|
||||
|
||||
.. _property: http://docs.python.org/2/library/functions.html#property
|
||||
|
||||
|
||||
Custom Verify view
|
||||
------------------
|
||||
|
||||
You can override which class is the view class for doing the
|
||||
verification. This can be useful in the case where you want to
|
||||
override certain methods that you need to work differently. To do
|
||||
this, set ``BROWSERID_VERIFY_CLASS`` to the path of your own preferred
|
||||
class.
|
||||
|
||||
Here's an example::
|
||||
|
||||
# settings.py
|
||||
BROWSERID_VERIFY_CLASS = 'myapp.MyVerifyClass'
|
||||
|
||||
# myapp.py
|
||||
from django_browserid.views import Verify
|
||||
class MyVerifyClass(Verify):
|
||||
@property
|
||||
def success_url(self):
|
||||
if self.user.username == 'Satan':
|
||||
return '/hell'
|
||||
# the default behaviour
|
||||
return getattr(settings, 'LOGIN_REDIRECT_URL', '/')
|
||||
|
|
@ -0,0 +1,68 @@
|
|||
API
|
||||
===
|
||||
|
||||
Template Helpers
|
||||
----------------
|
||||
|
||||
.. autofunction:: django_browserid.helpers.browserid_login
|
||||
|
||||
.. autofunction:: django_browserid.helpers.browserid_logout
|
||||
|
||||
.. autofunction:: django_browserid.helpers.browserid_js
|
||||
|
||||
.. autofunction:: django_browserid.helpers.browserid_css
|
||||
|
||||
|
||||
Admin Site
|
||||
----------
|
||||
|
||||
.. autoclass:: django_browserid.admin.BrowserIDAdminSite
|
||||
:members: include_password_form, copy_registry
|
||||
|
||||
.. autodata:: django_browserid.admin.site
|
||||
:annotation:
|
||||
|
||||
|
||||
Verification
|
||||
------------
|
||||
|
||||
.. autoclass:: django_browserid.RemoteVerifier
|
||||
:members: verify
|
||||
|
||||
.. autoclass:: django_browserid.MockVerifier
|
||||
:members: __init__, verify
|
||||
|
||||
.. autoclass:: django_browserid.VerificationResult
|
||||
:members: expires
|
||||
|
||||
.. autofunction:: django_browserid.get_audience
|
||||
|
||||
|
||||
Views
|
||||
-----
|
||||
|
||||
.. autoclass:: django_browserid.views.Verify
|
||||
:members:
|
||||
:show-inheritance:
|
||||
|
||||
.. autoclass:: django_browserid.views.Logout
|
||||
:members:
|
||||
:show-inheritance:
|
||||
|
||||
.. autoclass:: django_browserid.views.Info
|
||||
:members:
|
||||
:show-inheritance:
|
||||
|
||||
|
||||
Signals
|
||||
-------
|
||||
|
||||
.. automodule:: django_browserid.signals
|
||||
:members:
|
||||
|
||||
|
||||
Exceptions
|
||||
----------
|
||||
|
||||
.. autoexception:: django_browserid.base.BrowserIDException
|
||||
:members:
|
||||
|
|
@ -0,0 +1,52 @@
|
|||
JavaScript API
|
||||
==============
|
||||
|
||||
The JavaScript file that comes with django-browserid, ``browserid.js``,
|
||||
includes a a few public functions that are exposed through the
|
||||
``django_browserid`` global object.
|
||||
|
||||
Most functions return `jQuery Deferreds`_, which allow you to execute code
|
||||
asynchronously after the login or logout actions finish using a Promise
|
||||
interface.
|
||||
|
||||
.. note:: Most of the JavaScript API depends on the
|
||||
:class:`django_browserid.views.Info` view to retrieve some information from
|
||||
the backend. It assumes the Info view is available at ``/browserid/info/``
|
||||
on your site. If you are having issues with the JavaScript, make sure the
|
||||
Info view is available at that URL (typically by ensuring there is no regex
|
||||
in front of the django-browserid include in your ``urls.py``).
|
||||
|
||||
.. _`jQuery Deferreds`: https://api.jquery.com/jQuery.Deferred/
|
||||
|
||||
|
||||
.. js:function:: django_browserid.login([requestArgs])
|
||||
|
||||
Retrieve an assertion and use it to log the user into your site.
|
||||
|
||||
:param object requestArgs: Options to pass to `navigator.id.request`_.
|
||||
:returns: Deferred that resolves once the user has been logged in.
|
||||
|
||||
.. _`navigator.id.request`: https://developer.mozilla.org/en-US/docs/DOM/navigator.id.request
|
||||
|
||||
|
||||
.. js:function:: django_browserid.logout()
|
||||
|
||||
Log the user out of your site.
|
||||
|
||||
:returns: Deferred that resolves once the user has been logged out.
|
||||
|
||||
|
||||
.. js:function:: django_browserid.getAssertion([requestArgs])
|
||||
|
||||
Retrieve an assertion via BrowserID.
|
||||
|
||||
:returns: Deferred that resolves with the assertion once it is retrieved.
|
||||
|
||||
|
||||
.. js:function:: django_browserid.verifyAssertion(assertion)
|
||||
|
||||
Verify that the given assertion is valid, and log the user in.
|
||||
|
||||
:param string assertion: Assertion to verify.
|
||||
:returns: Deferred that resolves with the login view response once login is
|
||||
complete.
|
||||
|
|
@ -0,0 +1,105 @@
|
|||
Settings
|
||||
========
|
||||
|
||||
.. module:: django.conf.settings
|
||||
|
||||
|
||||
Core Settings
|
||||
-------------
|
||||
|
||||
.. data:: BROWSERID_AUDIENCES
|
||||
|
||||
**Default:** No default
|
||||
|
||||
List of audiences that your site accepts. An audience is the protocol,
|
||||
domain name, and (optionally) port that users access your site from. This
|
||||
list is used to determine the audience a user is part of (how they are
|
||||
accessing your site), which is used during verification to ensure that the
|
||||
assertion given to you by the user was intended for your site.
|
||||
|
||||
Without this, other sites that the user has authenticated with via Persona
|
||||
could use their assertions to impersonate the user on your site.
|
||||
|
||||
Note that this does not have to be a publicly accessible URL, so local URLs
|
||||
like ``http://localhost:8000`` or ``http://127.0.0.1`` are acceptable as
|
||||
long as they match what you are using to access your site.
|
||||
|
||||
|
||||
Redirect URLs
|
||||
-------------
|
||||
|
||||
.. data:: LOGIN_REDIRECT_URL
|
||||
|
||||
**Default:** ``'/accounts/profile'``
|
||||
|
||||
Path to redirect to on successful login. If you don't specify this, the
|
||||
default Django value will be used.
|
||||
|
||||
.. data:: LOGIN_REDIRECT_URL_FAILURE
|
||||
|
||||
**Default:** ``'/'``
|
||||
|
||||
Path to redirect to on an unsuccessful login attempt.
|
||||
|
||||
.. data:: LOGOUT_REDIRECT_URL
|
||||
|
||||
**Default:** ``'/'``
|
||||
|
||||
Path to redirect to on logout.
|
||||
|
||||
|
||||
Customizing the Login Popup
|
||||
---------------------------
|
||||
|
||||
.. data:: BROWSERID_REQUEST_ARGS
|
||||
|
||||
**Default:** ``{}``
|
||||
|
||||
Controls the arguments passed to ``navigator.id.request``, which are used to
|
||||
customize the login popup box. To see a list of valid keys and what they do,
|
||||
check out the `navigator.id.request documentation`_.
|
||||
|
||||
.. _navigator.id.request documentation: https://developer.mozilla.org/en-US/docs/DOM/navigator.id.request
|
||||
|
||||
|
||||
Customizing the Verify View
|
||||
---------------------------
|
||||
|
||||
.. data:: BROWSERID_VERIFY_VIEW
|
||||
|
||||
**Default:** ``django_browserid.views.Verify``
|
||||
|
||||
Allows you to substitute a custom class-based view for verifying assertions.
|
||||
For example, the string 'myapp.users.views.Verify' would import `Verify`
|
||||
from `myapp.users.views` and use it in place of the default view.
|
||||
|
||||
When using a custom view, it is generally a good idea to subclass the
|
||||
default Verify and override the methods you want to change.
|
||||
|
||||
.. data:: BROWSERID_CREATE_USER
|
||||
|
||||
**Default:** ``True``
|
||||
|
||||
If ``True`` or ``False``, enables or disables automatic user creation during
|
||||
authentication.
|
||||
|
||||
If set to a string, it is treated as an import path pointing to a custom
|
||||
user creation function. See :ref:`auto-user` for more information.
|
||||
|
||||
.. data:: BROWSERID_DISABLE_SANITY_CHECKS
|
||||
|
||||
**Default:** False
|
||||
|
||||
Controls whether the ``Verify`` view performs some helpful checks for common
|
||||
mistakes. Useful if you're getting warnings for things you know aren't
|
||||
errors.
|
||||
|
||||
|
||||
Using a Different Identity Provider
|
||||
-----------------------------------
|
||||
|
||||
.. data:: BROWSERID_SHIM
|
||||
|
||||
**Default:** 'https://login.persona.org/include.js'
|
||||
|
||||
The URL to use for the BrowserID JavaScript shim.
|
||||
|
|
@ -0,0 +1,85 @@
|
|||
Troubleshooting
|
||||
===============
|
||||
|
||||
CSP WARN: Directive "..." violated by https://browserid.org/include.js
|
||||
----------------------------------------------------------------------
|
||||
|
||||
This warning appears in the Error Console when your site uses
|
||||
`Content Security Policy`_ without making an exception for the persona.org
|
||||
external JavaScript include.
|
||||
|
||||
To fix this, include https://persona.org in your script-src and frame-src
|
||||
directive. If you're using the `django-csp`_ library, the following settings
|
||||
will work::
|
||||
|
||||
CSP_SCRIPT_SRC = ("'self'", 'https://login.persona.org')
|
||||
CSP_FRAME_SRC = ("'self'", 'https://login.persona.org')
|
||||
|
||||
.. _Content Security Policy: https://developer.mozilla.org/en/Security/CSP
|
||||
.. _django-csp: https://github.com/mozilla/django-csp
|
||||
|
||||
|
||||
Login fails silently due to SESSION_COOKIE_SECURE
|
||||
-------------------------------------------------
|
||||
|
||||
If you try to login on a local instance of a site and login fails without any
|
||||
error (typically redirecting you back to the login page), check to see if you've
|
||||
set `SESSION_COOKIE_SECURE` to True in your settings.
|
||||
|
||||
`SESSION_COOKIE_SECURE` controls if the `secure` flag is set on the session
|
||||
cookie. If set to True on a local instance of a site that does not use HTTPS,
|
||||
the session cookie won't be sent by your browser because you're using an HTTP
|
||||
connection.
|
||||
|
||||
The solution is to set `SESSION_COOKIE_SECURE` to False on your local instance,
|
||||
typically by adding it to `settings/local.py`::
|
||||
|
||||
SESSION_COOKIE_SECURE = False
|
||||
|
||||
|
||||
Login fails silently due to cache issues
|
||||
----------------------------------------
|
||||
|
||||
Another possible cause of silently failing logins is an issue with having no
|
||||
cache configured locally. Several projects (especially projects based on
|
||||
playdoh_, which uses `django-session-csrf`_) store session info in the cache
|
||||
rather than the database, and if your local instance has no cache configured,
|
||||
the session information will not be stored and login will fail silently.
|
||||
|
||||
To solve this issue, you should configure your local instance to use an
|
||||
in-memory cache with the following in your local settings file::
|
||||
|
||||
CACHES = {
|
||||
'default': {
|
||||
'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
|
||||
'LOCATION': 'unique-snowflake'
|
||||
}
|
||||
}
|
||||
|
||||
.. _playdoh: https://github.com/mozilla/playdoh
|
||||
.. _django-session-csrf: https://github.com/mozilla/django-session-csrf
|
||||
|
||||
Your website uses HTTPS but django-browserid thinks it's http
|
||||
-------------------------------------------------------------
|
||||
|
||||
This will happen if you are using django-browserid behind a load balancer
|
||||
that is terminating your SSL connections, like nginx.
|
||||
The ``request.is_secure()`` method will return false,
|
||||
unless you tell Django that it is being served over https.
|
||||
You will need to configure Django's `SECURE_PROXY_SSL_HEADER`_
|
||||
with a value provided by your load balancer.
|
||||
An example configuration might look like this::
|
||||
|
||||
# settings.py
|
||||
SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTOCOL', 'https')
|
||||
|
||||
# nginx
|
||||
location / {
|
||||
proxy_pass http://127.0.0.1:8000;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Protocol https; # Tell django we're using https
|
||||
}
|
||||
|
||||
.. _SECURE_PROXY_SSL_HEADER: https://docs.djangoproject.com/en/dev/ref/settings/#secure-proxy-ssl-header
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
Authors
|
||||
=======
|
||||
|
||||
|
||||
.. include:: ../../AUTHORS.rst
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
Changelog
|
||||
=========
|
||||
|
||||
|
||||
.. include:: ../../CHANGELOG.rst
|
||||
|
|
@ -0,0 +1,32 @@
|
|||
===============
|
||||
Developer Setup
|
||||
===============
|
||||
|
||||
Check out the code from the `github project`_::
|
||||
|
||||
git clone git://github.com/mozilla/django-browserid.git
|
||||
cd django-browserid
|
||||
|
||||
Create a `virtualenv`_ (the example here uses `virtualenvwrapper`_)
|
||||
and install all development packages::
|
||||
|
||||
mkvirtualenv django-browserid
|
||||
pip install -r requirements.txt
|
||||
|
||||
Here is how to run the test suite::
|
||||
|
||||
python runtests.py
|
||||
|
||||
You can also run the tests in all the Python/Django environment
|
||||
combinations using tox::
|
||||
|
||||
pip install tox
|
||||
tox
|
||||
|
||||
Here is how to build the documentation::
|
||||
|
||||
make -C docs/ html
|
||||
|
||||
.. _`github project`: https://github.com/mozilla/django-browserid
|
||||
.. _virtualenv: http://www.virtualenv.org/
|
||||
.. _virtualenvwrapper: http://virtualenvwrapper.readthedocs.org/
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
django-browserid
|
||||
================
|
||||
|
||||
django-browserid is a library that integrates BrowserID_ authentication into
|
||||
Django_. By default it relies on the Persona_ Identity Provider.
|
||||
|
||||
django-browserid provides an authentication backend, ``BrowserIDBackend``, that
|
||||
verifies BrowserID assertions using a BrowserID verification service and
|
||||
authenticates users. It also provides ``verify``, which lets you build more
|
||||
complex authentication systems based on BrowserID.
|
||||
|
||||
django-browserid is a work in progress. Contributions are welcome. Feel free
|
||||
to fork_ and contribute!
|
||||
|
||||
.. _Django: http://www.djangoproject.com/
|
||||
.. _BrowserID: https://developer.mozilla.org/en-US/docs/Persona
|
||||
.. _Persona: https://persona.org
|
||||
.. _fork: https://github.com/mozilla/django-browserid
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
|
||||
setup
|
||||
details/advanced
|
||||
details/settings
|
||||
details/troubleshooting
|
||||
details/api
|
||||
details/js_api
|
||||
|
||||
Developer Guide
|
||||
---------------
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 1
|
||||
|
||||
dev/devsetup
|
||||
dev/changelog
|
||||
dev/authors
|
||||
|
|
@ -0,0 +1,170 @@
|
|||
@ECHO OFF
|
||||
|
||||
REM Command file for Sphinx documentation
|
||||
|
||||
if "%SPHINXBUILD%" == "" (
|
||||
set SPHINXBUILD=sphinx-build
|
||||
)
|
||||
set BUILDDIR=_build
|
||||
set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% .
|
||||
if NOT "%PAPER%" == "" (
|
||||
set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS%
|
||||
)
|
||||
|
||||
if "%1" == "" goto help
|
||||
|
||||
if "%1" == "help" (
|
||||
:help
|
||||
echo.Please use `make ^<target^>` where ^<target^> is one of
|
||||
echo. html to make standalone HTML files
|
||||
echo. dirhtml to make HTML files named index.html in directories
|
||||
echo. singlehtml to make a single large HTML file
|
||||
echo. pickle to make pickle files
|
||||
echo. json to make JSON files
|
||||
echo. htmlhelp to make HTML files and a HTML help project
|
||||
echo. qthelp to make HTML files and a qthelp project
|
||||
echo. devhelp to make HTML files and a Devhelp project
|
||||
echo. epub to make an epub
|
||||
echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter
|
||||
echo. text to make text files
|
||||
echo. man to make manual pages
|
||||
echo. changes to make an overview over all changed/added/deprecated items
|
||||
echo. linkcheck to check all external links for integrity
|
||||
echo. doctest to run all doctests embedded in the documentation if enabled
|
||||
goto end
|
||||
)
|
||||
|
||||
if "%1" == "clean" (
|
||||
for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i
|
||||
del /q /s %BUILDDIR%\*
|
||||
goto end
|
||||
)
|
||||
|
||||
if "%1" == "html" (
|
||||
%SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html
|
||||
if errorlevel 1 exit /b 1
|
||||
echo.
|
||||
echo.Build finished. The HTML pages are in %BUILDDIR%/html.
|
||||
goto end
|
||||
)
|
||||
|
||||
if "%1" == "dirhtml" (
|
||||
%SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml
|
||||
if errorlevel 1 exit /b 1
|
||||
echo.
|
||||
echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml.
|
||||
goto end
|
||||
)
|
||||
|
||||
if "%1" == "singlehtml" (
|
||||
%SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml
|
||||
if errorlevel 1 exit /b 1
|
||||
echo.
|
||||
echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml.
|
||||
goto end
|
||||
)
|
||||
|
||||
if "%1" == "pickle" (
|
||||
%SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle
|
||||
if errorlevel 1 exit /b 1
|
||||
echo.
|
||||
echo.Build finished; now you can process the pickle files.
|
||||
goto end
|
||||
)
|
||||
|
||||
if "%1" == "json" (
|
||||
%SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json
|
||||
if errorlevel 1 exit /b 1
|
||||
echo.
|
||||
echo.Build finished; now you can process the JSON files.
|
||||
goto end
|
||||
)
|
||||
|
||||
if "%1" == "htmlhelp" (
|
||||
%SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp
|
||||
if errorlevel 1 exit /b 1
|
||||
echo.
|
||||
echo.Build finished; now you can run HTML Help Workshop with the ^
|
||||
.hhp project file in %BUILDDIR%/htmlhelp.
|
||||
goto end
|
||||
)
|
||||
|
||||
if "%1" == "qthelp" (
|
||||
%SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp
|
||||
if errorlevel 1 exit /b 1
|
||||
echo.
|
||||
echo.Build finished; now you can run "qcollectiongenerator" with the ^
|
||||
.qhcp project file in %BUILDDIR%/qthelp, like this:
|
||||
echo.^> qcollectiongenerator %BUILDDIR%\qthelp\django-browserid.qhcp
|
||||
echo.To view the help file:
|
||||
echo.^> assistant -collectionFile %BUILDDIR%\qthelp\django-browserid.ghc
|
||||
goto end
|
||||
)
|
||||
|
||||
if "%1" == "devhelp" (
|
||||
%SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp
|
||||
if errorlevel 1 exit /b 1
|
||||
echo.
|
||||
echo.Build finished.
|
||||
goto end
|
||||
)
|
||||
|
||||
if "%1" == "epub" (
|
||||
%SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub
|
||||
if errorlevel 1 exit /b 1
|
||||
echo.
|
||||
echo.Build finished. The epub file is in %BUILDDIR%/epub.
|
||||
goto end
|
||||
)
|
||||
|
||||
if "%1" == "latex" (
|
||||
%SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex
|
||||
if errorlevel 1 exit /b 1
|
||||
echo.
|
||||
echo.Build finished; the LaTeX files are in %BUILDDIR%/latex.
|
||||
goto end
|
||||
)
|
||||
|
||||
if "%1" == "text" (
|
||||
%SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text
|
||||
if errorlevel 1 exit /b 1
|
||||
echo.
|
||||
echo.Build finished. The text files are in %BUILDDIR%/text.
|
||||
goto end
|
||||
)
|
||||
|
||||
if "%1" == "man" (
|
||||
%SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man
|
||||
if errorlevel 1 exit /b 1
|
||||
echo.
|
||||
echo.Build finished. The manual pages are in %BUILDDIR%/man.
|
||||
goto end
|
||||
)
|
||||
|
||||
if "%1" == "changes" (
|
||||
%SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes
|
||||
if errorlevel 1 exit /b 1
|
||||
echo.
|
||||
echo.The overview file is in %BUILDDIR%/changes.
|
||||
goto end
|
||||
)
|
||||
|
||||
if "%1" == "linkcheck" (
|
||||
%SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck
|
||||
if errorlevel 1 exit /b 1
|
||||
echo.
|
||||
echo.Link check complete; look for any errors in the above output ^
|
||||
or in %BUILDDIR%/linkcheck/output.txt.
|
||||
goto end
|
||||
)
|
||||
|
||||
if "%1" == "doctest" (
|
||||
%SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest
|
||||
if errorlevel 1 exit /b 1
|
||||
echo.
|
||||
echo.Testing of doctests in the sources finished, look at the ^
|
||||
results in %BUILDDIR%/doctest/output.txt.
|
||||
goto end
|
||||
)
|
||||
|
||||
:end
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
# Django settings file for building the documentation.
|
||||
SECRET_KEY = 'asdf'
|
||||
|
|
@ -0,0 +1,281 @@
|
|||
Setup
|
||||
=====
|
||||
|
||||
Installation
|
||||
------------
|
||||
You can use pip to install django-browserid and requirements::
|
||||
|
||||
pip install django-browserid
|
||||
|
||||
|
||||
Configuration
|
||||
-------------
|
||||
To use ``django-browserid``, you'll need to make a few changes to your
|
||||
``settings.py`` file::
|
||||
|
||||
# Add 'django_browserid' to INSTALLED_APPS.
|
||||
INSTALLED_APPS = (
|
||||
# ...
|
||||
'django.contrib.auth',
|
||||
'django_browserid', # Load after auth
|
||||
# ...
|
||||
)
|
||||
|
||||
# Add the django_browserid authentication backend.
|
||||
AUTHENTICATION_BACKENDS = (
|
||||
# ...
|
||||
'django.contrib.auth.backends.ModelBackend', # required for admin
|
||||
'django_browserid.auth.BrowserIDBackend',
|
||||
# ...
|
||||
)
|
||||
|
||||
# Specify audiences (protocol, domain, port) that your site will handle.
|
||||
# Note! This is only needed if DEBUG = False
|
||||
BROWSERID_AUDIENCES = ['http://example.com:8000', 'https://my.example.com']
|
||||
|
||||
.. note:: For security reasons, it is *very important* that you set
|
||||
``BROWSERID_AUDIENCES`` correctly. If it is incorrect, other sites could use
|
||||
assertions to impersonate users on your own site.
|
||||
|
||||
Next, edit your ``urls.py`` file and add the following::
|
||||
|
||||
urlpatterns = patterns('',
|
||||
# ...
|
||||
(r'', include('django_browserid.urls')),
|
||||
# ...
|
||||
)
|
||||
|
||||
.. note:: The django-browserid urlconf *must not* have a regex with the
|
||||
include. Use a blank string, as shown above.
|
||||
|
||||
You can also set the following optional settings in ``settings.py``::
|
||||
|
||||
# Path to redirect to on successful login.
|
||||
LOGIN_REDIRECT_URL = '/'
|
||||
|
||||
# Path to redirect to on unsuccessful login attempt.
|
||||
LOGIN_REDIRECT_URL_FAILURE = '/'
|
||||
|
||||
# Path to redirect to on logout.
|
||||
LOGOUT_REDIRECT_URL = '/'
|
||||
|
||||
Finally, you'll need to add the login button to your templates. There are three
|
||||
things you will need to add to your templates:
|
||||
|
||||
1. ``{% browserid_js %}``: Outputs the ``<script>`` tags for the button
|
||||
JavaScript. Must be somewhere on the page, typically at the bottom right
|
||||
before the ``</body>`` tag to allow the page to visibly load before
|
||||
executing.
|
||||
|
||||
2. ``{% browserid_css %}``: Outputs ``<link>`` tags for optional CSS that
|
||||
styles login buttons to match Persona.
|
||||
|
||||
3. ``{% browserid_login %}`` and ``{% browserid_logout %}``: Outputs the HTML
|
||||
for the login and logout buttons.
|
||||
|
||||
A complete example:
|
||||
|
||||
.. code-block:: html+django
|
||||
|
||||
{% load browserid %}
|
||||
<html>
|
||||
<head>
|
||||
{% browserid_css %}
|
||||
</head>
|
||||
<body>
|
||||
<header>
|
||||
<h1>My Site</h1>
|
||||
<div class="authentication">
|
||||
{% if user.is_authenticated %}
|
||||
{% browserid_logout text='Logout' %}
|
||||
{% else %}
|
||||
{% browserid_login text='Login' color='dark' %}
|
||||
{% endif %}
|
||||
</div>
|
||||
</header>
|
||||
<article>
|
||||
<p>Welcome to my site!</p>
|
||||
</article>
|
||||
<script src="http://code.jquery.com/jquery-1.9.1.min.js"></script>
|
||||
{% browserid_js %}
|
||||
</body>
|
||||
</html>
|
||||
|
||||
If you're using `Jinja2`_ as your templating system, you can use the functions
|
||||
passed to your template by the context processor:
|
||||
|
||||
.. code-block:: html+jinja
|
||||
|
||||
<html>
|
||||
<head>
|
||||
{{ browserid_css() }}
|
||||
</head>
|
||||
<body>
|
||||
<header>
|
||||
<h1>My Site</h1>
|
||||
<div class="authentication">
|
||||
{% if user.is_authenticated() %}
|
||||
{{ browserid_logout(text='Logout') }}
|
||||
{% else %}
|
||||
{{ browserid_login(text='Login', color='dark') }}
|
||||
{% endif %}
|
||||
</div>
|
||||
</header>
|
||||
<article>
|
||||
<p>Welcome to my site!</p>
|
||||
</article>
|
||||
<script src="http://code.jquery.com/jquery-1.9.1.min.js"></script>
|
||||
{{ browserid_js() }}
|
||||
</body>
|
||||
</html>
|
||||
|
||||
.. note:: The JavaScript assumes you have `jQuery`_ 1.7 or higher on your site.
|
||||
|
||||
.. note:: For more information about the template helper functions, check out
|
||||
the :doc:`details/api` document.
|
||||
|
||||
.. _jQuery: http://jquery.com/
|
||||
.. _Jinja2: http://jinja.pocoo.org/
|
||||
.. _`Context Processor documentation`: https://docs.djangoproject.com/en/dev/ref/settings/#template-context-processors
|
||||
|
||||
|
||||
BrowserID in the Django Admin
|
||||
-----------------------------
|
||||
You can add support for logging in to the Django admin interface with BrowserID
|
||||
by using :data:`django_browserid.admin.site` instead of
|
||||
:data:`django.contrib.admin.site`. In your ``admin.py`` files, register
|
||||
ModelAdmin classes with the django-browserid site:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
from django.contrib import admin
|
||||
|
||||
from django_browserid.admin import site as browserid_admin
|
||||
|
||||
from myapp.foo.models import Bar
|
||||
|
||||
|
||||
class BarAdmin(admin.ModelAdmin):
|
||||
pass
|
||||
browserid_admin.register(Bar, BarAdmin)
|
||||
|
||||
Then, use the django-browserid admin site in your ``urls.py`` as well:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
from django.conf.urls import patterns, include, url
|
||||
|
||||
# Autodiscover admin.py files in your project.
|
||||
from django.contrib import admin
|
||||
admin.autodiscover()
|
||||
|
||||
# copy_registry copies ModelAdmins registered with the default site, like
|
||||
# the built-in Django User model.
|
||||
from django_browserid.admin import site as browserid_admin
|
||||
browserid_admin.copy_registry(admin.site)
|
||||
|
||||
urlpatterns = patterns('',
|
||||
# ...
|
||||
url(r'^admin/', include(browserid_admin.urls)),
|
||||
)
|
||||
|
||||
See :class:`django_browserid.admin.BrowserIDAdminSite` for details on how to
|
||||
customize the login page, such as including a normal login form.
|
||||
|
||||
|
||||
Deploying to Production
|
||||
-----------------------
|
||||
There are a few changes you need to make when deploying your app to production:
|
||||
|
||||
- BrowserID uses an assertion and an audience to verify the user. The
|
||||
``BROWSERID_AUDIENCES`` setting is used to determine the audience. For
|
||||
security reasons, it is *very important* that you set ``BROWSERID_AUDIENCES``
|
||||
correctly.
|
||||
|
||||
``BROWSERID_AUDIENCES`` should be set to the domains and protocols
|
||||
users will use to access your site, such as``https://affiliates.mozilla.org``.
|
||||
This URL does not have to be publicly available, however, so sites limited to
|
||||
a certain network can still use django-browserid.
|
||||
|
||||
|
||||
Static Files
|
||||
------------
|
||||
``browserid_js`` and ``browserid_css`` the Django `staticfiles`_ app to serve
|
||||
the static files for the buttons. If you don't want to use the static files
|
||||
framework, you'll need to include the JavaScript and CSS manually on any page
|
||||
you use the ``browserid_button`` function.
|
||||
|
||||
For ``browserid_js`` the files needed are the Persona JavaScript shim, which
|
||||
should be loaded from
|
||||
``https://login.persona.org/include.js`` in a script tag, and
|
||||
``django_browserid/static/browserid/browserid.js``, which is part of the
|
||||
django-browserid library.
|
||||
|
||||
For ``browserid_css`` the file needed is
|
||||
``django_browserid/static/browserid/persona-buttons.css``, which is also part of
|
||||
the django-browserid library.
|
||||
|
||||
.. _staticfiles: https://docs.djangoproject.com/en/dev/howto/static-files/
|
||||
|
||||
|
||||
Content Security Policy
|
||||
-----------------------
|
||||
If your site uses `Content Security Policy`_, you will have to add directives
|
||||
to allow the external persona.org JavaScript, as well as an iframe used as part
|
||||
of the login process.
|
||||
|
||||
If you're using `django-csp`_, the following settings will work::
|
||||
|
||||
CSP_SCRIPT_SRC = ("'self'", 'https://login.persona.org')
|
||||
CSP_FRAME_SRC = ("'self'", 'https://login.persona.org')
|
||||
|
||||
.. _Content Security Policy: https://developer.mozilla.org/en/Security/CSP
|
||||
.. _django-csp: https://github.com/mozilla/django-csp
|
||||
|
||||
|
||||
Alternate Template Languages (Jingo/Jinja)
|
||||
------------------------------------------
|
||||
If you are using a library like `Jingo`_ in order to use a template language
|
||||
besides the Django template language, you may need to configure the library to
|
||||
use the Django template language for django-browserid templates. With Jingo,
|
||||
you can do this using the ``JINGO_EXCLUDE_APPS`` setting::
|
||||
|
||||
JINGO_EXCLUDE_APPS = ('browserid',)
|
||||
|
||||
.. _Jingo: https://github.com/jbalogh/jingo
|
||||
|
||||
|
||||
Troubleshooting Issues
|
||||
----------------------
|
||||
If you run into any issues while setting up django-browserid, try the following
|
||||
steps:
|
||||
|
||||
1. Check for any warnings in the server log. You may have to edit your
|
||||
development server's logging settings to output ``django_browserid`` log
|
||||
entries. Here's an example ``LOGGING`` setup to start with::
|
||||
|
||||
LOGGING = {
|
||||
'version': 1,
|
||||
'handlers': {
|
||||
'console':{
|
||||
'level': 'DEBUG',
|
||||
'class': 'logging.StreamHandler'
|
||||
},
|
||||
},
|
||||
'loggers': {
|
||||
'django_browserid': {
|
||||
'handlers': ['console'],
|
||||
'level': 'DEBUG',
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
2. Check the :doc:`details/troubleshooting` document for commonly-reported
|
||||
issues.
|
||||
|
||||
3. Ask for help in the `#webdev`_ channel on irc.mozilla.org.
|
||||
|
||||
4. Post an issue on the `django-browserid Issue Tracker`_.
|
||||
|
||||
.. _#webdev: http://chat.mibbit.com/?channel=%23chat&server=irc.mozilla.org
|
||||
.. _django-browserid Issue Tracker: https://github.com/mozilla/django-browserid/issues
|
||||
|
|
@ -20,7 +20,7 @@ from jsonschema.validators import (
|
|||
)
|
||||
|
||||
|
||||
__version__ = "2.3.0"
|
||||
__version__ = "2.4.0"
|
||||
|
||||
|
||||
# flake8: noqa
|
||||
|
|
|
|||
|
|
@ -94,10 +94,10 @@ def maximum(validator, maximum, instance, schema):
|
|||
return
|
||||
|
||||
if schema.get("exclusiveMaximum", False):
|
||||
failed = float(instance) >= maximum
|
||||
failed = instance >= maximum
|
||||
cmp = "greater than or equal to"
|
||||
else:
|
||||
failed = float(instance) > maximum
|
||||
failed = instance > maximum
|
||||
cmp = "greater than"
|
||||
|
||||
if failed:
|
||||
|
|
|
|||
|
|
@ -40,8 +40,8 @@ parser.add_argument(
|
|||
"-V", "--validator",
|
||||
type=_namedAnyWithDefault,
|
||||
help="the fully qualified object name of a validator to use, or, for "
|
||||
"validators that are registered with jsonschema, simply the name "
|
||||
"of the class.",
|
||||
"validators that are registered with jsonschema, simply the name "
|
||||
"of the class.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"schema",
|
||||
|
|
|
|||
|
|
@ -18,14 +18,19 @@ def fake_validator(*errors):
|
|||
|
||||
|
||||
class TestParser(unittest.TestCase):
|
||||
|
||||
FakeValidator = fake_validator()
|
||||
|
||||
def setUp(self):
|
||||
self.open = mock.mock_open(read_data='{}')
|
||||
patch = mock.patch.object(cli, "open", self.open, create=True)
|
||||
patch.start()
|
||||
self.addCleanup(patch.stop)
|
||||
mock_open = mock.mock_open()
|
||||
patch_open = mock.patch.object(cli, "open", mock_open, create=True)
|
||||
patch_open.start()
|
||||
self.addCleanup(patch_open.stop)
|
||||
|
||||
mock_json_load = mock.Mock()
|
||||
mock_json_load.return_value = {}
|
||||
patch_json_load = mock.patch("json.load")
|
||||
patch_json_load.start()
|
||||
self.addCleanup(patch_json_load.stop)
|
||||
|
||||
def test_find_validator_by_fully_qualified_object_name(self):
|
||||
arguments = cli.parse_args(
|
||||
|
|
@ -54,10 +59,10 @@ class TestCLI(unittest.TestCase):
|
|||
stdout, stderr = StringIO(), StringIO()
|
||||
exit_code = cli.run(
|
||||
{
|
||||
"validator" : fake_validator(),
|
||||
"schema" : {},
|
||||
"instances" : [1],
|
||||
"error_format" : "{error.message}",
|
||||
"validator": fake_validator(),
|
||||
"schema": {},
|
||||
"instances": [1],
|
||||
"error_format": "{error.message}",
|
||||
},
|
||||
stdout=stdout,
|
||||
stderr=stderr,
|
||||
|
|
@ -71,10 +76,10 @@ class TestCLI(unittest.TestCase):
|
|||
stdout, stderr = StringIO(), StringIO()
|
||||
exit_code = cli.run(
|
||||
{
|
||||
"validator" : fake_validator([error]),
|
||||
"schema" : {},
|
||||
"instances" : [1],
|
||||
"error_format" : "{error.instance} - {error.message}",
|
||||
"validator": fake_validator([error]),
|
||||
"schema": {},
|
||||
"instances": [1],
|
||||
"error_format": "{error.instance} - {error.message}",
|
||||
},
|
||||
stdout=stdout,
|
||||
stderr=stderr,
|
||||
|
|
@ -92,10 +97,10 @@ class TestCLI(unittest.TestCase):
|
|||
stdout, stderr = StringIO(), StringIO()
|
||||
exit_code = cli.run(
|
||||
{
|
||||
"validator" : fake_validator(first_errors, second_errors),
|
||||
"schema" : {},
|
||||
"instances" : [1, 2],
|
||||
"error_format" : "{error.instance} - {error.message}\t",
|
||||
"validator": fake_validator(first_errors, second_errors),
|
||||
"schema": {},
|
||||
"instances": [1, 2],
|
||||
"error_format": "{error.instance} - {error.message}\t",
|
||||
},
|
||||
stdout=stdout,
|
||||
stderr=stderr,
|
||||
|
|
|
|||
|
|
@ -42,8 +42,8 @@ is at <http://python-requests.org>.
|
|||
"""
|
||||
|
||||
__title__ = 'requests'
|
||||
__version__ = '2.3.0'
|
||||
__build__ = 0x020300
|
||||
__version__ = '2.4.1'
|
||||
__build__ = 0x020401
|
||||
__author__ = 'Kenneth Reitz'
|
||||
__license__ = 'Apache 2.0'
|
||||
__copyright__ = 'Copyright 2014 Kenneth Reitz'
|
||||
|
|
|
|||
|
|
@ -11,20 +11,24 @@ and maintain connections.
|
|||
import socket
|
||||
|
||||
from .models import Response
|
||||
from .packages.urllib3 import Retry
|
||||
from .packages.urllib3.poolmanager import PoolManager, proxy_from_url
|
||||
from .packages.urllib3.response import HTTPResponse
|
||||
from .packages.urllib3.util import Timeout as TimeoutSauce
|
||||
from .compat import urlparse, basestring, urldefrag, unquote
|
||||
from .compat import urlparse, basestring, urldefrag
|
||||
from .utils import (DEFAULT_CA_BUNDLE_PATH, get_encoding_from_headers,
|
||||
prepend_scheme_if_needed, get_auth_from_url)
|
||||
from .structures import CaseInsensitiveDict
|
||||
from .packages.urllib3.exceptions import MaxRetryError
|
||||
from .packages.urllib3.exceptions import TimeoutError
|
||||
from .packages.urllib3.exceptions import SSLError as _SSLError
|
||||
from .packages.urllib3.exceptions import ConnectTimeoutError
|
||||
from .packages.urllib3.exceptions import HTTPError as _HTTPError
|
||||
from .packages.urllib3.exceptions import MaxRetryError
|
||||
from .packages.urllib3.exceptions import ProxyError as _ProxyError
|
||||
from .packages.urllib3.exceptions import ProtocolError
|
||||
from .packages.urllib3.exceptions import ReadTimeoutError
|
||||
from .packages.urllib3.exceptions import SSLError as _SSLError
|
||||
from .cookies import extract_cookies_to_jar
|
||||
from .exceptions import ConnectionError, Timeout, SSLError, ProxyError
|
||||
from .exceptions import (ConnectionError, ConnectTimeout, ReadTimeout, SSLError,
|
||||
ProxyError)
|
||||
from .auth import _basic_auth_str
|
||||
|
||||
DEFAULT_POOLBLOCK = False
|
||||
|
|
@ -101,14 +105,17 @@ class HTTPAdapter(BaseAdapter):
|
|||
self.init_poolmanager(self._pool_connections, self._pool_maxsize,
|
||||
block=self._pool_block)
|
||||
|
||||
def init_poolmanager(self, connections, maxsize, block=DEFAULT_POOLBLOCK):
|
||||
"""Initializes a urllib3 PoolManager. This method should not be called
|
||||
from user code, and is only exposed for use when subclassing the
|
||||
def init_poolmanager(self, connections, maxsize, block=DEFAULT_POOLBLOCK, **pool_kwargs):
|
||||
"""Initializes a urllib3 PoolManager.
|
||||
|
||||
This method should not be called from user code, and is only
|
||||
exposed for use when subclassing the
|
||||
:class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
|
||||
|
||||
:param connections: The number of urllib3 connection pools to cache.
|
||||
:param maxsize: The maximum number of connections to save in the pool.
|
||||
:param block: Block when no free connections are available.
|
||||
:param pool_kwargs: Extra keyword arguments used to initialize the Pool Manager.
|
||||
"""
|
||||
# save these values for pickling
|
||||
self._pool_connections = connections
|
||||
|
|
@ -116,7 +123,30 @@ class HTTPAdapter(BaseAdapter):
|
|||
self._pool_block = block
|
||||
|
||||
self.poolmanager = PoolManager(num_pools=connections, maxsize=maxsize,
|
||||
block=block)
|
||||
block=block, **pool_kwargs)
|
||||
|
||||
def proxy_manager_for(self, proxy, **proxy_kwargs):
|
||||
"""Return urllib3 ProxyManager for the given proxy.
|
||||
|
||||
This method should not be called from user code, and is only
|
||||
exposed for use when subclassing the
|
||||
:class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
|
||||
|
||||
:param proxy: The proxy to return a urllib3 ProxyManager for.
|
||||
:param proxy_kwargs: Extra keyword arguments used to configure the Proxy Manager.
|
||||
:returns: ProxyManager
|
||||
"""
|
||||
if not proxy in self.proxy_manager:
|
||||
proxy_headers = self.proxy_headers(proxy)
|
||||
self.proxy_manager[proxy] = proxy_from_url(
|
||||
proxy,
|
||||
proxy_headers=proxy_headers,
|
||||
num_pools=self._pool_connections,
|
||||
maxsize=self._pool_maxsize,
|
||||
block=self._pool_block,
|
||||
**proxy_kwargs)
|
||||
|
||||
return self.proxy_manager[proxy]
|
||||
|
||||
def cert_verify(self, conn, url, verify, cert):
|
||||
"""Verify a SSL certificate. This method should not be called from user
|
||||
|
|
@ -204,17 +234,8 @@ class HTTPAdapter(BaseAdapter):
|
|||
|
||||
if proxy:
|
||||
proxy = prepend_scheme_if_needed(proxy, 'http')
|
||||
proxy_headers = self.proxy_headers(proxy)
|
||||
|
||||
if not proxy in self.proxy_manager:
|
||||
self.proxy_manager[proxy] = proxy_from_url(
|
||||
proxy,
|
||||
proxy_headers=proxy_headers,
|
||||
num_pools=self._pool_connections,
|
||||
maxsize=self._pool_maxsize,
|
||||
block=self._pool_block)
|
||||
|
||||
conn = self.proxy_manager[proxy].connection_from_url(url)
|
||||
proxy_manager = self.proxy_manager_for(proxy)
|
||||
conn = proxy_manager.connection_from_url(url)
|
||||
else:
|
||||
# Only scheme should be lower case
|
||||
parsed = urlparse(url)
|
||||
|
|
@ -296,7 +317,10 @@ class HTTPAdapter(BaseAdapter):
|
|||
|
||||
:param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
|
||||
:param stream: (optional) Whether to stream the request content.
|
||||
:param timeout: (optional) The timeout on the request.
|
||||
:param timeout: (optional) How long to wait for the server to send
|
||||
data before giving up, as a float, or a (`connect timeout, read
|
||||
timeout <user/advanced.html#timeouts>`_) tuple.
|
||||
:type timeout: float or tuple
|
||||
:param verify: (optional) Whether to verify SSL certificates.
|
||||
:param cert: (optional) Any user-provided SSL certificate to be trusted.
|
||||
:param proxies: (optional) The proxies dictionary to apply to the request.
|
||||
|
|
@ -310,7 +334,18 @@ class HTTPAdapter(BaseAdapter):
|
|||
|
||||
chunked = not (request.body is None or 'Content-Length' in request.headers)
|
||||
|
||||
timeout = TimeoutSauce(connect=timeout, read=timeout)
|
||||
if isinstance(timeout, tuple):
|
||||
try:
|
||||
connect, read = timeout
|
||||
timeout = TimeoutSauce(connect=connect, read=read)
|
||||
except ValueError as e:
|
||||
# this may raise a string formatting error.
|
||||
err = ("Invalid timeout {0}. Pass a (connect, read) "
|
||||
"timeout tuple, or a single float to set "
|
||||
"both timeouts to the same value".format(timeout))
|
||||
raise ValueError(err)
|
||||
else:
|
||||
timeout = TimeoutSauce(connect=timeout, read=timeout)
|
||||
|
||||
try:
|
||||
if not chunked:
|
||||
|
|
@ -323,7 +358,7 @@ class HTTPAdapter(BaseAdapter):
|
|||
assert_same_host=False,
|
||||
preload_content=False,
|
||||
decode_content=False,
|
||||
retries=self.max_retries,
|
||||
retries=Retry(self.max_retries, read=False),
|
||||
timeout=timeout
|
||||
)
|
||||
|
||||
|
|
@ -368,10 +403,13 @@ class HTTPAdapter(BaseAdapter):
|
|||
# All is well, return the connection to the pool.
|
||||
conn._put_conn(low_conn)
|
||||
|
||||
except socket.error as sockerr:
|
||||
raise ConnectionError(sockerr, request=request)
|
||||
except (ProtocolError, socket.error) as err:
|
||||
raise ConnectionError(err, request=request)
|
||||
|
||||
except MaxRetryError as e:
|
||||
if isinstance(e.reason, ConnectTimeoutError):
|
||||
raise ConnectTimeout(e, request=request)
|
||||
|
||||
raise ConnectionError(e, request=request)
|
||||
|
||||
except _ProxyError as e:
|
||||
|
|
@ -380,8 +418,8 @@ class HTTPAdapter(BaseAdapter):
|
|||
except (_SSLError, _HTTPError) as e:
|
||||
if isinstance(e, _SSLError):
|
||||
raise SSLError(e, request=request)
|
||||
elif isinstance(e, TimeoutError):
|
||||
raise Timeout(e, request=request)
|
||||
elif isinstance(e, ReadTimeoutError):
|
||||
raise ReadTimeout(e, request=request)
|
||||
else:
|
||||
raise
|
||||
|
||||
|
|
|
|||
|
|
@ -24,10 +24,14 @@ def request(method, url, **kwargs):
|
|||
:param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`.
|
||||
:param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`.
|
||||
:param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`.
|
||||
:param files: (optional) Dictionary of 'name': file-like-objects (or {'name': ('filename', fileobj)}) for multipart encoding upload.
|
||||
:param files: (optional) Dictionary of ``'name': file-like-objects`` (or ``{'name': ('filename', fileobj)}``) for multipart encoding upload.
|
||||
:param auth: (optional) Auth tuple to enable Basic/Digest/Custom HTTP Auth.
|
||||
:param timeout: (optional) Float describing the timeout of the request in seconds.
|
||||
:param timeout: (optional) How long to wait for the server to send data
|
||||
before giving up, as a float, or a (`connect timeout, read timeout
|
||||
<user/advanced.html#timeouts>`_) tuple.
|
||||
:type timeout: float or tuple
|
||||
:param allow_redirects: (optional) Boolean. Set to True if POST/PUT/DELETE redirect following is allowed.
|
||||
:type allow_redirects: bool
|
||||
:param proxies: (optional) Dictionary mapping protocol to the URL of the proxy.
|
||||
:param verify: (optional) if ``True``, the SSL cert will be verified. A CA_BUNDLE path can also be provided.
|
||||
:param stream: (optional) if ``False``, the response content will be immediately downloaded.
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ from base64 import b64encode
|
|||
|
||||
from .compat import urlparse, str
|
||||
from .cookies import extract_cookies_to_jar
|
||||
from .utils import parse_dict_header
|
||||
from .utils import parse_dict_header, to_native_string
|
||||
|
||||
CONTENT_TYPE_FORM_URLENCODED = 'application/x-www-form-urlencoded'
|
||||
CONTENT_TYPE_MULTI_PART = 'multipart/form-data'
|
||||
|
|
@ -25,7 +25,11 @@ CONTENT_TYPE_MULTI_PART = 'multipart/form-data'
|
|||
def _basic_auth_str(username, password):
|
||||
"""Returns a Basic Auth string."""
|
||||
|
||||
return 'Basic ' + b64encode(('%s:%s' % (username, password)).encode('latin1')).strip().decode('latin1')
|
||||
authstr = 'Basic ' + to_native_string(
|
||||
b64encode(('%s:%s' % (username, password)).encode('latin1')).strip()
|
||||
)
|
||||
|
||||
return authstr
|
||||
|
||||
|
||||
class AuthBase(object):
|
||||
|
|
|
|||
|
|
@ -11,14 +11,15 @@ If you are packaging Requests, e.g., for a Linux distribution or a managed
|
|||
environment, you can change the definition of where() to return a separately
|
||||
packaged CA bundle.
|
||||
"""
|
||||
|
||||
import os.path
|
||||
|
||||
|
||||
def where():
|
||||
"""Return the preferred certificate bundle."""
|
||||
# vendored bundle inside Requests
|
||||
return os.path.join(os.path.dirname(__file__), 'cacert.pem')
|
||||
try:
|
||||
from certifi import where
|
||||
except ImportError:
|
||||
def where():
|
||||
"""Return the preferred certificate bundle."""
|
||||
# vendored bundle inside Requests
|
||||
return os.path.join(os.path.dirname(__file__), 'cacert.pem')
|
||||
|
||||
if __name__ == '__main__':
|
||||
print(where())
|
||||
|
|
|
|||
|
|
@ -75,7 +75,9 @@ is_solaris = ('solar==' in str(sys.platform).lower()) # Complete guess.
|
|||
|
||||
try:
|
||||
import simplejson as json
|
||||
except ImportError:
|
||||
except (ImportError, SyntaxError):
|
||||
# simplejson does not support Python 3.2, it thows a SyntaxError
|
||||
# because of u'...' Unicode literals.
|
||||
import json
|
||||
|
||||
# ---------
|
||||
|
|
@ -90,7 +92,6 @@ if is_py2:
|
|||
from Cookie import Morsel
|
||||
from StringIO import StringIO
|
||||
from .packages.urllib3.packages.ordered_dict import OrderedDict
|
||||
from httplib import IncompleteRead
|
||||
|
||||
builtin_str = str
|
||||
bytes = str
|
||||
|
|
@ -106,7 +107,6 @@ elif is_py3:
|
|||
from http.cookies import Morsel
|
||||
from io import StringIO
|
||||
from collections import OrderedDict
|
||||
from http.client import IncompleteRead
|
||||
|
||||
builtin_str = str
|
||||
str = str
|
||||
|
|
|
|||
|
|
@ -44,7 +44,23 @@ class SSLError(ConnectionError):
|
|||
|
||||
|
||||
class Timeout(RequestException):
|
||||
"""The request timed out."""
|
||||
"""The request timed out.
|
||||
|
||||
Catching this error will catch both
|
||||
:exc:`~requests.exceptions.ConnectTimeout` and
|
||||
:exc:`~requests.exceptions.ReadTimeout` errors.
|
||||
"""
|
||||
|
||||
|
||||
class ConnectTimeout(ConnectionError, Timeout):
|
||||
"""The request timed out while trying to connect to the remote server.
|
||||
|
||||
Requests that produced this error are safe to retry.
|
||||
"""
|
||||
|
||||
|
||||
class ReadTimeout(Timeout):
|
||||
"""The server did not send any data in the allotted amount of time."""
|
||||
|
||||
|
||||
class URLRequired(RequestException):
|
||||
|
|
|
|||
|
|
@ -19,26 +19,28 @@ from .cookies import cookiejar_from_dict, get_cookie_header
|
|||
from .packages.urllib3.fields import RequestField
|
||||
from .packages.urllib3.filepost import encode_multipart_formdata
|
||||
from .packages.urllib3.util import parse_url
|
||||
from .packages.urllib3.exceptions import DecodeError
|
||||
from .packages.urllib3.exceptions import (
|
||||
DecodeError, ReadTimeoutError, ProtocolError)
|
||||
from .exceptions import (
|
||||
HTTPError, RequestException, MissingSchema, InvalidURL,
|
||||
ChunkedEncodingError, ContentDecodingError)
|
||||
ChunkedEncodingError, ContentDecodingError, ConnectionError)
|
||||
from .utils import (
|
||||
guess_filename, get_auth_from_url, requote_uri,
|
||||
stream_decode_response_unicode, to_key_val_list, parse_header_links,
|
||||
iter_slices, guess_json_utf, super_len, to_native_string)
|
||||
from .compat import (
|
||||
cookielib, urlunparse, urlsplit, urlencode, str, bytes, StringIO,
|
||||
is_py2, chardet, json, builtin_str, basestring, IncompleteRead)
|
||||
is_py2, chardet, json, builtin_str, basestring)
|
||||
from .status_codes import codes
|
||||
|
||||
#: The set of HTTP status codes that indicate an automatically
|
||||
#: processable redirect.
|
||||
REDIRECT_STATI = (
|
||||
codes.moved, # 301
|
||||
codes.found, # 302
|
||||
codes.other, # 303
|
||||
codes.temporary_moved, # 307
|
||||
codes.moved, # 301
|
||||
codes.found, # 302
|
||||
codes.other, # 303
|
||||
codes.temporary_redirect, # 307
|
||||
codes.permanent_redirect, # 308
|
||||
)
|
||||
DEFAULT_REDIRECT_LIMIT = 30
|
||||
CONTENT_CHUNK_SIZE = 10 * 1024
|
||||
|
|
@ -309,8 +311,8 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
|
|||
p = PreparedRequest()
|
||||
p.method = self.method
|
||||
p.url = self.url
|
||||
p.headers = self.headers.copy()
|
||||
p._cookies = self._cookies.copy()
|
||||
p.headers = self.headers.copy() if self.headers is not None else None
|
||||
p._cookies = self._cookies.copy() if self._cookies is not None else None
|
||||
p.body = self.body
|
||||
p.hooks = self.hooks
|
||||
return p
|
||||
|
|
@ -433,7 +435,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin):
|
|||
else:
|
||||
if data:
|
||||
body = self._encode_params(data)
|
||||
if isinstance(data, str) or isinstance(data, builtin_str) or hasattr(data, 'read'):
|
||||
if isinstance(data, basestring) or hasattr(data, 'read'):
|
||||
content_type = None
|
||||
else:
|
||||
content_type = 'application/x-www-form-urlencoded'
|
||||
|
|
@ -556,6 +558,10 @@ class Response(object):
|
|||
#: and the arrival of the response (as a timedelta)
|
||||
self.elapsed = datetime.timedelta(0)
|
||||
|
||||
#: The :class:`PreparedRequest <PreparedRequest>` object to which this
|
||||
#: is a response.
|
||||
self.request = None
|
||||
|
||||
def __getstate__(self):
|
||||
# Consume everything; accessing the content attribute makes
|
||||
# sure the content has been fully read.
|
||||
|
|
@ -605,6 +611,11 @@ class Response(object):
|
|||
"""
|
||||
return ('location' in self.headers and self.status_code in REDIRECT_STATI)
|
||||
|
||||
@property
|
||||
def is_permanent_redirect(self):
|
||||
"""True if this Response one of the permanant versions of redirect"""
|
||||
return ('location' in self.headers and self.status_code in (codes.moved_permanently, codes.permanent_redirect))
|
||||
|
||||
@property
|
||||
def apparent_encoding(self):
|
||||
"""The apparent encoding, provided by the chardet library"""
|
||||
|
|
@ -626,10 +637,12 @@ class Response(object):
|
|||
try:
|
||||
for chunk in self.raw.stream(chunk_size, decode_content=True):
|
||||
yield chunk
|
||||
except IncompleteRead as e:
|
||||
except ProtocolError as e:
|
||||
raise ChunkedEncodingError(e)
|
||||
except DecodeError as e:
|
||||
raise ContentDecodingError(e)
|
||||
except ReadTimeoutError as e:
|
||||
raise ConnectionError(e)
|
||||
except AttributeError:
|
||||
# Standard file-like object.
|
||||
while True:
|
||||
|
|
|
|||
|
|
@ -1,9 +1,3 @@
|
|||
# urllib3/__init__.py
|
||||
# Copyright 2008-2013 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
|
||||
#
|
||||
# This module is part of urllib3 and is released under
|
||||
# the MIT License: http://www.opensource.org/licenses/mit-license.php
|
||||
|
||||
"""
|
||||
urllib3 - Thread-safe connection pooling and re-using.
|
||||
"""
|
||||
|
|
@ -23,7 +17,10 @@ from . import exceptions
|
|||
from .filepost import encode_multipart_formdata
|
||||
from .poolmanager import PoolManager, ProxyManager, proxy_from_url
|
||||
from .response import HTTPResponse
|
||||
from .util import make_headers, get_host, Timeout
|
||||
from .util.request import make_headers
|
||||
from .util.url import get_host
|
||||
from .util.timeout import Timeout
|
||||
from .util.retry import Retry
|
||||
|
||||
|
||||
# Set default logging handler to avoid "No handler found" warnings.
|
||||
|
|
@ -51,8 +48,19 @@ def add_stderr_logger(level=logging.DEBUG):
|
|||
handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s'))
|
||||
logger.addHandler(handler)
|
||||
logger.setLevel(level)
|
||||
logger.debug('Added an stderr logging handler to logger: %s' % __name__)
|
||||
logger.debug('Added a stderr logging handler to logger: %s' % __name__)
|
||||
return handler
|
||||
|
||||
# ... Clean up.
|
||||
del NullHandler
|
||||
|
||||
|
||||
# Set security warning to only go off once by default.
|
||||
import warnings
|
||||
warnings.simplefilter('module', exceptions.SecurityWarning)
|
||||
|
||||
def disable_warnings(category=exceptions.HTTPWarning):
|
||||
"""
|
||||
Helper for quickly disabling all urllib3 warnings.
|
||||
"""
|
||||
warnings.simplefilter('ignore', category)
|
||||
|
|
|
|||
|
|
@ -1,9 +1,3 @@
|
|||
# urllib3/_collections.py
|
||||
# Copyright 2008-2013 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
|
||||
#
|
||||
# This module is part of urllib3 and is released under
|
||||
# the MIT License: http://www.opensource.org/licenses/mit-license.php
|
||||
|
||||
from collections import Mapping, MutableMapping
|
||||
try:
|
||||
from threading import RLock
|
||||
|
|
@ -116,7 +110,7 @@ class HTTPHeaderDict(MutableMapping):
|
|||
A ``dict`` like container for storing HTTP Headers.
|
||||
|
||||
Field names are stored and compared case-insensitively in compliance with
|
||||
RFC 2616. Iteration provides the first case-sensitive key seen for each
|
||||
RFC 7230. Iteration provides the first case-sensitive key seen for each
|
||||
case-insensitive pair.
|
||||
|
||||
Using ``__setitem__`` syntax overwrites fields that compare equal
|
||||
|
|
|
|||
|
|
@ -1,95 +1,133 @@
|
|||
# urllib3/connection.py
|
||||
# Copyright 2008-2013 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
|
||||
#
|
||||
# This module is part of urllib3 and is released under
|
||||
# the MIT License: http://www.opensource.org/licenses/mit-license.php
|
||||
|
||||
import datetime
|
||||
import sys
|
||||
import socket
|
||||
from socket import timeout as SocketTimeout
|
||||
import warnings
|
||||
|
||||
try: # Python 3
|
||||
try: # Python 3
|
||||
from http.client import HTTPConnection as _HTTPConnection, HTTPException
|
||||
except ImportError:
|
||||
from httplib import HTTPConnection as _HTTPConnection, HTTPException
|
||||
|
||||
|
||||
class DummyConnection(object):
|
||||
"Used to detect a failed ConnectionCls import."
|
||||
pass
|
||||
|
||||
try: # Compiled with SSL?
|
||||
ssl = None
|
||||
|
||||
try: # Compiled with SSL?
|
||||
HTTPSConnection = DummyConnection
|
||||
import ssl
|
||||
BaseSSLError = ssl.SSLError
|
||||
except (ImportError, AttributeError): # Platform-specific: No SSL.
|
||||
ssl = None
|
||||
|
||||
class BaseSSLError(BaseException):
|
||||
pass
|
||||
|
||||
try: # Python 3
|
||||
from http.client import HTTPSConnection as _HTTPSConnection
|
||||
except ImportError:
|
||||
from httplib import HTTPSConnection as _HTTPSConnection
|
||||
|
||||
import ssl
|
||||
BaseSSLError = ssl.SSLError
|
||||
|
||||
except (ImportError, AttributeError): # Platform-specific: No SSL.
|
||||
pass
|
||||
|
||||
from .exceptions import (
|
||||
ConnectTimeoutError,
|
||||
SystemTimeWarning,
|
||||
)
|
||||
from .packages.ssl_match_hostname import match_hostname
|
||||
from .packages import six
|
||||
from .util import (
|
||||
assert_fingerprint,
|
||||
|
||||
from .util.ssl_ import (
|
||||
resolve_cert_reqs,
|
||||
resolve_ssl_version,
|
||||
ssl_wrap_socket,
|
||||
assert_fingerprint,
|
||||
)
|
||||
|
||||
from .util import connection
|
||||
|
||||
|
||||
port_by_scheme = {
|
||||
'http': 80,
|
||||
'https': 443,
|
||||
}
|
||||
|
||||
RECENT_DATE = datetime.date(2014, 1, 1)
|
||||
|
||||
|
||||
class HTTPConnection(_HTTPConnection, object):
|
||||
"""
|
||||
Based on httplib.HTTPConnection but provides an extra constructor
|
||||
backwards-compatibility layer between older and newer Pythons.
|
||||
|
||||
Additional keyword parameters are used to configure attributes of the connection.
|
||||
Accepted parameters include:
|
||||
|
||||
- ``strict``: See the documentation on :class:`urllib3.connectionpool.HTTPConnectionPool`
|
||||
- ``source_address``: Set the source address for the current connection.
|
||||
|
||||
.. note:: This is ignored for Python 2.6. It is only applied for 2.7 and 3.x
|
||||
|
||||
- ``socket_options``: Set specific options on the underlying socket. If not specified, then
|
||||
defaults are loaded from ``HTTPConnection.default_socket_options`` which includes disabling
|
||||
Nagle's algorithm (sets TCP_NODELAY to 1) unless the connection is behind a proxy.
|
||||
|
||||
For example, if you wish to enable TCP Keep Alive in addition to the defaults,
|
||||
you might pass::
|
||||
|
||||
HTTPConnection.default_socket_options + [
|
||||
(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1),
|
||||
]
|
||||
|
||||
Or you may want to disable the defaults by passing an empty list (e.g., ``[]``).
|
||||
"""
|
||||
|
||||
default_port = port_by_scheme['http']
|
||||
|
||||
# By default, disable Nagle's Algorithm.
|
||||
tcp_nodelay = 1
|
||||
#: Disable Nagle's algorithm by default.
|
||||
#: ``[(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)]``
|
||||
default_socket_options = [(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)]
|
||||
|
||||
#: Whether this connection verifies the host's certificate.
|
||||
is_verified = False
|
||||
|
||||
def __init__(self, *args, **kw):
|
||||
if six.PY3: # Python 3
|
||||
kw.pop('strict', None)
|
||||
if sys.version_info < (2, 7): # Python 2.6 and older
|
||||
kw.pop('source_address', None)
|
||||
|
||||
# Pre-set source_address in case we have an older Python like 2.6.
|
||||
self.source_address = kw.get('source_address')
|
||||
|
||||
if sys.version_info < (2, 7): # Python 2.6
|
||||
# _HTTPConnection on Python 2.6 will balk at this keyword arg, but
|
||||
# not newer versions. We can still use it when creating a
|
||||
# connection though, so we pop it *after* we have saved it as
|
||||
# self.source_address.
|
||||
kw.pop('source_address', None)
|
||||
|
||||
#: The socket options provided by the user. If no options are
|
||||
#: provided, we use the default options.
|
||||
self.socket_options = kw.pop('socket_options', self.default_socket_options)
|
||||
|
||||
# Superclass also sets self.source_address in Python 2.7+.
|
||||
_HTTPConnection.__init__(self, *args, **kw)
|
||||
_HTTPConnection.__init__(self, *args, **kw)
|
||||
|
||||
def _new_conn(self):
|
||||
""" Establish a socket connection and set nodelay settings on it.
|
||||
|
||||
:return: a new socket connection
|
||||
:return: New socket connection.
|
||||
"""
|
||||
extra_args = []
|
||||
if self.source_address: # Python 2.7+
|
||||
extra_args.append(self.source_address)
|
||||
extra_kw = {}
|
||||
if self.source_address:
|
||||
extra_kw['source_address'] = self.source_address
|
||||
|
||||
conn = socket.create_connection(
|
||||
(self.host, self.port), self.timeout, *extra_args)
|
||||
conn.setsockopt(
|
||||
socket.IPPROTO_TCP, socket.TCP_NODELAY, self.tcp_nodelay)
|
||||
if self.socket_options:
|
||||
extra_kw['socket_options'] = self.socket_options
|
||||
|
||||
try:
|
||||
conn = connection.create_connection(
|
||||
(self.host, self.port), self.timeout, **extra_kw)
|
||||
|
||||
except SocketTimeout:
|
||||
raise ConnectTimeoutError(
|
||||
self, "Connection to %s timed out. (connect timeout=%s)" %
|
||||
(self.host, self.timeout))
|
||||
|
||||
return conn
|
||||
|
||||
|
|
@ -101,6 +139,8 @@ class HTTPConnection(_HTTPConnection, object):
|
|||
if getattr(self, '_tunnel_host', None):
|
||||
# TODO: Fix tunnel so it doesn't depend on self.sock state.
|
||||
self._tunnel()
|
||||
# Mark this connection as not reusable
|
||||
self.auto_open = 0
|
||||
|
||||
def connect(self):
|
||||
conn = self._new_conn()
|
||||
|
|
@ -137,7 +177,7 @@ class VerifiedHTTPSConnection(HTTPSConnection):
|
|||
cert_reqs = None
|
||||
ca_certs = None
|
||||
ssl_version = None
|
||||
conn_kw = {}
|
||||
assert_fingerprint = None
|
||||
|
||||
def set_cert(self, key_file=None, cert_file=None,
|
||||
cert_reqs=None, ca_certs=None,
|
||||
|
|
@ -152,18 +192,7 @@ class VerifiedHTTPSConnection(HTTPSConnection):
|
|||
|
||||
def connect(self):
|
||||
# Add certificate verification
|
||||
|
||||
try:
|
||||
sock = socket.create_connection(
|
||||
address=(self.host, self.port), timeout=self.timeout,
|
||||
**self.conn_kw)
|
||||
except SocketTimeout:
|
||||
raise ConnectTimeoutError(
|
||||
self, "Connection to %s timed out. (connect timeout=%s)" %
|
||||
(self.host, self.timeout))
|
||||
|
||||
sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY,
|
||||
self.tcp_nodelay)
|
||||
conn = self._new_conn()
|
||||
|
||||
resolved_cert_reqs = resolve_cert_reqs(self.cert_reqs)
|
||||
resolved_ssl_version = resolve_ssl_version(self.ssl_version)
|
||||
|
|
@ -173,29 +202,42 @@ class VerifiedHTTPSConnection(HTTPSConnection):
|
|||
# _tunnel_host was added in Python 2.6.3
|
||||
# (See: http://hg.python.org/cpython/rev/0f57b30a152f)
|
||||
|
||||
self.sock = sock
|
||||
self.sock = conn
|
||||
# Calls self._set_hostport(), so self.host is
|
||||
# self._tunnel_host below.
|
||||
self._tunnel()
|
||||
# Mark this connection as not reusable
|
||||
self.auto_open = 0
|
||||
|
||||
# Override the host with the one we're requesting data from.
|
||||
hostname = self._tunnel_host
|
||||
|
||||
is_time_off = datetime.date.today() < RECENT_DATE
|
||||
if is_time_off:
|
||||
warnings.warn((
|
||||
'System time is way off (before {0}). This will probably '
|
||||
'lead to SSL verification errors').format(RECENT_DATE),
|
||||
SystemTimeWarning
|
||||
)
|
||||
|
||||
# Wrap socket using verification with the root certs in
|
||||
# trusted_root_certs
|
||||
self.sock = ssl_wrap_socket(sock, self.key_file, self.cert_file,
|
||||
self.sock = ssl_wrap_socket(conn, self.key_file, self.cert_file,
|
||||
cert_reqs=resolved_cert_reqs,
|
||||
ca_certs=self.ca_certs,
|
||||
server_hostname=hostname,
|
||||
ssl_version=resolved_ssl_version)
|
||||
|
||||
if resolved_cert_reqs != ssl.CERT_NONE:
|
||||
if self.assert_fingerprint:
|
||||
assert_fingerprint(self.sock.getpeercert(binary_form=True),
|
||||
self.assert_fingerprint)
|
||||
elif self.assert_hostname is not False:
|
||||
match_hostname(self.sock.getpeercert(),
|
||||
self.assert_hostname or hostname)
|
||||
if self.assert_fingerprint:
|
||||
assert_fingerprint(self.sock.getpeercert(binary_form=True),
|
||||
self.assert_fingerprint)
|
||||
elif resolved_cert_reqs != ssl.CERT_NONE \
|
||||
and self.assert_hostname is not False:
|
||||
match_hostname(self.sock.getpeercert(),
|
||||
self.assert_hostname or hostname)
|
||||
|
||||
self.is_verified = (resolved_cert_reqs == ssl.CERT_REQUIRED
|
||||
or self.assert_fingerprint is not None)
|
||||
|
||||
|
||||
if ssl:
|
||||
|
|
|
|||
|
|
@ -1,17 +1,12 @@
|
|||
# urllib3/connectionpool.py
|
||||
# Copyright 2008-2013 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
|
||||
#
|
||||
# This module is part of urllib3 and is released under
|
||||
# the MIT License: http://www.opensource.org/licenses/mit-license.php
|
||||
|
||||
import sys
|
||||
import errno
|
||||
import logging
|
||||
import sys
|
||||
import warnings
|
||||
|
||||
from socket import error as SocketError, timeout as SocketTimeout
|
||||
import socket
|
||||
|
||||
try: # Python 3
|
||||
try: # Python 3
|
||||
from queue import LifoQueue, Empty, Full
|
||||
except ImportError:
|
||||
from Queue import LifoQueue, Empty, Full
|
||||
|
|
@ -20,16 +15,16 @@ except ImportError:
|
|||
|
||||
from .exceptions import (
|
||||
ClosedPoolError,
|
||||
ConnectionError,
|
||||
ConnectTimeoutError,
|
||||
ProtocolError,
|
||||
EmptyPoolError,
|
||||
HostChangedError,
|
||||
LocationParseError,
|
||||
LocationValueError,
|
||||
MaxRetryError,
|
||||
ProxyError,
|
||||
ReadTimeoutError,
|
||||
SSLError,
|
||||
TimeoutError,
|
||||
ReadTimeoutError,
|
||||
ProxyError,
|
||||
InsecureRequestWarning,
|
||||
)
|
||||
from .packages.ssl_match_hostname import CertificateError
|
||||
from .packages import six
|
||||
|
|
@ -41,11 +36,11 @@ from .connection import (
|
|||
)
|
||||
from .request import RequestMethods
|
||||
from .response import HTTPResponse
|
||||
from .util import (
|
||||
get_host,
|
||||
is_connection_dropped,
|
||||
Timeout,
|
||||
)
|
||||
|
||||
from .util.connection import is_connection_dropped
|
||||
from .util.retry import Retry
|
||||
from .util.timeout import Timeout
|
||||
from .util.url import get_host
|
||||
|
||||
|
||||
xrange = six.moves.xrange
|
||||
|
|
@ -54,8 +49,8 @@ log = logging.getLogger(__name__)
|
|||
|
||||
_Default = object()
|
||||
|
||||
## Pool objects
|
||||
|
||||
## Pool objects
|
||||
class ConnectionPool(object):
|
||||
"""
|
||||
Base class for all connection pools, such as
|
||||
|
|
@ -66,13 +61,11 @@ class ConnectionPool(object):
|
|||
QueueCls = LifoQueue
|
||||
|
||||
def __init__(self, host, port=None):
|
||||
if host is None:
|
||||
raise LocationParseError(host)
|
||||
if not host:
|
||||
raise LocationValueError("No host specified.")
|
||||
|
||||
# httplib doesn't like it when we include brackets in ipv6 addresses
|
||||
host = host.strip('[]')
|
||||
|
||||
self.host = host
|
||||
self.host = host.strip('[]')
|
||||
self.port = port
|
||||
|
||||
def __str__(self):
|
||||
|
|
@ -82,6 +75,7 @@ class ConnectionPool(object):
|
|||
# This is taken from http://hg.python.org/cpython/file/7aaba721ebc0/Lib/socket.py#l252
|
||||
_blocking_errnos = set([errno.EAGAIN, errno.EWOULDBLOCK])
|
||||
|
||||
|
||||
class HTTPConnectionPool(ConnectionPool, RequestMethods):
|
||||
"""
|
||||
Thread-safe connection pool for one host.
|
||||
|
|
@ -126,6 +120,9 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
|
|||
Headers to include with all requests, unless other headers are given
|
||||
explicitly.
|
||||
|
||||
:param retries:
|
||||
Retry configuration to use by default with requests in this pool.
|
||||
|
||||
:param _proxy:
|
||||
Parsed proxy URL, should not be used directly, instead, see
|
||||
:class:`urllib3.connectionpool.ProxyManager`"
|
||||
|
|
@ -133,6 +130,10 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
|
|||
:param _proxy_headers:
|
||||
A dictionary with proxy headers, should not be used directly,
|
||||
instead, see :class:`urllib3.connectionpool.ProxyManager`"
|
||||
|
||||
:param \**conn_kw:
|
||||
Additional parameters are used to create fresh :class:`urllib3.connection.HTTPConnection`,
|
||||
:class:`urllib3.connection.HTTPSConnection` instances.
|
||||
"""
|
||||
|
||||
scheme = 'http'
|
||||
|
|
@ -140,18 +141,22 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
|
|||
|
||||
def __init__(self, host, port=None, strict=False,
|
||||
timeout=Timeout.DEFAULT_TIMEOUT, maxsize=1, block=False,
|
||||
headers=None, _proxy=None, _proxy_headers=None, **conn_kw):
|
||||
headers=None, retries=None,
|
||||
_proxy=None, _proxy_headers=None,
|
||||
**conn_kw):
|
||||
ConnectionPool.__init__(self, host, port)
|
||||
RequestMethods.__init__(self, headers)
|
||||
|
||||
self.strict = strict
|
||||
|
||||
# This is for backwards compatibility and can be removed once a timeout
|
||||
# can only be set to a Timeout object
|
||||
if not isinstance(timeout, Timeout):
|
||||
timeout = Timeout.from_float(timeout)
|
||||
|
||||
if retries is None:
|
||||
retries = Retry.DEFAULT
|
||||
|
||||
self.timeout = timeout
|
||||
self.retries = retries
|
||||
|
||||
self.pool = self.QueueCls(maxsize)
|
||||
self.block = block
|
||||
|
|
@ -166,11 +171,14 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
|
|||
# These are mostly for testing and debugging purposes.
|
||||
self.num_connections = 0
|
||||
self.num_requests = 0
|
||||
|
||||
if sys.version_info < (2, 7): # Python 2.6 and older
|
||||
conn_kw.pop('source_address', None)
|
||||
self.conn_kw = conn_kw
|
||||
|
||||
if self.proxy:
|
||||
# Enable Nagle's algorithm for proxies, to avoid packet fragmentation.
|
||||
# We cannot know if the user has added default socket options, so we cannot replace the
|
||||
# list.
|
||||
self.conn_kw.setdefault('socket_options', [])
|
||||
|
||||
def _new_conn(self):
|
||||
"""
|
||||
Return a fresh :class:`HTTPConnection`.
|
||||
|
|
@ -182,10 +190,6 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
|
|||
conn = self.ConnectionCls(host=self.host, port=self.port,
|
||||
timeout=self.timeout.connect_timeout,
|
||||
strict=self.strict, **self.conn_kw)
|
||||
if self.proxy is not None:
|
||||
# Enable Nagle's algorithm for proxies, to avoid packet
|
||||
# fragmentation.
|
||||
conn.tcp_nodelay = 0
|
||||
return conn
|
||||
|
||||
def _get_conn(self, timeout=None):
|
||||
|
|
@ -204,7 +208,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
|
|||
try:
|
||||
conn = self.pool.get(block=self.block, timeout=timeout)
|
||||
|
||||
except AttributeError: # self.pool is None
|
||||
except AttributeError: # self.pool is None
|
||||
raise ClosedPoolError(self, "Pool is closed.")
|
||||
|
||||
except Empty:
|
||||
|
|
@ -218,6 +222,11 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
|
|||
if conn and is_connection_dropped(conn):
|
||||
log.info("Resetting dropped connection: %s" % self.host)
|
||||
conn.close()
|
||||
if getattr(conn, 'auto_open', 1) == 0:
|
||||
# This is a proxied connection that has been mutated by
|
||||
# httplib._tunnel() and cannot be reused (since it would
|
||||
# attempt to bypass the proxy)
|
||||
conn = None
|
||||
|
||||
return conn or self._new_conn()
|
||||
|
||||
|
|
@ -237,7 +246,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
|
|||
"""
|
||||
try:
|
||||
self.pool.put(conn, block=False)
|
||||
return # Everything is dandy, done.
|
||||
return # Everything is dandy, done.
|
||||
except AttributeError:
|
||||
# self.pool is None.
|
||||
pass
|
||||
|
|
@ -251,6 +260,12 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
|
|||
if conn:
|
||||
conn.close()
|
||||
|
||||
def _validate_conn(self, conn):
|
||||
"""
|
||||
Called right before a request is made, after the socket is created.
|
||||
"""
|
||||
pass
|
||||
|
||||
def _get_timeout(self, timeout):
|
||||
""" Helper that always returns a :class:`urllib3.util.Timeout` """
|
||||
if timeout is _Default:
|
||||
|
|
@ -282,23 +297,21 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
|
|||
self.num_requests += 1
|
||||
|
||||
timeout_obj = self._get_timeout(timeout)
|
||||
timeout_obj.start_connect()
|
||||
conn.timeout = timeout_obj.connect_timeout
|
||||
|
||||
try:
|
||||
timeout_obj.start_connect()
|
||||
conn.timeout = timeout_obj.connect_timeout
|
||||
# conn.request() calls httplib.*.request, not the method in
|
||||
# urllib3.request. It also calls makefile (recv) on the socket.
|
||||
conn.request(method, url, **httplib_request_kw)
|
||||
except SocketTimeout:
|
||||
raise ConnectTimeoutError(
|
||||
self, "Connection to %s timed out. (connect timeout=%s)" %
|
||||
(self.host, timeout_obj.connect_timeout))
|
||||
# Trigger any extra validation we need to do.
|
||||
self._validate_conn(conn)
|
||||
|
||||
# conn.request() calls httplib.*.request, not the method in
|
||||
# urllib3.request. It also calls makefile (recv) on the socket.
|
||||
conn.request(method, url, **httplib_request_kw)
|
||||
|
||||
# Reset the timeout for the recv() on the socket
|
||||
read_timeout = timeout_obj.read_timeout
|
||||
|
||||
# App Engine doesn't have a sock attr
|
||||
if hasattr(conn, 'sock'):
|
||||
if getattr(conn, 'sock', None):
|
||||
# In Python 3 socket.py will catch EAGAIN and return None when you
|
||||
# try and read into the file pointer created by http.client, which
|
||||
# instead raises a BadStatusLine exception. Instead of catching
|
||||
|
|
@ -306,18 +319,17 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
|
|||
# timeouts, check for a zero timeout before making the request.
|
||||
if read_timeout == 0:
|
||||
raise ReadTimeoutError(
|
||||
self, url,
|
||||
"Read timed out. (read timeout=%s)" % read_timeout)
|
||||
self, url, "Read timed out. (read timeout=%s)" % read_timeout)
|
||||
if read_timeout is Timeout.DEFAULT_TIMEOUT:
|
||||
conn.sock.settimeout(socket.getdefaulttimeout())
|
||||
else: # None or a value
|
||||
else: # None or a value
|
||||
conn.sock.settimeout(read_timeout)
|
||||
|
||||
# Receive the response from the server
|
||||
try:
|
||||
try: # Python 2.7+, use buffering of HTTP responses
|
||||
try: # Python 2.7+, use buffering of HTTP responses
|
||||
httplib_response = conn.getresponse(buffering=True)
|
||||
except TypeError: # Python 2.6 and older
|
||||
except TypeError: # Python 2.6 and older
|
||||
httplib_response = conn.getresponse()
|
||||
except SocketTimeout:
|
||||
raise ReadTimeoutError(
|
||||
|
|
@ -329,17 +341,17 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
|
|||
# http://bugs.python.org/issue10272
|
||||
if 'timed out' in str(e) or \
|
||||
'did not complete (read)' in str(e): # Python 2.6
|
||||
raise ReadTimeoutError(self, url, "Read timed out.")
|
||||
raise ReadTimeoutError(
|
||||
self, url, "Read timed out. (read timeout=%s)" % read_timeout)
|
||||
|
||||
raise
|
||||
|
||||
except SocketError as e: # Platform-specific: Python 2
|
||||
except SocketError as e: # Platform-specific: Python 2
|
||||
# See the above comment about EAGAIN in Python 3. In Python 2 we
|
||||
# have to specifically catch it and throw the timeout error
|
||||
if e.errno in _blocking_errnos:
|
||||
raise ReadTimeoutError(
|
||||
self, url,
|
||||
"Read timed out. (read timeout=%s)" % read_timeout)
|
||||
self, url, "Read timed out. (read timeout=%s)" % read_timeout)
|
||||
|
||||
raise
|
||||
|
||||
|
|
@ -364,7 +376,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
|
|||
conn.close()
|
||||
|
||||
except Empty:
|
||||
pass # Done.
|
||||
pass # Done.
|
||||
|
||||
def is_same_host(self, url):
|
||||
"""
|
||||
|
|
@ -385,7 +397,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
|
|||
|
||||
return (scheme, host, port) == (self.scheme, self.host, self.port)
|
||||
|
||||
def urlopen(self, method, url, body=None, headers=None, retries=3,
|
||||
def urlopen(self, method, url, body=None, headers=None, retries=None,
|
||||
redirect=True, assert_same_host=True, timeout=_Default,
|
||||
pool_timeout=None, release_conn=None, **response_kw):
|
||||
"""
|
||||
|
|
@ -419,9 +431,20 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
|
|||
these headers completely replace any pool-specific headers.
|
||||
|
||||
:param retries:
|
||||
Number of retries to allow before raising a MaxRetryError exception.
|
||||
If `False`, then retries are disabled and any exception is raised
|
||||
immediately.
|
||||
Configure the number of retries to allow before raising a
|
||||
:class:`~urllib3.exceptions.MaxRetryError` exception.
|
||||
|
||||
Pass ``None`` to retry until you receive a response. Pass a
|
||||
:class:`~urllib3.util.retry.Retry` object for fine-grained control
|
||||
over different types of retries.
|
||||
Pass an integer number to retry connection errors that many times,
|
||||
but no other types of errors. Pass zero to never retry.
|
||||
|
||||
If ``False``, then retries are disabled and any exception is raised
|
||||
immediately. Also, instead of raising a MaxRetryError on redirects,
|
||||
the redirect response will be returned.
|
||||
|
||||
:type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.
|
||||
|
||||
:param redirect:
|
||||
If True, automatically handle redirects (status codes 301, 302,
|
||||
|
|
@ -460,15 +483,15 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
|
|||
if headers is None:
|
||||
headers = self.headers
|
||||
|
||||
if retries < 0 and retries is not False:
|
||||
raise MaxRetryError(self, url)
|
||||
if not isinstance(retries, Retry):
|
||||
retries = Retry.from_int(retries, redirect=redirect, default=self.retries)
|
||||
|
||||
if release_conn is None:
|
||||
release_conn = response_kw.get('preload_content', True)
|
||||
|
||||
# Check host
|
||||
if assert_same_host and not self.is_same_host(url):
|
||||
raise HostChangedError(self, url, retries - 1)
|
||||
raise HostChangedError(self, url, retries)
|
||||
|
||||
conn = None
|
||||
|
||||
|
|
@ -484,10 +507,10 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
|
|||
err = None
|
||||
|
||||
try:
|
||||
# Request a connection from the queue
|
||||
# Request a connection from the queue.
|
||||
conn = self._get_conn(timeout=pool_timeout)
|
||||
|
||||
# Make the request on the httplib connection object
|
||||
# Make the request on the httplib connection object.
|
||||
httplib_response = self._make_request(conn, method, url,
|
||||
timeout=timeout,
|
||||
body=body, headers=headers)
|
||||
|
|
@ -526,21 +549,15 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
|
|||
conn.close()
|
||||
conn = None
|
||||
|
||||
if not retries:
|
||||
if isinstance(e, TimeoutError):
|
||||
# TimeoutError is exempt from MaxRetryError-wrapping.
|
||||
# FIXME: ... Not sure why. Add a reason here.
|
||||
raise
|
||||
stacktrace = sys.exc_info()[2]
|
||||
if isinstance(e, SocketError) and self.proxy:
|
||||
e = ProxyError('Cannot connect to proxy.', e)
|
||||
elif isinstance(e, (SocketError, HTTPException)):
|
||||
e = ProtocolError('Connection aborted.', e)
|
||||
|
||||
# Wrap unexpected exceptions with the most appropriate
|
||||
# module-level exception and re-raise.
|
||||
if isinstance(e, SocketError) and self.proxy:
|
||||
raise ProxyError('Cannot connect to proxy.', e)
|
||||
|
||||
if retries is False:
|
||||
raise ConnectionError('Connection failed.', e)
|
||||
|
||||
raise MaxRetryError(self, url, e)
|
||||
retries = retries.increment(method, url, error=e,
|
||||
_pool=self, _stacktrace=stacktrace)
|
||||
retries.sleep()
|
||||
|
||||
# Keep track of the error for the retry warning.
|
||||
err = e
|
||||
|
|
@ -554,23 +571,43 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
|
|||
|
||||
if not conn:
|
||||
# Try again
|
||||
log.warning("Retrying (%d attempts remain) after connection "
|
||||
log.warning("Retrying (%r) after connection "
|
||||
"broken by '%r': %s" % (retries, err, url))
|
||||
return self.urlopen(method, url, body, headers, retries - 1,
|
||||
return self.urlopen(method, url, body, headers, retries,
|
||||
redirect, assert_same_host,
|
||||
timeout=timeout, pool_timeout=pool_timeout,
|
||||
release_conn=release_conn, **response_kw)
|
||||
|
||||
# Handle redirect?
|
||||
redirect_location = redirect and response.get_redirect_location()
|
||||
if redirect_location and retries is not False:
|
||||
if redirect_location:
|
||||
if response.status == 303:
|
||||
method = 'GET'
|
||||
|
||||
try:
|
||||
retries = retries.increment(method, url, response=response, _pool=self)
|
||||
except MaxRetryError:
|
||||
if retries.raise_on_redirect:
|
||||
raise
|
||||
return response
|
||||
|
||||
log.info("Redirecting %s -> %s" % (url, redirect_location))
|
||||
return self.urlopen(method, redirect_location, body, headers,
|
||||
retries - 1, redirect, assert_same_host,
|
||||
timeout=timeout, pool_timeout=pool_timeout,
|
||||
release_conn=release_conn, **response_kw)
|
||||
retries=retries, redirect=redirect,
|
||||
assert_same_host=assert_same_host,
|
||||
timeout=timeout, pool_timeout=pool_timeout,
|
||||
release_conn=release_conn, **response_kw)
|
||||
|
||||
# Check if we should retry the HTTP response.
|
||||
if retries.is_forced_retry(method, status_code=response.status):
|
||||
retries = retries.increment(method, url, response=response, _pool=self)
|
||||
retries.sleep()
|
||||
log.info("Forced retry: %s" % url)
|
||||
return self.urlopen(method, url, body, headers,
|
||||
retries=retries, redirect=redirect,
|
||||
assert_same_host=assert_same_host,
|
||||
timeout=timeout, pool_timeout=pool_timeout,
|
||||
release_conn=release_conn, **response_kw)
|
||||
|
||||
return response
|
||||
|
||||
|
|
@ -597,19 +634,17 @@ class HTTPSConnectionPool(HTTPConnectionPool):
|
|||
ConnectionCls = HTTPSConnection
|
||||
|
||||
def __init__(self, host, port=None,
|
||||
strict=False, timeout=None, maxsize=1,
|
||||
block=False, headers=None,
|
||||
strict=False, timeout=Timeout.DEFAULT_TIMEOUT, maxsize=1,
|
||||
block=False, headers=None, retries=None,
|
||||
_proxy=None, _proxy_headers=None,
|
||||
key_file=None, cert_file=None, cert_reqs=None,
|
||||
ca_certs=None, ssl_version=None,
|
||||
assert_hostname=None, assert_fingerprint=None,
|
||||
**conn_kw):
|
||||
|
||||
if sys.version_info < (2, 7): # Python 2.6 or older
|
||||
conn_kw.pop('source_address', None)
|
||||
|
||||
HTTPConnectionPool.__init__(self, host, port, strict, timeout, maxsize,
|
||||
block, headers, _proxy, _proxy_headers, **conn_kw)
|
||||
block, headers, retries, _proxy, _proxy_headers,
|
||||
**conn_kw)
|
||||
self.key_file = key_file
|
||||
self.cert_file = cert_file
|
||||
self.cert_reqs = cert_reqs
|
||||
|
|
@ -617,7 +652,6 @@ class HTTPSConnectionPool(HTTPConnectionPool):
|
|||
self.ssl_version = ssl_version
|
||||
self.assert_hostname = assert_hostname
|
||||
self.assert_fingerprint = assert_fingerprint
|
||||
self.conn_kw = conn_kw
|
||||
|
||||
def _prepare_conn(self, conn):
|
||||
"""
|
||||
|
|
@ -633,7 +667,6 @@ class HTTPSConnectionPool(HTTPConnectionPool):
|
|||
assert_hostname=self.assert_hostname,
|
||||
assert_fingerprint=self.assert_fingerprint)
|
||||
conn.ssl_version = self.ssl_version
|
||||
conn.conn_kw = self.conn_kw
|
||||
|
||||
if self.proxy is not None:
|
||||
# Python 2.7+
|
||||
|
|
@ -641,7 +674,12 @@ class HTTPSConnectionPool(HTTPConnectionPool):
|
|||
set_tunnel = conn.set_tunnel
|
||||
except AttributeError: # Platform-specific: Python 2.6
|
||||
set_tunnel = conn._set_tunnel
|
||||
set_tunnel(self.host, self.port, self.proxy_headers)
|
||||
|
||||
if sys.version_info <= (2, 6, 4) and not self.proxy_headers: # Python 2.6.4 and older
|
||||
set_tunnel(self.host, self.port)
|
||||
else:
|
||||
set_tunnel(self.host, self.port, self.proxy_headers)
|
||||
|
||||
# Establish tunnel connection early, because otherwise httplib
|
||||
# would improperly set Host: header to proxy's IP:port.
|
||||
conn.connect()
|
||||
|
|
@ -667,21 +705,30 @@ class HTTPSConnectionPool(HTTPConnectionPool):
|
|||
actual_host = self.proxy.host
|
||||
actual_port = self.proxy.port
|
||||
|
||||
extra_params = {}
|
||||
if not six.PY3: # Python 2
|
||||
extra_params['strict'] = self.strict
|
||||
extra_params.update(self.conn_kw)
|
||||
|
||||
conn = self.ConnectionCls(host=actual_host, port=actual_port,
|
||||
timeout=self.timeout.connect_timeout,
|
||||
**extra_params)
|
||||
if self.proxy is not None:
|
||||
# Enable Nagle's algorithm for proxies, to avoid packet
|
||||
# fragmentation.
|
||||
conn.tcp_nodelay = 0
|
||||
strict=self.strict, **self.conn_kw)
|
||||
|
||||
return self._prepare_conn(conn)
|
||||
|
||||
def _validate_conn(self, conn):
|
||||
"""
|
||||
Called right before a request is made, after the socket is created.
|
||||
"""
|
||||
super(HTTPSConnectionPool, self)._validate_conn(conn)
|
||||
|
||||
# Force connect early to allow us to validate the connection.
|
||||
if not getattr(conn, 'sock', None): # AppEngine might not have `.sock`
|
||||
conn.connect()
|
||||
|
||||
if not conn.is_verified:
|
||||
warnings.warn((
|
||||
'Unverified HTTPS request is being made. '
|
||||
'Adding certificate verification is strongly advised. See: '
|
||||
'https://urllib3.readthedocs.org/en/latest/security.html '
|
||||
'(This warning will only appear once by default.)'),
|
||||
InsecureRequestWarning)
|
||||
|
||||
|
||||
def connection_from_url(url, **kw):
|
||||
"""
|
||||
|
|
@ -698,7 +745,7 @@ def connection_from_url(url, **kw):
|
|||
:class:`.ConnectionPool`. Useful for specifying things like
|
||||
timeout, maxsize, headers, etc.
|
||||
|
||||
Example: ::
|
||||
Example::
|
||||
|
||||
>>> conn = connection_from_url('http://google.com/')
|
||||
>>> r = conn.request('GET', '/')
|
||||
|
|
|
|||
|
|
@ -1,9 +1,3 @@
|
|||
# urllib3/contrib/ntlmpool.py
|
||||
# Copyright 2008-2013 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
|
||||
#
|
||||
# This module is part of urllib3 and is released under
|
||||
# the MIT License: http://www.opensource.org/licenses/mit-license.php
|
||||
|
||||
"""
|
||||
NTLM authenticating pool, contributed by erikcederstran
|
||||
|
||||
|
|
|
|||
|
|
@ -46,15 +46,18 @@ Module Variables
|
|||
|
||||
'''
|
||||
|
||||
from ndg.httpsclient.ssl_peer_verification import SUBJ_ALT_NAME_SUPPORT
|
||||
from ndg.httpsclient.subj_alt_name import SubjectAltName as BaseSubjectAltName
|
||||
try:
|
||||
from ndg.httpsclient.ssl_peer_verification import SUBJ_ALT_NAME_SUPPORT
|
||||
from ndg.httpsclient.subj_alt_name import SubjectAltName as BaseSubjectAltName
|
||||
except SyntaxError as e:
|
||||
raise ImportError(e)
|
||||
|
||||
import OpenSSL.SSL
|
||||
from pyasn1.codec.der import decoder as der_decoder
|
||||
from pyasn1.type import univ, constraint
|
||||
from socket import _fileobject, timeout
|
||||
import ssl
|
||||
import select
|
||||
from cStringIO import StringIO
|
||||
|
||||
from .. import connection
|
||||
from .. import util
|
||||
|
|
@ -155,196 +158,43 @@ def get_subj_alt_name(peer_cert):
|
|||
return dns_name
|
||||
|
||||
|
||||
class fileobject(_fileobject):
|
||||
|
||||
def _wait_for_sock(self):
|
||||
rd, wd, ed = select.select([self._sock], [], [],
|
||||
self._sock.gettimeout())
|
||||
if not rd:
|
||||
raise timeout()
|
||||
|
||||
|
||||
def read(self, size=-1):
|
||||
# Use max, disallow tiny reads in a loop as they are very inefficient.
|
||||
# We never leave read() with any leftover data from a new recv() call
|
||||
# in our internal buffer.
|
||||
rbufsize = max(self._rbufsize, self.default_bufsize)
|
||||
# Our use of StringIO rather than lists of string objects returned by
|
||||
# recv() minimizes memory usage and fragmentation that occurs when
|
||||
# rbufsize is large compared to the typical return value of recv().
|
||||
buf = self._rbuf
|
||||
buf.seek(0, 2) # seek end
|
||||
if size < 0:
|
||||
# Read until EOF
|
||||
self._rbuf = StringIO() # reset _rbuf. we consume it via buf.
|
||||
while True:
|
||||
try:
|
||||
data = self._sock.recv(rbufsize)
|
||||
except OpenSSL.SSL.WantReadError:
|
||||
self._wait_for_sock()
|
||||
continue
|
||||
if not data:
|
||||
break
|
||||
buf.write(data)
|
||||
return buf.getvalue()
|
||||
else:
|
||||
# Read until size bytes or EOF seen, whichever comes first
|
||||
buf_len = buf.tell()
|
||||
if buf_len >= size:
|
||||
# Already have size bytes in our buffer? Extract and return.
|
||||
buf.seek(0)
|
||||
rv = buf.read(size)
|
||||
self._rbuf = StringIO()
|
||||
self._rbuf.write(buf.read())
|
||||
return rv
|
||||
|
||||
self._rbuf = StringIO() # reset _rbuf. we consume it via buf.
|
||||
while True:
|
||||
left = size - buf_len
|
||||
# recv() will malloc the amount of memory given as its
|
||||
# parameter even though it often returns much less data
|
||||
# than that. The returned data string is short lived
|
||||
# as we copy it into a StringIO and free it. This avoids
|
||||
# fragmentation issues on many platforms.
|
||||
try:
|
||||
data = self._sock.recv(left)
|
||||
except OpenSSL.SSL.WantReadError:
|
||||
self._wait_for_sock()
|
||||
continue
|
||||
if not data:
|
||||
break
|
||||
n = len(data)
|
||||
if n == size and not buf_len:
|
||||
# Shortcut. Avoid buffer data copies when:
|
||||
# - We have no data in our buffer.
|
||||
# AND
|
||||
# - Our call to recv returned exactly the
|
||||
# number of bytes we were asked to read.
|
||||
return data
|
||||
if n == left:
|
||||
buf.write(data)
|
||||
del data # explicit free
|
||||
break
|
||||
assert n <= left, "recv(%d) returned %d bytes" % (left, n)
|
||||
buf.write(data)
|
||||
buf_len += n
|
||||
del data # explicit free
|
||||
#assert buf_len == buf.tell()
|
||||
return buf.getvalue()
|
||||
|
||||
def readline(self, size=-1):
|
||||
buf = self._rbuf
|
||||
buf.seek(0, 2) # seek end
|
||||
if buf.tell() > 0:
|
||||
# check if we already have it in our buffer
|
||||
buf.seek(0)
|
||||
bline = buf.readline(size)
|
||||
if bline.endswith('\n') or len(bline) == size:
|
||||
self._rbuf = StringIO()
|
||||
self._rbuf.write(buf.read())
|
||||
return bline
|
||||
del bline
|
||||
if size < 0:
|
||||
# Read until \n or EOF, whichever comes first
|
||||
if self._rbufsize <= 1:
|
||||
# Speed up unbuffered case
|
||||
buf.seek(0)
|
||||
buffers = [buf.read()]
|
||||
self._rbuf = StringIO() # reset _rbuf. we consume it via buf.
|
||||
data = None
|
||||
recv = self._sock.recv
|
||||
while True:
|
||||
try:
|
||||
while data != "\n":
|
||||
data = recv(1)
|
||||
if not data:
|
||||
break
|
||||
buffers.append(data)
|
||||
except OpenSSL.SSL.WantReadError:
|
||||
self._wait_for_sock()
|
||||
continue
|
||||
break
|
||||
return "".join(buffers)
|
||||
|
||||
buf.seek(0, 2) # seek end
|
||||
self._rbuf = StringIO() # reset _rbuf. we consume it via buf.
|
||||
while True:
|
||||
try:
|
||||
data = self._sock.recv(self._rbufsize)
|
||||
except OpenSSL.SSL.WantReadError:
|
||||
self._wait_for_sock()
|
||||
continue
|
||||
if not data:
|
||||
break
|
||||
nl = data.find('\n')
|
||||
if nl >= 0:
|
||||
nl += 1
|
||||
buf.write(data[:nl])
|
||||
self._rbuf.write(data[nl:])
|
||||
del data
|
||||
break
|
||||
buf.write(data)
|
||||
return buf.getvalue()
|
||||
else:
|
||||
# Read until size bytes or \n or EOF seen, whichever comes first
|
||||
buf.seek(0, 2) # seek end
|
||||
buf_len = buf.tell()
|
||||
if buf_len >= size:
|
||||
buf.seek(0)
|
||||
rv = buf.read(size)
|
||||
self._rbuf = StringIO()
|
||||
self._rbuf.write(buf.read())
|
||||
return rv
|
||||
self._rbuf = StringIO() # reset _rbuf. we consume it via buf.
|
||||
while True:
|
||||
try:
|
||||
data = self._sock.recv(self._rbufsize)
|
||||
except OpenSSL.SSL.WantReadError:
|
||||
self._wait_for_sock()
|
||||
continue
|
||||
if not data:
|
||||
break
|
||||
left = size - buf_len
|
||||
# did we just receive a newline?
|
||||
nl = data.find('\n', 0, left)
|
||||
if nl >= 0:
|
||||
nl += 1
|
||||
# save the excess data to _rbuf
|
||||
self._rbuf.write(data[nl:])
|
||||
if buf_len:
|
||||
buf.write(data[:nl])
|
||||
break
|
||||
else:
|
||||
# Shortcut. Avoid data copy through buf when returning
|
||||
# a substring of our first recv().
|
||||
return data[:nl]
|
||||
n = len(data)
|
||||
if n == size and not buf_len:
|
||||
# Shortcut. Avoid data copy through buf when
|
||||
# returning exactly all of our first recv().
|
||||
return data
|
||||
if n >= left:
|
||||
buf.write(data[:left])
|
||||
self._rbuf.write(data[left:])
|
||||
break
|
||||
buf.write(data)
|
||||
buf_len += n
|
||||
#assert buf_len == buf.tell()
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
class WrappedSocket(object):
|
||||
'''API-compatibility wrapper for Python OpenSSL's Connection-class.'''
|
||||
'''API-compatibility wrapper for Python OpenSSL's Connection-class.
|
||||
|
||||
def __init__(self, connection, socket):
|
||||
Note: _makefile_refs, _drop() and _reuse() are needed for the garbage
|
||||
collector of pypy.
|
||||
'''
|
||||
|
||||
def __init__(self, connection, socket, suppress_ragged_eofs=True):
|
||||
self.connection = connection
|
||||
self.socket = socket
|
||||
self.suppress_ragged_eofs = suppress_ragged_eofs
|
||||
self._makefile_refs = 0
|
||||
|
||||
def fileno(self):
|
||||
return self.socket.fileno()
|
||||
|
||||
def makefile(self, mode, bufsize=-1):
|
||||
return fileobject(self.connection, mode, bufsize)
|
||||
self._makefile_refs += 1
|
||||
return _fileobject(self, mode, bufsize, close=True)
|
||||
|
||||
def recv(self, *args, **kwargs):
|
||||
try:
|
||||
data = self.connection.recv(*args, **kwargs)
|
||||
except OpenSSL.SSL.SysCallError as e:
|
||||
if self.suppress_ragged_eofs and e.args == (-1, 'Unexpected EOF'):
|
||||
return b''
|
||||
else:
|
||||
raise
|
||||
except OpenSSL.SSL.WantReadError:
|
||||
rd, wd, ed = select.select(
|
||||
[self.socket], [], [], self.socket.gettimeout())
|
||||
if not rd:
|
||||
raise timeout('The read operation timed out')
|
||||
else:
|
||||
return self.recv(*args, **kwargs)
|
||||
else:
|
||||
return data
|
||||
|
||||
def settimeout(self, timeout):
|
||||
return self.socket.settimeout(timeout)
|
||||
|
|
@ -353,7 +203,10 @@ class WrappedSocket(object):
|
|||
return self.connection.sendall(data)
|
||||
|
||||
def close(self):
|
||||
return self.connection.shutdown()
|
||||
if self._makefile_refs < 1:
|
||||
return self.connection.shutdown()
|
||||
else:
|
||||
self._makefile_refs -= 1
|
||||
|
||||
def getpeercert(self, binary_form=False):
|
||||
x509 = self.connection.get_peer_certificate()
|
||||
|
|
@ -376,6 +229,15 @@ class WrappedSocket(object):
|
|||
]
|
||||
}
|
||||
|
||||
def _reuse(self):
|
||||
self._makefile_refs += 1
|
||||
|
||||
def _drop(self):
|
||||
if self._makefile_refs < 1:
|
||||
self.close()
|
||||
else:
|
||||
self._makefile_refs -= 1
|
||||
|
||||
|
||||
def _verify_callback(cnx, x509, err_no, err_depth, return_code):
|
||||
return err_no == 0
|
||||
|
|
|
|||
|
|
@ -1,9 +1,3 @@
|
|||
# urllib3/exceptions.py
|
||||
# Copyright 2008-2013 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
|
||||
#
|
||||
# This module is part of urllib3 and is released under
|
||||
# the MIT License: http://www.opensource.org/licenses/mit-license.php
|
||||
|
||||
|
||||
## Base Exceptions
|
||||
|
||||
|
|
@ -11,6 +5,11 @@ class HTTPError(Exception):
|
|||
"Base exception used by this module."
|
||||
pass
|
||||
|
||||
class HTTPWarning(Warning):
|
||||
"Base warning used by this module."
|
||||
pass
|
||||
|
||||
|
||||
|
||||
class PoolError(HTTPError):
|
||||
"Base exception for errors caused within a pool."
|
||||
|
|
@ -44,27 +43,38 @@ class ProxyError(HTTPError):
|
|||
pass
|
||||
|
||||
|
||||
class ConnectionError(HTTPError):
|
||||
"Raised when a normal connection fails."
|
||||
pass
|
||||
|
||||
|
||||
class DecodeError(HTTPError):
|
||||
"Raised when automatic decoding based on Content-Type fails."
|
||||
pass
|
||||
|
||||
|
||||
class ProtocolError(HTTPError):
|
||||
"Raised when something unexpected happens mid-request/response."
|
||||
pass
|
||||
|
||||
|
||||
#: Renamed to ProtocolError but aliased for backwards compatibility.
|
||||
ConnectionError = ProtocolError
|
||||
|
||||
|
||||
## Leaf Exceptions
|
||||
|
||||
class MaxRetryError(RequestError):
|
||||
"Raised when the maximum number of retries is exceeded."
|
||||
"""Raised when the maximum number of retries is exceeded.
|
||||
|
||||
:param pool: The connection pool
|
||||
:type pool: :class:`~urllib3.connectionpool.HTTPConnectionPool`
|
||||
:param string url: The requested Url
|
||||
:param exceptions.Exception reason: The underlying error
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, pool, url, reason=None):
|
||||
self.reason = reason
|
||||
|
||||
message = "Max retries exceeded with url: %s" % url
|
||||
if reason:
|
||||
message += " (Caused by %s: %s)" % (type(reason), reason)
|
||||
message += " (Caused by %r)" % reason
|
||||
else:
|
||||
message += " (Caused by redirect)"
|
||||
|
||||
|
|
@ -116,7 +126,12 @@ class ClosedPoolError(PoolError):
|
|||
pass
|
||||
|
||||
|
||||
class LocationParseError(ValueError, HTTPError):
|
||||
class LocationValueError(ValueError, HTTPError):
|
||||
"Raised when there is something wrong with a given URL input."
|
||||
pass
|
||||
|
||||
|
||||
class LocationParseError(LocationValueError):
|
||||
"Raised when get_host or similar fails to parse the URL input."
|
||||
|
||||
def __init__(self, location):
|
||||
|
|
@ -124,3 +139,18 @@ class LocationParseError(ValueError, HTTPError):
|
|||
HTTPError.__init__(self, message)
|
||||
|
||||
self.location = location
|
||||
|
||||
|
||||
class SecurityWarning(HTTPWarning):
|
||||
"Warned when perfoming security reducing actions"
|
||||
pass
|
||||
|
||||
|
||||
class InsecureRequestWarning(SecurityWarning):
|
||||
"Warned when making an unverified HTTPS request."
|
||||
pass
|
||||
|
||||
|
||||
class SystemTimeWarning(SecurityWarning):
|
||||
"Warned when system time is suspected to be wrong"
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -1,9 +1,3 @@
|
|||
# urllib3/fields.py
|
||||
# Copyright 2008-2013 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
|
||||
#
|
||||
# This module is part of urllib3 and is released under
|
||||
# the MIT License: http://www.opensource.org/licenses/mit-license.php
|
||||
|
||||
import email.utils
|
||||
import mimetypes
|
||||
|
||||
|
|
@ -78,9 +72,10 @@ class RequestField(object):
|
|||
"""
|
||||
A :class:`~urllib3.fields.RequestField` factory from old-style tuple parameters.
|
||||
|
||||
Supports constructing :class:`~urllib3.fields.RequestField` from parameter
|
||||
of key/value strings AND key/filetuple. A filetuple is a (filename, data, MIME type)
|
||||
tuple where the MIME type is optional. For example: ::
|
||||
Supports constructing :class:`~urllib3.fields.RequestField` from
|
||||
parameter of key/value strings AND key/filetuple. A filetuple is a
|
||||
(filename, data, MIME type) tuple where the MIME type is optional.
|
||||
For example::
|
||||
|
||||
'foo': 'bar',
|
||||
'fakefile': ('foofile.txt', 'contents of foofile'),
|
||||
|
|
@ -125,8 +120,8 @@ class RequestField(object):
|
|||
'Content-Disposition' fields.
|
||||
|
||||
:param header_parts:
|
||||
A sequence of (k, v) typles or a :class:`dict` of (k, v) to format as
|
||||
`k1="v1"; k2="v2"; ...`.
|
||||
A sequence of (k, v) typles or a :class:`dict` of (k, v) to format
|
||||
as `k1="v1"; k2="v2"; ...`.
|
||||
"""
|
||||
parts = []
|
||||
iterable = header_parts
|
||||
|
|
@ -158,7 +153,8 @@ class RequestField(object):
|
|||
lines.append('\r\n')
|
||||
return '\r\n'.join(lines)
|
||||
|
||||
def make_multipart(self, content_disposition=None, content_type=None, content_location=None):
|
||||
def make_multipart(self, content_disposition=None, content_type=None,
|
||||
content_location=None):
|
||||
"""
|
||||
Makes this request field into a multipart request field.
|
||||
|
||||
|
|
@ -172,6 +168,10 @@ class RequestField(object):
|
|||
|
||||
"""
|
||||
self.headers['Content-Disposition'] = content_disposition or 'form-data'
|
||||
self.headers['Content-Disposition'] += '; '.join(['', self._render_parts((('name', self._name), ('filename', self._filename)))])
|
||||
self.headers['Content-Disposition'] += '; '.join([
|
||||
'', self._render_parts(
|
||||
(('name', self._name), ('filename', self._filename))
|
||||
)
|
||||
])
|
||||
self.headers['Content-Type'] = content_type
|
||||
self.headers['Content-Location'] = content_location
|
||||
|
|
|
|||
|
|
@ -1,11 +1,4 @@
|
|||
# urllib3/filepost.py
|
||||
# Copyright 2008-2013 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
|
||||
#
|
||||
# This module is part of urllib3 and is released under
|
||||
# the MIT License: http://www.opensource.org/licenses/mit-license.php
|
||||
|
||||
import codecs
|
||||
import mimetypes
|
||||
|
||||
from uuid import uuid4
|
||||
from io import BytesIO
|
||||
|
|
@ -38,10 +31,10 @@ def iter_field_objects(fields):
|
|||
i = iter(fields)
|
||||
|
||||
for field in i:
|
||||
if isinstance(field, RequestField):
|
||||
yield field
|
||||
else:
|
||||
yield RequestField.from_tuples(*field)
|
||||
if isinstance(field, RequestField):
|
||||
yield field
|
||||
else:
|
||||
yield RequestField.from_tuples(*field)
|
||||
|
||||
|
||||
def iter_fields(fields):
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@
|
|||
# Passes Python2.7's test suite and incorporates all the latest updates.
|
||||
# Copyright 2009 Raymond Hettinger, released under the MIT License.
|
||||
# http://code.activestate.com/recipes/576693/
|
||||
|
||||
try:
|
||||
from thread import get_ident as _get_ident
|
||||
except ImportError:
|
||||
|
|
|
|||
|
|
@ -1,9 +1,3 @@
|
|||
# urllib3/poolmanager.py
|
||||
# Copyright 2008-2014 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
|
||||
#
|
||||
# This module is part of urllib3 and is released under
|
||||
# the MIT License: http://www.opensource.org/licenses/mit-license.php
|
||||
|
||||
import logging
|
||||
|
||||
try: # Python 3
|
||||
|
|
@ -14,8 +8,10 @@ except ImportError:
|
|||
from ._collections import RecentlyUsedContainer
|
||||
from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool
|
||||
from .connectionpool import port_by_scheme
|
||||
from .exceptions import LocationValueError
|
||||
from .request import RequestMethods
|
||||
from .util import parse_url
|
||||
from .util.url import parse_url
|
||||
from .util.retry import Retry
|
||||
|
||||
|
||||
__all__ = ['PoolManager', 'ProxyManager', 'proxy_from_url']
|
||||
|
|
@ -49,7 +45,7 @@ class PoolManager(RequestMethods):
|
|||
Additional parameters are used to create fresh
|
||||
:class:`urllib3.connectionpool.ConnectionPool` instances.
|
||||
|
||||
Example: ::
|
||||
Example::
|
||||
|
||||
>>> manager = PoolManager(num_pools=2)
|
||||
>>> r = manager.request('GET', 'http://google.com/')
|
||||
|
|
@ -102,10 +98,11 @@ class PoolManager(RequestMethods):
|
|||
``urllib3.connectionpool.port_by_scheme``.
|
||||
"""
|
||||
|
||||
if not host:
|
||||
raise LocationValueError("No host specified.")
|
||||
|
||||
scheme = scheme or 'http'
|
||||
|
||||
port = port or port_by_scheme.get(scheme, 80)
|
||||
|
||||
pool_key = (scheme, host, port)
|
||||
|
||||
with self.pools.lock:
|
||||
|
|
@ -118,6 +115,7 @@ class PoolManager(RequestMethods):
|
|||
# Make a fresh ConnectionPool of the desired type
|
||||
pool = self._new_pool(scheme, host, port)
|
||||
self.pools[pool_key] = pool
|
||||
|
||||
return pool
|
||||
|
||||
def connection_from_url(self, url):
|
||||
|
|
@ -161,13 +159,18 @@ class PoolManager(RequestMethods):
|
|||
# Support relative URLs for redirecting.
|
||||
redirect_location = urljoin(url, redirect_location)
|
||||
|
||||
# RFC 2616, Section 10.3.4
|
||||
# RFC 7231, Section 6.4.4
|
||||
if response.status == 303:
|
||||
method = 'GET'
|
||||
|
||||
log.info("Redirecting %s -> %s" % (url, redirect_location))
|
||||
kw['retries'] = kw.get('retries', 3) - 1 # Persist retries countdown
|
||||
retries = kw.get('retries')
|
||||
if not isinstance(retries, Retry):
|
||||
retries = Retry.from_int(retries, redirect=redirect)
|
||||
|
||||
kw['retries'] = retries.increment(method, redirect_location)
|
||||
kw['redirect'] = redirect
|
||||
|
||||
log.info("Redirecting %s -> %s" % (url, redirect_location))
|
||||
return self.urlopen(method, redirect_location, **kw)
|
||||
|
||||
|
||||
|
|
@ -208,12 +211,16 @@ class ProxyManager(PoolManager):
|
|||
if not proxy.port:
|
||||
port = port_by_scheme.get(proxy.scheme, 80)
|
||||
proxy = proxy._replace(port=port)
|
||||
|
||||
assert proxy.scheme in ("http", "https"), \
|
||||
'Not supported proxy scheme %s' % proxy.scheme
|
||||
|
||||
self.proxy = proxy
|
||||
self.proxy_headers = proxy_headers or {}
|
||||
assert self.proxy.scheme in ("http", "https"), \
|
||||
'Not supported proxy scheme %s' % self.proxy.scheme
|
||||
|
||||
connection_pool_kw['_proxy'] = self.proxy
|
||||
connection_pool_kw['_proxy_headers'] = self.proxy_headers
|
||||
|
||||
super(ProxyManager, self).__init__(
|
||||
num_pools, headers, **connection_pool_kw)
|
||||
|
||||
|
|
@ -248,10 +255,10 @@ class ProxyManager(PoolManager):
|
|||
# For proxied HTTPS requests, httplib sets the necessary headers
|
||||
# on the CONNECT to the proxy. For HTTP, we'll definitely
|
||||
# need to set 'Host' at the very least.
|
||||
kw['headers'] = self._set_proxy_headers(url, kw.get('headers',
|
||||
self.headers))
|
||||
headers = kw.get('headers', self.headers)
|
||||
kw['headers'] = self._set_proxy_headers(url, headers)
|
||||
|
||||
return super(ProxyManager, self).urlopen(method, url, redirect, **kw)
|
||||
return super(ProxyManager, self).urlopen(method, url, redirect=redirect, **kw)
|
||||
|
||||
|
||||
def proxy_from_url(url, **kw):
|
||||
|
|
|
|||
|
|
@ -1,9 +1,3 @@
|
|||
# urllib3/request.py
|
||||
# Copyright 2008-2013 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
|
||||
#
|
||||
# This module is part of urllib3 and is released under
|
||||
# the MIT License: http://www.opensource.org/licenses/mit-license.php
|
||||
|
||||
try:
|
||||
from urllib.parse import urlencode
|
||||
except ImportError:
|
||||
|
|
@ -26,8 +20,8 @@ class RequestMethods(object):
|
|||
|
||||
Specifically,
|
||||
|
||||
:meth:`.request_encode_url` is for sending requests whose fields are encoded
|
||||
in the URL (such as GET, HEAD, DELETE).
|
||||
:meth:`.request_encode_url` is for sending requests whose fields are
|
||||
encoded in the URL (such as GET, HEAD, DELETE).
|
||||
|
||||
:meth:`.request_encode_body` is for sending requests whose fields are
|
||||
encoded in the *body* of the request using multipart or www-form-urlencoded
|
||||
|
|
@ -51,7 +45,7 @@ class RequestMethods(object):
|
|||
|
||||
def urlopen(self, method, url, body=None, headers=None,
|
||||
encode_multipart=True, multipart_boundary=None,
|
||||
**kw): # Abstract
|
||||
**kw): # Abstract
|
||||
raise NotImplemented("Classes extending RequestMethods must implement "
|
||||
"their own ``urlopen`` method.")
|
||||
|
||||
|
|
@ -61,8 +55,8 @@ class RequestMethods(object):
|
|||
``fields`` based on the ``method`` used.
|
||||
|
||||
This is a convenience method that requires the least amount of manual
|
||||
effort. It can be used in most situations, while still having the option
|
||||
to drop down to more specific methods when necessary, such as
|
||||
effort. It can be used in most situations, while still having the
|
||||
option to drop down to more specific methods when necessary, such as
|
||||
:meth:`request_encode_url`, :meth:`request_encode_body`,
|
||||
or even the lowest level :meth:`urlopen`.
|
||||
"""
|
||||
|
|
@ -70,12 +64,12 @@ class RequestMethods(object):
|
|||
|
||||
if method in self._encode_url_methods:
|
||||
return self.request_encode_url(method, url, fields=fields,
|
||||
headers=headers,
|
||||
**urlopen_kw)
|
||||
headers=headers,
|
||||
**urlopen_kw)
|
||||
else:
|
||||
return self.request_encode_body(method, url, fields=fields,
|
||||
headers=headers,
|
||||
**urlopen_kw)
|
||||
headers=headers,
|
||||
**urlopen_kw)
|
||||
|
||||
def request_encode_url(self, method, url, fields=None, **urlopen_kw):
|
||||
"""
|
||||
|
|
@ -94,18 +88,18 @@ class RequestMethods(object):
|
|||
the body. This is useful for request methods like POST, PUT, PATCH, etc.
|
||||
|
||||
When ``encode_multipart=True`` (default), then
|
||||
:meth:`urllib3.filepost.encode_multipart_formdata` is used to encode the
|
||||
payload with the appropriate content type. Otherwise
|
||||
:meth:`urllib3.filepost.encode_multipart_formdata` is used to encode
|
||||
the payload with the appropriate content type. Otherwise
|
||||
:meth:`urllib.urlencode` is used with the
|
||||
'application/x-www-form-urlencoded' content type.
|
||||
|
||||
Multipart encoding must be used when posting files, and it's reasonably
|
||||
safe to use it in other times too. However, it may break request signing,
|
||||
such as with OAuth.
|
||||
safe to use it in other times too. However, it may break request
|
||||
signing, such as with OAuth.
|
||||
|
||||
Supports an optional ``fields`` parameter of key/value strings AND
|
||||
key/filetuple. A filetuple is a (filename, data, MIME type) tuple where
|
||||
the MIME type is optional. For example: ::
|
||||
the MIME type is optional. For example::
|
||||
|
||||
fields = {
|
||||
'foo': 'bar',
|
||||
|
|
@ -119,17 +113,17 @@ class RequestMethods(object):
|
|||
When uploading a file, providing a filename (the first parameter of the
|
||||
tuple) is optional but recommended to best mimick behavior of browsers.
|
||||
|
||||
Note that if ``headers`` are supplied, the 'Content-Type' header will be
|
||||
overwritten because it depends on the dynamic random boundary string
|
||||
Note that if ``headers`` are supplied, the 'Content-Type' header will
|
||||
be overwritten because it depends on the dynamic random boundary string
|
||||
which is used to compose the body of the request. The random boundary
|
||||
string can be explicitly set with the ``multipart_boundary`` parameter.
|
||||
"""
|
||||
if encode_multipart:
|
||||
body, content_type = encode_multipart_formdata(fields or {},
|
||||
boundary=multipart_boundary)
|
||||
body, content_type = encode_multipart_formdata(
|
||||
fields or {}, boundary=multipart_boundary)
|
||||
else:
|
||||
body, content_type = (urlencode(fields or {}),
|
||||
'application/x-www-form-urlencoded')
|
||||
'application/x-www-form-urlencoded')
|
||||
|
||||
if headers is None:
|
||||
headers = self.headers
|
||||
|
|
|
|||
|
|
@ -1,22 +1,14 @@
|
|||
# urllib3/response.py
|
||||
# Copyright 2008-2013 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
|
||||
#
|
||||
# This module is part of urllib3 and is released under
|
||||
# the MIT License: http://www.opensource.org/licenses/mit-license.php
|
||||
|
||||
|
||||
import logging
|
||||
import zlib
|
||||
import io
|
||||
from socket import timeout as SocketTimeout
|
||||
|
||||
from ._collections import HTTPHeaderDict
|
||||
from .exceptions import DecodeError
|
||||
from .exceptions import ProtocolError, DecodeError, ReadTimeoutError
|
||||
from .packages.six import string_types as basestring, binary_type
|
||||
from .util import is_fp_closed
|
||||
from .connection import HTTPException, BaseSSLError
|
||||
from .util.response import is_fp_closed
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DeflateDecoder(object):
|
||||
|
||||
|
|
@ -56,7 +48,10 @@ class HTTPResponse(io.IOBase):
|
|||
HTTP Response container.
|
||||
|
||||
Backwards-compatible to httplib's HTTPResponse but the response ``body`` is
|
||||
loaded and decoded on-demand when the ``data`` property is accessed.
|
||||
loaded and decoded on-demand when the ``data`` property is accessed. This
|
||||
class is also compatible with the Python standard library's :mod:`io`
|
||||
module, and can hence be treated as a readable object in the context of that
|
||||
framework.
|
||||
|
||||
Extra parameters for behaviour not present in httplib.HTTPResponse:
|
||||
|
||||
|
|
@ -91,11 +86,14 @@ class HTTPResponse(io.IOBase):
|
|||
self.decode_content = decode_content
|
||||
|
||||
self._decoder = None
|
||||
self._body = body if body and isinstance(body, basestring) else None
|
||||
self._body = None
|
||||
self._fp = None
|
||||
self._original_response = original_response
|
||||
self._fp_bytes_read = 0
|
||||
|
||||
if body and isinstance(body, (basestring, binary_type)):
|
||||
self._body = body
|
||||
|
||||
self._pool = pool
|
||||
self._connection = connection
|
||||
|
||||
|
|
@ -163,8 +161,8 @@ class HTTPResponse(io.IOBase):
|
|||
after having ``.read()`` the file object. (Overridden if ``amt`` is
|
||||
set.)
|
||||
"""
|
||||
# Note: content-encoding value should be case-insensitive, per RFC 2616
|
||||
# Section 3.5
|
||||
# Note: content-encoding value should be case-insensitive, per RFC 7230
|
||||
# Section 3.2
|
||||
content_encoding = self.headers.get('content-encoding', '').lower()
|
||||
if self._decoder is None:
|
||||
if content_encoding in self.CONTENT_DECODERS:
|
||||
|
|
@ -178,23 +176,42 @@ class HTTPResponse(io.IOBase):
|
|||
flush_decoder = False
|
||||
|
||||
try:
|
||||
if amt is None:
|
||||
# cStringIO doesn't like amt=None
|
||||
data = self._fp.read()
|
||||
flush_decoder = True
|
||||
else:
|
||||
cache_content = False
|
||||
data = self._fp.read(amt)
|
||||
if amt != 0 and not data: # Platform-specific: Buggy versions of Python.
|
||||
# Close the connection when no data is returned
|
||||
#
|
||||
# This is redundant to what httplib/http.client _should_
|
||||
# already do. However, versions of python released before
|
||||
# December 15, 2012 (http://bugs.python.org/issue16298) do not
|
||||
# properly close the connection in all cases. There is no harm
|
||||
# in redundantly calling close.
|
||||
self._fp.close()
|
||||
try:
|
||||
if amt is None:
|
||||
# cStringIO doesn't like amt=None
|
||||
data = self._fp.read()
|
||||
flush_decoder = True
|
||||
else:
|
||||
cache_content = False
|
||||
data = self._fp.read(amt)
|
||||
if amt != 0 and not data: # Platform-specific: Buggy versions of Python.
|
||||
# Close the connection when no data is returned
|
||||
#
|
||||
# This is redundant to what httplib/http.client _should_
|
||||
# already do. However, versions of python released before
|
||||
# December 15, 2012 (http://bugs.python.org/issue16298) do
|
||||
# not properly close the connection in all cases. There is
|
||||
# no harm in redundantly calling close.
|
||||
self._fp.close()
|
||||
flush_decoder = True
|
||||
|
||||
except SocketTimeout:
|
||||
# FIXME: Ideally we'd like to include the url in the ReadTimeoutError but
|
||||
# there is yet no clean way to get at it from this context.
|
||||
raise ReadTimeoutError(self._pool, None, 'Read timed out.')
|
||||
|
||||
except BaseSSLError as e:
|
||||
# FIXME: Is there a better way to differentiate between SSLErrors?
|
||||
if not 'read operation timed out' in str(e): # Defensive:
|
||||
# This shouldn't happen but just in case we're missing an edge
|
||||
# case, let's avoid swallowing SSL errors.
|
||||
raise
|
||||
|
||||
raise ReadTimeoutError(self._pool, None, 'Read timed out.')
|
||||
|
||||
except HTTPException as e:
|
||||
# This includes IncompleteRead.
|
||||
raise ProtocolError('Connection broken: %r' % e, e)
|
||||
|
||||
self._fp_bytes_read += len(data)
|
||||
|
||||
|
|
@ -204,8 +221,7 @@ class HTTPResponse(io.IOBase):
|
|||
except (IOError, zlib.error) as e:
|
||||
raise DecodeError(
|
||||
"Received response with content-encoding: %s, but "
|
||||
"failed to decode it." % content_encoding,
|
||||
e)
|
||||
"failed to decode it." % content_encoding, e)
|
||||
|
||||
if flush_decoder and decode_content and self._decoder:
|
||||
buf = self._decoder.decompress(binary_type())
|
||||
|
|
@ -242,7 +258,6 @@ class HTTPResponse(io.IOBase):
|
|||
if data:
|
||||
yield data
|
||||
|
||||
|
||||
@classmethod
|
||||
def from_httplib(ResponseCls, r, **response_kw):
|
||||
"""
|
||||
|
|
@ -297,7 +312,7 @@ class HTTPResponse(io.IOBase):
|
|||
elif hasattr(self._fp, "fileno"):
|
||||
return self._fp.fileno()
|
||||
else:
|
||||
raise IOError("The file-like object this HTTPResponse is wrapped "
|
||||
raise IOError("The file-like object this HTTPResponse is wrapped "
|
||||
"around has no file descriptor")
|
||||
|
||||
def flush(self):
|
||||
|
|
@ -305,4 +320,14 @@ class HTTPResponse(io.IOBase):
|
|||
return self._fp.flush()
|
||||
|
||||
def readable(self):
|
||||
# This method is required for `io` module compatibility.
|
||||
return True
|
||||
|
||||
def readinto(self, b):
|
||||
# This method is required for `io` module compatibility.
|
||||
temp = self.read(len(b))
|
||||
if len(temp) == 0:
|
||||
return 0
|
||||
else:
|
||||
b[:len(temp)] = temp
|
||||
return len(temp)
|
||||
|
|
|
|||
|
|
@ -1,9 +1,4 @@
|
|||
# urllib3/util/__init__.py
|
||||
# Copyright 2008-2014 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
|
||||
#
|
||||
# This module is part of urllib3 and is released under
|
||||
# the MIT License: http://www.opensource.org/licenses/mit-license.php
|
||||
|
||||
# For backwards compatibility, provide imports that used to be here.
|
||||
from .connection import is_connection_dropped
|
||||
from .request import make_headers
|
||||
from .response import is_fp_closed
|
||||
|
|
@ -19,6 +14,8 @@ from .timeout import (
|
|||
current_time,
|
||||
Timeout,
|
||||
)
|
||||
|
||||
from .retry import Retry
|
||||
from .url import (
|
||||
get_host,
|
||||
parse_url,
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
from socket import error as SocketError
|
||||
import socket
|
||||
try:
|
||||
from select import poll, POLLIN
|
||||
except ImportError: # `poll` doesn't exist on OSX and other platforms
|
||||
|
|
@ -8,6 +8,7 @@ except ImportError: # `poll` doesn't exist on OSX and other platforms
|
|||
except ImportError: # `select` doesn't exist on AppEngine.
|
||||
select = False
|
||||
|
||||
|
||||
def is_connection_dropped(conn): # Platform-specific
|
||||
"""
|
||||
Returns True if the connection is dropped and should be closed.
|
||||
|
|
@ -22,7 +23,7 @@ def is_connection_dropped(conn): # Platform-specific
|
|||
if sock is False: # Platform-specific: AppEngine
|
||||
return False
|
||||
if sock is None: # Connection already closed (such as by httplib).
|
||||
return False
|
||||
return True
|
||||
|
||||
if not poll:
|
||||
if not select: # Platform-specific: AppEngine
|
||||
|
|
@ -30,7 +31,7 @@ def is_connection_dropped(conn): # Platform-specific
|
|||
|
||||
try:
|
||||
return select([sock], [], [], 0.0)[0]
|
||||
except SocketError:
|
||||
except socket.error:
|
||||
return True
|
||||
|
||||
# This version is better on platforms that support it.
|
||||
|
|
@ -42,4 +43,55 @@ def is_connection_dropped(conn): # Platform-specific
|
|||
return True
|
||||
|
||||
|
||||
# This function is copied from socket.py in the Python 2.7 standard
|
||||
# library test suite. Added to its signature is only `socket_options`.
|
||||
def create_connection(address, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
|
||||
source_address=None, socket_options=None):
|
||||
"""Connect to *address* and return the socket object.
|
||||
|
||||
Convenience function. Connect to *address* (a 2-tuple ``(host,
|
||||
port)``) and return the socket object. Passing the optional
|
||||
*timeout* parameter will set the timeout on the socket instance
|
||||
before attempting to connect. If no *timeout* is supplied, the
|
||||
global default timeout setting returned by :func:`getdefaulttimeout`
|
||||
is used. If *source_address* is set it must be a tuple of (host, port)
|
||||
for the socket to bind as a source address before making the connection.
|
||||
An host of '' or port 0 tells the OS to use the default.
|
||||
"""
|
||||
|
||||
host, port = address
|
||||
err = None
|
||||
for res in socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM):
|
||||
af, socktype, proto, canonname, sa = res
|
||||
sock = None
|
||||
try:
|
||||
sock = socket.socket(af, socktype, proto)
|
||||
|
||||
# If provided, set socket level options before connecting.
|
||||
# This is the only addition urllib3 makes to this function.
|
||||
_set_socket_options(sock, socket_options)
|
||||
|
||||
if timeout is not socket._GLOBAL_DEFAULT_TIMEOUT:
|
||||
sock.settimeout(timeout)
|
||||
if source_address:
|
||||
sock.bind(source_address)
|
||||
sock.connect(sa)
|
||||
return sock
|
||||
|
||||
except socket.error as _:
|
||||
err = _
|
||||
if sock is not None:
|
||||
sock.close()
|
||||
|
||||
if err is not None:
|
||||
raise err
|
||||
else:
|
||||
raise socket.error("getaddrinfo returns an empty list")
|
||||
|
||||
|
||||
def _set_socket_options(sock, options):
|
||||
if options is None:
|
||||
return
|
||||
|
||||
for opt in options:
|
||||
sock.setsockopt(*opt)
|
||||
|
|
|
|||
|
|
@ -1,13 +1,12 @@
|
|||
from base64 import b64encode
|
||||
|
||||
from ..packages import six
|
||||
|
||||
from ..packages.six import b
|
||||
|
||||
ACCEPT_ENCODING = 'gzip,deflate'
|
||||
|
||||
|
||||
def make_headers(keep_alive=None, accept_encoding=None, user_agent=None,
|
||||
basic_auth=None, proxy_basic_auth=None):
|
||||
basic_auth=None, proxy_basic_auth=None, disable_cache=None):
|
||||
"""
|
||||
Shortcuts for generating request headers.
|
||||
|
||||
|
|
@ -32,7 +31,10 @@ def make_headers(keep_alive=None, accept_encoding=None, user_agent=None,
|
|||
Colon-separated username:password string for 'proxy-authorization: basic ...'
|
||||
auth header.
|
||||
|
||||
Example: ::
|
||||
:param disable_cache:
|
||||
If ``True``, adds 'cache-control: no-cache' header.
|
||||
|
||||
Example::
|
||||
|
||||
>>> make_headers(keep_alive=True, user_agent="Batman/1.0")
|
||||
{'connection': 'keep-alive', 'user-agent': 'Batman/1.0'}
|
||||
|
|
@ -57,12 +59,13 @@ def make_headers(keep_alive=None, accept_encoding=None, user_agent=None,
|
|||
|
||||
if basic_auth:
|
||||
headers['authorization'] = 'Basic ' + \
|
||||
b64encode(six.b(basic_auth)).decode('utf-8')
|
||||
b64encode(b(basic_auth)).decode('utf-8')
|
||||
|
||||
if proxy_basic_auth:
|
||||
headers['proxy-authorization'] = 'Basic ' + \
|
||||
b64encode(six.b(proxy_basic_auth)).decode('utf-8')
|
||||
b64encode(b(proxy_basic_auth)).decode('utf-8')
|
||||
|
||||
if disable_cache:
|
||||
headers['cache-control'] = 'no-cache'
|
||||
|
||||
return headers
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -5,9 +5,18 @@ def is_fp_closed(obj):
|
|||
:param obj:
|
||||
The file-like object to check.
|
||||
"""
|
||||
if hasattr(obj, 'fp'):
|
||||
# Object is a container for another file-like object that gets released
|
||||
# on exhaustion (e.g. HTTPResponse)
|
||||
return obj.fp is None
|
||||
|
||||
return obj.closed
|
||||
try:
|
||||
# Check via the official file-like-object way.
|
||||
return obj.closed
|
||||
except AttributeError:
|
||||
pass
|
||||
|
||||
try:
|
||||
# Check if the object is a container for another file-like object that
|
||||
# gets released on exhaustion (e.g. HTTPResponse).
|
||||
return obj.fp is None
|
||||
except AttributeError:
|
||||
pass
|
||||
|
||||
raise ValueError("Unable to determine whether fp is closed.")
|
||||
|
|
|
|||
|
|
@ -0,0 +1,279 @@
|
|||
import time
|
||||
import logging
|
||||
|
||||
from ..exceptions import (
|
||||
ProtocolError,
|
||||
ConnectTimeoutError,
|
||||
ReadTimeoutError,
|
||||
MaxRetryError,
|
||||
)
|
||||
from ..packages import six
|
||||
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Retry(object):
|
||||
""" Retry configuration.
|
||||
|
||||
Each retry attempt will create a new Retry object with updated values, so
|
||||
they can be safely reused.
|
||||
|
||||
Retries can be defined as a default for a pool::
|
||||
|
||||
retries = Retry(connect=5, read=2, redirect=5)
|
||||
http = PoolManager(retries=retries)
|
||||
response = http.request('GET', 'http://example.com/')
|
||||
|
||||
Or per-request (which overrides the default for the pool)::
|
||||
|
||||
response = http.request('GET', 'http://example.com/', retries=Retry(10))
|
||||
|
||||
Retries can be disabled by passing ``False``::
|
||||
|
||||
response = http.request('GET', 'http://example.com/', retries=False)
|
||||
|
||||
Errors will be wrapped in :class:`~urllib3.exceptions.MaxRetryError` unless
|
||||
retries are disabled, in which case the causing exception will be raised.
|
||||
|
||||
|
||||
:param int total:
|
||||
Total number of retries to allow. Takes precedence over other counts.
|
||||
|
||||
Set to ``None`` to remove this constraint and fall back on other
|
||||
counts. It's a good idea to set this to some sensibly-high value to
|
||||
account for unexpected edge cases and avoid infinite retry loops.
|
||||
|
||||
Set to ``0`` to fail on the first retry.
|
||||
|
||||
Set to ``False`` to disable and imply ``raise_on_redirect=False``.
|
||||
|
||||
:param int connect:
|
||||
How many connection-related errors to retry on.
|
||||
|
||||
These are errors raised before the request is sent to the remote server,
|
||||
which we assume has not triggered the server to process the request.
|
||||
|
||||
Set to ``0`` to fail on the first retry of this type.
|
||||
|
||||
:param int read:
|
||||
How many times to retry on read errors.
|
||||
|
||||
These errors are raised after the request was sent to the server, so the
|
||||
request may have side-effects.
|
||||
|
||||
Set to ``0`` to fail on the first retry of this type.
|
||||
|
||||
:param int redirect:
|
||||
How many redirects to perform. Limit this to avoid infinite redirect
|
||||
loops.
|
||||
|
||||
A redirect is a HTTP response with a status code 301, 302, 303, 307 or
|
||||
308.
|
||||
|
||||
Set to ``0`` to fail on the first retry of this type.
|
||||
|
||||
Set to ``False`` to disable and imply ``raise_on_redirect=False``.
|
||||
|
||||
:param iterable method_whitelist:
|
||||
Set of uppercased HTTP method verbs that we should retry on.
|
||||
|
||||
By default, we only retry on methods which are considered to be
|
||||
indempotent (multiple requests with the same parameters end with the
|
||||
same state). See :attr:`Retry.DEFAULT_METHOD_WHITELIST`.
|
||||
|
||||
:param iterable status_forcelist:
|
||||
A set of HTTP status codes that we should force a retry on.
|
||||
|
||||
By default, this is disabled with ``None``.
|
||||
|
||||
:param float backoff_factor:
|
||||
A backoff factor to apply between attempts. urllib3 will sleep for::
|
||||
|
||||
{backoff factor} * (2 ^ ({number of total retries} - 1))
|
||||
|
||||
seconds. If the backoff_factor is 0.1, then :func:`.sleep` will sleep
|
||||
for [0.1s, 0.2s, 0.4s, ...] between retries. It will never be longer
|
||||
than :attr:`Retry.MAX_BACKOFF`.
|
||||
|
||||
By default, backoff is disabled (set to 0).
|
||||
|
||||
:param bool raise_on_redirect: Whether, if the number of redirects is
|
||||
exhausted, to raise a MaxRetryError, or to return a response with a
|
||||
response code in the 3xx range.
|
||||
"""
|
||||
|
||||
DEFAULT_METHOD_WHITELIST = frozenset([
|
||||
'HEAD', 'GET', 'PUT', 'DELETE', 'OPTIONS', 'TRACE'])
|
||||
|
||||
#: Maximum backoff time.
|
||||
BACKOFF_MAX = 120
|
||||
|
||||
def __init__(self, total=10, connect=None, read=None, redirect=None,
|
||||
method_whitelist=DEFAULT_METHOD_WHITELIST, status_forcelist=None,
|
||||
backoff_factor=0, raise_on_redirect=True, _observed_errors=0):
|
||||
|
||||
self.total = total
|
||||
self.connect = connect
|
||||
self.read = read
|
||||
|
||||
if redirect is False or total is False:
|
||||
redirect = 0
|
||||
raise_on_redirect = False
|
||||
|
||||
self.redirect = redirect
|
||||
self.status_forcelist = status_forcelist or set()
|
||||
self.method_whitelist = method_whitelist
|
||||
self.backoff_factor = backoff_factor
|
||||
self.raise_on_redirect = raise_on_redirect
|
||||
self._observed_errors = _observed_errors # TODO: use .history instead?
|
||||
|
||||
def new(self, **kw):
|
||||
params = dict(
|
||||
total=self.total,
|
||||
connect=self.connect, read=self.read, redirect=self.redirect,
|
||||
method_whitelist=self.method_whitelist,
|
||||
status_forcelist=self.status_forcelist,
|
||||
backoff_factor=self.backoff_factor,
|
||||
raise_on_redirect=self.raise_on_redirect,
|
||||
_observed_errors=self._observed_errors,
|
||||
)
|
||||
params.update(kw)
|
||||
return type(self)(**params)
|
||||
|
||||
@classmethod
|
||||
def from_int(cls, retries, redirect=True, default=None):
|
||||
""" Backwards-compatibility for the old retries format."""
|
||||
if retries is None:
|
||||
retries = default if default is not None else cls.DEFAULT
|
||||
|
||||
if isinstance(retries, Retry):
|
||||
return retries
|
||||
|
||||
redirect = bool(redirect) and None
|
||||
new_retries = cls(retries, redirect=redirect)
|
||||
log.debug("Converted retries value: %r -> %r" % (retries, new_retries))
|
||||
return new_retries
|
||||
|
||||
def get_backoff_time(self):
|
||||
""" Formula for computing the current backoff
|
||||
|
||||
:rtype: float
|
||||
"""
|
||||
if self._observed_errors <= 1:
|
||||
return 0
|
||||
|
||||
backoff_value = self.backoff_factor * (2 ** (self._observed_errors - 1))
|
||||
return min(self.BACKOFF_MAX, backoff_value)
|
||||
|
||||
def sleep(self):
|
||||
""" Sleep between retry attempts using an exponential backoff.
|
||||
|
||||
By default, the backoff factor is 0 and this method will return
|
||||
immediately.
|
||||
"""
|
||||
backoff = self.get_backoff_time()
|
||||
if backoff <= 0:
|
||||
return
|
||||
time.sleep(backoff)
|
||||
|
||||
def _is_connection_error(self, err):
|
||||
""" Errors when we're fairly sure that the server did not receive the
|
||||
request, so it should be safe to retry.
|
||||
"""
|
||||
return isinstance(err, ConnectTimeoutError)
|
||||
|
||||
def _is_read_error(self, err):
|
||||
""" Errors that occur after the request has been started, so we can't
|
||||
assume that the server did not process any of it.
|
||||
"""
|
||||
return isinstance(err, (ReadTimeoutError, ProtocolError))
|
||||
|
||||
def is_forced_retry(self, method, status_code):
|
||||
""" Is this method/response retryable? (Based on method/codes whitelists)
|
||||
"""
|
||||
if self.method_whitelist and method.upper() not in self.method_whitelist:
|
||||
return False
|
||||
|
||||
return self.status_forcelist and status_code in self.status_forcelist
|
||||
|
||||
def is_exhausted(self):
|
||||
""" Are we out of retries?
|
||||
"""
|
||||
retry_counts = (self.total, self.connect, self.read, self.redirect)
|
||||
retry_counts = list(filter(None, retry_counts))
|
||||
if not retry_counts:
|
||||
return False
|
||||
|
||||
return min(retry_counts) < 0
|
||||
|
||||
def increment(self, method=None, url=None, response=None, error=None, _pool=None, _stacktrace=None):
|
||||
""" Return a new Retry object with incremented retry counters.
|
||||
|
||||
:param response: A response object, or None, if the server did not
|
||||
return a response.
|
||||
:type response: :class:`~urllib3.response.HTTPResponse`
|
||||
:param Exception error: An error encountered during the request, or
|
||||
None if the response was received successfully.
|
||||
|
||||
:return: A new ``Retry`` object.
|
||||
"""
|
||||
if self.total is False and error:
|
||||
# Disabled, indicate to re-raise the error.
|
||||
raise six.reraise(type(error), error, _stacktrace)
|
||||
|
||||
total = self.total
|
||||
if total is not None:
|
||||
total -= 1
|
||||
|
||||
_observed_errors = self._observed_errors
|
||||
connect = self.connect
|
||||
read = self.read
|
||||
redirect = self.redirect
|
||||
|
||||
if error and self._is_connection_error(error):
|
||||
# Connect retry?
|
||||
if connect is False:
|
||||
raise six.reraise(type(error), error, _stacktrace)
|
||||
elif connect is not None:
|
||||
connect -= 1
|
||||
_observed_errors += 1
|
||||
|
||||
elif error and self._is_read_error(error):
|
||||
# Read retry?
|
||||
if read is False:
|
||||
raise six.reraise(type(error), error, _stacktrace)
|
||||
elif read is not None:
|
||||
read -= 1
|
||||
_observed_errors += 1
|
||||
|
||||
elif response and response.get_redirect_location():
|
||||
# Redirect retry?
|
||||
if redirect is not None:
|
||||
redirect -= 1
|
||||
|
||||
else:
|
||||
# FIXME: Nothing changed, scenario doesn't make sense.
|
||||
_observed_errors += 1
|
||||
|
||||
new_retry = self.new(
|
||||
total=total,
|
||||
connect=connect, read=read, redirect=redirect,
|
||||
_observed_errors=_observed_errors)
|
||||
|
||||
if new_retry.is_exhausted():
|
||||
raise MaxRetryError(_pool, url, error)
|
||||
|
||||
log.debug("Incremented Retry for (url='%s'): %r" % (url, new_retry))
|
||||
|
||||
return new_retry
|
||||
|
||||
|
||||
def __repr__(self):
|
||||
return ('{cls.__name__}(total={self.total}, connect={self.connect}, '
|
||||
'read={self.read}, redirect={self.redirect})').format(
|
||||
cls=type(self), self=self)
|
||||
|
||||
|
||||
# For backwards compatibility (equivalent to pre-v1.9):
|
||||
Retry.DEFAULT = Retry(3)
|
||||
|
|
@ -34,10 +34,9 @@ def assert_fingerprint(cert, fingerprint):
|
|||
}
|
||||
|
||||
fingerprint = fingerprint.replace(':', '').lower()
|
||||
digest_length, odd = divmod(len(fingerprint), 2)
|
||||
|
||||
digest_length, rest = divmod(len(fingerprint), 2)
|
||||
|
||||
if rest or digest_length not in hashfunc_map:
|
||||
if odd or digest_length not in hashfunc_map:
|
||||
raise SSLError('Fingerprint is of invalid length.')
|
||||
|
||||
# We need encode() here for py32; works on py2 and p33.
|
||||
|
|
|
|||
|
|
@ -1,32 +1,49 @@
|
|||
# The default socket timeout, used by httplib to indicate that no timeout was
|
||||
# specified by the user
|
||||
from socket import _GLOBAL_DEFAULT_TIMEOUT
|
||||
import time
|
||||
|
||||
from ..exceptions import TimeoutStateError
|
||||
|
||||
# A sentinel value to indicate that no timeout was specified by the user in
|
||||
# urllib3
|
||||
_Default = object()
|
||||
|
||||
def current_time():
|
||||
"""
|
||||
Retrieve the current time, this function is mocked out in unit testing.
|
||||
Retrieve the current time. This function is mocked out in unit testing.
|
||||
"""
|
||||
return time.time()
|
||||
|
||||
|
||||
_Default = object()
|
||||
# The default timeout to use for socket connections. This is the attribute used
|
||||
# by httplib to define the default timeout
|
||||
|
||||
|
||||
class Timeout(object):
|
||||
"""
|
||||
Utility object for storing timeout values.
|
||||
""" Timeout configuration.
|
||||
|
||||
Example usage:
|
||||
Timeouts can be defined as a default for a pool::
|
||||
|
||||
.. code-block:: python
|
||||
timeout = Timeout(connect=2.0, read=7.0)
|
||||
http = PoolManager(timeout=timeout)
|
||||
response = http.request('GET', 'http://example.com/')
|
||||
|
||||
timeout = urllib3.util.Timeout(connect=2.0, read=7.0)
|
||||
pool = HTTPConnectionPool('www.google.com', 80, timeout=timeout)
|
||||
pool.request(...) # Etc, etc
|
||||
Or per-request (which overrides the default for the pool)::
|
||||
|
||||
response = http.request('GET', 'http://example.com/', timeout=Timeout(10))
|
||||
|
||||
Timeouts can be disabled by setting all the parameters to ``None``::
|
||||
|
||||
no_timeout = Timeout(connect=None, read=None)
|
||||
response = http.request('GET', 'http://example.com/, timeout=no_timeout)
|
||||
|
||||
|
||||
:param total:
|
||||
This combines the connect and read timeouts into one; the read timeout
|
||||
will be set to the time leftover from the connect attempt. In the
|
||||
event that both a connect timeout and a total are specified, or a read
|
||||
timeout and a total are specified, the shorter timeout will be applied.
|
||||
|
||||
Defaults to None.
|
||||
|
||||
:type total: integer, float, or None
|
||||
|
||||
:param connect:
|
||||
The maximum amount of time to wait for a connection attempt to a server
|
||||
|
|
@ -47,25 +64,15 @@ class Timeout(object):
|
|||
|
||||
:type read: integer, float, or None
|
||||
|
||||
:param total:
|
||||
This combines the connect and read timeouts into one; the read timeout
|
||||
will be set to the time leftover from the connect attempt. In the
|
||||
event that both a connect timeout and a total are specified, or a read
|
||||
timeout and a total are specified, the shorter timeout will be applied.
|
||||
|
||||
Defaults to None.
|
||||
|
||||
:type total: integer, float, or None
|
||||
|
||||
.. note::
|
||||
|
||||
Many factors can affect the total amount of time for urllib3 to return
|
||||
an HTTP response. Specifically, Python's DNS resolver does not obey the
|
||||
timeout specified on the socket. Other factors that can affect total
|
||||
request time include high CPU load, high swap, the program running at a
|
||||
low priority level, or other behaviors. The observed running time for
|
||||
urllib3 to return a response may be greater than the value passed to
|
||||
`total`.
|
||||
an HTTP response.
|
||||
|
||||
For example, Python's DNS resolver does not obey the timeout specified
|
||||
on the socket. Other factors that can affect total request time include
|
||||
high CPU load, high swap, the program running at a low priority level,
|
||||
or other behaviors.
|
||||
|
||||
In addition, the read and total timeouts only measure the time between
|
||||
read operations on the socket connecting the client and the server,
|
||||
|
|
@ -73,8 +80,8 @@ class Timeout(object):
|
|||
response. For most requests, the timeout is raised because the server
|
||||
has not sent the first byte in the specified time. This is not always
|
||||
the case; if a server streams one byte every fifteen seconds, a timeout
|
||||
of 20 seconds will not ever trigger, even though the request will
|
||||
take several minutes to complete.
|
||||
of 20 seconds will not trigger, even though the request will take
|
||||
several minutes to complete.
|
||||
|
||||
If your goal is to cut off any request after a set amount of wall clock
|
||||
time, consider having a second "watcher" thread to cut off a slow
|
||||
|
|
@ -94,17 +101,16 @@ class Timeout(object):
|
|||
return '%s(connect=%r, read=%r, total=%r)' % (
|
||||
type(self).__name__, self._connect, self._read, self.total)
|
||||
|
||||
|
||||
@classmethod
|
||||
def _validate_timeout(cls, value, name):
|
||||
""" Check that a timeout attribute is valid
|
||||
""" Check that a timeout attribute is valid.
|
||||
|
||||
:param value: The timeout value to validate
|
||||
:param name: The name of the timeout attribute to validate. This is used
|
||||
for clear error messages
|
||||
:return: the value
|
||||
:raises ValueError: if the type is not an integer or a float, or if it
|
||||
is a numeric value less than zero
|
||||
:param name: The name of the timeout attribute to validate. This is
|
||||
used to specify in error messages.
|
||||
:return: The validated and casted version of the given value.
|
||||
:raises ValueError: If the type is not an integer or a float, or if it
|
||||
is a numeric value less than zero.
|
||||
"""
|
||||
if value is _Default:
|
||||
return cls.DEFAULT_TIMEOUT
|
||||
|
|
@ -123,7 +129,7 @@ class Timeout(object):
|
|||
raise ValueError("Attempted to set %s timeout to %s, but the "
|
||||
"timeout cannot be set to a value less "
|
||||
"than 0." % (name, value))
|
||||
except TypeError: # Python 3
|
||||
except TypeError: # Python 3
|
||||
raise ValueError("Timeout value %s was %s, but it must be an "
|
||||
"int or float." % (name, value))
|
||||
|
||||
|
|
@ -135,12 +141,12 @@ class Timeout(object):
|
|||
|
||||
The timeout value used by httplib.py sets the same timeout on the
|
||||
connect(), and recv() socket requests. This creates a :class:`Timeout`
|
||||
object that sets the individual timeouts to the ``timeout`` value passed
|
||||
to this function.
|
||||
object that sets the individual timeouts to the ``timeout`` value
|
||||
passed to this function.
|
||||
|
||||
:param timeout: The legacy timeout value
|
||||
:param timeout: The legacy timeout value.
|
||||
:type timeout: integer, float, sentinel default object, or None
|
||||
:return: a Timeout object
|
||||
:return: Timeout object
|
||||
:rtype: :class:`Timeout`
|
||||
"""
|
||||
return Timeout(read=timeout, connect=timeout)
|
||||
|
|
@ -174,7 +180,7 @@ class Timeout(object):
|
|||
def get_connect_duration(self):
|
||||
""" Gets the time elapsed since the call to :meth:`start_connect`.
|
||||
|
||||
:return: the elapsed time
|
||||
:return: Elapsed time.
|
||||
:rtype: float
|
||||
:raises urllib3.exceptions.TimeoutStateError: if you attempt
|
||||
to get duration for a timer that hasn't been started.
|
||||
|
|
@ -191,7 +197,7 @@ class Timeout(object):
|
|||
This will be a positive float or integer, the value None
|
||||
(never timeout), or the default system timeout.
|
||||
|
||||
:return: the connect timeout
|
||||
:return: Connect timeout.
|
||||
:rtype: int, float, :attr:`Timeout.DEFAULT_TIMEOUT` or None
|
||||
"""
|
||||
if self.total is None:
|
||||
|
|
@ -214,7 +220,7 @@ class Timeout(object):
|
|||
established, a :exc:`~urllib3.exceptions.TimeoutStateError` will be
|
||||
raised.
|
||||
|
||||
:return: the value to use for the read timeout
|
||||
:return: Value to use for the read timeout.
|
||||
:rtype: int, float, :attr:`Timeout.DEFAULT_TIMEOUT` or None
|
||||
:raises urllib3.exceptions.TimeoutStateError: If :meth:`start_connect`
|
||||
has not yet been called on this object.
|
||||
|
|
@ -223,7 +229,7 @@ class Timeout(object):
|
|||
self.total is not self.DEFAULT_TIMEOUT and
|
||||
self._read is not None and
|
||||
self._read is not self.DEFAULT_TIMEOUT):
|
||||
# in case the connect timeout has not yet been established.
|
||||
# In case the connect timeout has not yet been established.
|
||||
if self._start_connect is None:
|
||||
return self._read
|
||||
return max(0, min(self.total - self.get_connect_duration(),
|
||||
|
|
|
|||
|
|
@ -3,15 +3,20 @@ from collections import namedtuple
|
|||
from ..exceptions import LocationParseError
|
||||
|
||||
|
||||
class Url(namedtuple('Url', ['scheme', 'auth', 'host', 'port', 'path', 'query', 'fragment'])):
|
||||
url_attrs = ['scheme', 'auth', 'host', 'port', 'path', 'query', 'fragment']
|
||||
|
||||
|
||||
class Url(namedtuple('Url', url_attrs)):
|
||||
"""
|
||||
Datastructure for representing an HTTP URL. Used as a return value for
|
||||
:func:`parse_url`.
|
||||
"""
|
||||
slots = ()
|
||||
|
||||
def __new__(cls, scheme=None, auth=None, host=None, port=None, path=None, query=None, fragment=None):
|
||||
return super(Url, cls).__new__(cls, scheme, auth, host, port, path, query, fragment)
|
||||
def __new__(cls, scheme=None, auth=None, host=None, port=None, path=None,
|
||||
query=None, fragment=None):
|
||||
return super(Url, cls).__new__(cls, scheme, auth, host, port, path,
|
||||
query, fragment)
|
||||
|
||||
@property
|
||||
def hostname(self):
|
||||
|
|
@ -43,7 +48,7 @@ def split_first(s, delims):
|
|||
|
||||
If not found, then the first part is the full input string.
|
||||
|
||||
Example: ::
|
||||
Example::
|
||||
|
||||
>>> split_first('foo/bar?baz', '?/=')
|
||||
('foo', 'bar?baz', '/')
|
||||
|
|
@ -76,7 +81,7 @@ def parse_url(url):
|
|||
|
||||
Partly backwards-compatible with :mod:`urlparse`.
|
||||
|
||||
Example: ::
|
||||
Example::
|
||||
|
||||
>>> parse_url('http://google.com/mail/')
|
||||
Url(scheme='http', host='google.com', port=None, path='/', ...)
|
||||
|
|
@ -91,6 +96,10 @@ def parse_url(url):
|
|||
# Additionally, this implementations does silly things to be optimal
|
||||
# on CPython.
|
||||
|
||||
if not url:
|
||||
# Empty
|
||||
return Url()
|
||||
|
||||
scheme = None
|
||||
auth = None
|
||||
host = None
|
||||
|
|
|
|||
|
|
@ -91,10 +91,17 @@ class SessionRedirectMixin(object):
|
|||
"""Receives a Response. Returns a generator of Responses."""
|
||||
|
||||
i = 0
|
||||
hist = [] # keep track of history
|
||||
|
||||
while resp.is_redirect:
|
||||
prepared_request = req.copy()
|
||||
|
||||
if i > 0:
|
||||
# Update history and keep track of redirects.
|
||||
hist.append(resp)
|
||||
new_hist = list(hist)
|
||||
resp.history = new_hist
|
||||
|
||||
try:
|
||||
resp.content # Consume socket so it can be released
|
||||
except (ChunkedEncodingError, ContentDecodingError, RuntimeError):
|
||||
|
|
@ -118,7 +125,7 @@ class SessionRedirectMixin(object):
|
|||
parsed = urlparse(url)
|
||||
url = parsed.geturl()
|
||||
|
||||
# Facilitate non-RFC2616-compliant 'location' headers
|
||||
# Facilitate relative 'location' headers, as allowed by RFC 7231.
|
||||
# (e.g. '/path/to/resource' instead of 'http://domain.tld/path/to/resource')
|
||||
# Compliant with RFC3986, we percent encode the url.
|
||||
if not urlparse(url).netloc:
|
||||
|
|
@ -127,8 +134,11 @@ class SessionRedirectMixin(object):
|
|||
url = requote_uri(url)
|
||||
|
||||
prepared_request.url = to_native_string(url)
|
||||
# Cache the url, unless it redirects to itself.
|
||||
if resp.is_permanent_redirect and req.url != prepared_request.url:
|
||||
self.redirect_cache[req.url] = prepared_request.url
|
||||
|
||||
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.4
|
||||
# http://tools.ietf.org/html/rfc7231#section-6.4.4
|
||||
if (resp.status_code == codes.see_other and
|
||||
method != 'HEAD'):
|
||||
method = 'GET'
|
||||
|
|
@ -146,7 +156,7 @@ class SessionRedirectMixin(object):
|
|||
prepared_request.method = method
|
||||
|
||||
# https://github.com/kennethreitz/requests/issues/1084
|
||||
if resp.status_code not in (codes.temporary, codes.resume):
|
||||
if resp.status_code not in (codes.temporary_redirect, codes.permanent_redirect):
|
||||
if 'Content-Length' in prepared_request.headers:
|
||||
del prepared_request.headers['Content-Length']
|
||||
|
||||
|
|
@ -263,7 +273,7 @@ class Session(SessionRedirectMixin):
|
|||
__attrs__ = [
|
||||
'headers', 'cookies', 'auth', 'timeout', 'proxies', 'hooks',
|
||||
'params', 'verify', 'cert', 'prefetch', 'adapters', 'stream',
|
||||
'trust_env', 'max_redirects']
|
||||
'trust_env', 'max_redirects', 'redirect_cache']
|
||||
|
||||
def __init__(self):
|
||||
|
||||
|
|
@ -316,6 +326,8 @@ class Session(SessionRedirectMixin):
|
|||
self.mount('https://', HTTPAdapter())
|
||||
self.mount('http://', HTTPAdapter())
|
||||
|
||||
self.redirect_cache = {}
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
|
|
@ -388,13 +400,16 @@ class Session(SessionRedirectMixin):
|
|||
:class:`Request`.
|
||||
:param cookies: (optional) Dict or CookieJar object to send with the
|
||||
:class:`Request`.
|
||||
:param files: (optional) Dictionary of 'filename': file-like-objects
|
||||
:param files: (optional) Dictionary of ``'filename': file-like-objects``
|
||||
for multipart encoding upload.
|
||||
:param auth: (optional) Auth tuple or callable to enable
|
||||
Basic/Digest/Custom HTTP Auth.
|
||||
:param timeout: (optional) Float describing the timeout of the
|
||||
request in seconds.
|
||||
:param allow_redirects: (optional) Boolean. Set to True by default.
|
||||
:param timeout: (optional) How long to wait for the server to send
|
||||
data before giving up, as a float, or a (`connect timeout, read
|
||||
timeout <user/advanced.html#timeouts>`_) tuple.
|
||||
:type timeout: float or tuple
|
||||
:param allow_redirects: (optional) Set to True by default.
|
||||
:type allow_redirects: bool
|
||||
:param proxies: (optional) Dictionary mapping protocol to the URL of
|
||||
the proxy.
|
||||
:param stream: (optional) whether to immediately download the response
|
||||
|
|
@ -423,36 +438,16 @@ class Session(SessionRedirectMixin):
|
|||
|
||||
proxies = proxies or {}
|
||||
|
||||
# Gather clues from the surrounding environment.
|
||||
if self.trust_env:
|
||||
# Set environment's proxies.
|
||||
env_proxies = get_environ_proxies(url) or {}
|
||||
for (k, v) in env_proxies.items():
|
||||
proxies.setdefault(k, v)
|
||||
|
||||
# Look for configuration.
|
||||
if not verify and verify is not False:
|
||||
verify = os.environ.get('REQUESTS_CA_BUNDLE')
|
||||
|
||||
# Curl compatibility.
|
||||
if not verify and verify is not False:
|
||||
verify = os.environ.get('CURL_CA_BUNDLE')
|
||||
|
||||
# Merge all the kwargs.
|
||||
proxies = merge_setting(proxies, self.proxies)
|
||||
stream = merge_setting(stream, self.stream)
|
||||
verify = merge_setting(verify, self.verify)
|
||||
cert = merge_setting(cert, self.cert)
|
||||
settings = self.merge_environment_settings(
|
||||
prep.url, proxies, stream, verify, cert
|
||||
)
|
||||
|
||||
# Send the request.
|
||||
send_kwargs = {
|
||||
'stream': stream,
|
||||
'timeout': timeout,
|
||||
'verify': verify,
|
||||
'cert': cert,
|
||||
'proxies': proxies,
|
||||
'allow_redirects': allow_redirects,
|
||||
}
|
||||
send_kwargs.update(settings)
|
||||
resp = self.send(prep, **send_kwargs)
|
||||
|
||||
return resp
|
||||
|
|
@ -540,6 +535,9 @@ class Session(SessionRedirectMixin):
|
|||
if not isinstance(request, PreparedRequest):
|
||||
raise ValueError('You can only send PreparedRequests.')
|
||||
|
||||
while request.url in self.redirect_cache:
|
||||
request.url = self.redirect_cache.get(request.url)
|
||||
|
||||
# Set up variables needed for resolve_redirects and dispatching of hooks
|
||||
allow_redirects = kwargs.pop('allow_redirects', True)
|
||||
stream = kwargs.get('stream')
|
||||
|
|
@ -597,6 +595,30 @@ class Session(SessionRedirectMixin):
|
|||
|
||||
return r
|
||||
|
||||
def merge_environment_settings(self, url, proxies, stream, verify, cert):
|
||||
"""Check the environment and merge it with some settings."""
|
||||
# Gather clues from the surrounding environment.
|
||||
if self.trust_env:
|
||||
# Set environment's proxies.
|
||||
env_proxies = get_environ_proxies(url) or {}
|
||||
for (k, v) in env_proxies.items():
|
||||
proxies.setdefault(k, v)
|
||||
|
||||
# Look for requests environment configuration and be compatible
|
||||
# with cURL.
|
||||
if verify is True or verify is None:
|
||||
verify = (os.environ.get('REQUESTS_CA_BUNDLE') or
|
||||
os.environ.get('CURL_CA_BUNDLE'))
|
||||
|
||||
# Merge all the kwargs.
|
||||
proxies = merge_setting(proxies, self.proxies)
|
||||
stream = merge_setting(stream, self.stream)
|
||||
verify = merge_setting(verify, self.verify)
|
||||
cert = merge_setting(cert, self.cert)
|
||||
|
||||
return {'verify': verify, 'proxies': proxies, 'stream': stream,
|
||||
'cert': cert}
|
||||
|
||||
def get_adapter(self, url):
|
||||
"""Returns the appropriate connnection adapter for the given URL."""
|
||||
for (prefix, adapter) in self.adapters.items():
|
||||
|
|
|
|||
|
|
@ -30,7 +30,8 @@ _codes = {
|
|||
305: ('use_proxy',),
|
||||
306: ('switch_proxy',),
|
||||
307: ('temporary_redirect', 'temporary_moved', 'temporary'),
|
||||
308: ('resume_incomplete', 'resume'),
|
||||
308: ('permanent_redirect',
|
||||
'resume_incomplete', 'resume',), # These 2 to be removed in 3.0
|
||||
|
||||
# Client Error.
|
||||
400: ('bad_request', 'bad'),
|
||||
|
|
|
|||
|
|
@ -8,30 +8,7 @@ Data structures that power Requests.
|
|||
|
||||
"""
|
||||
|
||||
import os
|
||||
import collections
|
||||
from itertools import islice
|
||||
|
||||
|
||||
class IteratorProxy(object):
|
||||
"""docstring for IteratorProxy"""
|
||||
def __init__(self, i):
|
||||
self.i = i
|
||||
# self.i = chain.from_iterable(i)
|
||||
|
||||
def __iter__(self):
|
||||
return self.i
|
||||
|
||||
def __len__(self):
|
||||
if hasattr(self.i, '__len__'):
|
||||
return len(self.i)
|
||||
if hasattr(self.i, 'len'):
|
||||
return self.i.len
|
||||
if hasattr(self.i, 'fileno'):
|
||||
return os.fstat(self.i.fileno()).st_size
|
||||
|
||||
def read(self, n):
|
||||
return "".join(islice(self.i, None, n))
|
||||
|
||||
|
||||
class CaseInsensitiveDict(collections.MutableMapping):
|
||||
|
|
@ -46,7 +23,7 @@ class CaseInsensitiveDict(collections.MutableMapping):
|
|||
case of the last key to be set, and ``iter(instance)``,
|
||||
``keys()``, ``items()``, ``iterkeys()``, and ``iteritems()``
|
||||
will contain case-sensitive keys. However, querying and contains
|
||||
testing is case insensitive:
|
||||
testing is case insensitive::
|
||||
|
||||
cid = CaseInsensitiveDict()
|
||||
cid['Accept'] = 'application/json'
|
||||
|
|
|
|||
|
|
@ -554,7 +554,8 @@ def default_headers():
|
|||
return CaseInsensitiveDict({
|
||||
'User-Agent': default_user_agent(),
|
||||
'Accept-Encoding': ', '.join(('gzip', 'deflate')),
|
||||
'Accept': '*/*'
|
||||
'Accept': '*/*',
|
||||
'Connection': 'keep-alive',
|
||||
})
|
||||
|
||||
|
||||
|
|
|
|||
0
vendor/rest_framework_extensions/bulk_operations/__init__.py
поставляемый
Executable file → Normal file
0
vendor/rest_framework_extensions/bulk_operations/__init__.py
поставляемый
Executable file → Normal file
0
vendor/rest_framework_extensions/bulk_operations/mixins.py
поставляемый
Executable file → Normal file
0
vendor/rest_framework_extensions/bulk_operations/mixins.py
поставляемый
Executable file → Normal file
|
|
@ -2,8 +2,8 @@
|
|||
from functools import wraps
|
||||
|
||||
from django.utils.decorators import available_attrs
|
||||
from django.core.cache import get_cache
|
||||
|
||||
from rest_framework_extensions.utils import get_cache
|
||||
from rest_framework_extensions.settings import extensions_api_settings
|
||||
from rest_framework_extensions.compat import six
|
||||
|
||||
|
|
|
|||
0
vendor/rest_framework_extensions/key_constructor/__init__.py
поставляемый
Executable file → Normal file
0
vendor/rest_framework_extensions/key_constructor/__init__.py
поставляемый
Executable file → Normal file
0
vendor/rest_framework_extensions/key_constructor/bits.py
поставляемый
Executable file → Normal file
0
vendor/rest_framework_extensions/key_constructor/bits.py
поставляемый
Executable file → Normal file
0
vendor/rest_framework_extensions/key_constructor/constructors.py
поставляемый
Executable file → Normal file
0
vendor/rest_framework_extensions/key_constructor/constructors.py
поставляемый
Executable file → Normal file
|
|
@ -23,14 +23,14 @@ class DetailSerializerMixin(object):
|
|||
|
||||
def get_object(self, queryset=None):
|
||||
if queryset is None:
|
||||
queryset = self.get_queryset_detail()
|
||||
queryset = self.filter_queryset(self.get_queryset(is_for_detail=True))
|
||||
return super(DetailSerializerMixin, self).get_object(queryset=queryset)
|
||||
|
||||
def get_queryset_detail(self):
|
||||
if self.queryset_detail is not None:
|
||||
def get_queryset(self, is_for_detail=False):
|
||||
if self.queryset_detail is not None and is_for_detail:
|
||||
return self.queryset_detail._clone() # todo: test _clone()
|
||||
else:
|
||||
return self.get_queryset()
|
||||
return super(DetailSerializerMixin, self).get_queryset()
|
||||
|
||||
|
||||
class PaginateByMaxMixin(object):
|
||||
|
|
@ -76,4 +76,4 @@ class NestedViewSetMixin(object):
|
|||
)
|
||||
query_value = self.kwargs.get(kwarg_name)
|
||||
result[query_lookup] = query_value
|
||||
return result
|
||||
return result
|
||||
0
vendor/rest_framework_extensions/permissions/__init__.py
поставляемый
Executable file → Normal file
0
vendor/rest_framework_extensions/permissions/__init__.py
поставляемый
Executable file → Normal file
0
vendor/rest_framework_extensions/permissions/extended_django_object_permissions.py
поставляемый
Executable file → Normal file
0
vendor/rest_framework_extensions/permissions/extended_django_object_permissions.py
поставляемый
Executable file → Normal file
|
|
@ -29,8 +29,7 @@ def get_rest_framework_features():
|
|||
def get_django_features():
|
||||
# todo: test me
|
||||
return {
|
||||
'has_odd_space_in_sql_query': django_version < (1, 7, 0),
|
||||
'caches_singleton': django_version >= (1, 7, 0), # https://docs.djangoproject.com/en/dev/releases/1.7/#cache
|
||||
'has_odd_space_in_sql_query': django_version < (1, 7, 0)
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -76,15 +75,6 @@ def compose_parent_pk_kwarg_name(value):
|
|||
value
|
||||
)
|
||||
|
||||
def get_cache(alias):
|
||||
# todo: test me
|
||||
if get_django_features()['caches_singleton']:
|
||||
from django.core.cache import caches
|
||||
return caches[alias]
|
||||
else:
|
||||
from django.core.cache import get_cache as _get_cache
|
||||
return _get_cache(alias)
|
||||
|
||||
|
||||
default_cache_key_func = DefaultKeyConstructor()
|
||||
default_object_cache_key_func = DefaultObjectKeyConstructor()
|
||||
|
|
|
|||
|
|
@ -20,13 +20,15 @@
|
|||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
from __future__ import absolute_import
|
||||
|
||||
import functools
|
||||
import operator
|
||||
import sys
|
||||
import types
|
||||
|
||||
__author__ = "Benjamin Peterson <benjamin@python.org>"
|
||||
__version__ = "1.7.3"
|
||||
__version__ = "1.8.0"
|
||||
|
||||
|
||||
# Useful for very coarse version differentiation.
|
||||
|
|
@ -225,10 +227,12 @@ _moved_attributes = [
|
|||
MovedAttribute("filter", "itertools", "builtins", "ifilter", "filter"),
|
||||
MovedAttribute("filterfalse", "itertools", "itertools", "ifilterfalse", "filterfalse"),
|
||||
MovedAttribute("input", "__builtin__", "builtins", "raw_input", "input"),
|
||||
MovedAttribute("intern", "__builtin__", "sys"),
|
||||
MovedAttribute("map", "itertools", "builtins", "imap", "map"),
|
||||
MovedAttribute("range", "__builtin__", "builtins", "xrange", "range"),
|
||||
MovedAttribute("reload_module", "__builtin__", "imp", "reload"),
|
||||
MovedAttribute("reduce", "__builtin__", "functools"),
|
||||
MovedAttribute("shlex_quote", "pipes", "shlex", "quote"),
|
||||
MovedAttribute("StringIO", "StringIO", "io"),
|
||||
MovedAttribute("UserDict", "UserDict", "collections"),
|
||||
MovedAttribute("UserList", "UserList", "collections"),
|
||||
|
|
@ -248,6 +252,7 @@ _moved_attributes = [
|
|||
MovedModule("html_parser", "HTMLParser", "html.parser"),
|
||||
MovedModule("http_client", "httplib", "http.client"),
|
||||
MovedModule("email_mime_multipart", "email.MIMEMultipart", "email.mime.multipart"),
|
||||
MovedModule("email_mime_nonmultipart", "email.MIMENonMultipart", "email.mime.nonmultipart"),
|
||||
MovedModule("email_mime_text", "email.MIMEText", "email.mime.text"),
|
||||
MovedModule("email_mime_base", "email.MIMEBase", "email.mime.base"),
|
||||
MovedModule("BaseHTTPServer", "BaseHTTPServer", "http.server"),
|
||||
|
|
@ -317,6 +322,13 @@ _urllib_parse_moved_attributes = [
|
|||
MovedAttribute("unquote_plus", "urllib", "urllib.parse"),
|
||||
MovedAttribute("urlencode", "urllib", "urllib.parse"),
|
||||
MovedAttribute("splitquery", "urllib", "urllib.parse"),
|
||||
MovedAttribute("splittag", "urllib", "urllib.parse"),
|
||||
MovedAttribute("splituser", "urllib", "urllib.parse"),
|
||||
MovedAttribute("uses_fragment", "urlparse", "urllib.parse"),
|
||||
MovedAttribute("uses_netloc", "urlparse", "urllib.parse"),
|
||||
MovedAttribute("uses_params", "urlparse", "urllib.parse"),
|
||||
MovedAttribute("uses_query", "urlparse", "urllib.parse"),
|
||||
MovedAttribute("uses_relative", "urlparse", "urllib.parse"),
|
||||
]
|
||||
for attr in _urllib_parse_moved_attributes:
|
||||
setattr(Module_six_moves_urllib_parse, attr.name, attr)
|
||||
|
|
@ -606,6 +618,8 @@ if PY3:
|
|||
|
||||
|
||||
def reraise(tp, value, tb=None):
|
||||
if value is None:
|
||||
value = tp()
|
||||
if value.__traceback__ is not tb:
|
||||
raise value.with_traceback(tb)
|
||||
raise value
|
||||
|
|
@ -687,7 +701,8 @@ if print_ is None:
|
|||
_add_doc(reraise, """Reraise an exception.""")
|
||||
|
||||
if sys.version_info[0:2] < (3, 4):
|
||||
def wraps(wrapped):
|
||||
def wraps(wrapped, assigned=functools.WRAPPER_ASSIGNMENTS,
|
||||
updated=functools.WRAPPER_UPDATES):
|
||||
def wrapper(f):
|
||||
f = functools.wraps(wrapped)(f)
|
||||
f.__wrapped__ = wrapped
|
||||
|
|
@ -711,14 +726,14 @@ def add_metaclass(metaclass):
|
|||
"""Class decorator for creating a class with a metaclass."""
|
||||
def wrapper(cls):
|
||||
orig_vars = cls.__dict__.copy()
|
||||
orig_vars.pop('__dict__', None)
|
||||
orig_vars.pop('__weakref__', None)
|
||||
slots = orig_vars.get('__slots__')
|
||||
if slots is not None:
|
||||
if isinstance(slots, str):
|
||||
slots = [slots]
|
||||
for slots_var in slots:
|
||||
orig_vars.pop(slots_var)
|
||||
orig_vars.pop('__dict__', None)
|
||||
orig_vars.pop('__weakref__', None)
|
||||
return metaclass(cls.__name__, cls.__bases__, orig_vars)
|
||||
return wrapper
|
||||
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче