breaking: Python 3.12 compatibility & remove custom SSL adapter (#3185)

Add support for Python 3.12.

`match_hostname` is gone in Python 3.12 and has been unused by
Python since 3.7.

The custom SSL adapter allows passing a specific SSL version; this
was first introduced a looong time ago to handle some SSL issues
at the time.

Closes #3176.

---------

Signed-off-by: Hugo van Kemenade <hugovk@users.noreply.github.com>
Signed-off-by: Milas Bowman <milas.bowman@docker.com>
Co-authored-by: Hugo van Kemenade <hugovk@users.noreply.github.com>
This commit is contained in:
Milas Bowman 2023-11-21 10:42:53 -05:00 коммит произвёл GitHub
Родитель 976c84c481
Коммит db4878118b
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
19 изменённых файлов: 41 добавлений и 353 удалений

12
.github/workflows/ci.yml поставляемый
Просмотреть файл

@ -4,15 +4,16 @@ on: [push, pull_request]
env:
DOCKER_BUILDKIT: '1'
FORCE_COLOR: 1
jobs:
lint:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- uses: actions/setup-python@v4
with:
python-version: '3.11'
python-version: '3.x'
- run: pip install -U ruff==0.0.284
- name: Run ruff
run: ruff docker tests
@ -21,14 +22,15 @@ jobs:
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
allow-prereleases: true
- name: Install dependencies
run: |
python3 -m pip install --upgrade pip
@ -46,7 +48,7 @@ jobs:
variant: [ "integration-dind", "integration-dind-ssl", "integration-dind-ssh" ]
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- name: make ${{ matrix.variant }}
run: |
docker logout

6
.github/workflows/release.yml поставляемый
Просмотреть файл

@ -12,11 +12,15 @@ on:
type: boolean
default: true
env:
DOCKER_BUILDKIT: '1'
FORCE_COLOR: 1
jobs:
publish:
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- uses: actions/setup-python@v4
with:

Просмотреть файл

@ -1,6 +1,6 @@
# syntax=docker/dockerfile:1
ARG PYTHON_VERSION=3.10
ARG PYTHON_VERSION=3.12
FROM python:${PYTHON_VERSION}

Просмотреть файл

@ -1,6 +1,6 @@
# syntax=docker/dockerfile:1
ARG PYTHON_VERSION=3.10
ARG PYTHON_VERSION=3.12
FROM python:${PYTHON_VERSION}

147
Jenkinsfile поставляемый
Просмотреть файл

@ -1,147 +0,0 @@
#!groovy
def imageNameBase = "dockerpinata/docker-py"
def imageNamePy3
def imageDindSSH
def images = [:]
def buildImage = { name, buildargs, pyTag ->
img = docker.image(name)
try {
img.pull()
} catch (Exception exc) {
img = docker.build(name, buildargs)
img.push()
}
if (pyTag?.trim()) images[pyTag] = img.id
}
def buildImages = { ->
wrappedNode(label: "amd64 && ubuntu-2004 && overlay2", cleanWorkspace: true) {
stage("build image") {
checkout(scm)
imageNamePy3 = "${imageNameBase}:py3-${gitCommit()}"
imageDindSSH = "${imageNameBase}:sshdind-${gitCommit()}"
withDockerRegistry(credentialsId:'dockerbuildbot-index.docker.io') {
buildImage(imageDindSSH, "-f tests/Dockerfile-ssh-dind .", "")
buildImage(imageNamePy3, "-f tests/Dockerfile --build-arg PYTHON_VERSION=3.10 .", "py3.10")
}
}
}
}
def getDockerVersions = { ->
def dockerVersions = ["19.03.12"]
wrappedNode(label: "amd64 && ubuntu-2004 && overlay2") {
def result = sh(script: """docker run --rm \\
--entrypoint=python \\
${imageNamePy3} \\
/src/scripts/versions.py
""", returnStdout: true
)
dockerVersions = dockerVersions + result.trim().tokenize(' ')
}
return dockerVersions
}
def getAPIVersion = { engineVersion ->
def versionMap = [
'18.09': '1.39',
'19.03': '1.40'
]
def result = versionMap[engineVersion.substring(0, 5)]
if (!result) {
return '1.40'
}
return result
}
def runTests = { Map settings ->
def dockerVersion = settings.get("dockerVersion", null)
def pythonVersion = settings.get("pythonVersion", null)
def testImage = settings.get("testImage", null)
def apiVersion = getAPIVersion(dockerVersion)
if (!testImage) {
throw new Exception("Need test image object, e.g.: `runTests(testImage: img)`")
}
if (!dockerVersion) {
throw new Exception("Need Docker version to test, e.g.: `runTests(dockerVersion: '19.03.12')`")
}
if (!pythonVersion) {
throw new Exception("Need Python version being tested, e.g.: `runTests(pythonVersion: 'py3.x')`")
}
{ ->
wrappedNode(label: "amd64 && ubuntu-2004 && overlay2", cleanWorkspace: true) {
stage("test python=${pythonVersion} / docker=${dockerVersion}") {
checkout(scm)
def dindContainerName = "dpy-dind-\$BUILD_NUMBER-\$EXECUTOR_NUMBER-${pythonVersion}-${dockerVersion}"
def testContainerName = "dpy-tests-\$BUILD_NUMBER-\$EXECUTOR_NUMBER-${pythonVersion}-${dockerVersion}"
def testNetwork = "dpy-testnet-\$BUILD_NUMBER-\$EXECUTOR_NUMBER-${pythonVersion}-${dockerVersion}"
withDockerRegistry(credentialsId:'dockerbuildbot-index.docker.io') {
try {
// unit tests
sh """docker run --rm \\
-e 'DOCKER_TEST_API_VERSION=${apiVersion}' \\
${testImage} \\
py.test -v -rxs --cov=docker tests/unit
"""
// integration tests
sh """docker network create ${testNetwork}"""
sh """docker run --rm -d --name ${dindContainerName} -v /tmp --privileged --network ${testNetwork} \\
${imageDindSSH} dockerd -H tcp://0.0.0.0:2375
"""
sh """docker run --rm \\
--name ${testContainerName} \\
-e "DOCKER_HOST=tcp://${dindContainerName}:2375" \\
-e 'DOCKER_TEST_API_VERSION=${apiVersion}' \\
--network ${testNetwork} \\
--volumes-from ${dindContainerName} \\
-v $DOCKER_CONFIG/config.json:/root/.docker/config.json \\
${testImage} \\
py.test -v -rxs --cov=docker tests/integration
"""
sh """docker stop ${dindContainerName}"""
// start DIND container with SSH
sh """docker run --rm -d --name ${dindContainerName} -v /tmp --privileged --network ${testNetwork} \\
${imageDindSSH} dockerd --experimental"""
sh """docker exec ${dindContainerName} sh -c /usr/sbin/sshd """
// run SSH tests only
sh """docker run --rm \\
--name ${testContainerName} \\
-e "DOCKER_HOST=ssh://${dindContainerName}:22" \\
-e 'DOCKER_TEST_API_VERSION=${apiVersion}' \\
--network ${testNetwork} \\
--volumes-from ${dindContainerName} \\
-v $DOCKER_CONFIG/config.json:/root/.docker/config.json \\
${testImage} \\
py.test -v -rxs --cov=docker tests/ssh
"""
} finally {
sh """
docker stop ${dindContainerName}
docker network rm ${testNetwork}
"""
}
}
}
}
}
}
buildImages()
def dockerVersions = getDockerVersions()
def testMatrix = [failFast: false]
for (imgKey in new ArrayList(images.keySet())) {
for (version in dockerVersions) {
testMatrix["${imgKey}_${version}"] = runTests([testImage: images[imgKey], dockerVersion: version, pythonVersion: imgKey])
}
}
parallel(testMatrix)

Просмотреть файл

@ -4,6 +4,7 @@ import urllib
from functools import partial
import requests
import requests.adapters
import requests.exceptions
from .. import auth
@ -14,7 +15,7 @@ from ..constants import (DEFAULT_NUM_POOLS, DEFAULT_NUM_POOLS_SSH,
from ..errors import (DockerException, InvalidVersion, TLSParameterError,
create_api_error_from_http_exception)
from ..tls import TLSConfig
from ..transport import SSLHTTPAdapter, UnixHTTPAdapter
from ..transport import UnixHTTPAdapter
from ..utils import check_resource, config, update_headers, utils
from ..utils.json_stream import json_stream
from ..utils.proxy import ProxyConfig
@ -183,7 +184,7 @@ class APIClient(
if isinstance(tls, TLSConfig):
tls.configure_client(self)
elif tls:
self._custom_adapter = SSLHTTPAdapter(
self._custom_adapter = requests.adapters.HTTPAdapter(
pool_connections=num_pools)
self.mount('https://', self._custom_adapter)
self.base_url = base_url

Просмотреть файл

@ -71,8 +71,6 @@ class DockerClient:
timeout (int): Default timeout for API calls, in seconds.
max_pool_size (int): The maximum number of connections
to save in the pool.
ssl_version (int): A valid `SSL version`_.
assert_hostname (bool): Verify the hostname of the server.
environment (dict): The environment to read environment variables
from. Default: the value of ``os.environ``
credstore_env (dict): Override environment variables when calling

Просмотреть файл

@ -1,8 +1,6 @@
import os
import ssl
from . import errors
from .transport import SSLHTTPAdapter
class TLSConfig:
@ -15,35 +13,18 @@ class TLSConfig:
verify (bool or str): This can be a bool or a path to a CA cert
file to verify against. If ``True``, verify using ca_cert;
if ``False`` or not specified, do not verify.
ssl_version (int): A valid `SSL version`_.
assert_hostname (bool): Verify the hostname of the server.
.. _`SSL version`:
https://docs.python.org/3.5/library/ssl.html#ssl.PROTOCOL_TLSv1
"""
cert = None
ca_cert = None
verify = None
ssl_version = None
def __init__(self, client_cert=None, ca_cert=None, verify=None,
ssl_version=None, assert_hostname=None,
assert_fingerprint=None):
def __init__(self, client_cert=None, ca_cert=None, verify=None):
# Argument compatibility/mapping with
# https://docs.docker.com/engine/articles/https/
# This diverges from the Docker CLI in that users can specify 'tls'
# here, but also disable any public/default CA pool verification by
# leaving verify=False
self.assert_hostname = assert_hostname
self.assert_fingerprint = assert_fingerprint
# If the user provides an SSL version, we should use their preference
if ssl_version:
self.ssl_version = ssl_version
else:
self.ssl_version = ssl.PROTOCOL_TLS_CLIENT
# "client_cert" must have both or neither cert/key files. In
# either case, Alert the user when both are expected, but any are
# missing.
@ -77,8 +58,6 @@ class TLSConfig:
"""
Configure a client with these TLS options.
"""
client.ssl_version = self.ssl_version
if self.verify and self.ca_cert:
client.verify = self.ca_cert
else:
@ -86,9 +65,3 @@ class TLSConfig:
if self.cert:
client.cert = self.cert
client.mount('https://', SSLHTTPAdapter(
ssl_version=self.ssl_version,
assert_hostname=self.assert_hostname,
assert_fingerprint=self.assert_fingerprint,
))

Просмотреть файл

@ -1,5 +1,4 @@
from .unixconn import UnixHTTPAdapter
from .ssladapter import SSLHTTPAdapter
try:
from .npipeconn import NpipeHTTPAdapter
from .npipesocket import NpipeSocket

Просмотреть файл

@ -1,62 +0,0 @@
""" Resolves OpenSSL issues in some servers:
https://lukasa.co.uk/2013/01/Choosing_SSL_Version_In_Requests/
https://github.com/kennethreitz/requests/pull/799
"""
from packaging.version import Version
from requests.adapters import HTTPAdapter
from docker.transport.basehttpadapter import BaseHTTPAdapter
import urllib3
PoolManager = urllib3.poolmanager.PoolManager
class SSLHTTPAdapter(BaseHTTPAdapter):
'''An HTTPS Transport Adapter that uses an arbitrary SSL version.'''
__attrs__ = HTTPAdapter.__attrs__ + ['assert_fingerprint',
'assert_hostname',
'ssl_version']
def __init__(self, ssl_version=None, assert_hostname=None,
assert_fingerprint=None, **kwargs):
self.ssl_version = ssl_version
self.assert_hostname = assert_hostname
self.assert_fingerprint = assert_fingerprint
super().__init__(**kwargs)
def init_poolmanager(self, connections, maxsize, block=False):
kwargs = {
'num_pools': connections,
'maxsize': maxsize,
'block': block,
'assert_hostname': self.assert_hostname,
'assert_fingerprint': self.assert_fingerprint,
}
if self.ssl_version and self.can_override_ssl_version():
kwargs['ssl_version'] = self.ssl_version
self.poolmanager = PoolManager(**kwargs)
def get_connection(self, *args, **kwargs):
"""
Ensure assert_hostname is set correctly on our pool
We already take care of a normal poolmanager via init_poolmanager
But we still need to take care of when there is a proxy poolmanager
"""
conn = super().get_connection(*args, **kwargs)
if conn.assert_hostname != self.assert_hostname:
conn.assert_hostname = self.assert_hostname
return conn
def can_override_ssl_version(self):
urllib_ver = urllib3.__version__.split('-')[0]
if urllib_ver is None:
return False
if urllib_ver == 'dev':
return True
return Version(urllib_ver) > Version('1.5')

Просмотреть файл

@ -341,7 +341,7 @@ def parse_devices(devices):
return device_list
def kwargs_from_env(ssl_version=None, assert_hostname=None, environment=None):
def kwargs_from_env(environment=None):
if not environment:
environment = os.environ
host = environment.get('DOCKER_HOST')
@ -369,18 +369,11 @@ def kwargs_from_env(ssl_version=None, assert_hostname=None, environment=None):
if not cert_path:
cert_path = os.path.join(os.path.expanduser('~'), '.docker')
if not tls_verify and assert_hostname is None:
# assert_hostname is a subset of TLS verification,
# so if it's not set already then set it to false.
assert_hostname = False
params['tls'] = TLSConfig(
client_cert=(os.path.join(cert_path, 'cert.pem'),
os.path.join(cert_path, 'key.pem')),
ca_cert=os.path.join(cert_path, 'ca.pem'),
verify=tls_verify,
ssl_version=ssl_version,
assert_hostname=assert_hostname,
)
return params

Просмотреть файл

@ -74,6 +74,7 @@ setup(
'Programming Language :: Python :: 3.9',
'Programming Language :: Python :: 3.10',
'Programming Language :: Python :: 3.11',
'Programming Language :: Python :: 3.12',
'Topic :: Software Development',
'Topic :: Utilities',
'License :: OSI Approved :: Apache Software License',

Просмотреть файл

@ -1,6 +1,6 @@
setuptools==65.5.1
coverage==6.4.2
coverage==7.2.7
ruff==0.0.284
pytest==7.1.2
pytest-cov==3.0.0
pytest==7.4.2
pytest-cov==4.1.0
pytest-timeout==2.1.0

Просмотреть файл

@ -1,6 +1,6 @@
# syntax=docker/dockerfile:1
ARG PYTHON_VERSION=3.10
ARG PYTHON_VERSION=3.12
FROM python:${PYTHON_VERSION}

Просмотреть файл

@ -1,6 +1,6 @@
# syntax=docker/dockerfile:1
ARG PYTHON_VERSION=3.10
ARG PYTHON_VERSION=3.12
FROM python:${PYTHON_VERSION}
RUN mkdir /tmp/certs

Просмотреть файл

@ -10,8 +10,8 @@ class NetworkCollectionTest(unittest.TestCase):
client = make_fake_client()
network = client.networks.create("foobar", labels={'foo': 'bar'})
assert network.id == FAKE_NETWORK_ID
assert client.api.inspect_network.called_once_with(FAKE_NETWORK_ID)
assert client.api.create_network.called_once_with(
client.api.inspect_network.assert_called_once_with(FAKE_NETWORK_ID)
client.api.create_network.assert_called_once_with(
"foobar",
labels={'foo': 'bar'}
)
@ -20,21 +20,21 @@ class NetworkCollectionTest(unittest.TestCase):
client = make_fake_client()
network = client.networks.get(FAKE_NETWORK_ID)
assert network.id == FAKE_NETWORK_ID
assert client.api.inspect_network.called_once_with(FAKE_NETWORK_ID)
client.api.inspect_network.assert_called_once_with(FAKE_NETWORK_ID)
def test_list(self):
client = make_fake_client()
networks = client.networks.list()
assert networks[0].id == FAKE_NETWORK_ID
assert client.api.networks.called_once_with()
client.api.networks.assert_called_once_with()
client = make_fake_client()
client.networks.list(ids=["abc"])
assert client.api.networks.called_once_with(ids=["abc"])
client.api.networks.assert_called_once_with(ids=["abc"])
client = make_fake_client()
client.networks.list(names=["foobar"])
assert client.api.networks.called_once_with(names=["foobar"])
client.api.networks.assert_called_once_with(names=["foobar"])
class NetworkTest(unittest.TestCase):
@ -43,7 +43,7 @@ class NetworkTest(unittest.TestCase):
client = make_fake_client()
network = client.networks.get(FAKE_NETWORK_ID)
network.connect(FAKE_CONTAINER_ID)
assert client.api.connect_container_to_network.called_once_with(
client.api.connect_container_to_network.assert_called_once_with(
FAKE_CONTAINER_ID,
FAKE_NETWORK_ID
)
@ -52,7 +52,7 @@ class NetworkTest(unittest.TestCase):
client = make_fake_client()
network = client.networks.get(FAKE_NETWORK_ID)
network.disconnect(FAKE_CONTAINER_ID)
assert client.api.disconnect_container_from_network.called_once_with(
client.api.disconnect_container_from_network.assert_called_once_with(
FAKE_CONTAINER_ID,
FAKE_NETWORK_ID
)
@ -61,4 +61,4 @@ class NetworkTest(unittest.TestCase):
client = make_fake_client()
network = client.networks.get(FAKE_NETWORK_ID)
network.remove()
assert client.api.remove_network.called_once_with(FAKE_NETWORK_ID)
client.api.remove_network.assert_called_once_with(FAKE_NETWORK_ID)

Просмотреть файл

@ -1,71 +0,0 @@
import unittest
from ssl import match_hostname, CertificateError
import pytest
from docker.transport import ssladapter
try:
from ssl import OP_NO_SSLv3, OP_NO_SSLv2, OP_NO_TLSv1
except ImportError:
OP_NO_SSLv2 = 0x1000000
OP_NO_SSLv3 = 0x2000000
OP_NO_TLSv1 = 0x4000000
class SSLAdapterTest(unittest.TestCase):
def test_only_uses_tls(self):
ssl_context = ssladapter.urllib3.util.ssl_.create_urllib3_context()
assert ssl_context.options & OP_NO_SSLv3
# if OpenSSL is compiled without SSL2 support, OP_NO_SSLv2 will be 0
assert not bool(OP_NO_SSLv2) or ssl_context.options & OP_NO_SSLv2
assert not ssl_context.options & OP_NO_TLSv1
class MatchHostnameTest(unittest.TestCase):
cert = {
'issuer': (
(('countryName', 'US'),),
(('stateOrProvinceName', 'California'),),
(('localityName', 'San Francisco'),),
(('organizationName', 'Docker Inc'),),
(('organizationalUnitName', 'Docker-Python'),),
(('commonName', 'localhost'),),
(('emailAddress', 'info@docker.com'),)
),
'notAfter': 'Mar 25 23:08:23 2030 GMT',
'notBefore': 'Mar 25 23:08:23 2016 GMT',
'serialNumber': 'BD5F894C839C548F',
'subject': (
(('countryName', 'US'),),
(('stateOrProvinceName', 'California'),),
(('localityName', 'San Francisco'),),
(('organizationName', 'Docker Inc'),),
(('organizationalUnitName', 'Docker-Python'),),
(('commonName', 'localhost'),),
(('emailAddress', 'info@docker.com'),)
),
'subjectAltName': (
('DNS', 'localhost'),
('DNS', '*.gensokyo.jp'),
('IP Address', '127.0.0.1'),
),
'version': 3
}
def test_match_ip_address_success(self):
assert match_hostname(self.cert, '127.0.0.1') is None
def test_match_localhost_success(self):
assert match_hostname(self.cert, 'localhost') is None
def test_match_dns_success(self):
assert match_hostname(self.cert, 'touhou.gensokyo.jp') is None
def test_match_ip_address_failure(self):
with pytest.raises(CertificateError):
match_hostname(self.cert, '192.168.0.25')
def test_match_dns_failure(self):
with pytest.raises(CertificateError):
match_hostname(self.cert, 'foobar.co.uk')

Просмотреть файл

@ -75,13 +75,12 @@ class KwargsFromEnvTest(unittest.TestCase):
os.environ.update(DOCKER_HOST='tcp://192.168.59.103:2376',
DOCKER_CERT_PATH=TEST_CERT_DIR,
DOCKER_TLS_VERIFY='1')
kwargs = kwargs_from_env(assert_hostname=False)
kwargs = kwargs_from_env()
assert 'tcp://192.168.59.103:2376' == kwargs['base_url']
assert 'ca.pem' in kwargs['tls'].ca_cert
assert 'cert.pem' in kwargs['tls'].cert[0]
assert 'key.pem' in kwargs['tls'].cert[1]
assert kwargs['tls'].assert_hostname is False
assert kwargs['tls'].verify
assert kwargs['tls'].verify is True
parsed_host = parse_host(kwargs['base_url'], IS_WINDOWS_PLATFORM, True)
kwargs['version'] = DEFAULT_DOCKER_API_VERSION
@ -97,12 +96,11 @@ class KwargsFromEnvTest(unittest.TestCase):
os.environ.update(DOCKER_HOST='tcp://192.168.59.103:2376',
DOCKER_CERT_PATH=TEST_CERT_DIR,
DOCKER_TLS_VERIFY='')
kwargs = kwargs_from_env(assert_hostname=True)
kwargs = kwargs_from_env()
assert 'tcp://192.168.59.103:2376' == kwargs['base_url']
assert 'ca.pem' in kwargs['tls'].ca_cert
assert 'cert.pem' in kwargs['tls'].cert[0]
assert 'key.pem' in kwargs['tls'].cert[1]
assert kwargs['tls'].assert_hostname is True
assert kwargs['tls'].verify is False
parsed_host = parse_host(kwargs['base_url'], IS_WINDOWS_PLATFORM, True)
kwargs['version'] = DEFAULT_DOCKER_API_VERSION
@ -123,12 +121,12 @@ class KwargsFromEnvTest(unittest.TestCase):
HOME=temp_dir,
DOCKER_TLS_VERIFY='')
os.environ.pop('DOCKER_CERT_PATH', None)
kwargs = kwargs_from_env(assert_hostname=True)
kwargs = kwargs_from_env()
assert 'tcp://192.168.59.103:2376' == kwargs['base_url']
def test_kwargs_from_env_no_cert_path(self):
temp_dir = tempfile.mkdtemp()
try:
temp_dir = tempfile.mkdtemp()
cert_dir = os.path.join(temp_dir, '.docker')
shutil.copytree(TEST_CERT_DIR, cert_dir)
@ -142,8 +140,7 @@ class KwargsFromEnvTest(unittest.TestCase):
assert cert_dir in kwargs['tls'].cert[0]
assert cert_dir in kwargs['tls'].cert[1]
finally:
if temp_dir:
shutil.rmtree(temp_dir)
shutil.rmtree(temp_dir)
def test_kwargs_from_env_alternate_env(self):
# Values in os.environ are entirely ignored if an alternate is

Просмотреть файл

@ -1,5 +1,5 @@
[tox]
envlist = py{37,38,39,310,311}, ruff
envlist = py{37,38,39,310,311,312}, ruff
skipsdist=True
[testenv]