diff --git a/.gitignore b/.gitignore index 943e7f3f68..93a6df48e7 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,7 @@ *.exe *.exe~ *.orig -*.test +test.main .*.swp .DS_Store # a .bashrc may be added to customize the build environment diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 917214cd1b..75d14cda41 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -160,6 +160,10 @@ it! Take a look at existing tests for inspiration. [Run the full test suite](https://docs.docker.com/opensource/project/test-and-docs/) on your branch before submitting a pull request. +If your changes need integration tests, write them against the API. The `cli` +integration tests are slowly either migrated to API tests or moved away as unit +tests in `docker/cli` and end-to-end tests for docker. + Update the documentation when creating or modifying features. Test your documentation changes for clarity, concision, and correctness, as well as a clean documentation build. See our contributors guide for [our style diff --git a/Dockerfile b/Dockerfile index 33e88dce26..62ab2affec 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ # docker run -v `pwd`:/go/src/github.com/docker/docker --privileged -i -t docker bash # # # Run the test suite: -# docker run -e DOCKER_GITCOMMIT=foo --privileged docker hack/make.sh test-unit test-integration-cli test-docker-py +# docker run -e DOCKER_GITCOMMIT=foo --privileged docker hack/make.sh test-unit test-integration test-docker-py # # # Publish a release: # docker run --privileged \ diff --git a/Dockerfile.aarch64 b/Dockerfile.aarch64 index cabcda28bd..4526a1b257 100644 --- a/Dockerfile.aarch64 +++ b/Dockerfile.aarch64 @@ -9,7 +9,7 @@ # docker run -v `pwd`:/go/src/github.com/docker/docker --privileged -i -t docker bash # # # Run the test suite: -# docker run --privileged docker hack/make.sh test-unit test-integration-cli test-docker-py +# docker run --privileged docker hack/make.sh test-unit test-integration test-docker-py # # Note: AppArmor used to mess with privileged mode, but this is no longer # the case. Therefore, you don't have to disable it anymore. diff --git a/Dockerfile.armhf b/Dockerfile.armhf index dd1f536191..b19dbe5d30 100644 --- a/Dockerfile.armhf +++ b/Dockerfile.armhf @@ -9,7 +9,7 @@ # docker run -v `pwd`:/go/src/github.com/docker/docker --privileged -i -t docker bash # # # Run the test suite: -# docker run --privileged docker hack/make.sh test-unit test-integration-cli test-docker-py +# docker run --privileged docker hack/make.sh test-unit test-integration test-docker-py # # Note: AppArmor used to mess with privileged mode, but this is no longer # the case. Therefore, you don't have to disable it anymore. diff --git a/Dockerfile.ppc64le b/Dockerfile.ppc64le index 43b84e4501..940369f967 100644 --- a/Dockerfile.ppc64le +++ b/Dockerfile.ppc64le @@ -9,7 +9,7 @@ # docker run -v `pwd`:/go/src/github.com/docker/docker --privileged -i -t docker bash # # # Run the test suite: -# docker run --privileged docker hack/make.sh test-unit test-integration-cli test-docker-py +# docker run --privileged docker hack/make.sh test-unit test-integration test-docker-py # # Note: AppArmor used to mess with privileged mode, but this is no longer # the case. Therefore, you don't have to disable it anymore. diff --git a/Dockerfile.s390x b/Dockerfile.s390x index 35ec683739..63f45d78e6 100644 --- a/Dockerfile.s390x +++ b/Dockerfile.s390x @@ -9,7 +9,7 @@ # docker run -v `pwd`:/go/src/github.com/docker/docker --privileged -i -t docker bash # # # Run the test suite: -# docker run --privileged docker hack/make.sh test-unit test-integration-cli test-docker-py +# docker run --privileged docker hack/make.sh test-unit test-integration test-docker-py # # Note: AppArmor used to mess with privileged mode, but this is no longer # the case. Therefore, you don't have to disable it anymore. diff --git a/Dockerfile.simple b/Dockerfile.simple index b4682d4cbc..f84f3c565c 100644 --- a/Dockerfile.simple +++ b/Dockerfile.simple @@ -1,7 +1,7 @@ # docker build -t docker:simple -f Dockerfile.simple . # docker run --rm docker:simple hack/make.sh dynbinary # docker run --rm --privileged docker:simple hack/dind hack/make.sh test-unit -# docker run --rm --privileged -v /var/lib/docker docker:simple hack/dind hack/make.sh dynbinary test-integration-cli +# docker run --rm --privileged -v /var/lib/docker docker:simple hack/dind hack/make.sh dynbinary test-integration # This represents the bare minimum required to build and test Docker. diff --git a/Makefile b/Makefile index 0d99606cc2..85877bdd2e 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: all binary dynbinary build cross deb help init-go-pkg-cache install manpages rpm run shell test test-docker-py test-integration-cli test-unit tgz validate win +.PHONY: all binary dynbinary build cross deb help init-go-pkg-cache install manpages rpm run shell test test-docker-py test-integration test-unit tgz validate win # set the graph driver as the current graphdriver if not set DOCKER_GRAPHDRIVER := $(if $(DOCKER_GRAPHDRIVER),$(DOCKER_GRAPHDRIVER),$(shell docker info 2>&1 | grep "Storage Driver" | sed 's/.*: //')) @@ -149,13 +149,15 @@ shell: build ## start a shell inside the build env $(DOCKER_RUN_DOCKER) bash test: build ## run the unit, integration and docker-py tests - $(DOCKER_RUN_DOCKER) hack/make.sh dynbinary cross test-unit test-integration-cli test-docker-py + $(DOCKER_RUN_DOCKER) hack/make.sh dynbinary cross test-unit test-integration test-docker-py test-docker-py: build ## run the docker-py tests $(DOCKER_RUN_DOCKER) hack/make.sh dynbinary test-docker-py -test-integration-cli: build ## run the integration tests - $(DOCKER_RUN_DOCKER) hack/make.sh build-integration-test-binary dynbinary test-integration-cli +test-integration-cli: test-integration ## (DEPRECATED) use test-integration + +test-integration: build ## run the integration tests + $(DOCKER_RUN_DOCKER) hack/make.sh dynbinary test-integration test-unit: build ## run the unit tests $(DOCKER_RUN_DOCKER) hack/make.sh test-unit diff --git a/hack/README.md b/hack/README.md index 802395d533..9e588db256 100644 --- a/hack/README.md +++ b/hack/README.md @@ -37,14 +37,14 @@ More information is found within `make.ps1` by the author, @jhowardmsft - Referenced via `make test` when running tests on a local machine, or directly referenced when running tests inside a Docker development container. - When running on a local machine, `make test` to run all tests found in -`test`, `test-unit`, `test-integration-cli`, and `test-docker-py` on +`test`, `test-unit`, `test-integration`, and `test-docker-py` on your local machine. The default timeout is set in `make.sh` to 60 minutes (`${TIMEOUT:=60m}`), since it currently takes up to an hour to run all of the tests. - When running inside a Docker development container, `hack/make.sh` does not have a single target that runs all the tests. You need to provide a single command line with multiple targets that performs the same thing. -An example referenced from [Run targets inside a development container](https://docs.docker.com/opensource/project/test-and-docs/#run-targets-inside-a-development-container): `root@5f8630b873fe:/go/src/github.com/moby/moby# hack/make.sh dynbinary binary cross test-unit test-integration-cli test-docker-py` +An example referenced from [Run targets inside a development container](https://docs.docker.com/opensource/project/test-and-docs/#run-targets-inside-a-development-container): `root@5f8630b873fe:/go/src/github.com/moby/moby# hack/make.sh dynbinary binary cross test-unit test-integration test-docker-py` - For more information related to testing outside the scope of this README, refer to [Run tests and test documentation](https://docs.docker.com/opensource/project/test-and-docs/) diff --git a/hack/make.ps1 b/hack/make.ps1 index ac3e369048..f3f45e3d00 100644 --- a/hack/make.ps1 +++ b/hack/make.ps1 @@ -318,7 +318,7 @@ Function Run-UnitTests() { $pkgList = $pkgList | Select-String -Pattern "github.com/docker/docker" $pkgList = $pkgList | Select-String -NotMatch "github.com/docker/docker/vendor" $pkgList = $pkgList | Select-String -NotMatch "github.com/docker/docker/man" - $pkgList = $pkgList | Select-String -NotMatch "github.com/docker/docker/integration-cli" + $pkgList = $pkgList | Select-String -NotMatch "github.com/docker/docker/integration" $pkgList = $pkgList -replace "`r`n", " " $goTestCommand = "go test" + $raceParm + " -cover -ldflags -w -tags """ + "autogen daemon" + """ -a """ + "-test.timeout=10m" + """ $pkgList" Invoke-Expression $goTestCommand diff --git a/hack/make.sh b/hack/make.sh index e625b863de..ffcd7abdb3 100755 --- a/hack/make.sh +++ b/hack/make.sh @@ -60,7 +60,7 @@ DEFAULT_BUNDLES=( dynbinary test-unit - test-integration-cli + test-integration test-docker-py cross diff --git a/hack/make/.ensure-emptyfs b/hack/make/.ensure-emptyfs index 7b00b9d455..898cc22834 100644 --- a/hack/make/.ensure-emptyfs +++ b/hack/make/.ensure-emptyfs @@ -1,23 +1,23 @@ #!/usr/bin/env bash set -e -if ! docker inspect -t image emptyfs &> /dev/null; then - # let's build a "docker save" tarball for "emptyfs" +if ! docker image inspect emptyfs > /dev/null; then + # build a "docker save" tarball for "emptyfs" # see https://github.com/docker/docker/pull/5262 # and also https://github.com/docker/docker/issues/4242 dir="$DEST/emptyfs" - mkdir -p "$dir" + uuid=511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158 + mkdir -p "$dir/$uuid" ( - cd "$dir" - echo '{"emptyfs":{"latest":"511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158"}}' > repositories - mkdir -p 511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158 - ( - cd 511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158 - echo '{"id":"511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158","comment":"Imported from -","created":"2013-06-13T14:03:50.821769-07:00","container_config":{"Hostname":"","Domainname":"","User":"","Memory":0,"MemorySwap":0,"CpuShares":0,"AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"PortSpecs":null,"ExposedPorts":null,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":null,"Cmd":null,"Image":"","Volumes":null,"WorkingDir":"","Entrypoint":null,"NetworkDisabled":false,"OnBuild":null},"docker_version":"0.4.0","architecture":"x86_64","Size":0}' > json - echo '1.0' > VERSION - tar -cf layer.tar --files-from /dev/null - ) + echo '{"emptyfs":{"latest":"511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158"}}' > "$dir/repositories" + cd "$dir/$uuid" + echo '{"id":"511136ea3c5a64f264b78b5433614aec563103b4d4702f3ba7d4d2698e22c158","comment":"Imported from -","created":"2013-06-13T14:03:50.821769-07:00","container_config":{"Hostname":"","Domainname":"","User":"","Memory":0,"MemorySwap":0,"CpuShares":0,"AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"PortSpecs":null,"ExposedPorts":null,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":null,"Cmd":null,"Image":"","Volumes":null,"WorkingDir":"","Entrypoint":null,"NetworkDisabled":false,"OnBuild":null},"docker_version":"0.4.0","architecture":"x86_64","Size":0}' > json + echo '1.0' > VERSION + tar -cf layer.tar --files-from /dev/null + ) + ( + [ -n "$TESTDEBUG" ] && set -x + tar -cC "$dir" . | docker load ) - ( set -x; tar -cC "$dir" . | docker load ) rm -rf "$dir" fi diff --git a/hack/make/.integration-daemon-setup b/hack/make/.integration-daemon-setup index 5134e4c2db..c130e23560 100644 --- a/hack/make/.integration-daemon-setup +++ b/hack/make/.integration-daemon-setup @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -e -bundle .detect-daemon-osarch +source "$MAKEDIR/.detect-daemon-osarch" if [ "$DOCKER_ENGINE_GOOS" != "windows" ]; then bundle .ensure-emptyfs fi diff --git a/hack/make/.integration-daemon-start b/hack/make/.integration-daemon-start index dafd0533d9..b4e6684854 100644 --- a/hack/make/.integration-daemon-start +++ b/hack/make/.integration-daemon-start @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# see test-integration-cli for example usage of this script +# see test-integration for example usage of this script base="$ABS_DEST/.." export PATH="$base/binary-daemon:$base/dynbinary-daemon:$PATH" @@ -76,24 +76,26 @@ if [ -z "$DOCKER_TEST_HOST" ]; then # see https://github.com/docker/libcontainer/blob/master/apparmor/apparmor.go#L16 export container="" ( - set -x + [ -n "$TESTDEBUG" ] && set -x /etc/init.d/apparmor start ) fi - export DOCKER_HOST="unix://$(cd "$DEST" && pwd)/docker.sock" # "pwd" tricks to make sure $DEST is an absolute path, not a relative one - ( set -x; exec \ - dockerd --debug \ - --host "$DOCKER_HOST" \ - --storage-driver "$DOCKER_GRAPHDRIVER" \ - --pidfile "$DEST/docker.pid" \ - --userland-proxy="$DOCKER_USERLANDPROXY" \ - $storage_params \ - $extra_params \ - &> "$DEST/docker.log" + # "pwd" tricks to make sure $DEST is an absolute path, not a relative one + export DOCKER_HOST="unix://$(cd "$DEST" && pwd)/docker.sock" + ( + echo "Starting dockerd" + [ -n "$TESTDEBUG" ] && set -x + exec \ + dockerd --debug \ + --host "$DOCKER_HOST" \ + --storage-driver "$DOCKER_GRAPHDRIVER" \ + --pidfile "$DEST/docker.pid" \ + --userland-proxy="$DOCKER_USERLANDPROXY" \ + $storage_params \ + $extra_params \ + &> "$DEST/docker.log" ) & - # make sure that if the script exits unexpectedly, we stop this daemon we just started - trap 'bundle .integration-daemon-stop' EXIT else export DOCKER_HOST="$DOCKER_TEST_HOST" fi diff --git a/hack/make/.integration-daemon-stop b/hack/make/.integration-daemon-stop index 591a8d6bec..c1d43e1a5e 100644 --- a/hack/make/.integration-daemon-stop +++ b/hack/make/.integration-daemon-stop @@ -1,11 +1,12 @@ #!/usr/bin/env bash if [ ! "$(go env GOOS)" = 'windows' ]; then - trap - EXIT # reset EXIT trap applied in .integration-daemon-start - for pidFile in $(find "$DEST" -name docker.pid); do - pid=$(set -x; cat "$pidFile") - ( set -x; kill "$pid" ) + pid=$([ -n "$TESTDEBUG" ] && set -x; cat "$pidFile") + ( + [ -n "$TESTDEBUG" ] && set -x + kill "$pid" + ) if ! wait "$pid"; then echo >&2 "warning: PID $pid from $pidFile had a nonzero exit code" fi @@ -15,7 +16,7 @@ if [ ! "$(go env GOOS)" = 'windows' ]; then # Stop apparmor if it is enabled if [ -e "/sys/module/apparmor/parameters/enabled" ] && [ "$(cat /sys/module/apparmor/parameters/enabled)" == "Y" ]; then ( - set -x + [ -n "$TESTDEBUG" ] && set -x /etc/init.d/apparmor stop ) fi diff --git a/hack/make/.integration-test-helpers b/hack/make/.integration-test-helpers index 4ff9677c79..288be863ce 100644 --- a/hack/make/.integration-test-helpers +++ b/hack/make/.integration-test-helpers @@ -1,65 +1,72 @@ #!/usr/bin/env bash - -: ${TEST_REPEAT:=0} - -bundle_test_integration_cli() { - TESTFLAGS="$TESTFLAGS -check.v -check.timeout=${TIMEOUT} -test.timeout=360m" - go_test_dir integration-cli $DOCKER_INTEGRATION_TESTS_VERIFIED -} - -# If $TESTFLAGS is set in the environment, it is passed as extra arguments to 'go test'. -# You can use this to select certain tests to run, e.g. -# -# TESTFLAGS='-test.run ^TestBuild$' ./hack/make.sh test-unit # # For integration-cli test, we use [gocheck](https://labix.org/gocheck), if you want # to run certain tests on your local host, you should run with command: # -# TESTFLAGS='-check.f DockerSuite.TestBuild*' ./hack/make.sh binary test-integration-cli +# TESTFLAGS='-check.f DockerSuite.TestBuild*' ./hack/make.sh binary test-integration # -go_test_dir() { - dir=$1 - precompiled=$2 - testbinary="$ABS_DEST/test.main" - testcover=() - testcoverprofile=() + +source "$SCRIPTDIR/make/.go-autogen" + +: ${TEST_REPEAT:=1} + +integration_api_dirs=("$( + find ./integration -type d | + grep -vE '^(./integration$|./integration/util)')") + +run_test_integration() { + local flags="-test.v -test.timeout=${TIMEOUT} $TESTFLAGS" + for dir in $integration_api_dirs; do + ( + cd $dir + echo "Running $PWD" + test_env ./test.main $flags + ) + done + ( - set -e - mkdir -p "$DEST/coverprofiles" - export DEST="$ABS_DEST" # in a subshell this is safe -- our integration-cli tests need DEST, and "cd" screws it up - if [ -z $precompiled ]; then - ensure_test_dir $1 $testbinary - fi - cd "$dir" - i=0 - while ((++i)); do - test_env "$testbinary" $TESTFLAGS - if [ $i -gt "$TEST_REPEAT" ]; then - break - fi - echo "Repeating test ($i)" - done + flags="-check.v -check.timeout=${TIMEOUT} -test.timeout=360m $TESTFLAGS" + cd integration-cli + echo "Running $PWD" + test_env ./test.main $flags ) } -ensure_test_dir() { - ( - # make sure a test dir will compile - dir="$1" - out="$2" - echo Building test dir: "$dir" - set -xe - cd "$dir" - go test -c -o "$out" -ldflags "$LDFLAGS" "${BUILDFLAGS[@]}" - ) +build_test_suite_binaries() { + build_test_suite_binary ./integration-cli "test.main" + for dir in $integration_api_dirs; do + build_test_suite_binary "$dir" "test.main" + done } +# Build a binary for a test suite package +build_test_suite_binary() { + local dir="$1" + local out="$2" + echo Building test suite binary "$dir/$out" + go test -c -o "$dir/$out" -ldflags "$LDFLAGS" "${BUILDFLAGS[@]}" "$dir" +} + +cleanup_test_suite_binaries() { + [ -n "$TESTDEBUG" ] && return + echo "Removing test suite binaries" + find integration* -name test.main | xargs -r rm +} + +repeat() { + for i in $(seq 1 $TEST_REPEAT); do + echo "Running integration-test (iteration $i)" + $@ + done +} + +# use "env -i" to tightly control the environment variables that bleed into the tests test_env() { ( - set -xe - # use "env -i" to tightly control the environment variables that bleed into the tests + set -e + [ -n "$TESTDEBUG" ] && set -x env -i \ - DEST="$DEST" \ + DEST="$ABS_DEST" \ DOCKER_CLI_VERSION="$DOCKER_CLI_VERSION" \ DOCKER_API_VERSION="$DOCKER_API_VERSION" \ DOCKER_INTEGRATION_DAEMON_DEST="$DOCKER_INTEGRATION_DAEMON_DEST" \ @@ -82,3 +89,19 @@ test_env() { "$@" ) } + + +error_on_leaked_containerd_shims() { + if [ "$(go env GOOS)" == 'windows' ]; then + return + fi + + leftovers=$(ps -ax -o pid,cmd | + awk '$2 == "docker-containerd-shim" && $4 ~ /.*\/bundles\/.*\/test-integration/ { print $1 }') + if [ -n "$leftovers" ]; then + ps aux + kill -9 $leftovers 2> /dev/null + echo "!!!! WARNING you have left over shim(s), Cleanup your test !!!!" + exit 1 + fi +} diff --git a/hack/make/build-integration-test-binary b/hack/make/build-integration-test-binary deleted file mode 100644 index a842e8cce8..0000000000 --- a/hack/make/build-integration-test-binary +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash -set -e - -rm -rf "$DEST" -DEST="$ABS_DEST/../test-integration-cli" - -source "$SCRIPTDIR/make/.go-autogen" - -if [ -z $DOCKER_INTEGRATION_TESTS_VERIFIED ]; then - source ${MAKEDIR}/.integration-test-helpers - ensure_test_dir integration-cli "$DEST/test.main" - export DOCKER_INTEGRATION_TESTS_VERIFIED=1 -fi diff --git a/hack/make/test-integration b/hack/make/test-integration new file mode 100755 index 0000000000..e419d66c6e --- /dev/null +++ b/hack/make/test-integration @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -e + +source "${MAKEDIR}/.go-autogen" +source hack/make/.integration-test-helpers + +( + build_test_suite_binaries + bundle .integration-daemon-start + bundle .integration-daemon-setup + + local testexit=0 + ( repeat run_test_integration ) || testexit=$? + + # Always run cleanup, even if the subshell fails + bundle .integration-daemon-stop + cleanup_test_suite_binaries + error_on_leaked_containerd_shims + + exit $testexit + +) 2>&1 | tee -a "$DEST/test.log" diff --git a/hack/make/test-integration-cli b/hack/make/test-integration-cli index 61e2f7a7f8..4cc79d095d 100755 --- a/hack/make/test-integration-cli +++ b/hack/make/test-integration-cli @@ -1,29 +1,6 @@ #!/usr/bin/env bash set -e +echo "WARNING: test-integration-cli is DEREPCATED. Use test-integration." >&2 -source "${MAKEDIR}/.go-autogen" -source hack/make/.integration-test-helpers - -# subshell so that we can export PATH without breaking other things -( - bundle .integration-daemon-start - - bundle .integration-daemon-setup - - bundle_test_integration_cli - - bundle .integration-daemon-stop - - if [ "$(go env GOOS)" != 'windows' ] - then - leftovers=$(ps -ax -o pid,cmd | awk '$2 == "docker-containerd-shim" && $4 ~ /.*\/bundles\/.*\/test-integration-cli/ { print $1 }') - if [ -n "$leftovers" ] - then - ps aux - kill -9 $leftovers 2> /dev/null - echo "!!!! WARNING you have left over shim(s), Cleanup your test !!!!" - exit 1 - fi - fi - -) 2>&1 | tee -a "$DEST/test.log" +# TODO: remove this and exit 1 once CI has changed to use test-integration +bundle test-integration diff --git a/hack/make/test-integration-shell b/hack/make/test-integration-shell index 2201f5eb37..bcfa4682eb 100644 --- a/hack/make/test-integration-shell +++ b/hack/make/test-integration-shell @@ -5,3 +5,5 @@ bundle .integration-daemon-setup export ABS_DEST bash +e + +bundle .integration-daemon-stop diff --git a/hack/release.sh b/hack/release.sh index 5d7363044b..04f15be02b 100755 --- a/hack/release.sh +++ b/hack/release.sh @@ -57,7 +57,7 @@ if [ "$1" != '--release-regardless-of-test-failure' ]; then RELEASE_BUNDLES=( test-unit "${RELEASE_BUNDLES[@]}" - test-integration-cli + test-integration ) fi diff --git a/hack/test/unit b/hack/test/unit index 0018e88454..2b07089bcf 100755 --- a/hack/test/unit +++ b/hack/test/unit @@ -16,7 +16,7 @@ TESTFLAGS+=" -test.timeout=${TIMEOUT:-5m}" BUILDFLAGS=( -tags "netgo seccomp libdm_no_deferred_remove" ) TESTDIRS="${TESTDIRS:-"./..."}" -exclude_paths="/vendor/|/integration-cli" +exclude_paths="/vendor/|/integration" if [ "$(go env GOHOSTOS)" = 'solaris' ]; then exclude_paths="$exclude_paths|/daemon/graphdriver" fi diff --git a/integration-cli/check_test.go b/integration-cli/check_test.go index f05b6504e9..6af5229f04 100644 --- a/integration-cli/check_test.go +++ b/integration-cli/check_test.go @@ -4,10 +4,8 @@ import ( "fmt" "net/http/httptest" "os" - "os/exec" "path" "path/filepath" - "strings" "sync" "syscall" "testing" @@ -72,17 +70,7 @@ func TestMain(m *testing.M) { func Test(t *testing.T) { cli.EnsureTestEnvIsLoaded(t) fakestorage.EnsureTestEnvIsLoaded(t) - cmd := exec.Command(dockerBinary, "images", "-f", "dangling=false", "--format", "{{.Repository}}:{{.Tag}}") - cmd.Env = appendBaseEnv(true) - out, err := cmd.CombinedOutput() - if err != nil { - panic(fmt.Errorf("err=%v\nout=%s\n", err, out)) - } - images := strings.Split(strings.TrimSpace(string(out)), "\n") - testEnv.ProtectImage(t, images...) - if testEnv.DaemonPlatform() == "linux" { - ensureFrozenImagesLinux(t) - } + environment.ProtectImages(t, testEnv) check.TestingT(t) } diff --git a/integration-cli/docker_api_create_test.go b/integration-cli/docker_api_create_test.go index e404b6cf58..c2152d32a4 100644 --- a/integration-cli/docker_api_create_test.go +++ b/integration-cli/docker_api_create_test.go @@ -11,82 +11,6 @@ import ( "github.com/go-check/check" ) -func (s *DockerSuite) TestAPICreateWithNotExistImage(c *check.C) { - name := "test" - config := map[string]interface{}{ - "Image": "test456:v1", - "Volumes": map[string]struct{}{"/tmp": {}}, - } - - status, body, err := request.SockRequest("POST", "/containers/create?name="+name, config, daemonHost()) - c.Assert(err, check.IsNil) - c.Assert(status, check.Equals, http.StatusNotFound) - expected := "No such image: test456:v1" - c.Assert(getErrorMessage(c, body), checker.Contains, expected) - - config2 := map[string]interface{}{ - "Image": "test456", - "Volumes": map[string]struct{}{"/tmp": {}}, - } - - status, body, err = request.SockRequest("POST", "/containers/create?name="+name, config2, daemonHost()) - c.Assert(err, check.IsNil) - c.Assert(status, check.Equals, http.StatusNotFound) - expected = "No such image: test456:latest" - c.Assert(getErrorMessage(c, body), checker.Equals, expected) - - config3 := map[string]interface{}{ - "Image": "sha256:0cb40641836c461bc97c793971d84d758371ed682042457523e4ae701efeaaaa", - } - - status, body, err = request.SockRequest("POST", "/containers/create?name="+name, config3, daemonHost()) - c.Assert(err, check.IsNil) - c.Assert(status, check.Equals, http.StatusNotFound) - expected = "No such image: sha256:0cb40641836c461bc97c793971d84d758371ed682042457523e4ae701efeaaaa" - c.Assert(getErrorMessage(c, body), checker.Equals, expected) - -} - -// Test for #25099 -func (s *DockerSuite) TestAPICreateEmptyEnv(c *check.C) { - name := "test1" - config := map[string]interface{}{ - "Image": "busybox", - "Env": []string{"", "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"}, - "Cmd": []string{"true"}, - } - - status, body, err := request.SockRequest("POST", "/containers/create?name="+name, config, daemonHost()) - c.Assert(err, check.IsNil) - c.Assert(status, check.Equals, http.StatusInternalServerError) - expected := "invalid environment variable:" - c.Assert(getErrorMessage(c, body), checker.Contains, expected) - - name = "test2" - config = map[string]interface{}{ - "Image": "busybox", - "Env": []string{"=", "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"}, - "Cmd": []string{"true"}, - } - status, body, err = request.SockRequest("POST", "/containers/create?name="+name, config, daemonHost()) - c.Assert(err, check.IsNil) - c.Assert(status, check.Equals, http.StatusInternalServerError) - expected = "invalid environment variable: =" - c.Assert(getErrorMessage(c, body), checker.Contains, expected) - - name = "test3" - config = map[string]interface{}{ - "Image": "busybox", - "Env": []string{"=foo", "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"}, - "Cmd": []string{"true"}, - } - status, body, err = request.SockRequest("POST", "/containers/create?name="+name, config, daemonHost()) - c.Assert(err, check.IsNil) - c.Assert(status, check.Equals, http.StatusInternalServerError) - expected = "invalid environment variable: =foo" - c.Assert(getErrorMessage(c, body), checker.Contains, expected) -} - func (s *DockerSuite) TestAPICreateWithInvalidHealthcheckParams(c *check.C) { // test invalid Interval in Healthcheck: less than 0s name := "test1" diff --git a/integration-cli/environment/protect.go b/integration-cli/environment/protect.go index 2b0dd6df2f..05c86b7e4a 100644 --- a/integration-cli/environment/protect.go +++ b/integration-cli/environment/protect.go @@ -1,5 +1,16 @@ package environment +import ( + "strings" + + "github.com/docker/docker/integration-cli/fixtures/load" + icmd "github.com/docker/docker/pkg/testutil/cmd" +) + +type protectedElements struct { + images map[string]struct{} +} + // ProtectImage adds the specified image(s) to be protected in case of clean func (e *Execution) ProtectImage(t testingT, images ...string) { for _, image := range images { @@ -7,6 +18,31 @@ func (e *Execution) ProtectImage(t testingT, images ...string) { } } -type protectedElements struct { - images map[string]struct{} +// ProtectImages protects existing images and on linux frozen images from being +// cleaned up at the end of test runs +func ProtectImages(t testingT, testEnv *Execution) { + images := getExistingImages(t, testEnv) + + if testEnv.DaemonPlatform() == "linux" { + images = append(images, ensureFrozenImagesLinux(t, testEnv)...) + } + testEnv.ProtectImage(t, images...) +} + +func getExistingImages(t testingT, testEnv *Execution) []string { + // TODO: use API instead of cli + result := icmd.RunCommand(testEnv.dockerBinary, "images", "-f", "dangling=false", "--format", "{{.Repository}}:{{.Tag}}") + result.Assert(t, icmd.Success) + return strings.Split(strings.TrimSpace(result.Stdout()), "\n") +} + +func ensureFrozenImagesLinux(t testingT, testEnv *Execution) []string { + images := []string{"busybox:latest", "hello-world:frozen", "debian:jessie"} + err := load.FrozenImagesLinux(testEnv.DockerBinary(), images...) + if err != nil { + result := icmd.RunCommand(testEnv.DockerBinary(), "image", "ls") + t.Logf(result.String()) + t.Fatalf("%+v", err) + } + return images } diff --git a/integration-cli/fixtures_linux_daemon_test.go b/integration-cli/fixtures_linux_daemon_test.go index 895f976a18..0011797c0e 100644 --- a/integration-cli/fixtures_linux_daemon_test.go +++ b/integration-cli/fixtures_linux_daemon_test.go @@ -24,16 +24,6 @@ type logT interface { Logf(string, ...interface{}) } -func ensureFrozenImagesLinux(t testingT) { - images := []string{"busybox:latest", "hello-world:frozen", "debian:jessie"} - err := load.FrozenImagesLinux(dockerBinary, images...) - if err != nil { - t.Logf(dockerCmdWithError("images")) - t.Fatalf("%+v", err) - } - defer testEnv.ProtectImage(t, images...) -} - var ensureSyscallTestOnce sync.Once func ensureSyscallTest(c *check.C) { diff --git a/integration/container/create_test.go b/integration/container/create_test.go new file mode 100644 index 0000000000..4b83af41d4 --- /dev/null +++ b/integration/container/create_test.go @@ -0,0 +1,93 @@ +package container + +import ( + "context" + "strconv" + "testing" + + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/network" + "github.com/docker/docker/integration/util/request" + "github.com/docker/docker/pkg/testutil" +) + +func TestCreateFailsWhenIdentifierDoesNotExist(t *testing.T) { + defer setupTest(t)() + client := request.NewAPIClient(t) + + testCases := []struct { + doc string + image string + expectedError string + }{ + { + doc: "image and tag", + image: "test456:v1", + expectedError: "No such image: test456:v1", + }, + { + doc: "image no tag", + image: "test456", + expectedError: "No such image: test456", + }, + { + doc: "digest", + image: "sha256:0cb40641836c461bc97c793971d84d758371ed682042457523e4ae701efeaaaa", + expectedError: "No such image: sha256:0cb40641836c461bc97c793971d84d758371ed682042457523e4ae701efeaaaa", + }, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.doc, func(t *testing.T) { + t.Parallel() + _, err := client.ContainerCreate(context.Background(), + &container.Config{Image: tc.image}, + &container.HostConfig{}, + &network.NetworkingConfig{}, + "foo", + ) + testutil.ErrorContains(t, err, tc.expectedError) + }) + } +} + +func TestCreateWithInvalidEnv(t *testing.T) { + defer setupTest(t)() + client := request.NewAPIClient(t) + + testCases := []struct { + env string + expectedError string + }{ + { + env: "", + expectedError: "invalid environment variable:", + }, + { + env: "=", + expectedError: "invalid environment variable: =", + }, + { + env: "=foo", + expectedError: "invalid environment variable: =foo", + }, + } + + for index, tc := range testCases { + tc := tc + t.Run(strconv.Itoa(index), func(t *testing.T) { + t.Parallel() + _, err := client.ContainerCreate(context.Background(), + &container.Config{ + Image: "busybox", + Env: []string{tc.env}, + }, + &container.HostConfig{}, + &network.NetworkingConfig{}, + "foo", + ) + testutil.ErrorContains(t, err, tc.expectedError) + }) + } +} diff --git a/integration/container/main_test.go b/integration/container/main_test.go new file mode 100644 index 0000000000..a54b8d9094 --- /dev/null +++ b/integration/container/main_test.go @@ -0,0 +1,37 @@ +package container + +import ( + "fmt" + "os" + "testing" + + "github.com/docker/docker/integration-cli/environment" +) + +var ( + testEnv *environment.Execution +) + +func TestMain(m *testing.M) { + var err error + testEnv, err = environment.New() + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + // TODO: replace this with `testEnv.Print()` to print the full env + if testEnv.LocalDaemon() { + fmt.Println("INFO: Testing against a local daemon") + } else { + fmt.Println("INFO: Testing against a remote daemon") + } + + res := m.Run() + os.Exit(res) +} + +func setupTest(t *testing.T) func() { + environment.ProtectImages(t, testEnv) + return func() { testEnv.Clean(t, testEnv.DockerBinary()) } +} diff --git a/integration/doc.go b/integration/doc.go new file mode 100644 index 0000000000..2fdf62eef7 --- /dev/null +++ b/integration/doc.go @@ -0,0 +1,3 @@ +// Package integration provides integrations tests for Moby (API). +// These tests require a daemon (dockerd for now) to run. +package integration diff --git a/integration/util/request/client.go b/integration/util/request/client.go new file mode 100644 index 0000000000..771563673e --- /dev/null +++ b/integration/util/request/client.go @@ -0,0 +1,15 @@ +package request + +import ( + "testing" + + "github.com/docker/docker/client" + "github.com/stretchr/testify/require" +) + +// NewAPIClient returns a docker API client configured from environment variables +func NewAPIClient(t *testing.T) client.APIClient { + clt, err := client.NewEnvClient() + require.NoError(t, err) + return clt +} diff --git a/project/ARM.md b/project/ARM.md index e61e3b1847..c876231d1e 100644 --- a/project/ARM.md +++ b/project/ARM.md @@ -17,7 +17,7 @@ From the root of the Docker/Docker repo one can use make to execute the followin - make default - make shell - make test-unit -- make test-integration-cli +- make test-integration - make The Makefile does include logic to determine on which OS and architecture the Docker Development Image is built.