KAFKA-4809: docker/run_tests.sh should set up /opt/kafka-dev to be the source directory

…0.x and not 0.8

Author: Colin P. Mccabe <cmccabe@confluent.io>

Reviewers: Ewen Cheslack-Postava <ewen@confluent.io>

Closes #2602 from cmccabe/KAFKA-4809

tests/docker/Dockerfile

@@ -16,22 +16,16 @@
 FROM openjdk:8
 
 MAINTAINER Apache Kafka dev@kafka.apache.org
-RUN apt update
-RUN apt install -y unzip wget curl jq coreutils openssh-server net-tools vim openjdk-8-jdk python-pip python-dev libffi-dev libssl-dev
-RUN pip install -U pip
-RUN pip install --upgrade cffi
-RUN pip install ducktape==0.6.0
-
-VOLUME ["/kafka_src"]
-
+VOLUME ["/opt/kafka-dev"]
 ENV MIRROR="http://apache.cs.utah.edu/"
-RUN wget -q "${MIRROR}kafka/0.8.2.2/kafka_2.10-0.8.2.2.tgz" -O "/tmp/kafka_2.10-0.8.2.2.tgz" && tar xfz /tmp/kafka_2.10-0.8.2.2.tgz -C /opt && mv "/opt/kafka_2.10-0.8.2.2" "/opt/kafka-0.8.2.2"
-RUN wget -q "${MIRROR}kafka/0.9.0.1/kafka_2.10-0.9.0.1.tgz" -O "/tmp/kafka_2.10-0.9.0.1.tgz" && tar xfz /tmp/kafka_2.10-0.9.0.1.tgz -C /opt && mv "/opt/kafka_2.10-0.9.0.1" "/opt/kafka-0.9.0.1"
-RUN wget -q "${MIRROR}kafka/0.10.0.1/kafka_2.10-0.10.0.1.tgz" -O "/tmp/kafka_2.10-0.10.0.1.tgz" && tar xfz /tmp/kafka_2.10-0.10.0.1.tgz -C /opt && mv "/opt/kafka_2.10-0.10.0.1" "/opt/kafka-0.10.0.1"
-RUN wget -q "${MIRROR}kafka/0.10.1.1/kafka_2.10-0.10.1.1.tgz" -O "/tmp/kafka_2.10-0.10.1.1.tgz" && tar xfz /tmp/kafka_2.10-0.10.1.1.tgz -C /opt && mv "/opt/kafka_2.10-0.10.1.1" "/opt/kafka-0.10.1.1"
 
-RUN rm /tmp/kafka_*.tgz
 ADD ssh /root/.ssh
 RUN chmod 600 /root/.ssh/id_rsa
+RUN apt update && apt install -y unzip wget curl jq coreutils openssh-server net-tools vim openjdk-8-jdk python-pip python-dev libffi-dev libssl-dev
+RUN pip install -U pip && pip install --upgrade cffi ducktape==0.6.0
+RUN mkdir -p "/opt/kafka_2.10-0.8.2.2" && curl "${MIRROR}kafka/0.8.2.2/kafka_2.10-0.8.2.2.tgz" | tar xz --strip-components=1 -C "/opt/kafka_2.10-0.8.2.2"
+RUN mkdir -p "/opt/kafka_2.10-0.9.0.1" && curl "${MIRROR}kafka/0.9.0.1/kafka_2.10-0.9.0.1.tgz" | tar xz --strip-components=1 -C "/opt/kafka_2.10-0.9.0.1"
+RUN mkdir -p "/opt/kafka_2.10-0.10.0.1" && curl "${MIRROR}kafka/0.10.0.1/kafka_2.10-0.10.0.1.tgz" | tar xz --strip-components=1 -C "/opt/kafka_2.10-0.10.0.1"
+RUN mkdir -p "/opt/kafka_2.10-0.10.1.1" && curl "${MIRROR}kafka/0.10.1.1/kafka_2.10-0.10.1.1.tgz" | tar xz --strip-components=1 -C "/opt/kafka_2.10-0.10.1.1"
 
 CMD service ssh start && tail -f /dev/null

tests/docker/run_tests.sh

@@ -46,7 +46,7 @@ fi
 echo "Using kafka image: ${KAFKA_IMAGE}"
 docker inspect ${KAFKA_IMAGE}
 for i in $(seq -w 1 ${KAFKA_NUM_CONTAINERS}); do
-  docker run -d -t --name knode${i} --network knw -v ${KAFKA_SRC}:/kafka_src ${KAFKA_IMAGE}
+  docker run -d -t --name knode${i} --network knw -v ${KAFKA_SRC}:/opt/kafka-dev ${KAFKA_IMAGE}
 done
 
 docker info
@@ -55,18 +55,8 @@ docker network inspect knw
 
 for i in $(seq -w 1 ${KAFKA_NUM_CONTAINERS}); do
   echo knode${i}
-  docker exec knode${i} bash -c "(tar xfz /kafka_src/core/build/distributions/kafka_*SNAPSHOT.tgz -C /opt || echo missing kafka tgz did you build kafka tarball) && mv /opt/kafka*SNAPSHOT /opt/kafka-dev && ls -l /opt"
-  docker exec knode01 bash -c "ssh knode$i hostname"
-done
-
-# hack to copy test dependencies
-# this is required for running MiniKDC
-(cd ${KAFKA_SRC} && ./gradlew copyDependantTestLibs)
-for i in $(seq -w 1 ${KAFKA_NUM_CONTAINERS}); do
-  echo knode${i}
-  docker exec knode${i} bash -c "cp /kafka_src/core/build/dependant-testlibs/* /opt/kafka-dev/libs/"
   docker exec knode01 bash -c "ssh knode$i hostname"
 done
 
 bash tests/cluster_file_generator.sh > tests/cluster_file.json
-docker exec knode01 bash -c "cd /kafka_src; ducktape ${_DUCKTAPE_OPTIONS} --cluster-file tests/cluster_file.json ${TC_PATHS:-tests/kafkatest/tests}"
+docker exec knode01 bash -c "cd /opt/kafka-dev; ducktape ${_DUCKTAPE_OPTIONS} --cluster-file tests/cluster_file.json ${TC_PATHS:-tests/kafkatest/tests}"

tests/kafkatest/sanity_checks/test_verifiable_producer.py

@@ -23,7 +23,7 @@ from kafkatest.services.kafka import KafkaService
 from kafkatest.services.verifiable_producer import VerifiableProducer
 from kafkatest.services.zookeeper import ZookeeperService
 from kafkatest.utils import is_version
-from kafkatest.version import LATEST_0_8_2, LATEST_0_9, DEV_BRANCH, KafkaVersion
+from kafkatest.version import LATEST_0_8_2, LATEST_0_9, LATEST_0_10_0, LATEST_0_10_1, DEV_BRANCH, KafkaVersion
 
 
 class TestVerifiableProducer(Test):
@@ -48,6 +48,8 @@ class TestVerifiableProducer(Test):
     @cluster(num_nodes=3)
     @parametrize(producer_version=str(LATEST_0_8_2))
     @parametrize(producer_version=str(LATEST_0_9))
+    @parametrize(producer_version=str(LATEST_0_10_0))
+    @parametrize(producer_version=str(LATEST_0_10_1))
     @parametrize(producer_version=str(DEV_BRANCH))
     def test_simple_run(self, producer_version=DEV_BRANCH):
         """