зеркало из https://github.com/Azure/WALinuxAgent.git
Python configuration for tests (#2793)
* Python configuration for tests --------- Co-authored-by: narrieta <narrieta>
This commit is contained in:
Norberto Arrieta
GitHub
Родитель
08f86dbfd4
Коммит
1601e88cd1
|
|
@ -274,57 +274,81 @@ class AgentTestSuite(LisaTestSuite):
|
|||
log.info("************************************** [Node Setup] **************************************")
|
||||
log.info("")
|
||||
log.info("Test Node: %s", self.context.vm.name)
|
||||
log.info("IP Address: %s", self.context.vm_ip_address)
|
||||
log.info("Resource Group: %s", self.context.vm.resource_group)
|
||||
log.info("")
|
||||
|
||||
self.context.ssh_client.run_command("mkdir -p ~/bin/tests_e2e/tests; touch ~/bin/agent-env")
|
||||
|
||||
# Copy the test tools
|
||||
tools_path = self.context.test_source_directory/"orchestrator"/"scripts"
|
||||
tools_target_path = Path("~/bin")
|
||||
log.info("Copying %s to %s:%s", tools_path, self.context.node.name, tools_target_path)
|
||||
self.context.ssh_client.copy_to_node(tools_path, tools_target_path, recursive=True)
|
||||
|
||||
# Copy the test libraries
|
||||
lib_path = self.context.test_source_directory/"tests"/"lib"
|
||||
lib_target_path = Path("~/bin/tests_e2e/tests")
|
||||
log.info("Copying %s to %s:%s", lib_path, self.context.node.name, lib_target_path)
|
||||
self.context.ssh_client.copy_to_node(lib_path, lib_target_path, recursive=True)
|
||||
|
||||
# Copy the test agent
|
||||
agent_package_path: Path = self._get_agent_package_path()
|
||||
agent_package_target_path = Path("~/bin")/agent_package_path.name
|
||||
log.info("Copying %s to %s:%s", agent_package_path, self.context.node.name, agent_package_target_path)
|
||||
self.context.ssh_client.copy_to_node(agent_package_path, agent_package_target_path)
|
||||
|
||||
# Copy Pypy
|
||||
# NOTE: Pypy is pre-downloaded to /tmp on the container image used for Azure Pipelines runs. For dev runs,
|
||||
# if we don't find Pypy under /tmp, then we download it a few lines below.
|
||||
#
|
||||
# Ensure that the correct version (x84 vs ARM64) Pypy has been downloaded; it is pre-downloaded to /tmp on the container image
|
||||
# used for Azure Pipelines runs, but for developer runs it may need to be downloaded.
|
||||
#
|
||||
if self.context.ssh_client.get_architecture() == "aarch64":
|
||||
pypy_path = Path("/tmp/pypy3.7-arm64.tar.bz2")
|
||||
pypy_download = "https://downloads.python.org/pypy/pypy3.7-v7.3.5-aarch64.tar.bz2"
|
||||
else:
|
||||
pypy_path = Path("/tmp/pypy3.7-x64.tar.bz2")
|
||||
pypy_download = "https://downloads.python.org/pypy/pypy3.7-v7.3.5-linux64.tar.bz2"
|
||||
|
||||
if not pypy_path.exists():
|
||||
if pypy_path.exists():
|
||||
log.info("Found Pypy at %s", pypy_path)
|
||||
else:
|
||||
log.info("Downloading %s to %s", pypy_download, pypy_path)
|
||||
run_command(["wget", pypy_download, "-O", pypy_path])
|
||||
pypy_target_path = Path("~/bin/pypy3.7.tar.bz2")
|
||||
log.info("Copying %s to %s:%s", pypy_path, self.context.node.name, pypy_target_path)
|
||||
self.context.ssh_client.copy_to_node(pypy_path, pypy_target_path)
|
||||
|
||||
# Install the tools and libraries
|
||||
install_command = lambda: self.context.ssh_client.run_command(f"~/bin/scripts/install-tools --agent-package {agent_package_target_path}")
|
||||
log.info('Installing tools on the test node\n%s', install_command())
|
||||
log.info('Remote commands will use %s', self.context.ssh_client.run_command("which python3"))
|
||||
#
|
||||
# Create a tarball with the files we need to copy to the test node. The tarball includes two directories:
|
||||
#
|
||||
# * bin - Executables file (Bash and Python scripts)
|
||||
# * lib - Library files (Python modules)
|
||||
#
|
||||
# After extracting the tarball on the test node, 'bin' will be added to PATH and PYTHONPATH will be set to 'lib'.
|
||||
#
|
||||
# Note that executables are placed directly under 'bin', while the path for Python modules is preserved under 'lib.
|
||||
#
|
||||
tarball_path: Path = Path("/tmp/waagent.tar")
|
||||
log.info("Creating %s with the files need on the test node", tarball_path)
|
||||
log.info("Adding orchestrator/scripts")
|
||||
run_command(['tar', 'cvf', str(tarball_path), '--transform=s,.*/,bin/,', '-C', str(self.context.test_source_directory/"orchestrator"/"scripts"), '.'])
|
||||
# log.info("Adding tests/scripts")
|
||||
# run_command(['tar', 'rvf', str(tarball_path), '--transform=s,.*/,bin/,', '-C', str(self.context.test_source_directory/"tests"/"scripts"), '.'])
|
||||
log.info("Adding tests/lib")
|
||||
run_command(['tar', 'rvf', str(tarball_path), '--transform=s,^,lib/,', '-C', str(self.context.test_source_directory.parent), '--exclude=__pycache__', 'tests_e2e/tests/lib'])
|
||||
log.info("Contents of %s:\n\n%s", tarball_path, run_command(['tar', 'tvf', str(tarball_path)]))
|
||||
|
||||
#
|
||||
# Cleanup the test node (useful for developer runs)
|
||||
#
|
||||
log.info('Preparing the test node for setup')
|
||||
# Note that removing lib requires sudo, since a Python cache may have been created by tests using sudo
|
||||
self.context.ssh_client.run_command("rm -rvf ~/{bin,lib,tmp}", use_sudo=True)
|
||||
|
||||
#
|
||||
# Copy the tarball, Pypy and the test Agent to the test node
|
||||
#
|
||||
target_path = Path("~")/"tmp"
|
||||
self.context.ssh_client.run_command(f"mkdir {target_path}")
|
||||
log.info("Copying %s to %s:%s", tarball_path, self.context.node.name, target_path)
|
||||
self.context.ssh_client.copy_to_node(tarball_path, target_path)
|
||||
log.info("Copying %s to %s:%s", pypy_path, self.context.node.name, target_path)
|
||||
self.context.ssh_client.copy_to_node(pypy_path, target_path)
|
||||
agent_package_path: Path = self._get_agent_package_path()
|
||||
log.info("Copying %s to %s:%s", agent_package_path, self.context.node.name, target_path)
|
||||
self.context.ssh_client.copy_to_node(agent_package_path, target_path)
|
||||
|
||||
#
|
||||
# Extract the tarball and execute the install scripts
|
||||
#
|
||||
log.info('Installing tools on the test node')
|
||||
command = f"tar xf {target_path/tarball_path.name} && ~/bin/install-tools"
|
||||
log.info("%s\n%s", command, self.context.ssh_client.run_command(command))
|
||||
|
||||
# Install the agent
|
||||
if self.context.is_vhd:
|
||||
log.info("Using a VHD; will not install the Test Agent.")
|
||||
else:
|
||||
install_command = lambda: self.context.ssh_client.run_command(f"install-agent --package {agent_package_target_path} --version {AGENT_VERSION}", use_sudo=True)
|
||||
log.info("Installing the Test Agent on %s\n%s", self.context.node.name, install_command())
|
||||
log.info("Installing the Test Agent on the test node")
|
||||
command = f"install-agent --package ~/tmp/{agent_package_path.name} --version {AGENT_VERSION}"
|
||||
log.info("%s\n%s", command, self.context.ssh_client.run_command(command, use_sudo=True))
|
||||
|
||||
log.info("Completed test node setup")
|
||||
|
||||
def _collect_node_logs(self) -> None:
|
||||
"""
|
||||
|
|
@ -393,6 +417,8 @@ class AgentTestSuite(LisaTestSuite):
|
|||
# pylint seems to think self.context.test_suites is not iterable. Suppressing warning, since its type is List[AgentTestSuite]
|
||||
# E1133: Non-iterable value self.context.test_suites is used in an iterating context (not-an-iterable)
|
||||
for suite in self.context.test_suites: # pylint: disable=E1133
|
||||
log.info("Executing test suite %s", suite.name)
|
||||
self.context.lisa_log.info("Executing Test Suite %s", suite.name)
|
||||
test_suite_success = self._execute_test_suite(suite) and test_suite_success
|
||||
|
||||
test_suite_success = self._check_agent_log() and test_suite_success
|
||||
|
|
@ -419,7 +445,7 @@ class AgentTestSuite(LisaTestSuite):
|
|||
finally:
|
||||
self._clean_up()
|
||||
if not success:
|
||||
self._mark_log_as_failed(log_path)
|
||||
self._mark_log_as_failed()
|
||||
|
||||
def _execute_test_suite(self, suite: TestSuiteInfo) -> bool:
|
||||
"""
|
||||
|
|
@ -432,7 +458,7 @@ class AgentTestSuite(LisaTestSuite):
|
|||
success: bool = True # True if all the tests succeed
|
||||
|
||||
with _set_thread_name(suite_full_name): # The thread name is added to the LISA log
|
||||
log_path:Path = self.context.log_path/f"{suite_full_name}.log"
|
||||
log_path: Path = self.context.log_path/f"{suite_full_name}.log"
|
||||
with set_current_thread_log(log_path):
|
||||
try:
|
||||
log.info("")
|
||||
|
|
@ -447,7 +473,7 @@ class AgentTestSuite(LisaTestSuite):
|
|||
test_start_time: datetime.datetime = datetime.datetime.now()
|
||||
|
||||
log.info("******** Executing %s", test_name)
|
||||
self.context.lisa_log.info("******** Executing %s", test_full_name)
|
||||
self.context.lisa_log.info("Executing test %s", test_full_name)
|
||||
|
||||
try:
|
||||
|
||||
|
|
@ -455,7 +481,7 @@ class AgentTestSuite(LisaTestSuite):
|
|||
|
||||
summary.append(f"[Passed] {test_name}")
|
||||
log.info("******** [Passed] %s", test_name)
|
||||
self.context.lisa_log.info("******** [Passed] %s", test_full_name)
|
||||
self.context.lisa_log.info("[Passed] %s", test_full_name)
|
||||
self._report_test_result(
|
||||
suite_full_name,
|
||||
test_name,
|
||||
|
|
@ -514,7 +540,7 @@ class AgentTestSuite(LisaTestSuite):
|
|||
add_exception_stack_trace=True)
|
||||
finally:
|
||||
if not success:
|
||||
self._mark_log_as_failed(log_path)
|
||||
self._mark_log_as_failed()
|
||||
|
||||
return success
|
||||
|
||||
|
|
@ -551,21 +577,17 @@ class AgentTestSuite(LisaTestSuite):
|
|||
log.info("There are no errors in the agent log")
|
||||
return True
|
||||
|
||||
log_path: Path = self.context.log_path/f"CheckAgentLog-{self.context.environment_name}.log"
|
||||
message = f"Detected {len(errors)} error(s) in the agent log. See {log_path} for a full report."
|
||||
self.context.lisa_log.info(message)
|
||||
log.info(message)
|
||||
|
||||
with set_current_thread_log(log_path):
|
||||
log.info("Detected %s error(s) in the agent log:\n\n%s", len(errors), '\n'.join(['\t' + e.text for e in errors]))
|
||||
self._mark_log_as_failed(log_path)
|
||||
message = f"Detected {len(errors)} error(s) in the agent log"
|
||||
self.context.lisa_log.error(message)
|
||||
log.error("%s:\n\n%s\n", message, '\n'.join(['\t\t' + e.text.replace('\n', '\n\t\t') for e in errors]))
|
||||
self._mark_log_as_failed()
|
||||
|
||||
self._report_test_result(
|
||||
self.context.environment_name,
|
||||
"CheckAgentLog",
|
||||
TestStatus.FAILED,
|
||||
start_time,
|
||||
message=message + '\n' + '\n'.join([e.text for e in errors[0:3]]))
|
||||
message=message + ' - First few errors:\n' + '\n'.join([e.text for e in errors[0:3]]))
|
||||
except: # pylint: disable=bare-except
|
||||
log.exception("Error checking agent log")
|
||||
self._report_test_result(
|
||||
|
|
@ -579,11 +601,11 @@ class AgentTestSuite(LisaTestSuite):
|
|||
return False
|
||||
|
||||
@staticmethod
|
||||
def _mark_log_as_failed(log_path: Path):
|
||||
def _mark_log_as_failed():
|
||||
"""
|
||||
Renames the given log to prefix it with "_".
|
||||
Adds a message to indicate the log contains errors.
|
||||
"""
|
||||
log_path.rename(log_path.parent / ("_" + log_path.name))
|
||||
log.info("MARKER-LOG-WITH-ERRORS")
|
||||
|
||||
@staticmethod
|
||||
def _report_test_result(
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3
|
||||
#!/usr/bin/env pypy3
|
||||
|
||||
# Microsoft Azure Linux Agent
|
||||
#
|
||||
|
|
|
|||
|
|
@ -18,34 +18,20 @@
|
|||
#
|
||||
|
||||
#
|
||||
# Returns the path to the Python executable.
|
||||
# Returns the PYTHONPATH on which the azurelinuxagent and associated modules are located.
|
||||
#
|
||||
# To do this, the script walks the site packages for the Python used to execute the agent,
|
||||
# looking for the directory that contains "azurelinuxagent".
|
||||
#
|
||||
set -euo pipefail
|
||||
|
||||
# python3 is available on most distros
|
||||
if which python3 2> /dev/null; then
|
||||
exit 0
|
||||
fi
|
||||
$(get-agent-python) -c '
|
||||
import site
|
||||
import os
|
||||
|
||||
# try python
|
||||
if which python 2> /dev/null; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# try some well-known locations
|
||||
declare -a known_locations=(
|
||||
"/usr/share/oem/python/bin/python"
|
||||
"/usr/share/oem/python/bin/python3"
|
||||
)
|
||||
|
||||
for python in "${known_locations[@]}"
|
||||
do
|
||||
if [[ -e $python ]]; then
|
||||
echo "$python"
|
||||
exit 0
|
||||
fi
|
||||
done
|
||||
|
||||
echo "Can't find the python executable" >&2
|
||||
|
||||
exit 1
|
||||
for dir in site.getsitepackages():
|
||||
if os.path.isdir(dir + "/azurelinuxagent"):
|
||||
print(dir)
|
||||
exit(0)
|
||||
exit(1)
|
||||
'
|
||||
|
|
@ -0,0 +1,59 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
# Microsoft Azure Linux Agent
|
||||
#
|
||||
# Copyright 2018 Microsoft Corporation
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
#
|
||||
# Returns the path of the Python executable used to start the Agent.
|
||||
#
|
||||
set -euo pipefail
|
||||
|
||||
# if the agent is running, get the python command from 'exe' in the /proc file system
|
||||
if test -e /run/waagent.pid; then
|
||||
exe="/proc/$(cat /run/waagent.pid)/exe"
|
||||
if test -e "$exe"; then
|
||||
# exe is a symbolic link; return its target
|
||||
readlink -f "$exe"
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
|
||||
# try all the instances of 'python' and 'python3' in $PATH
|
||||
for path in $(echo "$PATH" | tr ':' '\n'); do
|
||||
if [[ -e $path ]]; then
|
||||
for python in $(find "$path" -maxdepth 1 -name python3 -or -name python); do
|
||||
if $python -c 'import azurelinuxagent' 2> /dev/null; then
|
||||
echo "$python"
|
||||
exit 0
|
||||
fi
|
||||
done
|
||||
fi
|
||||
done
|
||||
|
||||
# try some well-known locations
|
||||
declare -a known_locations=(
|
||||
"/usr/share/oem/python/bin/python"
|
||||
)
|
||||
for python in "${known_locations[@]}"
|
||||
do
|
||||
if $python -c 'import azurelinuxagent' 2> /dev/null; then
|
||||
echo "$python"
|
||||
exit 0
|
||||
fi
|
||||
done
|
||||
|
||||
exit 1
|
||||
|
|
@ -1,74 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
# Microsoft Azure Linux Agent
|
||||
#
|
||||
# Copyright 2018 Microsoft Corporation
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
#
|
||||
# Returns the PYTHONPATH on which the azurelinuxagent and associated modules are located.
|
||||
#
|
||||
# To do this, the script tries to find the python command used to start the agent and then
|
||||
# returns the value of site.getsitepackages().
|
||||
#
|
||||
set -euo pipefail
|
||||
|
||||
find-agent-python() {
|
||||
# if the agent is running, get the python command from 'exe' in the /proc file system
|
||||
if test -e /run/waagent.pid; then
|
||||
exe="/proc/$(cat /run/waagent.pid)/exe"
|
||||
if test -e "$exe"; then
|
||||
# exe is a symbolic link; return its target
|
||||
readlink -f "$exe"
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
|
||||
# try all the instances of 'python' and 'python3' in $PATH
|
||||
for path in $(echo "$PATH" | tr ':' '\n'); do
|
||||
if [[ -e $path ]]; then
|
||||
for python in $(find "$path" -maxdepth 1 -name python3 -or -name python); do
|
||||
if $python -c 'import azurelinuxagent' 2> /dev/null; then
|
||||
echo "$python"
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
fi
|
||||
done
|
||||
|
||||
# try some well-known locations
|
||||
declare -a known_locations=(
|
||||
"/usr/share/oem/python/bin/python"
|
||||
"/usr/share/oem/python/bin/python3"
|
||||
)
|
||||
|
||||
for python in "${known_locations[@]}"
|
||||
do
|
||||
if $python -c 'import azurelinuxagent' 2> /dev/null; then
|
||||
echo "$python"
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
|
||||
|
||||
return 1
|
||||
}
|
||||
|
||||
if ! python=$(find-agent-python); then
|
||||
echo "Can't find the python command used to start the agent" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
$python -c 'import site; print(":".join(site.getsitepackages()))'
|
||||
|
|
@ -73,12 +73,13 @@ fi
|
|||
echo "Service name: $service_name"
|
||||
|
||||
#
|
||||
# Find the path to the Agent's executable file
|
||||
# Output the initial version of the agent
|
||||
#
|
||||
waagent=$(get-agent-path)
|
||||
python=$(get-agent-python)
|
||||
waagent=$(get-agent-bin-path)
|
||||
echo "Agent's path: $waagent"
|
||||
$waagent --version
|
||||
echo ""
|
||||
$python "$waagent" --version
|
||||
printf "\n"
|
||||
|
||||
#
|
||||
# Install the package
|
||||
|
|
@ -112,7 +113,7 @@ echo "Verifying agent installation..."
|
|||
check-version() {
|
||||
for i in {0..5}
|
||||
do
|
||||
if $waagent --version | grep -E "Goal state agent:\s+$version" > /dev/null; then
|
||||
if $python "$waagent" --version | grep -E "Goal state agent:\s+$version" > /dev/null; then
|
||||
return 0
|
||||
fi
|
||||
sleep 10
|
||||
|
|
@ -129,7 +130,7 @@ else
|
|||
exit_code=1
|
||||
fi
|
||||
|
||||
$waagent --version
|
||||
$python "$waagent" --version
|
||||
printf "\n"
|
||||
service-status $service_name
|
||||
|
||||
|
|
|
|||
|
|
@ -25,112 +25,111 @@
|
|||
|
||||
set -euo pipefail
|
||||
|
||||
usage() (
|
||||
echo "Usage: install-tools -p|--agent-package <path>"
|
||||
exit 1
|
||||
)
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
-p|--agent-package)
|
||||
shift
|
||||
if [ "$#" -lt 1 ]; then
|
||||
usage
|
||||
fi
|
||||
agent_package=$1
|
||||
shift
|
||||
;;
|
||||
*)
|
||||
usage
|
||||
esac
|
||||
done
|
||||
if [ "$#" -ne 0 ] || [ -z ${agent_package+x} ]; then
|
||||
usage
|
||||
fi
|
||||
|
||||
echo "Installing scripts to ~/bin"
|
||||
mv ~/bin/scripts/* ~/bin
|
||||
rm -r ~/bin/scripts
|
||||
PATH="$HOME/bin:$PATH"
|
||||
|
||||
# If the system's Python is <= 3.7, install Pypy and make it the default Python for the user executing the tests
|
||||
python=$(~/bin/find-python)
|
||||
python_version=$($python -c 'import sys; print("{0:02}.{1:02}".format(sys.version_info[0], sys.version_info[1]))')
|
||||
echo "Python: $python ($python_version)"
|
||||
if [[ $python_version < "03.07" ]]; then
|
||||
echo "Installing Pypy 3.7 to ~/bin and making it the default Python for user $USER"
|
||||
# bzip2/lbzip2 (used by tar to uncompress *.bz2 files) are not available by default in some distros;
|
||||
# use Python to uncompress the Pypy tarball.
|
||||
$python ~/bin/uncompress.py ~/bin/pypy3.7.tar.bz2 ~/bin/pypy3.7.tar
|
||||
tar xf ~/bin/pypy3.7.tar -C ~/bin
|
||||
rm ~/bin/pypy3.7.tar ~/bin/pypy3.7.tar.bz2
|
||||
python=$(get-agent-python)
|
||||
echo "Python executable: $python"
|
||||
echo "Python version: $($python --version)"
|
||||
|
||||
if [[ -e ~/bin/python ]]; then
|
||||
rm ~/bin/python
|
||||
fi
|
||||
ln -s ~/bin/pypy*/bin/pypy3.7 ~/bin/python
|
||||
#
|
||||
# Install Pypy as ~/bin/pypy3
|
||||
#
|
||||
# Note that bzip2/lbzip2 (used by tar to uncompress *.bz2 files) are not available by default in some distros;
|
||||
# use Python to uncompress the Pypy tarball.
|
||||
#
|
||||
echo "Installing Pypy 3.7"
|
||||
$python ~/bin/uncompress.py ~/tmp/pypy3.7-*.tar.bz2 ~/tmp/pypy3.7.tar
|
||||
tar xf ~/tmp/pypy3.7.tar -C ~/bin
|
||||
echo "Pypy was installed in $(ls -d ~/bin/pypy*)"
|
||||
ln -s ~/bin/pypy*/bin/pypy3.7 ~/bin/pypy3
|
||||
echo "Creating symbolic link to Pypy: ~/bin/pypy3"
|
||||
|
||||
if [[ -e ~/bin/python3 ]]; then
|
||||
rm ~/bin/python3
|
||||
fi
|
||||
ln -s ~/bin/pypy*/bin/pypy3.7 ~/bin/python3
|
||||
|
||||
echo "Installing the 'distro' module"
|
||||
python3 -m ensurepip
|
||||
python3 -mpip install -U pip wheel
|
||||
python3 -mpip install distro
|
||||
#
|
||||
# The 'distro' and 'platform' modules in Pypy have small differences with the ones in the system's Python.
|
||||
# This can create problems in tests that use the get_distro() method in the Agent's 'version.py' module.
|
||||
# To work around this, we copy the system's 'distro' module to Pypy.
|
||||
#
|
||||
# In the case of 'platform', the 'linux_distribution' method was removed on Python 3.7 so we check the
|
||||
# system's module and, if the method does not exist, we also remove it from Pypy. Ubuntu 16 and 18 are
|
||||
# special cases in that the 'platform' module in Pypy identifies the distro as 'debian'; in this case we
|
||||
# copy the system's 'platform' module to Pypy.
|
||||
#
|
||||
distro_path=$($python -c '
|
||||
try:
|
||||
import distro
|
||||
except:
|
||||
exit(0)
|
||||
print(distro.__file__.replace("__init__.py", "distro.py"))
|
||||
exit(0)
|
||||
')
|
||||
if [[ "$distro_path" != "" ]]; then
|
||||
echo "Copying the system's distro module to Pypy"
|
||||
cp -v "$distro_path" ~/bin/pypy*/site-packages
|
||||
else
|
||||
# In some distros (e.g. Flatcar), Python is not in PATH; in that case create a symlink under ~/bin
|
||||
if ! which python3 > /dev/null 2>&1; then
|
||||
if [[ ! $python_version < "03.00" ]]; then
|
||||
echo "Python ($python) is not in PATH; creating symbolic links as ~/bin/python and ~/bin/python3"
|
||||
ln -s "$python" ~/bin/python
|
||||
ln -s "$python" ~/bin/python3
|
||||
fi
|
||||
echo "The distro module is not is not installing on the system; skipping."
|
||||
fi
|
||||
|
||||
has_linux_distribution=$($python -c 'import platform; print(hasattr(platform, "linux_distribution"))')
|
||||
if [[ "$has_linux_distribution" == "False" ]]; then
|
||||
echo "Python does not have platform.linux_distribution; removing it from Pypy"
|
||||
sed -i 's/def linux_distribution(/def __linux_distribution__(/' ~/bin/pypy*/lib-python/3/platform.py
|
||||
else
|
||||
echo "Python has platform.linux_distribution"
|
||||
uname=$(uname -v)
|
||||
if [[ "$uname" == *~18*-Ubuntu* || "$uname" == *~16*-Ubuntu* ]]; then
|
||||
echo "Copying the system's platform module to Pypy"
|
||||
pypy_platform=$(pypy3 -c 'import platform; print(platform.__file__)')
|
||||
python_platform=$($python -c 'import platform; print(platform.__file__)')
|
||||
cp -v "$python_platform" "$pypy_platform"
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "Installing Agent modules to ~/bin"
|
||||
unzip.py "$agent_package" ~/bin/WALinuxAgent
|
||||
unzip.py ~/bin/WALinuxAgent/bin/WALinuxAgent-*.egg ~/bin/WALinuxAgent/bin/WALinuxAgent.egg
|
||||
mv ~/bin/WALinuxAgent/bin/WALinuxAgent.egg/azurelinuxagent ~/bin
|
||||
rm -rf ~/bin/WALinuxAgent
|
||||
#
|
||||
# Now install the test Agent as a module package in Pypy.
|
||||
#
|
||||
echo "Installing Agent modules to Pypy"
|
||||
unzip.py ~/tmp/WALinuxAgent-*.zip ~/tmp/WALinuxAgent
|
||||
unzip.py ~/tmp/WALinuxAgent/bin/WALinuxAgent-*.egg ~/tmp/WALinuxAgent/bin/WALinuxAgent.egg
|
||||
mv ~/tmp/WALinuxAgent/bin/WALinuxAgent.egg/azurelinuxagent ~/bin/pypy*/site-packages
|
||||
|
||||
#
|
||||
# Create ~/bin/agent-env to set PATH and PYTHONPATH.
|
||||
# Log the results of get_distro() in Pypy and Python.
|
||||
#
|
||||
# We add $HOME/bin to the front of PATH, so tools in ~/bin (including python) will
|
||||
# take precedence over system tools with the same name.
|
||||
pypy_get_distro=$(pypy3 -c 'from azurelinuxagent.common.version import get_distro; print(get_distro())')
|
||||
python_get_distro=$($python -c 'from azurelinuxagent.common.version import get_distro; print(get_distro())')
|
||||
echo "Pypy get_distro(): $pypy_get_distro"
|
||||
echo "Python get_distro(): $python_get_distro"
|
||||
|
||||
#
|
||||
# We set PYTHONPATH to include the location of the agent modules installed on the VM image and also
|
||||
# the test modules we copied to ~/bin.
|
||||
# Create ~/bin/set-agent-env to set PATH and PYTHONPATH.
|
||||
#
|
||||
# We append $HOME/bin to PATH and set PYTHONPATH to $HOME/lib (bin contains the scripts used by tests, while
|
||||
# lib contains the Python libraries used by tests).
|
||||
#
|
||||
echo "Creating ~/bin/agent-env to set PATH and PYTHONPATH"
|
||||
echo '
|
||||
if [[ $PATH != *"$HOME/bin"* ]]; then
|
||||
PATH="$HOME/bin:$PATH:"
|
||||
echo "Creating ~/bin/set-agent-env to set PATH and PYTHONPATH"
|
||||
|
||||
echo "
|
||||
if [[ \$PATH != *\"$HOME/bin\"* ]]; then
|
||||
PATH=\"$HOME/bin:\$PATH:\"
|
||||
fi
|
||||
|
||||
export PYTHONPATH="$HOME/bin"
|
||||
' > ~/bin/agent-env
|
||||
chmod u+x ~/bin/agent-env
|
||||
export PYTHONPATH=\"$HOME/lib\"
|
||||
" > ~/bin/set-agent-env
|
||||
|
||||
echo "Adding ~/bin/agent-env to ~/.bash_profile"
|
||||
# In some distros, e.g. Flatcar, .bash_profile is a symbolic link to a read-only file. Make a copy in
|
||||
# that case.
|
||||
chmod u+x ~/bin/set-agent-env
|
||||
|
||||
#
|
||||
# Add ~/bin/set-agent-env to .bash_profile to simplify interactive debugging sessions
|
||||
#
|
||||
# Note that in some distros .bash_profile is a symbolic link to a read-only file. Make a copy in that case.
|
||||
#
|
||||
echo "Adding ~/bin/set-agent-env to ~/.bash_profile"
|
||||
if test -e ~/.bash_profile && ls -l .bash_profile | grep '\->'; then
|
||||
cp ~/.bash_profile ~/.bash_profile-bk
|
||||
rm ~/.bash_profile
|
||||
mv ~/.bash_profile-bk ~/.bash_profile
|
||||
fi
|
||||
if ! test -e ~/.bash_profile || ! grep '~/bin/agent-env' ~/.bash_profile > /dev/null; then
|
||||
echo 'source ~/bin/agent-env
|
||||
if ! test -e ~/.bash_profile || ! grep '~/bin/set-agent-env' ~/.bash_profile > /dev/null; then
|
||||
echo 'source ~/bin/set-agent-env
|
||||
' >> ~/.bash_profile
|
||||
fi
|
||||
|
||||
source ~/bin/agent-env
|
||||
echo "PATH=$PATH"
|
||||
echo "PYTHONPATH=$PYTHONPATH"
|
||||
echo "python3 -> $(which python3)"
|
||||
python3 --version
|
||||
|
|
|
|||
|
|
@ -1,5 +1,3 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
# Microsoft Azure Linux Agent
|
||||
#
|
||||
# Copyright 2018 Microsoft Corporation
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env python3
|
||||
#!/usr/bin/env pypy3
|
||||
|
||||
# Microsoft Azure Linux Agent
|
||||
#
|
||||
|
|
|
|||
|
|
@ -86,18 +86,27 @@ sudo find "$LOGS_DIRECTORY" -exec chown "$USER" {} \;
|
|||
#
|
||||
# Move the relevant logs to the staging directory
|
||||
#
|
||||
# Move the logs for failed tests to a temporary location
|
||||
mkdir "$BUILD_ARTIFACTSTAGINGDIRECTORY"/tmp
|
||||
for log in $(grep -l MARKER-LOG-WITH-ERRORS "$LOGS_DIRECTORY"/*.log); do
|
||||
mv "$log" "$BUILD_ARTIFACTSTAGINGDIRECTORY"/tmp
|
||||
done
|
||||
# Move the environment logs to "environment_logs"
|
||||
if ls "$LOGS_DIRECTORY"/env-*.log > /dev/null 2>&1; then
|
||||
mkdir "$BUILD_ARTIFACTSTAGINGDIRECTORY"/environment_logs
|
||||
mv "$LOGS_DIRECTORY"/env-*.log "$BUILD_ARTIFACTSTAGINGDIRECTORY"/environment_logs
|
||||
fi
|
||||
# Move the rest of the logs to "test_logs"
|
||||
if ls "$LOGS_DIRECTORY"/*.log > /dev/null 2>&1; then
|
||||
mkdir "$BUILD_ARTIFACTSTAGINGDIRECTORY"/test_logs
|
||||
mv "$LOGS_DIRECTORY"/*.log "$BUILD_ARTIFACTSTAGINGDIRECTORY"/test_logs
|
||||
fi
|
||||
# Move the logs for failed tests to the main directory
|
||||
if ls "$BUILD_ARTIFACTSTAGINGDIRECTORY"/test_logs/_*.log > /dev/null 2>&1; then
|
||||
mv "$BUILD_ARTIFACTSTAGINGDIRECTORY"/test_logs/_*.log "$BUILD_ARTIFACTSTAGINGDIRECTORY"
|
||||
if ls "$BUILD_ARTIFACTSTAGINGDIRECTORY"/tmp/*.log > /dev/null 2>&1; then
|
||||
mv "$BUILD_ARTIFACTSTAGINGDIRECTORY"/tmp/*.log "$BUILD_ARTIFACTSTAGINGDIRECTORY"
|
||||
fi
|
||||
rmdir "$BUILD_ARTIFACTSTAGINGDIRECTORY"/tmp
|
||||
# Move the logs collected from the test VMs to vm_logs
|
||||
if ls "$LOGS_DIRECTORY"/*.tgz > /dev/null 2>&1; then
|
||||
mkdir "$BUILD_ARTIFACTSTAGINGDIRECTORY"/vm_logs
|
||||
mv "$LOGS_DIRECTORY"/*.tgz "$BUILD_ARTIFACTSTAGINGDIRECTORY"/vm_logs
|
||||
|
|
|
|||
|
|
@ -46,7 +46,7 @@ class SshClient(object):
|
|||
sudo = "sudo env PATH=$PATH PYTHONPATH=$PYTHONPATH" if use_sudo else ''
|
||||
return retry_ssh_run(lambda: shell.run_command([
|
||||
"ssh", "-o", "StrictHostKeyChecking=no", "-i", self._private_key_file, destination,
|
||||
f"source ~/bin/agent-env;{sudo} {command}"]))
|
||||
f"if [[ -e ~/bin/set-agent-env ]]; then source ~/bin/set-agent-env; fi; {sudo} {command}"]))
|
||||
|
||||
@staticmethod
|
||||
def generate_ssh_key(private_key_file: Path):
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче