mozilla
/
addons-server
зеркало из https://github.com/mozilla/addons-server.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Better defaults for make setup with better testing of python. (#22804)

This commit is contained in:
Kevin Meinhardt 2024-11-04 14:08:08 +01:00 коммит произвёл GitHub
Родитель 7782a55d4a
Коммит a83aeadd6f
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: B5690EEEBB952194
5 изменённых файлов: 266 добавлений и 175 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

10
.github/workflows/ci.yml поставляемый
Просмотреть файл

@ -88,7 +88,7 @@ jobs:
shell: bash
run: |
docker compose version
npm exec jest -- ./tests/make --runInBand
make test_setup
test_run_docker_action:
runs-on: ubuntu-latest
@ -158,6 +158,14 @@ jobs:
exit 1
fi
- name: Test setup
uses: ./.github/actions/run-docker
with:
digest: ${{ needs.build.outputs.digest }}
version: ${{ needs.build.outputs.version }}
run: |
pytest tests/make/
docs_build:
runs-on: ubuntu-latest
needs: build

5
Makefile-os
Просмотреть файл

@ -7,7 +7,6 @@
DOCKER_PROGRESS ?= auto
DOCKER_METADATA_FILE ?= buildx-bake-metadata.json
DOCKER_PUSH ?=
export DEBUG ?= True
export DOCKER_COMMIT ?=
export DOCKER_BUILD ?=
export DOCKER_VERSION ?=
@ -68,6 +67,10 @@ help_submake:
@echo "\nAll other commands will be passed through to the docker 'web' container make:"
@make -f Makefile-docker help_submake
.PHONY: test_setup
test_setup:
npm exec jest -- ./tests/make --runInBand
.PHONY: setup
setup: ## create configuration files version.json and .env required to run this project
for path in $(CLEAN_PATHS); do rm -rf "$(PWD)/$$path" && echo "$$path removed"; done

85
scripts/setup.py
Просмотреть файл

@ -22,39 +22,44 @@ def get_env_file():
return env
env = get_env_file()
def get_value(key, default_value):
if key in os.environ:
return os.environ[key]
if key in env:
return env[key]
from_file = get_env_file()
if key in from_file:
return from_file[key]
return default_value
def get_docker_tag():
image_name = 'mozilla/addons-server'
version = os.environ.get('DOCKER_VERSION')
digest = os.environ.get('DOCKER_DIGEST')
image = 'mozilla/addons-server'
version = 'local'
tag = f'{image_name}:local'
if digest:
tag = f'{image_name}@{digest}'
elif version:
tag = f'{image_name}:{version}'
else:
tag = get_value('DOCKER_TAG', tag)
# First get the tag from the full tag variable
tag = get_value('DOCKER_TAG', f'{image}:{version}')
# extract version or digest from existing tag
if '@' in tag:
digest = tag.split('@')[1]
image, digest = tag.split('@')
version = None
elif ':' in tag:
version = tag.split(':')[1]
image, version = tag.split(':')
digest = None
print('Docker tag: ', tag)
# DOCKER_DIGEST or DOCKER_VERSION can override the extracted version or digest
# Note: it will inherit the image from the provided DOCKER_TAG if also provided
if bool(os.environ.get('DOCKER_DIGEST', False)):
digest = os.environ['DOCKER_DIGEST']
tag = f'{image}@{digest}'
version = None
elif bool(os.environ.get('DOCKER_VERSION', False)):
version = os.environ['DOCKER_VERSION']
tag = f'{image}:{version}'
digest = None
print('tag: ', tag)
print('version: ', version)
print('digest: ', digest)
@ -74,16 +79,48 @@ def get_docker_tag():
# 3. the value defined in the environment variable
# 4. the value defined in the make args.
docker_tag, docker_version, docker_digest = get_docker_tag()
docker_target = get_value('DOCKER_TARGET', 'development')
compose_file = get_value('COMPOSE_FILE', ('docker-compose.yml'))
def main():
docker_tag, docker_version, _ = get_docker_tag()
set_env_file(
is_local = docker_version == 'local'
# The default target should be inferred from the version
# but can be freely overridden by the user.
# E.g running local image in production mode
docker_target = get_value(
'DOCKER_TARGET', ('development' if is_local else 'production')
)
is_production = docker_target == 'production'
# The default value for which compose files to use is based on the target
# but can be freely overridden by the user.
# E.g running a production image in development mode with source code changes
compose_file = get_value(
'COMPOSE_FILE',
(
'docker-compose.yml:docker-compose.ci.yml'
if is_production
else 'docker-compose.yml'
),
)
# DEBUG is special, as we should allow the user to override it
# but we should not set a default to the previously set value but instead
# to the most sensible default.
debug = os.environ.get('DEBUG', str(False if is_production else True))
set_env_file(
{
'COMPOSE_FILE': compose_file,
'DOCKER_TAG': docker_tag,
'DOCKER_TARGET': docker_target,
'HOST_UID': get_value('HOST_UID', os.getuid()),
'DEBUG': debug,
}
)
)
if __name__ == '__main__':
main()

147
tests/make/make.spec.js
Просмотреть файл

@ -8,24 +8,19 @@ const rootPath = path.join(__dirname, '..', '..');
const envPath = path.join(rootPath, '.env');
function runSetup(env) {
fs.writeFileSync(envPath, '');
spawnSync('make', ['setup'], {
env: { ...process.env, ...env },
encoding: 'utf-8',
});
}
function readEnvFile(name) {
return parse(fs.readFileSync(envPath, { encoding: 'utf-8' }))[name];
return parse(fs.readFileSync(envPath, { encoding: 'utf-8' }));
}
test('map docker compose config', () => {
const values = {
values = runSetup({
DOCKER_VERSION: 'version',
HOST_UID: 'uid',
};
fs.writeFileSync(envPath, '');
runSetup(values);
});
const { stdout: rawConfig } = spawnSync(
'docker',
@ -36,11 +31,9 @@ test('map docker compose config', () => {
const config = JSON.parse(rawConfig);
const { web } = config.services;
expect(web.image).toStrictEqual(
`mozilla/addons-server:${values.DOCKER_VERSION}`,
);
expect(web.image).toStrictEqual(`mozilla/addons-server:version`);
expect(web.platform).toStrictEqual('linux/amd64');
expect(web.environment.HOST_UID).toStrictEqual(values.HOST_UID);
expect(web.environment.HOST_UID).toStrictEqual('9500');
expect(config.volumes.data_mysqld.name).toStrictEqual(
'addons-server_data_mysqld',
);
@ -48,7 +41,6 @@ test('map docker compose config', () => {
describe('docker-bake.hcl', () => {
function getBakeConfig(env = {}) {
fs.writeFileSync(envPath, '');
runSetup(env);
const { stdout: output } = spawnSync(
'make',
@ -101,130 +93,3 @@ describe('docker-bake.hcl', () => {
expect(output).toContain(`"target": "${target}"`);
});
});
function standardPermutations(name, defaultValue) {
return [
{
name,
file: undefined,
env: undefined,
expected: defaultValue,
},
{
name,
file: 'file',
env: undefined,
expected: 'file',
},
{
name,
file: undefined,
env: 'env',
expected: 'env',
},
{
name,
file: 'file',
env: 'env',
expected: 'env',
},
];
}
describe.each([
{
version: undefined,
digest: undefined,
tag: undefined,
expected: 'mozilla/addons-server:local',
},
{
version: 'version',
digest: undefined,
tag: undefined,
expected: 'mozilla/addons-server:version',
},
{
version: undefined,
digest: 'sha256:digest',
tag: undefined,
expected: 'mozilla/addons-server@sha256:digest',
},
{
version: 'version',
digest: 'sha256:digest',
tag: undefined,
expected: 'mozilla/addons-server@sha256:digest',
},
{
version: 'version',
digest: 'sha256:digest',
tag: 'previous',
expected: 'mozilla/addons-server@sha256:digest',
},
{
version: undefined,
digest: undefined,
tag: 'previous',
expected: 'previous',
},
])('DOCKER_TAG', ({ version, digest, tag, expected }) => {
it(`version:${version}_digest:${digest}_tag:${tag}`, () => {
fs.writeFileSync(envPath, '');
runSetup({
DOCKER_VERSION: version,
DOCKER_DIGEST: digest,
DOCKER_TAG: tag,
});
const actual = readEnvFile('DOCKER_TAG');
expect(actual).toStrictEqual(expected);
});
});
const testCases = [
...standardPermutations('DOCKER_TAG', 'mozilla/addons-server:local'),
...standardPermutations('DOCKER_TARGET', 'development'),
...standardPermutations('HOST_UID', process.getuid().toString()),
...standardPermutations('COMPOSE_FILE', 'docker-compose.yml'),
];
describe.each(testCases)('.env file', ({ name, file, env, expected }) => {
it(`name:${name}_file:${file}_env:${env}`, () => {
fs.writeFileSync(envPath, file ? `${name}=${file}` : '');
runSetup({ [name]: env });
const actual = readEnvFile(name);
expect(actual).toStrictEqual(expected);
});
});
const testedKeys = new Set(testCases.map(({ name }) => name));
// Keys testsed outside the scope of testCases
const skippedKeys = ['DOCKER_COMMIT', 'DOCKER_VERSION', 'DOCKER_BUILD', 'PWD'];
test('All dynamic properties in any docker compose file are referenced in the test', () => {
const composeFiles = globSync('docker-compose*.yml', { cwd: rootPath });
const variableDefinitions = [];
for (let file of composeFiles) {
const fileContent = fs.readFileSync(path.join(rootPath, file), {
encoding: 'utf-8',
});
for (let line of fileContent.split('\n')) {
const regex = /\${(.*?)(?::-.*)?}/g;
let match;
while ((match = regex.exec(line)) !== null) {
const variable = match[1];
if (!skippedKeys.includes(variable)) variableDefinitions.push(variable);
}
}
}
for (let variable of variableDefinitions) {
expect(testedKeys).toContain(variable);
}
});

178
tests/make/test_setup.py Normal file
Просмотреть файл

@ -0,0 +1,178 @@
import os
import unittest
from unittest import mock
from scripts.setup import get_docker_tag, main
def override_env(**kwargs):
return mock.patch.dict(os.environ, kwargs, clear=True)
keys = ['COMPOSE_FILE', 'DOCKER_TAG', 'DOCKER_TARGET', 'HOST_UID', 'DEBUG']
class BaseTestClass(unittest.TestCase):
def assert_set_env_file_called_with(self, **kwargs):
expected = {key: kwargs.get(key, mock.ANY) for key in keys}
assert mock.call(expected) in self.mock_set_env_file.call_args_list
def setUp(self):
patch = mock.patch('scripts.setup.set_env_file')
self.addCleanup(patch.stop)
self.mock_set_env_file = patch.start()
patch_two = mock.patch('scripts.setup.get_env_file', return_value={})
self.addCleanup(patch_two.stop)
self.mock_get_env_file = patch_two.start()
@override_env()
class TestGetDockerTag(BaseTestClass):
def test_default_value_is_local(self):
tag, version, digest = get_docker_tag()
self.assertEqual(tag, 'mozilla/addons-server:local')
self.assertEqual(version, 'local')
self.assertEqual(digest, None)
@override_env(DOCKER_VERSION='test')
def test_version_overrides_default(self):
tag, version, digest = get_docker_tag()
self.assertEqual(tag, 'mozilla/addons-server:test')
self.assertEqual(version, 'test')
self.assertEqual(digest, None)
@override_env(DOCKER_DIGEST='sha256:123')
def test_digest_overrides_version_and_default(self):
tag, version, digest = get_docker_tag()
self.assertEqual(tag, 'mozilla/addons-server@sha256:123')
self.assertEqual(version, None)
self.assertEqual(digest, 'sha256:123')
with override_env(DOCKER_VERSION='test', DOCKER_DIGEST='sha256:123'):
tag, version, digest = get_docker_tag()
self.assertEqual(tag, 'mozilla/addons-server@sha256:123')
self.assertEqual(version, None)
self.assertEqual(digest, 'sha256:123')
@override_env(DOCKER_TAG='image:latest')
def test_tag_overrides_default_version(self):
tag, version, digest = get_docker_tag()
self.assertEqual(tag, 'image:latest')
self.assertEqual(version, 'latest')
self.assertEqual(digest, None)
with override_env(DOCKER_TAG='image:latest', DOCKER_VERSION='test'):
tag, version, digest = get_docker_tag()
self.assertEqual(tag, 'image:test')
self.assertEqual(version, 'test')
self.assertEqual(digest, None)
@override_env(DOCKER_TAG='image@sha256:123')
def test_tag_overrides_default_digest(self):
tag, version, digest = get_docker_tag()
self.assertEqual(tag, 'image@sha256:123')
self.assertEqual(version, None)
self.assertEqual(digest, 'sha256:123')
with mock.patch.dict(os.environ, {'DOCKER_DIGEST': 'test'}):
tag, version, digest = get_docker_tag()
self.assertEqual(tag, 'image@test')
self.assertEqual(version, None)
self.assertEqual(digest, 'test')
def test_version_from_env_file(self):
self.mock_get_env_file.return_value = {'DOCKER_TAG': 'image:latest'}
tag, version, digest = get_docker_tag()
self.assertEqual(tag, 'image:latest')
self.assertEqual(version, 'latest')
self.assertEqual(digest, None)
def test_digest_from_env_file(self):
self.mock_get_env_file.return_value = {'DOCKER_TAG': 'image@sha256:123'}
tag, version, digest = get_docker_tag()
self.assertEqual(tag, 'image@sha256:123')
self.assertEqual(version, None)
self.assertEqual(digest, 'sha256:123')
@override_env(DOCKER_VERSION='')
def test_default_when_version_is_empty(self):
tag, version, digest = get_docker_tag()
self.assertEqual(tag, 'mozilla/addons-server:local')
self.assertEqual(version, 'local')
self.assertEqual(digest, None)
@override_env(DOCKER_DIGEST='')
def test_default_when_digest_is_empty(self):
self.mock_get_env_file.return_value = {'DOCKER_TAG': 'image@sha256:123'}
tag, version, digest = get_docker_tag()
self.assertEqual(tag, 'image@sha256:123')
self.assertEqual(version, None)
self.assertEqual(digest, 'sha256:123')
@override_env()
class TestDockerTarget(BaseTestClass):
def test_default_development_target(self):
main()
self.assert_set_env_file_called_with(DOCKER_TARGET='development')
@override_env(DOCKER_VERSION='test')
def test_default_production_target(self):
main()
self.assert_set_env_file_called_with(DOCKER_TARGET='production')
def test_default_env_file(self):
self.mock_get_env_file.return_value = {
'DOCKER_TAG': 'mozilla/addons-server:test'
}
main()
self.assert_set_env_file_called_with(DOCKER_TARGET='production')
@override_env()
class TestComposeFile(BaseTestClass):
def test_default_compose_file(self):
main()
self.assert_set_env_file_called_with(COMPOSE_FILE='docker-compose.yml')
@override_env(DOCKER_TARGET='production')
def test_default_target_production(self):
main()
self.assert_set_env_file_called_with(
COMPOSE_FILE='docker-compose.yml:docker-compose.ci.yml'
)
@override_env(COMPOSE_FILE='test')
def test_compose_file_override(self):
main()
self.assert_set_env_file_called_with(COMPOSE_FILE='test')
@override_env()
class TestDebug(BaseTestClass):
def test_default_debug(self):
main()
self.assert_set_env_file_called_with(DEBUG='True')
@override_env(DOCKER_TARGET='production')
def test_production_debug(self):
main()
self.assert_set_env_file_called_with(DEBUG='False')
@override_env(DOCKER_TARGET='production')
def test_override_env_debug_false_on_target_production(self):
self.mock_get_env_file.return_value = {'DEBUG': 'True'}
main()
self.assert_set_env_file_called_with(DEBUG='False')
@override_env(DOCKER_TARGET='development')
def test_override_env_debug_true_on_target_development(self):
self.mock_get_env_file.return_value = {'DEBUG': 'False'}
main()
self.assert_set_env_file_called_with(DEBUG='True')
@override_env(DEBUG='test')
def test_debug_override(self):
main()
self.assert_set_env_file_called_with(DEBUG='test')