From fa73e5a9536cf8725a1122e312c0687d7d85b8af Mon Sep 17 00:00:00 2001 From: Christopher Grebs Date: Mon, 15 Oct 2018 17:30:02 +0200 Subject: [PATCH] Unify storage path related settings. (#9655) * Unify storage path related settings. * Moves all storage related settings to settings_base * Keeps ability to overload base path by setting `NETAPP_STORAGE_ROOT` * Removes unused PACKAGER_PATH Fixes #9654 * Sort settings, rename according to their actual path settings * explicitly set all paths in conftest and document it's usage * Remove reviewer_attachments folder reference --- conftest.py | 21 +++++++++++++---- src/olympia/amo/monitors.py | 1 - src/olympia/amo/tests/test_settings.py | 2 +- src/olympia/conf/dev/settings.py | 12 ---------- src/olympia/conf/prod/settings.py | 12 ---------- src/olympia/conf/stage/settings.py | 12 ---------- src/olympia/files/tests/test_models.py | 2 +- src/olympia/lib/settings_base.py | 23 +++++++++++-------- .../commands/download_counts_from_file.py | 2 +- .../commands/theme_update_counts_from_file.py | 2 +- .../commands/update_counts_from_file.py | 2 +- 11 files changed, 34 insertions(+), 57 deletions(-) diff --git a/conftest.py b/conftest.py index 998c3bcfe7..67f56abd94 100644 --- a/conftest.py +++ b/conftest.py @@ -4,6 +4,7 @@ pytest hooks and fixtures used for our unittests. Please note that there should not be any Django/Olympia related imports on module-level, they should instead be added to hooks or fixtures directly. """ +import os import responses import pytest @@ -118,11 +119,21 @@ def test_pre_setup(request, tmpdir, settings): translation.trans_real._translations = {} translation.trans_real.activate(settings.LANGUAGE_CODE) - settings.MEDIA_ROOT = str(tmpdir.mkdir('media')) - settings.TMP_PATH = str(tmpdir.mkdir('tmp')) - settings.STATIC_ROOT = str(tmpdir.mkdir('site-static')) - settings.NETAPP_STORAGE = settings.TMP_PATH - settings.GIT_FILE_STORAGE_PATH = str(tmpdir.mkdir('git-storage')) + def _path(*args): + path = str(os.path.join(*args)) + if not os.path.exists(path): + os.makedirs(path) + return path + + settings.STORAGE_ROOT = storage_root = _path(tmpdir.mkdir('storage')) + settings.SHARED_STORAGE = shared_storage = _path( + storage_root, 'shared_storage') + + settings.ADDONS_PATH = _path(storage_root, 'files') + settings.GUARDED_ADDONS_PATH = _path(storage_root, 'guarded-addons') + settings.GIT_FILE_STORAGE_PATH = _path(storage_root, 'git-storage') + settings.MEDIA_ROOT = _path(shared_storage, 'uploads') + settings.TMP_PATH = _path(shared_storage, 'tmp') # Reset the prefixer and urlconf after updating media root default_prefixer(settings) diff --git a/src/olympia/amo/monitors.py b/src/olympia/amo/monitors.py index 436e34c69e..6b7c81ec49 100644 --- a/src/olympia/amo/monitors.py +++ b/src/olympia/amo/monitors.py @@ -102,7 +102,6 @@ def path(): user_media_path('collection_icons'), user_media_path('previews'), user_media_path('userpics'), - user_media_path('reviewer_attachments'), dump_apps.Command.get_json_path(),) r = [os.path.join(settings.ROOT, 'locale'), # The deploy process will want write access to this. diff --git a/src/olympia/amo/tests/test_settings.py b/src/olympia/amo/tests/test_settings.py index e92fc6d9a9..b46b7cc472 100644 --- a/src/olympia/amo/tests/test_settings.py +++ b/src/olympia/amo/tests/test_settings.py @@ -10,7 +10,7 @@ from olympia.lib.settings_base import get_raven_release @pytest.mark.parametrize('key', ( - 'NETAPP_STORAGE', 'GUARDED_ADDONS_PATH', 'TMP_PATH', 'MEDIA_ROOT')) + 'SHARED_STORAGE', 'GUARDED_ADDONS_PATH', 'TMP_PATH', 'MEDIA_ROOT')) def test_base_paths_bytestring(key): """Make sure all relevant base paths are bytestrings. diff --git a/src/olympia/conf/dev/settings.py b/src/olympia/conf/dev/settings.py index 1811e079d8..a1f985cf41 100644 --- a/src/olympia/conf/dev/settings.py +++ b/src/olympia/conf/dev/settings.py @@ -1,5 +1,4 @@ import logging -import os from olympia.lib.settings_base import * # noqa @@ -45,17 +44,6 @@ SESSION_COOKIE_DOMAIN = ".%s" % DOMAIN INBOUND_EMAIL_DOMAIN = env('INBOUND_EMAIL_DOMAIN', default='addons-dev.allizom.org') -NETAPP_STORAGE_ROOT = env('NETAPP_STORAGE_ROOT') -NETAPP_STORAGE = os.path.join(NETAPP_STORAGE_ROOT, 'shared_storage') -GUARDED_ADDONS_PATH = os.path.join(NETAPP_STORAGE_ROOT, 'guarded-addons') -MEDIA_ROOT = os.path.join(NETAPP_STORAGE, 'uploads') -TMP_PATH = os.path.join(NETAPP_STORAGE, 'tmp') -PACKAGER_PATH = os.path.join(TMP_PATH, 'packager') - -ADDONS_PATH = os.path.join(NETAPP_STORAGE_ROOT, 'files') - -REVIEWER_ATTACHMENTS_PATH = os.path.join(MEDIA_ROOT, 'reviewer_attachment') - DATABASES = { 'default': get_db_config('DATABASES_DEFAULT_URL'), 'slave': get_db_config('DATABASES_SLAVE_URL', atomic_requests=False), diff --git a/src/olympia/conf/prod/settings.py b/src/olympia/conf/prod/settings.py index d5af18b7f1..2ceded79fd 100644 --- a/src/olympia/conf/prod/settings.py +++ b/src/olympia/conf/prod/settings.py @@ -1,5 +1,4 @@ import logging -import os from olympia.lib.settings_base import * # noqa @@ -34,17 +33,6 @@ SESSION_COOKIE_DOMAIN = ".%s" % DOMAIN INBOUND_EMAIL_DOMAIN = env('INBOUND_EMAIL_DOMAIN', default='addons.mozilla.org') -NETAPP_STORAGE_ROOT = env('NETAPP_STORAGE_ROOT') -NETAPP_STORAGE = os.path.join(NETAPP_STORAGE_ROOT, 'shared_storage') -GUARDED_ADDONS_PATH = os.path.join(NETAPP_STORAGE_ROOT, 'guarded-addons') -MEDIA_ROOT = os.path.join(NETAPP_STORAGE, 'uploads') -TMP_PATH = os.path.join(NETAPP_STORAGE, 'tmp') -PACKAGER_PATH = os.path.join(TMP_PATH, 'packager') - -ADDONS_PATH = os.path.join(NETAPP_STORAGE_ROOT, 'files') - -REVIEWER_ATTACHMENTS_PATH = os.path.join(MEDIA_ROOT, 'reviewer_attachment') - DATABASES = { 'default': get_db_config('DATABASES_DEFAULT_URL'), 'slave': get_db_config('DATABASES_SLAVE_URL', atomic_requests=False), diff --git a/src/olympia/conf/stage/settings.py b/src/olympia/conf/stage/settings.py index 17e674ae9d..5270885ef3 100644 --- a/src/olympia/conf/stage/settings.py +++ b/src/olympia/conf/stage/settings.py @@ -1,5 +1,4 @@ import logging -import os from olympia.lib.settings_base import * # noqa @@ -43,17 +42,6 @@ SESSION_COOKIE_DOMAIN = ".%s" % DOMAIN INBOUND_EMAIL_DOMAIN = env('INBOUND_EMAIL_DOMAIN', default='addons.allizom.org') -NETAPP_STORAGE_ROOT = env('NETAPP_STORAGE_ROOT') -NETAPP_STORAGE = os.path.join(NETAPP_STORAGE_ROOT, 'shared_storage') -GUARDED_ADDONS_PATH = os.path.join(NETAPP_STORAGE_ROOT, 'guarded-addons') -MEDIA_ROOT = os.path.join(NETAPP_STORAGE, 'uploads') -TMP_PATH = os.path.join(NETAPP_STORAGE, 'tmp') -PACKAGER_PATH = os.path.join(TMP_PATH, 'packager') - -ADDONS_PATH = os.path.join(NETAPP_STORAGE_ROOT, 'files') - -REVIEWER_ATTACHMENTS_PATH = os.path.join(MEDIA_ROOT, 'reviewer_attachment') - DATABASES = { 'default': get_db_config('DATABASES_DEFAULT_URL'), 'slave': get_db_config('DATABASES_SLAVE_URL', atomic_requests=False), diff --git a/src/olympia/files/tests/test_models.py b/src/olympia/files/tests/test_models.py index 8243b29f62..36a351addf 100644 --- a/src/olympia/files/tests/test_models.py +++ b/src/olympia/files/tests/test_models.py @@ -336,7 +336,7 @@ class TestFile(TestCase, amo.tests.AMOPaths): u'iamstring', u'iamnutherstring', u'laststring!'] def test_current_file_path(self): - public_fp = '/media/addons/3615/delicious_bookmarks-2.1.072-fx.xpi' + public_fp = '/storage/files/3615/delicious_bookmarks-2.1.072-fx.xpi' guarded_fp = '/guarded-addons/3615/delicious_bookmarks-2.1.072-fx.xpi' # Add-on enabled, file approved diff --git a/src/olympia/lib/settings_base.py b/src/olympia/lib/settings_base.py index 93108a1473..2a5078a6b8 100644 --- a/src/olympia/lib/settings_base.py +++ b/src/olympia/lib/settings_base.py @@ -306,18 +306,11 @@ INBOUND_EMAIL_VALIDATION_KEY = env('INBOUND_EMAIL_VALIDATION_KEY', default='') # Domain emails should be sent to. INBOUND_EMAIL_DOMAIN = env('INBOUND_EMAIL_DOMAIN', default=DOMAIN) -# Absolute path to the directory that holds media. -# Example: "/home/media/media.lawrence.com/" -MEDIA_ROOT = path('user-media') - # URL that handles the media served from MEDIA_ROOT. Make sure to use a # trailing slash if there is a path component (optional in other cases). # Examples: "http://media.lawrence.com", "http://example.com/media/" MEDIA_URL = '/user-media/' -# Absolute path to a temporary storage area -TMP_PATH = path('tmp') - # Tarballs in DUMPED_APPS_PATH deleted 30 days after they have been written. DUMPED_APPS_DAYS_DELETE = 3600 * 24 * 30 @@ -1745,10 +1738,20 @@ STATICFILES_DIRS = ( path('static'), ) -NETAPP_STORAGE = TMP_PATH -GUARDED_ADDONS_PATH = os.path.join(ROOT, 'guarded-addons') +# Path related settings. In dev/stage/prod `NETAPP_STORAGE_ROOT` environment +# variable will be set and point to our NFS/EFS storage +# Make sure to check overwrites in conftest.py if new settings are added +# or changed. +STORAGE_ROOT = env('NETAPP_STORAGE_ROOT', default=path('storage')) -GIT_FILE_STORAGE_PATH = os.path.join(MEDIA_ROOT, 'git-storage') +ADDONS_PATH = os.path.join(STORAGE_ROOT, 'files') +GUARDED_ADDONS_PATH = os.path.join(STORAGE_ROOT, 'guarded-addons') +GIT_FILE_STORAGE_PATH = os.path.join(STORAGE_ROOT, 'git-storage') + +SHARED_STORAGE = os.path.join(STORAGE_ROOT, 'shared_storage') + +MEDIA_ROOT = os.path.join(SHARED_STORAGE, 'uploads') +TMP_PATH = os.path.join(SHARED_STORAGE, 'tmp') # These are key files that must be present on disk to encrypt/decrypt certain # database fields. diff --git a/src/olympia/stats/management/commands/download_counts_from_file.py b/src/olympia/stats/management/commands/download_counts_from_file.py index 4320d8c7e2..842b468510 100644 --- a/src/olympia/stats/management/commands/download_counts_from_file.py +++ b/src/olympia/stats/management/commands/download_counts_from_file.py @@ -41,7 +41,7 @@ class Command(BaseCommand): If stats_source is file: If not folder is specified, the default is `hive_results/YYYY-MM-DD/`. - This folder will be located in `/tmp`. + This folder will be located in `/tmp`. If stats_source is s3: This file will be located in diff --git a/src/olympia/stats/management/commands/theme_update_counts_from_file.py b/src/olympia/stats/management/commands/theme_update_counts_from_file.py index 2dd03bebbf..ce7619491b 100644 --- a/src/olympia/stats/management/commands/theme_update_counts_from_file.py +++ b/src/olympia/stats/management/commands/theme_update_counts_from_file.py @@ -29,7 +29,7 @@ class Command(BaseCommand): If stats_source is file: If not folder is specified, the default is `hive_results/YYYY-MM-DD/`. - This folder will be located in `/tmp`. + This folder will be located in `/tmp`. File processed: - theme_update_counts.hive diff --git a/src/olympia/stats/management/commands/update_counts_from_file.py b/src/olympia/stats/management/commands/update_counts_from_file.py index e10eaf45de..51dfc01cf3 100644 --- a/src/olympia/stats/management/commands/update_counts_from_file.py +++ b/src/olympia/stats/management/commands/update_counts_from_file.py @@ -48,7 +48,7 @@ class Command(BaseCommand): If stats_source is file: If not folder is specified, the default is `hive_results//`. - This folder will be located in `/tmp`. + This folder will be located in `/tmp`. Five files are processed: - update_counts_by_version.hive