Unify storage path related settings. (#9655)

* Unify storage path related settings.

* Moves all storage related settings to settings_base
* Keeps ability to overload base path by setting `NETAPP_STORAGE_ROOT`
* Removes unused PACKAGER_PATH

Fixes #9654

* Sort settings, rename according to their actual path settings
* explicitly set all paths in conftest and document it's usage
* Remove reviewer_attachments folder reference

conftest.py

@@ -4,6 +4,7 @@ pytest hooks and fixtures used for our unittests.
 Please note that there should not be any Django/Olympia related imports
 on module-level, they should instead be added to hooks or fixtures directly.
 """
+import os
 
 import responses
 import pytest
@@ -118,11 +119,21 @@ def test_pre_setup(request, tmpdir, settings):
     translation.trans_real._translations = {}
     translation.trans_real.activate(settings.LANGUAGE_CODE)
 
-    settings.MEDIA_ROOT = str(tmpdir.mkdir('media'))
-    settings.TMP_PATH = str(tmpdir.mkdir('tmp'))
-    settings.STATIC_ROOT = str(tmpdir.mkdir('site-static'))
-    settings.NETAPP_STORAGE = settings.TMP_PATH
-    settings.GIT_FILE_STORAGE_PATH = str(tmpdir.mkdir('git-storage'))
+    def _path(*args):
+        path = str(os.path.join(*args))
+        if not os.path.exists(path):
+            os.makedirs(path)
+        return path
+
+    settings.STORAGE_ROOT = storage_root = _path(tmpdir.mkdir('storage'))
+    settings.SHARED_STORAGE = shared_storage = _path(
+        storage_root, 'shared_storage')
+
+    settings.ADDONS_PATH = _path(storage_root, 'files')
+    settings.GUARDED_ADDONS_PATH = _path(storage_root, 'guarded-addons')
+    settings.GIT_FILE_STORAGE_PATH = _path(storage_root, 'git-storage')
+    settings.MEDIA_ROOT = _path(shared_storage, 'uploads')
+    settings.TMP_PATH = _path(shared_storage, 'tmp')
 
     # Reset the prefixer and urlconf after updating media root
     default_prefixer(settings)

src/olympia/amo/monitors.py

@@ -102,7 +102,6 @@ def path():
           user_media_path('collection_icons'),
           user_media_path('previews'),
           user_media_path('userpics'),
-          user_media_path('reviewer_attachments'),
           dump_apps.Command.get_json_path(),)
     r = [os.path.join(settings.ROOT, 'locale'),
          # The deploy process will want write access to this.

src/olympia/amo/tests/test_settings.py

@@ -10,7 +10,7 @@ from olympia.lib.settings_base import get_raven_release
 
 
 @pytest.mark.parametrize('key', (
-    'NETAPP_STORAGE', 'GUARDED_ADDONS_PATH', 'TMP_PATH', 'MEDIA_ROOT'))
+    'SHARED_STORAGE', 'GUARDED_ADDONS_PATH', 'TMP_PATH', 'MEDIA_ROOT'))
 def test_base_paths_bytestring(key):
     """Make sure all relevant base paths are bytestrings.
 

src/olympia/conf/dev/settings.py

@@ -1,5 +1,4 @@
 import logging
-import os
 
 from olympia.lib.settings_base import *  # noqa
 
@@ -45,17 +44,6 @@ SESSION_COOKIE_DOMAIN = ".%s" % DOMAIN
 INBOUND_EMAIL_DOMAIN = env('INBOUND_EMAIL_DOMAIN',
                            default='addons-dev.allizom.org')
 
-NETAPP_STORAGE_ROOT = env('NETAPP_STORAGE_ROOT')
-NETAPP_STORAGE = os.path.join(NETAPP_STORAGE_ROOT, 'shared_storage')
-GUARDED_ADDONS_PATH = os.path.join(NETAPP_STORAGE_ROOT, 'guarded-addons')
-MEDIA_ROOT = os.path.join(NETAPP_STORAGE, 'uploads')
-TMP_PATH = os.path.join(NETAPP_STORAGE, 'tmp')
-PACKAGER_PATH = os.path.join(TMP_PATH, 'packager')
-
-ADDONS_PATH = os.path.join(NETAPP_STORAGE_ROOT, 'files')
-
-REVIEWER_ATTACHMENTS_PATH = os.path.join(MEDIA_ROOT, 'reviewer_attachment')
-
 DATABASES = {
     'default': get_db_config('DATABASES_DEFAULT_URL'),
     'slave': get_db_config('DATABASES_SLAVE_URL', atomic_requests=False),

src/olympia/conf/prod/settings.py

@@ -1,5 +1,4 @@
 import logging
-import os
 
 from olympia.lib.settings_base import *  # noqa
 
@@ -34,17 +33,6 @@ SESSION_COOKIE_DOMAIN = ".%s" % DOMAIN
 INBOUND_EMAIL_DOMAIN = env('INBOUND_EMAIL_DOMAIN',
                            default='addons.mozilla.org')
 
-NETAPP_STORAGE_ROOT = env('NETAPP_STORAGE_ROOT')
-NETAPP_STORAGE = os.path.join(NETAPP_STORAGE_ROOT, 'shared_storage')
-GUARDED_ADDONS_PATH = os.path.join(NETAPP_STORAGE_ROOT, 'guarded-addons')
-MEDIA_ROOT = os.path.join(NETAPP_STORAGE, 'uploads')
-TMP_PATH = os.path.join(NETAPP_STORAGE, 'tmp')
-PACKAGER_PATH = os.path.join(TMP_PATH, 'packager')
-
-ADDONS_PATH = os.path.join(NETAPP_STORAGE_ROOT, 'files')
-
-REVIEWER_ATTACHMENTS_PATH = os.path.join(MEDIA_ROOT, 'reviewer_attachment')
-
 DATABASES = {
     'default': get_db_config('DATABASES_DEFAULT_URL'),
     'slave': get_db_config('DATABASES_SLAVE_URL', atomic_requests=False),

src/olympia/conf/stage/settings.py

@@ -1,5 +1,4 @@
 import logging
-import os
 
 from olympia.lib.settings_base import *  # noqa
 
@@ -43,17 +42,6 @@ SESSION_COOKIE_DOMAIN = ".%s" % DOMAIN
 INBOUND_EMAIL_DOMAIN = env('INBOUND_EMAIL_DOMAIN',
                            default='addons.allizom.org')
 
-NETAPP_STORAGE_ROOT = env('NETAPP_STORAGE_ROOT')
-NETAPP_STORAGE = os.path.join(NETAPP_STORAGE_ROOT, 'shared_storage')
-GUARDED_ADDONS_PATH = os.path.join(NETAPP_STORAGE_ROOT, 'guarded-addons')
-MEDIA_ROOT = os.path.join(NETAPP_STORAGE, 'uploads')
-TMP_PATH = os.path.join(NETAPP_STORAGE, 'tmp')
-PACKAGER_PATH = os.path.join(TMP_PATH, 'packager')
-
-ADDONS_PATH = os.path.join(NETAPP_STORAGE_ROOT, 'files')
-
-REVIEWER_ATTACHMENTS_PATH = os.path.join(MEDIA_ROOT, 'reviewer_attachment')
-
 DATABASES = {
     'default': get_db_config('DATABASES_DEFAULT_URL'),
     'slave': get_db_config('DATABASES_SLAVE_URL', atomic_requests=False),

src/olympia/files/tests/test_models.py

@@ -336,7 +336,7 @@ class TestFile(TestCase, amo.tests.AMOPaths):
             u'iamstring', u'iamnutherstring', u'laststring!']
 
     def test_current_file_path(self):
-        public_fp = '/media/addons/3615/delicious_bookmarks-2.1.072-fx.xpi'
+        public_fp = '/storage/files/3615/delicious_bookmarks-2.1.072-fx.xpi'
         guarded_fp = '/guarded-addons/3615/delicious_bookmarks-2.1.072-fx.xpi'
 
         # Add-on enabled, file approved

src/olympia/lib/settings_base.py

@@ -306,18 +306,11 @@ INBOUND_EMAIL_VALIDATION_KEY = env('INBOUND_EMAIL_VALIDATION_KEY', default='')
 # Domain emails should be sent to.
 INBOUND_EMAIL_DOMAIN = env('INBOUND_EMAIL_DOMAIN', default=DOMAIN)
 
-# Absolute path to the directory that holds media.
-# Example: "/home/media/media.lawrence.com/"
-MEDIA_ROOT = path('user-media')
-
 # URL that handles the media served from MEDIA_ROOT. Make sure to use a
 # trailing slash if there is a path component (optional in other cases).
 # Examples: "http://media.lawrence.com", "http://example.com/media/"
 MEDIA_URL = '/user-media/'
 
-# Absolute path to a temporary storage area
-TMP_PATH = path('tmp')
-
 # Tarballs in DUMPED_APPS_PATH deleted 30 days after they have been written.
 DUMPED_APPS_DAYS_DELETE = 3600 * 24 * 30
 
@@ -1745,10 +1738,20 @@ STATICFILES_DIRS = (
     path('static'),
 )
 
-NETAPP_STORAGE = TMP_PATH
-GUARDED_ADDONS_PATH = os.path.join(ROOT, 'guarded-addons')
+# Path related settings. In dev/stage/prod `NETAPP_STORAGE_ROOT` environment
+# variable will be set and point to our NFS/EFS storage
+# Make sure to check overwrites in conftest.py if new settings are added
+# or changed.
+STORAGE_ROOT = env('NETAPP_STORAGE_ROOT', default=path('storage'))
 
-GIT_FILE_STORAGE_PATH = os.path.join(MEDIA_ROOT, 'git-storage')
+ADDONS_PATH = os.path.join(STORAGE_ROOT, 'files')
+GUARDED_ADDONS_PATH = os.path.join(STORAGE_ROOT, 'guarded-addons')
+GIT_FILE_STORAGE_PATH = os.path.join(STORAGE_ROOT, 'git-storage')
+
+SHARED_STORAGE = os.path.join(STORAGE_ROOT, 'shared_storage')
+
+MEDIA_ROOT = os.path.join(SHARED_STORAGE, 'uploads')
+TMP_PATH = os.path.join(SHARED_STORAGE, 'tmp')
 
 # These are key files that must be present on disk to encrypt/decrypt certain
 # database fields.

src/olympia/stats/management/commands/download_counts_from_file.py

@@ -41,7 +41,7 @@ class Command(BaseCommand):
 
     If stats_source is file:
         If not folder is specified, the default is `hive_results/YYYY-MM-DD/`.
-        This folder will be located in `<settings.NETAPP_STORAGE>/tmp`.
+        This folder will be located in `<settings.SHARED_STORAGE>/tmp`.
 
     If stats_source is s3:
         This file will be located in

src/olympia/stats/management/commands/theme_update_counts_from_file.py

@@ -29,7 +29,7 @@ class Command(BaseCommand):
 
     If stats_source is file:
         If not folder is specified, the default is `hive_results/YYYY-MM-DD/`.
-        This folder will be located in `<settings.NETAPP_STORAGE>/tmp`.
+        This folder will be located in `<settings.SHARED_STORAGE>/tmp`.
 
         File processed:
         - theme_update_counts.hive

src/olympia/stats/management/commands/update_counts_from_file.py

@@ -48,7 +48,7 @@ class Command(BaseCommand):
     If stats_source is file:
         If not folder is specified, the default is
             `hive_results/<YYYY-MM-DD>/`.
-        This folder will be located in `<settings.NETAPP_STORAGE>/tmp`.
+        This folder will be located in `<settings.SHARED_STORAGE>/tmp`.
 
         Five files are processed:
         - update_counts_by_version.hive