Feature/TP1-569: Add Invoke command to copy Staging DB to a specific Review App (#12493)

* Feature: Add Invoke command to copy Staging DB to a specific Review App

* Fix: Change Domain and Hostnames for Review Apps in cleanup.sql

* Refactor: Avoid duplicate code for PLATFORM_ARG

* Feature: Add the staging-db-to-review-app invoke command to the docs

* Fix: Linting and formatting

cleanup.sql

@@ -0,0 +1,59 @@
+-- noinspection SqlNoDataSourceInspectionForFile
+
+CREATE EXTENSION IF NOT EXISTS pgcrypto;
+
+CREATE OR REPLACE FUNCTION clean_user_data()
+RETURNS VOID AS $$
+DECLARE
+    user_row RECORD;
+    new_email varchar;
+    new_hash varchar;
+    new_username varchar;
+    counter integer := 1;
+BEGIN
+--     scrub the user table
+    TRUNCATE django_session;
+
+--     clean up non-staff social auth data
+    DELETE FROM social_auth_usersocialauth
+    WHERE uid NOT LIKE '%@mozillafoundation.org';
+
+--     Update the site domain
+    UPDATE django_site
+    SET domain = '{DOMAIN}.mofostaging.net'
+    WHERE domain = 'foundation.mofostaging.net';
+
+    UPDATE wagtailcore_site
+    SET hostname = '{HOSTNAME}.mofostaging.net'
+    WHERE hostname = 'foundation.mofostaging.net';
+
+    UPDATE wagtailcore_site
+    SET hostname = 'mozfest-{HOSTNAME}.mofostaging.net'
+    WHERE hostname = 'mozillafestival.mofostaging.net';
+
+--     Iterate over each non-staff user and remove any PII
+    FOR user_row IN
+        SELECT id
+        FROM auth_user
+        WHERE email NOT LIKE '%@mozillafoundation.org'
+    LOOP
+        new_email := concat(encode(gen_random_bytes(12), 'base64'), '@example.com');
+        new_hash := crypt(encode(gen_random_bytes(32), 'base64'), gen_salt('bf', 6));
+        new_username := concat('anonymouse', counter::varchar);
+
+        UPDATE auth_user
+        SET
+          email = new_email,
+          password = new_hash,
+          username = new_username,
+          first_name = 'anony',
+          last_name = 'mouse'
+        Where id = user_row.id;
+
+--         Increase the counter
+        counter := counter + 1;
+    END LOOP;
+END;
+$$ LANGUAGE plpgsql;
+
+SELECT clean_user_data();

copy_staging_db_to_review_app.py

@@ -0,0 +1,103 @@
+import tempfile
+from time import sleep
+
+from tasks import PLATFORM_ARG
+
+STAGING_APP = "foundation-mofostaging-net"
+
+
+def execute_command(ctx, command: str, custom_error: str = ""):
+    try:
+        result = ctx.run(command, hide=False, warn=True, **PLATFORM_ARG)
+        if result.failed:
+            raise Exception(f"{custom_error}: {result.stderr}")
+        return result.stdout.strip()
+    except Exception as e:
+        raise Exception(f"{custom_error}: {e}") from e
+
+
+def log_step(message: str):
+    print(f"--> {message}\n", flush=True)
+
+
+def log_step_completed(message: str):
+    print(f"✔️  {message} completed.\n", flush=True)
+
+
+def replace_placeholders_in_sql(review_app_name: str, input_file: str) -> str:
+    with open(input_file, "r") as file:
+        sql_content = file.read()
+
+    sql_content = sql_content.replace("{DOMAIN}", review_app_name)
+    sql_content = sql_content.replace("{HOSTNAME}", review_app_name)
+
+    return sql_content
+
+
+def main(ctx, review_app_name):
+    log_step(f"The review app name is: {review_app_name}, if not, please cancel now...")
+    sleep(5)
+
+    log_step("Verifying if logged in Heroku")
+    heroku_user = execute_command(ctx, "heroku whoami", "Verify that you are logged in Heroku CLI")
+    print(f"Heroku user: {heroku_user}\n", flush=True)
+    log_step_completed("Heroku login verification")
+
+    log_step("Verifying if psql is installed")
+    execute_command(ctx, "psql --version", "Verify that you have 'psql' installed")
+    log_step_completed("psql installation verification")
+
+    try:
+        log_step("Enabling maintenance mode on the Review App")
+        execute_command(ctx, f"heroku maintenance:on -a {review_app_name}")
+        log_step_completed("Maintenance mode enabling")
+
+        log_step("Scaling web dynos on Review App to 0")
+        execute_command(ctx, f"heroku ps:scale -a {review_app_name} web=0")
+        log_step_completed("Web dynos scaling to 0")
+
+        log_step("Backing up Staging DB")
+        execute_command(ctx, f"heroku pg:backups:capture -a {STAGING_APP}")
+        log_step_completed("Staging DB backup")
+
+        log_step("Backing up Review App DB")
+        execute_command(ctx, f"heroku pg:backups:capture -a {review_app_name}")
+        log_step_completed("Review App DB backup")
+
+        log_step("Restoring the latest Staging backup to Review App")
+        backup_staging_url = execute_command(ctx, f"heroku pg:backups:url -a {STAGING_APP}")
+        execute_command(
+            ctx, f"heroku pg:backups:restore --confirm {review_app_name} -a {review_app_name} '{backup_staging_url}'"
+        )
+        log_step_completed("Staging backup restoration to Review App")
+
+        log_step("Executing cleanup SQL script")
+        review_app_db_url = execute_command(ctx, f"heroku config:get -a {review_app_name} DATABASE_URL")
+
+        # Replace placeholders and write to a temporary file
+        sql_content = replace_placeholders_in_sql(review_app_name, "./cleanup.sql")
+        with tempfile.NamedTemporaryFile(suffix=".sql", mode="w", delete=True) as temp_sql_file:
+            temp_sql_file.write(sql_content)
+            temp_sql_file.flush()
+            execute_command(ctx, f"psql {review_app_db_url} -f {temp_sql_file.name}")
+
+        log_step_completed("Cleanup SQL script execution")
+
+        log_step("Running migrations")
+        execute_command(ctx, f"heroku run -a {review_app_name} -- python network-api/manage.py migrate --no-input")
+        log_step_completed("Migrations running")
+
+    except Exception as e:
+        log_step("Rolling back Review App")
+        execute_command(ctx, f"heroku pg:backups:restore -a {review_app_name} --confirm {review_app_name}")
+        print(e, flush=True)
+        log_step_completed("Review App rollback")
+
+    finally:
+        log_step("Scaling web dynos on Review App to 1")
+        execute_command(ctx, f"heroku ps:scale -a {review_app_name} web=1")
+        log_step_completed("Web dynos scaling to 1")
+
+        log_step("Disabling maintenance mode on Review App")
+        execute_command(ctx, f"heroku maintenance:off -a {review_app_name}")
+        log_step_completed("Maintenance mode disabling")

docs/local_development.md

@@ -25,26 +25,27 @@ The general workflow is:
 To get a list of invoke commands available, run `invoke -l`:
 
 ```
-  catch-up (catchup, docker-catchup)              Rebuild images, install dependencies, and apply migrations
+  catch-up (catchup, docker-catchup)                Rebuild images, install dependencies, and apply migrations
-  compilemessages (docker-compilemessages)        Compile the latest translations
+  compilemessages (docker-compilemessages)          Compile the latest translations
-  makemessages (docker-makemessages)              Extract all template messages in .po files for localization
+  makemessages (docker-makemessages)                Extract all template messages in .po files for localization
-  makemigrations (docker-makemigrations)          Creates new migration(s) for apps
+  makemigrations (docker-makemigrations)            Creates new migration(s) for apps
-  manage (docker-manage)                          Shorthand to manage.py. inv docker-manage "[COMMAND] [ARG]"
+  manage (docker-manage)                            Shorthand to manage.py. inv docker-manage "[COMMAND] [ARG]"
-  migrate (docker-migrate)                        Updates database schema
+  migrate (docker-migrate)                          Updates database schema
-  new-db (docker-new-db)                          Delete your database and create a new one with fake data
+  new-db (docker-new-db)                            Delete your database and create a new one with fake data
-  copy-stage-db                                   Overwrite your local docker postgres DB with a copy of the staging database
+  copy-stage-db                                     Overwrite your local docker postgres DB with a copy of the staging database
-  copy-prod-db                                    Overwrite your local docker postgres DB with a copy of the production database
+  copy-prod-db                                      Overwrite your local docker postgres DB with a copy of the production database
-  new-env (docker-new-env)                        Get a new dev environment and a new database with fake data
+  new-env (docker-new-env)                          Get a new dev environment and a new database with fake data
-  npm (docker-npm)                                Shorthand to npm. inv docker-npm "[COMMAND] [ARG]"
+  npm (docker-npm)                                  Shorthand to npm. inv docker-npm "[COMMAND] [ARG]"
-  npm-install (docker-npm-install)                Install Node dependencies
+  npm-install (docker-npm-install)                  Install Node dependencies
-  pip-compile (docker-pip-compile)                Shorthand to pip-tools. inv pip-compile "[filename]" "[COMMAND] [ARG]"
+  pip-compile (docker-pip-compile)                  Shorthand to pip-tools. inv pip-compile "[filename]" "[COMMAND] [ARG]"
-  pip-compile-lock (docker-pip-compile-lock)      Lock prod and dev dependencies
+  pip-compile-lock (docker-pip-compile-lock)        Lock prod and dev dependencies
-  pip-sync (docker-pip-sync)                      Sync your python virtualenv
+  pip-sync (docker-pip-sync)                        Sync your python virtualenv
-  start-dev (docker-start, start)                 Start the dev server
+  staging-db-to-review-app (staging-to-review-app)  Copy Staging DB to a specific Review App. inv staging-to-review-app "[REVIEW_APP_NAME]"
-  start-lean-dev (docker-start-lean, start-lean)  Start the dev server without rebuilding
+  start-dev (docker-start, start)                   Start the dev server
-  test (docker-test)                              Run both Node and Python tests
+  start-lean-dev (docker-start-lean, start-lean)    Start the dev server without rebuilding
-  test-node (docker-test-node)                    Run node tests
+  test (docker-test)                                Run both Node and Python tests
-  test-python (docker-test-python)                Run python tests
+  test-node (docker-test-node)                      Run node tests
+  test-python (docker-test-python)                  Run python tests
 ```
 
 Note the above commands carefully, as they should cover the majority of what you'd need for local development.

tasks.py

@@ -585,3 +585,13 @@ def compilemessages(ctx):
             "../dockerpythonvenv/bin/python manage.py compilemessages",
             **PLATFORM_ARG,
         )
+
+
+@task(aliases=["staging-to-review-app"])
+def staging_db_to_review_app(ctx, review_app_name):
+    """
+    Copy Staging DB to a specific Review App. inv staging-to-review-app \"[REVIEW_APP_NAME]\"
+    """
+    from copy_staging_db_to_review_app import main
+
+    main(ctx, review_app_name)