Feature/TP1-569: Add Invoke command to copy Staging DB to a specific Review App (#12493)
* Feature: Add Invoke command to copy Staging DB to a specific Review App * Fix: Change Domain and Hostnames for Review Apps in cleanup.sql * Refactor: Avoid duplicate code for PLATFORM_ARG * Feature: Add the staging-db-to-review-app invoke command to the docs * Fix: Linting and formatting
This commit is contained in:
Adalberto Vazquez
GitHub
Родитель
d066a7cfa4
Коммит
da2d9129d3
|
|
@ -0,0 +1,59 @@
|
|||
-- noinspection SqlNoDataSourceInspectionForFile
|
||||
|
||||
CREATE EXTENSION IF NOT EXISTS pgcrypto;
|
||||
|
||||
CREATE OR REPLACE FUNCTION clean_user_data()
|
||||
RETURNS VOID AS $$
|
||||
DECLARE
|
||||
user_row RECORD;
|
||||
new_email varchar;
|
||||
new_hash varchar;
|
||||
new_username varchar;
|
||||
counter integer := 1;
|
||||
BEGIN
|
||||
-- scrub the user table
|
||||
TRUNCATE django_session;
|
||||
|
||||
-- clean up non-staff social auth data
|
||||
DELETE FROM social_auth_usersocialauth
|
||||
WHERE uid NOT LIKE '%@mozillafoundation.org';
|
||||
|
||||
-- Update the site domain
|
||||
UPDATE django_site
|
||||
SET domain = '{DOMAIN}.mofostaging.net'
|
||||
WHERE domain = 'foundation.mofostaging.net';
|
||||
|
||||
UPDATE wagtailcore_site
|
||||
SET hostname = '{HOSTNAME}.mofostaging.net'
|
||||
WHERE hostname = 'foundation.mofostaging.net';
|
||||
|
||||
UPDATE wagtailcore_site
|
||||
SET hostname = 'mozfest-{HOSTNAME}.mofostaging.net'
|
||||
WHERE hostname = 'mozillafestival.mofostaging.net';
|
||||
|
||||
-- Iterate over each non-staff user and remove any PII
|
||||
FOR user_row IN
|
||||
SELECT id
|
||||
FROM auth_user
|
||||
WHERE email NOT LIKE '%@mozillafoundation.org'
|
||||
LOOP
|
||||
new_email := concat(encode(gen_random_bytes(12), 'base64'), '@example.com');
|
||||
new_hash := crypt(encode(gen_random_bytes(32), 'base64'), gen_salt('bf', 6));
|
||||
new_username := concat('anonymouse', counter::varchar);
|
||||
|
||||
UPDATE auth_user
|
||||
SET
|
||||
email = new_email,
|
||||
password = new_hash,
|
||||
username = new_username,
|
||||
first_name = 'anony',
|
||||
last_name = 'mouse'
|
||||
Where id = user_row.id;
|
||||
|
||||
-- Increase the counter
|
||||
counter := counter + 1;
|
||||
END LOOP;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
|
||||
SELECT clean_user_data();
|
||||
|
|
@ -0,0 +1,103 @@
|
|||
import tempfile
|
||||
from time import sleep
|
||||
|
||||
from tasks import PLATFORM_ARG
|
||||
|
||||
STAGING_APP = "foundation-mofostaging-net"
|
||||
|
||||
|
||||
def execute_command(ctx, command: str, custom_error: str = ""):
|
||||
try:
|
||||
result = ctx.run(command, hide=False, warn=True, **PLATFORM_ARG)
|
||||
if result.failed:
|
||||
raise Exception(f"{custom_error}: {result.stderr}")
|
||||
return result.stdout.strip()
|
||||
except Exception as e:
|
||||
raise Exception(f"{custom_error}: {e}") from e
|
||||
|
||||
|
||||
def log_step(message: str):
|
||||
print(f"--> {message}\n", flush=True)
|
||||
|
||||
|
||||
def log_step_completed(message: str):
|
||||
print(f"✔️ {message} completed.\n", flush=True)
|
||||
|
||||
|
||||
def replace_placeholders_in_sql(review_app_name: str, input_file: str) -> str:
|
||||
with open(input_file, "r") as file:
|
||||
sql_content = file.read()
|
||||
|
||||
sql_content = sql_content.replace("{DOMAIN}", review_app_name)
|
||||
sql_content = sql_content.replace("{HOSTNAME}", review_app_name)
|
||||
|
||||
return sql_content
|
||||
|
||||
|
||||
def main(ctx, review_app_name):
|
||||
log_step(f"The review app name is: {review_app_name}, if not, please cancel now...")
|
||||
sleep(5)
|
||||
|
||||
log_step("Verifying if logged in Heroku")
|
||||
heroku_user = execute_command(ctx, "heroku whoami", "Verify that you are logged in Heroku CLI")
|
||||
print(f"Heroku user: {heroku_user}\n", flush=True)
|
||||
log_step_completed("Heroku login verification")
|
||||
|
||||
log_step("Verifying if psql is installed")
|
||||
execute_command(ctx, "psql --version", "Verify that you have 'psql' installed")
|
||||
log_step_completed("psql installation verification")
|
||||
|
||||
try:
|
||||
log_step("Enabling maintenance mode on the Review App")
|
||||
execute_command(ctx, f"heroku maintenance:on -a {review_app_name}")
|
||||
log_step_completed("Maintenance mode enabling")
|
||||
|
||||
log_step("Scaling web dynos on Review App to 0")
|
||||
execute_command(ctx, f"heroku ps:scale -a {review_app_name} web=0")
|
||||
log_step_completed("Web dynos scaling to 0")
|
||||
|
||||
log_step("Backing up Staging DB")
|
||||
execute_command(ctx, f"heroku pg:backups:capture -a {STAGING_APP}")
|
||||
log_step_completed("Staging DB backup")
|
||||
|
||||
log_step("Backing up Review App DB")
|
||||
execute_command(ctx, f"heroku pg:backups:capture -a {review_app_name}")
|
||||
log_step_completed("Review App DB backup")
|
||||
|
||||
log_step("Restoring the latest Staging backup to Review App")
|
||||
backup_staging_url = execute_command(ctx, f"heroku pg:backups:url -a {STAGING_APP}")
|
||||
execute_command(
|
||||
ctx, f"heroku pg:backups:restore --confirm {review_app_name} -a {review_app_name} '{backup_staging_url}'"
|
||||
)
|
||||
log_step_completed("Staging backup restoration to Review App")
|
||||
|
||||
log_step("Executing cleanup SQL script")
|
||||
review_app_db_url = execute_command(ctx, f"heroku config:get -a {review_app_name} DATABASE_URL")
|
||||
|
||||
# Replace placeholders and write to a temporary file
|
||||
sql_content = replace_placeholders_in_sql(review_app_name, "./cleanup.sql")
|
||||
with tempfile.NamedTemporaryFile(suffix=".sql", mode="w", delete=True) as temp_sql_file:
|
||||
temp_sql_file.write(sql_content)
|
||||
temp_sql_file.flush()
|
||||
execute_command(ctx, f"psql {review_app_db_url} -f {temp_sql_file.name}")
|
||||
|
||||
log_step_completed("Cleanup SQL script execution")
|
||||
|
||||
log_step("Running migrations")
|
||||
execute_command(ctx, f"heroku run -a {review_app_name} -- python network-api/manage.py migrate --no-input")
|
||||
log_step_completed("Migrations running")
|
||||
|
||||
except Exception as e:
|
||||
log_step("Rolling back Review App")
|
||||
execute_command(ctx, f"heroku pg:backups:restore -a {review_app_name} --confirm {review_app_name}")
|
||||
print(e, flush=True)
|
||||
log_step_completed("Review App rollback")
|
||||
|
||||
finally:
|
||||
log_step("Scaling web dynos on Review App to 1")
|
||||
execute_command(ctx, f"heroku ps:scale -a {review_app_name} web=1")
|
||||
log_step_completed("Web dynos scaling to 1")
|
||||
|
||||
log_step("Disabling maintenance mode on Review App")
|
||||
execute_command(ctx, f"heroku maintenance:off -a {review_app_name}")
|
||||
log_step_completed("Maintenance mode disabling")
|
||||
|
|
@ -25,26 +25,27 @@ The general workflow is:
|
|||
To get a list of invoke commands available, run `invoke -l`:
|
||||
|
||||
```
|
||||
catch-up (catchup, docker-catchup) Rebuild images, install dependencies, and apply migrations
|
||||
compilemessages (docker-compilemessages) Compile the latest translations
|
||||
makemessages (docker-makemessages) Extract all template messages in .po files for localization
|
||||
makemigrations (docker-makemigrations) Creates new migration(s) for apps
|
||||
manage (docker-manage) Shorthand to manage.py. inv docker-manage "[COMMAND] [ARG]"
|
||||
migrate (docker-migrate) Updates database schema
|
||||
new-db (docker-new-db) Delete your database and create a new one with fake data
|
||||
copy-stage-db Overwrite your local docker postgres DB with a copy of the staging database
|
||||
copy-prod-db Overwrite your local docker postgres DB with a copy of the production database
|
||||
new-env (docker-new-env) Get a new dev environment and a new database with fake data
|
||||
npm (docker-npm) Shorthand to npm. inv docker-npm "[COMMAND] [ARG]"
|
||||
npm-install (docker-npm-install) Install Node dependencies
|
||||
pip-compile (docker-pip-compile) Shorthand to pip-tools. inv pip-compile "[filename]" "[COMMAND] [ARG]"
|
||||
pip-compile-lock (docker-pip-compile-lock) Lock prod and dev dependencies
|
||||
pip-sync (docker-pip-sync) Sync your python virtualenv
|
||||
start-dev (docker-start, start) Start the dev server
|
||||
start-lean-dev (docker-start-lean, start-lean) Start the dev server without rebuilding
|
||||
test (docker-test) Run both Node and Python tests
|
||||
test-node (docker-test-node) Run node tests
|
||||
test-python (docker-test-python) Run python tests
|
||||
catch-up (catchup, docker-catchup) Rebuild images, install dependencies, and apply migrations
|
||||
compilemessages (docker-compilemessages) Compile the latest translations
|
||||
makemessages (docker-makemessages) Extract all template messages in .po files for localization
|
||||
makemigrations (docker-makemigrations) Creates new migration(s) for apps
|
||||
manage (docker-manage) Shorthand to manage.py. inv docker-manage "[COMMAND] [ARG]"
|
||||
migrate (docker-migrate) Updates database schema
|
||||
new-db (docker-new-db) Delete your database and create a new one with fake data
|
||||
copy-stage-db Overwrite your local docker postgres DB with a copy of the staging database
|
||||
copy-prod-db Overwrite your local docker postgres DB with a copy of the production database
|
||||
new-env (docker-new-env) Get a new dev environment and a new database with fake data
|
||||
npm (docker-npm) Shorthand to npm. inv docker-npm "[COMMAND] [ARG]"
|
||||
npm-install (docker-npm-install) Install Node dependencies
|
||||
pip-compile (docker-pip-compile) Shorthand to pip-tools. inv pip-compile "[filename]" "[COMMAND] [ARG]"
|
||||
pip-compile-lock (docker-pip-compile-lock) Lock prod and dev dependencies
|
||||
pip-sync (docker-pip-sync) Sync your python virtualenv
|
||||
staging-db-to-review-app (staging-to-review-app) Copy Staging DB to a specific Review App. inv staging-to-review-app "[REVIEW_APP_NAME]"
|
||||
start-dev (docker-start, start) Start the dev server
|
||||
start-lean-dev (docker-start-lean, start-lean) Start the dev server without rebuilding
|
||||
test (docker-test) Run both Node and Python tests
|
||||
test-node (docker-test-node) Run node tests
|
||||
test-python (docker-test-python) Run python tests
|
||||
```
|
||||
|
||||
Note the above commands carefully, as they should cover the majority of what you'd need for local development.
|
||||
|
|
|
|||
10
tasks.py
10
tasks.py
|
|
@ -585,3 +585,13 @@ def compilemessages(ctx):
|
|||
"../dockerpythonvenv/bin/python manage.py compilemessages",
|
||||
**PLATFORM_ARG,
|
||||
)
|
||||
|
||||
|
||||
@task(aliases=["staging-to-review-app"])
|
||||
def staging_db_to_review_app(ctx, review_app_name):
|
||||
"""
|
||||
Copy Staging DB to a specific Review App. inv staging-to-review-app \"[REVIEW_APP_NAME]\"
|
||||
"""
|
||||
from copy_staging_db_to_review_app import main
|
||||
|
||||
main(ctx, review_app_name)
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче