From e8d6ca674b6893fb36de1f412f0cf891ba68b2cd Mon Sep 17 00:00:00 2001 From: Paul McLanahan Date: Wed, 22 Feb 2017 15:53:31 -0500 Subject: [PATCH] Cleanup bin dir and move run scripts to bin --- Makefile | 6 +- Procfile | 4 +- bin/circleci-docker-build.sh | 6 -- bin/jenkins.sh | 65 -------------- bin/pipstrap.py | 118 -------------------------- {docker => bin}/run-clock.sh | 0 {docker => bin}/run-prod.sh | 0 {docker => bin}/run-supervisor.sh | 0 {docker => bin}/run-tests.sh | 0 {docker => bin}/run.sh | 4 +- bin/{sync_all => sync-all.sh} | 0 bin/upgrade-pip.sh | 10 --- docker/dockerfiles/bedrock_base | 2 +- docker/dockerfiles/bedrock_test | 2 +- docker/jenkins/build_images.sh | 2 +- docs/install.rst | 2 +- etc/supervisor_available/bedrock.conf | 2 +- etc/supervisor_available/cron_db.conf | 2 +- 18 files changed, 13 insertions(+), 212 deletions(-) delete mode 100755 bin/circleci-docker-build.sh delete mode 100755 bin/jenkins.sh delete mode 100755 bin/pipstrap.py rename {docker => bin}/run-clock.sh (100%) rename {docker => bin}/run-prod.sh (100%) rename {docker => bin}/run-supervisor.sh (100%) rename {docker => bin}/run-tests.sh (100%) rename {docker => bin}/run.sh (70%) rename bin/{sync_all => sync-all.sh} (100%) delete mode 100755 bin/upgrade-pip.sh diff --git a/Makefile b/Makefile index 03596e19aa..0469cf8aef 100644 --- a/Makefile +++ b/Makefile @@ -44,7 +44,7 @@ shell: .docker-build docker run --user `id -u` -it --env-file docker/dev.env -v "$$PWD:/app" ${DEV_IMG_NAME} bash sync-all: .docker-build - docker run --user `id -u` --env-file docker/demo.env -v "$$PWD:/app" ${DEV_IMG_NAME} bin/sync_all + docker run --user `id -u` --env-file docker/demo.env -v "$$PWD:/app" ${DEV_IMG_NAME} bin/sync-all.sh clean: # python related things @@ -69,10 +69,10 @@ clean: -rm -f .docker-build-final test: .docker-build - docker run --user `id -u` --env-file docker/test.env -v "$$PWD:/app" ${DEV_IMG_NAME} docker/run-tests.sh + docker run --user `id -u` --env-file docker/test.env -v "$$PWD:/app" ${DEV_IMG_NAME} bin/run-tests.sh test-image: .docker-build-final - docker run --env-file docker/test.env ${FINAL_IMG_NAME} docker/run-tests.sh + docker run --env-file docker/test.env ${FINAL_IMG_NAME} bin/run-tests.sh docs: docker run --user `id -u` --env-file docker/dev.env -v "$$PWD:/app" ${DEV_IMG_NAME} bash -c "make -C docs/ clean && make -C docs/ html" diff --git a/Procfile b/Procfile index 672d17aff9..64fa3617ba 100644 --- a/Procfile +++ b/Procfile @@ -1,2 +1,2 @@ -web: ./docker/run.sh -clock: ./docker/run-clock.sh +web: ./bin/run.sh +clock: ./bin/run-clock.sh diff --git a/bin/circleci-docker-build.sh b/bin/circleci-docker-build.sh deleted file mode 100755 index 7f3f559bc6..0000000000 --- a/bin/circleci-docker-build.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -set -ex - -make clean -echo "ENV GIT_SHA ${CIRCLE_SHA1}" >> docker/dockerfiles/bedrock_dev_final -make build-final diff --git a/bin/jenkins.sh b/bin/jenkins.sh deleted file mode 100755 index 7020bdc72c..0000000000 --- a/bin/jenkins.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/bin/sh -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -# This script makes sure that Jenkins can properly run your tests against your -# codebase. -set -e - -if [ -z $WORKSPACE ] ; then - WORKSPACE=`/bin/pwd` -fi - -cd $WORKSPACE -VENV=$WORKSPACE/venv - -export DB_HOST="localhost" -export DB_USER="hudson" - -echo "Starting build on executor $EXECUTOR_NUMBER..." - -# Make sure there's no old pyc files around. -find . -name '*.pyc' -exec rm {} \; - -if [ ! -d "$VENV/bin" ]; then - echo "No virtualenv found. Making one..." - virtualenv $VENV --no-site-packages - . $VENV/bin/activate - pip install --upgrade pip - pip install coverage -fi - -git submodule sync -q -git submodule update --init --recursive - -if [ ! -d "$WORKSPACE/vendor" ]; then - echo "No /vendor... crap." - exit 1 -fi - -if [ -d "$WORKSPACE/locale" ]; then - svn up locale -else - svn checkout https://svn.mozilla.org/projects/mozilla.com/trunk/locales/ locale -fi - -. $VENV/bin/activate -pip install -q -r requirements/compiled.txt -pip install -q -r requirements/dev.txt - -echo "Creating database if we need it..." -echo "CREATE DATABASE IF NOT EXISTS \`${JOB_NAME}\`"|mysql -u $DB_USER -h $DB_HOST - -echo "Update product_details" -python manage.py update_product_details_files --database bedrock - -echo "Check PEP-8" -flake8 bedrock lib - -echo "Starting tests..." -export FORCE_DB=1 -coverage run py.test lib bedrock -coverage xml $(find bedrock lib -name '*.py') - -echo "FIN" diff --git a/bin/pipstrap.py b/bin/pipstrap.py deleted file mode 100755 index 4c4b2d5969..0000000000 --- a/bin/pipstrap.py +++ /dev/null @@ -1,118 +0,0 @@ -#!/usr/bin/env python -"""A small script that can act as a trust root for installing pip 8 - -Embed this in your project, and your VCS checkout is all you have to trust. In -a post-peep era, this lets you claw your way to a hash-checking version of pip, -with which you can install the rest of your dependencies safely. All it assumes -is Python 2.7 or better and *some* version of pip already installed. If -anything goes wrong, it will exit with a non-zero status code. - -""" -# This is here so embedded copies are MIT-compliant: -# Copyright (c) 2016 Erik Rose -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to -# deal in the Software without restriction, including without limitation the -# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or -# sell copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -from __future__ import print_function -from hashlib import sha256 -from os.path import join -from pipes import quote -from shutil import rmtree -from subprocess import check_output -from sys import exit -from tempfile import mkdtemp -try: - from urllib2 import build_opener, HTTPHandler, HTTPSHandler -except ImportError: - from urllib.request import build_opener, HTTPHandler, HTTPSHandler -try: - from urlparse import urlparse -except ImportError: - from urllib.parse import urlparse # 3.4 - - -PACKAGES = [ - # Pip has no dependencies, as it vendors everything: - ('https://pypi.python.org/packages/source/p/pip/pip-8.0.2.tar.gz', - '46f4bd0d8dfd51125a554568d646fe4200a3c2c6c36b9f2d06d2212148439521'), - # This version of setuptools has only optional dependencies: - ('https://pypi.python.org/packages/source/s/setuptools/' - 'setuptools-19.4.tar.gz', - '214bf29933f47cf25e6faa569f710731728a07a19cae91ea64f826051f68a8cf'), - # We require Python 2.7 or later because we don't support wheel's - # conditional dep on argparse. This version of wheel has no other - # dependencies: - ('https://pypi.python.org/packages/source/w/wheel/wheel-0.26.0.tar.gz', - 'eaad353805c180a47545a256e6508835b65a8e830ba1093ed8162f19a50a530c') -] - - -class HashError(Exception): - def __str__(self): - url, path, actual, expected = self.args - return ('{url} did not match the expected hash {expected}. Instead, ' - 'it was {actual}. The file (left at {path}) may have been ' - 'tampered with.'.format(**locals())) - - -def hashed_download(url, temp, digest): - """Download ``url`` to ``temp``, make sure it has the SHA-256 ``digest``, - and return its path.""" - # Based on pip 1.4.1's URLOpener but with cert verification removed - def opener(): - opener = build_opener(HTTPSHandler()) - # Strip out HTTPHandler to prevent MITM spoof: - for handler in opener.handlers: - if isinstance(handler, HTTPHandler): - opener.handlers.remove(handler) - return opener - - def read_chunks(response, chunk_size): - while True: - chunk = response.read(chunk_size) - if not chunk: - break - yield chunk - - response = opener().open(url) - path = join(temp, urlparse(url).path.split('/')[-1]) - actual_hash = sha256() - with open(path, 'wb') as file: - for chunk in read_chunks(response, 4096): - file.write(chunk) - actual_hash.update(chunk) - - actual_digest = actual_hash.hexdigest() - if actual_digest != digest: - raise HashError(url, path, actual_digest, digest) - return path - - -def main(): - temp = mkdtemp(prefix='pipstrap-') - try: - downloads = [hashed_download(url, temp, digest) - for url, digest in PACKAGES] - check_output('pip install --no-index --no-deps -U ' + - ' '.join(quote(d) for d in downloads), - shell=True) - except HashError as exc: - print(exc) - except Exception: - rmtree(temp) - raise - else: - rmtree(temp) - return 0 - return 1 - - -if __name__ == '__main__': - exit(main()) diff --git a/docker/run-clock.sh b/bin/run-clock.sh similarity index 100% rename from docker/run-clock.sh rename to bin/run-clock.sh diff --git a/docker/run-prod.sh b/bin/run-prod.sh similarity index 100% rename from docker/run-prod.sh rename to bin/run-prod.sh diff --git a/docker/run-supervisor.sh b/bin/run-supervisor.sh similarity index 100% rename from docker/run-supervisor.sh rename to bin/run-supervisor.sh diff --git a/docker/run-tests.sh b/bin/run-tests.sh similarity index 100% rename from docker/run-tests.sh rename to bin/run-tests.sh diff --git a/docker/run.sh b/bin/run.sh similarity index 70% rename from docker/run.sh rename to bin/run.sh index 3b189e7ce5..ee34c86479 100755 --- a/docker/run.sh +++ b/bin/run.sh @@ -3,7 +3,7 @@ RUN_SUPERVISOR=$(echo "$RUN_SUPERVISOR" | tr '[:upper:]' '[:lower:]') if [[ "$RUN_SUPERVISOR" == "true" ]]; then - exec docker/run-supervisor.sh + exec bin/run-supervisor.sh else - exec docker/run-prod.sh + exec bin/run-prod.sh fi diff --git a/bin/sync_all b/bin/sync-all.sh similarity index 100% rename from bin/sync_all rename to bin/sync-all.sh diff --git a/bin/upgrade-pip.sh b/bin/upgrade-pip.sh deleted file mode 100755 index 95bca98af8..0000000000 --- a/bin/upgrade-pip.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -PIP_MAJOR_VERSION=$(pip --version | cut -d ' ' -f 2 | cut -d '.' -f 1) - -if [[ "$PIP_MAJOR_VERSION" != "8" ]]; then - echo "Upgrading pip to version 8" - bin/pipstrap.py -else - echo "No need to upgrade pip" -fi diff --git a/docker/dockerfiles/bedrock_base b/docker/dockerfiles/bedrock_base index ae3426199b..17b28a3d62 100644 --- a/docker/dockerfiles/bedrock_base +++ b/docker/dockerfiles/bedrock_base @@ -10,7 +10,7 @@ RUN adduser --uid 1000 --disabled-password --gecos '' --no-create-home webdev WORKDIR /app EXPOSE 8000 -CMD ["./docker/run.sh"] +CMD ["./bin/run.sh"] RUN apt-get update && \ apt-get install -y --no-install-recommends \ diff --git a/docker/dockerfiles/bedrock_test b/docker/dockerfiles/bedrock_test index 6d3c82ecde..4c84b6a1e1 100644 --- a/docker/dockerfiles/bedrock_test +++ b/docker/dockerfiles/bedrock_test @@ -1,6 +1,6 @@ FROM mozorg/bedrock_code:${GIT_COMMIT} -CMD ["./docker/run-tests.sh"] +CMD ["./bin/run-tests.sh"] USER root RUN pip install --no-cache-dir -r requirements/test.txt diff --git a/docker/jenkins/build_images.sh b/docker/jenkins/build_images.sh index d355ce637e..b7b05164ac 100755 --- a/docker/jenkins/build_images.sh +++ b/docker/jenkins/build_images.sh @@ -74,7 +74,7 @@ fi # include the data that the deployments need if $DEMO_MODE && ! imageExists "demo"; then - dockerRun demo code bin/sync_all + dockerRun demo code bin/sync-all.sh docker/jenkins/docker_build.sh "demo" fi if $PROD_MODE && ! imageExists "l10n"; then diff --git a/docs/install.rst b/docs/install.rst index 56c8317d87..b0a4d1a513 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -45,7 +45,7 @@ You shouldn't need to customize anything in there yet. Sync the database and all of the external data locally. This gets product-details, security-advisories, credits, release notes, etc:: - $ bin/sync_all + $ bin/sync-all.sh Lastly, you need to have `Node.js `_ and `NPM `_ installed. The node diff --git a/etc/supervisor_available/bedrock.conf b/etc/supervisor_available/bedrock.conf index 245b805a72..d75d88a35c 100644 --- a/etc/supervisor_available/bedrock.conf +++ b/etc/supervisor_available/bedrock.conf @@ -1,5 +1,5 @@ [program:bedrock] -command = /app/docker/run-prod.sh +command = /app/bin/run-prod.sh numprocs = 1 autostart = true redirect_stderr = true diff --git a/etc/supervisor_available/cron_db.conf b/etc/supervisor_available/cron_db.conf index 8d6fd4ad1c..2a12a5d9b5 100644 --- a/etc/supervisor_available/cron_db.conf +++ b/etc/supervisor_available/cron_db.conf @@ -1,5 +1,5 @@ [program:cron_db] -command = /app/docker/run-clock.sh +command = /app/bin/run-clock.sh numprocs = 1 autostart = true redirect_stderr = true