diff --git a/.dockerignore b/.dockerignore index 1889e87..560be38 100644 --- a/.dockerignore +++ b/.dockerignore @@ -26,4 +26,9 @@ node_modules .vscode/ .tox venv -.doit.db \ No newline at end of file +.doit.db + +# Dockerfiles +src/sub/Dockerfile +src/hub/Dockerfile +Dockerfile.base diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..ad959b4 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,11 @@ +# https://editorconfig.org/ + +root = true + +[*] +indent_style = space +indent_size = 4 +insert_final_newline = true +trim_trailing_whitespace = true +end_of_line = lf +charset = utf-8 diff --git a/.gitignore b/.gitignore index c19b20f..a45904e 100644 --- a/.gitignore +++ b/.gitignore @@ -128,4 +128,10 @@ node_modules/ # GraphViz *.png -*.dot \ No newline at end of file +*.dot + +# src tarballs +.src.tar.gz + +# OSX +.DS_Store diff --git a/.idea/runConfigurations/behave.xml b/.idea/runConfigurations/behave.xml index 01bedcc..4f15809 100644 --- a/.idea/runConfigurations/behave.xml +++ b/.idea/runConfigurations/behave.xml @@ -1,6 +1,6 @@ - + diff --git a/.travis.yml b/.travis.yml index 36f7200..c79b2bd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -40,4 +40,4 @@ jobs: - doit draw - stage: Unit Test script: - - doit test \ No newline at end of file + - doit test diff --git a/Dockerfile.base b/Dockerfile.base new file mode 100644 index 0000000..a78803f --- /dev/null +++ b/Dockerfile.base @@ -0,0 +1,24 @@ +FROM python:3.7-alpine +MAINTAINER Stewart Henderson + +ENV PYTHONDONTWRITEBYTECODE 1 +ENV PYTHONUNBUFFERED 1 + +RUN mkdir -p /base/etc +COPY etc /base/etc + +RUN mkdir -p /base/bin +COPY bin /base/bin + +WORKDIR /base +COPY automation_requirements.txt /base +COPY src/app_requirements.txt /base +COPY src/test_requirements.txt /base + +RUN apk add bash==5.0.0-r0 && \ + bin/install-packages.sh && \ + pip3 install -r automation_requirements.txt && \ + pip3 install -r app_requirements.txt && \ + pip3 install -r test_requirements.txt && \ + pip3 install awscli==1.16.213 && \ + pip3 install "connexion[swagger-ui]" diff --git a/Pipfile b/Pipfile new file mode 100644 index 0000000..b723d01 --- /dev/null +++ b/Pipfile @@ -0,0 +1,11 @@ +[[source]] +name = "pypi" +url = "https://pypi.org/simple" +verify_ssl = true + +[dev-packages] + +[packages] + +[requires] +python_version = "3.7" diff --git a/README.md b/README.md index 1648b0e..ec37b99 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,8 @@ Payment subscription REST api for customers: - yarn (https://yarnpkg.com): package manager for node modules for setting up serverless for running and deploying subhub - cloc - [GraphViz](https://graphviz.org/) +- Docker +- Docker Compose ## Important Environment Variables The CFG object is for accessing values either from the `subhub/.env` file and|or superseded by env vars. @@ -123,7 +125,7 @@ The `check` task have several subtasks: - `doit check:json` This makes sure all of the json files in the git repo can be loaded. - `doit check:yaml` This makes sure all of the yaml files in the git repo can be loaded. - `doit check:black` This runs `black --check` to ensure formatting. -- `doit check:reqs` This compares subhub/requirements.txt vs what is installed via pip freeze. +- `doit check:reqs` This compares automation_requirements.txt vs what is installed via pip freeze. ## setup the virtualenv (venv) This task will create the virtual env and install all of the requirements for use in running code locally. @@ -177,13 +179,23 @@ This run the `serverless deploy` command and requires the user to be logged into doit deploy ``` +Alternatively you may deploy a subset of the `deploy` function by specifying the component as such: + +``` +doit deploy SERVICE FUNCTION +``` + +Where, + SERVICE is the service that you are deploying from the set of fxa. + FUNCTION is the function that you are deploying from the set of sub, hub, mia. + ## dependency graph This command will generate a GraphViz `dot` file that can be used to generate a media file. ``` doit graph ``` -## dependency graph image +## dependency graph image This command will generate a PNG of the dependency graph. ``` doit draw @@ -202,12 +214,12 @@ doit draw A [Postman](https://www.getpostman.com/) URL collection is available for testing, learning, etc [here](https://www.getpostman.com/collections/ab233178aa256e424668). -## [Performance Tests](./subhub/tests/performance/README.md) +## [Performance Tests](./{sub,hub}/tests/performance/README.md) ## Behave Tests -The `behave` tests for this project are located in the `subhub/tests/bdd` directory. The +The `behave` tests for this project are located in the `src/{sub,hub}/tests/bdd` directory. The steps that are available presently are available in the `steps`subdirectory. You can run this in a few ways: * Jetbrains PyCharm: A runtime configuration is loaded in that allows for debugging and running of the feature files. - * Command line: `cd subhub/tests/bdd && behave` after satisfying the `requirements.txt` in that directory. + * Command line: `cd src/{sub,hub}/tests/bdd && behave` after satisfying the `src/test_requirements.txt`. diff --git a/SRE_INFO.md b/SRE_INFO.md index 0ce7e64..02feb9c 100644 --- a/SRE_INFO.md +++ b/SRE_INFO.md @@ -12,7 +12,7 @@ administered by the Mozilla IT SRE team. We are available on #it-sre on slack. ## Secrets Secrets in this project all reside in AWS Secrets Manager. There is one set of secrets for each environment: prod, stage, qa, dev. These secrets are loaded as environment variables via the -subhub/secrets.py file and then generally used via the env loading mechanism in suhub/cfg.py which +src/shared/secrets.py file and then generally used via the env loading mechanism in src/shared/cfg.py which uses decouple to load them as fields. ## Source Repos diff --git a/docker-compose.yml b/docker-compose.yml index 426fc5b..28b1c82 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,19 +1,57 @@ version: "3.7" services: - subhub: - container_name: subhub - image: mozilla/subhub - # The `command` section below can take any command from - # `doit list` and run them here. - command: local + base: + image: mozilla/subhub-base + container_name: base build: context: . + dockerfile: Dockerfile.base + sub: + container_name: sub + image: mozilla/sub + command: python3 sub/app.py + build: + context: src/sub + dockerfile: Dockerfile args: + LOCAL_FLASK_PORT: 5000 + DYNALITE_PORT: 4567 + environment: AWS_ACCESS_KEY_ID: "fake-id" AWS_SECRET_ACCESS_KEY: "fake-key" - STRIPE_API_KEY: "sk_test_123" - SUPPORT_API_KEY: "support_test" - LOCAL_FLASK_PORT: 5000 + STRIPE_API_KEY: $STRIPE_API_KEY + PAYMENT_API_KEY: $PAYMENT_API_KEY + SUPPORT_API_KEY: $SUPPORT_API_KEY ports: - - "5000:5000" \ No newline at end of file + - "5000:5000" + depends_on: + - base + - dynalite + + hub: + container_name: hub + image: mozilla/hub + command: python3 hub/app.py + build: + context: src/hub + dockerfile: Dockerfile + args: + LOCAL_FLASK_PORT: 5001 + DYNALITE_PORT: 4567 + environment: + AWS_ACCESS_KEY_ID: "fake-id" + AWS_SECRET_ACCESS_KEY: "fake-key" + STRIPE_API_KEY: $STRIPE_API_KEY + HUB_API_KEY: $HUB_API_KEY + ports: + - "5001:5001" + depends_on: + - base + - dynalite + + dynalite: + build: https://github.com/vitarn/docker-dynalite.git + ports: + - 4567 + diff --git a/dodo.py b/dodo.py index 559aaea..ca12e03 100644 --- a/dodo.py +++ b/dodo.py @@ -14,8 +14,11 @@ from functools import lru_cache from doit.tools import LongRunning from pathlib import Path from pkg_resources import parse_version +from os.path import join, dirname, realpath -from subhub.cfg import CFG, call, CalledProcessError +sys.path.insert(0, join(dirname(realpath(__file__)), 'src')) + +from shared.cfg import CFG, call, CalledProcessError DOIT_CONFIG = { 'default_tasks': [ @@ -45,6 +48,10 @@ SVCS = [ svc for svc in os.listdir('services') if os.path.isdir(f'services/{svc}') if os.path.isfile(f'services/{svc}/serverless.yml') ] +SRCS = [ + src for src in os.listdir('src/') + if os.path.isdir(f'src/{src}') if src != 'shared' +] mutex = threading.Lock() @@ -106,6 +113,59 @@ def pyfiles(path, exclude=None): pyfiles = set(Path(path).rglob('*.py')) - set(Path(exclude).rglob('*.py') if exclude else []) return [pyfile.as_posix() for pyfile in pyfiles] +def load_serverless(svc): + return yaml.safe_load(open(f'services/{svc}/serverless.yml')) + +def get_svcs_to_funcs(): + return {svc: list(load_serverless(svc)['functions'].keys()) for svc in SVCS} + +def svc_func(svc, func=None): + assert svc in SVCS, f"svc '{svc}' not in {SVCS}" + funcs = get_svcs_to_funcs()[svc] + if func: + assert func in funcs, f"for svc '{svc}', func '{func}' not in {funcs}" + return svc, func + +def svc_action(svc, action=None): + assert svc in SVCS, f"svc '{svc}' not in {SVCS}" + assert action in ('create', 'delete') + return svc, action + +def parameterized(dec): + def layer(*args, **kwargs): + def repl(f): + return dec(f, *args, **kwargs) + return repl + layer.__name__ = dec.__name__ + layer.__doc__ = dec.__doc__ + return layer + +@parameterized +def guard(func, env): + def wrapper(*args, **kwargs): + task_dict = func(*args, **kwargs) + if CFG.DEPLOYED_ENV == env and CFG('DEPLOY_TO', None) != env: + task_dict['actions'] = [ + f'attempting to run {func.__name__} without env var DEPLOY_TO={env} set', + 'false', + ] + return task_dict + wrapper.__name__ = func.__name__ + wrapper.__doc__ = func.__doc__ + return wrapper + +@parameterized +def skip(func, taskname): + def wrapper(*args, **kwargs): + task_dict = func(*args, **kwargs) + envvar = f'SKIP_{taskname.upper()}' + if CFG(envvar, None): + task_dict['uptodate'] = [True] + return task_dict + wrapper.__name__ = func.__name__ + wrapper.__doc__ = func.__doc__ + return wrapper + # TODO: This needs to check for the existence of the dependency prior to execution or update project requirements. def task_count(): ''' @@ -203,9 +263,9 @@ def gen_file_check(name, func, *patterns, message=None): def check_black(): ''' - run black --check in subhub directory + run black --check in src/ directory ''' - black_check = f'black --check {CFG.PROJECT_PATH}' + black_check = f'black --check src/' return { 'name': 'black', 'task_dep': [ @@ -273,7 +333,7 @@ def task_check(): yield gen_file_check('json', json.load, 'services/**/*.json') yield gen_file_check('yaml', yaml.safe_load, 'services/**/*.yaml', 'services/**/*.yml') header_message = "consider running 'doit header:'" - yield gen_file_check('header', check_header, 'subhub/**/*.py', message=header_message) + yield gen_file_check('header', check_header, 'src/**/*.py', message=header_message) yield check_black() yield check_reqs() @@ -300,13 +360,12 @@ def task_creds(): ], } +@skip('test') def task_stripe(): ''' check to see if STRIPE_API_KEY is set ''' def stripe_check(): - if os.environ.get('SKIP_TESTS', None): - return True try: CFG.STRIPE_API_KEY except: @@ -327,20 +386,20 @@ def task_stripe(): def task_black(): ''' - run black on subhub/ + run black on src/ ''' return { 'actions': [ - f'black {CFG.PROJECT_PATH}', + f'black src/' ], } def task_header(): ''' - apply the HEADER to all the py files under subhub/ + apply the HEADER to all the py files under src/ ''' def ensure_headers(): - for pyfile in pyfiles('subhub/'): + for pyfile in pyfiles('src/'): with open(pyfile, 'r') as old: content = old.read() if has_header(content): @@ -356,12 +415,13 @@ def task_header(): ], } +@skip('venv') def task_venv(): ''' setup virtual env ''' - app_requirements = f'{CFG.PROJECT_PATH}/requirements.txt' - test_requirements = f'{CFG.PROJECT_PATH}/tests/requirements.txt' + app_requirements = f'src/app_requirements.txt' + test_requirements = f'src/test_requirements.txt' return { 'task_dep': [ 'check', @@ -372,9 +432,6 @@ def task_venv(): f'[ -f "{app_requirements}" ] && {PIP3} install -r "{app_requirements}"', f'[ -f "{test_requirements}" ] && {PIP3} install -r "{test_requirements}"', ], - 'uptodate': [ - lambda: os.environ.get('SKIP_VENV', None), - ], } def task_dynalite(): @@ -435,22 +492,13 @@ def task_local(): ''' run local deployment ''' - ENVS=envs( - AWS_ACCESS_KEY_ID='fake-id', - AWS_SECRET_ACCESS_KEY='fake-key', - PYTHONPATH='.' - ) return { 'task_dep': [ 'check', - 'stripe', - 'venv', - 'dynalite:start', #FIXME: test removed as a dep due to concurrency bug + 'tar' ], 'actions': [ - f'{PYTHON3} -m setup develop', - 'echo $PATH', - f'env {ENVS} {PYTHON3} subhub/app.py', + f'docker-compose up --build' ], } @@ -479,7 +527,7 @@ def task_perf_local(): AWS_SECRET_ACCESS_KEY='fake-key', PYTHONPATH='.' ) - cmd = f'env {ENVS} {PYTHON3} subhub/app.py' + cmd = f'env {ENVS} {PYTHON3} src/sub/app.py' #FIXME: should work on hub too... return { 'basename': 'perf-local', 'task_dep':[ @@ -491,8 +539,8 @@ def task_perf_local(): 'actions':[ f'{PYTHON3} -m setup develop', 'echo $PATH', - LongRunning(f'nohup env {envs} {PYTHON3} subhub/app.py > /dev/null &'), - f'cd subhub/tests/performance && locust -f locustfile.py --host=http://localhost:{FLASK_PORT}' + LongRunning(f'nohup env {envs} {PYTHON3} src/sub/app.py > /dev/null &'), #FIXME: same as above + f'cd src/sub/tests/performance && locust -f locustfile.py --host=http://localhost:{FLASK_PORT}' #FIXME: same ] } @@ -509,25 +557,29 @@ def task_perf_remote(): ], 'actions':[ f'{PYTHON3} -m setup develop', - f'cd subhub/tests/performance && locust -f locustfile.py --host=https://{CFG.DEPLOY_DOMAIN}' + f'cd src/sub/tests/performance && locust -f locustfile.py --host=https://{CFG.DEPLOY_DOMAIN}' #FIXME: same as above ] } +@skip('mypy') def task_mypy(): ''' run mpyp, a static type checker for Python 3 ''' - return { - 'task_dep': [ - 'check', - 'yarn', - 'venv', - ], - 'actions': [ - f'cd {CFG.REPO_ROOT} && {envs(MYPYPATH="venv")} {MYPY} -p subhub' - ], - } + for pkg in ('sub', 'hub'): + yield { + 'name': pkg, + 'task_dep': [ + 'check', + 'yarn', + 'venv', + ], + 'actions': [ + f'cd {CFG.REPO_ROOT}/src && env {envs(MYPYPATH=VENV)} {MYPY} -p {pkg}', + ], + } +@skip('test') def task_test(): ''' run tox in tests/ @@ -539,21 +591,18 @@ def task_test(): 'yarn', 'venv', 'dynalite:stop', - 'mypy', + #'mypy', FIXME: this needs to be activated once mypy is figured out ], 'actions': [ f'cd {CFG.REPO_ROOT} && tox', ], - 'uptodate': [ - lambda: os.environ.get('SKIP_TESTS', None), - ], } def task_pytest(): ''' run pytest per test file ''' - for filename in Path('subhub/tests').glob('**/*.py'): + for filename in Path('src/sub/tests').glob('**/*.py'): #FIXME: should work on hub too... yield { 'name': filename, 'task_dep': [ @@ -585,41 +634,77 @@ def task_package(): ], } -def task_deploy(): +def task_tar(): ''' - run serverless deploy -v for every service + tar up source files, dereferncing symlinks ''' - def deploy_to_prod(): - if CFG.DEPLOYED_ENV == 'prod': - if CFG('DEPLOY_TO', None) == 'prod': - return True - return False - return True - for svc in SVCS: - servicepath = f'services/{svc}' - curl = f'curl --silent https://{CFG.DEPLOYED_ENV}.{svc}.mozilla-subhub.app/v1/version' - describe = 'git describe --abbrev=7' + excludes = ' '.join([ + f'--exclude={CFG.SRCTAR}', + '--exclude=__pycache__', + '--exclude=*.pyc', + '--exclude=.env', + '--exclude=.git', + ]) + for src in SRCS: + ## it is important to note that this is required to keep the tarballs from + ## genereating different checksums and therefore different layers in docker + cmd = f'cd {CFG.REPO_ROOT}/src/{src} && echo "$(git status -s)" > {CFG.REVISION} && tar cvh {excludes} . | gzip -n > {CFG.SRCTAR} && rm {CFG.REVISION}' yield { - 'name': svc, + 'name': src, 'task_dep': [ - 'check', - 'creds', - 'stripe', - 'yarn', - 'test', + 'check:noroot', + # 'test', ], 'actions': [ - f'cd {servicepath} && env {envs()} {SLS} deploy --stage {CFG.DEPLOYED_ENV} --aws-s3-accelerate -v', - f'echo "{curl}"', - f'{curl}', - f'echo "{describe}"', - f'{describe}', - ] if deploy_to_prod() else [ - f'attempting to deploy to prod without env var DEPLOY_TO=prod set', - 'false', + f'echo "{cmd}"', + f'{cmd}', ], } +@guard('prod') +def task_deploy(): + ''' + deploy [] + ''' + def deploy(args): + svc, func = svc_func(*args) + if func: + deploy_cmd = f'cd services/{svc} && env {envs()} {SLS} deploy function --stage {CFG.DEPLOYED_ENV} --aws-s3-accelerate -v --function {func}' + else: + deploy_cmd = f'cd services/{svc} && env {envs()} {SLS} deploy --stage {CFG.DEPLOYED_ENV} --aws-s3-accelerate -v' + call(deploy_cmd, stdout=None, stderr=None) + return { + 'task_dep': [ + 'check', + 'creds', + 'stripe', + 'yarn', + 'test', + ], + 'pos_arg': 'args', + 'actions': [(deploy,)], + } + +@guard('prod') +def task_domain(): + ''' + domain [create|delete] + ''' + def domain(args): + svc, action = svc_action(*args) + assert action in ('create', 'delete'), "provide 'create' or 'delete'" + domain_cmd = f'cd services/{svc} && env {envs()} {SLS} {action}_domain --stage {CFG.DEPLOYED_ENV} -v' + call(domain_cmd, stdout=None, stderr=None) + return { + 'task_dep': [ + 'check', + 'creds', + 'yarn', + ], + 'pos_arg': 'args', + 'actions': [(domain,)], + } + def task_remove(): ''' run serverless remove -v for every service @@ -655,13 +740,17 @@ def task_curl(): ''' curl again remote deployment url: /version, /deployed ''' - for route in ('deployed', 'version'): - yield { - 'name': route, - 'actions': [ - f'curl --silent https://{CFG.DEPLOYED_ENV}.fxa.mozilla-subhub.app/v1/{route}', - ], - } + def curl(args): + svc, func = svc_func(*args) + funcs = [func] if func else [func for func in get_svcs_to_funcs()[svc] if func != 'mia'] + for func in funcs: + for route in ('version', 'deployed'): + cmd = f'curl --silent https://{CFG.DEPLOYED_ENV}.{svc}.mozilla-subhub.app/v1/{func}/{route}' + call(f'echo "{cmd}"; {cmd}', stdout=None, stderr=None) + return { + 'pos_arg': 'args', + 'actions': [(curl,)], + } def task_rmrf(): ''' @@ -687,7 +776,8 @@ def task_rmrf(): yield { 'name': name, 'actions': [ - f'sudo find {CFG.REPO_ROOT} -depth -name {name} -type {type} -exec {rmrf}' for name, type in targets.items() + f'sudo find {CFG.REPO_ROOT} -depth -name {name} -type {type} -exec {rmrf}' + for name, type in targets.items() ], } @@ -713,4 +803,4 @@ def task_draw(): 'file_dep': ['tasks.dot'], 'targets': ['tasks.png'], 'actions': ['dot -Tpng %(dependencies)s -o %(targets)s'], - } \ No newline at end of file + } diff --git a/etc/alpine-packages b/etc/alpine-packages index 584dd09..4244f59 100644 --- a/etc/alpine-packages +++ b/etc/alpine-packages @@ -6,7 +6,7 @@ libffi-dev==3.2.1-r6 openssl-dev==1.1.1c-r0 zeromq-dev==4.3.2-r1 linux-headers==4.19.36-r0 -nodejs==10.16.0-r0 +nodejs curl==7.65.1-r0 yarn==1.16.0-r0 gcc==8.3.0-r0 @@ -15,4 +15,4 @@ musl-dev==1.1.22-r3 pkgconfig git==2.22.0-r0 graphviz-dev==2.40.1-r1 -graphviz==2.40.1-r1 \ No newline at end of file +graphviz==2.40.1-r1 diff --git a/package.json b/package.json index 82ded8c..bd0e325 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,8 @@ "serverless-domain-manager": "3.2.7", "serverless-offline": "5.10.1", "serverless-plugin-tracing": "2.0.0", - "serverless-python-requirements": "5.0.0" + "serverless-python-requirements": "5.0.0", + "serverless-package-external": "1.1.1", + "serverless-wsgi": "1.7.3" } } diff --git a/services/fxa/README.md b/services/fxa/README.md new file mode 100644 index 0000000..91a1def --- /dev/null +++ b/services/fxa/README.md @@ -0,0 +1,71 @@ +# Serverless + +## Commands + +From this (`services/fxa`) directory, execute the following commands of interest. NOTE: If you require extra detail of the Serverless framework, +you will need to set the follow environment variable. + +`export SLS_DEBUG=*` + +### Offline Testing + +Start DynamoDB locally, `sls dynamodb start &` + +Start offline services, `serverless offline start --host 0.0.0.0 --stage dev` + +Once this is done, you can access the DynamoDB Javascript Shell by +navigating [here](http://localhost:8000/shell/). Additionally, you may interact with the application as you would on AWS via commands such as: + * Perform a HTTP GET of `http://localhost:3000/v1/sub/version` + +### Domain Creation + +`sls create_domain` + +### Packaging + +`sls package --stage ENVIRONMENT` + +Where `ENVIRONMENT` is in the set of (dev, staging, prod). + +You may inspect the contents of each packages with: + +`zipinfo .serverless/{ARCHIVE}.zip` + +Where `ARCHIVE` is a member of + +* sub +* hub +* mia + +### Logs + +You can inspect the Serverless logs by function via the command: + +`sls logs -f {FUNCTION}` + +Where `FUNCTION` is a member of + +* sub +* hub +* mia + +#### Live Tailing of the Logs + +`serverless logs -f {FUNCTION} --tail` + +### Running + +`sls wsgi serve` + +### To-do + +* [Investigate Serverless Termination Protection for Production](https://www.npmjs.com/package/serverless-termination-protection) +* [Investigate metering requests via apiKeySourceType](https://serverless.com/framework/docs/providers/aws/events/apigateway/) + +## References + +1. [SLS_DEBUG](https://github.com/serverless/serverless/pull/1729/files) +2. [API Gateway Resource Policy Support](https://github.com/serverless/serverless/issues/4926) +3. [Add apig resource policy](https://github.com/serverless/serverless/pull/5071) +4. [add PRIVATE endpointType](https://github.com/serverless/serverless/pull/5080) +5. [Serverless AWS Lambda Events](https://serverless.com/framework/docs/providers/aws/events/) diff --git a/services/fxa/accounts.yml b/services/fxa/accounts.yml index 7fdd9c1..bcd3f3c 100644 --- a/services/fxa/accounts.yml +++ b/services/fxa/accounts.yml @@ -3,3 +3,4 @@ fxa: stage: 142069644989 qa: 142069644989 dev: 927034868273 + fab: 927034868273 diff --git a/services/fxa/config/config.dev.json b/services/fxa/config/config.dev.json new file mode 100644 index 0000000..56908e8 --- /dev/null +++ b/services/fxa/config/config.dev.json @@ -0,0 +1,6 @@ +{ + "LAMBDA_MEMORY_SIZE": 256, + "LAMBDA_RESERVED_CONCURRENCY": 1, + "LAMBDA_TIMEOUT": 5, + "MIA_RATE_SCHEDULE": "24 hours" +} diff --git a/services/fxa/config/config.fab.json b/services/fxa/config/config.fab.json new file mode 100644 index 0000000..e116d03 --- /dev/null +++ b/services/fxa/config/config.fab.json @@ -0,0 +1,6 @@ +{ + "LAMBDA_MEMORY_SIZE": 512, + "LAMBDA_RESERVED_CONCURRENCY": 5, + "LAMBDA_TIMEOUT": 5, + "MIA_RATE_SCHEDULE": "30 days" +} diff --git a/services/fxa/config/config.prod.json b/services/fxa/config/config.prod.json new file mode 100644 index 0000000..784d19c --- /dev/null +++ b/services/fxa/config/config.prod.json @@ -0,0 +1,6 @@ +{ + "LAMBDA_MEMORY_SIZE": 512, + "LAMBDA_RESERVED_CONCURRENCY": 5, + "LAMBDA_TIMEOUT": 5, + "MIA_RATE_SCHEDULE": "6 hours" +} diff --git a/services/fxa/config/config.qa.json b/services/fxa/config/config.qa.json new file mode 100644 index 0000000..ee1f1aa --- /dev/null +++ b/services/fxa/config/config.qa.json @@ -0,0 +1,6 @@ +{ + "LAMBDA_MEMORY_SIZE": 256, + "LAMBDA_RESERVED_CONCURRENCY": 1, + "LAMBDA_TIMEOUT": 5, + "MIA_RATE_SCHEDULE": "30 days" +} diff --git a/services/fxa/config/config.stage.json b/services/fxa/config/config.stage.json new file mode 100644 index 0000000..784d19c --- /dev/null +++ b/services/fxa/config/config.stage.json @@ -0,0 +1,6 @@ +{ + "LAMBDA_MEMORY_SIZE": 512, + "LAMBDA_RESERVED_CONCURRENCY": 5, + "LAMBDA_TIMEOUT": 5, + "MIA_RATE_SCHEDULE": "6 hours" +} diff --git a/services/fxa/env.yml b/services/fxa/env.yml new file mode 100644 index 0000000..27920c9 --- /dev/null +++ b/services/fxa/env.yml @@ -0,0 +1,17 @@ +default: + DEPLOYED_BY: ${env:DEPLOYED_BY} + DEPLOYED_ENV: ${env:DEPLOYED_ENV} + DEPLOYED_WHEN: ${env:DEPLOYED_WHEN} + STAGE: ${self:provider.stage} + PROJECT_NAME: ${env:PROJECT_NAME} + BRANCH: ${env:BRANCH} + REVISION: ${env:REVISION} + VERSION: ${env:VERSION} + REMOTE_ORIGIN_URL: ${env:REMOTE_ORIGIN_URL} + LOG_LEVEL: ${env:LOG_LEVEL} + NEW_RELIC_ACCOUNT_ID: ${env:NEW_RELIC_ACCOUNT_ID} + NEW_RELIC_TRUSTED_ACCOUNT_ID: ${env:NEW_RELIC_TRUSTED_ACCOUNT_ID} + NEW_RELIC_SERVERLESS_MODE_ENABLED: ${env:NEW_RELIC_SERVERLESS_MODE_ENABLED} + NEW_RELIC_DISTRIBUTED_TRACING_ENABLED: ${env:NEW_RELIC_DISTRIBUTED_TRACING_ENABLED} + PROFILING_ENABLED: ${env:PROFILING_ENABLED} + PROCESS_EVENTS_HOURS: 6 diff --git a/services/fxa/handler.py b/services/fxa/handler.py deleted file mode 100644 index dd3f8e5..0000000 --- a/services/fxa/handler.py +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -import os -import sys - -import awsgi -import newrelic.agent - -from aws_xray_sdk.core import xray_recorder -from aws_xray_sdk.ext.flask.middleware import XRayMiddleware - -newrelic.agent.initialize() - -# First some funky path manipulation so that we can work properly in -# the AWS environment -dir_path = os.path.dirname(os.path.realpath(__file__)) -sys.path.append(dir_path) - -from subhub.app import create_app -from subhub.log import get_logger - -logger = get_logger() - -xray_recorder.configure(service="subhub") - -# Create app at module scope to cache it for repeat requests -try: - app = create_app() - XRayMiddleware(app.app, xray_recorder) -except Exception: # pylint: disable=broad-except - logger.exception("Exception occurred while loading app") - # TODO: Add Sentry exception catch here - raise - -@newrelic.agent.lambda_handler() -def handle(event, context): - try: - logger.info("handling event", subhub_event=event, context=context) - return awsgi.response(app, event, context) - except Exception as e: # pylint: disable=broad-except - logger.exception("exception occurred", subhub_event=event, context=context, error=e) - # TODO: Add Sentry exception catch here - raise diff --git a/services/fxa/hubhandler.py b/services/fxa/hubhandler.py new file mode 100644 index 0000000..7f3e28c --- /dev/null +++ b/services/fxa/hubhandler.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +import os +import sys + +# TODO! +# import newrelic.agent +import serverless_wsgi + +serverless_wsgi.TEXT_MIME_TYPES.append("application/custom+json") + +from os.path import join, dirname, realpath +# First some funky path manipulation so that we can work properly in +# the AWS environment +sys.path.insert(0, join(dirname(realpath(__file__)), 'src')) + +# TODO! +# newrelic.agent.initialize() + +from aws_xray_sdk.core import xray_recorder, patch_all +from aws_xray_sdk.core.context import Context +from aws_xray_sdk.ext.flask.middleware import XRayMiddleware + +from hub.app import create_app +from shared.log import get_logger + +logger = get_logger() + +xray_recorder.configure(service="fxa.hub") +patch_all() + +hub_app = create_app() +XRayMiddleware(hub_app.app, xray_recorder) + +# TODO! +# @newrelic.agent.lambda_handler() +def handle(event, context): + try: + logger.info("handling hub event", subhub_event=event, context=context) + return serverless_wsgi.handle_request(hub_app.app, event, context) + except Exception as e: # pylint: disable=broad-except + logger.exception("exception occurred", subhub_event=event, context=context, error=e) + # TODO: Add Sentry exception catch here + raise diff --git a/services/fxa/miahandler.py b/services/fxa/miahandler.py new file mode 100644 index 0000000..a5ec0a0 --- /dev/null +++ b/services/fxa/miahandler.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +import os +import sys + +# TODO! +# import newrelic.agent +import serverless_wsgi + +from os.path import join, dirname, realpath +# First some funky path manipulation so that we can work properly in +# the AWS environment +sys.path.insert(0, join(dirname(realpath(__file__)), 'src')) + +# TODO! +# newrelic.agent.initialize() + +from aws_xray_sdk.core import xray_recorder, patch_all +from aws_xray_sdk.core.context import Context +from aws_xray_sdk.ext.flask.middleware import XRayMiddleware + +from hub.verifications import events_check +from shared.log import get_logger + +logger = get_logger() + +xray_recorder.configure(service="fxa.mia") +patch_all() + +# TODO! +# @newrelic.agent.lambda_handler() +def handle_mia(event, context): + try: + logger.info("handling mia event", subhub_event=event, context=context) + processing_duration=int(os.getenv('PROCESS_EVENTS_HOURS', '6')) + events_check.process_events(processing_duration) + except Exception as e: # pylint: disable=broad-except + logger.exception("exception occurred", subhub_event=event, context=context, error=e) + # TODO: Add Sentry exception catch here + raise + diff --git a/services/fxa/resources/dynamodb-table.yml b/services/fxa/resources/dynamodb-table.yml new file mode 100644 index 0000000..f9b0aaa --- /dev/null +++ b/services/fxa/resources/dynamodb-table.yml @@ -0,0 +1,43 @@ +Resources: + UsersTable: + Type: 'AWS::DynamoDB::Table' + Properties: + TableName: ${self:custom.usersTable} + AttributeDefinitions: + - AttributeName: user_id + AttributeType: S + KeySchema: + - AttributeName: user_id + KeyType: HASH + PointInTimeRecoverySpecification: + PointInTimeRecoveryEnabled: true + # Set the capacity to auto-scale + BillingMode: PAY_PER_REQUEST + DeletedUsersTable: + Type: 'AWS::DynamoDB::Table' + Properties: + TableName: ${self:custom.deletedUsersTable} + AttributeDefinitions: + - AttributeName: user_id + AttributeType: S + KeySchema: + - AttributeName: user_id + KeyType: HASH + PointInTimeRecoverySpecification: + PointInTimeRecoveryEnabled: true + # Set the capacity to auto-scale + BillingMode: PAY_PER_REQUEST + EventsTable: + Type: 'AWS::DynamoDB::Table' + Properties: + TableName: ${self:custom.eventsTable} + AttributeDefinitions: + - AttributeName: event_id + AttributeType: S + KeySchema: + - AttributeName: event_id + KeyType: HASH + PointInTimeRecoverySpecification: + PointInTimeRecoveryEnabled: true + # Set the capacity to auto-scale + BillingMode: PAY_PER_REQUEST \ No newline at end of file diff --git a/services/fxa/resources/sns-topic.yml b/services/fxa/resources/sns-topic.yml new file mode 100644 index 0000000..77c375d --- /dev/null +++ b/services/fxa/resources/sns-topic.yml @@ -0,0 +1,24 @@ +Resources: + SubHubSNS: + Type: AWS::SNS::Topic + Properties: + DisplayName: FxA ${self:provider.stage} Event Data + TopicName: ${self:provider.stage}-fxa-event-data + SubHubTopicPolicy: + Type: AWS::SNS::TopicPolicy + Properties: + PolicyDocument: + Id: AWSAccountTopicAccess + Version: '2008-10-17' + Statement: + - Sid: FxAStageAccess + Effect: Allow + Principal: + AWS: arn:aws:iam::${self:provider.snsaccount}:root + Action: + - SNS:Subscribe + - SNS:Receive + - SNS:GetTopicAttributes + Resource: arn:aws:sns:us-west-2:903937621340:${self:provider.stage}-fxa-event-data + Topics: + - Ref: SubHubSNS \ No newline at end of file diff --git a/services/fxa/resources/tags.yml b/services/fxa/resources/tags.yml new file mode 100644 index 0000000..e16c50d --- /dev/null +++ b/services/fxa/resources/tags.yml @@ -0,0 +1,15 @@ +default: + cost-center: 1440 + project-name: Subhub + project-desc: Payment subscription REST API for customers + project-email: subhub@mozilla.com + deployed-by: ${env:DEPLOYED_BY} + deployed-env: ${env:DEPLOYED_ENV} + deployed-when: ${env:DEPLOYED_WHEN} + deployed-method: serverless + sources: https://github.com/mozilla/subhub + urls: prod.fxa.mozilla-subhub.app/v1 + keywords: subscriptions:flask:serverless:swagger + branch: ${env:BRANCH} + revision: ${env:REVISION} + version: ${env:VERSION} diff --git a/services/fxa/serverless.yml b/services/fxa/serverless.yml index 49105bf..bdf5648 100644 --- a/services/fxa/serverless.yml +++ b/services/fxa/serverless.yml @@ -2,133 +2,42 @@ service: name: fxa -plugins: - - serverless-python-requirements - - serverless-domain-manager - - serverless-plugin-tracing - - serverless-dynamodb-local - - serverless-offline - -provider: - name: aws - runtime: python3.7 - region: us-west-2 - stage: ${opt:stage, 'dev'} - stackName: ${self:custom.prefix}-stack - apiName: ${self:custom.prefix}-apigw - deploymentPrefix: ${self:custom.prefix} - endpointType: regional - logRetentionInDays: 90 - logs: - restApi: true - memorySize: 512 - reservedConcurrency: 5 - timeout: 5 - tracing: true - snsaccount: ${file(./accounts.yml):fxa.${self:provider.stage}} - environment: - DEPLOYED_BY: ${env:DEPLOYED_BY} - DEPLOYED_ENV: ${env:DEPLOYED_ENV} - DEPLOYED_WHEN: ${env:DEPLOYED_WHEN} - STAGE: ${self:provider.stage} - PROJECT_NAME: ${env:PROJECT_NAME} - BRANCH: ${env:BRANCH} - REVISION: ${env:REVISION} - VERSION: ${env:VERSION} - REMOTE_ORIGIN_URL: ${env:REMOTE_ORIGIN_URL} - LOG_LEVEL: ${env:LOG_LEVEL} - NEW_RELIC_ACCOUNT_ID: ${env:NEW_RELIC_ACCOUNT_ID} - NEW_RELIC_TRUSTED_ACCOUNT_ID: ${env:NEW_RELIC_TRUSTED_ACCOUNT_ID} - NEW_RELIC_SERVERLESS_MODE_ENABLED: ${env:NEW_RELIC_SERVERLESS_MODE_ENABLED} - NEW_RELIC_DISTRIBUTED_TRACING_ENABLED: ${env:NEW_RELIC_DISTRIBUTED_TRACING_ENABLED} - PROFILING_ENABLED: ${env:PROFILING_ENABLED} - USER_TABLE: - Ref: 'Users' - EVENT_TABLE: - Ref: 'Events' - DELETED_USER_TABLE: - Ref: 'DeletedUsers' - tags: - cost-center: 1440 - project-name: subhub - project-desc: payment subscription REST api for customers - project-email: subhub@mozilla.com - deployed-by: ${env:DEPLOYED_BY} - deployed-env: ${env:DEPLOYED_ENV} - deployed-when: ${env:DEPLOYED_WHEN} - deployed-method: serverless - sources: https://github.com/mozilla/subhub - urls: prod.fxa.mozilla-subhub.app/v1 - keywords: subhub:subscriptions:flask:serverless:swagger - branch: ${env:BRANCH} - revision: ${env:REVISION} - version: ${env:VERSION} - stackTags: - service: ${self:service} - iamRoleStatements: - - Effect: Allow - Action: - - 'dynamodb:Query' - - 'dynamodb:Scan' - - 'dynamodb:GetItem' - - 'dynamodb:PutItem' - - 'dynamodb:UpdateItem' - - 'dynamodb:DeleteItem' - - 'dynamodb:DescribeTable' - - 'dynamodb:CreateTable' - Resource: - - { 'Fn::GetAtt': ['Users', 'Arn'] } - - { 'Fn::GetAtt': ['Events', 'Arn'] } - - { 'Fn::GetAtt': ['DeletedUsers', 'Arn']} - - Effect: Allow - Action: - - 'secretsmanager:GetSecretValue' - Resource: - - 'Fn::Join': [':', ['arn:aws:secretsmanager', Ref: AWS::Region, Ref: AWS::AccountId, 'secret:${self:provider.stage}/*']] - - Effect: Allow - Action: - - logs:CreateLogGroup - - logs:CreateLogStream - - logs:PutLogEvents - Resource: - - 'Fn::Join': [':', ['arn:aws:logs', Ref: AWS::Region, Ref: AWS::AccountId, 'log-group:/aws/lambda/*:*:*']] - - Effect: Allow - Action: - - kms:Decrypt - Resource: - - 'Fn::Join': [':', ['arn:aws:kms', Ref: AWS::Region, Ref: AWS::AccountId, 'alias/*']] - - 'Fn::Join': [':', ['arn:aws:kms', Ref: AWS::Region, Ref: AWS::AccountId, 'key/*']] - - Effect: Allow - Action: - - 'xray:PutTraceSegments' - - 'xray:PutTelemetryRecords' - Resource: - - '*' - - Effect: Allow - Action: - - sns:Publish - Resource: - - 'Fn::Join': [':', ['arn:aws:sns', Ref: AWS::Region, Ref: AWS::AccountId, '${self:provider.stage}-fxa-event-data']] - resourcePolicy: ${self:custom.resourcePolicies.${self:custom.access.${self:provider.stage}}} - package: + individually: true exclude: - - '**/*' + - 'node_modules/*' include: - 'handler.py' - - 'subhub/**' + - 'src/**' custom: stage: ${opt:stage, self:provider.stage} + usersTable: ${self:custom.stage}-users + deletedUsersTable: ${self:custom.stage}-deletedUsers + eventsTable: ${self:custom.stage}-events prefix: ${self:provider.stage}-${self:service.name} subdomain: ${self:provider.stage}.${self:service.name} pythonRequirements: dockerizePip: 'non-linux' - fileName: subhub/requirements.txt - git-repo: https://github.com/mozilla/subhub + fileName: "../../src/app_requirements.txt" + packageExternal: + external: + - '../../src/sub' + - '../../src/hub' + - '../../src/shared' + - '../../src' + git-repo: "https://github.com/mozilla/subhub" dynamodb: + stages: + - dev start: + port: 8000 + inMemory: true + heapInitial: 200m + heapMax: 1g migrate: true + seed: true + convertEmptyValues: true customDomain: domainName: ${self:custom.subdomain}.mozilla-subhub.app certificateName: ${self:custom.subdomain}.mozilla-subhub.app @@ -142,6 +51,7 @@ custom: stage: restricted qa: restricted dev: unfettered + fab: unfettered resourcePolicies: unfettered: - Effect: Allow @@ -177,7 +87,7 @@ custom: - execute-api:/*/*/support/* Condition: IpAddress: - aws:SourceIp: ${file(./whitelist.yml):support.${self:provider.stage}} + aws:SourceIp: ${file(whitelist.yml):support.${self:provider.stage}} - Effect: Allow Principal: "*" Action: execute-api:Invoke @@ -186,7 +96,7 @@ custom: - execute-api:/*/*/plans Condition: IpAddress: - aws:SourceIp: ${file(./whitelist.yml):payments.${self:provider.stage}} + aws:SourceIp: ${file(whitelist.yml):payments.${self:provider.stage}} - Effect: Allow Principal: "*" Action: execute-api:Invoke @@ -194,108 +104,143 @@ custom: - execute-api:/*/*/hub Condition: IpAddress: - aws:SourceIp: ${file(./whitelist.yml):hub} + aws:SourceIp: ${file(whitelist.yml):hub} +plugins: + - serverless-python-requirements + - serverless-domain-manager + - serverless-plugin-tracing + - serverless-dynamodb-local + - serverless-package-external + - serverless-offline +provider: + name: aws + runtime: python3.7 + region: us-west-2 + stage: ${opt:stage} + stackName: ${self:custom.prefix}-stack + apiName: ${self:custom.prefix}-apigw + deploymentPrefix: ${self:custom.prefix} + endpointType: regional + logRetentionInDays: 90 + logs: + # NOTE: https://github.com/serverless/serverless/issues/6112 + # Logging documentation: + # https://serverless.com/framework/docs/providers/aws/events/apigateway/ + restApi: true + tracing: + lambda: true + apiGateway: true + memorySize: ${file(config/config.${self:custom.stage}.json):LAMBDA_MEMORY_SIZE} + reservedConcurrency: ${file(config/config.${self:custom.stage}.json):LAMBDA_RESERVED_CONCURRENCY} + timeout: ${file(config/config.${self:custom.stage}.json):LAMBDA_TIMEOUT} + snsaccount: ${file(accounts.yml):fxa.${self:provider.stage}} + environment: ${file(env.yml):${self:custom.stage}, file(env.yml):default} + tags: ${file(resources/tags.yml):${self:custom.stage}, file(resources/tags.yml):default} + stackTags: + service: ${self:service} + # Reference: https://serverless.com/blog/abcs-of-iam-permissions/ + iamRoleStatements: + - Effect: Allow + Action: + - 'dynamodb:Query' + - 'dynamodb:Scan' + - 'dynamodb:GetItem' + - 'dynamodb:PutItem' + - 'dynamodb:UpdateItem' + - 'dynamodb:DeleteItem' + - 'dynamodb:DescribeTable' + - 'dynamodb:CreateTable' + Resource: 'arn:aws:dynamodb:us-west-2:*:*' + - Effect: Allow + Action: + - 'secretsmanager:GetSecretValue' + Resource: + - 'Fn::Join': [':', ['arn:aws:secretsmanager', Ref: AWS::Region, Ref: AWS::AccountId, 'secret:${self:provider.stage}/*']] + - Effect: Allow + Action: + - logs:CreateLogGroup + - logs:CreateLogStream + - logs:PutLogEvents + Resource: + - 'Fn::Join': [':', ['arn:aws:logs', Ref: AWS::Region, Ref: AWS::AccountId, 'log-group:/aws/lambda/*:*:*']] + - Effect: Allow + Action: + - kms:Decrypt + Resource: + - 'Fn::Join': [':', ['arn:aws:kms', Ref: AWS::Region, Ref: AWS::AccountId, 'alias/*']] + - 'Fn::Join': [':', ['arn:aws:kms', Ref: AWS::Region, Ref: AWS::AccountId, 'key/*']] + - Effect: Allow + Action: + - 'xray:PutTraceSegments' + - 'xray:PutTelemetryRecords' + Resource: + - '*' + - Effect: Allow + Action: + - sns:Publish + Resource: + - 'Fn::Join': [':', ['arn:aws:sns', Ref: AWS::Region, Ref: AWS::AccountId, '${self:provider.stage}-fxa-event-data']] + resourcePolicy: ${self:custom.resourcePolicies.${self:custom.access.${self:provider.stage}}} functions: - subhub: - name: ${self:custom.prefix}-function - description: > - subhub service for handling subscription services interactions - handler: handler.handle - timeout: 30 + sub: + name: '${self:custom.prefix}-sub' + description: "Function for handling subscription services interactions\n" + handler: subhandler.handle events: - - http: - method: ANY - path: / - cors: true - - http: - method: ANY - path: '{proxy+}' - cors: true - + - http: { + path: "/sub/{proxy+}", + method: get, + cors: false, + private: false + } + hub: + name: ${self:custom.prefix}-hub + description: > + Function for handling subscription services interactions + handler: hubhandler.handle + events: + - http: { + path: /hub, + method: post, + cors: false, + private: false + } + - http: { + path: "/hub/{proxy+}", + method: any, + cors: false, + private: false + } + mia: + name: ${self:custom.prefix}-mia + description: > + Function for reconcilation of missing hub events + handler: miahandler.handle + events: + # Invoke Lambda function on a schedule (either cron or rate limited). This fires an + # + # Reference: Serverless Event Scheduling + # https://serverless.com/framework/docs/providers/aws/events/schedule/ + # Reference: AWS Cloudwatch Scheduled Event: + # https://docs.aws.amazon.com/AmazonCloudWatch/latest/events/EventTypes.html#schedule_event_type + # + # Rate Syntax, http://docs.aws.amazon.com/AmazonCloudWatch/latest/events/ScheduledEvents.html#RateExpressions + # rate(value unit) + # where value is an unsigned integer + # and the unit is a unit of time in the set of (minute, minutes, hour, hours, day, days) + # + # Cron Syntax, http://docs.aws.amazon.com/lambda/latest/dg/tutorial-scheduled-events-schedule-expressions.html + # cron(minutes day-of-month(DOM) month day-of-week(DOW) year) + # where + # Field | Values | Wildcards + # Minutes | 0-59 | ,-*/ + # Hours | 0-23 | ,-*/ + # DOM | 1-31 | ,-*/?LW + # Month | 1-12 | ,-*/ + # DOW | 1-7 | ,-*?/L# + # Year | 192199 | ,-*/ + - schedule: rate(${file(config/config.${self:custom.stage}.json):MIA_RATE_SCHEDULE}) resources: - Resources: - SubHubSNS: - Type: AWS::SNS::Topic - Properties: - DisplayName: FxA ${self:provider.stage} Event Data - TopicName: ${self:provider.stage}-fxa-event-data - SubHubTopicPolicy: - Type: AWS::SNS::TopicPolicy - Properties: - PolicyDocument: - Id: AWSAccountTopicAccess - Version: '2008-10-17' - Statement: - - Sid: FxAStageAccess - Effect: Allow - Principal: - AWS: arn:aws:iam::${self:provider.snsaccount}:root - Action: - - SNS:Subscribe - - SNS:Receive - - SNS:GetTopicAttributes - Resource: arn:aws:sns:us-west-2:903937621340:${self:provider.stage}-fxa-event-data - Topics: - - Ref: SubHubSNS - Users: - Type: 'AWS::DynamoDB::Table' - DeletionPolicy: Retain - Properties: - AttributeDefinitions: - - - AttributeName: user_id - AttributeType: S - KeySchema: - - - AttributeName: user_id - KeyType: HASH - BillingMode: PAY_PER_REQUEST - PointInTimeRecoverySpecification: - PointInTimeRecoveryEnabled: true - DeletedUsers: - Type: 'AWS::DynamoDB::Table' - DeletionPolicy: Retain - Properties: - AttributeDefinitions: - - AttributeName: user_id - AttributeType: S - KeySchema: - - AttributeName: user_id - KeyType: HASH - BillingMode: PAY_PER_REQUEST - PointInTimeRecoverySpecification: - PointInTimeRecoveryEnabled: true - Events: - Type: 'AWS::DynamoDB::Table' - DeletionPolicy: Retain - Properties: - AttributeDefinitions: - - - AttributeName: event_id - AttributeType: S - KeySchema: - - - AttributeName: event_id - KeyType: HASH - BillingMode: PAY_PER_REQUEST - PointInTimeRecoverySpecification: - PointInTimeRecoveryEnabled: true - Outputs: - SubHubSNS: - Value: - Ref: SubHubSNS - Export: - Name: ${self:custom.stage}-SubHubSNS - SubHubTopicPolicy: - Value: - Ref: SubHubTopicPolicy - Users: - Value: - Ref: Users - Events: - Value: - Ref: Events - DeletedUsers: - Value: - Ref: DeletedUsers \ No newline at end of file + - ${file(resources/dynamodb-table.yml)} + - ${file(resources/sns-topic.yml)} diff --git a/services/fxa/subhandler.py b/services/fxa/subhandler.py new file mode 100644 index 0000000..59f4476 --- /dev/null +++ b/services/fxa/subhandler.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +import os +import sys + +# TODO! +# import newrelic.agent +import serverless_wsgi + +serverless_wsgi.TEXT_MIME_TYPES.append("application/custom+json") + +from os.path import join, dirname, realpath +# First some funky path manipulation so that we can work properly in +# the AWS environment +sys.path.insert(0, join(dirname(realpath(__file__)), 'src')) + +# TODO! +# newrelic.agent.initialize() + +from aws_xray_sdk.core import xray_recorder, patch_all +from aws_xray_sdk.core.context import Context +from aws_xray_sdk.ext.flask.middleware import XRayMiddleware + + +from sub.app import create_app +from shared.log import get_logger + +logger = get_logger() + +xray_recorder.configure(service="fxa.sub") +patch_all() + +sub_app = create_app() +XRayMiddleware(sub_app.app, xray_recorder) + +# TODO! +# @newrelic.agent.lambda_handler() +def handle(event, context): + try: + logger.info("handling sub event", subhub_event=event, context=context) + return serverless_wsgi.handle_request(sub_app.app, event, context) + except Exception as e: # pylint: disable=broad-except + logger.exception("exception occurred", subhub_event=event, context=context, error=e) + # TODO: Add Sentry exception catch here + raise diff --git a/services/fxa/subhub b/services/fxa/subhub deleted file mode 120000 index 6c20adc..0000000 --- a/services/fxa/subhub +++ /dev/null @@ -1 +0,0 @@ -../../subhub \ No newline at end of file diff --git a/services/fxa/whitelist.yml b/services/fxa/whitelist.yml index 195bfef..e42f278 100644 --- a/services/fxa/whitelist.yml +++ b/services/fxa/whitelist.yml @@ -3,6 +3,10 @@ payments: - 54.68.203.164 - 35.164.199.47 - 52.36.241.207 + prod-test: + - 3.217.6.148 + - 3.214.25.122 + - 3.94.151.18 support: prod: - 13.210.202.182/32 diff --git a/setup.cfg b/setup.cfg index a7a417b..6690c67 100644 --- a/setup.cfg +++ b/setup.cfg @@ -21,4 +21,3 @@ warn_no_return = False warn_redundant_casts = True warn_unused_ignores = True warn_unreachable = False - diff --git a/setup.py b/setup.py index 19a5795..2eab6d7 100644 --- a/setup.py +++ b/setup.py @@ -7,10 +7,10 @@ from setuptools import setup, find_packages with open("README.md", "r") as fh: long_description = fh.read() -with open('subhub/requirements.txt') as f: +with open('src/app_requirements.txt') as f: app_requirements = f.read().splitlines() -with open('subhub/tests/requirements.txt') as f: +with open('src/test_requirements.txt') as f: test_requirements = f.read().splitlines() setup_requirements = [ @@ -37,7 +37,7 @@ setup( install_requires=app_requirements, license='Mozilla Public License 2.0', include_package_data=True, - packages=find_packages(include=['subhub']), + packages=find_packages(include=['src']), setup_requires=setup_requirements, test_suite='tests', tests_require=test_requirements, diff --git a/subhub/requirements.txt b/src/app_requirements.txt similarity index 86% rename from subhub/requirements.txt rename to src/app_requirements.txt index 65b37f6..786fa23 100644 --- a/subhub/requirements.txt +++ b/src/app_requirements.txt @@ -1,7 +1,6 @@ -# requirements for subhub application to run +# requirements for hub.application to run # do not add testing reqs or automation reqs here attrdict==2.0.1 -aws-wsgi==0.0.8 boto3==1.9.184 botocore==1.12.184 certifi==2019.6.16 @@ -31,9 +30,10 @@ PyYAML==5.1.1 requests==2.22.0 s3transfer==0.2.1 six==1.12.0 -stripe==2.33.0 +stripe==2.35.1 structlog==19.1.0 urllib3==1.25.3 pyinstrument==3.0.3 aws-xray-sdk==2.4.2 -cachetools==3.1.1 \ No newline at end of file +cachetools==3.1.1 +serverless-wsgi==1.7.3 diff --git a/src/hub/Dockerfile b/src/hub/Dockerfile new file mode 100644 index 0000000..4852be3 --- /dev/null +++ b/src/hub/Dockerfile @@ -0,0 +1,31 @@ +FROM mozilla/subhub-base:latest +MAINTAINER Stewart Henderson + +ARG STRIPE_API_KEY +ARG AWS_ACCESS_KEY_ID +ARG AWS_SECRET_ACCESS_KEY +ARG LOCAL_FLASK_PORT +ARG SUPPORT_API_KEY +ARG DYNALITE_PORT + +ENV STRIPE_API_KEY=$STRIPE_API_KEY +ENV AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID +ENV AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY +ENV LOCAL_FLASK_PORT=$LOCAL_FLASK_PORT +ENV HUB_API_KEY=$HUB_API_KEY +ENV DYNALITE_PORT=$DYNALITE_PORT +ENV FLASK_ENV=development + +ENV DEPLOYED_ENV=local +ENV BRANCH=local +ENV REVISION=latest +ENV VERSION=latest +ENV PROJECT_NAME=subhub +ENV REMOTE_ORIGIN_URL=git@github.com:mozilla/subhub.git + +EXPOSE $LOCAL_FLASK_PORT + +RUN mkdir -p /subhub/hub +ADD .src.tar.gz /subhub/hub +WORKDIR /subhub/ +ENV PYTHONPATH=. diff --git a/subhub/__init__.py b/src/hub/__init__.py similarity index 100% rename from subhub/__init__.py rename to src/hub/__init__.py diff --git a/src/hub/app.py b/src/hub/app.py new file mode 100644 index 0000000..72cd0b8 --- /dev/null +++ b/src/hub/app.py @@ -0,0 +1,146 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import os +import sys + +import connexion +import stripe + +from flask import current_app, g, jsonify +from flask_cors import CORS +from flask import request + +from shared import secrets +from shared.cfg import CFG +from shared.exceptions import SubHubError +from shared.db import HubEvent +from shared.log import get_logger + +logger = get_logger() + +# Setup Stripe Error handlers +def intermittent_stripe_error(e): + logger.error("intermittent stripe error", error=e) + return jsonify({"message": f"{e.user_message}"}), 503 + + +def server_stripe_error(e): + logger.error("server stripe error", error=e) + return ( + jsonify({"message": f"{e.user_message}", "params": None, "code": f"{e.code}"}), + 500, + ) + + +def server_stripe_error_with_params(e): + logger.error("server stripe error with params", error=e) + return ( + jsonify( + { + "message": f"{e.user_message}", + "params": f"{e.param}", + "code": f"{e.code}", + } + ), + 500, + ) + + +def server_stripe_card_error(e): + logger.error("server stripe card error", error=e) + return jsonify({"message": f"{e.user_message}", "code": f"{e.code}"}), 402 + + +def is_container() -> bool: + import requests + + try: + requests.get(f"http://dynalite:{CFG.DYNALITE_PORT}") + return True + except Exception as e: + pass + return False + + +def create_app(config=None): + logger.info("creating flask app", config=config) + region = "localhost" + host = f"http://localhost:{CFG.DYNALITE_PORT}" + if is_container(): + host = f"http://dynalite:{CFG.DYNALITE_PORT}" + stripe.api_key = CFG.STRIPE_API_KEY + if CFG.AWS_EXECUTION_ENV: + region = "us-west-2" + host = None + options = dict(swagger_ui=CFG.SWAGGER_UI) + + app = connexion.FlaskApp(__name__, specification_dir=".", options=options) + app.add_api("swagger.yaml", pass_context_arg_name="request", strict_validation=True) + + app.app.hub_table = HubEvent(table_name=CFG.EVENT_TABLE, region=region, host=host) + + if not app.app.hub_table.model.exists(): + app.app.hub_table.model.create_table( + read_capacity_units=1, write_capacity_units=1, wait=True + ) + + @app.app.errorhandler(SubHubError) + def display_subhub_errors(e: SubHubError): + if e.status_code == 500: + logger.error("display subhub errors", error=e) + response = jsonify(e.to_dict()) + response.status_code = e.status_code + return response + + for error in ( + stripe.error.APIConnectionError, + stripe.error.APIError, + stripe.error.RateLimitError, + stripe.error.IdempotencyError, + ): + app.app.errorhandler(error)(intermittent_stripe_error) + + for error in (stripe.error.AuthenticationError,): + app.app.errorhandler(error)(server_stripe_error) + + for error in ( + stripe.error.InvalidRequestError, + stripe.error.StripeErrorWithParamCode, + ): + app.app.errorhandler(error)(server_stripe_error_with_params) + + for error in (stripe.error.CardError,): + app.app.errorhandler(error)(server_stripe_card_error) + + @app.app.before_request + def before_request(): + g.hub_table = current_app.hub_table + g.app_system_id = None + if CFG.PROFILING_ENABLED: + if "profile" in request.args and not hasattr(sys, "_called_from_test"): + from pyinstrument import Profiler + + g.profiler = Profiler() + g.profiler.start() + + @app.app.after_request + def after_request(response): + if not hasattr(g, "profiler") or hasattr(sys, "_called_from_test"): + return response + if CFG.PROFILING_ENABLED: + g.profiler.stop() + output_html = g.profiler.output_html() + return app.app.make_response(output_html) + return response + + CORS(app.app) + return app + + +if __name__ == "__main__": + app = create_app() + app.debug = True + app.use_reloader = True + app.run(host="0.0.0.0", port=CFG.LOCAL_FLASK_PORT) diff --git a/subhub/hub/__init__.py b/src/hub/routes/__init__.py similarity index 100% rename from subhub/hub/__init__.py rename to src/hub/routes/__init__.py diff --git a/subhub/hub/routes/abstract.py b/src/hub/routes/abstract.py similarity index 96% rename from subhub/hub/routes/abstract.py rename to src/hub/routes/abstract.py index 3596887..a7ab084 100644 --- a/subhub/hub/routes/abstract.py +++ b/src/hub/routes/abstract.py @@ -6,7 +6,7 @@ from abc import ABC import flask -from subhub.log import get_logger +from hub.shared.log import get_logger logger = get_logger() diff --git a/subhub/hub/routes/firefox.py b/src/hub/routes/firefox.py similarity index 91% rename from subhub/hub/routes/firefox.py rename to src/hub/routes/firefox.py index f2f1fdb..830c589 100644 --- a/subhub/hub/routes/firefox.py +++ b/src/hub/routes/firefox.py @@ -10,9 +10,9 @@ from typing import Dict from botocore.exceptions import ClientError from stripe.error import APIConnectionError -from subhub.hub.routes.abstract import AbstractRoute -from subhub.cfg import CFG -from subhub.log import get_logger +from hub.routes.abstract import AbstractRoute +from hub.shared.cfg import CFG +from hub.shared.log import get_logger logger = get_logger() diff --git a/subhub/hub/routes/pipeline.py b/src/hub/routes/pipeline.py similarity index 81% rename from subhub/hub/routes/pipeline.py rename to src/hub/routes/pipeline.py index b9394db..3121b28 100644 --- a/subhub/hub/routes/pipeline.py +++ b/src/hub/routes/pipeline.py @@ -2,9 +2,9 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. -from subhub.hub.routes.firefox import FirefoxRoute -from subhub.hub.routes.salesforce import SalesforceRoute -from subhub.hub.routes.static import StaticRoutes +from hub.routes.firefox import FirefoxRoute +from hub.routes.salesforce import SalesforceRoute +from hub.routes.static import StaticRoutes class RoutesPipeline: diff --git a/subhub/hub/routes/salesforce.py b/src/hub/routes/salesforce.py similarity index 85% rename from subhub/hub/routes/salesforce.py rename to src/hub/routes/salesforce.py index 640030d..ca996d6 100644 --- a/subhub/hub/routes/salesforce.py +++ b/src/hub/routes/salesforce.py @@ -7,10 +7,9 @@ import requests from typing import Dict -from subhub.hub.routes.abstract import AbstractRoute -from subhub.cfg import CFG - -from subhub.log import get_logger +from hub.routes.abstract import AbstractRoute +from shared.cfg import CFG +from shared.log import get_logger logger = get_logger() diff --git a/subhub/hub/routes/static.py b/src/hub/routes/static.py similarity index 100% rename from subhub/hub/routes/static.py rename to src/hub/routes/static.py diff --git a/src/hub/shared b/src/hub/shared new file mode 120000 index 0000000..8fba6b6 --- /dev/null +++ b/src/hub/shared @@ -0,0 +1 @@ +../shared \ No newline at end of file diff --git a/src/hub/swagger.yaml b/src/hub/swagger.yaml new file mode 100644 index 0000000..22e7a81 --- /dev/null +++ b/src/hub/swagger.yaml @@ -0,0 +1,228 @@ +swagger: "2.0" + +info: + title: "SubHub - Hub API" + version: "1.0" + +consumes: + - "application/json" +produces: + - "application/json" + +basePath: /v1 + +securityDefinitions: + HubApiKey: + type: apiKey + in: header + name: Authorization + description: | + Hub validation + x-apikeyInfoFunc: shared.authentication.hub_auth +parameters: + uidParam: + in: path + name: uid + type: string + required: true + description: User ID + subIdParam: + in: path + name: sub_id + type: string + required: true + description: Subscription ID +paths: + /hub/version: + get: + operationId: shared.version.get_version + tags: + - Version + summary: SubHub - Hub API version + description: Show Subhub version string (git describe --abbrev=7) + produces: + - application/json + responses: + 200: + description: Success + schema: + $ref: '#/definitions/Version' + /hub/deployed: + get: + operationId: shared.deployed.get_deployed + tags: + - Deployed + summary: SubHub deployed + description: Show Subhub deployed env vars + produces: + - application/json + responses: + 200: + description: Success + schema: + $ref: '#/definitions/Deployed' + /hub: + post: + operationId: hub.vendor.controller.view + tags: + - Hub + summary: Receives hub calls + description: Receives hub calls. + produces: + - application/json + responses: + 200: + description: Hub call received successfully. + schema: + type: object + properties: + message: + type: string + 500: + description: Error - unable to receive webhook. + schema: + $ref: '#/definitions/Errormessage' + parameters: + - in: body + name: data + schema: + type: object +definitions: + Version: + type: object + properties: + BRANCH: + type: string + example: master + VERSION: + type: string + example: v0.0.3-11-gaf8af91 + REVISION: + type: string + example: af8af912255d204bcd178fe47e9a1af3215e09d4 + Deployed: + type: object + properties: + DEPLOYED_BY: + type: string + example: sidler@76 + DEPLOYED_ENV: + type: string + example: dev + DEPLOYED_WHEN: + type: string + example: 2019-07-26T21:21:16.180200 + Plans: + type: array + items: + type: object + properties: + plan_id: + type: string + example: pro_basic_823 + product_id: + type: string + example: pro_basic + product_name: + type: string + example: Moz Sub + interval: + type: string + example: month + enum: + - day + - week + - month + - year + amount: + type: integer + example: 500 + description: A positive number in cents representing how much to charge on a recurring basis. + currency: + type: string + example: usd + plan_name: + type: string + example: Monthly Rocket Launches + Subscriptions: + type: object + properties: + subscriptions: + type: array + required: [ + "subscription_id", + "status", + "plan_name", + "plan_id", + "ended_at", + "current_period_start", + "current_period_end", + "cancel_at_period_end" + ] + items: + type: object + properties: + subscription_id: + type: string + example: sub_abc123 + plan_id: + type: string + example: pro_basic_823 + plan_name: + type: string + example: "pro_basic" + current_period_end: + type: number + description: Seconds since UNIX epoch. + example: 1557361022 + current_period_start: + type: number + description: Seconds since UNIX epoch. + example: 1557361022 + end_at: + type: number + description: Non-null if the subscription is ending at a period in time. + example: 1557361022 + status: + type: string + description: Subscription status. + example: active + cancel_at_period_end: + type: boolean + description: Shows if subscription will be cancelled at the end of the period. + example: true + failure_code: + type: string + description: Shows the failure code for subscription that is incomplete. This is an optional field. + example: Card declined + failure_message: + type: string + description: Shows the failure message for subscription that is incomplete. This is an optional field. + example: Your card was declined. + Errormessage: + type: object + properties: + message: + type: string + example: The resource is not available. + code: + type: number + example: 404 + IntermittentError: + type: object + properties: + message: + type: string + example: Connection cannot be completed. + ServerError: + type: object + properties: + message: + type: string + example: Server not available + param: + type: string + example: Customer not found + code: + type: string + example: Invalid Account diff --git a/subhub/hub/routes/__init__.py b/src/hub/tests/__init__.py similarity index 100% rename from subhub/hub/routes/__init__.py rename to src/hub/tests/__init__.py diff --git a/subhub/tests/conftest.py b/src/hub/tests/conftest.py similarity index 63% rename from subhub/tests/conftest.py rename to src/hub/tests/conftest.py index 692ff48..33e8603 100644 --- a/subhub/tests/conftest.py +++ b/src/hub/tests/conftest.py @@ -6,7 +6,6 @@ import os import sys import signal import subprocess -import uuid import logging import psutil @@ -14,12 +13,9 @@ import pytest import stripe from flask import g -from subhub.sub import payments -from subhub.app import create_app -from subhub.cfg import CFG -from subhub.customer import create_customer - -from subhub.log import get_logger +from hub.app import create_app +from hub.shared.cfg import CFG +from hub.shared.log import get_logger logger = get_logger() @@ -37,7 +33,6 @@ def pytest_configure(): # Latest boto3 now wants fake credentials around, so here we are. os.environ["AWS_ACCESS_KEY_ID"] = "fake" os.environ["AWS_SECRET_ACCESS_KEY"] = "fake" - os.environ["USER_TABLE"] = "users-testing" os.environ["EVENT_TABLE"] = "events-testing" os.environ["ALLOWED_ORIGIN_SYSTEMS"] = "Test_system,Test_System,Test_System1" sys._called_from_test = True @@ -73,37 +68,5 @@ def pytest_unconfigure(): def app(): app = create_app() with app.app.app_context(): - g.subhub_account = app.app.subhub_account g.hub_table = app.app.hub_table - g.subhub_deleted_users = app.app.subhub_deleted_users yield app - - -@pytest.fixture() -def create_customer_for_processing(): - uid = uuid.uuid4() - customer = create_customer( - g.subhub_account, - user_id="process_customer", - source_token="tok_visa", - email="test_fixture@{}tester.com".format(uid.hex), - origin_system="Test_system", - display_name="John Tester", - ) - yield customer - - -@pytest.fixture(scope="function") -def create_subscription_for_processing(): - uid = uuid.uuid4() - subscription = payments.subscribe_to_plan( - "process_test", - { - "pmt_token": "tok_visa", - "plan_id": "plan_EtMcOlFMNWW4nd", - "origin_system": "Test_system", - "email": "subtest@{}tester.com".format(uid), - "display_name": "John Tester", - }, - ) - yield subscription diff --git a/subhub/tests/pylint.rc b/src/hub/tests/pylint.rc similarity index 100% rename from subhub/tests/pylint.rc rename to src/hub/tests/pylint.rc diff --git a/subhub/tests/pytest.ini b/src/hub/tests/pytest.ini similarity index 100% rename from subhub/tests/pytest.ini rename to src/hub/tests/pytest.ini diff --git a/subhub/hub/stripe/__init__.py b/src/hub/tests/unit/__init__.py similarity index 100% rename from subhub/hub/stripe/__init__.py rename to src/hub/tests/unit/__init__.py diff --git a/subhub/tests/unit/fixtures/invalid_plan_response.json b/src/hub/tests/unit/fixtures/invalid_plan_response.json similarity index 100% rename from subhub/tests/unit/fixtures/invalid_plan_response.json rename to src/hub/tests/unit/fixtures/invalid_plan_response.json diff --git a/subhub/tests/unit/fixtures/stripe_ch_test1.json b/src/hub/tests/unit/fixtures/stripe_ch_test1.json similarity index 100% rename from subhub/tests/unit/fixtures/stripe_ch_test1.json rename to src/hub/tests/unit/fixtures/stripe_ch_test1.json diff --git a/subhub/tests/unit/fixtures/stripe_cust_test1.json b/src/hub/tests/unit/fixtures/stripe_cust_test1.json similarity index 100% rename from subhub/tests/unit/fixtures/stripe_cust_test1.json rename to src/hub/tests/unit/fixtures/stripe_cust_test1.json diff --git a/subhub/tests/unit/fixtures/stripe_in_test1.json b/src/hub/tests/unit/fixtures/stripe_in_test1.json similarity index 100% rename from subhub/tests/unit/fixtures/stripe_in_test1.json rename to src/hub/tests/unit/fixtures/stripe_in_test1.json diff --git a/subhub/tests/unit/fixtures/stripe_plan_test1.json b/src/hub/tests/unit/fixtures/stripe_plan_test1.json similarity index 100% rename from subhub/tests/unit/fixtures/stripe_plan_test1.json rename to src/hub/tests/unit/fixtures/stripe_plan_test1.json diff --git a/subhub/tests/unit/fixtures/stripe_plan_test2.json b/src/hub/tests/unit/fixtures/stripe_plan_test2.json similarity index 100% rename from subhub/tests/unit/fixtures/stripe_plan_test2.json rename to src/hub/tests/unit/fixtures/stripe_plan_test2.json diff --git a/subhub/tests/unit/fixtures/stripe_plan_test3.json b/src/hub/tests/unit/fixtures/stripe_plan_test3.json similarity index 100% rename from subhub/tests/unit/fixtures/stripe_plan_test3.json rename to src/hub/tests/unit/fixtures/stripe_plan_test3.json diff --git a/subhub/tests/unit/fixtures/stripe_prod_test1.json b/src/hub/tests/unit/fixtures/stripe_prod_test1.json similarity index 100% rename from subhub/tests/unit/fixtures/stripe_prod_test1.json rename to src/hub/tests/unit/fixtures/stripe_prod_test1.json diff --git a/subhub/tests/unit/fixtures/stripe_prod_test2.json b/src/hub/tests/unit/fixtures/stripe_prod_test2.json similarity index 100% rename from subhub/tests/unit/fixtures/stripe_prod_test2.json rename to src/hub/tests/unit/fixtures/stripe_prod_test2.json diff --git a/subhub/tests/unit/fixtures/stripe_sub_test1.json b/src/hub/tests/unit/fixtures/stripe_sub_test1.json similarity index 100% rename from subhub/tests/unit/fixtures/stripe_sub_test1.json rename to src/hub/tests/unit/fixtures/stripe_sub_test1.json diff --git a/subhub/tests/unit/fixtures/stripe_sub_test2.json b/src/hub/tests/unit/fixtures/stripe_sub_test2.json similarity index 100% rename from subhub/tests/unit/fixtures/stripe_sub_test2.json rename to src/hub/tests/unit/fixtures/stripe_sub_test2.json diff --git a/subhub/tests/unit/fixtures/valid_plan_response.json b/src/hub/tests/unit/fixtures/valid_plan_response.json similarity index 100% rename from subhub/tests/unit/fixtures/valid_plan_response.json rename to src/hub/tests/unit/fixtures/valid_plan_response.json diff --git a/subhub/hub/verifications/__init__.py b/src/hub/tests/unit/stripe/__init__.py similarity index 100% rename from subhub/hub/verifications/__init__.py rename to src/hub/tests/unit/stripe/__init__.py diff --git a/subhub/sub/__init__.py b/src/hub/tests/unit/stripe/charge/__init__.py similarity index 100% rename from subhub/sub/__init__.py rename to src/hub/tests/unit/stripe/charge/__init__.py diff --git a/subhub/tests/unit/stripe/charge/badpayload.json b/src/hub/tests/unit/stripe/charge/badpayload.json similarity index 100% rename from subhub/tests/unit/stripe/charge/badpayload.json rename to src/hub/tests/unit/stripe/charge/badpayload.json diff --git a/subhub/tests/unit/stripe/charge/charge-captured.json b/src/hub/tests/unit/stripe/charge/charge-captured.json similarity index 100% rename from subhub/tests/unit/stripe/charge/charge-captured.json rename to src/hub/tests/unit/stripe/charge/charge-captured.json diff --git a/subhub/tests/unit/stripe/charge/charge-dispute-closed.json b/src/hub/tests/unit/stripe/charge/charge-dispute-closed.json similarity index 100% rename from subhub/tests/unit/stripe/charge/charge-dispute-closed.json rename to src/hub/tests/unit/stripe/charge/charge-dispute-closed.json diff --git a/subhub/tests/unit/stripe/charge/charge-dispute-created.json b/src/hub/tests/unit/stripe/charge/charge-dispute-created.json similarity index 100% rename from subhub/tests/unit/stripe/charge/charge-dispute-created.json rename to src/hub/tests/unit/stripe/charge/charge-dispute-created.json diff --git a/subhub/tests/unit/stripe/charge/charge-dispute-funds_reinstated.json b/src/hub/tests/unit/stripe/charge/charge-dispute-funds_reinstated.json similarity index 100% rename from subhub/tests/unit/stripe/charge/charge-dispute-funds_reinstated.json rename to src/hub/tests/unit/stripe/charge/charge-dispute-funds_reinstated.json diff --git a/subhub/tests/unit/stripe/charge/charge-dispute-funds_withdrawn.json b/src/hub/tests/unit/stripe/charge/charge-dispute-funds_withdrawn.json similarity index 100% rename from subhub/tests/unit/stripe/charge/charge-dispute-funds_withdrawn.json rename to src/hub/tests/unit/stripe/charge/charge-dispute-funds_withdrawn.json diff --git a/subhub/tests/unit/stripe/charge/charge-dispute-updated.json b/src/hub/tests/unit/stripe/charge/charge-dispute-updated.json similarity index 100% rename from subhub/tests/unit/stripe/charge/charge-dispute-updated.json rename to src/hub/tests/unit/stripe/charge/charge-dispute-updated.json diff --git a/subhub/tests/unit/stripe/charge/charge-expired.json b/src/hub/tests/unit/stripe/charge/charge-expired.json similarity index 100% rename from subhub/tests/unit/stripe/charge/charge-expired.json rename to src/hub/tests/unit/stripe/charge/charge-expired.json diff --git a/subhub/tests/unit/stripe/charge/charge-failed.json b/src/hub/tests/unit/stripe/charge/charge-failed.json similarity index 100% rename from subhub/tests/unit/stripe/charge/charge-failed.json rename to src/hub/tests/unit/stripe/charge/charge-failed.json diff --git a/subhub/tests/unit/stripe/charge/charge-pending.json b/src/hub/tests/unit/stripe/charge/charge-pending.json similarity index 100% rename from subhub/tests/unit/stripe/charge/charge-pending.json rename to src/hub/tests/unit/stripe/charge/charge-pending.json diff --git a/subhub/tests/unit/stripe/charge/charge-refund-updated.json b/src/hub/tests/unit/stripe/charge/charge-refund-updated.json similarity index 100% rename from subhub/tests/unit/stripe/charge/charge-refund-updated.json rename to src/hub/tests/unit/stripe/charge/charge-refund-updated.json diff --git a/subhub/tests/unit/stripe/charge/charge-refunded.json b/src/hub/tests/unit/stripe/charge/charge-refunded.json similarity index 100% rename from subhub/tests/unit/stripe/charge/charge-refunded.json rename to src/hub/tests/unit/stripe/charge/charge-refunded.json diff --git a/subhub/tests/unit/stripe/charge/charge-succeeded.json b/src/hub/tests/unit/stripe/charge/charge-succeeded.json similarity index 100% rename from subhub/tests/unit/stripe/charge/charge-succeeded.json rename to src/hub/tests/unit/stripe/charge/charge-succeeded.json diff --git a/subhub/tests/unit/stripe/charge/charge-updated.json b/src/hub/tests/unit/stripe/charge/charge-updated.json similarity index 100% rename from subhub/tests/unit/stripe/charge/charge-updated.json rename to src/hub/tests/unit/stripe/charge/charge-updated.json diff --git a/subhub/tests/unit/stripe/charge/test_stripe_charge.py b/src/hub/tests/unit/stripe/charge/test_stripe_charge.py similarity index 94% rename from subhub/tests/unit/stripe/charge/test_stripe_charge.py rename to src/hub/tests/unit/stripe/charge/test_stripe_charge.py index c83f1aa..e4e7833 100644 --- a/subhub/tests/unit/stripe/charge/test_stripe_charge.py +++ b/src/hub/tests/unit/stripe/charge/test_stripe_charge.py @@ -6,9 +6,9 @@ import mockito import requests import boto3 import flask -from subhub.cfg import CFG +from hub.shared.cfg import CFG -from subhub.tests.unit.stripe.utils import run_test, MockSqsClient +from hub.tests.unit.stripe.utils import run_test, MockSqsClient def test_stripe_hub_succeeded(mocker): diff --git a/subhub/tests/__init__.py b/src/hub/tests/unit/stripe/customer/__init__.py similarity index 100% rename from subhub/tests/__init__.py rename to src/hub/tests/unit/stripe/customer/__init__.py diff --git a/subhub/tests/unit/stripe/customer/customer-created.json b/src/hub/tests/unit/stripe/customer/customer-created.json similarity index 100% rename from subhub/tests/unit/stripe/customer/customer-created.json rename to src/hub/tests/unit/stripe/customer/customer-created.json diff --git a/subhub/tests/unit/stripe/customer/customer-deleted.json b/src/hub/tests/unit/stripe/customer/customer-deleted.json similarity index 100% rename from subhub/tests/unit/stripe/customer/customer-deleted.json rename to src/hub/tests/unit/stripe/customer/customer-deleted.json diff --git a/subhub/tests/unit/stripe/customer/customer-source-expiring.json b/src/hub/tests/unit/stripe/customer/customer-source-expiring.json similarity index 100% rename from subhub/tests/unit/stripe/customer/customer-source-expiring.json rename to src/hub/tests/unit/stripe/customer/customer-source-expiring.json diff --git a/subhub/tests/unit/stripe/customer/customer-subscription-created.json b/src/hub/tests/unit/stripe/customer/customer-subscription-created.json similarity index 100% rename from subhub/tests/unit/stripe/customer/customer-subscription-created.json rename to src/hub/tests/unit/stripe/customer/customer-subscription-created.json diff --git a/subhub/tests/unit/stripe/customer/customer-subscription-deleted.json b/src/hub/tests/unit/stripe/customer/customer-subscription-deleted.json similarity index 100% rename from subhub/tests/unit/stripe/customer/customer-subscription-deleted.json rename to src/hub/tests/unit/stripe/customer/customer-subscription-deleted.json diff --git a/subhub/tests/unit/stripe/customer/customer-subscription-updated-no-cancel.json b/src/hub/tests/unit/stripe/customer/customer-subscription-updated-no-cancel.json similarity index 100% rename from subhub/tests/unit/stripe/customer/customer-subscription-updated-no-cancel.json rename to src/hub/tests/unit/stripe/customer/customer-subscription-updated-no-cancel.json diff --git a/subhub/tests/unit/stripe/customer/customer-subscription-updated.json b/src/hub/tests/unit/stripe/customer/customer-subscription-updated.json similarity index 100% rename from subhub/tests/unit/stripe/customer/customer-subscription-updated.json rename to src/hub/tests/unit/stripe/customer/customer-subscription-updated.json diff --git a/subhub/tests/unit/stripe/customer/customer-updated.json b/src/hub/tests/unit/stripe/customer/customer-updated.json similarity index 100% rename from subhub/tests/unit/stripe/customer/customer-updated.json rename to src/hub/tests/unit/stripe/customer/customer-updated.json diff --git a/subhub/tests/unit/stripe/customer/test_stripe_customer.py b/src/hub/tests/unit/stripe/customer/test_stripe_customer.py similarity index 98% rename from subhub/tests/unit/stripe/customer/test_stripe_customer.py rename to src/hub/tests/unit/stripe/customer/test_stripe_customer.py index 1063e0d..713f931 100644 --- a/subhub/tests/unit/stripe/customer/test_stripe_customer.py +++ b/src/hub/tests/unit/stripe/customer/test_stripe_customer.py @@ -13,9 +13,9 @@ import requests from mockito import when, mock, unstub -from subhub.tests.unit.stripe.utils import run_test, MockSqsClient, MockSnsClient -from subhub.cfg import CFG -from subhub.log import get_logger +from hub.tests.unit.stripe.utils import run_test, MockSqsClient, MockSnsClient +from hub.shared.cfg import CFG +from hub.shared.log import get_logger logger = get_logger() diff --git a/subhub/tests/unit/__init__.py b/src/hub/tests/unit/stripe/event/__init__.py similarity index 100% rename from subhub/tests/unit/__init__.py rename to src/hub/tests/unit/stripe/event/__init__.py diff --git a/subhub/tests/unit/stripe/event/event.json b/src/hub/tests/unit/stripe/event/event.json similarity index 100% rename from subhub/tests/unit/stripe/event/event.json rename to src/hub/tests/unit/stripe/event/event.json diff --git a/subhub/tests/unit/stripe/event/more_event.json b/src/hub/tests/unit/stripe/event/more_event.json similarity index 100% rename from subhub/tests/unit/stripe/event/more_event.json rename to src/hub/tests/unit/stripe/event/more_event.json diff --git a/subhub/tests/unit/stripe/event/test_stripe_events.py b/src/hub/tests/unit/stripe/event/test_stripe_events.py similarity index 92% rename from subhub/tests/unit/stripe/event/test_stripe_events.py rename to src/hub/tests/unit/stripe/event/test_stripe_events.py index fa310c2..ff4814f 100644 --- a/subhub/tests/unit/stripe/event/test_stripe_events.py +++ b/src/hub/tests/unit/stripe/event/test_stripe_events.py @@ -3,22 +3,21 @@ # file, You can obtain one at https://mozilla.org/MPL/2.0/. import time -from datetime import datetime, timedelta import os import json - import boto3 import flask -from flask import Response import stripe import requests +from flask import Response from mockito import when, mock, unstub +from datetime import datetime, timedelta -from subhub.tests.unit.stripe.utils import run_view, run_event_process -from subhub.cfg import CFG -from subhub.hub.verifications.events_check import EventCheck, process_events -from subhub.log import get_logger +from hub.tests.unit.stripe.utils import run_view, run_event_process +from hub.verifications.events_check import EventCheck, process_events +from hub.shared.cfg import CFG +from hub.shared.log import get_logger logger = get_logger() diff --git a/subhub/tests/unit/customer/__init__.py b/src/hub/tests/unit/stripe/invoice/__init__.py similarity index 100% rename from subhub/tests/unit/customer/__init__.py rename to src/hub/tests/unit/stripe/invoice/__init__.py diff --git a/src/hub/tests/unit/stripe/invoice/invoice-finalized.json b/src/hub/tests/unit/stripe/invoice/invoice-finalized.json new file mode 100644 index 0000000..e2144aa --- /dev/null +++ b/src/hub/tests/unit/stripe/invoice/invoice-finalized.json @@ -0,0 +1,137 @@ +{ + "created": 1326853478, + "livemode": false, + "id": "evt_00000000000000", + "type": "invoice.finalized", + "object": "event", + "request": null, + "pending_webhooks": 1, + "api_version": "2018-02-06", + "data": { + "object": { + "id": "in_0000000", + "object": "invoice", + "account_country": "US", + "account_name": "Mozilla Corporation", + "amount_due": 1000, + "amount_paid": 1000, + "amount_remaining": 0, + "application_fee_amount": null, + "attempt_count": 1, + "attempted": true, + "auto_advance": false, + "billing": "charge_automatically", + "billing_reason": "subscription_create", + "charge": "ch_0000000", + "collection_method": "charge_automatically", + "created": 1559568873, + "currency": "usd", + "custom_fields": null, + "customer": "cus_00000000000", + "customer_address": null, + "customer_email": "john@gmail.com", + "customer_name": null, + "customer_phone": null, + "customer_shipping": null, + "customer_tax_exempt": "none", + "customer_tax_ids": [ + ], + "default_payment_method": null, + "default_source": null, + "default_tax_rates": [ + ], + "description": null, + "discount": null, + "due_date": null, + "ending_balance": 0, + "footer": null, + "hosted_invoice_url": "https://pay.stripe.com/invoice/invst_000000", + "invoice_pdf": "https://pay.stripe.com/invoice/invst_000000/pdf", + "lines": { + "object": "list", + "data": [ + { + "id": "sli_000000", + "object": "line_item", + "amount": 1000, + "currency": "usd", + "description": "1 Moz-Sub × Moz_Sub (at $10.00 / month)", + "discountable": true, + "livemode": false, + "metadata": { + }, + "period": { + "end": 1562160873, + "start": 1559568873 + }, + "plan": { + "id": "plan_000000", + "object": "plan", + "active": true, + "aggregate_usage": null, + "amount": 1000, + "billing_scheme": "per_unit", + "created": 1555354251, + "currency": "usd", + "interval": "month", + "interval_count": 1, + "livemode": false, + "metadata": { + "service1": "VPN", + "service2": "Water Bottles" + }, + "nickname": "Mozilla_Subscription", + "product": "prod_000000", + "tiers": null, + "tiers_mode": null, + "transform_usage": null, + "trial_period_days": 30, + "usage_type": "licensed" + }, + "proration": false, + "quantity": 1, + "subscription": "sub_000000", + "subscription_item": "si_000000", + "tax_amounts": [ + ], + "tax_rates": [ + ], + "type": "subscription" + } + ], + "has_more": false, + "total_count": 1, + "url": "/v1/invoices/in_0000000/lines" + }, + "livemode": false, + "metadata": { + }, + "next_payment_attempt": null, + "number": "C8828DAC-0001", + "paid": true, + "payment_intent": "pi_000000", + "period_end": 1559568873, + "period_start": 1559568873, + "post_payment_credit_notes_amount": 0, + "pre_payment_credit_notes_amount": 0, + "receipt_number": null, + "starting_balance": 0, + "statement_descriptor": null, + "status": "paid", + "status_transitions": { + "finalized_at": 1559568873, + "marked_uncollectible_at": null, + "paid_at": 1559568874, + "voided_at": null + }, + "subscription": "sub_000000", + "subtotal": 1000, + "tax": null, + "tax_percent": null, + "total": 1000, + "total_tax_amounts": [ + ], + "webhooks_delivered_at": null + } + } +} diff --git a/subhub/tests/unit/stripe/invoice/invoice-payment-failed.json b/src/hub/tests/unit/stripe/invoice/invoice-payment-failed.json similarity index 100% rename from subhub/tests/unit/stripe/invoice/invoice-payment-failed.json rename to src/hub/tests/unit/stripe/invoice/invoice-payment-failed.json diff --git a/subhub/tests/unit/stripe/invoice/test_stripe_invoice.py b/src/hub/tests/unit/stripe/invoice/test_stripe_invoice.py similarity index 92% rename from subhub/tests/unit/stripe/invoice/test_stripe_invoice.py rename to src/hub/tests/unit/stripe/invoice/test_stripe_invoice.py index c1da25c..dcccfcc 100644 --- a/subhub/tests/unit/stripe/invoice/test_stripe_invoice.py +++ b/src/hub/tests/unit/stripe/invoice/test_stripe_invoice.py @@ -9,10 +9,10 @@ import requests import boto3 import flask -from subhub.cfg import CFG -from subhub import secrets +from hub.shared.cfg import CFG +from hub.shared import secrets -from subhub.tests.unit.stripe.utils import run_test, MockSqsClient +from hub.tests.unit.stripe.utils import run_test, MockSqsClient def run_customer(mocker, data, filename): diff --git a/subhub/tests/unit/stripe/__init__.py b/src/hub/tests/unit/stripe/payment/__init__.py similarity index 100% rename from subhub/tests/unit/stripe/__init__.py rename to src/hub/tests/unit/stripe/payment/__init__.py diff --git a/subhub/tests/unit/stripe/payment/payment-intent-succeeded.json b/src/hub/tests/unit/stripe/payment/payment-intent-succeeded.json similarity index 100% rename from subhub/tests/unit/stripe/payment/payment-intent-succeeded.json rename to src/hub/tests/unit/stripe/payment/payment-intent-succeeded.json diff --git a/subhub/tests/unit/stripe/payment/test_stripe_payments.py b/src/hub/tests/unit/stripe/payment/test_stripe_payments.py similarity index 93% rename from subhub/tests/unit/stripe/payment/test_stripe_payments.py rename to src/hub/tests/unit/stripe/payment/test_stripe_payments.py index 03a1b93..f861052 100644 --- a/subhub/tests/unit/stripe/payment/test_stripe_payments.py +++ b/src/hub/tests/unit/stripe/payment/test_stripe_payments.py @@ -10,10 +10,10 @@ import flask import stripe.error from mockito import when, mock, unstub -from subhub.cfg import CFG -from subhub import secrets +from hub.shared.cfg import CFG +from hub.shared import secrets -from subhub.tests.unit.stripe.utils import run_test, MockSqsClient +from hub.tests.unit.stripe.utils import run_test, MockSqsClient def run_customer(mocker, data, filename): diff --git a/subhub/tests/unit/stripe/utils.py b/src/hub/tests/unit/stripe/utils.py similarity index 87% rename from subhub/tests/unit/stripe/utils.py rename to src/hub/tests/unit/stripe/utils.py index db2d09d..e31d205 100644 --- a/subhub/tests/unit/stripe/utils.py +++ b/src/hub/tests/unit/stripe/utils.py @@ -5,12 +5,10 @@ import os import json -from typing import Dict, List, Optional, Any - -from subhub.cfg import CFG -from subhub import secrets -from subhub.hub.stripe.controller import StripeHubEventPipeline, event_process, view from flask import Response +from typing import Dict, Any + +from hub.vendor.controller import StripeHubEventPipeline, event_process, view __location__ = os.path.realpath(os.path.dirname(__file__)) diff --git a/subhub/tests/unit/test_stripe_controller.py b/src/hub/tests/unit/test_stripe_controller.py similarity index 90% rename from subhub/tests/unit/test_stripe_controller.py rename to src/hub/tests/unit/test_stripe_controller.py index 4036955..f57e231 100644 --- a/subhub/tests/unit/test_stripe_controller.py +++ b/src/hub/tests/unit/test_stripe_controller.py @@ -3,12 +3,12 @@ # file, You can obtain one at https://mozilla.org/MPL/2.0/. import flask + from flask import Response +from mockito import unstub -from mockito import when, mock, unstub - -from subhub.tests.unit.stripe.utils import run_event_process -from subhub.log import get_logger +from hub.tests.unit.stripe.utils import run_event_process +from hub.shared.log import get_logger logger = get_logger() diff --git a/subhub/tests/unit/stripe/charge/__init__.py b/src/hub/vendor/__init__.py similarity index 100% rename from subhub/tests/unit/stripe/charge/__init__.py rename to src/hub/vendor/__init__.py diff --git a/subhub/hub/stripe/abstract.py b/src/hub/vendor/abstract.py similarity index 92% rename from subhub/hub/stripe/abstract.py rename to src/hub/vendor/abstract.py index 79a04a3..8b5a65b 100644 --- a/subhub/hub/stripe/abstract.py +++ b/src/hub/vendor/abstract.py @@ -7,9 +7,9 @@ from abc import ABC, abstractmethod from typing import Dict from attrdict import AttrDict -from subhub.hub.routes.pipeline import RoutesPipeline -from subhub.cfg import CFG -from subhub.log import get_logger +from hub.routes.pipeline import RoutesPipeline +from shared.cfg import CFG +from shared.log import get_logger logger = get_logger() diff --git a/subhub/hub/stripe/controller.py b/src/hub/vendor/controller.py similarity index 80% rename from subhub/hub/stripe/controller.py rename to src/hub/vendor/controller.py index 4319389..32075f1 100644 --- a/subhub/hub/stripe/controller.py +++ b/src/hub/vendor/controller.py @@ -2,19 +2,21 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. -from typing import Dict, Any, Union, Iterable +import stripe from flask import request, Response +from typing import Dict, Any, Union, Iterable -import stripe -from subhub.cfg import CFG -from subhub.hub.stripe.customer import StripeCustomerCreated -from subhub.hub.stripe.customer import StripeCustomerSubscriptionCreated -from subhub.hub.stripe.customer import StripeCustomerSubscriptionUpdated -from subhub.hub.stripe.customer import StripeCustomerSubscriptionDeleted -from subhub.hub.stripe.customer import StripeCustomerSourceExpiring -from subhub.hub.stripe.invoices import StripeInvoicePaymentFailed -from subhub.log import get_logger +from shared.cfg import CFG +from shared.log import get_logger +from hub.vendor.customer import ( + StripeCustomerCreated, + StripeCustomerSubscriptionUpdated, + StripeCustomerSourceExpiring, + StripeCustomerSubscriptionCreated, + StripeCustomerSubscriptionDeleted, +) +from hub.vendor.invoices import StripeInvoicePaymentFailed logger = get_logger() @@ -49,18 +51,18 @@ def view() -> tuple: logger.info("payload type", type=type(payload)) sig_header = request.headers["Stripe-Signature"] event = stripe.Webhook.construct_event(payload, sig_header, CFG.HUB_API_KEY) - return {"message": ""}, 200 + return Response("", status=200) except ValueError as e: # Invalid payload logger.error("ValueError", error=e) - return {"message": e}, 400 + return Response(str(e), status=400) except stripe.error.SignatureVerificationError as e: # Invalid signature logger.error("SignatureVerificationError", error=e) - return {"message": e}, 400 + return Response(str(e), status=400) except Exception as e: logger.error("General Exception", error=e) - return {"message": e}, 500 + return Response(str(e), status=500) finally: pipeline = StripeHubEventPipeline(event) pipeline.run() diff --git a/subhub/hub/stripe/customer.py b/src/hub/vendor/customer.py similarity index 98% rename from subhub/hub/stripe/customer.py rename to src/hub/vendor/customer.py index 7eb844d..7680db6 100644 --- a/subhub/hub/stripe/customer.py +++ b/src/hub/vendor/customer.py @@ -4,18 +4,16 @@ import json import time +import stripe + from datetime import datetime from typing import Optional, Dict, Any -import stripe -from stripe.error import InvalidRequestError - -from subhub.hub.stripe.abstract import AbstractStripeHubEvent -from subhub.hub.routes.static import StaticRoutes -from subhub.universal import format_plan_nickname -from subhub.exceptions import ClientError - -from subhub.log import get_logger +from hub.vendor.abstract import AbstractStripeHubEvent +from hub.routes.static import StaticRoutes +from hub.shared.exceptions import ClientError +from hub.shared.universal import format_plan_nickname +from hub.shared.log import get_logger logger = get_logger() diff --git a/subhub/hub/stripe/invoices.py b/src/hub/vendor/invoices.py similarity index 87% rename from subhub/hub/stripe/invoices.py rename to src/hub/vendor/invoices.py index 4d31c45..9a3b3f1 100644 --- a/subhub/hub/stripe/invoices.py +++ b/src/hub/vendor/invoices.py @@ -7,10 +7,10 @@ import json from stripe.error import InvalidRequestError from stripe import Product -from subhub.hub.stripe.abstract import AbstractStripeHubEvent -from subhub.hub.routes.static import StaticRoutes -from subhub.universal import format_plan_nickname -from subhub.log import get_logger +from hub.vendor.abstract import AbstractStripeHubEvent +from hub.routes.static import StaticRoutes +from hub.shared.universal import format_plan_nickname +from hub.shared.log import get_logger logger = get_logger() diff --git a/src/hub/vendor/subscription.py b/src/hub/vendor/subscription.py new file mode 100644 index 0000000..15644b7 --- /dev/null +++ b/src/hub/vendor/subscription.py @@ -0,0 +1,28 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import json + +from hub.vendor.abstract import AbstractStripeHubEvent +from hub.routes.static import StaticRoutes +from hub.shared.log import get_logger + +logger = get_logger() + + +class StripeSubscriptionCreated(AbstractStripeHubEvent): + def run(self): + data = self.create_data( + customer_id=self.payload.data.object.id, + subscription_created=self.payload.data.items.data[0].created, + current_period_start=self.payload.data.current_period_start, + current_period_end=self.payload.data.current_period_end, + plan_amount=self.payload.data.plan.amount, + plan_currency=self.payload.data.plan.currency, + plan_name=self.payload.data.plan.nickname, + created=self.payload.data.object.created, + ) + logger.info("subscription created", data=data) + routes = [StaticRoutes.SALESFORCE_ROUTE] + self.send_to_routes(routes, json.dumps(data)) diff --git a/subhub/tests/unit/stripe/customer/__init__.py b/src/hub/verifications/__init__.py similarity index 100% rename from subhub/tests/unit/stripe/customer/__init__.py rename to src/hub/verifications/__init__.py diff --git a/subhub/hub/verifications/events_check.py b/src/hub/verifications/events_check.py similarity index 94% rename from subhub/hub/verifications/events_check.py rename to src/hub/verifications/events_check.py index db5c800..037800d 100644 --- a/subhub/hub/verifications/events_check.py +++ b/src/hub/verifications/events_check.py @@ -14,10 +14,10 @@ from flask import current_app from aws_xray_sdk.core import xray_recorder from aws_xray_sdk.ext.flask.middleware import XRayMiddleware -from subhub.app import create_app, g -from subhub.hub.stripe.controller import event_process -from subhub.cfg import CFG -from subhub.log import get_logger +from hub.app import create_app, g +from hub.vendor.controller import event_process +from shared.cfg import CFG +from shared.log import get_logger logger = get_logger() diff --git a/src/mypy.ini b/src/mypy.ini new file mode 100644 index 0000000..bc64796 --- /dev/null +++ b/src/mypy.ini @@ -0,0 +1,10 @@ +[mypy] +disallow_untyped_calls = True +follow_imports = normal +ignore_missing_imports = True +python_version = 3.7 +strict_optional = True +warn_no_return = False +warn_redundant_casts = True +warn_unused_ignores = True +warn_unreachable = False diff --git a/subhub/tests/unit/stripe/event/__init__.py b/src/shared/__init__.py similarity index 100% rename from subhub/tests/unit/stripe/event/__init__.py rename to src/shared/__init__.py diff --git a/subhub/authentication.py b/src/shared/authentication.py similarity index 89% rename from subhub/authentication.py rename to src/shared/authentication.py index df6646b..4add2f1 100644 --- a/subhub/authentication.py +++ b/src/shared/authentication.py @@ -2,9 +2,9 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. -from subhub import secrets -from subhub.cfg import CFG -from subhub.log import get_logger +from shared import secrets +from shared.cfg import CFG +from shared.log import get_logger logger = get_logger() diff --git a/subhub/cfg.py b/src/shared/cfg.py similarity index 97% rename from subhub/cfg.py rename to src/shared/cfg.py index cfc7aa0..b113e39 100644 --- a/subhub/cfg.py +++ b/src/shared/cfg.py @@ -59,7 +59,10 @@ def call( if nerf: return (None, "nerfed", "nerfed") process = Popen(cmd, stdout=stdout, stderr=stderr, shell=shell) - _stdout, _stderr = [stream.decode("utf-8") for stream in process.communicate()] + _stdout, _stderr = [ + stream.decode("utf-8") if stream != None else None + for stream in process.communicate() + ] exitcode = process.poll() if verbose: if _stdout: @@ -119,6 +122,7 @@ class AutoConfigPlus(AutoConfig): # pylint: disable=too-many-public-methods "stage": "INFO", "qa": "INFO", "dev": "DEBUG", + "fab": "DEBUG", }.get(self.DEPLOYED_ENV, "NOTSET") return self("LOG_LEVEL", default_level) @@ -159,6 +163,21 @@ class AutoConfigPlus(AutoConfig): # pylint: disable=too-many-public-methods return "qa" return "dev" + @property + def DEPLOYED_BY(self): + """ + DEPLOYED_BY + """ + return self("DEPLOYED_BY", f"{self.USER}@{self.HOSTNAME}") + + @property # type: ignore + @lru_cache() + def DEPLOYED_WHEN(self): + """ + DEPLOYED_WHEN + """ + return self("DEPLOYED_WHEN", datetime.utcnow().isoformat()) + @property def REVISION(self): """ @@ -419,6 +438,13 @@ class AutoConfigPlus(AutoConfig): # pylint: disable=too-many-public-methods """ return self("DEPLOY_DOMAIN", "localhost") + @property + def SRCTAR(self): + """ + SRCTAR + """ + return self("SRCTAR", ".src.tar.gz") + @property def USER(self): """ @@ -439,21 +465,6 @@ class AutoConfigPlus(AutoConfig): # pylint: disable=too-many-public-methods except: return "unknown" - @property - def DEPLOYED_BY(self): - """ - DEPLOYED_BY - """ - return self("DEPLOYED_BY", f"{self.USER}@{self.HOSTNAME}") - - @property # type: ignore - @lru_cache() - def DEPLOYED_WHEN(self): - """ - DEPLOYED_WHEN - """ - return self("DEPLOYED_WHEN", datetime.utcnow().isoformat()) - def __getattr__(self, attr): """ getattr diff --git a/subhub/db.py b/src/shared/db.py similarity index 99% rename from subhub/db.py rename to src/shared/db.py index 9647df9..162256d 100644 --- a/subhub/db.py +++ b/src/shared/db.py @@ -9,7 +9,7 @@ from pynamodb.connection import Connection from pynamodb.models import Model, DoesNotExist from pynamodb.exceptions import PutError, DeleteError -from subhub.log import get_logger +from shared.log import get_logger logger = get_logger() @@ -90,7 +90,7 @@ class SubHubAccount: self.model.Meta.table_name, hash_key=uid, range_key=None ) # Note that range key is optional return True - except DeleteError: + except DeleteError as e: logger.error("failed to remove user from db", uid=uid) return False diff --git a/subhub/sub/deployed.py b/src/shared/deployed.py similarity index 82% rename from subhub/sub/deployed.py rename to src/shared/deployed.py index b8777a0..148aa7b 100644 --- a/subhub/sub/deployed.py +++ b/src/shared/deployed.py @@ -2,9 +2,9 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. -from subhub.cfg import CFG -from subhub.sub.types import FlaskResponse -from subhub.log import get_logger +from shared.types import FlaskResponse +from shared.cfg import CFG +from shared.log import get_logger logger = get_logger() diff --git a/subhub/exceptions.py b/src/shared/exceptions.py similarity index 100% rename from subhub/exceptions.py rename to src/shared/exceptions.py diff --git a/subhub/log.py b/src/shared/log.py similarity index 98% rename from subhub/log.py rename to src/shared/log.py index e71e598..ca02325 100644 --- a/subhub/log.py +++ b/src/shared/log.py @@ -4,7 +4,7 @@ """ Usage example: - from subhub.log import get_logger + from shared.log import get_logger log = get_logger() log.info('my_event', my_key1='val 1', my_key2=5, my_key3=[1, 2, 3], my_key4={'a': 1, 'b': 2}) List of metadata keys in each log message: @@ -34,7 +34,7 @@ import structlog from typing import Any, Dict, Optional -from subhub.cfg import CFG +from shared.cfg import CFG IS_CONFIGURED = False EVENT_UUID = str(uuid.uuid4()) diff --git a/subhub/secrets.py b/src/shared/secrets.py similarity index 90% rename from subhub/secrets.py rename to src/shared/secrets.py index d47eddb..245a672 100644 --- a/subhub/secrets.py +++ b/src/shared/secrets.py @@ -9,8 +9,8 @@ import json from typing import Dict, Any -from subhub.cfg import CFG -from subhub.exceptions import SecretStringMissingError +from shared.cfg import CFG +from shared.exceptions import SecretStringMissingError def get_secret(secret_id) -> Dict[str, Any]: diff --git a/subhub/tracing.py b/src/shared/tracing.py similarity index 98% rename from subhub/tracing.py rename to src/shared/tracing.py index 9c75a6a..1f39f23 100644 --- a/subhub/tracing.py +++ b/src/shared/tracing.py @@ -2,25 +2,26 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. -import functools -import time -import cProfile -from subhub.cfg import CFG - -from functools import wraps - """ Memory Profile a function -Requires the environment variable, PYTHONTRACEMALLOC to be set prior +Requires the environment variable, PYTHONTRACEMALLOC to be set prior or you will get the following run-tine exception: `the tracemalloc module must be tracing memory allocations to take a snapshot` - + Calling syntax: @mprofiled def some_function(): pass """ +import functools +import time +import cProfile + +from functools import wraps + +from shared.cfg import CFG + def mprofiled(func): import tracemalloc diff --git a/subhub/sub/types.py b/src/shared/types.py similarity index 100% rename from subhub/sub/types.py rename to src/shared/types.py diff --git a/subhub/universal.py b/src/shared/universal.py similarity index 100% rename from subhub/universal.py rename to src/shared/universal.py diff --git a/subhub/sub/version.py b/src/shared/version.py similarity index 79% rename from subhub/sub/version.py rename to src/shared/version.py index 1b70c57..0e0357c 100644 --- a/subhub/sub/version.py +++ b/src/shared/version.py @@ -2,9 +2,9 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. -from subhub.cfg import CFG -from subhub.sub.types import FlaskResponse -from subhub.log import get_logger +from shared.types import FlaskResponse +from shared.cfg import CFG +from shared.log import get_logger logger = get_logger() diff --git a/subhub/.coveragerc b/src/sub/.coveragerc similarity index 100% rename from subhub/.coveragerc rename to src/sub/.coveragerc diff --git a/Dockerfile b/src/sub/Dockerfile similarity index 54% rename from Dockerfile rename to src/sub/Dockerfile index 65aeeb1..3b291cb 100644 --- a/Dockerfile +++ b/src/sub/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.7-alpine +FROM mozilla/subhub-base:latest MAINTAINER Stewart Henderson ARG STRIPE_API_KEY @@ -6,26 +6,24 @@ ARG AWS_ACCESS_KEY_ID ARG AWS_SECRET_ACCESS_KEY ARG LOCAL_FLASK_PORT ARG SUPPORT_API_KEY +ARG DYNALITE_PORT ENV STRIPE_API_KEY=$STRIPE_API_KEY ENV AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID ENV AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY ENV LOCAL_FLASK_PORT=$LOCAL_FLASK_PORT ENV SUPPORT_API_KEY=$SUPPORT_API_KEY +ENV PAYMENT_API_KEY=$PAYMENT_API_KEY +ENV DYNALITE_PORT=$DYNALITE_PORT ENV FLASK_ENV=development +ENV BRANCH=local +ENV REMOTE_ORIGIN_URL=git@github.com:mozilla/subhub.git +ENV REVISION=latest +ENV VERSION=latest EXPOSE $LOCAL_FLASK_PORT -RUN mkdir -p /subhub +RUN mkdir -p /subhub/sub +ADD .src.tar.gz /subhub/sub WORKDIR /subhub -COPY . /subhub -RUN apk add bash==5.0.0-r0 && \ - bin/install-packages.sh && \ - pip install -r automation_requirements.txt && \ - pip install awscli==1.16.213 -RUN yarn install -RUN addgroup -g 10001 subhub && \ - adduser -D -G subhub -h /subhub -u 10001 subhub -USER subhub -ENTRYPOINT ["doit"] -CMD ["local"] \ No newline at end of file +ENV PYTHONPATH=. diff --git a/subhub/tests/unit/stripe/invoice/__init__.py b/src/sub/__init__.py similarity index 100% rename from subhub/tests/unit/stripe/invoice/__init__.py rename to src/sub/__init__.py diff --git a/subhub/app.py b/src/sub/app.py similarity index 86% rename from subhub/app.py rename to src/sub/app.py index a427ad6..5a1a68b 100644 --- a/subhub/app.py +++ b/src/sub/app.py @@ -14,12 +14,11 @@ from flask import current_app, g, jsonify from flask_cors import CORS from flask import request -from subhub import secrets -from subhub.cfg import CFG -from subhub.exceptions import SubHubError -from subhub.db import SubHubAccount, HubEvent, SubHubDeletedAccount - -from subhub.log import get_logger +from sub.shared import secrets +from sub.shared.cfg import CFG +from sub.shared.exceptions import SubHubError +from sub.shared.db import SubHubAccount, SubHubDeletedAccount +from sub.shared.log import get_logger logger = get_logger() @@ -56,15 +55,29 @@ def server_stripe_card_error(e): return jsonify({"message": f"{e.user_message}", "code": f"{e.code}"}), 402 +def is_container() -> bool: + import requests + + try: + requests.get(f"http://dynalite:{CFG.DYNALITE_PORT}") + return True + except Exception as e: + pass + return False + + def create_app(config=None) -> connexion.FlaskApp: logger.info("creating flask app", config=config) + region = "localhost" + host = f"http://localhost:{CFG.DYNALITE_PORT}" + if is_container(): + host = f"http://dynalite:{CFG.DYNALITE_PORT}" stripe.api_key = CFG.STRIPE_API_KEY + logger.info("aws", aws=CFG.AWS_EXECUTION_ENV) if CFG.AWS_EXECUTION_ENV: region = "us-west-2" host = None - else: - region = "localhost" - host = f"http://localhost:{CFG.DYNALITE_PORT}" + logger.info("app", port=CFG.DYNALITE_PORT, table_name=CFG.DELETED_USER_TABLE) options = dict(swagger_ui=CFG.SWAGGER_UI) app = connexion.FlaskApp(__name__, specification_dir="./", options=options) @@ -78,7 +91,6 @@ def create_app(config=None) -> connexion.FlaskApp: app.app.subhub_account = SubHubAccount( table_name=CFG.USER_TABLE, region=region, host=host ) - app.app.hub_table = HubEvent(table_name=CFG.EVENT_TABLE, region=region, host=host) app.app.subhub_deleted_users = SubHubDeletedAccount( table_name=CFG.DELETED_USER_TABLE, region=region, host=host ) @@ -86,10 +98,6 @@ def create_app(config=None) -> connexion.FlaskApp: app.app.subhub_account.model.create_table( read_capacity_units=1, write_capacity_units=1, wait=True ) - if not app.app.hub_table.model.exists(): - app.app.hub_table.model.create_table( - read_capacity_units=1, write_capacity_units=1, wait=True - ) if not app.app.subhub_deleted_users.model.exists(): app.app.subhub_deleted_users.model.create_table( read_capacity_units=1, write_capacity_units=1, wait=True @@ -127,7 +135,6 @@ def create_app(config=None) -> connexion.FlaskApp: @app.app.before_request def before_request(): g.subhub_account = current_app.subhub_account - g.hub_table = current_app.hub_table g.subhub_deleted_users = current_app.subhub_deleted_users g.app_system_id = None if CFG.PROFILING_ENABLED: diff --git a/subhub/customer.py b/src/sub/customer.py similarity index 96% rename from subhub/customer.py rename to src/sub/customer.py index 207ce7c..d395fe8 100644 --- a/subhub/customer.py +++ b/src/sub/customer.py @@ -2,16 +2,16 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. -"""Customer functions""" -from typing import Union, Optional -from stripe import Customer, Subscription import stripe -from stripe.error import InvalidRequestError -from subhub.cfg import CFG -from subhub.exceptions import IntermittentError, ServerError -from subhub.db import SubHubAccount -from subhub.log import get_logger +from stripe import Customer, Subscription +from stripe.error import InvalidRequestError +from typing import Union, Optional + +from sub.shared.exceptions import IntermittentError, ServerError +from sub.shared.db import SubHubAccount +from sub.shared.cfg import CFG +from sub.shared.log import get_logger logger = get_logger() diff --git a/subhub/sub/payments.py b/src/sub/payments.py similarity index 95% rename from subhub/sub/payments.py rename to src/sub/payments.py index 6c176d6..ad1bff4 100644 --- a/subhub/sub/payments.py +++ b/src/sub/payments.py @@ -2,18 +2,18 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. +import cachetools + from datetime import datetime from typing import List, Dict, Any, Optional - -import cachetools from stripe import Charge, Customer, Invoice, Plan, Product, Subscription from flask import g -from subhub.sub.types import JsonDict, FlaskResponse, FlaskListResponse -from subhub.customer import existing_or_new_customer, has_existing_plan, fetch_customer -from subhub.log import get_logger -from subhub.universal import format_plan_nickname -from subhub.db import SubHubDeletedAccount +from sub.shared.types import JsonDict, FlaskResponse, FlaskListResponse +from sub.shared.universal import format_plan_nickname +from sub.customer import existing_or_new_customer, has_existing_plan, fetch_customer +from sub.shared.db import SubHubDeletedAccount +from sub.shared.log import get_logger logger = get_logger() @@ -322,12 +322,16 @@ def update_payment_method(uid, data) -> FlaskResponse: :return: Success or failure message. """ customer = fetch_customer(g.subhub_account, uid) + logger.info("customer", customer=customer) if not customer: return dict(message="Customer does not exist."), 404 - if customer["metadata"]["userid"] == uid: - customer.modify(customer.id, source=data["pmt_token"]) - return {"message": "Payment method updated successfully."}, 201 + metadata = customer.get("metadata") + logger.info("metadata", metadata=metadata, customer=type(customer)) + if metadata: + if metadata["userid"] == uid: + Customer.modify(customer.id, source=data["pmt_token"]) + return {"message": "Payment method updated successfully."}, 201 return dict(message="Customer mismatch."), 400 diff --git a/src/sub/shared b/src/sub/shared new file mode 120000 index 0000000..8fba6b6 --- /dev/null +++ b/src/sub/shared @@ -0,0 +1 @@ +../shared \ No newline at end of file diff --git a/subhub/swagger.yaml b/src/sub/swagger.yaml similarity index 89% rename from subhub/swagger.yaml rename to src/sub/swagger.yaml index 39f29d4..e620bde 100644 --- a/subhub/swagger.yaml +++ b/src/sub/swagger.yaml @@ -1,7 +1,7 @@ swagger: "2.0" info: - title: "SubHub" + title: "SubHub - Sub API" version: "1.0" consumes: @@ -20,21 +20,14 @@ securityDefinitions: Ops issued token. An example of the Authorization header would be: ```Authorization: Bearer 00secret00``` - x-apikeyInfoFunc: subhub.authentication.payment_auth - HubApiKey: - type: apiKey - in: header - name: Authorization - description: | - Hub validation - x-apikeyInfoFunc: subhub.authentication.hub_auth + x-apikeyInfoFunc: shared.authentication.payment_auth SupportApiKey: type: apiKey in: header name: Authorization description: | Sending application identifier - x-apikeyInfoFunc: subhub.authentication.support_auth + x-apikeyInfoFunc: shared.authentication.support_auth parameters: uidParam: in: path @@ -49,13 +42,13 @@ parameters: required: true description: Subscription ID paths: - /version: + /sub/version: get: - operationId: subhub.sub.version.get_version + operationId: shared.version.get_version tags: - Version - summary: SubHub version - description: Show Subhub version string (git desribe --abbrev=7) + summary: SubHub - Sub API Version + description: Show Subhub version string (git describe --abbrev=7) produces: - application/json responses: @@ -63,9 +56,9 @@ paths: description: Success schema: $ref: '#/definitions/Version' - /deployed: + /sub/deployed: get: - operationId: subhub.sub.deployed.get_deployed + operationId: shared.deployed.get_deployed tags: - Deployed summary: SubHub deployed @@ -77,9 +70,9 @@ paths: description: Success schema: $ref: '#/definitions/Deployed' - /support/{uid}/subscriptions: + /sub/support/{uid}/subscriptions: get: - operationId: subhub.sub.payments.support_status + operationId: sub.payments.support_status tags: - Support summary: Support view of user Subscriptions @@ -111,9 +104,9 @@ paths: $ref: '#/definitions/IntermittentError' parameters: - $ref: '#/parameters/uidParam' - /customer/{uid}/subscriptions: + /sub/customer/{uid}/subscriptions: get: - operationId: subhub.sub.payments.subscription_status + operationId: sub.payments.subscription_status tags: - Subscriptions summary: List of Subscriptions @@ -146,7 +139,7 @@ paths: parameters: - $ref: '#/parameters/uidParam' post: - operationId: subhub.sub.payments.subscribe_to_plan + operationId: sub.payments.subscribe_to_plan tags: - Subscriptions summary: Subscribe to Plan @@ -213,9 +206,9 @@ paths: type: string description: User Display Name example: Joe User - /plans: + /sub/plans: get: - operationId: subhub.sub.payments.list_all_plans + operationId: sub.payments.list_all_plans tags: - Subscriptions summary: List all Stripe Plans @@ -237,9 +230,9 @@ paths: description: Intermittent Error schema: $ref: '#/definitions/IntermittentError' - /customer/{uid}/subscriptions/{sub_id}: + /sub/customer/{uid}/subscriptions/{sub_id}: post: - operationId: subhub.sub.payments.reactivate_subscription + operationId: sub.payments.reactivate_subscription tags: - Subscriptions summary: Reactivate a Subscription @@ -272,7 +265,7 @@ paths: - $ref: '#/parameters/uidParam' - $ref: '#/parameters/subIdParam' delete: - operationId: subhub.sub.payments.cancel_subscription + operationId: sub.payments.cancel_subscription tags: - Subscriptions summary: Cancel a Subscription @@ -308,9 +301,9 @@ paths: parameters: - $ref: '#/parameters/uidParam' - $ref: '#/parameters/subIdParam' - /customer/{uid}: + /sub/customer/{uid}: get: - operationId: subhub.sub.payments.customer_update + operationId: sub.payments.customer_update tags: - Subscriptions summary: Customer Update @@ -411,7 +404,7 @@ paths: parameters: - $ref: '#/parameters/uidParam' post: - operationId: subhub.sub.payments.update_payment_method + operationId: sub.payments.update_payment_method tags: - Subscriptions summary: Update Payment Method @@ -458,7 +451,7 @@ paths: description: Pay Token. example: tok_KPte7942xySKBKyrBu11yEpf delete: - operationId: subhub.sub.payments.delete_customer + operationId: sub.payments.delete_customer tags: - Subscriptions summary: Delete a customer @@ -494,36 +487,6 @@ paths: $ref: '#/definitions/IntermittentError' parameters: - $ref: '#/parameters/uidParam' - /hub: - post: - operationId: subhub.hub.stripe.controller.view - tags: - - Hub - summary: Receives hub calls - description: Receives hub calls. - produces: - - application/json - responses: - 200: - description: Hub call received successfully. - schema: - type: object - properties: - message: - type: string - 400: - description: Error - webhook not valid. - schema: - $ref: '#/definitions/Errormessage' - 500: - description: Error - unable to receive webhook. - schema: - $ref: '#/definitions/Errormessage' - parameters: - - in: body - name: data - schema: - type: object definitions: Version: type: object diff --git a/subhub/tests/unit/stripe/payment/__init__.py b/src/sub/tests/__init__.py similarity index 100% rename from subhub/tests/unit/stripe/payment/__init__.py rename to src/sub/tests/__init__.py diff --git a/subhub/tests/bdd/customer.feature b/src/sub/tests/bdd/customer.feature similarity index 100% rename from subhub/tests/bdd/customer.feature rename to src/sub/tests/bdd/customer.feature diff --git a/subhub/tests/bdd/environment.py b/src/sub/tests/bdd/environment.py similarity index 100% rename from subhub/tests/bdd/environment.py rename to src/sub/tests/bdd/environment.py diff --git a/subhub/tests/bdd/steps/rest.py b/src/sub/tests/bdd/steps/rest.py similarity index 100% rename from subhub/tests/bdd/steps/rest.py rename to src/sub/tests/bdd/steps/rest.py diff --git a/subhub/tests/bdd/subscriptions.feature b/src/sub/tests/bdd/subscriptions.feature similarity index 100% rename from subhub/tests/bdd/subscriptions.feature rename to src/sub/tests/bdd/subscriptions.feature diff --git a/subhub/tests/bdd/version.feature b/src/sub/tests/bdd/version.feature similarity index 100% rename from subhub/tests/bdd/version.feature rename to src/sub/tests/bdd/version.feature diff --git a/src/sub/tests/conftest.py b/src/sub/tests/conftest.py new file mode 100644 index 0000000..4a64810 --- /dev/null +++ b/src/sub/tests/conftest.py @@ -0,0 +1,164 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import os +import sys +import signal +import subprocess +import uuid +import logging +import json + +from unittest.mock import Mock, MagicMock, PropertyMock + +import psutil +import pytest +import stripe + +from flask import g + +from sub import payments +from sub.app import create_app +from sub.shared.cfg import CFG +from sub.customer import create_customer + +from sub.shared.log import get_logger + +logger = get_logger() + +ddb_process = None +THIS_PATH = os.path.join(os.path.realpath(os.path.dirname(__file__))) +UID = str(uuid.uuid4()) + + +class MockCustomer: + id = None + object = "customer" + subscriptions = [{"data": "somedata"}] + + def properties(self, cls): + return [i for i in cls.__dict__.keys() if i[:1] != "_"] + + def get(self, key, default=None): + properties = self.properties(MockCustomer) + if key in properties: + return key + else: + return default + + +def get_file(filename, path=THIS_PATH, **overrides): + with open(f"{path}/unit/customer/{filename}") as f: + obj = json.load(f) + return dict(obj, **overrides) + + +def pytest_configure(): + """Called before testing begins""" + global ddb_process + for name in ("boto3", "botocore", "stripe"): + logging.getLogger(name).setLevel(logging.CRITICAL) + if os.getenv("AWS_LOCAL_DYNAMODB") is None: + os.environ["AWS_LOCAL_DYNAMODB"] = f"http://127.0.0.1:{CFG.DYNALITE_PORT}" + + # Latest boto3 now wants fake credentials around, so here we are. + os.environ["AWS_ACCESS_KEY_ID"] = "fake" + os.environ["AWS_SECRET_ACCESS_KEY"] = "fake" + os.environ["USER_TABLE"] = "users-testing" + os.environ["DELETED_USER_TABLE"] = "deleted-users-testing" + os.environ["ALLOWED_ORIGIN_SYSTEMS"] = "Test_system,Test_System,Test_System1" + sys._called_from_test = True + + # Set stripe api key + stripe.api_key = CFG.STRIPE_API_KEY + + # Locate absolute path of dynalite + dynalite = f"{CFG.REPO_ROOT}/node_modules/.bin/dynalite" + + cmd = f"{dynalite} --port {CFG.DYNALITE_PORT}" + ddb_process = subprocess.Popen( + cmd, shell=True, env=os.environ, stdout=subprocess.PIPE + ) + while 1: + line = ddb_process.stdout.readline() + if line.startswith(b"Listening"): + break + + +def pytest_unconfigure(): + del sys._called_from_test + global ddb_process + """Called after all tests run and warnings displayed""" + proc = psutil.Process(pid=ddb_process.pid) + child_procs = proc.children(recursive=True) + for p in [proc] + child_procs: + os.kill(p.pid, signal.SIGTERM) + ddb_process.wait() + + +@pytest.fixture(autouse=True, scope="module") +def app(): + app = create_app() + with app.app.app_context(): + g.subhub_account = app.app.subhub_account + g.subhub_deleted_users = app.app.subhub_deleted_users + yield app + + +@pytest.fixture() +def create_customer_for_processing(): + uid = uuid.uuid4() + customer = create_customer( + g.subhub_account, + user_id="process_customer", + source_token="tok_visa", + email="test_fixture@{}tester.com".format(uid.hex), + origin_system="Test_system", + display_name="John Tester", + ) + yield customer + + +@pytest.fixture(scope="function") +def create_subscription_for_processing(monkeypatch): + subhub_account = MagicMock() + + get_user = MagicMock() + user_id = PropertyMock(return_value=UID) + cust_id = PropertyMock(return_value="cust123") + type(get_user).user_id = user_id + type(get_user).cust_id = cust_id + + subhub_account.get_user = get_user + + customer = Mock(return_value=MockCustomer()) + none = Mock(return_value=None) + updated_customer = Mock( + return_value={ + "subscriptions": {"data": [get_file("subscription1.json")]}, + "metadata": {"userid": "process_test"}, + "id": "cust_123", + } + ) + product = Mock(return_value={"name": "Mozilla Product"}) + + monkeypatch.setattr("flask.g.subhub_account", subhub_account) + monkeypatch.setattr("sub.payments.existing_or_new_customer", customer) + monkeypatch.setattr("sub.payments.has_existing_plan", none) + monkeypatch.setattr("stripe.Subscription.create", Mock) + monkeypatch.setattr("stripe.Customer.retrieve", updated_customer) + monkeypatch.setattr("stripe.Product.retrieve", product) + + data = json.dumps( + { + "pmt_token": "tok_visa", + "plan_id": "plan_EtMcOlFMNWW4nd", + "origin_system": "Test_system", + "email": "subtest@tester.com", + "display_name": "John Tester", + } + ) + + subscription = payments.subscribe_to_plan("process_test", json.loads(data)) + yield subscription diff --git a/subhub/tests/performance/.gitignore b/src/sub/tests/performance/.gitignore similarity index 100% rename from subhub/tests/performance/.gitignore rename to src/sub/tests/performance/.gitignore diff --git a/subhub/tests/performance/README.md b/src/sub/tests/performance/README.md similarity index 76% rename from subhub/tests/performance/README.md rename to src/sub/tests/performance/README.md index b6189a8..ad2688e 100644 --- a/subhub/tests/performance/README.md +++ b/src/sub/tests/performance/README.md @@ -6,9 +6,9 @@ This directory contains the performance test framework for Subhub. After provisioning `doit` per the root directory's README.md` file: -* `doit remote_perf`: This command runs the performance tests against a deployed +* `doit remote_perf`: This command runs the performance tests against a deployed instance of the application running at the configuration value of `CFG.DEPLOY_DOMAIN`. -* `doit perf`: This command starts a local instance of the subhub application and also +* `doit perf`: This command starts a local instance of the hub.application and also instance of the performance test running against it. ## Author(s) diff --git a/subhub/tests/performance/locustfile.py b/src/sub/tests/performance/locustfile.py similarity index 100% rename from subhub/tests/performance/locustfile.py rename to src/sub/tests/performance/locustfile.py diff --git a/src/sub/tests/pylint.rc b/src/sub/tests/pylint.rc new file mode 100644 index 0000000..c183b52 --- /dev/null +++ b/src/sub/tests/pylint.rc @@ -0,0 +1,394 @@ +[MASTER] + +# Specify a configuration file. +#rcfile= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Profiled execution. +#profile=no + +# Add files or directories to the blacklist. They should be base names, not +# paths. +ignore=CVS + +# Pickle collected data for later comparisons. +persistent=yes + +# List of plugins (as comma separated values of python modules names) to load, +# usually to register additional checkers. +load-plugins= + +# Deprecated. It was used to include message's id in output. Use --msg-template +# instead. +# include-ids=no + +# Deprecated. It was used to include symbolic ids of messages in output. Use +# --msg-template instead. +# symbols=no + +# Use multiple processes to speed up Pylint. +jobs=1 + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code +extension-pkg-whitelist= + +# Allow optimization of some AST trees. This will activate a peephole AST +# optimizer, which will apply various small optimizations. For instance, it can +# be used to obtain the result of joining multiple strings with the addition +# operator. Joining a lot of strings can lead to a maximum recursion error in +# Pylint and this flag can prevent that. It has one side effect, the resulting +# AST will be different than the one from reality. +optimize-ast=no + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED +confidence= + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time. See also the "--disable" option for examples. +#enable= + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once).You can also use "--disable=all" to +# disable everything first and then reenable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use"--disable=all --enable=classes +# --disable=W" +disable=E1608,W1627,E1601,E1603,E1602,E1605,E1604,E1607,E1606,W1621,W1620,W1623,W1622,W1625,W1624,W1609,W1608,W1607,W1606,W1605,W1604,W1603,W1602,W1601,W1639,W1640,I0021,W1638,I0020,W1618,W1619,W1630,W1626,W1637,W1634,W1635,W1610,W1611,W1612,W1613,W1614,W1615,W1616,W1617,W1632,W1633,W0704,W1628,W1629,W1636,C0326,superfluous-parens,invalid-name,wrong-import-position,logging-fstring-interpolation + +[REPORTS] + +# Set the output format. Available formats are text, parseable, colorized, msvs +# (visual studio) and html. You can also give a reporter class, eg +# mypackage.mymodule.MyReporterClass. +output-format=text + +# Put messages in a separate file for each module / package specified on the +# command line instead of printing them on stdout. Reports (if any) will be +# written in a file name "pylint_global.[txt|html]". +files-output=no + +# Tells whether to display a full report or only the messages +reports=yes + +# Python expression which should return a note less than 10 (10 is the highest +# note). You have access to the variables errors warning, statement which +# respectively contain the number of errors / warnings messages and the total +# number of statements analyzed. This is used by the global evaluation report +# (RP0004). +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + +# Add a comment according to your evaluation note. This is used by the global +# evaluation report (RP0004). +#comment=no + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details +#msg-template= + + +[VARIABLES] + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# A regular expression matching the name of dummy variables (i.e. expectedly +# not used). +dummy-variables-rgx=_$|dummy + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid to define new builtins when possible. +additional-builtins= + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_,_cb + + +[TYPECHECK] + +# Tells whether missing members accessed in mixin class should be ignored. A +# mixin class is detected if its name ends with "mixin" (case insensitive). +ignore-mixin-members=yes + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis +ignored-modules=rpython,rpython.rtyper.lltypesystem.rffi + +# List of classes names for which member attributes should not be checked +# (useful for classes with attributes dynamically set). +ignored-classes=SQLObject + +# When zope mode is activated, add a predefined set of Zope acquired attributes +# to generated-members. +#zope=no + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E0201 when accessed. Python regular +# expressions are accepted. +generated-members=REQUEST,acl_users,aq_parent + + +[SPELLING] + +# Spelling dictionary name. Available dictionaries: en_US (myspell), en_GB +# (myspell), en_AG (myspell), en_AU (myspell), en_BS (myspell), en_BW +# (myspell), en_BZ (myspell), en_CA (myspell), en_DK (myspell), en_GH +# (myspell), en_HK (myspell), en_IE (myspell), en_IN (myspell), en_JM +# (myspell), en_MW (myspell), en_NA (myspell), en_NG (myspell), en_NZ +# (myspell), en_PH (myspell), en_SG (myspell), en_TT (myspell), en_ZA +# (myspell), en_ZM (myspell), en_ZW (myspell). +spelling-dict= + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to indicated private dictionary in +# --spelling-private-dict-file option instead of raising a message. +spelling-store-unknown-words=no + + +[SIMILARITIES] + +# Minimum lines number of a similarity. +min-similarity-lines=4 + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + +# Ignore imports when computing similarities. +ignore-imports=no + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME,XXX,TODO + + +[LOGGING] + +# Logging modules to check that the string format arguments are in logging +# function parameter format +logging-modules=logging + + +[FORMAT] + +# Maximum number of characters on a single line. +max-line-length=120 + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + +# List of optional constructs for which whitespace checking is disabled +no-space-check=trailing-comma,dict-separator + +# Maximum number of lines in a module +max-module-lines=1000 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= + + +[BASIC] + +# Required attributes for module, separated by a comma +#required-attributes= + +# List of builtins function names that should not be used, separated by a comma +bad-functions=map,filter,input + +# Good variable names which should always be accepted, separated by a comma +good-names=i,j,k,ex,Run,_ + +# Bad variable names which should always be refused, separated by a comma +bad-names=foo,bar,baz,toto,tutu,tata + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Include a hint for the correct naming format with invalid-name +include-naming-hint=no + +# Regular expression matching correct function names +function-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for function names +function-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct variable names +variable-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for variable names +variable-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct constant names +const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Naming hint for constant names +const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ + +# Regular expression matching correct attribute names +attr-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for attribute names +attr-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct argument names +argument-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for argument names +argument-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression matching correct class attribute names +class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ + +# Naming hint for class attribute names +class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ + +# Regular expression matching correct inline iteration names +inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ + +# Naming hint for inline iteration names +inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ + +# Regular expression matching correct class names +class-rgx=[A-Z_][a-zA-Z0-9]+$ + +# Naming hint for class names +class-name-hint=[A-Z_][a-zA-Z0-9]+$ + +# Regular expression matching correct module names +module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Naming hint for module names +module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Regular expression matching correct method names +method-rgx=[a-z_][a-z0-9_]{2,30}$ + +# Naming hint for method names +method-name-hint=[a-z_][a-z0-9_]{2,30}$ + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=__.*__ + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + + +[IMPORTS] + +# Deprecated modules which should not be used, separated by a comma +deprecated-modules=regsub,TERMIOS,Bastion,rexec + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report RP0402 must not be disabled) +import-graph= + +# Create a graph of external dependencies in the given file (report RP0402 must +# not be disabled) +ext-import-graph= + +# Create a graph of internal dependencies in the given file (report RP0402 must +# not be disabled) +int-import-graph= + + +[DESIGN] + +# Maximum number of arguments for function / method +max-args=10 + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore +ignored-argument-names=_.* + +# Maximum number of locals for function / method body +max-locals=15 + +# Maximum number of return / yield for function / method body +max-returns=6 + +# Maximum number of branch for function / method body +max-branches=12 + +# Maximum number of statements in function / method body +max-statements=50 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=1 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + + +[CLASSES] + +# List of interface methods to ignore, separated by a comma. This is used for +# instance to not check methods defines in Zope's Interface base class. +#ignore-iface-methods=isImplementedBy,deferred,extends,names,namesAndDescriptions,queryDescriptionFor,getBases,getDescriptionFor,getDoc,getName,getTaggedValue,getTaggedValueTags,isEqualOrExtendedBy,setTaggedValue,isImplementedByInstancesOf,adaptWith,is_implemented_by + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__,__new__,setUp + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=mcs + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict,_fields,_replace,_source,_make + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "Exception" +overgeneral-exceptions=Exception diff --git a/src/sub/tests/pytest.ini b/src/sub/tests/pytest.ini new file mode 100644 index 0000000..ee6e164 --- /dev/null +++ b/src/sub/tests/pytest.ini @@ -0,0 +1,16 @@ +# pytest.ini + +[pytest] +addopts = --maxfail=6 +norecursedirs = docs *.egg-info .git appdir .tox .venv env +log_format = %(asctime)s %(levelname)s %(message)s +log_date_format = %Y-%m-%d %H:%M:%S +log_level=INFO + +filterwarnings = + ignore::FutureWarning + ignore::DeprecationWarning + +[pytest-watch] +ignore = ./integration-tests .venv +nobeep = True diff --git a/src/sub/tests/unit/__init__.py b/src/sub/tests/unit/__init__.py new file mode 100644 index 0000000..448bb86 --- /dev/null +++ b/src/sub/tests/unit/__init__.py @@ -0,0 +1,3 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. diff --git a/src/sub/tests/unit/customer/__init__.py b/src/sub/tests/unit/customer/__init__.py new file mode 100644 index 0000000..448bb86 --- /dev/null +++ b/src/sub/tests/unit/customer/__init__.py @@ -0,0 +1,3 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. diff --git a/subhub/tests/unit/customer/subscription1.json b/src/sub/tests/unit/customer/subscription1.json similarity index 95% rename from subhub/tests/unit/customer/subscription1.json rename to src/sub/tests/unit/customer/subscription1.json index 907a427..ca8e19a 100644 --- a/subhub/tests/unit/customer/subscription1.json +++ b/src/sub/tests/unit/customer/subscription1.json @@ -42,7 +42,7 @@ "livemode": false, "metadata": {}, "nickname": null, - "product": "prod_BUthVRQ7KdFfa7", + "product": "prod_test1", "tiers": null, "tiers_mode": null, "transform_usage": null, @@ -59,7 +59,7 @@ }, "latest_invoice": null, "livemode": false, - "metadata": {}, + "metadata": null, "plan": { "id": "ivory-freelance-040", "object": "plan", @@ -74,7 +74,7 @@ "livemode": false, "metadata": {}, "nickname": null, - "product": "prod_BUthVRQ7KdFfa7", + "product": "prod_test1", "tiers": null, "tiers_mode": null, "transform_usage": null, @@ -88,4 +88,4 @@ "tax_percent": null, "trial_end": null, "trial_start": null -} \ No newline at end of file +} diff --git a/subhub/tests/unit/customer/subscription2.json b/src/sub/tests/unit/customer/subscription2.json similarity index 100% rename from subhub/tests/unit/customer/subscription2.json rename to src/sub/tests/unit/customer/subscription2.json diff --git a/subhub/tests/unit/customer/subscription3.json b/src/sub/tests/unit/customer/subscription3.json similarity index 100% rename from subhub/tests/unit/customer/subscription3.json rename to src/sub/tests/unit/customer/subscription3.json diff --git a/subhub/tests/unit/customer/subscription_active.json b/src/sub/tests/unit/customer/subscription_active.json similarity index 100% rename from subhub/tests/unit/customer/subscription_active.json rename to src/sub/tests/unit/customer/subscription_active.json diff --git a/subhub/tests/unit/customer/subscription_incomplete.json b/src/sub/tests/unit/customer/subscription_incomplete.json similarity index 100% rename from subhub/tests/unit/customer/subscription_incomplete.json rename to src/sub/tests/unit/customer/subscription_incomplete.json diff --git a/subhub/tests/unit/customer/test_customer_calls.py b/src/sub/tests/unit/customer/test_customer_calls.py similarity index 95% rename from subhub/tests/unit/customer/test_customer_calls.py rename to src/sub/tests/unit/customer/test_customer_calls.py index 4feefe5..dfe7443 100644 --- a/subhub/tests/unit/customer/test_customer_calls.py +++ b/src/sub/tests/unit/customer/test_customer_calls.py @@ -2,14 +2,15 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. +import os import uuid import json -from subhub.sub.payments import subscribe_to_plan, customer_update -from subhub.tests.unit.stripe.utils import MockSubhubAccount -from unittest.mock import Mock, MagicMock, PropertyMock -import os -from subhub.log import get_logger +from unittest.mock import Mock, MagicMock, PropertyMock + +from sub.payments import subscribe_to_plan, customer_update, create_update_data +from sub.tests.unit.utils import MockSubhubAccount +from sub.shared.log import get_logger logger = get_logger() @@ -68,8 +69,8 @@ def test_subscribe_to_plan_returns_newest(monkeypatch): product = Mock(return_value={"name": "Mozilla Product"}) monkeypatch.setattr("flask.g.subhub_account", subhub_account) - monkeypatch.setattr("subhub.sub.payments.existing_or_new_customer", customer) - monkeypatch.setattr("subhub.sub.payments.has_existing_plan", none) + monkeypatch.setattr("sub.payments.existing_or_new_customer", customer) + monkeypatch.setattr("sub.payments.has_existing_plan", none) monkeypatch.setattr("stripe.Subscription.create", Mock) monkeypatch.setattr("stripe.Customer.retrieve", updated_customer) monkeypatch.setattr("stripe.Product.retrieve", product) diff --git a/src/sub/tests/unit/fixtures/invalid_plan_response.json b/src/sub/tests/unit/fixtures/invalid_plan_response.json new file mode 100644 index 0000000..0f0d77e --- /dev/null +++ b/src/sub/tests/unit/fixtures/invalid_plan_response.json @@ -0,0 +1,7 @@ +[ + { + "amount": "500", + "currency": "usd", + "frequency": "month" + } +] \ No newline at end of file diff --git a/src/sub/tests/unit/fixtures/stripe_ch_test1.json b/src/sub/tests/unit/fixtures/stripe_ch_test1.json new file mode 100644 index 0000000..cbecfea --- /dev/null +++ b/src/sub/tests/unit/fixtures/stripe_ch_test1.json @@ -0,0 +1,143 @@ +{ + "id": "ch_test1", + "object": "charge", + "amount": 1000, + "amount_refunded": 0, + "application": null, + "application_fee": null, + "application_fee_amount": null, + "balance_transaction": "txn_1FCyC5JNcmPzuWtRZVJ3VRLC", + "billing_details": { + "address": { + "city": null, + "country": null, + "line1": null, + "line2": null, + "postal_code": null, + "state": null + }, + "email": null, + "name": null, + "phone": null + }, + "captured": false, + "created": 1558033578, + "currency": "usd", + "customer": "cus_F4yl2VV15h6ahh", + "description": "Payment for invoice 1D1C1268-0002", + "destination": null, + "dispute": null, + "failure_code": "card_declined", + "failure_message": "Your card was declined.", + "fraud_details": { + "stripe_report": "fraudulent" + }, + "invoice": "in_1EaouDJNcmPzuWtReWEFz6j0", + "livemode": false, + "metadata": {}, + "on_behalf_of": null, + "order": null, + "outcome": { + "network_status": "not_sent_to_network", + "reason": "merchant_blacklist", + "risk_level": "highest", + "risk_score": 76, + "seller_message": "Stripe blocked this payment.", + "type": "blocked" + }, + "paid": false, + "payment_intent": "pi_1EaouDJNcmPzuWtRSbo5ntSi", + "payment_method": "card_1EaosmJNcmPzuWtRa3ygzXId", + "payment_method_details": { + "card": { + "brand": "visa", + "checks": { + "address_line1_check": null, + "address_postal_code_check": null, + "cvc_check": "pass" + }, + "country": "US", + "exp_month": 12, + "exp_year": 2023, + "fingerprint": "bMt3PVLFsmMUsiqx", + "funding": "credit", + "last4": "0019", + "three_d_secure": { + "authenticated": false, + "succeeded": true, + "version": "1.0" + }, + "wallet": null + }, + "type": "card" + }, + "receipt_email": null, + "receipt_number": null, + "receipt_url": "https://pay.stripe.com/receipts/acct_1EJOaaJNcmPzuWtR/ch_1EaouEJNcmPzuWtRIabj08KN/rcpt_F4yrXrJVsUxcXUB932zUcgWhpkNOIDM", + "refunded": false, + "refunds": { + "object": "list", + "data": [], + "has_more": false, + "total_count": 0, + "url": "/v1/charges/ch_1EaouEJNcmPzuWtRIabj08KN/refunds" + }, + "review": null, + "shipping": null, + "source": { + "id": "src_1EaouDJNcmPzuWtRpQanmuk9", + "object": "source", + "amount": 1000, + "client_secret": "src_client_secret_F4yreVQ0YNwEjJ5TKFLQbDpe", + "created": 1558033578, + "currency": "usd", + "flow": "redirect", + "livemode": false, + "metadata": {}, + "owner": { + "address": null, + "email": null, + "name": null, + "phone": null, + "verified_address": null, + "verified_email": null, + "verified_name": null, + "verified_phone": null + }, + "redirect": { + "failure_reason": null, + "return_url": "stripejs://use_stripe_sdk/return_url", + "status": "not_required", + "url": "https://hooks.stripe.com/redirect/authenticate/src_1EaouDJNcmPzuWtRpQanmuk9?client_secret=src_client_secret_F4yreVQ0YNwEjJ5TKFLQbDpe" + }, + "statement_descriptor": null, + "status": "consumed", + "three_d_secure": { + "card": "card_1EaosmJNcmPzuWtRa3ygzXId", + "exp_month": 12, + "exp_year": 2023, + "last4": "0019", + "country": "US", + "brand": "Visa", + "cvc_check": "unavailable", + "funding": "credit", + "fingerprint": "bMt3PVLFsmMUsiqx", + "three_d_secure": "optional", + "customer": null, + "authenticated": false, + "name": null, + "address_line1_check": null, + "address_zip_check": null, + "tokenization_method": null, + "dynamic_last4": null + }, + "type": "three_d_secure", + "usage": "single_use" + }, + "source_transfer": null, + "statement_descriptor": null, + "statement_descriptor_suffix": null, + "status": "failed", + "transfer_data": null, + "transfer_group": null +} \ No newline at end of file diff --git a/src/sub/tests/unit/fixtures/stripe_cust_test1.json b/src/sub/tests/unit/fixtures/stripe_cust_test1.json new file mode 100644 index 0000000..e702a9a --- /dev/null +++ b/src/sub/tests/unit/fixtures/stripe_cust_test1.json @@ -0,0 +1,91 @@ +{ + "id": "cus_test1", + "object": "customer", + "account_balance": 0, + "address": null, + "balance": 0, + "created": 1567635511, + "currency": "usd", + "default_source": null, + "delinquent": false, + "description": null, + "discount": null, + "email": null, + "invoice_prefix": "F351DA7", + "invoice_settings": { + "custom_fields": null, + "default_payment_method": null, + "footer": null + }, + "livemode": false, + "metadata": {}, + "name": null, + "phone": null, + "preferred_locales": [], + "shipping": null, + "sources": { + "object": "list", + "data": [ + { + "funding": "credit", + "last4": "1234", + "exp_month": 12, + "exp_year": 2020 + } + ], + "has_more": false, + "total_count": 0, + "url": "/v1/customers/cus_test1/sources" + }, + "subscriptions": { + "object": "list", + "data": [ + { + "id": "sub_EtMhnV2YdEfMSS", + "object": "subscription", + "application_fee_percent": null, + "current_period_start": 1565895370, + "current_period_end": 1568573770, + "cancel_at_period_end": false, + "ended_at": null, + "plan": { + "id": "plan_EtMcOlFMNWW4nd", + "object": "plan", + "active": true, + "aggregate_usage": null, + "amount": 1000, + "amount_decimal": "1000", + "billing_scheme": "per_unit", + "created": 1555354251, + "currency": "usd", + "interval": "month", + "interval_count": 1, + "livemode": false, + "metadata": { + "service": "{\"service_id\": \"srv_001\", \"service_name\": \"Water Bottles\"}" + }, + "nickname": "Mozilla_Subscription", + "product": "prod_EtMczoDntN9YEa", + "tiers": null, + "tiers_mode": null, + "transform_usage": null, + "trial_period_days": 30, + "usage_type": "licensed" + } + } + ], + "has_more": false, + "total_count": 0, + "url": "/v1/customers/cus_test1/subscriptions" + }, + "tax_exempt": "none", + "tax_ids": { + "object": "list", + "data": [], + "has_more": false, + "total_count": 0, + "url": "/v1/customers/cus_test1/tax_ids" + }, + "tax_info": null, + "tax_info_verification": null +} diff --git a/src/sub/tests/unit/fixtures/stripe_in_test1.json b/src/sub/tests/unit/fixtures/stripe_in_test1.json new file mode 100644 index 0000000..fa1964c --- /dev/null +++ b/src/sub/tests/unit/fixtures/stripe_in_test1.json @@ -0,0 +1,94 @@ +{ + "id": "in_test1", + "object": "invoice", + "account_country": "US", + "account_name": "Mozilla Corporation", + "amount_due": 1000, + "amount_paid": 1000, + "amount_remaining": 0, + "application_fee_amount": null, + "attempt_count": 1, + "attempted": true, + "auto_advance": false, + "billing": "charge_automatically", + "billing_reason": "subscription_create", + "charge": "ch_test1", + "collection_method": "charge_automatically", + "created": 1555354567, + "currency": "usd", + "custom_fields": null, + "customer": "cus_test1", + "customer_address": null, + "customer_email": "test_fixture@tester.com", + "customer_name": null, + "customer_phone": null, + "customer_shipping": null, + "customer_tax_exempt": "none", + "customer_tax_ids": [], + "default_payment_method": null, + "default_source": null, + "default_tax_rates": [], + "description": null, + "discount": null, + "due_date": null, + "ending_balance": 0, + "footer": null, + "hosted_invoice_url": "https://pay.stripe.com/invoice/invst_ZPQW1BXsP9LEjkKf4oXxhNMeeS", + "invoice_pdf": "https://pay.stripe.com/invoice/invst_ZPQW1BXsP9LEjkKf4oXxhNMeeS/pdf", + "lines": { + "data": [ + { + "id": "sli_2182b279422cba", + "object": "line_item", + "amount": 1000, + "currency": "usd", + "description": "1 Moz-Sub × Moz_Sub (at $10.00 / month)", + "discountable": true, + "livemode": false, + "metadata": {}, + "period": { + "end": 1572905353, + "start": 1570226953 + }, + "plan": {}, + "proration": false, + "quantity": 1, + "subscription": "sub_FkbsOxUMt9qxhO", + "subscription_item": "si_FkbsnTorDMAHh3", + "tax_amounts": [], + "tax_rates": [], + "type": "subscription" + } + ], + "has_more": false, + "object": "list", + "url": "/v1/invoices/in_test1/lines" + }, + "livemode": false, + "metadata": {}, + "next_payment_attempt": null, + "number": "3B74E3D0-0001", + "paid": true, + "payment_intent": "pi_1EPZyNJNcmPzuWtR9U3SsJ4w", + "period_end": 1555354567, + "period_start": 1555354567, + "post_payment_credit_notes_amount": 0, + "pre_payment_credit_notes_amount": 0, + "receipt_number": null, + "starting_balance": 0, + "statement_descriptor": null, + "status": "paid", + "status_transitions": { + "finalized_at": 1555354567, + "marked_uncollectible_at": null, + "paid_at": 1555354568, + "voided_at": null + }, + "subscription": "sub_test2", + "subtotal": 1000, + "tax": null, + "tax_percent": null, + "total": 1000, + "total_tax_amounts": [], + "webhooks_delivered_at": 1555354569 +} \ No newline at end of file diff --git a/src/sub/tests/unit/fixtures/stripe_plan_test1.json b/src/sub/tests/unit/fixtures/stripe_plan_test1.json new file mode 100644 index 0000000..0b63475 --- /dev/null +++ b/src/sub/tests/unit/fixtures/stripe_plan_test1.json @@ -0,0 +1,22 @@ +{ + "id": "plan_test1", + "object": "plan", + "active": true, + "aggregate_usage": null, + "amount": 100, + "amount_decimal": "100", + "billing_scheme": "per_unit", + "created": 1561581476, + "currency": "usd", + "interval": "month", + "interval_count": 1, + "livemode": false, + "metadata": {}, + "nickname": "Free", + "product": "prod_test1", + "tiers": null, + "tiers_mode": null, + "transform_usage": null, + "trial_period_days": null, + "usage_type": "licensed" +} diff --git a/src/sub/tests/unit/fixtures/stripe_plan_test2.json b/src/sub/tests/unit/fixtures/stripe_plan_test2.json new file mode 100644 index 0000000..0f9c82d --- /dev/null +++ b/src/sub/tests/unit/fixtures/stripe_plan_test2.json @@ -0,0 +1,22 @@ +{ + "id": "plan_test2", + "object": "plan", + "active": true, + "aggregate_usage": null, + "amount": 1000, + "amount_decimal": "1000", + "billing_scheme": "per_unit", + "created": 1561581476, + "currency": "usd", + "interval": "year", + "interval_count": 1, + "livemode": false, + "metadata": {}, + "nickname": "Free", + "product": "prod_test1", + "tiers": null, + "tiers_mode": null, + "transform_usage": null, + "trial_period_days": null, + "usage_type": "licensed" + } \ No newline at end of file diff --git a/src/sub/tests/unit/fixtures/stripe_plan_test3.json b/src/sub/tests/unit/fixtures/stripe_plan_test3.json new file mode 100644 index 0000000..7e4d047 --- /dev/null +++ b/src/sub/tests/unit/fixtures/stripe_plan_test3.json @@ -0,0 +1,22 @@ +{ + "id": "plan_test3", + "object": "plan", + "active": true, + "aggregate_usage": null, + "amount": 1000, + "amount_decimal": "1000", + "billing_scheme": "per_unit", + "created": 1561581476, + "currency": "usd", + "interval": "year", + "interval_count": 1, + "livemode": false, + "metadata": {}, + "nickname": "Free", + "product": "prod_test2", + "tiers": null, + "tiers_mode": null, + "transform_usage": null, + "trial_period_days": null, + "usage_type": "licensed" + } \ No newline at end of file diff --git a/src/sub/tests/unit/fixtures/stripe_prod_test1.json b/src/sub/tests/unit/fixtures/stripe_prod_test1.json new file mode 100644 index 0000000..6a237dd --- /dev/null +++ b/src/sub/tests/unit/fixtures/stripe_prod_test1.json @@ -0,0 +1,21 @@ +{ + "id": "prod_test1", + "object": "product", + "active": true, + "attributes": [], + "caption": null, + "created": 1567100773, + "deactivate_on": [], + "description": null, + "images": [], + "livemode": false, + "metadata": {}, + "name": "Project Guardian", + "package_dimensions": null, + "shippable": null, + "statement_descriptor": "Firefox Guardian", + "type": "service", + "unit_label": null, + "updated": 1567100794, + "url": null +} diff --git a/src/sub/tests/unit/fixtures/stripe_prod_test2.json b/src/sub/tests/unit/fixtures/stripe_prod_test2.json new file mode 100644 index 0000000..9e171cf --- /dev/null +++ b/src/sub/tests/unit/fixtures/stripe_prod_test2.json @@ -0,0 +1,21 @@ +{ + "id": "prod_test2", + "object": "product", + "active": true, + "attributes": [], + "caption": null, + "created": 1567100773, + "deactivate_on": [], + "description": null, + "images": [], + "livemode": false, + "metadata": {}, + "name": "Firefox VPN", + "package_dimensions": null, + "shippable": null, + "statement_descriptor": "Firefox VPN", + "type": "service", + "unit_label": null, + "updated": 1567100794, + "url": null +} \ No newline at end of file diff --git a/src/sub/tests/unit/fixtures/stripe_sub_test1.json b/src/sub/tests/unit/fixtures/stripe_sub_test1.json new file mode 100644 index 0000000..a193519 --- /dev/null +++ b/src/sub/tests/unit/fixtures/stripe_sub_test1.json @@ -0,0 +1,36 @@ +{ + "id": "sub_test1", + "object": "subscription", + "application_fee_percent": null, + "billing": "charge_automatically", + "billing_cycle_anchor": 1567634953, + "billing_thresholds": null, + "cancel_at": null, + "cancel_at_period_end": false, + "canceled_at": null, + "collection_method": "charge_automatically", + "created": 1567634953, + "current_period_end": 1570226953, + "current_period_start": 1567634953, + "customer": "cus_test1", + "days_until_due": null, + "default_payment_method": null, + "default_source": null, + "default_tax_rates": [], + "discount": null, + "ended_at": null, + "items": {}, + "latest_invoice": "in_1FF6f7JNcmPzuWtRv2OaJjBO", + "livemode": false, + "metadata": {}, + "pending_setup_intent": null, + "plan": {}, + "quantity": 1, + "schedule": null, + "start": 1567634953, + "start_date": 1567634953, + "status": "active", + "tax_percent": null, + "trial_end": null, + "trial_start": null +} \ No newline at end of file diff --git a/src/sub/tests/unit/fixtures/stripe_sub_test2.json b/src/sub/tests/unit/fixtures/stripe_sub_test2.json new file mode 100644 index 0000000..930296b --- /dev/null +++ b/src/sub/tests/unit/fixtures/stripe_sub_test2.json @@ -0,0 +1,36 @@ +{ + "id": "sub_test2", + "object": "subscription", + "application_fee_percent": null, + "billing": "charge_automatically", + "billing_cycle_anchor": 1567634953, + "billing_thresholds": null, + "cancel_at": null, + "cancel_at_period_end": false, + "canceled_at": null, + "collection_method": "charge_automatically", + "created": 1567634953, + "current_period_end": 1570226953, + "current_period_start": 1567634953, + "customer": "cus_FkbsWZ82NIFGmc", + "days_until_due": null, + "default_payment_method": null, + "default_source": null, + "default_tax_rates": [], + "discount": null, + "ended_at": null, + "items": {}, + "latest_invoice": "in_test1", + "livemode": false, + "metadata": {}, + "pending_setup_intent": null, + "plan": {}, + "quantity": 1, + "schedule": null, + "start": 1567634953, + "start_date": 1567634953, + "status": "incomplete", + "tax_percent": null, + "trial_end": null, + "trial_start": null +} \ No newline at end of file diff --git a/src/sub/tests/unit/fixtures/valid_plan_response.json b/src/sub/tests/unit/fixtures/valid_plan_response.json new file mode 100644 index 0000000..7eb524d --- /dev/null +++ b/src/sub/tests/unit/fixtures/valid_plan_response.json @@ -0,0 +1,11 @@ +[ + { + "amount": 500, + "currency": "usd", + "interval": "month", + "plan_id": "pro_basic_823", + "plan_name": "Monthly Rocket Launches", + "product_id": "pro_basic", + "product_name": "Moz Sub" + } +] \ No newline at end of file diff --git a/subhub/tests/unit/test_app.py b/src/sub/tests/unit/test_app.py similarity index 86% rename from subhub/tests/unit/test_app.py rename to src/sub/tests/unit/test_app.py index 3e97505..f556635 100644 --- a/subhub/tests/unit/test_app.py +++ b/src/sub/tests/unit/test_app.py @@ -5,11 +5,11 @@ from flask import jsonify from stripe.error import AuthenticationError, CardError, StripeError -from subhub.app import create_app -from subhub.app import server_stripe_error -from subhub.app import intermittent_stripe_error -from subhub.app import server_stripe_error_with_params -from subhub.app import server_stripe_card_error +from sub.app import create_app +from sub.app import server_stripe_error +from sub.app import intermittent_stripe_error +from sub.app import server_stripe_error_with_params +from sub.app import server_stripe_card_error def test_create_app(): diff --git a/subhub/tests/unit/test_authentication.py b/src/sub/tests/unit/test_authentication.py similarity index 94% rename from subhub/tests/unit/test_authentication.py rename to src/sub/tests/unit/test_authentication.py index 997e691..4b63ae0 100644 --- a/subhub/tests/unit/test_authentication.py +++ b/src/sub/tests/unit/test_authentication.py @@ -2,8 +2,7 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. -from subhub import authentication -from subhub.cfg import CFG +from sub.shared import authentication def test_payment_auth(): diff --git a/subhub/tests/unit/test_cfg.py b/src/sub/tests/unit/test_cfg.py similarity index 98% rename from subhub/tests/unit/test_cfg.py rename to src/sub/tests/unit/test_cfg.py index ffd909a..e3f1fc3 100644 --- a/subhub/tests/unit/test_cfg.py +++ b/src/sub/tests/unit/test_cfg.py @@ -8,9 +8,9 @@ import pwd import sys import tempfile import contextlib -from subhub.cfg import CFG, call, git, NotGitRepoError, GitCommandNotFoundError -from subhub.log import get_logger +from sub.shared.cfg import CFG, call, git, NotGitRepoError, GitCommandNotFoundError +from sub.shared.log import get_logger logger = get_logger diff --git a/subhub/tests/unit/test_customer.py b/src/sub/tests/unit/test_customer.py similarity index 96% rename from subhub/tests/unit/test_customer.py rename to src/sub/tests/unit/test_customer.py index 40de352..b72ab47 100644 --- a/subhub/tests/unit/test_customer.py +++ b/src/sub/tests/unit/test_customer.py @@ -4,7 +4,7 @@ from unittest.mock import MagicMock, PropertyMock -from subhub.customer import fetch_customer +from sub.customer import fetch_customer def test_fetch_customer_no_account(monkeypatch): diff --git a/subhub/tests/unit/test_deployed.py b/src/sub/tests/unit/test_deployed.py similarity index 68% rename from subhub/tests/unit/test_deployed.py rename to src/sub/tests/unit/test_deployed.py index c451a18..0af09dc 100644 --- a/subhub/tests/unit/test_deployed.py +++ b/src/sub/tests/unit/test_deployed.py @@ -2,8 +2,8 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. -from subhub.cfg import CFG -from subhub.sub.deployed import get_deployed +from sub.shared.cfg import CFG +from sub.shared.deployed import get_deployed def test_get_deployed(): @@ -15,4 +15,5 @@ def test_get_deployed(): DEPLOYED_ENV=CFG.DEPLOYED_ENV, DEPLOYED_WHEN=CFG.DEPLOYED_WHEN, ) - assert get_deployed() == (deployed, 200) + current_deployed = get_deployed() + assert current_deployed[0]["DEPLOYED_BY"] == deployed["DEPLOYED_BY"] diff --git a/subhub/tests/unit/test_exceptions.py b/src/sub/tests/unit/test_exceptions.py similarity index 94% rename from subhub/tests/unit/test_exceptions.py rename to src/sub/tests/unit/test_exceptions.py index 7237612..ab1ea40 100644 --- a/subhub/tests/unit/test_exceptions.py +++ b/src/sub/tests/unit/test_exceptions.py @@ -2,8 +2,13 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. -from subhub.exceptions import SubHubError, IntermittentError, ClientError, ServerError -from subhub.exceptions import SecretStringMissingError +from sub.shared.exceptions import ( + SubHubError, + IntermittentError, + ClientError, + ServerError, + SecretStringMissingError, +) def test_subhub_error(): diff --git a/subhub/tests/unit/test_payment_calls.py b/src/sub/tests/unit/test_payment_calls.py similarity index 92% rename from subhub/tests/unit/test_payment_calls.py rename to src/sub/tests/unit/test_payment_calls.py index 08fd088..2f2700c 100644 --- a/subhub/tests/unit/test_payment_calls.py +++ b/src/sub/tests/unit/test_payment_calls.py @@ -2,27 +2,24 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. +import os import json -from mock import patch import pytest from flask import g -from stripe.error import InvalidRequestError +from mock import patch from stripe import Customer +from stripe.error import InvalidRequestError from unittest.mock import Mock, MagicMock, PropertyMock from mockito import when, mock -from subhub.sub import payments -from subhub.customer import ( - create_customer, - subscribe_customer, - existing_or_new_customer, -) -from subhub.tests.unit.stripe.utils import MockSubhubUser -from subhub.log import get_logger - +from sub import payments +from sub.customer import create_customer, subscribe_customer, existing_or_new_customer +from sub.tests.unit.utils import MockSubhubUser +from sub.shared.log import get_logger logger = get_logger() +THIS_PATH = os.path.join(os.path.realpath(os.path.dirname(__file__))) class MockCustomer: @@ -46,6 +43,12 @@ class MockCustomer: yield "subscriptions", self.subscriptions +def get_file(filename, path=THIS_PATH, **overrides): + with open(f"{path}/customer/{filename}") as f: + obj = json.load(f) + return dict(obj, **overrides) + + def test_create_customer_invalid_origin_system(monkeypatch): """ GIVEN create a stripe customer @@ -274,11 +277,11 @@ def test_subscribe_customer_existing(app, monkeypatch): monkeypatch.setattr("stripe.Plan.list", plans) monkeypatch.setattr("stripe.Product.retrieve", product) - monkeypatch.setattr("subhub.sub.payments.has_existing_plan", mock_true) + monkeypatch.setattr("sub.payments.has_existing_plan", mock_true) monkeypatch.setattr("flask.g.subhub_account", subhub_account) monkeypatch.setattr("stripe.Customer.retrieve", stripe_customer) - path = "v1/customer/user123/subscriptions" + path = "v1/sub/customer/user123/subscriptions" data = { "pmt_token": "tok_visa", "plan_id": "plan_123", @@ -341,8 +344,8 @@ def test_cancel_subscription_no_subscription_found(monkeypatch): ) m.setattr("flask.g.subhub_account.get_user", user) m.setattr("stripe.Customer.retrieve", customer) - m.setattr("subhub.customer.fetch_customer", customer) - m.setattr("subhub.sub.payments.retrieve_stripe_subscriptions", cancel_response) + m.setattr("sub.customer.fetch_customer", customer) + m.setattr("sub.payments.retrieve_stripe_subscriptions", cancel_response) cancel_sub, code = payments.cancel_subscription("123", "sub_123") logger.info("cancel sub", cancel_sub=cancel_sub) assert "Subscription not available." in cancel_sub["message"] @@ -356,12 +359,40 @@ def test_cancel_subscription_without_valid_user(monkeypatch): THEN return the appropriate message """ customer = Mock(return_value=None) - monkeypatch.setattr("subhub.customer.fetch_customer", customer) + monkeypatch.setattr("sub.customer.fetch_customer", customer) cancel_sub, code = payments.cancel_subscription("bob_123", "sub_123") assert "Customer does not exist." in cancel_sub["message"] assert code == 404 +def test_delete_user_from_db(app, create_subscription_for_processing): + """ + GIVEN should delete user from user table + WHEN provided with a valid user id + THEN add to deleted users table + """ + deleted_user = payments.delete_user("process_test", "sub_id", "origin") + logger.info("deleted user from db", deleted_user=deleted_user) + assert isinstance(deleted_user, MagicMock) + + +def test_delete_user_from_db2(app, create_subscription_for_processing, monkeypatch): + """ + GIVEN raise DeleteError + WHEN an entry cannot be removed from the database + THEN validate error message + """ + subhub_account = MagicMock() + subhub_account.remove_from_db.return_value = False + + delete_error = False + monkeypatch.setattr("flask.g.subhub_account", subhub_account) + monkeypatch.setattr("sub.shared.db.SubHubAccount.remove_from_db", delete_error) + + du = payments.delete_user("process_test_2", "sub_id", "origin") + assert du is False + + def test_add_user_to_deleted_users_record(monkeypatch): """ GIVEN Add user to deleted users record @@ -408,47 +439,20 @@ def test_update_payment_method_missing_stripe_customer(monkeypatch): subhub_account = MagicMock() get_user = MagicMock() - user_id = PropertyMock(return_value="process_test") + user_id = PropertyMock(return_value=None) cust_id = PropertyMock(return_value=None) type(get_user).user_id = user_id type(get_user).cust_id = cust_id subhub_account.get_user = get_user + monkeypatch.setattr("sub.customer.fetch_customer", subhub_account) updated_pmt, code = payments.update_payment_method( - "process_test", {"pmt_token": "tok_invalid"} + "process_test1", {"pmt_token": "tok_invalid"} ) assert 404 == code -def test_update_payment_method_invalid_stripe_customer( - app, create_subscription_for_processing -): - """ - GIVEN api_token, userid, pmt_token - WHEN provided invalid stripe data - THEN a StripeError is raised - """ - - subscription, code = create_subscription_for_processing - subhub_user = g.subhub_account.get_user("process_test") - subhub_user.cust_id = "bad_id" - g.subhub_account.save_user(subhub_user) - - exception = None - try: - updated_pmt, code = payments.update_payment_method( - "process_test", {"pmt_token": "tok_invalid"} - ) - except Exception as e: - exception = e - - g.subhub_account.remove_from_db("process_test") - - assert isinstance(exception, InvalidRequestError) - assert "No such customer:" in exception.user_message - - def test_customer_update_success(monkeypatch): subhub_account = MagicMock() diff --git a/subhub/tests/unit/test_payments_mock.py b/src/sub/tests/unit/test_payments_mock.py similarity index 97% rename from subhub/tests/unit/test_payments_mock.py rename to src/sub/tests/unit/test_payments_mock.py index cc0521d..3771377 100644 --- a/subhub/tests/unit/test_payments_mock.py +++ b/src/sub/tests/unit/test_payments_mock.py @@ -7,8 +7,8 @@ import pytest from mockito import when, mock, unstub, ANY -from subhub.sub import payments -from subhub.log import get_logger +from sub import payments +from sub.shared.log import get_logger logger = get_logger() diff --git a/subhub/tests/unit/test_secrets.py b/src/sub/tests/unit/test_secrets.py similarity index 91% rename from subhub/tests/unit/test_secrets.py rename to src/sub/tests/unit/test_secrets.py index 92e16ef..3b11790 100644 --- a/subhub/tests/unit/test_secrets.py +++ b/src/sub/tests/unit/test_secrets.py @@ -5,11 +5,10 @@ import json import boto3 -from mockito import when, mock, unstub +from mockito import when -from subhub import secrets -from subhub.cfg import CFG -from subhub.exceptions import SecretStringMissingError +from sub.shared.exceptions import SecretStringMissingError +from sub.shared import secrets EXPECTED = { diff --git a/subhub/tests/unit/test_subhub.py b/src/sub/tests/unit/test_subhub.py similarity index 86% rename from subhub/tests/unit/test_subhub.py rename to src/sub/tests/unit/test_subhub.py index 5150d78..d37ed43 100644 --- a/subhub/tests/unit/test_subhub.py +++ b/src/sub/tests/unit/test_subhub.py @@ -10,10 +10,10 @@ import connexion import stripe.error from stripe.util import convert_to_stripe_object -from subhub.app import create_app -from subhub.tests.unit.stripe.utils import MockSubhubUser +from sub.app import create_app +from sub.tests.unit.utils import MockSubhubUser -from subhub.log import get_logger +from sub.shared.log import get_logger logger = get_logger() @@ -67,7 +67,7 @@ def test_list_plans(mock_plans, mock_product, app): client = app.app.test_client() - path = "v1/plans" + path = "v1/sub/plans" response = client.get( path, @@ -80,7 +80,7 @@ def test_list_plans(mock_plans, mock_product, app): def test_update_customer_payment_server_stripe_error_with_params(app, monkeypatch): """ - GIVEN the route POST v1/customer/{id} is called + GIVEN the route POST v1/sub/customer/{id} is called WHEN the payment token provided is invalid THEN the StripeError should be handled by the app errorhandler """ @@ -98,7 +98,7 @@ def test_update_customer_payment_server_stripe_error_with_params(app, monkeypatc monkeypatch.setattr("flask.g.subhub_account.get_user", user) monkeypatch.setattr("stripe.Customer.retrieve", retrieve) - path = "v1/customer/123" + path = "v1/sub/customer/123" data = {"pmt_token": "token"} response = client.post( @@ -109,14 +109,13 @@ def test_update_customer_payment_server_stripe_error_with_params(app, monkeypatc ) data = json.loads(response.data) - assert response.status_code == 500 assert data["message"] == "Customer instance has invalid ID" def test_customer_signup_server_stripe_error_with_params(app, monkeypatch): """ - GIVEN the route POST v1/customer/{id}/subcriptions is called + GIVEN the route POST v1/sub/customer/{id}/subcriptions is called WHEN the plan id provided is invalid THEN the StripeError should be handled by the app errorhandler """ @@ -130,11 +129,11 @@ def test_customer_signup_server_stripe_error_with_params(app, monkeypatch): message="No such plan: invalid", param="plan_id", code="invalid_plan" ) ) - monkeypatch.setattr("subhub.sub.payments.has_existing_plan", none) - monkeypatch.setattr("subhub.sub.payments.existing_or_new_customer", customer) + monkeypatch.setattr("sub.payments.has_existing_plan", none) + monkeypatch.setattr("sub.payments.existing_or_new_customer", customer) monkeypatch.setattr("stripe.Subscription.create", create) - path = "v1/customer/process_test/subscriptions" + path = "v1/sub/customer/process_test/subscriptions" data = { "pmt_token": "tok_visa", "plan_id": "invalid", @@ -157,11 +156,11 @@ def test_customer_signup_server_stripe_error_with_params(app, monkeypatch): @mock.patch("stripe.Product.retrieve") -@mock.patch("subhub.sub.payments.find_newest_subscription") -@mock.patch("subhub.sub.payments.fetch_customer") +@mock.patch("sub.payments.find_newest_subscription") +@mock.patch("sub.payments.fetch_customer") @mock.patch("stripe.Subscription.create") -@mock.patch("subhub.sub.payments.has_existing_plan") -@mock.patch("subhub.sub.payments.existing_or_new_customer") +@mock.patch("sub.payments.has_existing_plan") +@mock.patch("sub.payments.existing_or_new_customer") def test_subscribe_success( mock_new_customer, mock_has_plan, @@ -201,7 +200,7 @@ def test_subscribe_success( fh.close() mock_product.return_value = prod_test1 - path = "v1/customer/subtest/subscriptions" + path = "v1/sub/customer/subtest/subscriptions" data = { "pmt_token": "tok_visa", "plan_id": "plan", @@ -231,9 +230,9 @@ def test_subscribe_customer_existing(app, monkeypatch): mock_true = Mock(return_value=True) - monkeypatch.setattr("subhub.sub.payments.has_existing_plan", mock_true) + monkeypatch.setattr("sub.payments.has_existing_plan", mock_true) - path = "v1/customer/subtest/subscriptions" + path = "v1/sub/customer/subtest/subscriptions" data = { "pmt_token": "tok_visa", "plan_id": "plan", @@ -269,11 +268,11 @@ def test_subscribe_card_declined_error_handler(app, monkeypatch): message="card declined", param="", code="generic_decline" ) ) - monkeypatch.setattr("subhub.sub.payments.has_existing_plan", none) - monkeypatch.setattr("subhub.sub.payments.existing_or_new_customer", customer) + monkeypatch.setattr("sub.payments.has_existing_plan", none) + monkeypatch.setattr("sub.payments.existing_or_new_customer", customer) monkeypatch.setattr("stripe.Subscription.create", create) - path = "v1/customer/subtest/subscriptions" + path = "v1/sub/customer/subtest/subscriptions" data = { "pmt_token": "tok_visa", "plan_id": "plan", @@ -294,7 +293,7 @@ def test_subscribe_card_declined_error_handler(app, monkeypatch): def test_customer_unsubscribe_server_stripe_error_with_params(app, monkeypatch): """ - GIVEN the route DELETE v1/customer/{id}/subcriptions/{sub_id} is called + GIVEN the route DELETE v1/sub/customer/{id}/subcriptions/{sub_id} is called WHEN the stripe customer id on the user object is invalid THEN the StripeError should be handled by the app errorhandler """ @@ -312,7 +311,7 @@ def test_customer_unsubscribe_server_stripe_error_with_params(app, monkeypatch): monkeypatch.setattr("flask.g.subhub_account.get_user", subhub_user) monkeypatch.setattr("stripe.Customer.retrieve", retrieve) - path = f"v1/customer/testuser/subscriptions/sub_123" + path = f"v1/sub/customer/testuser/subscriptions/sub_123" response = client.delete(path, headers={"Authorization": "fake_payment_api_key"}) @@ -322,7 +321,7 @@ def test_customer_unsubscribe_server_stripe_error_with_params(app, monkeypatch): assert "Customer instance has invalid ID" in data["message"] -@mock.patch("subhub.sub.payments._get_all_plans") +@mock.patch("sub.payments._get_all_plans") def test_plan_response_valid(mock_plans, app): fh = open("tests/unit/fixtures/valid_plan_response.json") valid_response = json.loads(fh.read()) @@ -332,7 +331,7 @@ def test_plan_response_valid(mock_plans, app): client = app.app.test_client() - path = "v1/plans" + path = "v1/sub/plans" response = client.get( path, @@ -343,7 +342,7 @@ def test_plan_response_valid(mock_plans, app): assert response.status_code == 200 -@mock.patch("subhub.sub.payments._get_all_plans") +@mock.patch("sub.payments._get_all_plans") def test_plan_response_invalid(mock_plans, app): fh = open("tests/unit/fixtures/invalid_plan_response.json") invalid_response = json.loads(fh.read()) @@ -353,7 +352,7 @@ def test_plan_response_invalid(mock_plans, app): client = app.app.test_client() - path = "v1/plans" + path = "v1/sub/plans" response = client.get( path, diff --git a/subhub/tests/unit/test_tracing.py b/src/sub/tests/unit/test_tracing.py similarity index 95% rename from subhub/tests/unit/test_tracing.py rename to src/sub/tests/unit/test_tracing.py index aa3a57f..f829fad 100644 --- a/subhub/tests/unit/test_tracing.py +++ b/src/sub/tests/unit/test_tracing.py @@ -4,9 +4,10 @@ import functools -from subhub.tracing import timed, mprofiled from random import randint, randrange +from sub.shared.tracing import timed, mprofiled + @timed def generate_random_value(): diff --git a/subhub/tests/unit/test_version.py b/src/sub/tests/unit/test_version.py similarity index 84% rename from subhub/tests/unit/test_version.py rename to src/sub/tests/unit/test_version.py index 7af9210..285199a 100644 --- a/subhub/tests/unit/test_version.py +++ b/src/sub/tests/unit/test_version.py @@ -2,8 +2,8 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. -from subhub.cfg import CFG -from subhub.sub.version import get_version +from sub.shared.version import get_version +from sub.shared.cfg import CFG def test_get_version(): diff --git a/src/sub/tests/unit/utils.py b/src/sub/tests/unit/utils.py new file mode 100644 index 0000000..c094b41 --- /dev/null +++ b/src/sub/tests/unit/utils.py @@ -0,0 +1,39 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import os +import json + +from sub.shared.cfg import CFG +from sub.shared import secrets + +__location__ = os.path.realpath(os.path.dirname(__file__)) + + +class MockSqsClient: + @staticmethod + def list_queues(QueueNamePrefix={}): + return {"QueueUrls": ["DevSub"]} + + @staticmethod + def send_message(QueueUrl={}, MessageBody={}): + return {"ResponseMetadata": {"HTTPStatusCode": 200}} + + +class MockSnsClient: + @staticmethod + def publish( + Message: dict = None, MessageStructure: str = "json", TopicArn: str = None + ): + return {"ResponseMetadata": {"HTTPStatusCode": 200}} + + +class MockSubhubAccount: + def subhub_account(self): + pass + + +class MockSubhubUser: + id = "123" + cust_id = "cust_123" diff --git a/subhub/tests/requirements.txt b/src/test_requirements.txt similarity index 70% rename from subhub/tests/requirements.txt rename to src/test_requirements.txt index 6d0e642..dd28e10 100644 --- a/subhub/tests/requirements.txt +++ b/src/test_requirements.txt @@ -1,5 +1,5 @@ # requirements for subhub testing to run -# do not add requirements for subhub, subhub/requirements.txt already handles that +# do not add requirements for subhub, src/app_requirements.txt already handles that mockito==1.1.1 pytest==5.0.1 pytest-cov==2.7.1 diff --git a/subhub/tests/performance/payment_api_keys b/subhub/tests/performance/payment_api_keys deleted file mode 100644 index e69de29..0000000 diff --git a/subhub/tests/performance/uids b/subhub/tests/performance/uids deleted file mode 100644 index e69de29..0000000 diff --git a/tox.ini b/tox.ini index 6a9abe6..0d169eb 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] minversion = 3.5.0 -envlist = py37 +envlist = py37-{sub,hub} skipsdist=true [testenv] @@ -10,7 +10,9 @@ run_before = export AWS_XRAY_SDK_ENABLED=false envdir = {toxinidir}/venv -changedir = {toxinidir}/subhub +changedir = + sub: {toxinidir}/src/sub + hub: {toxinidir}/src/hub passenv = STRIPE_API_KEY USER_TABLE @@ -22,11 +24,13 @@ passenv = AWS_REGION FXA_SQS_URI deps = - -r subhub/tests/requirements.txt + -r src/test_requirements.txt .[test] tox-run-before -commands = pytest --cov=subhub --cov-report term-missing --capture=no {posargs} +commands = + sub: pytest --cov=sub --cov-report term-missing --capture=no {posargs} + hub: pytest --cov=hub --cov-report term-missing --capture=no {posargs} [pytest] norecursedirs = docs *.egg-info .git appdir .tox .venv env diff --git a/yarn.lock b/yarn.lock index 749cbef..64e54d4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -604,7 +604,7 @@ async-limiter@^1.0.0: resolved "https://registry.yarnpkg.com/async-limiter/-/async-limiter-1.0.1.tgz#dd379e94f0db8310b08291f9d64c3209766617fd" integrity sha512-csOlWGAcRFJaI6m+F2WKdnMKr4HhdhFVBk0H/QbJFMCr+uO2kwohwXQPxw/9OCxp05r5ghVBFSyioixx3gfkNQ== -async@^1.5.2: +async@^1.5.2, async@~1.5: version "1.5.2" resolved "https://registry.yarnpkg.com/async/-/async-1.5.2.tgz#ec6a61ae56480c0c3cb241c95618e20892f9672a" integrity sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo= @@ -1886,6 +1886,15 @@ fs-extra@^7.0.0, fs-extra@^7.0.1: jsonfile "^4.0.0" universalify "^0.1.0" +fs-extra@^8.1.0: + version "8.1.0" + resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-8.1.0.tgz#49d43c45a88cd9677668cb7be1b46efdb8d2e1c0" + integrity sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g== + dependencies: + graceful-fs "^4.2.0" + jsonfile "^4.0.0" + universalify "^0.1.0" + fs.realpath@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" @@ -2175,6 +2184,13 @@ has@^1.0.1, has@^1.0.3: dependencies: function-bind "^1.1.1" +hasbin@^1.2.3: + version "1.2.3" + resolved "https://registry.yarnpkg.com/hasbin/-/hasbin-1.2.3.tgz#78c5926893c80215c2b568ae1fd3fcab7a2696b0" + integrity sha1-eMWSaJPIAhXCtWiuH9P8q3omlrA= + dependencies: + async "~1.5" + http-cache-semantics@^4.0.0: version "4.0.3" resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-4.0.3.tgz#495704773277eeef6e43f9ab2c2c7d259dda25c5" @@ -3945,6 +3961,14 @@ serverless-offline@5.10.1: update-notifier "^3.0.1" velocityjs "^1.1.3" +serverless-package-external@1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/serverless-package-external/-/serverless-package-external-1.1.1.tgz#46c268ad1b41bfc44cf15fc3d070d1cc98147518" + integrity sha512-9DnMWkafTua4a6YNwKfxq8m/JkVaUyPFCAdokp6NyZRh7djVrBmysyYf5Or/vF3eogcmRayBzT7SYa8AAjlYsQ== + dependencies: + rimraf "^2.6.2" + yesno "0.0.1" + serverless-plugin-tracing@2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/serverless-plugin-tracing/-/serverless-plugin-tracing-2.0.0.tgz#df6b8b3166ac9bb70a37c7fc875014b2369158f6" @@ -3970,6 +3994,16 @@ serverless-python-requirements@5.0.0: sha256-file "1.0.0" shell-quote "^1.6.1" +serverless-wsgi@1.7.3: + version "1.7.3" + resolved "https://registry.yarnpkg.com/serverless-wsgi/-/serverless-wsgi-1.7.3.tgz#34c7989718a3cb52a9148bdba8d7d6c2da0f0328" + integrity sha512-LKzAbLBFam9m7Iu9t5/Oa0AVfd9qRDRNKkrG35h2enlourDebooia3p638bu0nCOqs4VTQ2IqdBlv8devjFYGQ== + dependencies: + bluebird "^3.5.5" + fs-extra "^8.1.0" + hasbin "^1.2.3" + lodash "^4.17.15" + serverless@1.49.0: version "1.49.0" resolved "https://registry.yarnpkg.com/serverless/-/serverless-1.49.0.tgz#a4c8353c70db6dfe65018bb5e43ea5f44bc43795" @@ -4791,6 +4825,11 @@ yauzl@^2.4.2: buffer-crc32 "~0.2.3" fd-slicer "~1.1.0" +yesno@0.0.1: + version "0.0.1" + resolved "https://registry.yarnpkg.com/yesno/-/yesno-0.0.1.tgz#ffbc04ff3d6f99dad24f7463134e9b92ae41bef6" + integrity sha1-/7wE/z1vmdrST3RjE06bkq5BvvY= + zip-stream@^1.1.0: version "1.2.0" resolved "https://registry.yarnpkg.com/zip-stream/-/zip-stream-1.2.0.tgz#a8bc45f4c1b49699c6b90198baacaacdbcd4ba04"