зеркало из https://github.com/mozilla/treeherder.git
Test integrate black (#6277)
* Configuration for black * changes congiguration * change pyproject's directory * add files to be excluded and skip string normalization * removed isort from pre-commit * remove version locks for black * fix * remove all isort * update requirements Co-authored-by: SuyashSalampuria <suyash546@gmail.com> Co-authored-by: Kyle Lahnakoski <kyle@lahnakoski.com>
This commit is contained in:
Kyle Lahnakoski
GitHub
Родитель
775e4b679e
Коммит
53a806d3cc
|
|
@ -3,15 +3,6 @@ repos:
|
|||
rev: 3.7.9
|
||||
hooks:
|
||||
- id: flake8
|
||||
- repo: https://github.com/asottile/seed-isort-config
|
||||
rev: v2.1.0
|
||||
hooks:
|
||||
- id: seed-isort-config
|
||||
- repo: https://github.com/timothycrosley/isort
|
||||
rev: 4.3.21
|
||||
hooks:
|
||||
- id: isort
|
||||
args: ['--virtual-env=.venv']
|
||||
- repo: https://github.com/ryanrhee/shellcheck-py
|
||||
rev: v0.7.0.1-1
|
||||
hooks:
|
||||
|
|
@ -24,3 +15,8 @@ repos:
|
|||
rev: v0.22.0
|
||||
hooks:
|
||||
- id: markdownlint
|
||||
- repo: https://github.com/psf/black
|
||||
rev: stable
|
||||
hooks:
|
||||
- id: black
|
||||
language_version: python3.7
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@
|
|||
[](https://david-dm.org/mozilla/treeherder)
|
||||
[](https://david-dm.org/mozilla/treeherder?type=dev)
|
||||
[](https://treeherder.readthedocs.io/?badge=latest)
|
||||
[](https://github.com/psf/black)
|
||||
|
||||
## Description
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
## Running the tests
|
||||
|
||||
You can run flake8, isort and the pytest suite inside Docker, using:
|
||||
You can run flake8 and the pytest suite inside Docker, using:
|
||||
|
||||
```bash
|
||||
docker-compose run backend ./runtests.sh
|
||||
|
|
@ -54,20 +54,6 @@ Then run the individual tools within that shell, like so:
|
|||
flake8
|
||||
```
|
||||
|
||||
- [isort](https://github.com/timothycrosley/isort) (checks the [Python import style](code_style.md#python-imports)):
|
||||
|
||||
To run interactively:
|
||||
|
||||
```bash
|
||||
isort
|
||||
```
|
||||
|
||||
Or to apply all changes without confirmation:
|
||||
|
||||
```bash
|
||||
isort --apply
|
||||
```
|
||||
|
||||
## Hide Jobs with Tiers
|
||||
|
||||
To hide jobs we use the job's `tier` setting. Jobs with `tier` of 3 are
|
||||
|
|
|
|||
|
|
@ -1,34 +1,5 @@
|
|||
# Code Style
|
||||
|
||||
## Python imports
|
||||
|
||||
[isort](https://github.com/timothycrosley/isort) enforces the following Python global import order:
|
||||
|
||||
- `from __future__ import ...`
|
||||
- Python standard library
|
||||
- Third party modules
|
||||
- Local project imports (absolutely specified)
|
||||
- Local project imports (relative path, eg: `from .models import Credentials`)
|
||||
|
||||
In addition:
|
||||
|
||||
- Each group should be separated by a blank line.
|
||||
- Within each group, all `import ...` statements should be before `from ... import ...`.
|
||||
- After that, sort alphabetically by module name.
|
||||
- When importing multiple items from one module, use this style:
|
||||
|
||||
```python
|
||||
from django.db import (models,
|
||||
transaction)
|
||||
```
|
||||
|
||||
The quickest way to correct import style locally is to let isort make the changes for you - see
|
||||
[running the tests](common_tasks.md#running-the-tests).
|
||||
|
||||
Note: It's not possible to disable isort wrapping style checking, so for now we've chosen the
|
||||
most deterministic [wrapping mode](https://github.com/timothycrosley/isort#multi-line-output-modes)
|
||||
to reduce the line length guess-work when adding imports, even though it's not the most concise.
|
||||
|
||||
## UI
|
||||
|
||||
We use Prettier for JS/JSX formatting and the [Airbnb](https://github.com/airbnb/javascript)
|
||||
|
|
|
|||
|
|
@ -10,4 +10,4 @@ python = "^3.7"
|
|||
[tool.poetry.dev-dependencies]
|
||||
mkdocs = "1.0.4"
|
||||
mkdocs-material = "4.6.3"
|
||||
mdx_truly_sane_lists = "1.2"
|
||||
mdx_truly_sane_lists = "1.2"
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
[tool.black]
|
||||
line-length = 100
|
||||
target-version = ['py37']
|
||||
skip-string-normalization = true
|
||||
include = '\.pyi?$'
|
||||
exclude = '''
|
||||
/(
|
||||
treeherder/model/migrations
|
||||
| treeherder/perf/migrations
|
||||
| treeherder/changelog/migrations
|
||||
)/
|
||||
'''
|
||||
|
|
@ -9,12 +9,11 @@ PyPOM
|
|||
|
||||
# for git commit hooks
|
||||
pre-commit
|
||||
seed-isort-config
|
||||
|
||||
# Required by django-extension's runserver_plus command.
|
||||
flake8
|
||||
isort
|
||||
pytest-django
|
||||
black
|
||||
|
||||
# To test async code
|
||||
pytest-asyncio
|
||||
|
|
|
|||
|
|
@ -7,15 +7,11 @@
|
|||
appdirs==1.4.3 \
|
||||
--hash=sha256:9e5896d1372858f8dd3344faf4e5014d21849c756c8d5701f78f8a103b372d92 \
|
||||
--hash=sha256:d8b24664561d0d34ddfaec54636d502d7cea6e29c3eaf68f3df6180863e2166e \
|
||||
# via virtualenv
|
||||
# via black, virtualenv
|
||||
asgiref==3.2.7 \
|
||||
--hash=sha256:8036f90603c54e93521e5777b2b9a39ba1bad05773fcf2d208f0299d1df58ce5 \
|
||||
--hash=sha256:9ca8b952a0a9afa61d30aa6d3d9b570bb3fd6bafcf7ec9e6bed43b936133db1c \
|
||||
# via django
|
||||
aspy.refactor-imports==2.1.0 \
|
||||
--hash=sha256:4a1e75cdbe1502c898222f7ebf7d8a2d9649978715364364a066328d11c60450 \
|
||||
--hash=sha256:e8dd8a3ab3fd162fac539e752ecd9a609870254cddaaaa4434b0825bd8594857 \
|
||||
# via seed-isort-config
|
||||
atomicwrites==1.3.0 \
|
||||
--hash=sha256:03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4 \
|
||||
--hash=sha256:75a9445bac02d8d058d5e1fe689654ba5a6556a1dfd8ce6ec55a0ed79866cfa6 \
|
||||
|
|
@ -23,11 +19,11 @@ atomicwrites==1.3.0 \
|
|||
attrs==19.3.0 \
|
||||
--hash=sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c \
|
||||
--hash=sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72 \
|
||||
# via pytest
|
||||
cached-property==1.5.1 \
|
||||
--hash=sha256:3a026f1a54135677e7da5ce819b0c690f156f37976f3e30c5430740725203d7f \
|
||||
--hash=sha256:9217a59f14a5682da7c4b8829deadbfc194ac22e9908ccf7c8820234e80a1504 \
|
||||
# via aspy.refactor-imports
|
||||
# via black, pytest
|
||||
black==19.10b0 \
|
||||
--hash=sha256:1b30e59be925fafc1ee4565e5e08abef6b03fe455102883820fe5ee2e4734e0b \
|
||||
--hash=sha256:c2edb73a08e9e0e6f65a0e6af18b059b8b1cdd5bef997d7a0b181df93dc81539 \
|
||||
# via -r requirements/dev.in
|
||||
certifi==2020.4.5.1 \
|
||||
--hash=sha256:1d987a998c75633c40847cc966fcf5904906c920a7f17ef374f5aa4282abd304 \
|
||||
--hash=sha256:51fcb31174be6e6664c5f69e3e1691a2d72a1a12e90f872cbdb1567eb47b6519 \
|
||||
|
|
@ -43,7 +39,7 @@ chardet==3.0.4 \
|
|||
click==7.1.1 \
|
||||
--hash=sha256:8a18b4ea89d8820c5d0c7da8a64b2c324b4dabb695804dbfea19b9be9d88c0cc \
|
||||
--hash=sha256:e345d143d80bf5ee7534056164e5e112ea5e22716bbb1ce727941f4c8b471b9a \
|
||||
# via pip-tools
|
||||
# via black, pip-tools
|
||||
colorama==0.4.3 \
|
||||
--hash=sha256:7d73d2a99753107a36ac6b455ee49046802e59d9d076ef8e47b61499fa29afff \
|
||||
--hash=sha256:e96da0d330793e2cb9485e9ddfd918d456036c7149416295932478192f4436a1 \
|
||||
|
|
@ -120,10 +116,6 @@ importlib-metadata==1.6.0 \
|
|||
--hash=sha256:2a688cbaa90e0cc587f1df48bdc97a6eadccdcd9c35fb3f976a09e3b5016d90f \
|
||||
--hash=sha256:34513a8a0c4962bc66d35b359558fd8a5e10cd472d37aec5f66858addef32c1e \
|
||||
# via pluggy, pre-commit, pytest, virtualenv
|
||||
isort==4.3.21 \
|
||||
--hash=sha256:54da7e92468955c4fceacd0c86bd0ec997b0e1ee80d97f67c35a78b719dccab1 \
|
||||
--hash=sha256:6e811fcb295968434526407adb8796944f1988c5b65e8139058f2014cbe100fd \
|
||||
# via -r requirements/dev.in
|
||||
mccabe==0.6.1 \
|
||||
--hash=sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42 \
|
||||
--hash=sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f \
|
||||
|
|
@ -143,6 +135,10 @@ packaging==20.3 \
|
|||
--hash=sha256:3c292b474fda1671ec57d46d739d072bfd495a4f51ad01a055121d81e952b7a3 \
|
||||
--hash=sha256:82f77b9bee21c1bafbf35a84905d604d5d1223801d639cf3ed140bd651c08752 \
|
||||
# via pytest
|
||||
pathspec==0.7.0 \
|
||||
--hash=sha256:163b0632d4e31cef212976cf57b43d9fd6b0bac6e67c26015d611a647d5e7424 \
|
||||
--hash=sha256:562aa70af2e0d434367d9790ad37aed893de47f1693e4201fd1d3dca15d19b96 \
|
||||
# via black
|
||||
pip-tools==4.5.1 \
|
||||
--hash=sha256:693f30e451875796b1b25203247f0b4cf48a4c4a5ab7341f4f33ffd498cdcc98 \
|
||||
--hash=sha256:be9c796aa88b2eec5cabf1323ba1cb60a08212b84bfb75b8b4037a8ef8cb8cb6 \
|
||||
|
|
@ -228,6 +224,29 @@ pyyaml==5.3.1 \
|
|||
--hash=sha256:cc8955cfbfc7a115fa81d85284ee61147059a753344bc51098f3ccd69b0d7e0c \
|
||||
--hash=sha256:d13155f591e6fcc1ec3b30685d50bf0711574e2c0dfffd7644babf8b5102ca1a \
|
||||
# via pre-commit
|
||||
regex==2020.4.4 \
|
||||
--hash=sha256:08119f707f0ebf2da60d2f24c2f39ca616277bb67ef6c92b72cbf90cbe3a556b \
|
||||
--hash=sha256:0ce9537396d8f556bcfc317c65b6a0705320701e5ce511f05fc04421ba05b8a8 \
|
||||
--hash=sha256:1cbe0fa0b7f673400eb29e9ef41d4f53638f65f9a2143854de6b1ce2899185c3 \
|
||||
--hash=sha256:2294f8b70e058a2553cd009df003a20802ef75b3c629506be20687df0908177e \
|
||||
--hash=sha256:23069d9c07e115537f37270d1d5faea3e0bdded8279081c4d4d607a2ad393683 \
|
||||
--hash=sha256:24f4f4062eb16c5bbfff6a22312e8eab92c2c99c51a02e39b4eae54ce8255cd1 \
|
||||
--hash=sha256:295badf61a51add2d428a46b8580309c520d8b26e769868b922750cf3ce67142 \
|
||||
--hash=sha256:2a3bf8b48f8e37c3a40bb3f854bf0121c194e69a650b209628d951190b862de3 \
|
||||
--hash=sha256:4385f12aa289d79419fede43f979e372f527892ac44a541b5446617e4406c468 \
|
||||
--hash=sha256:5635cd1ed0a12b4c42cce18a8d2fb53ff13ff537f09de5fd791e97de27b6400e \
|
||||
--hash=sha256:5bfed051dbff32fd8945eccca70f5e22b55e4148d2a8a45141a3b053d6455ae3 \
|
||||
--hash=sha256:7e1037073b1b7053ee74c3c6c0ada80f3501ec29d5f46e42669378eae6d4405a \
|
||||
--hash=sha256:90742c6ff121a9c5b261b9b215cb476eea97df98ea82037ec8ac95d1be7a034f \
|
||||
--hash=sha256:a58dd45cb865be0ce1d5ecc4cfc85cd8c6867bea66733623e54bd95131f473b6 \
|
||||
--hash=sha256:c087bff162158536387c53647411db09b6ee3f9603c334c90943e97b1052a156 \
|
||||
--hash=sha256:c162a21e0da33eb3d31a3ac17a51db5e634fc347f650d271f0305d96601dc15b \
|
||||
--hash=sha256:c9423a150d3a4fc0f3f2aae897a59919acd293f4cb397429b120a5fcd96ea3db \
|
||||
--hash=sha256:ccccdd84912875e34c5ad2d06e1989d890d43af6c2242c6fcfa51556997af6cd \
|
||||
--hash=sha256:e91ba11da11cf770f389e47c3f5c30473e6d85e06d7fd9dcba0017d2867aab4a \
|
||||
--hash=sha256:ea4adf02d23b437684cd388d557bf76e3afa72f7fed5bbc013482cc00c816948 \
|
||||
--hash=sha256:fb95debbd1a824b2c4376932f2216cc186912e389bdb0e27147778cf6acb3f89 \
|
||||
# via black
|
||||
requests==2.23.0 \
|
||||
--hash=sha256:43999036bfa82904b6af1d99e4882b560e5e2c68e5c4b0aa03b655f3d7d73fee \
|
||||
--hash=sha256:b3f43d496c6daba4493e7c431722aeb7dbc6288f52a6e04e7b6023b0247817e6 \
|
||||
|
|
@ -236,10 +255,6 @@ responses==0.10.12 \
|
|||
--hash=sha256:0474ce3c897fbcc1aef286117c93499882d5c440f06a805947e4b1cb5ab3d474 \
|
||||
--hash=sha256:f83613479a021e233e82d52ffb3e2e0e2836d24b0cc88a0fa31978789f78d0e5 \
|
||||
# via -r requirements/dev.in
|
||||
seed-isort-config==2.1.0 \
|
||||
--hash=sha256:ae069c0ac7fa833838a951114a8126801a550c592ffa347ffaf47164c0eda21e \
|
||||
--hash=sha256:c9e5a444492757d98aab32f5a39a6ed79ed272820133f12c437ccb713b6fd2ec \
|
||||
# via -r requirements/dev.in
|
||||
selenium==3.141.0 \
|
||||
--hash=sha256:2d7131d7bc5a5b99a2d9b04aaf2612c411b03b8ca1b1ee8d3de5845a9be2cb3c \
|
||||
--hash=sha256:deaf32b60ad91a4611b98d8002757f29e6f2c2d5fcaf202e1c9ad06d6772300d \
|
||||
|
|
@ -255,7 +270,30 @@ sqlparse==0.3.1 \
|
|||
toml==0.10.0 \
|
||||
--hash=sha256:229f81c57791a41d65e399fc06bf0848bab550a9dfd5ed66df18ce5f05e73d5c \
|
||||
--hash=sha256:235682dd292d5899d361a811df37e04a8828a5b1da3115886b73cf81ebc9100e \
|
||||
# via pre-commit
|
||||
# via black, pre-commit
|
||||
typed-ast==1.4.1 \
|
||||
--hash=sha256:0666aa36131496aed8f7be0410ff974562ab7eeac11ef351def9ea6fa28f6355 \
|
||||
--hash=sha256:0c2c07682d61a629b68433afb159376e24e5b2fd4641d35424e462169c0a7919 \
|
||||
--hash=sha256:249862707802d40f7f29f6e1aad8d84b5aa9e44552d2cc17384b209f091276aa \
|
||||
--hash=sha256:24995c843eb0ad11a4527b026b4dde3da70e1f2d8806c99b7b4a7cf491612652 \
|
||||
--hash=sha256:269151951236b0f9a6f04015a9004084a5ab0d5f19b57de779f908621e7d8b75 \
|
||||
--hash=sha256:4083861b0aa07990b619bd7ddc365eb7fa4b817e99cf5f8d9cf21a42780f6e01 \
|
||||
--hash=sha256:498b0f36cc7054c1fead3d7fc59d2150f4d5c6c56ba7fb150c013fbc683a8d2d \
|
||||
--hash=sha256:4e3e5da80ccbebfff202a67bf900d081906c358ccc3d5e3c8aea42fdfdfd51c1 \
|
||||
--hash=sha256:6daac9731f172c2a22ade6ed0c00197ee7cc1221aa84cfdf9c31defeb059a907 \
|
||||
--hash=sha256:715ff2f2df46121071622063fc7543d9b1fd19ebfc4f5c8895af64a77a8c852c \
|
||||
--hash=sha256:73d785a950fc82dd2a25897d525d003f6378d1cb23ab305578394694202a58c3 \
|
||||
--hash=sha256:8c8aaad94455178e3187ab22c8b01a3837f8ee50e09cf31f1ba129eb293ec30b \
|
||||
--hash=sha256:8ce678dbaf790dbdb3eba24056d5364fb45944f33553dd5869b7580cdbb83614 \
|
||||
--hash=sha256:aaee9905aee35ba5905cfb3c62f3e83b3bec7b39413f0a7f19be4e547ea01ebb \
|
||||
--hash=sha256:bcd3b13b56ea479b3650b82cabd6b5343a625b0ced5429e4ccad28a8973f301b \
|
||||
--hash=sha256:c9e348e02e4d2b4a8b2eedb48210430658df6951fa484e59de33ff773fbd4b41 \
|
||||
--hash=sha256:d205b1b46085271b4e15f670058ce182bd1199e56b317bf2ec004b6a44f911f6 \
|
||||
--hash=sha256:d43943ef777f9a1c42bf4e552ba23ac77a6351de620aa9acf64ad54933ad4d34 \
|
||||
--hash=sha256:d5d33e9e7af3b34a40dc05f498939f0ebf187f07c385fd58d591c533ad8562fe \
|
||||
--hash=sha256:fc0fea399acb12edbf8a628ba8d2312f583bdbdb3335635db062fa98cf71fca4 \
|
||||
--hash=sha256:fe460b922ec15dd205595c9b5b99e2f056fd98ae8f9f56b888e7a17dc2b757e7 \
|
||||
# via black
|
||||
urllib3==1.25.8 \
|
||||
--hash=sha256:2f3db8b19923a873b3e5256dc9c2dedfa883e33d87c690d9c7913e1f40673cdc \
|
||||
--hash=sha256:87716c2d2a7121198ebcb7ce7cccf6ce5e9ba539041cfbaeecfb641dc0bf6acc \
|
||||
|
|
|
|||
|
|
@ -16,10 +16,6 @@ echo "Checking CELERY_TASK_QUEUES matches Procfile"
|
|||
echo "Running flake8"
|
||||
flake8 --show-source || { echo "flake8 errors found!"; exit 1; }
|
||||
|
||||
echo "Running isort"
|
||||
isort --check-only --diff --quiet \
|
||||
|| { echo "isort errors found! Run 'isort' with no options to fix."; exit 1; }
|
||||
|
||||
if hash shellcheck 2>/dev/null; then
|
||||
echo "Running shellcheck"
|
||||
git grep -El '^#!/.+\b(bash|sh)\b' | xargs shellcheck
|
||||
|
|
|
|||
|
|
@ -5,13 +5,6 @@ exclude = */.*/,.*/,__pycache__,node_modules
|
|||
extend_ignore = E129,E501
|
||||
max-line-length = 100
|
||||
|
||||
[tool:isort]
|
||||
known_third_party =_pytest,aiohttp,cache_memoize,celery,dateutil,deepdiff,django,django_filters,environ,first,furl,jinja2,jose,jsonschema,jx_base,jx_bigquery,jx_mysql,jx_python,kombu,mo_files,mo_future,mo_json,mo_logs,mo_math,mo_sql,mo_testing,mo_times,mock,mozlog,newrelic,pages,pypom,pytest,redis,requests,responses,rest_framework,selenium,setuptools,simplejson,slugid,taskcluster,taskcluster_urls,thclient,whitenoise,yaml
|
||||
skip = __pycache__,node_modules,migrations,misc,.vendor
|
||||
multi_line_output = 1
|
||||
force_grid_wrap = true
|
||||
line_length = 100
|
||||
|
||||
[tool:pytest]
|
||||
testpaths = tests
|
||||
norecursedirs = __pycache__ ui
|
||||
|
|
|
|||
|
|
@ -2,12 +2,11 @@ import binascii
|
|||
import json
|
||||
import os
|
||||
import re
|
||||
from datetime import (datetime,
|
||||
timedelta)
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
import responses
|
||||
|
||||
from treeherder.changelog.collector import collect # noqa isort:skip
|
||||
from treeherder.changelog.collector import collect
|
||||
|
||||
|
||||
def random_id():
|
||||
|
|
|
|||
|
|
@ -1,15 +1,10 @@
|
|||
from jx_bigquery import bigquery
|
||||
from jx_mysql.mysql import (MySQL,
|
||||
sql_query)
|
||||
from jx_mysql.mysql import MySQL, sql_query
|
||||
from jx_mysql.mysql_snowflake_extractor import MySqlSnowflakeExtractor
|
||||
from jx_python import jx
|
||||
from mo_files import File
|
||||
from mo_json import (json2value,
|
||||
value2json)
|
||||
from mo_logs import (Log,
|
||||
constants,
|
||||
startup,
|
||||
strings)
|
||||
from mo_json import json2value, value2json
|
||||
from mo_logs import Log, constants, startup, strings
|
||||
from mo_sql import SQL
|
||||
from mo_times import Timer
|
||||
from mo_times.dates import Date
|
||||
|
|
@ -17,6 +12,7 @@ from redis import Redis
|
|||
|
||||
from treeherder.config.settings import REDIS_URL
|
||||
|
||||
|
||||
CONFIG_FILE = (File.new_instance(__file__).parent / "extract_jobs.json").abspath
|
||||
|
||||
|
||||
|
|
@ -128,8 +124,7 @@ class ExtractJobs:
|
|||
last_doc = acc[-1]
|
||||
last_modified, job_id = last_doc.last_modified, last_doc.id
|
||||
redis.set(
|
||||
settings.extractor.key,
|
||||
value2json((last_modified, job_id)).encode("utf8"),
|
||||
settings.extractor.key, value2json((last_modified, job_id)).encode("utf8"),
|
||||
)
|
||||
|
||||
if len(acc) < settings.extractor.chunk_size:
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче