Generate requirements with python 3.8 (#1021)

and enforce via CI

.circleci/config.yml

@@ -1,7 +1,7 @@
 version: 2
 jobs:
   build:
-    docker:
+    docker: &docker
     - image: python:3.8
     steps:
     - checkout
@@ -18,7 +18,8 @@ jobs:
         name: Build
         command: |
           python3.8 -m venv venv/
-          venv/bin/pip install --upgrade -r requirements.txt
+          venv/bin/pip install pip-tools
+          venv/bin/pip-sync
     - run:
         name: PyTest with linters
         command: PATH="venv/bin:$PATH" script/entrypoint
@@ -27,25 +28,31 @@ jobs:
         - venv/
         key: *cache_key
   verify-format-sql:
-    docker:
-      - image: python:3.8
+    docker: *docker
     steps:
       - checkout
       - run:
           name: Verify that SQL is correctly formatted
-          # check all directories owned by git except sql
           command: script/format_sql --check $(git ls-tree -d HEAD --name-only)
+  verify-requirements:
+    docker: *docker
+    steps:
+      - checkout
+      - run:
+          name: Verify that requirements.txt contains the right dependencies for this python version
+          command: |
+            pip install pip-tools
+            pip-compile --quiet --generate-hashes requirements.in
+            git diff --exit-code requirements.txt
   dry-run-sql:
-    docker:
-      - image: python:3.8
+    docker: *docker
     steps:
       - checkout
       - run:
           name: Verify that BigQuery validates each query
           command: script/dryrun
   validate-metadata:
-    docker:
-      - image: python:3.8
+    docker: *docker
     steps:
       - checkout
       - *build
@@ -53,8 +60,7 @@ jobs:
           name: Verify that metadata files are valid
           command: PATH="venv/bin:$PATH" script/validate_metadata
   integration:
-    docker:
-    - image: python:3.8
+    docker: *docker
     steps:
     - checkout
     - &skip_forked_pr
@@ -130,6 +136,7 @@ workflows:
     - build:
         context: data-eng-circleci-tests
     - verify-format-sql
+    - verify-requirements
     - dry-run-sql
     - validate-metadata
     - integration

requirements.txt

@@ -107,10 +107,6 @@ idna==2.9 \
     --hash=sha256:7588d1c14ae4c77d74036e8c22ff447b26d0fde8f007354fd48a7814db15b7cb \
     --hash=sha256:a068a21ceac8a4d63dbfd964670474107f541babbd2250d61922f029858365fa \
     # via requests
-importlib-metadata==1.6.0 \
-    --hash=sha256:2a688cbaa90e0cc587f1df48bdc97a6eadccdcd9c35fb3f976a09e3b5016d90f \
-    --hash=sha256:34513a8a0c4962bc66d35b359558fd8a5e10cd472d37aec5f66858addef32c1e \
-    # via flake8, jsonschema, pluggy, pytest
 jinja2==2.11.2 \
     --hash=sha256:89aab215427ef59c34ad58735269eb58b1a5808103067f7bb9d5836c651b3bb0 \
     --hash=sha256:f0a4641d3cf955324a89c04f3d94663aa4d638abe8f733ecd3582848e1c37035 \
@@ -404,10 +400,6 @@ wcwidth==0.1.9 \
     --hash=sha256:cafe2186b3c009a04067022ce1dcd79cb38d8d65ee4f4791b8888d6599d1bbe1 \
     --hash=sha256:ee73862862a156bf77ff92b09034fc4825dd3af9cf81bc5b360668d425f3c5f1 \
     # via pytest
-zipp==3.1.0 \
-    --hash=sha256:aa36550ff0c0b7ef7fa639055d797116ee891440eac1a56f378e2d3179e0320b \
-    --hash=sha256:c599e4d75c98f6798c509911d08a22e6c021d074469042177c8c86fb92eefd96 \
-    # via importlib-metadata
 
 # WARNING: The following packages were not pinned, but pip requires them to be
 # pinned when the requirements file includes hashes. Consider using the --allow-unsafe flag.