Avoid needing to run py2 on workstations. (#1734)

* Avoid needing to run py2 on workstations.

* Use gunicorn instead of dev_appserver

* Add comment to explain that dev-default.yaml is not currently used.

* Use venv pip to install py3 deps.

* Auto-activate and stop referencing lib/.

* Make gunicorn use libs in cs-env.

README.md

@@ -13,7 +13,7 @@ Chrome Platform Status
 1. Before you begin, make sure that you have a java JRE (version 8 or greater) installed. JRE is required to use the DataStore Emulator.
 1. Install global CLIs in the home directory
     1. [Google App Engine SDK for Python](https://cloud.google.com/appengine/docs/standard/python3/setting-up-environment). Make sure to select Python 3.
-    1. pip, node, npm.
+    1. pip, node, npm, and gunicorn.
     1. Gulp `npm install --global gulp-cli`
 1. We recommend using an older node version, e.g. node 10
     1. Use `node -v` to check the default node version
@@ -25,7 +25,7 @@ Chrome Platform Status
 5. Install pip for python2
     1. curl https://bootstrap.pypa.io/pip/2.7/get-pip.py --output get-pip.py
     1. python2 get-pip.py
-6. Install other dependencies 
+6. Install other dependencies
     1. `npm run deps`
     1. `npm run dev-deps`
 

appengine_config.py

@@ -4,11 +4,6 @@ import importlib
 # name of the django settings module
 os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
 
-lib_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'lib')
-
-from google.appengine.ext import vendor
-vendor.add(lib_path) # add third party libs to "lib" folder.
-
 # Add libraries to pkg_resources working set to find the distribution.
 import pkg_resources
 pkg_resources.working_set.add_entry(lib_path)

dev-default.yaml

@@ -0,0 +1,23 @@
+# This GAE service is just an empty placeholder because we need a default
+# service in py3 when doing development on workstations.  On GAE, the
+# py2 service defined in py/app-py2.yaml is used.
+
+# As of 2022-02-14 this file is not actually used because start_server.sh
+# runs gunicorn directly instead of using dev_appserver.   If dev_appserver
+# is updated to work under py3, then we could start using it again and we
+# would use this file.
+
+
+runtime: python39
+
+automatic_scaling:
+  min_idle_instances: 1
+  max_pending_latency: 0.2s
+
+# No handlers
+
+includes:
+- env_vars.yaml
+
+env_variables:
+  GAE_USE_SOCKETS_HTTPLIB : ''

framework/basehandlers.py

@@ -460,10 +460,10 @@ def ndb_wsgi_middleware(wsgi_app):
   return middleware
 
 
-def FlaskApplication(routes, pattern_base='', debug=False):
+def FlaskApplication(import_name, routes, pattern_base='', debug=False):
   """Make a Flask app and add routes and handlers that work like webapp2."""
 
-  app = flask.Flask(__name__)
+  app = flask.Flask(import_name)
   app.wsgi_app = ndb_wsgi_middleware(app.wsgi_app) # For Cloud NDB Context
   client = ndb.Client()
   with client.context():

framework/basehandlers_test.py

@@ -54,6 +54,7 @@ class TestableFlaskHandler(basehandlers.FlaskHandler):
 
 
 test_app = basehandlers.FlaskApplication(
+    __name__,
     [('/test', TestableFlaskHandler),
      ('/old_path', basehandlers.Redirector,
       {'location': '/new_path'}),

framework/permissions_test.py

@@ -42,6 +42,7 @@ class MockHandler(basehandlers.BaseHandler):
 
 
 test_app = basehandlers.FlaskApplication(
+    __name__,
     [('/path', MockHandler),
      ],
     debug=True)

main.py

@@ -194,6 +194,7 @@ internals_routes = [
 
 # All requests to the app-py3 GAE service are handled by this Flask app.
 app = basehandlers.FlaskApplication(
+    __name__,
     (metrics_chart_routes + api_routes + page_routes +
      internals_routes))
 

package.json

@@ -7,9 +7,10 @@
     "node": ">=6.0.0"
   },
   "scripts": {
-    "deps": "python3 -m pip install -t lib -r requirements.txt --upgrade; cd py2; python3 -m pip install -t lib -r requirements.txt --upgrade",
+    "deps": "source cs-env/bin/activate; pip install -r requirements.txt --upgrade; cd py2; python3 -m pip install -t lib -r requirements.txt --upgrade",
     "dev-deps": "python2 -m pip install --no-deps -r requirements.dev.txt",
-    "do-tests": "curl -X POST 'http://localhost:15606/reset' && python3 -m unittest discover -p '*_test.py'  -b",
+    "do-tests": "source cs-env/bin/activate; curl -X POST 'http://localhost:15606/reset' && python3 -m unittest discover -p '*_test.py'  -b",
+    "start-emulator-persist": "gcloud beta emulators datastore start --host-port=:15606 --consistency=1.0",
     "start-emulator": "gcloud beta emulators datastore start --host-port=:15606 --no-store-on-disk --consistency=1.0",
     "stop-emulator": "curl -X POST 'http://localhost:15606/shutdown'",
     "test": "(npm run start-emulator > /dev/null 2>&1 &); sleep 6; curl --retry 4 http://localhost:15606/ --retry-connrefused; npm run do-tests; status=$?; npm run stop-emulator; exit $status",
@@ -23,7 +24,8 @@
     "watch": "gulp watch",
     "deploy": "npm run build && ./scripts/deploy_site.sh `git describe --always --abbrev=7 --match 'NOT A TAG' --dirty='-tainted'` && ./scripts/deploy_site.sh rc",
     "staging": "npm run build && ./scripts/deploy_site.sh `git describe --always --abbrev=7 --match 'NOT A TAG' --dirty='-tainted'` cr-status-staging && ./scripts/deploy_site.sh rc cr-status-staging",
-    "start": "npm run build && ./scripts/start_server.sh"
+    "start-app": "npm run build && ./scripts/start_server.sh",
+    "start": "source cs-env/bin/activate; (npm run start-emulator-persist > /dev/null 2>&1 &); sleep 6; curl --retry 4 http://localhost:15606/ --retry-connrefused; npm run start-app; status=$?; npm run stop-emulator; exit $status"
   },
   "repository": {
     "type": "git",

scripts/start_server.sh

@@ -4,18 +4,34 @@
 #
 # Copyright 2017 Eric Bidelman <ericbidelman@chromium.org>
 
+export PYTHONPATH=cs-env/lib/python3.9/site-packages:$PYTHONPATH
+export GOOGLE_CLOUD_PROJECT='cr-status-staging'
+export SERVER_SOFTWARE='gunicorn'
+export GAE_ENV='localdev'
+export DJANGO_SETTINGS_MODULE='settings'
+export DJANGO_SECRET='this-is-a-secret'
+export DATASTORE_EMULATOR_HOST='localhost:15606'
+
+
+gunicorn --bind :8080 --workers 4 main:app
+
+
+# TODO(jrobbins): Consider switching back to dev_appserver when
+# it no longer requires python2.
+#
 # The directory in which this script resides.
-readonly BASEDIR=$(dirname $BASH_SOURCE)
+#readonly BASEDIR=$(dirname $BASH_SOURCE)
+#
+#dev_appserver.py -A cr-status --enable_console=1 \
+#  --support_datastore_emulator=1 --datastore_emulator_port=15606 \
+#  --env_var DATASTORE_EMULATOR_HOST='localhost:15606' \
+#  $BASEDIR/../dispatch.yaml \
+#  $BASEDIR/../notifier.yaml \
+#  $BASEDIR/../dev-default.yaml \
+#  $BASEDIR/../app-py3.yaml
 
-
-dev_appserver.py -A cr-status --enable_console=1 \
-  --support_datastore_emulator=1 --datastore_emulator_port=15606 \
-  --env_var DATASTORE_EMULATOR_HOST='localhost:15606' \
-  $BASEDIR/../dispatch.yaml \
-  $BASEDIR/../notifier.yaml \
-  $BASEDIR/../py2/app-py2.yaml \
-  $BASEDIR/../app-py3.yaml
-
-# Note: We don't create tasks in dev mode, so the app-py2 service is
-# normally idle.  You can hit the URL http://localhost:8080/py2
-# to see if the app-py2 service is responding at all.
+# Note: When running locally, the default service is dev-default.yaml
+# which is a py3 service which does nothing.  That avoids needing py2
+# on the developer's workstation.
+# On GAE, the default service is py2/app-py2.yaml which uses the GAE
+# py2 runtime.

testing_config.py

@@ -1,6 +1,3 @@
-
-
-
 # Copyright 2020 Google Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License")
@@ -30,13 +27,6 @@ else:
         file=sys.stderr)
   sys.exit(1)
 
-import dev_appserver
-dev_appserver.fix_sys_path()
-
-lib_path = os.path.join(os.path.dirname(__file__), 'lib')
-from google.appengine.ext import vendor
-vendor.add(lib_path) # add third party libs to "lib" folder.
-
 from google.cloud import ndb
 
 os.environ['DJANGO_SECRET'] = 'test secret'