Feature owners: db notifications and task queue.

- Intercepts calls to Feature.put() and diffs changes in properties for updates
- Creates async task queue and email notifies feature owners
- Switch to pip and custom django install instead of GAE version

.gitignore

@@ -16,4 +16,5 @@ static/js/**/*.es6.min.js
 bulkloader-*
 css/
 static/dist/
-cookie
+cookie
+lib

README.md

@@ -11,10 +11,11 @@ Chrome Platform Status
 
 First, install the [Google App Engine SDK for Python](https://developers.google.com/appengine/downloads#Google_App_Engine_SDK_for_Python).
 
-You'll also need node/npm. Next, install `bower` and the npm deps:
+You'll also need pip, node, and npm. Next, install `bower` and the deps:
 
     npm install -g bower
     npm install
+    pip install -t lib -r requirements.txt
 
 This will also pull down bower_components and run `gulp` to build the site.
 

admin.py

@@ -35,11 +35,11 @@ from google.appengine.api import urlfetch
 from google.appengine.api import users
 from google.appengine.ext import blobstore
 from google.appengine.ext import db
+from google.appengine.api import taskqueue
 from google.appengine.ext.webapp import blobstore_handlers
 
 # File imports.
 import common
-import emailer
 import models
 import settings
 
@@ -431,12 +431,6 @@ class FeatureHandler(common.ContentHandler):
       redirect_url = '%s/%s?%s' % (self.LAUNCH_URL, key.id(),
                                    '&'.join(params))
 
-    try:
-      # Email feature owners.
-      emailer.email_feature_owners(feature, update=updating_existing_feature)
-    except:
-      logging.error('Error sending email to feature owners')
-
     return self.redirect(redirect_url)
 
 

app.yaml

@@ -1,5 +1,5 @@
 application: cr-status
-version: schedule2
+version: schedule-emailer
 runtime: python27
 threadsafe: true
 api_version: 1
@@ -9,16 +9,16 @@ api_version: 1
 builtins:
 - remote_api: on
 
-libraries:
-- name: webapp2
-  version: "latest"
-- name: django
-  version: "1.4" #"latest"
+#libraries:
+#- name: webapp2
+#  version: "latest"
+#- name: django
+#  version: "1.4" #"1.9"
 # - name: setuptools
 #   version: latest
 
-env_variables:
-  DJANGO_SETTINGS_MODULE: 'settings'
+#env_variables:
+#  DJANGO_SETTINGS_MODULE: 'settings'
 
 handlers:
 

appengine_config.py

@@ -0,0 +1,6 @@
+import os
+# name of the django settings module
+os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
+
+from google.appengine.ext import vendor
+vendor.add('lib') # add third party libs to "lib" folder.

common.py

@@ -24,12 +24,12 @@ import webapp2
 # App Engine imports.
 from google.appengine.api import users
 
+import settings
+import models
+
 from django.template.loader import render_to_string
 from django.utils import feedgenerator
 
-import models
-import settings
-
 
 def require_whitelisted_user(handler):
   """Handler decorator to require the user be whitelisted."""

emailer.py

@@ -1,104 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2017 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License")
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-__author__ = 'ericbidelman@chromium.org (Eric Bidelman)'
-
-import logging
-import datetime
-
-from google.appengine.api import mail
-
-import settings
-import models
-
-def email_feature_owners(feature, update=False):
-    for component_name in feature.blink_components:
-      component = models.BlinkComponent.get_by_name(component_name)
-      if not component:
-        logging.warn('Blink component %s not found. Not sending email to owners' % component_name)
-        return
-
-      message = mail.EmailMessage(sender='Chromestatus <admin@cr-status.appspotmail.com>',
-                                  subject='chromestatus update',
-                                  to=[owner.email for owner in component.owners])
-
-      owner_names = [owner.name for owner in component.owners]
-
-      if feature.shipped_milestone:
-        milestone_str = feature.shipped_milestone
-      elif feature.shipped_milestone is None and feature.shipped_android_milestone:
-        milestone_str = feature.shipped_android_milestone + ' (android)'
-      else:
-        milestone_str = 'not yet assigned'
-
-      created_on = datetime.datetime.strptime(str(feature.created), "%Y-%m-%d %H:%M:%S.%f").date()
-      new_msg = """
-Hi {owners},
-
-You are listed as a web platform owner for "{component_name}". {created_by} added a feature to chromestatus on {created}:
-
-  Feature: {name}
-  Implementation status: {status}
-  Milestone: {milestone}
-  Created: {created}
-
-  See https://www.chromestatus.com/feature/{id} for more details.
-
-Next steps:
-- Try the API, write a sample, provide early feedback to eng.
-- Consider authoring a new article/update for /web.
-- Write a <a href="https://github.com/GoogleChrome/lighthouse/tree/master/docs/recipes/custom-audit">new Lighthouse audit</a>. This can  help drive adoption of an API over time.
-- Add a sample to https://github.com/GoogleChrome/samples (see <a href="https://github.com/GoogleChrome/samples#contributing-samples">contributing</a>).
-  - Don't forget add your demo link to the <a href="https://www.chromestatus.com/admin/features/edit/{id}">chromestatus feature entry</a>.
-""".format(name=feature.name, id=feature.key().id(), created=created_on,
-           created_by=feature.created_by, component_name=component_name,
-           owners=', '.join(owner_names), milestone=milestone_str,
-           status=models.IMPLEMENTATION_STATUS[feature.impl_status_chrome])
-
-    updated_on = datetime.datetime.strptime(str(feature.updated), "%Y-%m-%d %H:%M:%S.%f").date()
-
-    # TODO: link to existing /web content tagged with component name.
-    update_msg = """
-Hi {owners},
-
-You are listed as a web platform owner for "{component_name}". {updated_by} updated a feature on chromestatus on {updated}:
-
-  Feature: <a href="https://www.chromestatus.com/feature/{id}">{name}</a>
-
-  Implementation status: {status}
-  Milestone: {milestone}
-  Updated: {updated}
-
-  See https://www.chromestatus.com/feature/{id} for more details.
-
-Next steps:
-- Check existing /web content for correctness.
-- Check existing <a href="https://github.com/GoogleChrome/lighthouse/tree/master/lighthouse-core/audits">Lighthouse audits</a> for correctness.
-""".format(name=feature.name, id=feature.key().id(), updated=updated_on,
-           updated_by=feature.updated_by, component_name=component_name,
-           owners=', '.join(owner_names), milestone=milestone_str,
-           status=models.IMPLEMENTATION_STATUS[feature.impl_status_chrome])
-
-    if update:
-      message.html = update_msg
-      message.subject = 'chromestatus: updated feature'
-    else:
-      message.html = new_msg
-      message.subject = 'chromestatus: new feature'
-
-    message.check_initialized()
-
-    if settings.SEND_EMAIL:
-      message.send()

models.py

@@ -4,19 +4,21 @@ import logging
 import re
 import time
 
+from google.appengine.ext import db
+# from google.appengine.ext.db import djangoforms
 from google.appengine.api import mail
 from google.appengine.api import memcache
 from google.appengine.api import urlfetch
+from google.appengine.api import taskqueue
 from google.appengine.api import users
-from google.appengine.ext import db
-#from google.appengine.ext.db import djangoforms
+
+import settings
+import util
 
 #from django.forms import ModelForm
 from collections import OrderedDict
 from django import forms
-
-import settings
-import util
+# import google.appengine.ext.django as django
 
 
 SIMPLE_TYPES = (int, long, float, bool, dict, basestring, list)
@@ -376,11 +378,14 @@ class Feature(DictModel):
     except Exception as e:
       logging.error(e)
 
-  def format_for_template(self, version=None):
+  def format_for_template(self, version=2):
     d = self.to_dict()
 
     if version == 2:
-      d['id'] = self.key().id()
+      if self.is_saved():
+        d['id'] = self.key().id()
+      else:
+        d['id'] = None
       d['category'] = FEATURE_CATEGORIES[self.category]
       d['created'] = {
         'by': d.pop('created', None),
@@ -473,7 +478,10 @@ class Feature(DictModel):
       del_none(d) # Further prune response by removing null/[] values.
 
     else:
-      d['id'] = self.key().id()
+      if self.is_saved():
+        d['id'] = self.key().id()
+      else:
+        d['id'] = None
       d['category'] = FEATURE_CATEGORIES[self.category]
       d['visibility'] = VISIBILITY_CHOICES[self.visibility]
       d['impl_status_chrome'] = IMPLEMENTATION_STATUS[self.impl_status_chrome]
@@ -707,6 +715,44 @@ class Feature(DictModel):
       params.append('cc=' + ','.join(self.owner))
     return url + '?' + '&'.join(params)
 
+  def __init__(self, *args, **kwargs):
+    super(Feature, self).__init__(*args, **kwargs)
+
+    # Stash existing values when entity is created so we can diff property
+    # values later in put() to know what's changed. https://stackoverflow.com/a/41344898
+    for prop_name, prop in self.properties().iteritems():
+      old_val = getattr(self, prop_name, None)
+      setattr(self, '_old_' + prop_name, old_val)
+
+  def __notify_feature_owners_of_changes(self, is_update):
+    """Async notifies owners of new features and property changes to features by
+       posting to a task queue."""
+    # Diff values to see what properties have changed.
+    changed_props = []
+    for prop_name, prop in self.properties().iteritems():
+      new_val = getattr(self, prop_name, None)
+      old_val = getattr(self, '_old_' + prop_name, None)
+      if new_val != old_val:
+        changed_props.append({
+            'prop_name': prop_name, 'old_val': old_val, 'new_val': new_val})
+
+    payload = json.dumps({
+      'changes': changed_props,
+      'is_update': is_update,
+      'feature': self.format_for_template()
+    })
+    queue = taskqueue.Queue(name='emailer')
+    # Create task to email owners.
+    task = taskqueue.Task(method="POST", url='/tasks/email-owners',
+        target='notifier', payload=payload)
+    queue.add(task)
+
+  def put(self, **kwargs):
+    is_update = self.is_saved()
+    key = super(Feature, self).put(**kwargs)
+    self.__notify_feature_owners_of_changes(is_update)
+    return key
+
   # Metadata.
   created = db.DateTimeProperty(auto_now_add=True)
   updated = db.DateTimeProperty(auto_now=True)

notifier.py

@@ -0,0 +1,143 @@
+# -*- coding: utf-8 -*-
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License")
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__author__ = 'ericbidelman@chromium.org (Eric Bidelman)'
+
+import logging
+import datetime
+import json
+import webapp2
+
+from google.appengine.api import mail
+from google.appengine.api import taskqueue
+
+import settings
+import models
+
+
+def email_feature_owners(feature, is_update=False, changes=[]):
+  for component_name in feature.blink_components:
+    component = models.BlinkComponent.get_by_name(component_name)
+    if not component:
+      logging.warn('Blink component %s not found. Not sending email to owners' % component_name)
+      return
+
+    owner_names = [owner.name for owner in component.owners]
+    if not owner_names:
+      logging.info('Blink component has no owners. Skipping email')
+      return
+
+    if feature.shipped_milestone:
+      milestone_str = feature.shipped_milestone
+    elif feature.shipped_milestone is None and feature.shipped_android_milestone:
+      milestone_str = feature.shipped_android_milestone + ' (android)'
+    else:
+      milestone_str = 'not yet assigned'
+
+    created_on = datetime.datetime.strptime(str(feature.created), "%Y-%m-%d %H:%M:%S.%f").date()
+    new_msg = """
+Hi {owners},
+
+{created_by} added a new feature to chromestatus. You are listed as a web platform owner for "{component_name}".
+See https://www.chromestatus.com/feature/{id} for more details.
+---
+
+Feature: {name}
+
+Created: {created}
+Implementation status: {status}
+Milestone: {milestone}
+
+---
+Next steps:
+- Try the API, write a sample, provide early feedback to eng.
+- Consider authoring a new article/update for /web.
+- Write a <a href="https://github.com/GoogleChrome/lighthouse/tree/master/docs/recipes/custom-audit">new Lighthouse audit</a>. This can  help drive adoption of an API over time.
+- Add a sample to https://github.com/GoogleChrome/samples (see <a href="https://github.com/GoogleChrome/samples#contributing-samples">contributing</a>).
+  - Don't forget add your demo link to the <a href="https://www.chromestatus.com/admin/features/edit/{id}">chromestatus feature entry</a>.
+""".format(name=feature.name, id=feature.key().id(), created=created_on,
+           created_by=feature.created_by, component_name=component_name,
+           owners=', '.join(owner_names), milestone=milestone_str,
+           status=models.IMPLEMENTATION_STATUS[feature.impl_status_chrome])
+
+  updated_on = datetime.datetime.strptime(str(feature.updated), "%Y-%m-%d %H:%M:%S.%f").date()
+  formatted_changes = ''
+  for prop in changes:
+    formatted_changes += '- %s: %s -> %s\n' % (prop['prop_name'], prop['old_val'], prop['new_val'])
+  if not formatted_changes:
+    formatted_changes = 'None'
+
+  # TODO: link to existing /web content tagged with component name.
+  update_msg = """
+Hi {owners},
+
+{updated_by} updated a feature on chromestatus. You are listed as a web platform owner for "{component_name}".
+See https://www.chromestatus.com/feature/{id} for more details.
+---
+
+Feature: <a href="https://www.chromestatus.com/feature/{id}">{name}</a>
+
+Updated: {updated}
+Implementation status: {status}
+Milestone: {milestone}
+
+Changes:
+{formatted_changes}
+
+---
+Next steps:
+- Check existing /web content for correctness.
+- Check existing <a href="https://github.com/GoogleChrome/lighthouse/tree/master/lighthouse-core/audits">Lighthouse audits</a> for correctness.
+""".format(name=feature.name, id=feature.key().id(), updated=updated_on,
+           updated_by=feature.updated_by, component_name=component_name,
+           owners=', '.join(owner_names), milestone=milestone_str,
+           status=models.IMPLEMENTATION_STATUS[feature.impl_status_chrome],
+           formatted_changes=formatted_changes)
+
+  message = mail.EmailMessage(sender='Chromestatus <admin@cr-status.appspotmail.com>',
+                              subject='chromestatus update',
+                              to=[owner.email for owner in component.owners])
+
+  if is_update:
+    message.html = update_msg
+    message.subject = 'chromestatus: updated feature'
+  else:
+    message.html = new_msg
+    message.subject = 'chromestatus: new feature'
+
+  message.check_initialized()
+
+  if settings.SEND_EMAIL:
+    message.send()
+
+
+class EmailOwnersHandler(webapp2.RequestHandler):
+  def post(self):
+    json_body = json.loads(self.request.body)
+    feature = json_body.get('feature') or None
+    is_update = json_body.get('is_update') or False
+    changes = json_body.get('changes') or []
+
+    # Email feature owners.
+    try:
+      feature = models.Feature.get_by_id(feature['id'])
+      email_feature_owners(feature, is_update=is_update, changes=changes)
+    except:
+      logging.error('Error sending email to feature owners')
+
+
+app = webapp2.WSGIApplication([
+  ('/tasks/email-owners', EmailOwnersHandler),
+], debug=settings.DEBUG)

notifier.yaml

@@ -0,0 +1,9 @@
+runtime: python27
+api_version: 1
+threadsafe: true
+service: notifier
+
+handlers:
+- url: /.*
+  script: notifier.app
+  login: admin

queue.yaml

@@ -0,0 +1,10 @@
+# total_storage_limit: 120M
+queue:
+- name: emailer
+  target: notifier
+  rate: 1/s
+  retry_parameters:
+    task_retry_limit: 3
+    task_age_limit: 2d
+  # bucket_size: 40
+  # max_concurrent_requests: 10

requirements.txt

@@ -0,0 +1 @@
+Django==1.4

scripts/deploy_site.sh

@@ -16,4 +16,4 @@ readonly BASEDIR=$(dirname $BASH_SOURCE)
 gulp
 
 $BASEDIR/oauthtoken.sh deploy
-appcfg.py update $BASEDIR/../
+appcfg.py update -A cr-status $BASEDIR/../app.yaml $BASEDIR/../notifier.yaml

scripts/start_server.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+#
+# Starts the dev server and services.
+#
+# Copyright 2017 Eric Bidelman <ericbidelman@chromium.org>
+
+# The directory in which this script resides.
+readonly BASEDIR=$(dirname $BASH_SOURCE)
+
+dev_appserver.py -A cr-status $BASEDIR/../app.yaml $BASEDIR/../notifier.yaml

server.py

@@ -20,9 +20,9 @@ import logging
 import os
 import webapp2
 
+import settings
 import common
 import models
-import settings
 import util
 
 import http2push.http2push as http2push