mozilla
/
addons-server
зеркало из https://github.com/mozilla/addons-server.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

implement impression api endpoint for sponsored shelf (#15687) 

* implement impression api endpoint for sponsored shelf

* update docs for impression endpoint
This commit is contained in:
Andrew Williamson 2020-10-12 11:11:29 +01:00 коммит произвёл GitHub
Родитель 87e82fa110
Коммит dfcde84625
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
6 изменённых файлов: 282 добавлений и 59 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

17
docs/topics/api/shelves.rst
Просмотреть файл

@ -101,4 +101,19 @@ Current implementation relies on Adzerk to determine which addons are returned a
:>json string results[].click_url: the url to ping if the sponsored addon's detail page is navigated to.
:>json string results[].click_data: the data payload to send to ``click_url`` that identifies the sponsored placement clicked on.
:>json string impression_url: the url to ping when the contents of this sponsored shelf is rendered on screen to the user.
:>json string impression_data: the data payload to send to ``impression_url`` that identifies the sponsored placements displayed.
:>json string impression_data: the signed data payload to send to ``impression_url`` that identifies the sponsored placements displayed.
---------------------------
Sponsored Shelf Impressions
---------------------------
.. _sponsored-shelf-impression:
When the sponsored shelf is displayed for the user this endpoint can be used to record the impressions.
The current implemenation forwards these impression pings to Adzerk.
.. http:post:: /api/v4/shelves/sponsored/impression/
:form string impression_data: the signed data payload that was sent in the :ref:`sponsored shelf <sponsored-shelf>` response.

2
src/olympia/lib/settings_base.py
Просмотреть файл

@ -1940,3 +1940,5 @@ ADZERK_TIMEOUT = 5 # seconds
ADZERK_NETWORK_ID = env('ADZERK_NETWORK_ID', default=10521)
ADZERK_SITE_ID = env('ADZERK_SITE_ID', default=1131244)
ADZERK_URL = f'https://e-{ADZERK_NETWORK_ID}.adzerk.net/api/v2'
ADZERK_IMPRESSION_URL = f'https://e-{ADZERK_NETWORK_ID}.adzerk.net/i.gif?'
ADZERK_IMPRESSION_TIMEOUT = 60 # seconds

153
src/olympia/shelves/tests/test_utils.py
Просмотреть файл

@ -1,14 +1,26 @@
import json
import os
from unittest import mock
from collections import namedtuple
from datetime import timedelta
from unittest import mock, TestCase
from django.conf import settings
from django.core.signing import TimestampSigner
from django.utils.encoding import force_bytes
from rest_framework.exceptions import APIException
import responses
from freezegun import freeze_time
from ..utils import (
call_adzerk_server, get_addons_from_adzerk, process_adzerk_results)
call_adzerk_server,
filter_adzerk_results_to_es_results_qs,
get_addons_from_adzerk,
get_signed_impression_blob_from_results,
get_impression_data_from_signed_blob,
process_adzerk_results,
send_impression_pings)
# This is a copy of a response from adzerk (with click and impressions trimmed)
@ -26,12 +38,12 @@ def test_get_addons_from_adzerk_full():
results = get_addons_from_adzerk(4)
assert results == {
'415198': {
'impression': 'e=eyJ2IjLtgBug',
'click': 'e=eyJ2IjoiMS42IiwiYXLCA',
'impression': 'e%3DeyJ2IjLtgBug',
'click': 'e%3DeyJ2IjoiMS42IiwiYXLCA',
'addon_id': '415198'},
'566314': {
'impression': 'e=eyJ2IjoNtSz8',
'click': 'e=eyJ2IjoiMS42IiwiYU5Hw',
'impression': 'e%3DeyJ2IjoNtSz8',
'click': 'e%3DeyJ2IjoiMS42IiwiYU5Hw',
'addon_id': '566314'},
}
@ -42,12 +54,10 @@ def test_call_adzerk_server(statsd_mock):
responses.POST,
settings.ADZERK_URL,
json=adzerk_json)
placeholders = ['div0', 'div1', 'div2']
site_id = settings.ADZERK_SITE_ID
network_id = settings.ADZERK_NETWORK_ID
results = call_adzerk_server(placeholders)
assert responses.calls[0].request.body == force_bytes(json.dumps(
{'placements': [
data = {
'placements': [
{
"divName": 'div0',
"networkId": network_id,
@ -63,7 +73,10 @@ def test_call_adzerk_server(statsd_mock):
"networkId": network_id,
"siteId": site_id,
"adTypes": [5]},
]}))
]}
results = call_adzerk_server(settings.ADZERK_URL, data)
assert responses.calls[0].request.body == force_bytes(json.dumps(
data))
assert 'div0' in results['decisions']
assert 'div1' in results['decisions']
# we're using a real response that only contains two divs
@ -72,12 +85,10 @@ def test_call_adzerk_server(statsd_mock):
@mock.patch('olympia.shelves.utils.statsd.incr')
def test_call_adzerk_server_empty_response(statsd_mock):
placeholders = ['div1', 'div2', 'div3']
responses.add(
responses.POST,
settings.ADZERK_URL)
results = call_adzerk_server(placeholders)
results = call_adzerk_server(settings.ADZERK_URL)
assert results == {}
statsd_mock.assert_called_with('services.adzerk.fail')
@ -87,7 +98,7 @@ def test_call_adzerk_server_empty_response(statsd_mock):
settings.ADZERK_URL,
status=500,
json={})
results = call_adzerk_server(placeholders)
results = call_adzerk_server(settings.ADZERK_URL)
assert results == {}
statsd_mock.assert_called_with('services.adzerk.fail')
@ -99,8 +110,8 @@ def test_process_adzerk_results():
response = {
'decisions': {
'foo1': {
"clickUrl": "https://e-9999.adzerk.net/r?e=eyJ2IjoiMS42IiwiA",
"impressionUrl": "https://e-9999.adzerk.net/i.gif?e=eyJ2IjLg",
"clickUrl": "https://e-9999.adzerk.net/r?e=eyJ2IjoiMS42I&iwiA",
"impressionUrl": "https://e-9999.adzerk.net/i.gif?e=eyJ2I&jLg",
"contents": [{
"data": {
"customData": {
@ -160,13 +171,13 @@ def test_process_adzerk_results():
results = process_adzerk_results(response, placeholders)
assert results == {
'1234': {
'impression': 'e=eyJ2IjLg',
'click': 'e=eyJ2IjoiMS42IiwiA',
'impression': 'e%3DeyJ2I%26jLg',
'click': 'e%3DeyJ2IjoiMS42I%26iwiA',
'addon_id': '1234',
},
'1': {
'impression': 'e=thesfsg',
'click': 'e=ey44545',
'impression': 'e%3Dthesfsg',
'click': 'e%3Dey44545',
'addon_id': '1',
}
}
@ -175,14 +186,110 @@ def test_process_adzerk_results():
@mock.patch('olympia.shelves.utils.process_adzerk_results')
@mock.patch('olympia.shelves.utils.call_adzerk_server')
def test_get_addons_from_adzerk(call_server_mock, process_mock):
site_id = settings.ADZERK_SITE_ID
network_id = settings.ADZERK_NETWORK_ID
data = {
'placements': [
{
"divName": 'div0',
"networkId": network_id,
"siteId": site_id,
"adTypes": [5]},
{
"divName": 'div1',
"networkId": network_id,
"siteId": site_id,
"adTypes": [5]},
]}
call_server_mock.return_value = None
process_mock.return_value = {'thing': {}}
assert get_addons_from_adzerk(2) == {}
call_server_mock.assert_called_with(['div0', 'div1'])
call_server_mock.assert_called_with(
settings.ADZERK_URL, data)
process_mock.assert_not_called()
call_server_mock.return_value = {'something': 'something'}
assert get_addons_from_adzerk(3) == {'thing': {}}
call_server_mock.assert_called_with(['div0', 'div1', 'div2'])
data['placements'].append(
{
"divName": 'div2',
"networkId": network_id,
"siteId": site_id,
"adTypes": [5]},
)
call_server_mock.assert_called_with(
settings.ADZERK_URL, data)
process_mock.assert_called_with(
{'something': 'something'}, ['div0', 'div1', 'div2'])
@mock.patch('olympia.shelves.utils.statsd.incr')
def test_send_impression_pings(incr_mock):
impressions = [
'e%3DeyJ2IjLtg%26Bug',
'e%3DeyJ2IjoNtSz8',
]
responses.add(
responses.GET,
settings.ADZERK_IMPRESSION_URL + 'e=eyJ2IjLtg&Bug')
responses.add(
responses.GET,
settings.ADZERK_IMPRESSION_URL + 'e=eyJ2IjoNtSz8')
send_impression_pings(impressions)
incr_mock.assert_called_with('services.adzerk.impression.success')
def test_filter_adzerk_results_to_es_results_qs():
results = {
'99': {},
'123': {},
'33': {},
}
hit = namedtuple('hit', 'id')
es_results = [
hit(99),
hit(66),
hit(33),
]
filter_adzerk_results_to_es_results_qs(results, es_results)
assert results == {
'99': {},
'33': {}
}
@freeze_time('2020-01-01')
def test_get_signed_impression_blob_from_results():
signer = TimestampSigner()
results = {
'66': {
'addon_id': 66,
'impression': '123456',
'click': 'abcdef'},
'99': {
'addon_id': 99,
'impression': '012345',
'click': 'bcdefg'},
}
blob = get_signed_impression_blob_from_results(results)
assert blob == signer.sign('123456,012345')
assert blob == '123456,012345:1imRQe:bYOCLk1ZS18trP34EnE8Ph5ykFI'
def test_get_impression_data_from_signed_blob():
blob = '123456,012345:1imRQe:bYOCLk1ZS18trP34EnE8Ph5ykFI'
with freeze_time('2020-01-01') as freezer:
# bad
with TestCase.assertRaises(None, APIException):
get_impression_data_from_signed_blob('.' + blob)
# good
impressions = get_impression_data_from_signed_blob(blob)
assert impressions == ['123456', '012345']
# good, but now stale
freezer.tick(delta=timedelta(seconds=61))
with TestCase.assertRaises(None, APIException):
get_impression_data_from_signed_blob(blob)

47
src/olympia/shelves/tests/test_views.py
Просмотреть файл

@ -1,9 +1,13 @@
import json
from datetime import timedelta
from unittest import mock
from django.conf import settings
from django.core.signing import TimestampSigner
from django.utils.encoding import force_text
from freezegun import freeze_time
from olympia import amo
from olympia.amo.tests import (
addon_factory, APITestClient, ESTestCase, reverse_ns)
@ -168,7 +172,6 @@ class TestSponsoredShelfViewSet(ESTestCase):
assert data['page_count'] == 1
assert data['impression_url'] == reverse_ns(
'sponsored-shelf-impression')
assert data['impression_data'] is None
return data
def test_no_adzerk_addons(self):
@ -178,8 +181,11 @@ class TestSponsoredShelfViewSet(ESTestCase):
get.assert_called_with(6)
assert data['count'] == 0
assert len(data['results']) == 0
assert data['impression_data'] is None
@freeze_time('2020-01-01')
def test_basic(self):
signer = TimestampSigner()
with mock.patch('olympia.shelves.views.get_addons_from_adzerk') as get:
get.return_value = {
str(self.sponsored_ext.id): {
@ -195,10 +201,13 @@ class TestSponsoredShelfViewSet(ESTestCase):
get.assert_called_with(6), get.call_args
assert data['count'] == 2
assert len(data['results']) == 2
assert data['impression_data'] == signer.sign('123456,012345')
assert {itm['id'] for itm in data['results']} == {
self.sponsored_ext.pk, self.sponsored_theme.pk}
@freeze_time('2020-01-01')
def test_adzerk_returns_none_sponsored(self):
signer = TimestampSigner()
with mock.patch('olympia.shelves.views.get_addons_from_adzerk') as get:
get.return_value = {
str(self.sponsored_ext.id): {
@ -227,6 +236,7 @@ class TestSponsoredShelfViewSet(ESTestCase):
# non sponsored are ignored
assert data['count'] == 2
assert len(data['results']) == 2
assert data['impression_data'] == signer.sign('123456,012345')
assert {itm['id'] for itm in data['results']} == {
self.sponsored_ext.pk, self.sponsored_theme.pk}
@ -238,15 +248,42 @@ class TestSponsoredShelfViewSet(ESTestCase):
get.assert_called_with(4)
assert data['count'] == 0
assert len(data['results']) == 0
assert data['impression_data'] is None
def test_impression_endpoint(self):
@mock.patch('olympia.shelves.views.send_impression_pings')
def test_impression_endpoint(self, send_mock):
url = reverse_ns('sponsored-shelf-impression')
with self.assertNumQueries(0):
response = self.client.post(url)
assert response.status_code == 200
# no data
response = self.client.post(url)
assert response.status_code == 400
send_mock.assert_not_called()
# bad data
response = self.client.post(url, {'impression_data': 'dfdfd:3434'})
assert response.status_code == 400
send_mock.assert_not_called()
# good data
signer = TimestampSigner()
impressions = ['assfsf', 'fwafsf']
data = signer.sign(','.join(impressions))
response = self.client.post(url, {'impression_data': data})
assert response.status_code == 202
send_mock.assert_called_with(impressions)
assert response.content == b''
# good data but stale
send_mock.reset_mock()
with freeze_time('2020-01-01') as freezer:
data = signer.sign(','.join(impressions))
freezer.tick(delta=timedelta(seconds=61))
response = self.client.post(url, {'impression_data': data})
assert response.status_code == 400
send_mock.assert_not_called()
def test_click_endpoint(self):
url = reverse_ns('sponsored-shelf-click')
with self.assertNumQueries(0):
response = self.client.post(url)
assert response.status_code == 200
assert response.content == b''

89
src/olympia/shelves/utils.py
Просмотреть файл

@ -1,9 +1,11 @@
from urllib.parse import urlparse
from urllib.parse import quote, unquote, urlparse
from django.conf import settings
from django.core.signing import BadSignature, SignatureExpired, TimestampSigner
import requests
from django_statsd.clients import statsd
from rest_framework.exceptions import APIException
import olympia.core.logger
@ -11,26 +13,15 @@ import olympia.core.logger
log = olympia.core.logger.getLogger('z.shelves')
def call_adzerk_server(placeholders):
"""Call adzerk server to get sponsored addon results.
`placeholders` is a list of arbitrary strings that we pass so we can
identify the order of the results in the response dict."""
site_id = settings.ADZERK_SITE_ID
network_id = settings.ADZERK_NETWORK_ID
placements = [
{"divName": ph,
"networkId": network_id,
"siteId": site_id,
"adTypes": [5]} for ph in placeholders]
def call_adzerk_server(url, json_data=None):
"""Call adzerk server to get sponsored addon results."""
json_response = {}
try:
log.info('Calling adzerk')
with statsd.timer('services.adzerk'):
response = requests.post(
settings.ADZERK_URL,
json={'placements': placements},
url,
json=json_data,
timeout=settings.ADZERK_TIMEOUT)
if response.status_code != 200:
raise requests.exceptions.RequestException()
@ -46,15 +37,31 @@ def call_adzerk_server(placeholders):
return json_response
def ping_adzerk_server(url, type='impression'):
"""Ping adzerk server for impression/clicks"""
try:
log.info('Calling adzerk')
with statsd.timer('services.adzerk'):
response = requests.get(
url,
timeout=settings.ADZERK_TIMEOUT)
if response.status_code != 200:
raise requests.exceptions.RequestException()
except requests.exceptions.RequestException as e:
log.exception('Calling adzerk failed: %s', e)
statsd.incr(f'services.adzerk.{type}.fail')
else:
statsd.incr(f'services.adzerk.{type}.success')
def process_adzerk_result(decision):
contents = (decision.get('contents') or [{}])[0]
return {
'impression': urlparse(decision.get('impressionUrl', '')).query,
'click': urlparse(decision.get('clickUrl', '')).query,
'impression': quote(urlparse(decision.get('impressionUrl', '')).query),
'click': quote(urlparse(decision.get('clickUrl', '')).query),
'addon_id':
contents.get('data', {}).get('customData', {}).get('id', None)
}
return
def process_adzerk_results(response, placeholders):
@ -72,8 +79,48 @@ def process_adzerk_results(response, placeholders):
def get_addons_from_adzerk(count):
placeholders = [f'div{i}' for i in range(count)]
response = call_adzerk_server(placeholders)
site_id = settings.ADZERK_SITE_ID
network_id = settings.ADZERK_NETWORK_ID
placements = [
{"divName": ph,
"networkId": network_id,
"siteId": site_id,
"adTypes": [5]} for ph in placeholders]
url = settings.ADZERK_URL
response = call_adzerk_server(url, {'placements': placements})
results_dict = (
process_adzerk_results(response, placeholders) if response else {})
log.debug(f'{results_dict=}')
return results_dict
def send_impression_pings(impressions):
base_url = settings.ADZERK_IMPRESSION_URL
urls = [f'{base_url}{unquote(impression)}' for impression in impressions]
for url in urls:
ping_adzerk_server(url, type='impression')
def filter_adzerk_results_to_es_results_qs(results, es_results_qs):
results_ids = [str(hit.id) for hit in es_results_qs]
for key in tuple(results.keys()):
if key not in results_ids:
results.pop(key)
def get_signed_impression_blob_from_results(adzerk_results):
impressions = [
result.get('impression') for result in adzerk_results.values()
if result.get('impression')]
if not impressions:
return None
signer = TimestampSigner()
return signer.sign(','.join(impressions))
def get_impression_data_from_signed_blob(blob):
signer = TimestampSigner()
try:
return signer.unsign(
blob, settings.ADZERK_IMPRESSION_TIMEOUT).split(',')
except (BadSignature, SignatureExpired) as e:
raise APIException(e)

33
src/olympia/shelves/views.py
Просмотреть файл

@ -1,8 +1,9 @@
from django.db.transaction import non_atomic_requests
from elasticsearch_dsl import Q, query
from rest_framework import mixins, viewsets
from rest_framework import mixins, status, viewsets
from rest_framework.decorators import action
from rest_framework.exceptions import APIException
from rest_framework.response import Response
from olympia.addons.views import AddonSearchView
@ -12,7 +13,12 @@ from olympia.search.filters import ReviewedContentFilter
from .models import Shelf
from .serializers import ESSponsoredAddonSerializer, ShelfSerializer
from .utils import get_addons_from_adzerk
from .utils import (
get_addons_from_adzerk,
get_impression_data_from_signed_blob,
get_signed_impression_blob_from_results,
filter_adzerk_results_to_es_results_qs,
send_impression_pings)
class ShelfViewSet(mixins.ListModelMixin, viewsets.GenericViewSet):
@ -35,28 +41,37 @@ class SponsoredShelfViewSet(viewsets.ViewSetMixin, AddonSearchView):
def as_view(cls, actions, **initkwargs):
return non_atomic_requests(super().as_view(actions, **initkwargs))
def get_impression_data(self):
return None
def get_paginated_response(self, data):
response = super().get_paginated_response(data)
response.data['impression_url'] = self.reverse_action('impression')
response.data['impression_data'] = self.get_impression_data()
response.data['impression_data'] = (
get_signed_impression_blob_from_results(self.adzerk_results))
return response
def filter_queryset(self, qs):
qs = super().filter_queryset(qs)
count = self.paginator.get_page_size(self.request)
results = get_addons_from_adzerk(count)
ids = list(results.keys())
self.adzerk_results = get_addons_from_adzerk(count)
ids = list(self.adzerk_results.keys())
results_qs = qs.query(query.Bool(must=[
Q('terms', id=ids),
Q('term', **{'promoted.group_id': VERIFIED_ONE.id})]))
results_qs.execute() # To cache the results.
filter_adzerk_results_to_es_results_qs(
self.adzerk_results, results_qs)
return results_qs
@action(detail=False, methods=['post'])
def impression(self, request):
return Response()
signed_impressions = request.data.get('impression_data', '')
try:
send_impression_pings(
get_impression_data_from_signed_blob(signed_impressions))
except APIException as e:
return Response(
f'Bad impression_data: {e}',
status=status.HTTP_400_BAD_REQUEST)
return Response(status=status.HTTP_202_ACCEPTED)
@action(detail=False, methods=['post'])
def click(self, request):