mozilla
/
addons-server
зеркало из https://github.com/mozilla/addons-server.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Remove obsolete/broken remote settings "test server mode" for dev (#22724) 

It's been broken for ages - that mode was designed to work around the fact
that the Kinto server was being regularly wiped out, so we had this special
user that created the user and collection dynamically, and then used its
own special bucket. But nowadays we can just use the same
user/bucket/collection everywhere, as the remote settigns server has the
same config everywhere.
This commit is contained in:
Mathieu Pillard 2024-10-01 10:30:38 +02:00 коммит произвёл GitHub
Родитель 780805ee84
Коммит 8bbd90c01f
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: B5690EEEBB952194
6 изменённых файлов: 7 добавлений и 176 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

2
src/olympia/conf/dev/settings.py
Просмотреть файл

@ -74,6 +74,4 @@ FXA_CONTENT_HOST = 'https://accounts.stage.mozaws.net'
FXA_OAUTH_HOST = 'https://oauth.stage.mozaws.net/v1'
FXA_PROFILE_HOST = 'https://profile.stage.mozaws.net/v1'
REMOTE_SETTINGS_IS_TEST_SERVER = True
SITEMAP_DEBUG_AVAILABLE = True

1
src/olympia/conf/prod/settings.py
Просмотреть файл

@ -73,7 +73,6 @@ REMOTE_SETTINGS_API_URL = 'https://firefox.settings.services.mozilla.com/v1/'
REMOTE_SETTINGS_WRITER_URL = env(
'REMOTE_SETTINGS_WRITER_URL', default='https://remote-settings.mozilla.org/v1/'
)
REMOTE_SETTINGS_WRITER_BUCKET = 'staging'
# See: https://bugzilla.mozilla.org/show_bug.cgi?id=1633746
BIGQUERY_AMO_DATASET = 'amo_prod'

1
src/olympia/conf/stage/settings.py
Просмотреть файл

@ -74,6 +74,5 @@ REMOTE_SETTINGS_API_URL = 'https://firefox.settings.services.allizom.org/v1/'
REMOTE_SETTINGS_WRITER_URL = env(
'REMOTE_SETTINGS_WRITER_URL', default='https://remote-settings.allizom.org/v1/'
)
REMOTE_SETTINGS_WRITER_BUCKET = 'staging'
CINDER_QUEUE_PREFIX = 'amo-stage-'

63
src/olympia/lib/remote_settings.py
Просмотреть файл

@ -28,15 +28,6 @@ class RemoteSettings:
self.collection = collection
self.sign_off_needed = sign_off_needed
def setup(self):
if self._setup_done:
return
if settings.REMOTE_SETTINGS_IS_TEST_SERVER:
self.setup_test_server_auth()
self.bucket = f'{self.bucket}_{self.username}'
self.setup_test_server_collection()
self._setup_done = True
@property
def headers(self):
b64 = b64encode(f'{self.username}:{self.password}'.encode()).decode()
@ -55,59 +46,10 @@ class RemoteSettings:
)
return 'id' in response.json().get('user', {})
def setup_test_server_auth(self):
# check if the user already exists in remote setting's accounts
host = settings.REMOTE_SETTINGS_WRITER_URL
response = requests.get(host, headers=self.headers)
user_id = response.json().get('user', {}).get('id')
if user_id != f'account:{self.username}':
# lets create it
log.info('Creating remote settings test account for %s' % self.username)
response = requests.put(
f'{host}accounts/{self.username}',
json={'data': {'password': self.password}},
headers={'Content-Type': 'application/json'},
)
if response.status_code != 201:
log.error(
'Creating remote settings test account for %s failed. [%s]'
% (self.username, response.content),
stack_info=True,
)
raise ConnectionError('Remote settings account not created')
def setup_test_server_collection(self):
# check if the bucket exists
bucket_url = f'{settings.REMOTE_SETTINGS_WRITER_URL}buckets/{self.bucket}'
headers = self.headers
response = requests.get(bucket_url, headers=headers)
data = {'permissions': {'read': ['system.Everyone']}}
if response.status_code == 403:
# lets create them
log.info(
'Creating remote settings bucket %s and collection %s'
% (self.bucket, self.collection)
)
response = requests.put(bucket_url, json=data, headers=headers)
# and the collection
collection_url = f'{bucket_url}/collections/{self.collection}'
response = requests.get(collection_url, headers=headers)
if response.status_code == 404:
response = requests.put(collection_url, json=data, headers=headers)
if response.status_code != 201:
log.error(
'Creating collection %s/%s failed: %s'
% (self.bucket, self.collection, response.content),
stack_info=True,
)
raise ConnectionError('Remote settings collection not created')
def publish_record(self, data, legacy_id=None):
"""Publish a record to remote settings. If `legacy_id` is not None the
existing record will be updated (PUT); otherwise a new record will be
created (POST)."""
self.setup()
add_url = (
f'{settings.REMOTE_SETTINGS_WRITER_URL}buckets/{self.bucket}/'
f'collections/{self.collection}/records'
@ -137,8 +79,6 @@ class RemoteSettings:
is not None the existing record will be updated; otherwise a new record
will be created.
`attachment` is a tuple of (filename, file object, content type)"""
self.setup()
if not legacy_id:
log.info('Creating record')
else:
@ -166,7 +106,6 @@ class RemoteSettings:
return response.json().get('data', {})
def delete_record(self, legacy_id):
self.setup()
url = (
f'{settings.REMOTE_SETTINGS_WRITER_URL}buckets/{self.bucket}/'
f'collections/{self.collection}/records/{legacy_id}'
@ -175,7 +114,6 @@ class RemoteSettings:
self._changes = True
def delete_all_records(self):
self.setup()
url = (
f'{settings.REMOTE_SETTINGS_WRITER_URL}buckets/{self.bucket}/'
f'collections/{self.collection}/records'
@ -186,7 +124,6 @@ class RemoteSettings:
def complete_session(self):
if not self._changes:
return
self.setup()
url = (
f'{settings.REMOTE_SETTINGS_WRITER_URL}buckets/{self.bucket}/'
f'collections/{self.collection}'

11
src/olympia/lib/settings_base.py
Просмотреть файл

@ -1536,14 +1536,15 @@ CUSTOMS_GIT_REPOSITORY = env('CUSTOMS_GIT_REPOSITORY', default=None)
DUAL_SIGNOFF_AVERAGE_DAILY_USERS_THRESHOLD = 100_000
REMOTE_SETTINGS_API_URL = 'https://remote-settings-dev.allizom.org/v1/'
REMOTE_SETTINGS_WRITER_URL = 'https://remote-settings-dev.allizom.org/v1/'
REMOTE_SETTINGS_WRITER_BUCKET = 'blocklists'
REMOTE_SETTINGS_WRITER_BUCKET = 'staging'
REMOTE_SETTINGS_CHECK_TIMEOUT_SECONDS = 10
# The remote settings test server needs accounts and setting up before using.
REMOTE_SETTINGS_IS_TEST_SERVER = False
BLOCKLIST_REMOTE_SETTINGS_USERNAME = env('BLOCKLIST_KINTO_USERNAME', default='amo_dev')
# Each env is expected to overwrite those credentials.
BLOCKLIST_REMOTE_SETTINGS_USERNAME = env(
'BLOCKLIST_KINTO_USERNAME', default='amo_remote_settings_username'
)
BLOCKLIST_REMOTE_SETTINGS_PASSWORD = env(
'BLOCKLIST_KINTO_PASSWORD', default='amo_dev_password'
'BLOCKLIST_KINTO_PASSWORD', default='amo_remote_settings_password'
)
# The path to the current google service account configuration for BigQuery.

105
src/olympia/lib/tests/test_kinto.py → src/olympia/lib/tests/test_remote_settings.py
Просмотреть файл

@ -16,114 +16,11 @@ from olympia.lib.remote_settings import RemoteSettings
BLOCKLIST_REMOTE_SETTINGS_PASSWORD='test_password',
)
class TestRemoteSettings(TestCase):
def test_setup_server_auth(self):
server = RemoteSettings('foo', 'baa')
responses.add(
responses.GET,
settings.REMOTE_SETTINGS_WRITER_URL,
content_type='application/json',
json={'user': {'id': ''}},
)
responses.add(
responses.PUT,
settings.REMOTE_SETTINGS_WRITER_URL + 'accounts/test_username',
content_type='application/json',
json={'data': {'password': 'test_password'}},
status=201,
)
server.setup_test_server_auth()
# If repeated then the account should exist the 2nd time
responses.add(
responses.GET,
settings.REMOTE_SETTINGS_WRITER_URL,
content_type='application/json',
json={'user': {'id': 'account:test_username'}},
)
server.setup_test_server_auth()
def test_setup_server_bucket(self):
server = RemoteSettings('foo', 'baa')
# if the server 403s on the bucket it's because it doesn't exist
responses.add(
responses.GET,
settings.REMOTE_SETTINGS_WRITER_URL + 'buckets/foo',
content_type='application/json',
status=403,
)
responses.add(
responses.PUT,
settings.REMOTE_SETTINGS_WRITER_URL + 'buckets/foo',
content_type='application/json',
)
# if the server 404s on the collection it's because it doesn't exist
responses.add(
responses.GET,
settings.REMOTE_SETTINGS_WRITER_URL + 'buckets/foo/collections/baa',
content_type='application/json',
status=404,
)
responses.add(
responses.PUT,
settings.REMOTE_SETTINGS_WRITER_URL + 'buckets/foo/collections/baa',
content_type='application/json',
status=201,
)
server.setup_test_server_collection()
def test_setup_server_collection(self):
server = RemoteSettings('foo', 'baa')
# But if the bucket exists then the collection should still be created
responses.add(
responses.GET,
settings.REMOTE_SETTINGS_WRITER_URL + 'buckets/foo',
content_type='application/json',
)
responses.add(
responses.GET,
settings.REMOTE_SETTINGS_WRITER_URL + 'buckets/foo/collections/baa',
content_type='application/json',
status=404,
)
responses.add(
responses.PUT,
settings.REMOTE_SETTINGS_WRITER_URL + 'buckets/foo/collections/baa',
content_type='application/json',
status=201,
)
server.setup_test_server_collection()
@override_settings(REMOTE_SETTINGS_IS_TEST_SERVER=False)
def test_setup_not_test_server(self):
def test_bucket_not_altered(self):
server = RemoteSettings('foo', 'baa')
server.setup() # will just return
assert server._setup_done
assert server.bucket == 'foo'
@override_settings(REMOTE_SETTINGS_IS_TEST_SERVER=True)
def test_setup(self):
server = RemoteSettings('foo', 'baa')
responses.add(
responses.GET,
settings.REMOTE_SETTINGS_WRITER_URL,
content_type='application/json',
json={'user': {'id': 'account:test_username'}},
)
bucket_url = settings.REMOTE_SETTINGS_WRITER_URL + 'buckets/foo_test_username'
responses.add(responses.GET, bucket_url, content_type='application/json')
responses.add(
responses.GET,
bucket_url + '/collections/baa',
content_type='application/json',
)
server.setup()
assert server._setup_done
assert server.bucket == 'foo_test_username'
server.setup() # a second time shouldn't make any requests
def test_publish_record(self):
server = RemoteSettings('foo', 'baa')
server._setup_done = True