зеркало из https://github.com/mozilla/gecko-dev.git
Bug 1563555 - Improve structure of generate_static_pref_list.py. r=glandium
generate_static_pref_list currently has two functions: - generate_header(): this checks the YAML and generates most of the code. - emit_header(): this does a bit of additional code generation and writes the code to file. This patch gives it a cleaner structure: - check_pref_list(): this checks the YAML. - generate_code(): this generates all the code. (It calls check_pref_list() first.) - emit_code(): this just writes the code to file. The patch also improves the testing in test_generate_static_pref_list.py, so that it checks the full contents of all generated files. All these improvements will help with the next patch, which adds two additional generated files. The patch also fixes a couple of minor errors in the comment at the top of StaticPrefList.yaml. Differential Revision: https://phabricator.services.mozilla.com/D40790 --HG-- extra : moz-landing-system : lando
This commit is contained in:
Nicholas Nethercote
Родитель
9c8bc605a0
Коммит
fb3388ccf2
|
|
@ -33,7 +33,7 @@
|
|||
# type: <cpp-type> # mandatory
|
||||
# value: <default-value> # mandatory
|
||||
# mirror: <never | once | always> # mandatory
|
||||
# do_not_use_directly: <True | False> # optional
|
||||
# do_not_use_directly: <true | false> # optional
|
||||
# include: <header-file> # optional
|
||||
#
|
||||
# - `name` is the name of the pref, without double-quotes, as it appears
|
||||
|
|
@ -88,6 +88,10 @@
|
|||
# not present. Cannot be used with a `never` mirror value, because there is
|
||||
# no getter function in that case.
|
||||
#
|
||||
# - `include` names a header file that must be included for the pref value to
|
||||
# compile correctly, e.g. because it refers to a code constant. System
|
||||
# headers should be surrounded with angle brackets, e.g. `<cmath>`.
|
||||
#
|
||||
# The getter function's base name is the same as the pref's name, but with
|
||||
# '.' or '-' chars converted to '_', to make a valid identifier. For example,
|
||||
# the getter for `foo.bar_baz` is `foo_bar_baz()`. This is ugly but clear,
|
||||
|
|
@ -100,10 +104,6 @@
|
|||
# - If the `do_not_use_directly` value is true, `_DoNotUseDirectly` is
|
||||
# appended.
|
||||
#
|
||||
# - `include` names a header file that must be included for the pref value to
|
||||
# compile correctly, e.g. because it refers to a code constant. System
|
||||
# headers should be surrounded with angle brackets, e.g. `<cmath>`.
|
||||
#
|
||||
# Preprocessor
|
||||
# ------------
|
||||
# Note finally that this file is preprocessed by preprocessor.py, not the C++
|
||||
|
|
|
|||
|
|
@ -44,9 +44,7 @@ VALID_TYPES = VALID_BOOL_TYPES.union({
|
|||
'AtomicFloat',
|
||||
})
|
||||
|
||||
FIRST_LINE = '''\
|
||||
// This file was generated by generate_static_pref_list.py. DO NOT EDIT.
|
||||
'''
|
||||
FIRST_LINE = '// This file was generated by generate_static_pref_list.py. DO NOT EDIT.'
|
||||
|
||||
MIRROR_TEMPLATES = {
|
||||
'never': '''\
|
||||
|
|
@ -72,15 +70,14 @@ ALWAYS_PREF(
|
|||
''',
|
||||
}
|
||||
|
||||
PREFS_FILE_TEMPLATE1 = '''\
|
||||
STATIC_PREFS_GROUP_H_TEMPLATE1 = '''\
|
||||
// Include it to gain access to StaticPrefs::{group}_*.
|
||||
|
||||
#ifndef mozilla_StaticPrefs_{group}_h
|
||||
#define mozilla_StaticPrefs_{group}_h
|
||||
|
||||
'''
|
||||
|
||||
PREFS_FILE_TEMPLATE2 = '''\
|
||||
STATIC_PREFS_GROUP_H_TEMPLATE2 = '''\
|
||||
#include "mozilla/StaticPrefListBegin.h"
|
||||
#include "mozilla/StaticPrefList_{group}.h"
|
||||
#include "mozilla/StaticPrefListEnd.h"
|
||||
|
|
@ -98,34 +95,18 @@ def mk_id(name):
|
|||
return name.replace('.', '_').replace('-', '_')
|
||||
|
||||
|
||||
def pref_ids(pref):
|
||||
if pref['mirror'] == 'never':
|
||||
if pref.get('do_not_use_directly'):
|
||||
error('`do_not_use_directly` uselessly set with `mirror` value '
|
||||
'`never` for pref `{}`'.format(pref['name']))
|
||||
return (None, None)
|
||||
|
||||
base_id = mk_id(pref['name'])
|
||||
full_id = base_id
|
||||
if pref['mirror'] == 'once':
|
||||
full_id += '_AtStartup'
|
||||
if pref.get('do_not_use_directly'):
|
||||
full_id += '_DoNotUseDirectly'
|
||||
|
||||
return (base_id, full_id)
|
||||
def mk_group(pref):
|
||||
name = pref['name']
|
||||
return mk_id(name.split('.', 1)[0])
|
||||
|
||||
|
||||
def generate_headers(pref_list):
|
||||
# The generated code, one list of lines per pref group.
|
||||
code = defaultdict(list)
|
||||
|
||||
# The required includes, one set of header file names per pref group.
|
||||
includes = defaultdict(set)
|
||||
|
||||
def check_pref_list(pref_list):
|
||||
# Pref names seen so far. Used to detect any duplicates.
|
||||
seen_names = set()
|
||||
|
||||
# The previous pref. Used to detect mis-ordered prefs.
|
||||
prev_pref = None
|
||||
|
||||
for pref in pref_list:
|
||||
# Check all given keys are known ones.
|
||||
for key in pref:
|
||||
|
|
@ -138,17 +119,15 @@ def generate_headers(pref_list):
|
|||
name = pref['name']
|
||||
if type(name) != str:
|
||||
error('non-string `name` value `{}`'.format(name))
|
||||
if '.' not in name:
|
||||
error('`name` value `{}` lacks a \'.\''.format(name))
|
||||
if name in seen_names:
|
||||
error('`{}` pref is defined more than once'.format(name))
|
||||
seen_names.add(name)
|
||||
|
||||
segs = name.split('.', 1)
|
||||
if len(segs) != 2:
|
||||
error('`name` value `{}` lacks a \'.\''.format(name))
|
||||
group = mk_id(segs[0])
|
||||
# Prefs must be ordered appropriately.
|
||||
if prev_pref:
|
||||
prev_pref_group = prev_pref['name'].split('.', 1)[0]
|
||||
if prev_pref_group > group:
|
||||
if mk_group(prev_pref) > mk_group(pref):
|
||||
error('`{}` pref must come before `{}` pref'
|
||||
.format(name, prev_pref['name']))
|
||||
|
||||
|
|
@ -168,14 +147,8 @@ def generate_headers(pref_list):
|
|||
error('non-string `value` value `{}` for `String` pref `{}`; '
|
||||
'add double quotes'
|
||||
.format(value, name))
|
||||
# Quote string literals, and escape double-quote chars.
|
||||
value = '"{}"'.format(value.replace('"', '\\"'))
|
||||
elif typ in VALID_BOOL_TYPES:
|
||||
if value is True: # Convert Python bools to C++ bools.
|
||||
value = 'true'
|
||||
elif value is False:
|
||||
value = 'false'
|
||||
else:
|
||||
if value not in (True, False):
|
||||
error('invalid boolean value `{}` for pref `{}`'
|
||||
.format(value, name))
|
||||
|
||||
|
|
@ -193,38 +166,123 @@ def generate_headers(pref_list):
|
|||
if type(do_not_use_directly) != bool:
|
||||
error('non-boolean `do_not_use_directly` value `{}` for pref '
|
||||
'`{}`'.format(do_not_use_directly, name))
|
||||
if do_not_use_directly and mirror == 'never':
|
||||
error('`do_not_use_directly` uselessly set with `mirror` value '
|
||||
'`never` for pref `{}`'.format(pref['name']))
|
||||
|
||||
# Check and process 'include' if present.
|
||||
# Check 'include' if present.
|
||||
if 'include' in pref:
|
||||
include = pref['include']
|
||||
if type(include) != str:
|
||||
error('non-string `include` value `{}` for pref `{}`'
|
||||
.format(include, name))
|
||||
if include.startswith('<'):
|
||||
if not include.endswith('>'):
|
||||
error('`include` value `{}` starts with `<` but does not '
|
||||
'end with `>` for pref `{}`'.format(include, name))
|
||||
else:
|
||||
if include.startswith('<') and not include.endswith('>'):
|
||||
error('`include` value `{}` starts with `<` but does not '
|
||||
'end with `>` for pref `{}`'.format(include, name))
|
||||
|
||||
prev_pref = pref
|
||||
|
||||
|
||||
def generate_code(pref_list):
|
||||
check_pref_list(pref_list)
|
||||
|
||||
# The required includes for StaticPrefs_<group>.h.
|
||||
includes = defaultdict(set)
|
||||
|
||||
# StaticPrefList_<group>.h contains all the pref definitions for this
|
||||
# group.
|
||||
static_pref_list_group_h = defaultdict(lambda: [FIRST_LINE, ''])
|
||||
|
||||
# Generate the per-pref code (spread across multiple files).
|
||||
for pref in pref_list:
|
||||
name = pref['name']
|
||||
typ = pref['type']
|
||||
value = pref['value']
|
||||
mirror = pref['mirror']
|
||||
do_not_use_directly = pref.get('do_not_use_directly')
|
||||
include = pref.get('include')
|
||||
|
||||
base_id = mk_id(pref['name'])
|
||||
full_id = base_id
|
||||
if mirror == 'once':
|
||||
full_id += '_AtStartup'
|
||||
if do_not_use_directly:
|
||||
full_id += '_DoNotUseDirectly'
|
||||
|
||||
group = mk_group(pref)
|
||||
|
||||
if include:
|
||||
if not include.startswith('<'):
|
||||
# It's not a system header. Add double quotes.
|
||||
include = '"{}"'.format(include)
|
||||
includes[group].add(include)
|
||||
|
||||
if typ == 'String':
|
||||
# Quote string literals, and escape double-quote chars.
|
||||
value = '"{}"'.format(value.replace('"', '\\"'))
|
||||
elif typ in VALID_BOOL_TYPES:
|
||||
# Convert Python bools to C++ bools.
|
||||
if value is True:
|
||||
value = 'true'
|
||||
elif value is False:
|
||||
value = 'false'
|
||||
|
||||
# Append the C++ definition to the relevant output file's code.
|
||||
ids = pref_ids(pref)
|
||||
code[group].append(MIRROR_TEMPLATES[mirror].format(
|
||||
static_pref_list_group_h[group].append(MIRROR_TEMPLATES[mirror].format(
|
||||
name=name,
|
||||
base_id=ids[0],
|
||||
full_id=ids[1],
|
||||
base_id=base_id,
|
||||
full_id=full_id,
|
||||
typ=typ,
|
||||
value=value,
|
||||
))
|
||||
|
||||
prev_pref = pref
|
||||
# Delete this so that `group` can be reused below without Flake8
|
||||
# complaining.
|
||||
del group
|
||||
|
||||
return (code, includes)
|
||||
# StaticPrefListAll.h contains one `#include "mozilla/StaticPrefList_X.h`
|
||||
# line per pref group.
|
||||
static_pref_list_all_h = [FIRST_LINE, '']
|
||||
static_pref_list_all_h.extend(
|
||||
'#include "mozilla/StaticPrefList_{}.h"'.format(group)
|
||||
for group in sorted(static_pref_list_group_h)
|
||||
)
|
||||
static_pref_list_all_h.append('')
|
||||
|
||||
# StaticPrefsAll.h contains one `#include "mozilla/StaticPrefs_X.h` line per
|
||||
# pref group.
|
||||
static_prefs_all_h = [FIRST_LINE, '']
|
||||
static_prefs_all_h.extend(
|
||||
'#include "mozilla/StaticPrefs_{}.h"'.format(group)
|
||||
for group in sorted(static_pref_list_group_h)
|
||||
)
|
||||
static_prefs_all_h.append('')
|
||||
|
||||
# StaticPrefs_<group>.h wraps StaticPrefList_<group>.h. It is the header
|
||||
# used directly by application code.
|
||||
static_prefs_group_h = defaultdict(list)
|
||||
for group in sorted(static_pref_list_group_h):
|
||||
static_prefs_group_h[group] = [FIRST_LINE]
|
||||
static_prefs_group_h[group].append(STATIC_PREFS_GROUP_H_TEMPLATE1.format(group=group))
|
||||
if group in includes:
|
||||
# Add any necessary includes, from 'h_include' values.
|
||||
for include in sorted(includes[group]):
|
||||
static_prefs_group_h[group].append('#include {}'.format(include))
|
||||
static_prefs_group_h[group].append('')
|
||||
static_prefs_group_h[group].append(STATIC_PREFS_GROUP_H_TEMPLATE2.format(group=group))
|
||||
|
||||
def fold(lines):
|
||||
return '\n'.join(lines)
|
||||
|
||||
return {
|
||||
'static_pref_list_all_h': fold(static_pref_list_all_h),
|
||||
'static_prefs_all_h': fold(static_prefs_all_h),
|
||||
'static_pref_list_group_h': {k: fold(v) for k, v in static_pref_list_group_h.items()},
|
||||
'static_prefs_group_h': {k: fold(v) for k, v in static_prefs_group_h.items()},
|
||||
}
|
||||
|
||||
|
||||
def emit_header(fd, pref_list_filename):
|
||||
def emit_code(fd, pref_list_filename):
|
||||
pp = Preprocessor()
|
||||
pp.context.update(buildconfig.defines['ALLDEFINES'])
|
||||
|
||||
|
|
@ -238,52 +296,31 @@ def emit_header(fd, pref_list_filename):
|
|||
|
||||
try:
|
||||
pref_list = yaml.safe_load(pp.out.getvalue())
|
||||
(code, includes) = generate_headers(pref_list)
|
||||
code = generate_code(pref_list)
|
||||
except (IOError, ValueError) as e:
|
||||
print('{}: error:\n {}\n'
|
||||
.format(pref_list_filename, e))
|
||||
sys.exit(1)
|
||||
|
||||
# When generating multiple files from a script, the build system treats the
|
||||
# first named output file (StaticPrefList.h in this case) specially -- it
|
||||
# first named output file (StaticPrefListAll.h in this case) specially -- it
|
||||
# is created elsewhere, and written to via `fd`.
|
||||
#
|
||||
# This file has one `#include "mozilla/StaticPrefList_X.h` per pref group.
|
||||
fd.write(FIRST_LINE)
|
||||
fd.write('\n')
|
||||
for group in sorted(code):
|
||||
fd.write('#include "mozilla/StaticPrefList_{}.h"\n'.format(group))
|
||||
fd.write(code['static_pref_list_all_h'])
|
||||
|
||||
# We must create the remaining output files ourselves. This requires
|
||||
# creating the output directory directly if it doesn't already exist.
|
||||
ensureParentDir(fd.name)
|
||||
init_dirname = os.path.dirname(fd.name)
|
||||
base_dirname = os.path.dirname(init_dirname)
|
||||
|
||||
# This file has one `#include "mozilla/StaticPrefs_X.h` per pref group.
|
||||
with FileAvoidWrite(os.path.join(base_dirname, 'StaticPrefsAll.h')) as fd:
|
||||
fd.write(FIRST_LINE)
|
||||
fd.write('\n')
|
||||
for group in sorted(code):
|
||||
fd.write('#include "mozilla/StaticPrefs_{}.h"\n'.format(group))
|
||||
with FileAvoidWrite('StaticPrefsAll.h') as fd:
|
||||
fd.write(code['static_prefs_all_h'])
|
||||
|
||||
for group, lines in sorted(code.items()):
|
||||
# This header contains all the definitions for the pref group.
|
||||
for group, text in sorted(code['static_pref_list_group_h'].items()):
|
||||
filename = 'StaticPrefList_{}.h'.format(group)
|
||||
with FileAvoidWrite(os.path.join(init_dirname, filename)) as fd:
|
||||
fd.write(FIRST_LINE)
|
||||
fd.write('\n')
|
||||
fd.write('\n'.join(lines))
|
||||
fd.write(text)
|
||||
|
||||
# This header wraps the previous header file. It is the header used
|
||||
# directly by application code.
|
||||
for group, text in sorted(code['static_prefs_group_h'].items()):
|
||||
filename = 'StaticPrefs_{}.h'.format(group)
|
||||
with FileAvoidWrite(filename) as fd:
|
||||
fd.write(FIRST_LINE)
|
||||
fd.write(PREFS_FILE_TEMPLATE1.format(group=group))
|
||||
if group in includes:
|
||||
# Add any necessary includes, from 'include' values.
|
||||
for include in sorted(includes[group]):
|
||||
fd.write('#include {}\n'.format(include))
|
||||
fd.write('\n')
|
||||
fd.write(PREFS_FILE_TEMPLATE2.format(group=group))
|
||||
fd.write(text)
|
||||
|
|
|
|||
|
|
@ -109,7 +109,7 @@ EXPORTS.mozilla += sorted(['!' + gf for gf in genfiles])
|
|||
GENERATED_FILES += [genfiles_tuple]
|
||||
|
||||
static_pref_list = GENERATED_FILES[genfiles_tuple]
|
||||
static_pref_list.script = 'init/generate_static_pref_list.py:emit_header'
|
||||
static_pref_list.script = 'init/generate_static_pref_list.py:emit_code'
|
||||
static_pref_list.inputs = ['init/StaticPrefList.yaml']
|
||||
|
||||
PYTHON_UNITTEST_MANIFESTS += [
|
||||
|
|
|
|||
|
|
@ -13,14 +13,12 @@ from os import path
|
|||
from StringIO import StringIO
|
||||
|
||||
sys.path.append(path.join(path.dirname(__file__), ".."))
|
||||
from init.generate_static_pref_list import generate_headers
|
||||
from init.generate_static_pref_list import generate_code
|
||||
|
||||
test_data_path = mozpath.abspath(mozpath.dirname(__file__))
|
||||
test_data_path = mozpath.join(test_data_path, 'data')
|
||||
|
||||
# A single good input with lots of different combinations. All the prefs start
|
||||
# with "my." so they end up in the same group, which simplifies the comparison
|
||||
# against the expected output.
|
||||
# A single good input with lots of different combinations.
|
||||
good_input = '''
|
||||
- name: my.bool
|
||||
type: bool
|
||||
|
|
@ -80,15 +78,34 @@ good_input = '''
|
|||
|
||||
# YAML+Python changes `.4455667` to `0.4455667` because it interprets the value
|
||||
# as a float.
|
||||
- name: my.atomic.float
|
||||
- name: my-dashed.atomic.float
|
||||
type: AtomicFloat
|
||||
value: .4455667
|
||||
mirror: never
|
||||
include: <math.h>
|
||||
'''
|
||||
|
||||
# The corresponding output for good_input.
|
||||
good_output = '''\
|
||||
# The corresponding code for good_input.
|
||||
good = {}
|
||||
|
||||
good['static_pref_list_all_h'] = '''\
|
||||
// This file was generated by generate_static_pref_list.py. DO NOT EDIT.
|
||||
|
||||
#include "mozilla/StaticPrefList_my.h"
|
||||
#include "mozilla/StaticPrefList_my_dashed.h"
|
||||
'''
|
||||
|
||||
good['static_prefs_all_h'] = '''\
|
||||
// This file was generated by generate_static_pref_list.py. DO NOT EDIT.
|
||||
|
||||
#include "mozilla/StaticPrefs_my.h"
|
||||
#include "mozilla/StaticPrefs_my_dashed.h"
|
||||
'''
|
||||
|
||||
good['static_pref_list_group_h'] = {
|
||||
'my': '''\
|
||||
// This file was generated by generate_static_pref_list.py. DO NOT EDIT.
|
||||
|
||||
NEVER_PREF("my.bool", bool, false)
|
||||
|
||||
ONCE_PREF(
|
||||
|
|
@ -136,22 +153,33 @@ ONCE_PREF(
|
|||
my_atomic_uint_AtStartup,
|
||||
SequentiallyConsistentAtomicUint32, 68
|
||||
)
|
||||
''',
|
||||
'my_dashed': '''\
|
||||
// This file was generated by generate_static_pref_list.py. DO NOT EDIT.
|
||||
|
||||
NEVER_PREF("my.atomic.float", AtomicFloat, 0.4455667)
|
||||
NEVER_PREF("my-dashed.atomic.float", AtomicFloat, 0.4455667)
|
||||
'''
|
||||
}
|
||||
|
||||
good['static_prefs_group_h'] = {'my': '''\
|
||||
// This file was generated by generate_static_pref_list.py. DO NOT EDIT.
|
||||
// Include it to gain access to StaticPrefs::my_*.
|
||||
|
||||
#ifndef mozilla_StaticPrefs_my_h
|
||||
#define mozilla_StaticPrefs_my_h
|
||||
|
||||
#include "foobar.h"
|
||||
|
||||
#include "mozilla/StaticPrefListBegin.h"
|
||||
#include "mozilla/StaticPrefList_my.h"
|
||||
#include "mozilla/StaticPrefListEnd.h"
|
||||
|
||||
#endif // mozilla_StaticPrefs_my_h
|
||||
'''}
|
||||
|
||||
# A lot of bad inputs, each with an accompanying error message. Listed in order
|
||||
# of the relevant `error` calls within generate_static_pref_list.py.
|
||||
bad_inputs = [
|
||||
('''
|
||||
- name: do_not_use_directly.uselessly.set
|
||||
type: int32_t
|
||||
value: 0
|
||||
mirror: never
|
||||
do_not_use_directly: true
|
||||
''', '`do_not_use_directly` uselessly set with `mirror` value `never` for '
|
||||
'pref `do_not_use_directly.uselessly.set`'),
|
||||
|
||||
('''
|
||||
- invalidkey: 3
|
||||
''', 'invalid key `invalidkey`'),
|
||||
|
|
@ -245,6 +273,15 @@ bad_inputs = [
|
|||
''', 'non-boolean `do_not_use_directly` value `0` for pref '
|
||||
'`non-boolean.do_not_use_directly.value`'),
|
||||
|
||||
('''
|
||||
- name: do_not_use_directly.uselessly.set
|
||||
type: int32_t
|
||||
value: 0
|
||||
mirror: never
|
||||
do_not_use_directly: true
|
||||
''', '`do_not_use_directly` uselessly set with `mirror` value `never` for '
|
||||
'pref `do_not_use_directly.uselessly.set`'),
|
||||
|
||||
('''
|
||||
- name: non-string.include.value
|
||||
type: bool
|
||||
|
|
@ -273,8 +310,21 @@ class TestGenerateStaticPrefList(unittest.TestCase):
|
|||
'Test various pieces of good input.'
|
||||
inp = StringIO(good_input)
|
||||
pref_list = yaml.safe_load(inp)
|
||||
(code, includes) = generate_headers(pref_list)
|
||||
self.assertEqual(good_output, '\n'.join(code['my']))
|
||||
code = generate_code(pref_list)
|
||||
|
||||
self.assertEqual(good['static_pref_list_all_h'],
|
||||
code['static_pref_list_all_h'])
|
||||
|
||||
self.assertEqual(good['static_prefs_all_h'],
|
||||
code['static_prefs_all_h'])
|
||||
|
||||
self.assertEqual(good['static_pref_list_group_h']['my'],
|
||||
code['static_pref_list_group_h']['my'])
|
||||
self.assertEqual(good['static_pref_list_group_h']['my_dashed'],
|
||||
code['static_pref_list_group_h']['my_dashed'])
|
||||
|
||||
self.assertEqual(good['static_prefs_group_h']['my'],
|
||||
code['static_prefs_group_h']['my'])
|
||||
|
||||
def test_bad(self):
|
||||
'Test various pieces of bad input.'
|
||||
|
|
@ -283,7 +333,7 @@ class TestGenerateStaticPrefList(unittest.TestCase):
|
|||
inp = StringIO(input_string)
|
||||
try:
|
||||
pref_list = yaml.safe_load(inp)
|
||||
generate_headers(pref_list)
|
||||
generate_code(pref_list)
|
||||
self.assertEqual(0, 1)
|
||||
except ValueError as e:
|
||||
self.assertEqual(str(e), expected)
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче