mozilla
/
gecko-dev
зеркало из https://github.com/mozilla/gecko-dev.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность
gecko-dev/gfx/angle/update-angle.py

594 строки
16 KiB
Python
Executable File
Исходник Ответственный История

#! /usr/bin/env python3
assert __name__ == '__main__'
'''
To update ANGLE in Gecko, use Windows with git-bash, and setup depot_tools, python2, and
python3. Because depot_tools expects `python` to be `python2` (shame!), python2 must come
before python3 in your path.
Upstream: https://chromium.googlesource.com/angle/angle
Our repo: https://github.com/mozilla/angle
It has branches like 'firefox-60' which is the branch we use for pulling into
Gecko with this script.
This script leaves a record of the merge-base and cherry-picks that we pull into
Gecko. (gfx/angle/cherries.log)
ANGLE<->Chrome version mappings are here: https://omahaproxy.appspot.com/
An easy choice is to grab Chrome's Beta's ANGLE branch.
## Usage
Prepare your env:
~~~
export PATH="$PATH:/path/to/depot_tools"
~~~
If this is a new repo, don't forget:
~~~
# In the angle repo:
./scripts/bootstrap.py
gclient sync
~~~
Update: (in the angle repo)
~~~
# In the angle repo:
/path/to/gecko/gfx/angle/update-angle.py origin/chromium/XXXX
git push moz # Push the firefox-XX branch to github.com/mozilla/angle
~~~~
'''
import json
import os
from pathlib import *
import re
import shutil
import subprocess
import sys
from vendor_from_git import *
REPO_DIR = Path.cwd()
GECKO_ANGLE_DIR = Path(__file__).parent
OUT_DIR = 'out'
COMMON_HEADER = [
'# Generated by update-angle.py',
'',
"include('../../moz.build.common')",
]
VENDOR_PREREQ_TARGETS = [
'//:commit_id', # Generate 'commit.h'.
]
ROOTS = ['//:translator', '//:libEGL', '//:libGLESv2']
DRY_RUN = '--dry' in sys.argv
ACTION_PREFIX = ''
if DRY_RUN:
ACTION_PREFIX = '(not) '
GN_ENV = dict(os.environ)
GN_ENV['DEPOT_TOOLS_WIN_TOOLCHAIN'] = '0'
(MERGE_BASE_ORIGIN, ) = sys.argv[1:] # Not always 'origin'!
# --------------------------------------
def sorted_items(x):
for k in sorted(x.keys()):
yield (k, x[k])
def collapse_dotdots(path):
split = path.split('/')
ret = []
for x in split:
if x == '..' and ret:
ret.pop()
continue
ret.append(x)
continue
return '/'.join(ret)
def traverse(roots, pre_recurse_func, key_func=id):
visited = set()
def recurse(cur):
key = key_func(cur)
if key in visited:
return
visited.add(key)
t = pre_recurse_func(cur)
post_recurse_func = None
try:
(children, post_recurse_func) = t
except ValueError:
(children,) = t
for x in children:
recurse(x)
if post_recurse_func:
post_recurse_func(cur)
return
for x in roots:
recurse(x)
return
# --------------------------------------
if not DRY_RUN:
record_cherry_picks(GECKO_ANGLE_DIR, MERGE_BASE_ORIGIN)
# --
print_now('Importing graph')
shutil.rmtree(OUT_DIR, True)
run_checked('gn', 'gen', OUT_DIR, shell=True, env=GN_ENV)
GN_ARGS = '''
# Build arguments go here.
# See "gn args <out_dir> --list" for available build arguments.
is_clang = false
angle_enable_gl = false
angle_enable_gl_null = false
angle_enable_null = false
angle_enable_vulkan = false
'''[1:]
with open(OUT_DIR + '/args.gn', 'wb') as f:
f.write(GN_ARGS.encode())
# --
p = run_checked('gn', 'desc', '--format=json', OUT_DIR, '*', stdout=subprocess.PIPE,
shell=True, env=GN_ENV)
print_now('Processing graph')
descs = json.loads(p.stdout.decode())
# HACKHACKHACK
common = descs['//:angle_common']
common['sources'] += [
'//src/common/system_utils_linux.cpp',
'//src/common/system_utils_mac.cpp',
]
# --
for (k, v) in descs.items():
for (k2, v2) in v.items():
if type(v2) == list:
v[k2] = tuple(v2) # Freeze lists
v['target_name'] = k
v['dep_nodes'] = tuple([descs[x] for x in v['deps']])
assert v['public'] == '*', k
# --
# Ready to traverse
ROOTS = [descs[k] for k in ROOTS]
# Gather real targets:
real_targets = []
def gather_real_targets(cur):
print_now(' ' + cur['type'], cur['target_name'])
if cur['type'] in ['shared_library', 'static_library']:
real_targets.append(cur)
def post(x):
x['sources_with_deps'] = x.get('sources', ())
x['include_dirs_with_deps'] = x.get('include_dirs', ())
for y in x['dep_nodes']:
x['sources_with_deps'] += y['sources_with_deps']
x['include_dirs_with_deps'] += y['include_dirs_with_deps']
return (cur['dep_nodes'], post)
traverse(ROOTS, gather_real_targets)
# --
def sortedi(x):
return sorted(x, key=str.lower)
def append_arr(dest, name, vals, indent=0):
if not vals:
return
dest.append('{}{} += ['.format(' '*4*indent, name))
for x in sortedi(vals):
dest.append("{}'{}',".format(' '*4*(indent+1), x))
dest.append('{}]'.format(' '*4*indent))
dest.append('')
return
INCLUDE_REGEX = re.compile('# *include +([<"])([^>"]+)[>"]')
IGNORED_INCLUDES = {
'compiler/translator/TranslatorVulkan.h',
'libANGLE/renderer/d3d/d3d11/winrt/NativeWindow11WinRT.h',
'libANGLE/renderer/gl/glx/DisplayGLX.h',
'libANGLE/renderer/gl/cgl/DisplayCGL.h',
'libANGLE/renderer/gl/egl/ozone/DisplayOzone.h',
'libANGLE/renderer/gl/egl/android/DisplayAndroid.h',
'libANGLE/renderer/gl/wgl/DisplayWGL.h',
'libANGLE/renderer/null/DisplayNULL.h',
'libANGLE/renderer/vulkan/android/DisplayVkAndroid.h',
'libANGLE/renderer/vulkan/win32/DisplayVkWin32.h',
'libANGLE/renderer/vulkan/xcb/DisplayVkXcb.h',
'kernel/image.h',
}
IGNORED_INCLUDE_PREFIXES = {
'android/',
'Carbon/',
'CoreFoundation/',
'CoreServices/',
'IOSurface/',
'mach/',
'mach-o/',
'OpenGL/',
'pci/',
'sys/',
'wrl/',
'X11/',
}
REGISTERED_DEFINES = {
'ANGLE_ENABLE_D3D11': True,
'ANGLE_ENABLE_D3D9': True,
'ANGLE_ENABLE_DEBUG_ANNOTATIONS': True,
'ANGLE_ENABLE_NULL': False,
'ANGLE_ENABLE_OPENGL': False,
'ANGLE_ENABLE_OPENGL_NULL': False,
'ANGLE_ENABLE_ESSL': True,
'ANGLE_ENABLE_GLSL': True,
'ANGLE_ENABLE_HLSL': True,
'ANGLE_GENERATE_SHADER_DEBUG_INFO': True,
'ANGLE_IS_64_BIT_CPU': False,
'ANGLE_PRELOADED_D3DCOMPILER_MODULE_NAMES': False,
'CERT_CHAIN_PARA_HAS_EXTRA_FIELDS': False,
'CHROMIUM_BUILD': False,
'COMPONENT_BUILD': False,
'constexpr14': False,
'DYNAMIC_ANNOTATIONS_ENABLED': True,
'EGL_EGLEXT_PROTOTYPES': True,
'EGLAPI': True,
'FIELDTRIAL_TESTING_ENABLED': False,
'FULL_SAFE_BROWSING': False,
'GL_API': True,
'GL_APICALL': True,
'GL_GLEXT_PROTOTYPES': True,
'GPU_INFO_USE_SETUPAPI': True,
'LIBANGLE_IMPLEMENTATION': True,
'LIBEGL_IMPLEMENTATION': True,
'LIBGLESV2_IMPLEMENTATION': True,
'NOMINMAX': True,
'NO_TCMALLOC': False,
# Else: gfx/angle/checkout/src/libANGLE/renderer/d3d/d3d11/win32/NativeWindow11Win32.cpp(89): error C2787: 'IDCompositionDevice': no GUID has been associated with this object
'NTDDI_VERSION': True,
'PSAPI_VERSION': False,
'SAFE_BROWSING_CSD': False,
'SAFE_BROWSING_DB_LOCAL': False,
'UNICODE': True,
'USE_AURA': False,
'V8_DEPRECATION_WARNINGS': False,
'WIN32': False,
'WIN32_LEAN_AND_MEAN': False,
'WINAPI_FAMILY': False,
'WINVER': False,
'WTF_USE_DYNAMIC_ANNOTATIONS': False,
'_ATL_NO_OPENGL': True,
'_CRT_RAND_S': True,
'_CRT_SECURE_NO_DEPRECATE': True,
'_DEBUG': False,
'_HAS_EXCEPTIONS': True,
'_HAS_ITERATOR_DEBUGGING': False,
'_SCL_SECURE_NO_DEPRECATE': True,
'_SECURE_ATL': True,
'_UNICODE': True,
'_USING_V110_SDK71_': False,
'_WIN32_WINNT': False,
'_WINDOWS': False,
'__STD_C': False,
}
SOURCE_FILE_EXTS = frozenset(['h', 'hpp', 'inc', 'inl', 'c', 'cc', 'cpp'])
def is_source_file(x):
e = x.split('.')[-1]
return e in SOURCE_FILE_EXTS
def assert_valid_includes(target_name, cur, avail_files, include_dirs):
assert cur.startswith('//'), cur
cur = PurePosixPath(cur[2:])
(cur_dir, _) = os.path.split(cur)
include_dirs = [
'//',
'//' + cur_dir + '/',
] + list(include_dirs)
def assert_one(inc, line_num):
attempts = []
for inc_dir in include_dirs:
assert inc_dir[-1] == '/'
inc_path = inc_dir + inc
inc_path = collapse_dotdots(inc_path)
attempts.append(inc_path)
if inc_path in avail_files:
return
print('Warning in {}: {}:{}: Invalid include: {}'.format(target_name, cur, line_num, inc))
print(' Tried:')
for x in attempts:
print(' {}'.format(x))
#print()
#print(avail_files)
exit(1)
line_num = 0
with open(cur, 'rb') as f:
for line in f:
line = line.decode()
line_num += 1
m = INCLUDE_REGEX.match(line)
if not m:
continue
inc = m.group(2)
if inc in IGNORED_INCLUDES:
continue
if m.group(1) == '<':
if '/' not in inc:
continue
if any((inc.startswith(x) for x in IGNORED_INCLUDE_PREFIXES)):
continue
assert_one(inc, line_num)
total_used_files = set()
vendor_prereq_outputs = set()
# --
print_now('Running prerequisite actions')
for k in VENDOR_PREREQ_TARGETS:
assert k.startswith('//')
run_checked('ninja', '-C', OUT_DIR, k[2:])
vendor_prereq_outputs |= set(descs[k]['outputs'])
total_used_files |= vendor_prereq_outputs
# --
# Export our targets
print_now('Export targets')
# Clear our dest directories
targets_dir = Path(GECKO_ANGLE_DIR, 'targets')
checkout_dir = Path(GECKO_ANGLE_DIR, 'checkout')
if not DRY_RUN:
shutil.rmtree(targets_dir, True)
shutil.rmtree(checkout_dir, True)
targets_dir.mkdir(exist_ok=True)
checkout_dir.mkdir(exist_ok=True)
def export_target(root):
name = root['target_name']
assert name.startswith('//:')
name = name[3:]
used_files = root['sources_with_deps']
used_files = [x for x in used_files if x.split('.')[-1] not in ['dll']]
global total_used_files
total_used_files |= set(used_files)
# Check includes, since `gn check` seems to be broken.
includable = set(root['sources_with_deps']) | vendor_prereq_outputs
for x in includable:
if is_source_file(x):
assert_valid_includes(name, x, includable, root['include_dirs_with_deps'])
# Accumulate a combined dict for the target including non-lib deps.
accum_desc = dict(root)
del accum_desc['dep_nodes']
use_libs = set()
checkable_sources = set()
target_includable_files = set()
def pre(cur):
assert not cur.get('allow_circular_includes_from', ()), cur['target_name']
deps = cur['dep_nodes']
if cur != root:
if cur['type'] in ['shared_library', 'static_library']:
deps = []
name = cur['target_name']
assert name.startswith('//:')
name = name[3:]
use_libs.add(name)
elif cur['type'] in ('source_set', 'group', 'action'):
for (k,v) in cur.items():
if k in ('dep_nodes', 'sources_with_deps', 'include_dirs_with_deps'):
continue
if type(v) in (list, tuple):
vs = accum_desc.setdefault(k, ())
vs += v
else:
accum_desc.setdefault(k, v)
return (deps,)
traverse([root], pre)
# Create our manifest lines
target_dir = Path(targets_dir, name)
target_dir.mkdir(exist_ok=True)
lines = COMMON_HEADER[:]
lines.append('')
for x in sorted(set(accum_desc['defines'])):
try:
(k, v) = x.split('=', 1)
v = "'{}'".format(v)
except ValueError:
(k, v) = (x, 'True')
try:
line = "DEFINES['{}'] = {}".format(k, v)
if REGISTERED_DEFINES[k] == False:
line = '#' + line
lines.append(line)
except KeyError:
print('[{}] Unrecognized define: {}'.format(name, k))
lines.append('')
cxxflags = set(accum_desc['cflags'] + accum_desc['cflags_cc'])
def fixup_paths(listt):
for x in set(listt):
assert x.startswith('//'), x
yield '../../checkout/' + x[2:]
sources_by_config = {}
extras = dict()
for x in fixup_paths(accum_desc['sources']):
(b, e) = x.rsplit('.', 1)
if e in ['h', 'y', 'l', 'inc', 'inl']:
continue
elif e in ['cpp', 'cc', 'c']:
if b.endswith('_win'):
config = "CONFIG['OS_ARCH'] == 'WINNT'"
elif b.endswith('_linux'):
# Include these on BSDs too.
config = "CONFIG['OS_ARCH'] not in ('Darwin', 'WINNT')"
elif b.endswith('_mac'):
config = "CONFIG['OS_ARCH'] == 'Darwin'"
else:
config = '' # None can't compare against str.
sources_by_config.setdefault(config, []).append(x)
continue
elif e == 'rc':
assert 'RCFILE' not in extras
extras['RCFILE'] = "'{}'".format(x)
continue
else:
assert False, "Unhandled ext: {}".format(x)
ldflags = set(accum_desc['ldflags'])
DEF_PREFIX = '/DEF:'
for x in set(ldflags):
if x.startswith(DEF_PREFIX):
assert 'DEFFILE' not in extras
ldflags.remove(x)
def_path = OUT_DIR + '/' + x[len(DEF_PREFIX):]
def_path = '//' + collapse_dotdots(def_path)
total_used_files.add(def_path)
def_rel_path = list(fixup_paths([def_path]))[0]
extras['DEFFILE'] = "'{}'".format(def_rel_path)
os_libs = list(map( lambda x: x[:-len('.lib')], set(accum_desc.get('libs', [])) ))
def append_arr_commented(dest, name, src):
lines = []
append_arr(lines, name, src)
def comment(x):
if x:
x = '#' + x
return x
lines = map(comment, lines)
dest += lines
append_arr(lines, 'LOCAL_INCLUDES', fixup_paths(accum_desc['include_dirs']))
append_arr_commented(lines, 'CXXFLAGS', cxxflags)
for (config,v) in sorted_items(sources_by_config):
indent = 0
if config:
lines.append("if {}:".format(config))
indent = 1
append_arr(lines, 'SOURCES', v, indent=indent)
append_arr(lines, 'USE_LIBS', use_libs)
append_arr(lines, 'DIRS', ['../' + x for x in use_libs])
append_arr(lines, 'OS_LIBS', os_libs)
append_arr_commented(lines, 'LDFLAGS', ldflags)
for (k,v) in sorted(extras.items()):
lines.append('{} = {}'.format(k, v))
lib_type = root['type']
if lib_type == 'shared_library':
lines.append("GeckoSharedLibrary('{}', linkage=None)".format(name))
elif lib_type == 'static_library':
lines.append("Library('{}')".format(name))
else:
assert False, lib_type
# Write it out
mozbuild = Path(target_dir, 'moz.build')
print_now(' {}Writing {}'.format(ACTION_PREFIX, mozbuild))
if not DRY_RUN:
with mozbuild.open('w', newline='\n') as f:
for x in lines:
f.write(x + '\n')
return
for x in real_targets:
export_target(x)
# Copy all the files
print_now('Migrate files')
total_used_files = sorted(total_used_files)
i = 0
for x in total_used_files:
i += 1
sys.stdout.write('\r {}Copying {}/{}'.format(ACTION_PREFIX, i, len(total_used_files)))
sys.stdout.flush()
assert x.startswith('//'), x
x = x[2:]
src = Path(REPO_DIR, x)
dest = Path(checkout_dir, x)
if not DRY_RUN:
dest.parent.mkdir(parents=True, exist_ok=True)
data = src.read_bytes()
data = data.replace(b'\r\n', b'\n')
dest.write_bytes(data)
print('\nDone')
Ссылка в новой задаче Показать git blame Копировать постоянную ссылку