Bug 1857234 - Have mach vendor's `**` globbing match hidden files r=tjr,devtools-reviewers,cubeb-reviewers,padenot,ochameau

This patch contains three changes:

1. a local version of glb.iglob in vendor_manifest.py that always
   matches hidden files for '**'. I've tested it on Python 3.5 up to
   3.12 and it works as expected.

2. a cleanup of many moz.yaml files that got convluted because of the
   previous behavior of '**'

3. removal of a few useless files that are now caught by the '**'
   globbing.

Differential Revision: https://phabricator.services.mozilla.com/D190195
This commit is contained in:
serge-sans-paille 2023-10-10 08:21:13 +00:00
Родитель dcd790469f
Коммит d3e574bc41
30 изменённых файлов: 25 добавлений и 630 удалений

Просмотреть файл

@ -24,10 +24,7 @@ vendoring:
# dirs
- test
# files
- .clang-format
- .clang-tidy
- .gitignore
- .travis.yml
- ".*"
- AppSandbox.md
- CMakeLists.txt
- LibrarySandbox.md

Просмотреть файл

@ -22,10 +22,7 @@ vendoring:
tracking: commit
exclude:
- ".*"
- "*"
- "**"
- "**/.gitignore"
keep:
- LICENSE

Просмотреть файл

@ -21,10 +21,7 @@ vendoring:
tracking: commit
exclude:
- "*"
- "**"
- ".*"
- "**/.gitignore"
keep:
- glue
@ -37,7 +34,7 @@ vendoring:
- src/README.md
include:
- "src/hunspell"
- src/hunspell
patches:
- patches/bug1410214.patch
@ -61,5 +58,3 @@ vendoring:
path: "src/hunzip.hxx"
- action: delete-path
path: "src/utf_info.hxx"
- action: delete-path
path: "tests/.gitattributes"

Просмотреть файл

@ -57,9 +57,6 @@ vendoring:
exclude:
- "**"
- "**/.*"
- ".*"
- ".*/**"
- "src/test*"
- src/hb-ucdn

Просмотреть файл

@ -38,8 +38,7 @@ vendoring:
vendor-directory: third_party/aom
exclude:
- build/.gitattributes
- build/.gitignore
- "build/.git*"
update-actions:
- action: delete-path

Просмотреть файл

@ -21,7 +21,7 @@ vendoring:
skip-vendoring-steps:
- update-moz-build
exclude:
- .clang-format
- ".*"
- CMakeLists.txt
- Config.cmake.in
- INSTALL.md

Просмотреть файл

@ -48,8 +48,7 @@ vendoring:
vendor-directory: third_party/dav1d
exclude:
- build/.gitattributes
- build/.gitignore
- "build/.git*"
- doc
- examples
- package

Просмотреть файл

@ -22,8 +22,6 @@ vendoring:
tracking: tag
exclude:
- ".*"
- "*"
- "**"
keep:
@ -51,8 +49,6 @@ vendoring:
- action: move-file
from: '{vendor_dir}/src/framing.c'
to: '{vendor_dir}/src/ogg_framing.c'
- action: delete-path
path: '{vendor_dir}/win32/.gitignore'
# Manually reproduce configure step here
- action: replace-in-file

Просмотреть файл

@ -31,7 +31,6 @@ vendoring:
exclude:
- "**"
- ".*"
include:
- arm

Просмотреть файл

@ -20,24 +20,7 @@ vendoring:
tracking: commit
exclude:
- bin
- bootstrap
- config
- configure.ac
- createsrcpack
- Makefile.am
- make-win.bat
- README.html
- readme.md
- soundtouch.m4
- soundtouch.pc.in
- ".*"
- "source/Android-lib"
- "source/*/*/*"
- "source/*/*"
- "source/*/.*"
- "source/*"
- "include/*"
- "**"
include:
- COPYING.TXT

Просмотреть файл

@ -22,8 +22,6 @@ vendoring:
tracking: commit
exclude:
- ".*"
- "*"
- "**"
keep:
@ -41,10 +39,6 @@ vendoring:
- COPYING
update-actions:
- action: delete-path
path: '{vendor_dir}/doc'
- action: delete-path
path: '{vendor_dir}/libspeexdsp/.cvsignore'
- action: move-file
from: '{vendor_dir}/libspeexdsp/resample.c'
to: '{vendor_dir}/src/resample.c'

Просмотреть файл

@ -23,9 +23,7 @@ vendoring:
tracking: tag
exclude:
- "*"
- "**"
- ".*"
include:
- lib/envelope.h

Просмотреть файл

@ -31,7 +31,6 @@ vendoring:
exclude:
- "**"
- ".*"
include:
- AUTHORS

Просмотреть файл

@ -1,6 +0,0 @@
# Defines the Chromium style for automatic reformatting.
# http://clang.llvm.org/docs/ClangFormatStyleOptions.html
BasedOnStyle: Chromium
---
Language: Java
BasedOnStyle: Google

Просмотреть файл

@ -1,38 +0,0 @@
# Copyright 2015 The LibYuv Project Authors. All rights reserved.
#
# Use of this source code is governed by a BSD-style license
# that can be found in the LICENSE file in the root of the source
# tree. An additional intellectual property rights grant can be found
# in the file PATENTS. All contributing project authors may
# be found in the AUTHORS file in the root of the source tree.
import("//build/dotfile_settings.gni")
# The location of the build configuration file.
buildconfig = "//build/config/BUILDCONFIG.gn"
# The python interpreter to use by default. On Windows, this will look
# for python3.exe and python3.bat.
script_executable = "python3"
# The secondary source root is a parallel directory tree where
# GN build files are placed when they can not be placed directly
# in the source tree, e.g. for third party source trees.
secondary_source = "//build/secondary/"
# These are the targets to check headers for by default. The files in targets
# matching these patterns (see "gn help label_pattern" for format) will have
# their includes checked for proper dependencies when you run either
# "gn check" or "gn gen --check".
check_targets = [ "//libyuv/*" ]
# These are the list of GN files that run exec_script. This whitelist exists
# to force additional review for new uses of exec_script, which is strongly
# discouraged except for gypi_to_gn calls.
exec_script_whitelist = build_dotfile_settings.exec_script_whitelist +
[ "//build_overrides/build.gni" ]
default_args = {
mac_sdk_min = "10.12"
ios_deployment_target = "12.0"
}

Просмотреть файл

@ -1,52 +0,0 @@
# This is a vpython "spec" file.
#
# It describes patterns for python wheel dependencies of the python scripts in
# the chromium repo, particularly for dependencies that have compiled components
# (since pure-python dependencies can be easily vendored into third_party).
#
# When vpython is invoked, it finds this file and builds a python VirtualEnv,
# containing all of the dependencies described in this file, fetching them from
# CIPD (the "Chrome Infrastructure Package Deployer" service). Unlike `pip`,
# this never requires the end-user machine to have a working python extension
# compilation environment. All of these packages are built using:
# https://chromium.googlesource.com/infra/infra/+/master/infra/tools/dockerbuild/
#
# All python scripts in the repo share this same spec, to avoid dependency
# fragmentation.
#
# If you have depot_tools installed in your $PATH, you can invoke python scripts
# in this repo by running them as you normally would run them, except
# substituting `vpython` instead of `python` on the command line, e.g.:
# vpython path/to/script.py some --arguments
#
# Read more about `vpython` and how to modify this file here:
# https://chromium.googlesource.com/infra/infra/+/master/doc/users/vpython.md
python_version: "2.7"
# Used by:
# third_party/catapult
wheel: <
name: "infra/python/wheels/psutil/${platform}_${py_python}_${py_abi}"
version: "version:5.2.2"
>
# Used by:
# third_party/catapult
wheel: <
name: "infra/python/wheels/pypiwin32/${vpython_platform}"
version: "version:219"
match_tag: <
platform: "win32"
>
match_tag: <
platform: "win_amd64"
>
>
# Used by:
# build/android
wheel: <
name: "infra/python/wheels/requests-py2_py3"
version: "version:2.13.0"
>

Просмотреть файл

@ -1,405 +0,0 @@
# This is a vpython "spec" file.
#
# It describes patterns for python wheel dependencies of the python scripts in
# the chromium repo, particularly for dependencies that have compiled components
# (since pure-python dependencies can be easily vendored into third_party).
#
# When vpython is invoked, it finds this file and builds a python VirtualEnv,
# containing all of the dependencies described in this file, fetching them from
# CIPD (the "Chrome Infrastructure Package Deployer" service). Unlike `pip`,
# this never requires the end-user machine to have a working python extension
# compilation environment. All of these packages are built using:
# https://chromium.googlesource.com/infra/infra/+/main/infra/tools/dockerbuild/
#
# All python scripts in the repo share this same spec, to avoid dependency
# fragmentation.
#
# If you have depot_tools installed in your $PATH, you can invoke python scripts
# in this repo by running them as you normally would run them, except
# substituting `vpython` instead of `python` on the command line, e.g.:
# vpython path/to/script.py some --arguments
#
# Read more about `vpython` and how to modify this file here:
# https://chromium.googlesource.com/infra/infra/+/main/doc/users/vpython.md
python_version: "3.8"
# The default set of platforms vpython checks does not yet include mac-arm64.
# Setting `verify_pep425_tag` to the list of platforms we explicitly must support
# allows us to ensure that vpython specs stay mac-arm64-friendly
verify_pep425_tag: [
{python: "cp38", abi: "cp38", platform: "manylinux1_x86_64"},
{python: "cp38", abi: "cp38", platform: "linux_arm64"},
{python: "cp38", abi: "cp38", platform: "macosx_10_10_intel"},
{python: "cp38", abi: "cp38", platform: "macosx_11_0_arm64"},
{python: "cp38", abi: "cp38", platform: "win32"},
{python: "cp38", abi: "cp38", platform: "win_amd64"}
]
# Used by:
# build/android/pylib/local/emulator/avd.py
# components/policy/test_support/policy_testserver.py
wheel: <
name: "infra/python/wheels/protobuf-py2_py3"
version: "version:3.15.8"
>
# TODO(https://crbug.com/898348): Add in necessary wheels as Python3 versions
# become available.
wheel: <
name: "infra/python/wheels/six-py2_py3"
version: "version:1.15.0"
>
# Common utilities.
# Use the same versions specified by //third_party/catapult/.vpython3 so that
# Chromium tests using Telemetry function properly.
wheel: <
name: "infra/python/wheels/numpy/${vpython_platform}"
version: "version:1.20.3"
# A newer version of numpy is required on ARM64, but it breaks older OS versions.
not_match_tag <
platform: "macosx_11_0_arm64"
>
>
wheel: <
name: "infra/python/wheels/numpy/mac-arm64_cp38_cp38"
version: "version:1.21.1"
match_tag <
platform: "macosx_11_0_arm64"
>
>
wheel: <
name: "infra/python/wheels/psutil/${vpython_platform}"
version: "version:5.8.0.chromium.2"
>
wheel: <
name: "infra/python/wheels/requests-py2_py3"
version: "version:2.26.0"
>
# Used by various python unit tests.
wheel: <
name: "infra/python/wheels/mock-py2_py3"
version: "version:2.0.0"
>
wheel: <
name: "infra/python/wheels/parameterized-py2_py3"
version: "version:0.7.1"
>
wheel: <
name: "infra/python/wheels/pbr-py2_py3"
version: "version:3.0.0"
>
wheel: <
name: "infra/python/wheels/pyfakefs-py2_py3"
version: "version:3.7.2"
>
# Used by:
# build/chromeos/test_runner.py
wheel: <
name: "infra/python/wheels/jsonlines-py2_py3"
version: "version:1.2.0"
>
wheel: <
name: "infra/python/wheels/python-dateutil-py2_py3"
version: "version:2.7.3"
>
# Used by WPT importer
wheel: <
name: "infra/python/wheels/charset_normalizer-py3"
version: "version:2.0.4"
>
wheel: <
name: "infra/python/wheels/pyasn1-py2_py3"
version: "version:0.4.5"
>
wheel: <
name: "infra/python/wheels/pyasn1_modules-py2_py3"
version: "version:0.2.4"
>
wheel: <
name: "infra/python/wheels/rsa-py2_py3"
version: "version:3.4.2"
>
wheel: <
name: "infra/python/wheels/cachetools-py2_py3"
version: "version:2.0.1"
>
wheel: <
name: "infra/python/wheels/uritemplate-py2_py3"
version: "version:3.0.0"
>
wheel: <
name: "infra/python/wheels/google-auth-py2_py3"
version: "version:1.25.0"
>
wheel: <
name: "infra/python/wheels/googleapis-common-protos-py2_py3"
version: "version:1.52.0"
>
wheel: <
name: "infra/python/wheels/google-api-core-py2_py3"
version: "version:1.25.1"
>
wheel: <
name: "infra/python/wheels/google-auth-httplib2-py2_py3"
version: "version:0.1.0"
>
wheel: <
name: "infra/python/wheels/google-api-python-client-py3"
version: "version:2.2.0"
>
wheel: <
name: "infra/python/wheels/oauth2client-py2_py3"
version: "version:3.0.0"
>
# Used by Web Platform Tests (WPT) codebase in
# //third_party/blink/web_tests/external/wpt/tools/
wheel: <
name: "infra/python/wheels/html5lib-py2_py3"
version: "version:1.0.1"
>
wheel: <
name: "infra/python/wheels/mozdebug-py2_py3"
version: "version:0.2"
>
wheel: <
name: "infra/python/wheels/mozinfo-py2_py3"
version: "version:1.2.2"
>
wheel: <
name: "infra/python/wheels/mozlog-py2_py3"
version: "version:7.1.0"
>
wheel: <
name: "infra/python/wheels/mozprocess-py2_py3"
version: "version:1.2.1"
>
wheel: <
name: "infra/python/wheels/urllib3-py2_py3"
version: "version:1.24.3"
>
wheel: <
name: "infra/python/wheels/blessings-py2_py3"
version: "version:1.7"
>
wheel: <
name: "infra/python/wheels/mozfile-py2_py3"
version: "version:2.0.0"
>
wheel: <
name: "infra/python/wheels/mozterm-py2_py3"
version: "version:1.0.0"
>
wheel: <
name: "infra/python/wheels/webencodings-py2_py3"
version: "version:0.5.1"
>
wheel: <
name: "infra/python/wheels/certifi-py2_py3"
version: "version:2020.11.8"
>
wheel: <
name: "infra/python/wheels/chardet-py2_py3"
version: "version:3.0.4"
>
wheel: <
name: "infra/python/wheels/idna-py2_py3"
version: "version:2.8"
>
wheel: <
name: "infra/python/wheels/distro-py2_py3"
version: "version:1.4.0"
>
wheel: <
name: "infra/python/wheels/pillow/linux-amd64_cp38_cp38"
version: "version:8.1.2"
>
wheel: <
name: "infra/python/wheels/aioquic/${vpython_platform}"
version: "version:0.9.15"
>
wheel: <
name: "infra/python/wheels/pylsqpack/${vpython_platform}"
version: "version:0.3.12"
>
wheel: <
name: "infra/python/wheels/cryptography/${vpython_platform}"
version: "version:3.3.1.chromium.1"
>
wheel: <
name: "infra/python/wheels/cffi/${vpython_platform}"
version: "version:1.14.5"
>
wheel: <
name: "infra/python/wheels/pycparser-py2_py3"
version: "version:2.19"
>
# Used by:
# chrome/test/chromedriver/test/run_webdriver_tests.py
wheel: <
name: "infra/python/wheels/iniconfig-py3"
version: "version:1.1.1"
>
wheel: <
name: "infra/python/wheels/packaging-py2_py3"
version: "version:16.8"
>
wheel: <
name: "infra/python/wheels/pyparsing-py2_py3"
version: "version:2.4.7"
>
wheel: <
name: "infra/python/wheels/toml-py3"
version: "version:0.10.1"
>
wheel <
name: "infra/python/wheels/pytest-py3"
version: "version:6.2.2"
>
wheel <
name: "infra/python/wheels/pytest-asyncio-py3"
version: "version:0.14.0"
>
wheel <
name: "infra/python/wheels/attrs-py2_py3"
version: "version:20.3.0"
>
wheel <
name: "infra/python/wheels/six-py2_py3"
version: "version:1.15.0"
>
wheel <
name: "infra/python/wheels/more-itertools-py2_py3"
version: "version:4.1.0"
>
wheel <
name: "infra/python/wheels/pluggy-py3"
version: "version:0.13.1"
>
wheel <
name: "infra/python/wheels/py-py2_py3"
version: "version:1.10.0"
>
wheel <
name: "infra/python/wheels/funcsigs-py2_py3"
version: "version:1.0.2"
>
wheel: <
name: "infra/python/wheels/atomicwrites-py2_py3"
version: "version:1.3.0"
>
wheel: <
name: "infra/python/wheels/colorama-py2_py3"
version: "version:0.4.1"
>
# Used by:
# testing/buildbot/generate_buildbot_json_coveragetest.py
wheel: <
name: "infra/python/wheels/coverage/${vpython_platform}"
version: "version:5.5.chromium.2"
>
# Used by:
# //content/test/gpu
wheel: <
name: "infra/python/wheels/pathos/${vpython_platform}"
version: "version:0.2.7.chromium.4"
not_match_tag <
abi: "cp27mu"
platform: "manylinux1_i686"
>
not_match_tag <
abi: "cp27mu"
platform: "linux_mips64"
>
not_match_tag <
abi: "cp27mu"
platform: "linux_armv6l"
>
not_match_tag <
abi: "cp27mu"
platform: "linux_armv7l"
>
>
# Used by:
# //tools/infra/find_bad_builds.py
wheel: <
name: "infra/python/wheels/pytz-py2_py3"
version: "version:2018.4"
>
# Used by:
# //third_party/blink/tools/blinkpy/web_tests/port/server_process.py
wheel: <
name: "infra/python/wheels/pywin32/${vpython_platform}"
version: "version:300"
match_tag: <
platform: "win32"
>
match_tag: <
platform: "win_amd64"
>
>
# Used by:
# //content/test/gpu/gpu_tests/color_profile_manager_mac.py
wheel: <
name: "infra/python/wheels/pyobjc/${vpython_platform}"
version: "version:7.3.chromium.1"
match_tag: <
platform: "macosx_10_10_intel"
>
>
# Used by:
# tools/perf/core/results_dashboard.py
wheel: <
name: "infra/python/wheels/httplib2-py3"
version: "version:0.19.1"
>
# Used by:
# tools/perf/flakiness_cli
wheel: <
name: "infra/python/wheels/pandas/${vpython_platform}"
version: "version:1.3.2.chromium.1"
match_tag: <
platform: "win32"
>
match_tag: <
platform: "win_amd64"
>
match_tag: <
platform: "manylinux1_i686"
>
match_tag: <
platform: "manylinux1_x86_64"
>
match_tag: <
platform: "macosx_10_6_intel"
>
>

Просмотреть файл

@ -48,8 +48,7 @@ vendoring:
skip-vendoring-steps: ['update-moz-build']
exclude:
- .gitattributes
- .gitignore
- ".*"
patches:
- fix_build_errors.patch

Просмотреть файл

@ -22,8 +22,6 @@ vendoring:
tracking: commit
exclude:
- ".*"
- "*"
- "**"
keep:

Просмотреть файл

@ -30,6 +30,16 @@ DEFAULT_KEEP_FILES = ["**/moz.build", "**/moz.yaml"]
DEFAULT_INCLUDE_FILES = []
def iglob_hidden(*args, **kwargs):
# glob._ishidden exists from 3.5 up to 3.12 (and beyond?)
old_ishidden = glob._ishidden
glob._ishidden = lambda x: False
try:
yield from glob.iglob(*args, **kwargs)
finally:
glob._ishidden = old_ishidden
def throwe():
raise Exception
@ -366,11 +376,11 @@ class VendorManifest(MozbuildObject):
# Append double asterisk to the end to make glob.iglob recursively match
# contents of directory
paths.extend(
glob.iglob(mozpath.join(pattern_full_path, "**"), recursive=True)
iglob_hidden(mozpath.join(pattern_full_path, "**"), recursive=True)
)
# Otherwise pattern is a file or wildcard expression so add it without altering it
else:
paths.extend(glob.iglob(pattern_full_path, recursive=True))
paths.extend(iglob_hidden(pattern_full_path, recursive=True))
# Remove folder names from list of paths in order to avoid prematurely
# truncating directories elsewhere
# Sort the final list to ensure we preserve 01_, 02_ ordering for e.g. *.patch globs

1
third_party/cups/moz.yaml поставляемый
Просмотреть файл

@ -21,7 +21,6 @@ vendoring:
tracking: commit
exclude:
- ".*"
- "**"
include:

1
third_party/drm/moz.yaml поставляемый
Просмотреть файл

@ -40,7 +40,6 @@ vendoring:
skip-vendoring-steps: ['update-moz-build']
exclude:
- '**'
- '**/.*'
include:
- include/drm/drm.h

3
third_party/gbm/moz.yaml поставляемый
Просмотреть файл

@ -40,9 +40,6 @@ vendoring:
skip-vendoring-steps: ['update-moz-build']
exclude:
- '**'
- '**/.*'
- '**/.*/**'
- '**/.*/**/.*'
include:
- src/gbm/main/gbm.h

4
third_party/js/cfworker/moz.yaml поставляемый
Просмотреть файл

@ -26,10 +26,6 @@ vendoring:
exclude:
- "**"
- ".*"
- ".changeset"
- ".github"
- ".vscode"
include:
- LICENSE.md

2
third_party/libsrtp/moz.yaml поставляемый
Просмотреть файл

@ -30,8 +30,6 @@ vendoring:
vendor-directory: third_party/libsrtp/src
exclude:
- ".*"
- ".github"
- "**"
include:

1
third_party/msgpack/moz.yaml поставляемый
Просмотреть файл

@ -22,7 +22,6 @@ vendoring:
tracking: commit
exclude:
- ".*"
- "**"
include:

2
third_party/rlbox/moz.yaml поставляемый
Просмотреть файл

@ -21,8 +21,6 @@ vendoring:
tracking: commit
exclude:
- ".*"
- "*"
- "**"
include:

11
third_party/simde/moz.yaml поставляемый
Просмотреть файл

@ -29,15 +29,7 @@ vendoring:
- test
- test/*
- test/.gitignore
- .all-contributorsrc
- .appveyor.yml
- .azure-pipelines.yml
- .cirrus.yml
- .drone.star
- .editorconfig
- .gitignore
- .gitmodules
- .travis.yml
- ".*"
- codecov.yml
- meson.build
- meson_options.txt
@ -45,4 +37,3 @@ vendoring:
- CONTRIBUTING.md
- amalgamate.py
- netlify-disable.toml
- .drone.star.disabled

Просмотреть файл

@ -24,7 +24,7 @@ origin:
# Revision to pull in
# Must be a long or short commit SHA (long preferred)
revision: 3697152e0fd772d9185697fdbd4a1d340ca5571d
revision: 3697152e0fd772d9185697fdbd4a1d340ca5571d
# The package's license, where possible using the mnemonic from
# https://spdx.org/licenses/

Просмотреть файл

@ -1,4 +1,6 @@
---
---
schema: 1
bugzilla:
@ -18,18 +20,16 @@ origin:
revision: 76d07503f0c69f6632e6d8d4736e2a4cb4055a92
license: Apache-2.0
license-file: LICENSE.txt
license-file: LICENSE.TXT
vendoring:
url: https://github.com/llvm/llvm-project
source-hosting: github
keep:
- LICENSE.txt
- LICENSE.TXT
exclude:
- ".*"
- "*"
- "**"
include:
@ -53,44 +53,3 @@ vendoring:
path: '{yaml_dir}/FuzzerMain*'
- action: delete-path
path: '{yaml_dir}/FuzzerInterceptors*'
# The globbing from exclude skips all dot files, remove them here
- action: delete-path
path: clang/
- action: delete-path
path: clang-tools-extra/
- action: delete-path
path: compiler-rt/
- action: delete-path
path: debuginfo-tests/
- action: delete-path
path: flang/
- action: delete-path
path: libc/
- action: delete-path
path: libclc/
- action: delete-path
path: libcxx/
- action: delete-path
path: libcxxabi/
- action: delete-path
path: libunwind/
- action: delete-path
path: lld/
- action: delete-path
path: lldb/
- action: delete-path
path: llvm/
- action: delete-path
path: mlir/
- action: delete-path
path: openmp/
- action: delete-path
path: parallel-libs/
- action: delete-path
path: polly/
- action: delete-path
path: pstl/
- action: delete-path
path: test/
- action: delete-path
path: utils/