Skip to content

Commit

Permalink
Update build configurations to latest
Browse files Browse the repository at this point in the history
  • Loading branch information
zcbenz committed Aug 26, 2023
1 parent 72de6dc commit 471436e
Show file tree
Hide file tree
Showing 319 changed files with 19,532 additions and 13,986 deletions.
18 changes: 9 additions & 9 deletions .gitmodules
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,12 @@
[submodule "tools/gn"]
path = tools/gn
url = https://github.com/yue/gn
[submodule "buildtools/third_party/libc++/trunk"]
path = buildtools/third_party/libc++/trunk
url = https://chromium.googlesource.com/external/github.com/llvm/llvm-project/libcxx.git
[submodule "buildtools/third_party/libc++abi/trunk"]
path = buildtools/third_party/libc++abi/trunk
url = https://chromium.googlesource.com/external/github.com/llvm/llvm-project/libcxxabi.git
[submodule "buildtools/third_party/libunwind/trunk"]
path = buildtools/third_party/libunwind/trunk
url = https://chromium.googlesource.com/external/github.com/llvm/llvm-project/libunwind.git
[submodule "third_party/libc++/src"]
path = third_party/libc++/src
url = https://chromium.googlesource.com/external/github.com/llvm/llvm-project/libcxx
[submodule "third_party/libc++abi/src"]
path = third_party/libc++abi/src
url = https://chromium.googlesource.com/external/github.com/llvm/llvm-project/libcxxabi
[submodule "third_party/libunwind/src"]
path = third_party/libunwind/src
url = https://chromium.googlesource.com/external/github.com/llvm/llvm-project/libunwind
2 changes: 2 additions & 0 deletions build/3pp/.style.yapf
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
[style]
based_on_style = pep8
5 changes: 5 additions & 0 deletions build/3pp/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
# 3pp_util

Contains helper scripts for 3pp configs.

See [`//docs/docs/cipd_and_3pp.md`](/docs/cipd_and_3pp.md)
94 changes: 94 additions & 0 deletions build/3pp/fetch_github_release.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
# Copyright 2023 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import argparse
import hashlib
import json
import os
import pathlib
import re
import urllib.request


def _fetch_json(url):
return json.load(urllib.request.urlopen(url))


def _latest(api_url, install_scripts=None):
# Make the version change every time this file changes.
md5 = hashlib.md5()
md5.update(pathlib.Path(__file__).read_bytes())
import __main__
md5.update(pathlib.Path(__main__.__file__).read_bytes())

if install_scripts:
for path in install_scripts:
md5.update(pathlib.Path(path).read_bytes())
file_hash = md5.hexdigest()[:10]

release = _fetch_json(f'{api_url}/releases/latest')['tag_name']
print('{}.{}'.format(release, file_hash))


def _get_url(api_url,
artifact_filename=None,
artifact_extension=None,
artifact_regex=None):
# Split off our md5 hash.
version = os.environ['_3PP_VERSION'].rsplit('.', 1)[0]
json_dict = _fetch_json(f'{api_url}/releases/tags/{version}')
urls = [x['browser_download_url'] for x in json_dict['assets']]

if artifact_regex:
urls = [x for x in urls if re.search(artifact_regex, x)]

if len(urls) != 1:
raise Exception('len(urls) != 1: \n' + '\n'.join(urls))

partial_manifest = {
'url': urls,
'ext': artifact_extension or '',
}
if artifact_filename:
partial_manifest['name'] = [artifact_filename]

print(json.dumps(partial_manifest))


def main(*,
project,
artifact_filename=None,
artifact_extension=None,
artifact_regex=None,
install_scripts=None,
extract_extension=None):
"""The fetch.py script for a 3pp module.
Args:
project: GitHub username for the repo. e.g. "google/protobuf".
artifact_filename: The name for the downloaded file. Required when not
setting "unpack_archive: true" in 3pp.pb.
artifact_extension: File extension of file being downloaded. Required when
setting "unpack_archive: true" in 3pp.pb.
artifact_regex: A regex to use to identify the desired artifact from the
list of artifacts on the release.
install_scripts: List of script to add to the md5 of the version. The main
module and this module are always included.
"""
parser = argparse.ArgumentParser()
parser.add_argument('action', choices=('latest', 'get_url'))
args = parser.parse_args()

api_url = f'https://api.github.com/repos/{project}'
if args.action == 'latest':
_latest(api_url, install_scripts=install_scripts)
else:
_get_url(api_url,
artifact_filename=artifact_filename,
artifact_extension=artifact_extension,
artifact_regex=artifact_regex)


if __name__ == '__main__':
main()
53 changes: 53 additions & 0 deletions build/3pp/print_cipd_version.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
#!/usr/bin/env python3
# Copyright 2023 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import argparse
import pathlib
import re
import subprocess

_DIR_SOURCE_ROOT = str(pathlib.Path(__file__).absolute().parents[2])


def main():
parser = argparse.ArgumentParser()
# Hide args set by wrappers so that using --help with the wrappers does not
# show them.
parser.add_argument('--subdir', required=True, help=argparse.SUPPRESS)
parser.add_argument('--cipd-package',
required=True,
help=argparse.SUPPRESS)
parser.add_argument('--git-log-url', help=argparse.SUPPRESS)
parser.add_argument('--cipd-instance',
help='Uses value from DEPS by default')
args = parser.parse_args()

if not args.cipd_instance:
cmd = [
'gclient', 'getdep', '-r', f'src/{args.subdir}:{args.cipd_package}'
]
args.cipd_instance = subprocess.check_output(cmd,
cwd=_DIR_SOURCE_ROOT,
text=True)

cmd = [
'cipd', 'describe', args.cipd_package, '-version', args.cipd_instance
]
print(' '.join(cmd))
output = subprocess.check_output(cmd, text=True)
print(output, end='')
if args.git_log_url:
git_hashes = re.findall(r'version:.*?@(\w+)', output)
if not git_hashes:
print('Could not find git hash from output.')
else:
# Multiple version tags exist when multiple versions have the same sha1.
last_version = git_hashes[-1]
print()
print('Recent commits:', args.git_log_url.format(last_version))


if __name__ == '__main__':
main()
2 changes: 2 additions & 0 deletions build/OWNERS
Original file line number Diff line number Diff line change
Expand Up @@ -32,3 +32,5 @@ per-file whitespace_file.txt=*
per-file OWNERS.status=*
per-file OWNERS.setnoparent=set noparent
per-file OWNERS.setnoparent=file://ATL_OWNERS

per-file nocompile.gni=file://tools/nocompile/OWNERS
8 changes: 5 additions & 3 deletions build/OWNERS.setnoparent
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ file://third_party/OWNERS
# Security reviews
file://build/fuchsia/SECURITY_OWNERS
file://chromeos/SECURITY_OWNERS
file://content/browser/SITE_ISOLATION_OWNERS
file://content/browser/CHILD_PROCESS_SECURITY_POLICY_OWNERS
file://ipc/SECURITY_OWNERS
file://net/base/SECURITY_OWNERS
file://sandbox/linux/OWNERS
Expand Down Expand Up @@ -40,12 +40,14 @@ file://extensions/common/api/API_OWNERS
# we don't want them.
file://ui/android/java/res/LAYOUT_OWNERS

# Updating policy_templates.json can have drastic effects for systems depending
# Enterprise policies
# Updating policy templates can have drastic effects for systems depending
# on policy definitions (for example, Google's cloud management tools for
# Chrome and Chrome OS).
# The rules are documented at:
# https://sites.google.com/a/chromium.org/dev/developers/how-tos/enterprise/adding-new-policies
file://components/policy/ENTERPRISE_POLICY_OWNERS
file://components/policy/resources/templates/OWNERS

# This restriction is in place due to the complicated compliance regulations
# around this code.
Expand Down Expand Up @@ -74,7 +76,7 @@ file://infra/config/groups/sheriff-rotations/CHROMIUM_OWNERS
# completed manually.
file://third_party/blink/common/origin_trials/OT_OWNERS

# New notifiers added to //ash/constants/notifier_catalogs.h and
# New notifiers added to //ash/constants/notifier_catalogs.h and
# //ash/constants/quick_settings_catalogs.h should be reviewed
# by //ash/system owners to ensure that the correct notifier is being used.
file://ash/system/OWNERS
Expand Down
14 changes: 6 additions & 8 deletions build/PRESUBMIT.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,6 @@

PRESUBMIT_VERSION = '2.0.0'

# This line is 'magic' in that git-cl looks for it to decide whether to
# use Python3 instead of Python2 when running the code in this file.
USE_PYTHON3 = True

import textwrap


Expand All @@ -17,6 +13,9 @@ def CheckNoBadDeps(input_api, output_api):
r'(.+/)?BUILD\.gn',
r'.+\.gni',
]
exclude_file_patterns = [
r'build/rust/tests',
]
blocklist_pattern = input_api.re.compile(r'^[^#]*"//(?!build).+?/.*"')
allowlist_pattern = input_api.re.compile(r'^[^#]*"//third_party/junit')

Expand All @@ -31,7 +30,8 @@ def CheckNoBadDeps(input_api, output_api):

def FilterFile(affected_file):
return input_api.FilterSourceFile(affected_file,
files_to_check=build_file_patterns)
files_to_check=build_file_patterns,
files_to_skip=exclude_file_patterns)

problems = []
for f in input_api.AffectedSourceFiles(FilterFile):
Expand All @@ -52,6 +52,4 @@ def CheckPythonTests(input_api, output_api):
input_api,
output_api,
input_api.PresubmitLocalPath(),
files_to_check=[r'.+_(?:unit)?test\.py$'],
run_on_python2=False,
run_on_python3=True))
files_to_check=[r'.+_(?:unit)?test\.py$']))
1 change: 0 additions & 1 deletion build/PRESUBMIT_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
from PRESUBMIT_test_mocks import MockAffectedFile
from PRESUBMIT_test_mocks import MockInputApi, MockOutputApi

USE_PYTHON3 = True


def _fails_deps_check(line, filename='BUILD.gn'):
Expand Down
126 changes: 126 additions & 0 deletions build/action_helpers.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,126 @@
# Copyright 2023 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Helper functions useful when writing scripts used by action() targets."""

import contextlib
import filecmp
import os
import pathlib
import posixpath
import shutil
import tempfile

import gn_helpers


@contextlib.contextmanager
def atomic_output(path, mode='w+b', only_if_changed=True):
"""Prevent half-written files and dirty mtimes for unchanged files.
Args:
path: Path to the final output file, which will be written atomically.
mode: The mode to open the file in (str).
only_if_changed: Whether to maintain the mtime if the file has not changed.
Returns:
A Context Manager that yields a NamedTemporaryFile instance. On exit, the
manager will check if the file contents is different from the destination
and if so, move it into place.
Example:
with action_helpers.atomic_output(output_path) as tmp_file:
subprocess.check_call(['prog', '--output', tmp_file.name])
"""
# Create in same directory to ensure same filesystem when moving.
dirname = os.path.dirname(path) or '.'
os.makedirs(dirname, exist_ok=True)
with tempfile.NamedTemporaryFile(mode,
suffix=os.path.basename(path),
dir=dirname,
delete=False) as f:
try:
yield f

# File should be closed before comparison/move.
f.close()
if not (only_if_changed and os.path.exists(path)
and filecmp.cmp(f.name, path)):
shutil.move(f.name, path)
finally:
f.close()
if os.path.exists(f.name):
os.unlink(f.name)


def add_depfile_arg(parser):
if hasattr(parser, 'add_option'):
func = parser.add_option
else:
func = parser.add_argument
func('--depfile', help='Path to depfile (refer to "gn help depfile")')


def write_depfile(depfile_path, first_gn_output, inputs=None):
"""Writes a ninja depfile.
See notes about how to use depfiles in //build/docs/writing_gn_templates.md.
Args:
depfile_path: Path to file to write.
first_gn_output: Path of first entry in action's outputs.
inputs: List of inputs to add to depfile.
"""
assert depfile_path != first_gn_output # http://crbug.com/646165
assert not isinstance(inputs, str) # Easy mistake to make

def _process_path(path):
assert not os.path.isabs(path), f'Found abs path in depfile: {path}'
if os.path.sep != posixpath.sep:
path = str(pathlib.Path(path).as_posix())
assert '\\' not in path, f'Found \\ in depfile: {path}'
return path.replace(' ', '\\ ')

sb = []
sb.append(_process_path(first_gn_output))
if inputs:
# Sort and uniquify to ensure file is hermetic.
# One path per line to keep it human readable.
sb.append(': \\\n ')
sb.append(' \\\n '.join(sorted(_process_path(p) for p in set(inputs))))
else:
sb.append(': ')
sb.append('\n')

path = pathlib.Path(depfile_path)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(''.join(sb))


def parse_gn_list(value):
"""Converts a "GN-list" command-line parameter into a list.
Conversions handled:
* None -> []
* '' -> []
* 'asdf' -> ['asdf']
* '["a", "b"]' -> ['a', 'b']
* ['["a", "b"]', 'c'] -> ['a', 'b', 'c'] (action='append')
This allows passing args like:
gn_list = [ "one", "two", "three" ]
args = [ "--items=$gn_list" ]
"""
# Convert None to [].
if not value:
return []
# Convert a list of GN lists to a flattened list.
if isinstance(value, list):
ret = []
for arg in value:
ret.extend(parse_gn_list(arg))
return ret
# Convert normal GN list.
if value.startswith('['):
return gn_helpers.GNValueParser(value).ParseList()
# Convert a single string value to a list.
return [value]
Loading

0 comments on commit 471436e

Please sign in to comment.