coffee/libyuv
1
0
Fork 0
mirror of https://chromium.googlesource.com/libyuv/libyuv synced 2025-12-06 08:46:47 +08:00
Code Issues Packages Projects Releases Wiki Activity

Update pylintrc to a pep-8 like style

Browse Source 
In particular, this requires reformatting
- autoroller/
- PRESUBMIT.py
- infra/config/PRESUBMIT.py
- get_landmines.py
- download_vs_toolchain.py
as well as some manual fixes.

This CL is preparation to updating the autoroller so that we can roll DEPS from chromium.

No-Try: True
Bug: libyuv:358992053
Change-Id: I090a09c6fa61beff2427a0537bca371f0839fa3e
Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/6105550
Reviewed-by: Mirko Bonadei <mbonadei@chromium.org>
Reviewed-by: Jeremy Leconte <jleconte@google.com>
Commit-Queue: Jeremy Leconte <jleconte@google.com>
This commit is contained in:
Björn Terelius 2024-12-18 14:02:45 +01:00 committed by libyuv LUCI CQ
parent 2c682f8d59
commit 37e88ecc68
7 changed files with 792 additions and 635 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
PRESUBMIT.py
View File

@ -9,11 +9,16 @@
# Runs PRESUBMIT.py in py3 mode by git cl presubmit. # Runs PRESUBMIT.py in py3 mode by git cl presubmit.
USE_PYTHON3 = True USE_PYTHON3 = True
def _CommonChecks(input_api, output_api): def _CommonChecks(input_api, output_api):
"""Checks common to both upload and commit.""" """Checks common to both upload and commit."""
results = [] results = []
results.extend(input_api.canned_checks.RunPylint(input_api, output_api, results.extend(
files_to_skip=(r'^base[\\\/].*\.py$', input_api.canned_checks.RunPylint(
input_api,
output_api,
files_to_skip=(
r'^base[\\\/].*\.py$',
r'^build[\\\/].*\.py$', r'^build[\\\/].*\.py$',
r'^buildtools[\\\/].*\.py$', r'^buildtools[\\\/].*\.py$',
r'^ios[\\\/].*\.py$', r'^ios[\\\/].*\.py$',
@ -23,13 +28,17 @@ def _CommonChecks(input_api, output_api):
r'^tools[\\\/].*\.py$', r'^tools[\\\/].*\.py$',
# TODO(kjellander): should arguably be checked. # TODO(kjellander): should arguably be checked.
r'^tools_libyuv[\\\/]valgrind[\\\/].*\.py$', r'^tools_libyuv[\\\/]valgrind[\\\/].*\.py$',
r'^xcodebuild.*[\\\/].*\.py$',), r'^xcodebuild.*[\\\/].*\.py$',
disabled_warnings=['F0401', # Failed to import x ),
disabled_warnings=[
'F0401', # Failed to import x
'E0611', # No package y in x 'E0611', # No package y in x
'W0232', # Class has no __init__ method 'W0232', # Class has no __init__ method
], ],
pylintrc='pylintrc', pylintrc='pylintrc',
version='2.7')) version='2.7',
)
)
return results return results
@ -37,7 +46,8 @@ def CheckChangeOnUpload(input_api, output_api):
results = [] results = []
results.extend(_CommonChecks(input_api, output_api)) results.extend(_CommonChecks(input_api, output_api))
results.extend( results.extend(
input_api.canned_checks.CheckGNFormatted(input_api, output_api)) input_api.canned_checks.CheckGNFormatted(input_api, output_api)
)
return results return results
@ -45,8 +55,11 @@ def CheckChangeOnCommit(input_api, output_api):
results = [] results = []
results.extend(_CommonChecks(input_api, output_api)) results.extend(_CommonChecks(input_api, output_api))
results.extend(input_api.canned_checks.CheckOwners(input_api, output_api)) results.extend(input_api.canned_checks.CheckOwners(input_api, output_api))
results.extend(input_api.canned_checks.CheckChangeWasUploaded( results.extend(
input_api, output_api)) input_api.canned_checks.CheckChangeWasUploaded(input_api, output_api)
results.extend(input_api.canned_checks.CheckChangeHasDescription( )
input_api, output_api)) results.extend(
input_api.canned_checks.CheckChangeHasDescription(input_api,
output_api)
)
return results return results

8
infra/config/PRESUBMIT.py
View File

@ -6,8 +6,12 @@ USE_PYTHON3 = True
def CheckChangeOnUpload(input_api, output_api): def CheckChangeOnUpload(input_api, output_api):
return input_api.canned_checks.CheckChangedLUCIConfigs(input_api, output_api) return input_api.canned_checks.CheckChangedLUCIConfigs(
input_api, output_api
)
def CheckChangeOnCommit(input_api, output_api): def CheckChangeOnCommit(input_api, output_api):
return input_api.canned_checks.CheckChangedLUCIConfigs(input_api, output_api) return input_api.canned_checks.CheckChangedLUCIConfigs(
input_api, output_api
)

34
pylintrc
View File

@ -13,5 +13,37 @@ reports=no
[FORMAT] [FORMAT]
# We use two spaces for indents, instead of the usual four spaces or tab. # Maximum number of characters on a single line.
max-line-length=79
# We use four spaces for indents.
indent-string=' ' indent-string=' '
[BASIC]
# List of builtins function names that should not be used, separated by a comma
bad-functions=map,filter,apply,input
# Naming style matching correct module names.
module-naming-style=snake_case
# Naming style matching correct constant names.
const-naming-style=UPPER_CASE
# Naming style matching correct class names.
class-naming-style=PascalCase
# Naming style matching correct function names.
function-naming-style=snake_case
# Regular expression matching correct method names.
method-rgx=([a-z_][a-z0-9_]{2,}|setUp|tearDown)$
# Naming style matching correct attribute names.
attr-naming-style=snake_case
# Naming style matching correct argument names.
argument-naming-style=snake_case
# Naming style matching correct variable names.
variable-naming-style=snake_case

319
tools_libyuv/autoroller/roll_deps.py
View File

@ -19,6 +19,7 @@ import re
import subprocess import subprocess
import sys import sys
import urllib.request import urllib.request
import find_depot_tools
def FindSrcDirPath(): def FindSrcDirPath():
@ -85,27 +86,30 @@ ANDROID_DEPS_END = r'=== ANDROID_DEPS Generated Code End ==='
ANDROID_DEPS_PATH = 'src/third_party/android_deps/' ANDROID_DEPS_PATH = 'src/third_party/android_deps/'
sys.path.append(os.path.join(CHECKOUT_SRC_DIR, 'build')) sys.path.append(os.path.join(CHECKOUT_SRC_DIR, 'build'))
import find_depot_tools
find_depot_tools.add_depot_tools_to_path() find_depot_tools.add_depot_tools_to_path()
CLANG_UPDATE_SCRIPT_URL_PATH = 'tools/clang/scripts/update.py' CLANG_UPDATE_SCRIPT_URL_PATH = 'tools/clang/scripts/update.py'
CLANG_UPDATE_SCRIPT_LOCAL_PATH = os.path.join(CHECKOUT_SRC_DIR, 'tools', CLANG_UPDATE_SCRIPT_LOCAL_PATH = os.path.join(
'clang', 'scripts', 'update.py') CHECKOUT_SRC_DIR, 'tools', 'clang', 'scripts', 'update.py'
)
DepsEntry = collections.namedtuple('DepsEntry', 'path url revision') DepsEntry = collections.namedtuple('DepsEntry', 'path url revision')
ChangedDep = collections.namedtuple('ChangedDep', ChangedDep = collections.namedtuple(
'path url current_rev new_rev') 'ChangedDep', 'path url current_rev new_rev'
)
CipdDepsEntry = collections.namedtuple('CipdDepsEntry', 'path packages') CipdDepsEntry = collections.namedtuple('CipdDepsEntry', 'path packages')
VersionEntry = collections.namedtuple('VersionEntry', 'version') VersionEntry = collections.namedtuple('VersionEntry', 'version')
ChangedCipdPackage = collections.namedtuple( ChangedCipdPackage = collections.namedtuple(
'ChangedCipdPackage', 'path package current_version new_version') 'ChangedCipdPackage', 'path package current_version new_version'
)
ChangedVersionEntry = collections.namedtuple( ChangedVersionEntry = collections.namedtuple(
'ChangedVersionEntry', 'path current_version new_version') 'ChangedVersionEntry', 'path current_version new_version'
)
ChromiumRevisionUpdate = collections.namedtuple('ChromiumRevisionUpdate', ChromiumRevisionUpdate = collections.namedtuple(
('current_chromium_rev ' 'ChromiumRevisionUpdate', 'current_chromium_rev new_chromium_rev '
'new_chromium_rev ')) )
class RollError(Exception): class RollError(Exception):
@ -142,16 +146,19 @@ def ParseCommitPosition(commit_message):
m = COMMIT_POSITION_RE.match(line.strip()) m = COMMIT_POSITION_RE.match(line.strip())
if m: if m:
return int(m.group(1)) return int(m.group(1))
logging.error('Failed to parse commit position id from:\n%s\n', logging.error(
commit_message) 'Failed to parse commit position id from:\n%s\n', commit_message
)
sys.exit(-1) sys.exit(-1)
def _RunCommand(command, def _RunCommand(
command,
working_dir=None, working_dir=None,
ignore_exit_code=False, ignore_exit_code=False,
extra_env=None, extra_env=None,
input_data=None): input_data=None,
):
"""Runs a command and returns the output from that command. """Runs a command and returns the output from that command.
If the command fails (exit code != 0), the function will exit the process. If the command fails (exit code != 0), the function will exit the process.
@ -166,20 +173,25 @@ def _RunCommand(command,
assert all(isinstance(value, str) for value in extra_env.values()) assert all(isinstance(value, str) for value in extra_env.values())
logging.debug('extra env: %s', extra_env) logging.debug('extra env: %s', extra_env)
env.update(extra_env) env.update(extra_env)
p = subprocess.Popen(command, p = subprocess.Popen(
command,
stdin=subprocess.PIPE, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, stderr=subprocess.PIPE,
env=env, env=env,
cwd=working_dir, cwd=working_dir,
universal_newlines=True) universal_newlines=True,
)
std_output, err_output = p.communicate(input_data) std_output, err_output = p.communicate(input_data)
p.stdout.close() p.stdout.close()
p.stderr.close() p.stderr.close()
if not ignore_exit_code and p.returncode != 0: if not ignore_exit_code and p.returncode != 0:
logging.error('Command failed: %s\n' logging.error(
'stdout:\n%s\n' 'Command failed: %s\nstdout:\n%s\nstderr:\n%s\n',
'stderr:\n%s\n', ' '.join(command), std_output, err_output) ' '.join(command),
std_output,
err_output,
)
sys.exit(p.returncode) sys.exit(p.returncode)
return std_output, err_output return std_output, err_output
@ -218,11 +230,13 @@ def ReadRemoteCrFile(path_below_src, revision):
Args: Args:
path_below_src: A path to the target file relative to src dir. path_below_src: A path to the target file relative to src dir.
revision: Revision to read. revision: Revision to read.
Returns: Returns:
A string with file content. A string with file content.
""" """
return _ReadGitilesContent(CHROMIUM_FILE_TEMPLATE % return _ReadGitilesContent(
(revision, path_below_src)) CHROMIUM_FILE_TEMPLATE % (revision, path_below_src)
)
def ReadRemoteCrCommit(revision): def ReadRemoteCrCommit(revision):
@ -235,6 +249,7 @@ def ReadUrlContent(url):
Args: Args:
url: URL to connect to. url: URL to connect to.
Returns: Returns:
A list of lines. A list of lines.
""" """
@ -267,7 +282,9 @@ def GetMatchingDepsEntries(depsentry_dict, dir_path):
result.append(depsentry) result.append(depsentry)
else: else:
parts = path.split('/') parts = path.split('/')
if all(part == parts[i] for i, part in enumerate(dir_path.split('/'))): if all(
part == parts[i] for i, part in enumerate(dir_path.split('/'))
):
result.append(depsentry) result.append(depsentry)
return result return result
@ -309,25 +326,29 @@ def BuildDepsentryDict(deps_dict):
def _FindChangedCipdPackages(path, old_pkgs, new_pkgs): def _FindChangedCipdPackages(path, old_pkgs, new_pkgs):
old_pkgs_names = {p['package'] for p in old_pkgs} old_pkgs_names = {p['package'] for p in old_pkgs}
new_pkgs_names = {p['package'] for p in new_pkgs} new_pkgs_names = {p['package'] for p in new_pkgs}
pkgs_equal = (old_pkgs_names == new_pkgs_names) pkgs_equal = old_pkgs_names == new_pkgs_names
added_pkgs = [p for p in new_pkgs_names if p not in old_pkgs_names] added_pkgs = [p for p in new_pkgs_names if p not in old_pkgs_names]
removed_pkgs = [p for p in old_pkgs_names if p not in new_pkgs_names] removed_pkgs = [p for p in old_pkgs_names if p not in new_pkgs_names]
assert pkgs_equal, ('Old: %s\n New: %s.\nYou need to do a manual roll ' assert pkgs_equal, (
'Old: %s\n New: %s.\nYou need to do a manual roll '
'and remove/add entries in DEPS so the old and new ' 'and remove/add entries in DEPS so the old and new '
'list match.\nMost likely, you should add \"%s\" and ' 'list match.\nMost likely, you should add "%s" and '
'remove \"%s\"' % 'remove "%s"' % (old_pkgs, new_pkgs, added_pkgs, removed_pkgs)
(old_pkgs, new_pkgs, added_pkgs, removed_pkgs)) )
for old_pkg in old_pkgs: for old_pkg in old_pkgs:
for new_pkg in new_pkgs: for new_pkg in new_pkgs:
old_version = old_pkg['version'] old_version = old_pkg['version']
new_version = new_pkg['version'] new_version = new_pkg['version']
if (old_pkg['package'] == new_pkg['package'] if (
and old_version != new_version): old_pkg['package'] == new_pkg['package']
and old_version != new_version
):
logging.debug('Roll dependency %s to %s', path, new_version) logging.debug('Roll dependency %s to %s', path, new_version)
yield ChangedCipdPackage(path, old_pkg['package'], old_version, yield ChangedCipdPackage(
new_version) path, old_pkg['package'], old_version, new_version
)
def _FindChangedVars(name, old_version, new_version): def _FindChangedVars(name, old_version, new_version):
@ -337,18 +358,18 @@ def _FindChangedVars(name, old_version, new_version):
def _FindNewDeps(old, new): def _FindNewDeps(old, new):
""" Gather dependencies only in `new` and return corresponding paths. """ """Gather dependencies only in `new` and return corresponding paths."""
old_entries = set(BuildDepsentryDict(old)) old_entries = set(BuildDepsentryDict(old))
new_entries = set(BuildDepsentryDict(new)) new_entries = set(BuildDepsentryDict(new))
return [ return [
path for path in new_entries - old_entries path
for path in new_entries - old_entries
if path not in DONT_AUTOROLL_THESE if path not in DONT_AUTOROLL_THESE
] ]
def FindAddedDeps(libyuv_deps, new_cr_deps): def FindAddedDeps(libyuv_deps, new_cr_deps):
""" """Calculate new deps entries of interest.
Calculate new deps entries of interest.
Ideally, that would mean: only appearing in chromium DEPS Ideally, that would mean: only appearing in chromium DEPS
but transitively used in LibYUV. but transitively used in LibYUV.
@ -379,8 +400,7 @@ def FindAddedDeps(libyuv_deps, new_cr_deps):
def FindRemovedDeps(libyuv_deps, new_cr_deps): def FindRemovedDeps(libyuv_deps, new_cr_deps):
""" """Calculate obsolete deps entries.
Calculate obsolete deps entries.
Ideally, that would mean: no more appearing in chromium DEPS Ideally, that would mean: no more appearing in chromium DEPS
and not used in LibYUV. and not used in LibYUV.
@ -403,19 +423,23 @@ def FindRemovedDeps(libyuv_deps, new_cr_deps):
A list of paths of unexpected disappearing dependencies. A list of paths of unexpected disappearing dependencies.
""" """
all_removed_deps = _FindNewDeps(new_cr_deps, libyuv_deps) all_removed_deps = _FindNewDeps(new_cr_deps, libyuv_deps)
generated_android_deps = sorted( generated_android_deps = sorted([
[path for path in all_removed_deps if path.startswith(ANDROID_DEPS_PATH)]) path for path in all_removed_deps if path.startswith(ANDROID_DEPS_PATH)
])
# Webrtc-only dependencies are handled in CalculateChangedDeps. # Webrtc-only dependencies are handled in CalculateChangedDeps.
other_deps = sorted([ other_deps = sorted([
path for path in all_removed_deps path
for path in all_removed_deps
if path not in generated_android_deps and path not in LIBYUV_ONLY_DEPS if path not in generated_android_deps and path not in LIBYUV_ONLY_DEPS
]) ])
return generated_android_deps, other_deps return generated_android_deps, other_deps
def CalculateChangedDeps(libyuv_deps, new_cr_deps): def CalculateChangedDeps(libyuv_deps, new_cr_deps):
""" """Calculate changed deps entries based on entries defined in the LibYUV
Calculate changed deps entries based on entries defined in the LibYUV DEPS
DEPS
file: file:
- If a shared dependency with the Chromium DEPS file: roll it to the same - If a shared dependency with the Chromium DEPS file: roll it to the same
revision as Chromium (i.e. entry in the new_cr_deps dict) revision as Chromium (i.e. entry in the new_cr_deps dict)
@ -440,26 +464,36 @@ def CalculateChangedDeps(libyuv_deps, new_cr_deps):
if isinstance(cr_deps_entry, CipdDepsEntry): if isinstance(cr_deps_entry, CipdDepsEntry):
result.extend( result.extend(
_FindChangedCipdPackages(path, libyuv_deps_entry.packages, _FindChangedCipdPackages(
cr_deps_entry.packages)) path, libyuv_deps_entry.packages, cr_deps_entry.packages # pylint: disable=line-too-long
)
)
continue continue
if isinstance(cr_deps_entry, VersionEntry): if isinstance(cr_deps_entry, VersionEntry):
result.extend( result.extend(
_FindChangedVars(path, libyuv_deps_entry.version, _FindChangedVars(
cr_deps_entry.version)) path, libyuv_deps_entry.version, cr_deps_entry.version
)
)
continue continue
# Use the revision from Chromium's DEPS file. # Use the revision from Chromium's DEPS file.
new_rev = cr_deps_entry.revision new_rev = cr_deps_entry.revision
assert libyuv_deps_entry.url == cr_deps_entry.url, ( assert libyuv_deps_entry.url == cr_deps_entry.url, (
'LibYUV DEPS entry %s has a different URL %s than Chromium %s.' % 'LibYUV DEPS entry %s has a different URL %s than Chromium %s.'
(path, libyuv_deps_entry.url, cr_deps_entry.url)) % (
path,
libyuv_deps_entry.url,
cr_deps_entry.url,
)
)
else: else:
if isinstance(libyuv_deps_entry, DepsEntry): if isinstance(libyuv_deps_entry, DepsEntry):
# Use the HEAD of the deps repo. # Use the HEAD of the deps repo.
stdout, _ = _RunCommand( stdout, _ = _RunCommand(
['git', 'ls-remote', libyuv_deps_entry.url, 'HEAD']) ['git', 'ls-remote', libyuv_deps_entry.url, 'HEAD']
)
new_rev = stdout.strip().split('\t')[0] new_rev = stdout.strip().split('\t')[0]
else: else:
# The dependency has been removed from chromium. # The dependency has been removed from chromium.
@ -470,8 +504,13 @@ def CalculateChangedDeps(libyuv_deps, new_cr_deps):
if libyuv_deps_entry.revision != new_rev: if libyuv_deps_entry.revision != new_rev:
logging.debug('Roll dependency %s to %s', path, new_rev) logging.debug('Roll dependency %s to %s', path, new_rev)
result.append( result.append(
ChangedDep(path, libyuv_deps_entry.url, libyuv_deps_entry.revision, ChangedDep(
new_rev)) path,
libyuv_deps_entry.url,
libyuv_deps_entry.revision,
new_rev,
)
)
return sorted(result) return sorted(result)
@ -488,10 +527,13 @@ def CalculateChangedClang(new_cr_rev):
current_lines = f.readlines() current_lines = f.readlines()
current_rev = GetClangRev(current_lines) current_rev = GetClangRev(current_lines)
new_clang_update_py = ReadRemoteCrFile(CLANG_UPDATE_SCRIPT_URL_PATH, new_clang_update_py = ReadRemoteCrFile(
new_cr_rev).splitlines() CLANG_UPDATE_SCRIPT_URL_PATH, new_cr_rev
).splitlines()
new_rev = GetClangRev(new_clang_update_py) new_rev = GetClangRev(new_clang_update_py)
return ChangedDep(CLANG_UPDATE_SCRIPT_LOCAL_PATH, None, current_rev, new_rev) return ChangedDep(
CLANG_UPDATE_SCRIPT_LOCAL_PATH, None, current_rev, new_rev
)
def GenerateCommitMessage( def GenerateCommitMessage(
@ -509,9 +551,10 @@ def GenerateCommitMessage(
git_number_interval = '%s:%s' % (current_commit_pos, new_commit_pos) git_number_interval = '%s:%s' % (current_commit_pos, new_commit_pos)
commit_msg = [ commit_msg = [
'Roll chromium_revision %s (%s)\n' % (rev_interval, git_number_interval), 'Roll chromium_revision %s (%s)\n'
% (rev_interval, git_number_interval),
'Change log: %s' % (CHROMIUM_LOG_TEMPLATE % rev_interval), 'Change log: %s' % (CHROMIUM_LOG_TEMPLATE % rev_interval),
'Full diff: %s\n' % (CHROMIUM_COMMIT_TEMPLATE % rev_interval) 'Full diff: %s\n' % (CHROMIUM_COMMIT_TEMPLATE % rev_interval),
] ]
def Section(adjective, deps): def Section(adjective, deps):
@ -523,14 +566,19 @@ def GenerateCommitMessage(
for c in changed_deps_list: for c in changed_deps_list:
if isinstance(c, ChangedCipdPackage): if isinstance(c, ChangedCipdPackage):
commit_msg.append('* %s: %s..%s' % commit_msg.append(
(c.path, c.current_version, c.new_version)) '* %s: %s..%s' % (c.path, c.current_version, c.new_version)
)
elif isinstance(c, ChangedVersionEntry): elif isinstance(c, ChangedVersionEntry):
commit_msg.append('* %s_vesion: %s..%s' % commit_msg.append(
(c.path, c.current_version, c.new_version)) '* %s_vesion: %s..%s'
% (c.path, c.current_version, c.new_version)
)
else: else:
commit_msg.append('* %s: %s/+log/%s..%s' % commit_msg.append(
(c.path, c.url, c.current_rev[0:10], c.new_rev[0:10])) '* %s: %s/+log/%s..%s'
% (c.path, c.url, c.current_rev[0:10], c.new_rev[0:10])
)
if added_deps_paths: if added_deps_paths:
Section('Added', added_deps_paths) Section('Added', added_deps_paths)
@ -547,10 +595,14 @@ def GenerateCommitMessage(
commit_msg.append('No dependencies changed.') commit_msg.append('No dependencies changed.')
if clang_change and clang_change.current_rev != clang_change.new_rev: if clang_change and clang_change.current_rev != clang_change.new_rev:
commit_msg.append('Clang version changed %s:%s' % commit_msg.append(
(clang_change.current_rev, clang_change.new_rev)) 'Clang version changed %s:%s'
change_url = CHROMIUM_FILE_TEMPLATE % (rev_interval, % (clang_change.current_rev, clang_change.new_rev)
CLANG_UPDATE_SCRIPT_URL_PATH) )
change_url = CHROMIUM_FILE_TEMPLATE % (
rev_interval,
CLANG_UPDATE_SCRIPT_URL_PATH,
)
commit_msg.append('Details: %s\n' % change_url) commit_msg.append('Details: %s\n' % change_url)
else: else:
commit_msg.append('No update to Clang.\n') commit_msg.append('No update to Clang.\n')
@ -566,25 +618,31 @@ def UpdateDepsFile(deps_filename, rev_update, changed_deps, new_cr_content):
deps_content = deps_file.read().decode('utf-8') deps_content = deps_file.read().decode('utf-8')
# Update the chromium_revision variable. # Update the chromium_revision variable.
deps_content = deps_content.replace(rev_update.current_chromium_rev, deps_content = deps_content.replace(
rev_update.new_chromium_rev) rev_update.current_chromium_rev, rev_update.new_chromium_rev
)
# Add and remove dependencies. For now: only generated android deps. # Add and remove dependencies. For now: only generated android deps.
# Since gclient cannot add or remove deps, we on the fact that # Since gclient cannot add or remove deps, we on the fact that
# these android deps are located in one place we can copy/paste. # these android deps are located in one place we can copy/paste.
deps_re = re.compile(ANDROID_DEPS_START + '.*' + ANDROID_DEPS_END, re.DOTALL) deps_re = re.compile(
ANDROID_DEPS_START + '.*' + ANDROID_DEPS_END, re.DOTALL
)
new_deps = deps_re.search(new_cr_content) new_deps = deps_re.search(new_cr_content)
old_deps = deps_re.search(deps_content) old_deps = deps_re.search(deps_content)
if not new_deps or not old_deps: if not new_deps or not old_deps:
faulty = 'Chromium' if not new_deps else 'LibYUV' faulty = 'Chromium' if not new_deps else 'LibYUV'
raise RollError('Was expecting to find "%s" and "%s"\n' raise RollError(
'in %s DEPS' % 'Was expecting to find "%s" and "%s"\nin %s DEPS'
(ANDROID_DEPS_START, ANDROID_DEPS_END, faulty)) % (ANDROID_DEPS_START, ANDROID_DEPS_END, faulty)
)
deps_content = deps_re.sub(new_deps.group(0), deps_content) deps_content = deps_re.sub(new_deps.group(0), deps_content)
for dep in changed_deps: for dep in changed_deps:
if isinstance(dep, ChangedVersionEntry): if isinstance(dep, ChangedVersionEntry):
deps_content = deps_content.replace(dep.current_version, dep.new_version) deps_content = deps_content.replace(
dep.current_version, dep.new_version
)
with open(deps_filename, 'wb') as deps_file: with open(deps_filename, 'wb') as deps_file:
deps_file.write(deps_content.encode('utf-8')) deps_file.write(deps_content.encode('utf-8'))
@ -599,17 +657,20 @@ def UpdateDepsFile(deps_filename, rev_update, changed_deps, new_cr_content):
raise RollError( raise RollError(
'Cannot find local directory %s. Either run\n' 'Cannot find local directory %s. Either run\n'
'gclient sync --deps=all\n' 'gclient sync --deps=all\n'
'or make sure the .gclient file for your solution contains all ' 'or make sure the .gclient file for your solution contains '
'platforms in the target_os list, i.e.\n' 'all platforms in the target_os list, i.e.\n'
'target_os = ["android", "unix", "mac", "ios", "win"];\n' 'target_os = ["android", "unix", "mac", "ios", "win"];\n'
'Then run "gclient sync" again.' % local_dep_dir) 'Then run "gclient sync" again.' % local_dep_dir
)
if isinstance(dep, ChangedCipdPackage): if isinstance(dep, ChangedCipdPackage):
package = dep.package.format() # Eliminate double curly brackets package = dep.package.format() # Eliminate double curly brackets
update = '%s:%s@%s' % (dep.path, package, dep.new_version) update = '%s:%s@%s' % (dep.path, package, dep.new_version)
else: else:
update = '%s@%s' % (dep.path, dep.new_rev) update = '%s@%s' % (dep.path, dep.new_rev)
_RunCommand(['gclient', 'setdep', '--revision', update], _RunCommand(
working_dir=CHECKOUT_SRC_DIR) ['gclient', 'setdep', '--revision', update],
working_dir=CHECKOUT_SRC_DIR,
)
def _IsTreeClean(): def _IsTreeClean():
@ -622,10 +683,11 @@ def _IsTreeClean():
def _EnsureUpdatedMainBranch(dry_run): def _EnsureUpdatedMainBranch(dry_run):
current_branch = _RunCommand(['git', 'rev-parse', '--abbrev-ref', current_branch = _RunCommand(['git', 'rev-parse', '--abbrev-ref', 'HEAD'])[
'HEAD'])[0].splitlines()[0] 0
].splitlines()[0]
if current_branch != 'main': if current_branch != 'main':
logging.error('Please checkout the main branch and re-run this script.') logging.error('Please checkout the main branch and re-run this script.') # pylint: disable=line-too-long
if not dry_run: if not dry_run:
sys.exit(-1) sys.exit(-1)
@ -666,10 +728,9 @@ def ChooseCQMode(skip_cq, cq_over, current_commit_pos, new_commit_pos):
def _GetCcRecipients(changed_deps_list): def _GetCcRecipients(changed_deps_list):
"""Returns a list of emails to notify based on the changed deps list. """Returns a list of emails to notify based on the changed deps list."""
"""
cc_recipients = [] cc_recipients = []
for c in changed_deps_list: for _ in changed_deps_list:
pass pass
return cc_recipients return cc_recipients
@ -701,8 +762,11 @@ def _UploadCL(commit_queue_mode, add_cc=None):
'SKIP_GCE_AUTH_FOR_GIT': '1', 'SKIP_GCE_AUTH_FOR_GIT': '1',
} }
stdout, stderr = _RunCommand(cmd, extra_env=extra_env) stdout, stderr = _RunCommand(cmd, extra_env=extra_env)
logging.debug('Output from "git cl upload":\nstdout:\n%s\n\nstderr:\n%s', logging.debug(
stdout, stderr) 'Output from "git cl upload":\nstdout:\n%s\n\nstderr:\n%s',
stdout,
stderr,
)
def GetRollRevisionRanges(opts, libyuv_deps): def GetRollRevisionRanges(opts, libyuv_deps):
@ -719,41 +783,63 @@ def GetRollRevisionRanges(opts, libyuv_deps):
def main(): def main():
p = argparse.ArgumentParser() p = argparse.ArgumentParser()
p.add_argument('--clean', p.add_argument(
'--clean',
action='store_true', action='store_true',
default=False, default=False,
help='Removes any previous local roll branch.') help='Removes any previous local roll branch.',
p.add_argument('-r', )
p.add_argument(
'-r',
'--revision', '--revision',
help=('Chromium Git revision to roll to. Defaults to the ' help=(
'Chromium HEAD revision if omitted.')) 'Chromium Git revision to roll to. Defaults to the '
p.add_argument('--dry-run', 'Chromium HEAD revision if omitted.'
),
)
p.add_argument(
'--dry-run',
action='store_true', action='store_true',
default=False, default=False,
help=('Calculate changes and modify DEPS, but don\'t create ' help=(
"Calculate changes and modify DEPS, but don't create "
'any local branch, commit, upload CL or send any ' 'any local branch, commit, upload CL or send any '
'tryjobs.')) 'tryjobs.'
p.add_argument('-i', ),
)
p.add_argument(
'-i',
'--ignore-unclean-workdir', '--ignore-unclean-workdir',
action='store_true', action='store_true',
default=False, default=False,
help=('Ignore if the current branch is not main or if there ' help=(
'are uncommitted changes (default: %(default)s).')) 'Ignore if the current branch is not main or if there '
'are uncommitted changes (default: %(default)s).'
),
)
grp = p.add_mutually_exclusive_group() grp = p.add_mutually_exclusive_group()
grp.add_argument('--skip-cq', grp.add_argument(
'--skip-cq',
action='store_true', action='store_true',
default=False, default=False,
help='Skip sending the CL to the CQ (default: %(default)s)') help='Skip sending the CL to the CQ (default: %(default)s)',
grp.add_argument('--cq-over', )
grp.add_argument(
'--cq-over',
type=int, type=int,
default=1, default=1,
help=('Commit queue dry run if the revision difference ' help=(
'is below this number (default: %(default)s)')) 'Commit queue dry run if the revision difference '
p.add_argument('-v', 'is below this number (default: %(default)s)'
),
)
p.add_argument(
'-v',
'--verbose', '--verbose',
action='store_true', action='store_true',
default=False, default=False,
help='Be extra verbose in printing of log messages.') help='Be extra verbose in printing of log messages.',
)
opts = p.parse_args() opts = p.parse_args()
if opts.verbose: if opts.verbose:
@ -777,9 +863,11 @@ def main():
rev_update = GetRollRevisionRanges(opts, libyuv_deps) rev_update = GetRollRevisionRanges(opts, libyuv_deps)
current_commit_pos = ParseCommitPosition( current_commit_pos = ParseCommitPosition(
ReadRemoteCrCommit(rev_update.current_chromium_rev)) ReadRemoteCrCommit(rev_update.current_chromium_rev)
)
new_commit_pos = ParseCommitPosition( new_commit_pos = ParseCommitPosition(
ReadRemoteCrCommit(rev_update.new_chromium_rev)) ReadRemoteCrCommit(rev_update.new_chromium_rev)
)
new_cr_content = ReadRemoteCrFile('DEPS', rev_update.new_chromium_rev) new_cr_content = ReadRemoteCrFile('DEPS', rev_update.new_chromium_rev)
new_cr_deps = ParseDepsDict(new_cr_content) new_cr_deps = ParseDepsDict(new_cr_content)
@ -787,11 +875,14 @@ def main():
# Discard other deps, assumed to be chromium-only dependencies. # Discard other deps, assumed to be chromium-only dependencies.
new_generated_android_deps, _ = FindAddedDeps(libyuv_deps, new_cr_deps) new_generated_android_deps, _ = FindAddedDeps(libyuv_deps, new_cr_deps)
removed_generated_android_deps, other_deps = FindRemovedDeps( removed_generated_android_deps, other_deps = FindRemovedDeps(
libyuv_deps, new_cr_deps) libyuv_deps, new_cr_deps
)
if other_deps: if other_deps:
raise RollError('LibYUV DEPS entries are missing from Chromium: %s.\n' raise RollError(
'LibYUV DEPS entries are missing from Chromium: %s.\n'
'Remove them or add them to either ' 'Remove them or add them to either '
'LIBYUV_ONLY_DEPS or DONT_AUTOROLL_THESE.' % other_deps) 'LIBYUV_ONLY_DEPS or DONT_AUTOROLL_THESE.' % other_deps
)
clang_change = CalculateChangedClang(rev_update.new_chromium_rev) clang_change = CalculateChangedClang(rev_update.new_chromium_rev)
commit_msg = GenerateCommitMessage( commit_msg = GenerateCommitMessage(
rev_update, rev_update,
@ -800,18 +891,20 @@ def main():
changed_deps, changed_deps,
added_deps_paths=new_generated_android_deps, added_deps_paths=new_generated_android_deps,
removed_deps_paths=removed_generated_android_deps, removed_deps_paths=removed_generated_android_deps,
clang_change=clang_change) clang_change=clang_change,
)
logging.debug('Commit message:\n%s', commit_msg) logging.debug('Commit message:\n%s', commit_msg)
_CreateRollBranch(opts.dry_run) _CreateRollBranch(opts.dry_run)
if not opts.dry_run: if not opts.dry_run:
UpdateDepsFile(deps_filename, rev_update, changed_deps, new_cr_content) UpdateDepsFile(deps_filename, rev_update, changed_deps, new_cr_content)
if _IsTreeClean(): if _IsTreeClean():
logging.info("No DEPS changes detected, skipping CL creation.") logging.info('No DEPS changes detected, skipping CL creation.')
else: else:
_LocalCommit(commit_msg, opts.dry_run) _LocalCommit(commit_msg, opts.dry_run)
commit_queue_mode = ChooseCQMode(opts.skip_cq, opts.cq_over, commit_queue_mode = ChooseCQMode(
current_commit_pos, new_commit_pos) opts.skip_cq, opts.cq_over, current_commit_pos, new_commit_pos
)
logging.info('Uploading CL...') logging.info('Uploading CL...')
if not opts.dry_run: if not opts.dry_run:
_UploadCL(commit_queue_mode, _GetCcRecipients(changed_deps)) _UploadCL(commit_queue_mode, _GetCcRecipients(changed_deps))

38
tools_libyuv/autoroller/unittests/roll_deps_test.py
View File

@ -16,8 +16,7 @@ import tempfile
import unittest import unittest
import roll_deps import roll_deps
from roll_deps import CalculateChangedDeps, GetMatchingDepsEntries, \ from roll_deps import CalculateChangedDeps, GetMatchingDepsEntries, ParseDepsDict, ParseLocalDepsFile, UpdateDepsFile # pylint: disable=line-too-long
ParseDepsDict, ParseLocalDepsFile, UpdateDepsFile
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
PARENT_DIR = os.path.join(SCRIPT_DIR, os.pardir) PARENT_DIR = os.path.join(SCRIPT_DIR, os.pardir)
@ -45,7 +44,8 @@ class TestError(Exception):
pass pass
class FakeCmd(): class FakeCmd:
def __init__(self): def __init__(self):
self.expectations = [] self.expectations = []
@ -58,20 +58,28 @@ class FakeCmd():
raise TestError('Got unexpected\n%s\n%s' % (args, kwargs)) raise TestError('Got unexpected\n%s\n%s' % (args, kwargs))
exp_args, exp_kwargs, exp_returns = self.expectations.pop(0) exp_args, exp_kwargs, exp_returns = self.expectations.pop(0)
if args != exp_args or kwargs != exp_kwargs: if args != exp_args or kwargs != exp_kwargs:
message = 'Expected:\n args: %s\n kwargs: %s\n' % (exp_args, exp_kwargs) message = 'Expected:\n args: %s\n kwargs: %s\n' % (
exp_args,
exp_kwargs,
)
message += 'Got:\n args: %s\n kwargs: %s\n' % (args, kwargs) message += 'Got:\n args: %s\n kwargs: %s\n' % (args, kwargs)
raise TestError(message) raise TestError(message)
return exp_returns return exp_returns
class TestRollChromiumRevision(unittest.TestCase): class TestRollChromiumRevision(unittest.TestCase):
def setUp(self): def setUp(self):
self._output_dir = tempfile.mkdtemp() self._output_dir = tempfile.mkdtemp()
for test_file in glob.glob(os.path.join(SCRIPT_DIR, 'testdata', '*')): for test_file in glob.glob(os.path.join(SCRIPT_DIR, 'testdata', '*')):
shutil.copy(test_file, self._output_dir) shutil.copy(test_file, self._output_dir)
self._libyuv_depsfile = os.path.join(self._output_dir, 'DEPS') self._libyuv_depsfile = os.path.join(self._output_dir, 'DEPS')
self._old_cr_depsfile = os.path.join(self._output_dir, 'DEPS.chromium.old') self._old_cr_depsfile = os.path.join(
self._new_cr_depsfile = os.path.join(self._output_dir, 'DEPS.chromium.new') self._output_dir, 'DEPS.chromium.old'
)
self._new_cr_depsfile = os.path.join(
self._output_dir, 'DEPS.chromium.new'
)
self.fake = FakeCmd() self.fake = FakeCmd()
self.old_RunCommand = getattr(roll_deps, '_RunCommand') self.old_RunCommand = getattr(roll_deps, '_RunCommand')
@ -94,8 +102,10 @@ class TestRollChromiumRevision(unittest.TestCase):
UpdateDepsFile(self._libyuv_depsfile, current_rev, new_rev, []) UpdateDepsFile(self._libyuv_depsfile, current_rev, new_rev, [])
with open(self._libyuv_depsfile, 'r') as deps_file: with open(self._libyuv_depsfile, 'r') as deps_file:
deps_contents = deps_file.read() deps_contents = deps_file.read()
self.assertTrue(new_rev in deps_contents, self.assertTrue(
'Failed to find %s in\n%s' % (new_rev, deps_contents)) new_rev in deps_contents,
'Failed to find %s in\n%s' % (new_rev, deps_contents),
)
def testParseDepsDict(self): def testParseDepsDict(self):
with open(self._libyuv_depsfile, 'r') as deps_file: with open(self._libyuv_depsfile, 'r') as deps_file:
@ -104,7 +114,10 @@ class TestRollChromiumRevision(unittest.TestCase):
vars_dict = local_scope['vars'] vars_dict = local_scope['vars']
def assertVar(variable_name): def assertVar(variable_name):
self.assertEqual(vars_dict[variable_name], TEST_DATA_VARS[variable_name]) self.assertEqual(
vars_dict[variable_name], TEST_DATA_VARS[variable_name]
)
assertVar('chromium_git') assertVar('chromium_git')
assertVar('chromium_revision') assertVar('chromium_revision')
self.assertEqual(len(local_scope['deps']), 3) self.assertEqual(len(local_scope['deps']), 3)
@ -124,8 +137,11 @@ class TestRollChromiumRevision(unittest.TestCase):
self.assertEqual(entries[0], DEPS_ENTRIES['src/build']) self.assertEqual(entries[0], DEPS_ENTRIES['src/build'])
def testCalculateChangedDeps(self): def testCalculateChangedDeps(self):
_SetupGitLsRemoteCall(self.fake, _SetupGitLsRemoteCall(
'https://chromium.googlesource.com/chromium/src/build', BUILD_NEW_REV) self.fake,
'https://chromium.googlesource.com/chromium/src/build',
BUILD_NEW_REV,
)
libyuv_deps = ParseLocalDepsFile(self._libyuv_depsfile) libyuv_deps = ParseLocalDepsFile(self._libyuv_depsfile)
new_cr_deps = ParseLocalDepsFile(self._new_cr_depsfile) new_cr_deps = ParseLocalDepsFile(self._new_cr_depsfile)
changed_deps = CalculateChangedDeps(libyuv_deps, new_cr_deps) changed_deps = CalculateChangedDeps(libyuv_deps, new_cr_deps)

15
tools_libyuv/get_landmines.py
View File

@ -9,23 +9,22 @@
# be found in the AUTHORS file in the root of the source tree. # be found in the AUTHORS file in the root of the source tree.
""" """
This file emits the list of reasons why a particular build needs to be clobbered This file emits the list of reasons why a particular build needs to be
(or a list of 'landmines'). clobbered (or a list of 'landmines').
""" """
import sys import sys
def print_landmines(): def print_landmines():
""" """ALL LANDMINES ARE EMITTED FROM HERE."""
ALL LANDMINES ARE EMITTED FROM HERE.
"""
# DO NOT add landmines as part of a regular CL. Landmines are a last-effort # DO NOT add landmines as part of a regular CL. Landmines are a last-effort
# bandaid fix if a CL that got landed has a build dependency bug and all bots # bandaid fix if a CL that got landed has a build dependency bug and all
# need to be cleaned up. If you're writing a new CL that causes build # bots need to be cleaned up. If you're writing a new CL that causes build
# dependency problems, fix the dependency problems instead of adding a # dependency problems, fix the dependency problems instead of adding a
# landmine. # landmine.
# See the Chromium version in src/build/get_landmines.py for usage examples. # See the Chromium version in src/build/get_landmines.py for usage
# examples.
print('Clobber to remove GYP artifacts after switching bots to GN.') print('Clobber to remove GYP artifacts after switching bots to GN.')
print('Another try to remove GYP artifacts after switching bots to GN.') print('Another try to remove GYP artifacts after switching bots to GN.')