From 37e88ecc688b2752e7d1b8053035d25ecec84389 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Terelius?= Date: Wed, 18 Dec 2024 14:02:45 +0100 Subject: [PATCH] Update pylintrc to a pep-8 like style In particular, this requires reformatting - autoroller/ - PRESUBMIT.py - infra/config/PRESUBMIT.py - get_landmines.py - download_vs_toolchain.py as well as some manual fixes. This CL is preparation to updating the autoroller so that we can roll DEPS from chromium. No-Try: True Bug: libyuv:358992053 Change-Id: I090a09c6fa61beff2427a0537bca371f0839fa3e Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/6105550 Reviewed-by: Mirko Bonadei Reviewed-by: Jeremy Leconte Commit-Queue: Jeremy Leconte --- PRESUBMIT.py | 81 +- download_vs_toolchain.py | 2 +- infra/config/PRESUBMIT.py | 8 +- pylintrc | 36 +- tools_libyuv/autoroller/roll_deps.py | 1079 +++++++++-------- .../autoroller/unittests/roll_deps_test.py | 190 +-- tools_libyuv/get_landmines.py | 31 +- 7 files changed, 792 insertions(+), 635 deletions(-) diff --git a/PRESUBMIT.py b/PRESUBMIT.py index d3901caf3..37df28a10 100644 --- a/PRESUBMIT.py +++ b/PRESUBMIT.py @@ -9,44 +9,57 @@ # Runs PRESUBMIT.py in py3 mode by git cl presubmit. USE_PYTHON3 = True + def _CommonChecks(input_api, output_api): - """Checks common to both upload and commit.""" - results = [] - results.extend(input_api.canned_checks.RunPylint(input_api, output_api, - files_to_skip=(r'^base[\\\/].*\.py$', - r'^build[\\\/].*\.py$', - r'^buildtools[\\\/].*\.py$', - r'^ios[\\\/].*\.py$', - r'^out.*[\\\/].*\.py$', - r'^testing[\\\/].*\.py$', - r'^third_party[\\\/].*\.py$', - r'^tools[\\\/].*\.py$', - # TODO(kjellander): should arguably be checked. - r'^tools_libyuv[\\\/]valgrind[\\\/].*\.py$', - r'^xcodebuild.*[\\\/].*\.py$',), - disabled_warnings=['F0401', # Failed to import x - 'E0611', # No package y in x - 'W0232', # Class has no __init__ method - ], - pylintrc='pylintrc', - version='2.7')) - return results + """Checks common to both upload and commit.""" + results = [] + results.extend( + input_api.canned_checks.RunPylint( + input_api, + output_api, + files_to_skip=( + r'^base[\\\/].*\.py$', + r'^build[\\\/].*\.py$', + r'^buildtools[\\\/].*\.py$', + r'^ios[\\\/].*\.py$', + r'^out.*[\\\/].*\.py$', + r'^testing[\\\/].*\.py$', + r'^third_party[\\\/].*\.py$', + r'^tools[\\\/].*\.py$', + # TODO(kjellander): should arguably be checked. + r'^tools_libyuv[\\\/]valgrind[\\\/].*\.py$', + r'^xcodebuild.*[\\\/].*\.py$', + ), + disabled_warnings=[ + 'F0401', # Failed to import x + 'E0611', # No package y in x + 'W0232', # Class has no __init__ method + ], + pylintrc='pylintrc', + version='2.7', + ) + ) + return results def CheckChangeOnUpload(input_api, output_api): - results = [] - results.extend(_CommonChecks(input_api, output_api)) - results.extend( - input_api.canned_checks.CheckGNFormatted(input_api, output_api)) - return results + results = [] + results.extend(_CommonChecks(input_api, output_api)) + results.extend( + input_api.canned_checks.CheckGNFormatted(input_api, output_api) + ) + return results def CheckChangeOnCommit(input_api, output_api): - results = [] - results.extend(_CommonChecks(input_api, output_api)) - results.extend(input_api.canned_checks.CheckOwners(input_api, output_api)) - results.extend(input_api.canned_checks.CheckChangeWasUploaded( - input_api, output_api)) - results.extend(input_api.canned_checks.CheckChangeHasDescription( - input_api, output_api)) - return results + results = [] + results.extend(_CommonChecks(input_api, output_api)) + results.extend(input_api.canned_checks.CheckOwners(input_api, output_api)) + results.extend( + input_api.canned_checks.CheckChangeWasUploaded(input_api, output_api) + ) + results.extend( + input_api.canned_checks.CheckChangeHasDescription(input_api, + output_api) + ) + return results diff --git a/download_vs_toolchain.py b/download_vs_toolchain.py index 6bc086d68..9fc89affe 100644 --- a/download_vs_toolchain.py +++ b/download_vs_toolchain.py @@ -26,4 +26,4 @@ import vs_toolchain # pylint: disable=wrong-import-position if __name__ == '__main__': - sys.exit(vs_toolchain.main()) + sys.exit(vs_toolchain.main()) diff --git a/infra/config/PRESUBMIT.py b/infra/config/PRESUBMIT.py index f79e08ad6..0aa465e24 100644 --- a/infra/config/PRESUBMIT.py +++ b/infra/config/PRESUBMIT.py @@ -6,8 +6,12 @@ USE_PYTHON3 = True def CheckChangeOnUpload(input_api, output_api): - return input_api.canned_checks.CheckChangedLUCIConfigs(input_api, output_api) + return input_api.canned_checks.CheckChangedLUCIConfigs( + input_api, output_api + ) def CheckChangeOnCommit(input_api, output_api): - return input_api.canned_checks.CheckChangedLUCIConfigs(input_api, output_api) + return input_api.canned_checks.CheckChangedLUCIConfigs( + input_api, output_api + ) diff --git a/pylintrc b/pylintrc index b8bea3348..57be7df05 100644 --- a/pylintrc +++ b/pylintrc @@ -13,5 +13,37 @@ reports=no [FORMAT] -# We use two spaces for indents, instead of the usual four spaces or tab. -indent-string=' ' +# Maximum number of characters on a single line. +max-line-length=79 + +# We use four spaces for indents. +indent-string=' ' + +[BASIC] + +# List of builtins function names that should not be used, separated by a comma +bad-functions=map,filter,apply,input + +# Naming style matching correct module names. +module-naming-style=snake_case + +# Naming style matching correct constant names. +const-naming-style=UPPER_CASE + +# Naming style matching correct class names. +class-naming-style=PascalCase + +# Naming style matching correct function names. +function-naming-style=snake_case + +# Regular expression matching correct method names. +method-rgx=([a-z_][a-z0-9_]{2,}|setUp|tearDown)$ + +# Naming style matching correct attribute names. +attr-naming-style=snake_case + +# Naming style matching correct argument names. +argument-naming-style=snake_case + +# Naming style matching correct variable names. +variable-naming-style=snake_case diff --git a/tools_libyuv/autoroller/roll_deps.py b/tools_libyuv/autoroller/roll_deps.py index d5c1089f8..f4fbf25a2 100755 --- a/tools_libyuv/autoroller/roll_deps.py +++ b/tools_libyuv/autoroller/roll_deps.py @@ -19,14 +19,15 @@ import re import subprocess import sys import urllib.request +import find_depot_tools def FindSrcDirPath(): - """Returns the abs path to the src/ dir of the project.""" - src_dir = os.path.dirname(os.path.abspath(__file__)) - while os.path.basename(src_dir) != 'src': - src_dir = os.path.normpath(os.path.join(src_dir, os.pardir)) - return src_dir + """Returns the abs path to the src/ dir of the project.""" + src_dir = os.path.dirname(os.path.abspath(__file__)) + while os.path.basename(src_dir) != 'src': + src_dir = os.path.normpath(os.path.join(src_dir, os.pardir)) + return src_dir # Skip these dependencies (list without solution name prefix). @@ -85,171 +86,185 @@ ANDROID_DEPS_END = r'=== ANDROID_DEPS Generated Code End ===' ANDROID_DEPS_PATH = 'src/third_party/android_deps/' sys.path.append(os.path.join(CHECKOUT_SRC_DIR, 'build')) -import find_depot_tools find_depot_tools.add_depot_tools_to_path() CLANG_UPDATE_SCRIPT_URL_PATH = 'tools/clang/scripts/update.py' -CLANG_UPDATE_SCRIPT_LOCAL_PATH = os.path.join(CHECKOUT_SRC_DIR, 'tools', - 'clang', 'scripts', 'update.py') +CLANG_UPDATE_SCRIPT_LOCAL_PATH = os.path.join( + CHECKOUT_SRC_DIR, 'tools', 'clang', 'scripts', 'update.py' +) DepsEntry = collections.namedtuple('DepsEntry', 'path url revision') -ChangedDep = collections.namedtuple('ChangedDep', - 'path url current_rev new_rev') +ChangedDep = collections.namedtuple( + 'ChangedDep', 'path url current_rev new_rev' +) CipdDepsEntry = collections.namedtuple('CipdDepsEntry', 'path packages') VersionEntry = collections.namedtuple('VersionEntry', 'version') ChangedCipdPackage = collections.namedtuple( - 'ChangedCipdPackage', 'path package current_version new_version') + 'ChangedCipdPackage', 'path package current_version new_version' +) ChangedVersionEntry = collections.namedtuple( - 'ChangedVersionEntry', 'path current_version new_version') + 'ChangedVersionEntry', 'path current_version new_version' +) -ChromiumRevisionUpdate = collections.namedtuple('ChromiumRevisionUpdate', - ('current_chromium_rev ' - 'new_chromium_rev ')) +ChromiumRevisionUpdate = collections.namedtuple( + 'ChromiumRevisionUpdate', 'current_chromium_rev new_chromium_rev ' +) class RollError(Exception): - pass + pass def StrExpansion(): - return lambda str_value: str_value + return lambda str_value: str_value def VarLookup(local_scope): - return lambda var_name: local_scope['vars'][var_name] + return lambda var_name: local_scope['vars'][var_name] def ParseDepsDict(deps_content): - local_scope = {} - global_scope = { - 'Str': StrExpansion(), - 'Var': VarLookup(local_scope), - 'deps_os': {}, - } - exec(deps_content, global_scope, local_scope) - return local_scope + local_scope = {} + global_scope = { + 'Str': StrExpansion(), + 'Var': VarLookup(local_scope), + 'deps_os': {}, + } + exec(deps_content, global_scope, local_scope) + return local_scope def ParseLocalDepsFile(filename): - with open(filename, 'rb') as f: - deps_content = f.read().decode('utf-8') - return ParseDepsDict(deps_content) + with open(filename, 'rb') as f: + deps_content = f.read().decode('utf-8') + return ParseDepsDict(deps_content) def ParseCommitPosition(commit_message): - for line in reversed(commit_message.splitlines()): - m = COMMIT_POSITION_RE.match(line.strip()) - if m: - return int(m.group(1)) - logging.error('Failed to parse commit position id from:\n%s\n', - commit_message) - sys.exit(-1) + for line in reversed(commit_message.splitlines()): + m = COMMIT_POSITION_RE.match(line.strip()) + if m: + return int(m.group(1)) + logging.error( + 'Failed to parse commit position id from:\n%s\n', commit_message + ) + sys.exit(-1) -def _RunCommand(command, - working_dir=None, - ignore_exit_code=False, - extra_env=None, - input_data=None): - """Runs a command and returns the output from that command. +def _RunCommand( + command, + working_dir=None, + ignore_exit_code=False, + extra_env=None, + input_data=None, +): + """Runs a command and returns the output from that command. If the command fails (exit code != 0), the function will exit the process. Returns: A tuple containing the stdout and stderr outputs as strings. """ - working_dir = working_dir or CHECKOUT_SRC_DIR - logging.debug('CMD: %s CWD: %s', ' '.join(command), working_dir) - env = os.environ.copy() - if extra_env: - assert all(isinstance(value, str) for value in extra_env.values()) - logging.debug('extra env: %s', extra_env) - env.update(extra_env) - p = subprocess.Popen(command, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - env=env, - cwd=working_dir, - universal_newlines=True) - std_output, err_output = p.communicate(input_data) - p.stdout.close() - p.stderr.close() - if not ignore_exit_code and p.returncode != 0: - logging.error('Command failed: %s\n' - 'stdout:\n%s\n' - 'stderr:\n%s\n', ' '.join(command), std_output, err_output) - sys.exit(p.returncode) - return std_output, err_output + working_dir = working_dir or CHECKOUT_SRC_DIR + logging.debug('CMD: %s CWD: %s', ' '.join(command), working_dir) + env = os.environ.copy() + if extra_env: + assert all(isinstance(value, str) for value in extra_env.values()) + logging.debug('extra env: %s', extra_env) + env.update(extra_env) + p = subprocess.Popen( + command, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env, + cwd=working_dir, + universal_newlines=True, + ) + std_output, err_output = p.communicate(input_data) + p.stdout.close() + p.stderr.close() + if not ignore_exit_code and p.returncode != 0: + logging.error( + 'Command failed: %s\nstdout:\n%s\nstderr:\n%s\n', + ' '.join(command), + std_output, + err_output, + ) + sys.exit(p.returncode) + return std_output, err_output def _GetBranches(): - """Returns a tuple of active,branches. + """Returns a tuple of active,branches. The 'active' is the name of the currently active branch and 'branches' is a list of all branches. """ - lines = _RunCommand(['git', 'branch'])[0].split('\n') - branches = [] - active = '' - for line in lines: - if '*' in line: - # The assumption is that the first char will always be the '*'. - active = line[1:].strip() - branches.append(active) - else: - branch = line.strip() - if branch: - branches.append(branch) - return active, branches + lines = _RunCommand(['git', 'branch'])[0].split('\n') + branches = [] + active = '' + for line in lines: + if '*' in line: + # The assumption is that the first char will always be the '*'. + active = line[1:].strip() + branches.append(active) + else: + branch = line.strip() + if branch: + branches.append(branch) + return active, branches def _ReadGitilesContent(url): - # Download and decode BASE64 content until - # https://code.google.com/p/gitiles/issues/detail?id=7 is fixed. - base64_content = ReadUrlContent(url + '?format=TEXT') - return base64.b64decode(base64_content[0]).decode('utf-8') + # Download and decode BASE64 content until + # https://code.google.com/p/gitiles/issues/detail?id=7 is fixed. + base64_content = ReadUrlContent(url + '?format=TEXT') + return base64.b64decode(base64_content[0]).decode('utf-8') def ReadRemoteCrFile(path_below_src, revision): - """Reads a remote Chromium file of a specific revision. + """Reads a remote Chromium file of a specific revision. Args: path_below_src: A path to the target file relative to src dir. revision: Revision to read. + Returns: A string with file content. """ - return _ReadGitilesContent(CHROMIUM_FILE_TEMPLATE % - (revision, path_below_src)) + return _ReadGitilesContent( + CHROMIUM_FILE_TEMPLATE % (revision, path_below_src) + ) def ReadRemoteCrCommit(revision): - """Reads a remote Chromium commit message. Returns a string.""" - return _ReadGitilesContent(CHROMIUM_COMMIT_TEMPLATE % revision) + """Reads a remote Chromium commit message. Returns a string.""" + return _ReadGitilesContent(CHROMIUM_COMMIT_TEMPLATE % revision) def ReadUrlContent(url): - """Connect to a remote host and read the contents. + """Connect to a remote host and read the contents. Args: url: URL to connect to. + Returns: A list of lines. """ - conn = urllib.request.urlopen(url) - try: - return conn.readlines() - except IOError as e: - logging.exception('Error connecting to %s. Error: %s', url, e) - raise - finally: - conn.close() + conn = urllib.request.urlopen(url) + try: + return conn.readlines() + except IOError as e: + logging.exception('Error connecting to %s. Error: %s', url, e) + raise + finally: + conn.close() def GetMatchingDepsEntries(depsentry_dict, dir_path): - """Gets all deps entries matching the provided path. + """Gets all deps entries matching the provided path. This list may contain more than one DepsEntry object. Example: dir_path='src/testing' would give results containing both @@ -261,94 +276,100 @@ def GetMatchingDepsEntries(depsentry_dict, dir_path): Returns: A list of DepsEntry objects. """ - result = [] - for path, depsentry in depsentry_dict.items(): - if path == dir_path: - result.append(depsentry) - else: - parts = path.split('/') - if all(part == parts[i] for i, part in enumerate(dir_path.split('/'))): - result.append(depsentry) - return result + result = [] + for path, depsentry in depsentry_dict.items(): + if path == dir_path: + result.append(depsentry) + else: + parts = path.split('/') + if all( + part == parts[i] for i, part in enumerate(dir_path.split('/')) + ): + result.append(depsentry) + return result def BuildDepsentryDict(deps_dict): - """Builds a dict of paths to DepsEntry objects from a raw deps dict.""" - result = {} + """Builds a dict of paths to DepsEntry objects from a raw deps dict.""" + result = {} - def AddDepsEntries(deps_subdict): - for path, dep in deps_subdict.items(): - if path in result: - continue - if not isinstance(dep, dict): - dep = {'url': dep} - if dep.get('dep_type') == 'cipd': - result[path] = CipdDepsEntry(path, dep['packages']) - else: - if '@' not in dep['url']: - continue - url, revision = dep['url'].split('@') - result[path] = DepsEntry(path, url, revision) + def AddDepsEntries(deps_subdict): + for path, dep in deps_subdict.items(): + if path in result: + continue + if not isinstance(dep, dict): + dep = {'url': dep} + if dep.get('dep_type') == 'cipd': + result[path] = CipdDepsEntry(path, dep['packages']) + else: + if '@' not in dep['url']: + continue + url, revision = dep['url'].split('@') + result[path] = DepsEntry(path, url, revision) - def AddVersionEntry(vars_subdict): - for key, value in vars_subdict.items(): - if key in result: - continue - if not key.endswith('_version'): - continue - key = re.sub('_version$', '', key) - result[key] = VersionEntry(value) + def AddVersionEntry(vars_subdict): + for key, value in vars_subdict.items(): + if key in result: + continue + if not key.endswith('_version'): + continue + key = re.sub('_version$', '', key) + result[key] = VersionEntry(value) - AddDepsEntries(deps_dict['deps']) - for deps_os in ['win', 'mac', 'unix', 'android', 'ios', 'unix']: - AddDepsEntries(deps_dict.get('deps_os', {}).get(deps_os, {})) - AddVersionEntry(deps_dict.get('vars', {})) - return result + AddDepsEntries(deps_dict['deps']) + for deps_os in ['win', 'mac', 'unix', 'android', 'ios', 'unix']: + AddDepsEntries(deps_dict.get('deps_os', {}).get(deps_os, {})) + AddVersionEntry(deps_dict.get('vars', {})) + return result def _FindChangedCipdPackages(path, old_pkgs, new_pkgs): - old_pkgs_names = {p['package'] for p in old_pkgs} - new_pkgs_names = {p['package'] for p in new_pkgs} - pkgs_equal = (old_pkgs_names == new_pkgs_names) - added_pkgs = [p for p in new_pkgs_names if p not in old_pkgs_names] - removed_pkgs = [p for p in old_pkgs_names if p not in new_pkgs_names] + old_pkgs_names = {p['package'] for p in old_pkgs} + new_pkgs_names = {p['package'] for p in new_pkgs} + pkgs_equal = old_pkgs_names == new_pkgs_names + added_pkgs = [p for p in new_pkgs_names if p not in old_pkgs_names] + removed_pkgs = [p for p in old_pkgs_names if p not in new_pkgs_names] - assert pkgs_equal, ('Old: %s\n New: %s.\nYou need to do a manual roll ' - 'and remove/add entries in DEPS so the old and new ' - 'list match.\nMost likely, you should add \"%s\" and ' - 'remove \"%s\"' % - (old_pkgs, new_pkgs, added_pkgs, removed_pkgs)) + assert pkgs_equal, ( + 'Old: %s\n New: %s.\nYou need to do a manual roll ' + 'and remove/add entries in DEPS so the old and new ' + 'list match.\nMost likely, you should add "%s" and ' + 'remove "%s"' % (old_pkgs, new_pkgs, added_pkgs, removed_pkgs) + ) - for old_pkg in old_pkgs: - for new_pkg in new_pkgs: - old_version = old_pkg['version'] - new_version = new_pkg['version'] - if (old_pkg['package'] == new_pkg['package'] - and old_version != new_version): - logging.debug('Roll dependency %s to %s', path, new_version) - yield ChangedCipdPackage(path, old_pkg['package'], old_version, - new_version) + for old_pkg in old_pkgs: + for new_pkg in new_pkgs: + old_version = old_pkg['version'] + new_version = new_pkg['version'] + if ( + old_pkg['package'] == new_pkg['package'] + and old_version != new_version + ): + logging.debug('Roll dependency %s to %s', path, new_version) + yield ChangedCipdPackage( + path, old_pkg['package'], old_version, new_version + ) def _FindChangedVars(name, old_version, new_version): - if old_version != new_version: - logging.debug('Roll dependency %s to %s', name, new_version) - yield ChangedVersionEntry(name, old_version, new_version) + if old_version != new_version: + logging.debug('Roll dependency %s to %s', name, new_version) + yield ChangedVersionEntry(name, old_version, new_version) def _FindNewDeps(old, new): - """ Gather dependencies only in `new` and return corresponding paths. """ - old_entries = set(BuildDepsentryDict(old)) - new_entries = set(BuildDepsentryDict(new)) - return [ - path for path in new_entries - old_entries - if path not in DONT_AUTOROLL_THESE - ] + """Gather dependencies only in `new` and return corresponding paths.""" + old_entries = set(BuildDepsentryDict(old)) + new_entries = set(BuildDepsentryDict(new)) + return [ + path + for path in new_entries - old_entries + if path not in DONT_AUTOROLL_THESE + ] def FindAddedDeps(libyuv_deps, new_cr_deps): - """ - Calculate new deps entries of interest. + """Calculate new deps entries of interest. Ideally, that would mean: only appearing in chromium DEPS but transitively used in LibYUV. @@ -368,19 +389,18 @@ def FindAddedDeps(libyuv_deps, new_cr_deps): A list of paths added dependencies sitting in `ANDROID_DEPS_PATH`. A list of paths for other added dependencies. """ - all_added_deps = _FindNewDeps(libyuv_deps, new_cr_deps) - generated_android_deps = [ - path for path in all_added_deps if path.startswith(ANDROID_DEPS_PATH) - ] - other_deps = [ - path for path in all_added_deps if path not in generated_android_deps - ] - return generated_android_deps, other_deps + all_added_deps = _FindNewDeps(libyuv_deps, new_cr_deps) + generated_android_deps = [ + path for path in all_added_deps if path.startswith(ANDROID_DEPS_PATH) + ] + other_deps = [ + path for path in all_added_deps if path not in generated_android_deps + ] + return generated_android_deps, other_deps def FindRemovedDeps(libyuv_deps, new_cr_deps): - """ - Calculate obsolete deps entries. + """Calculate obsolete deps entries. Ideally, that would mean: no more appearing in chromium DEPS and not used in LibYUV. @@ -402,20 +422,24 @@ def FindRemovedDeps(libyuv_deps, new_cr_deps): A list of paths of dependencies removed from `ANDROID_DEPS_PATH`. A list of paths of unexpected disappearing dependencies. """ - all_removed_deps = _FindNewDeps(new_cr_deps, libyuv_deps) - generated_android_deps = sorted( - [path for path in all_removed_deps if path.startswith(ANDROID_DEPS_PATH)]) - # Webrtc-only dependencies are handled in CalculateChangedDeps. - other_deps = sorted([ - path for path in all_removed_deps - if path not in generated_android_deps and path not in LIBYUV_ONLY_DEPS - ]) - return generated_android_deps, other_deps + all_removed_deps = _FindNewDeps(new_cr_deps, libyuv_deps) + generated_android_deps = sorted([ + path for path in all_removed_deps if path.startswith(ANDROID_DEPS_PATH) + ]) + # Webrtc-only dependencies are handled in CalculateChangedDeps. + other_deps = sorted([ + path + for path in all_removed_deps + if path not in generated_android_deps and path not in LIBYUV_ONLY_DEPS + ]) + return generated_android_deps, other_deps def CalculateChangedDeps(libyuv_deps, new_cr_deps): - """ - Calculate changed deps entries based on entries defined in the LibYUV DEPS + """Calculate changed deps entries based on entries defined in the LibYUV + + DEPS + file: - If a shared dependency with the Chromium DEPS file: roll it to the same revision as Chromium (i.e. entry in the new_cr_deps dict) @@ -428,254 +452,291 @@ def CalculateChangedDeps(libyuv_deps, new_cr_deps): Returns: A list of ChangedDep objects representing the changed deps. """ - result = [] - libyuv_entries = BuildDepsentryDict(libyuv_deps) - new_cr_entries = BuildDepsentryDict(new_cr_deps) - for path, libyuv_deps_entry in libyuv_entries.items(): - if path in DONT_AUTOROLL_THESE: - continue - cr_deps_entry = new_cr_entries.get(path) - if cr_deps_entry: - assert type(cr_deps_entry) is type(libyuv_deps_entry) + result = [] + libyuv_entries = BuildDepsentryDict(libyuv_deps) + new_cr_entries = BuildDepsentryDict(new_cr_deps) + for path, libyuv_deps_entry in libyuv_entries.items(): + if path in DONT_AUTOROLL_THESE: + continue + cr_deps_entry = new_cr_entries.get(path) + if cr_deps_entry: + assert type(cr_deps_entry) is type(libyuv_deps_entry) - if isinstance(cr_deps_entry, CipdDepsEntry): - result.extend( - _FindChangedCipdPackages(path, libyuv_deps_entry.packages, - cr_deps_entry.packages)) - continue + if isinstance(cr_deps_entry, CipdDepsEntry): + result.extend( + _FindChangedCipdPackages( + path, libyuv_deps_entry.packages, cr_deps_entry.packages # pylint: disable=line-too-long + ) + ) + continue - if isinstance(cr_deps_entry, VersionEntry): - result.extend( - _FindChangedVars(path, libyuv_deps_entry.version, - cr_deps_entry.version)) - continue + if isinstance(cr_deps_entry, VersionEntry): + result.extend( + _FindChangedVars( + path, libyuv_deps_entry.version, cr_deps_entry.version + ) + ) + continue - # Use the revision from Chromium's DEPS file. - new_rev = cr_deps_entry.revision - assert libyuv_deps_entry.url == cr_deps_entry.url, ( - 'LibYUV DEPS entry %s has a different URL %s than Chromium %s.' % - (path, libyuv_deps_entry.url, cr_deps_entry.url)) - else: - if isinstance(libyuv_deps_entry, DepsEntry): - # Use the HEAD of the deps repo. - stdout, _ = _RunCommand( - ['git', 'ls-remote', libyuv_deps_entry.url, 'HEAD']) - new_rev = stdout.strip().split('\t')[0] - else: - # The dependency has been removed from chromium. - # This is handled by FindRemovedDeps. - continue + # Use the revision from Chromium's DEPS file. + new_rev = cr_deps_entry.revision + assert libyuv_deps_entry.url == cr_deps_entry.url, ( + 'LibYUV DEPS entry %s has a different URL %s than Chromium %s.' + % ( + path, + libyuv_deps_entry.url, + cr_deps_entry.url, + ) + ) + else: + if isinstance(libyuv_deps_entry, DepsEntry): + # Use the HEAD of the deps repo. + stdout, _ = _RunCommand( + ['git', 'ls-remote', libyuv_deps_entry.url, 'HEAD'] + ) + new_rev = stdout.strip().split('\t')[0] + else: + # The dependency has been removed from chromium. + # This is handled by FindRemovedDeps. + continue - # Check if an update is necessary. - if libyuv_deps_entry.revision != new_rev: - logging.debug('Roll dependency %s to %s', path, new_rev) - result.append( - ChangedDep(path, libyuv_deps_entry.url, libyuv_deps_entry.revision, - new_rev)) - return sorted(result) + # Check if an update is necessary. + if libyuv_deps_entry.revision != new_rev: + logging.debug('Roll dependency %s to %s', path, new_rev) + result.append( + ChangedDep( + path, + libyuv_deps_entry.url, + libyuv_deps_entry.revision, + new_rev, + ) + ) + return sorted(result) def CalculateChangedClang(new_cr_rev): - def GetClangRev(lines): - for line in lines: - match = CLANG_REVISION_RE.match(line) - if match: - return match.group(1) - raise RollError('Could not parse Clang revision!') + def GetClangRev(lines): + for line in lines: + match = CLANG_REVISION_RE.match(line) + if match: + return match.group(1) + raise RollError('Could not parse Clang revision!') - with open(CLANG_UPDATE_SCRIPT_LOCAL_PATH, 'r') as f: - current_lines = f.readlines() - current_rev = GetClangRev(current_lines) + with open(CLANG_UPDATE_SCRIPT_LOCAL_PATH, 'r') as f: + current_lines = f.readlines() + current_rev = GetClangRev(current_lines) - new_clang_update_py = ReadRemoteCrFile(CLANG_UPDATE_SCRIPT_URL_PATH, - new_cr_rev).splitlines() - new_rev = GetClangRev(new_clang_update_py) - return ChangedDep(CLANG_UPDATE_SCRIPT_LOCAL_PATH, None, current_rev, new_rev) + new_clang_update_py = ReadRemoteCrFile( + CLANG_UPDATE_SCRIPT_URL_PATH, new_cr_rev + ).splitlines() + new_rev = GetClangRev(new_clang_update_py) + return ChangedDep( + CLANG_UPDATE_SCRIPT_LOCAL_PATH, None, current_rev, new_rev + ) def GenerateCommitMessage( - rev_update, - current_commit_pos, - new_commit_pos, - changed_deps_list, - added_deps_paths=None, - removed_deps_paths=None, - clang_change=None, + rev_update, + current_commit_pos, + new_commit_pos, + changed_deps_list, + added_deps_paths=None, + removed_deps_paths=None, + clang_change=None, ): - current_cr_rev = rev_update.current_chromium_rev[0:10] - new_cr_rev = rev_update.new_chromium_rev[0:10] - rev_interval = '%s..%s' % (current_cr_rev, new_cr_rev) - git_number_interval = '%s:%s' % (current_commit_pos, new_commit_pos) + current_cr_rev = rev_update.current_chromium_rev[0:10] + new_cr_rev = rev_update.new_chromium_rev[0:10] + rev_interval = '%s..%s' % (current_cr_rev, new_cr_rev) + git_number_interval = '%s:%s' % (current_commit_pos, new_commit_pos) - commit_msg = [ - 'Roll chromium_revision %s (%s)\n' % (rev_interval, git_number_interval), - 'Change log: %s' % (CHROMIUM_LOG_TEMPLATE % rev_interval), - 'Full diff: %s\n' % (CHROMIUM_COMMIT_TEMPLATE % rev_interval) - ] + commit_msg = [ + 'Roll chromium_revision %s (%s)\n' + % (rev_interval, git_number_interval), + 'Change log: %s' % (CHROMIUM_LOG_TEMPLATE % rev_interval), + 'Full diff: %s\n' % (CHROMIUM_COMMIT_TEMPLATE % rev_interval), + ] - def Section(adjective, deps): - noun = 'dependency' if len(deps) == 1 else 'dependencies' - commit_msg.append('%s %s' % (adjective, noun)) + def Section(adjective, deps): + noun = 'dependency' if len(deps) == 1 else 'dependencies' + commit_msg.append('%s %s' % (adjective, noun)) - if changed_deps_list: - Section('Changed', changed_deps_list) + if changed_deps_list: + Section('Changed', changed_deps_list) - for c in changed_deps_list: - if isinstance(c, ChangedCipdPackage): - commit_msg.append('* %s: %s..%s' % - (c.path, c.current_version, c.new_version)) - elif isinstance(c, ChangedVersionEntry): - commit_msg.append('* %s_vesion: %s..%s' % - (c.path, c.current_version, c.new_version)) - else: - commit_msg.append('* %s: %s/+log/%s..%s' % - (c.path, c.url, c.current_rev[0:10], c.new_rev[0:10])) + for c in changed_deps_list: + if isinstance(c, ChangedCipdPackage): + commit_msg.append( + '* %s: %s..%s' % (c.path, c.current_version, c.new_version) + ) + elif isinstance(c, ChangedVersionEntry): + commit_msg.append( + '* %s_vesion: %s..%s' + % (c.path, c.current_version, c.new_version) + ) + else: + commit_msg.append( + '* %s: %s/+log/%s..%s' + % (c.path, c.url, c.current_rev[0:10], c.new_rev[0:10]) + ) - if added_deps_paths: - Section('Added', added_deps_paths) - commit_msg.extend('* %s' % p for p in added_deps_paths) + if added_deps_paths: + Section('Added', added_deps_paths) + commit_msg.extend('* %s' % p for p in added_deps_paths) - if removed_deps_paths: - Section('Removed', removed_deps_paths) - commit_msg.extend('* %s' % p for p in removed_deps_paths) + if removed_deps_paths: + Section('Removed', removed_deps_paths) + commit_msg.extend('* %s' % p for p in removed_deps_paths) - if any([changed_deps_list, added_deps_paths, removed_deps_paths]): - change_url = CHROMIUM_FILE_TEMPLATE % (rev_interval, 'DEPS') - commit_msg.append('DEPS diff: %s\n' % change_url) - else: - commit_msg.append('No dependencies changed.') + if any([changed_deps_list, added_deps_paths, removed_deps_paths]): + change_url = CHROMIUM_FILE_TEMPLATE % (rev_interval, 'DEPS') + commit_msg.append('DEPS diff: %s\n' % change_url) + else: + commit_msg.append('No dependencies changed.') - if clang_change and clang_change.current_rev != clang_change.new_rev: - commit_msg.append('Clang version changed %s:%s' % - (clang_change.current_rev, clang_change.new_rev)) - change_url = CHROMIUM_FILE_TEMPLATE % (rev_interval, - CLANG_UPDATE_SCRIPT_URL_PATH) - commit_msg.append('Details: %s\n' % change_url) - else: - commit_msg.append('No update to Clang.\n') + if clang_change and clang_change.current_rev != clang_change.new_rev: + commit_msg.append( + 'Clang version changed %s:%s' + % (clang_change.current_rev, clang_change.new_rev) + ) + change_url = CHROMIUM_FILE_TEMPLATE % ( + rev_interval, + CLANG_UPDATE_SCRIPT_URL_PATH, + ) + commit_msg.append('Details: %s\n' % change_url) + else: + commit_msg.append('No update to Clang.\n') - commit_msg.append('BUG=None') - return '\n'.join(commit_msg) + commit_msg.append('BUG=None') + return '\n'.join(commit_msg) def UpdateDepsFile(deps_filename, rev_update, changed_deps, new_cr_content): - """Update the DEPS file with the new revision.""" + """Update the DEPS file with the new revision.""" - with open(deps_filename, 'rb') as deps_file: - deps_content = deps_file.read().decode('utf-8') + with open(deps_filename, 'rb') as deps_file: + deps_content = deps_file.read().decode('utf-8') - # Update the chromium_revision variable. - deps_content = deps_content.replace(rev_update.current_chromium_rev, - rev_update.new_chromium_rev) + # Update the chromium_revision variable. + deps_content = deps_content.replace( + rev_update.current_chromium_rev, rev_update.new_chromium_rev + ) - # Add and remove dependencies. For now: only generated android deps. - # Since gclient cannot add or remove deps, we on the fact that - # these android deps are located in one place we can copy/paste. - deps_re = re.compile(ANDROID_DEPS_START + '.*' + ANDROID_DEPS_END, re.DOTALL) - new_deps = deps_re.search(new_cr_content) - old_deps = deps_re.search(deps_content) - if not new_deps or not old_deps: - faulty = 'Chromium' if not new_deps else 'LibYUV' - raise RollError('Was expecting to find "%s" and "%s"\n' - 'in %s DEPS' % - (ANDROID_DEPS_START, ANDROID_DEPS_END, faulty)) - deps_content = deps_re.sub(new_deps.group(0), deps_content) + # Add and remove dependencies. For now: only generated android deps. + # Since gclient cannot add or remove deps, we on the fact that + # these android deps are located in one place we can copy/paste. + deps_re = re.compile( + ANDROID_DEPS_START + '.*' + ANDROID_DEPS_END, re.DOTALL + ) + new_deps = deps_re.search(new_cr_content) + old_deps = deps_re.search(deps_content) + if not new_deps or not old_deps: + faulty = 'Chromium' if not new_deps else 'LibYUV' + raise RollError( + 'Was expecting to find "%s" and "%s"\nin %s DEPS' + % (ANDROID_DEPS_START, ANDROID_DEPS_END, faulty) + ) + deps_content = deps_re.sub(new_deps.group(0), deps_content) - for dep in changed_deps: - if isinstance(dep, ChangedVersionEntry): - deps_content = deps_content.replace(dep.current_version, dep.new_version) + for dep in changed_deps: + if isinstance(dep, ChangedVersionEntry): + deps_content = deps_content.replace( + dep.current_version, dep.new_version + ) - with open(deps_filename, 'wb') as deps_file: - deps_file.write(deps_content.encode('utf-8')) + with open(deps_filename, 'wb') as deps_file: + deps_file.write(deps_content.encode('utf-8')) - # Update each individual DEPS entry. - for dep in changed_deps: - # ChangedVersionEntry types are already been processed. - if isinstance(dep, ChangedVersionEntry): - continue - local_dep_dir = os.path.join(CHECKOUT_ROOT_DIR, dep.path) - if not os.path.isdir(local_dep_dir): - raise RollError( - 'Cannot find local directory %s. Either run\n' - 'gclient sync --deps=all\n' - 'or make sure the .gclient file for your solution contains all ' - 'platforms in the target_os list, i.e.\n' - 'target_os = ["android", "unix", "mac", "ios", "win"];\n' - 'Then run "gclient sync" again.' % local_dep_dir) - if isinstance(dep, ChangedCipdPackage): - package = dep.package.format() # Eliminate double curly brackets - update = '%s:%s@%s' % (dep.path, package, dep.new_version) - else: - update = '%s@%s' % (dep.path, dep.new_rev) - _RunCommand(['gclient', 'setdep', '--revision', update], - working_dir=CHECKOUT_SRC_DIR) + # Update each individual DEPS entry. + for dep in changed_deps: + # ChangedVersionEntry types are already been processed. + if isinstance(dep, ChangedVersionEntry): + continue + local_dep_dir = os.path.join(CHECKOUT_ROOT_DIR, dep.path) + if not os.path.isdir(local_dep_dir): + raise RollError( + 'Cannot find local directory %s. Either run\n' + 'gclient sync --deps=all\n' + 'or make sure the .gclient file for your solution contains ' + 'all platforms in the target_os list, i.e.\n' + 'target_os = ["android", "unix", "mac", "ios", "win"];\n' + 'Then run "gclient sync" again.' % local_dep_dir + ) + if isinstance(dep, ChangedCipdPackage): + package = dep.package.format() # Eliminate double curly brackets + update = '%s:%s@%s' % (dep.path, package, dep.new_version) + else: + update = '%s@%s' % (dep.path, dep.new_rev) + _RunCommand( + ['gclient', 'setdep', '--revision', update], + working_dir=CHECKOUT_SRC_DIR, + ) def _IsTreeClean(): - stdout, _ = _RunCommand(['git', 'status', '--porcelain']) - if len(stdout) == 0: - return True + stdout, _ = _RunCommand(['git', 'status', '--porcelain']) + if len(stdout) == 0: + return True - logging.error('Dirty/unversioned files:\n%s', stdout) - return False + logging.error('Dirty/unversioned files:\n%s', stdout) + return False def _EnsureUpdatedMainBranch(dry_run): - current_branch = _RunCommand(['git', 'rev-parse', '--abbrev-ref', - 'HEAD'])[0].splitlines()[0] - if current_branch != 'main': - logging.error('Please checkout the main branch and re-run this script.') - if not dry_run: - sys.exit(-1) + current_branch = _RunCommand(['git', 'rev-parse', '--abbrev-ref', 'HEAD'])[ + 0 + ].splitlines()[0] + if current_branch != 'main': + logging.error('Please checkout the main branch and re-run this script.') # pylint: disable=line-too-long + if not dry_run: + sys.exit(-1) - logging.info('Updating main branch...') - _RunCommand(['git', 'pull']) + logging.info('Updating main branch...') + _RunCommand(['git', 'pull']) def _CreateRollBranch(dry_run): - logging.info('Creating roll branch: %s', ROLL_BRANCH_NAME) - if not dry_run: - _RunCommand(['git', 'checkout', '-b', ROLL_BRANCH_NAME]) + logging.info('Creating roll branch: %s', ROLL_BRANCH_NAME) + if not dry_run: + _RunCommand(['git', 'checkout', '-b', ROLL_BRANCH_NAME]) def _RemovePreviousRollBranch(dry_run): - active_branch, branches = _GetBranches() - if active_branch == ROLL_BRANCH_NAME: - active_branch = 'main' - if ROLL_BRANCH_NAME in branches: - logging.info('Removing previous roll branch (%s)', ROLL_BRANCH_NAME) - if not dry_run: - _RunCommand(['git', 'checkout', active_branch]) - _RunCommand(['git', 'branch', '-D', ROLL_BRANCH_NAME]) + active_branch, branches = _GetBranches() + if active_branch == ROLL_BRANCH_NAME: + active_branch = 'main' + if ROLL_BRANCH_NAME in branches: + logging.info('Removing previous roll branch (%s)', ROLL_BRANCH_NAME) + if not dry_run: + _RunCommand(['git', 'checkout', active_branch]) + _RunCommand(['git', 'branch', '-D', ROLL_BRANCH_NAME]) def _LocalCommit(commit_msg, dry_run): - logging.info('Committing changes locally.') - if not dry_run: - _RunCommand(['git', 'add', '--update', '.']) - _RunCommand(['git', 'commit', '-m', commit_msg]) + logging.info('Committing changes locally.') + if not dry_run: + _RunCommand(['git', 'add', '--update', '.']) + _RunCommand(['git', 'commit', '-m', commit_msg]) def ChooseCQMode(skip_cq, cq_over, current_commit_pos, new_commit_pos): - if skip_cq: - return 0 - if (new_commit_pos - current_commit_pos) < cq_over: - return 1 - return 2 + if skip_cq: + return 0 + if (new_commit_pos - current_commit_pos) < cq_over: + return 1 + return 2 def _GetCcRecipients(changed_deps_list): - """Returns a list of emails to notify based on the changed deps list. - """ - cc_recipients = [] - for c in changed_deps_list: - pass - return cc_recipients + """Returns a list of emails to notify based on the changed deps list.""" + cc_recipients = [] + for _ in changed_deps_list: + pass + return cc_recipients def _UploadCL(commit_queue_mode, add_cc=None): - """Upload the committed changes as a changelist to Gerrit. + """Upload the committed changes as a changelist to Gerrit. commit_queue_mode: - 2: Submit to commit queue. @@ -684,139 +745,171 @@ def _UploadCL(commit_queue_mode, add_cc=None): add_cc: A list of email addresses to add as CC recipients. """ - cc_recipients = [] - if add_cc: - cc_recipients.extend(add_cc) - cmd = ['git', 'cl', 'upload', '--force', '--bypass-hooks'] - if commit_queue_mode >= 2: - logging.info('Sending the CL to the CQ...') - cmd.extend(['-o', 'label=Bot-Commit+1']) - cmd.extend(['-o', 'label=Commit-Queue+2']) - cmd.extend(['--send-mail', '--cc', ','.join(cc_recipients)]) - elif commit_queue_mode >= 1: - logging.info('Starting CQ dry run...') - cmd.extend(['-o', 'label=Commit-Queue+1']) - extra_env = { - 'EDITOR': 'true', - 'SKIP_GCE_AUTH_FOR_GIT': '1', - } - stdout, stderr = _RunCommand(cmd, extra_env=extra_env) - logging.debug('Output from "git cl upload":\nstdout:\n%s\n\nstderr:\n%s', - stdout, stderr) + cc_recipients = [] + if add_cc: + cc_recipients.extend(add_cc) + cmd = ['git', 'cl', 'upload', '--force', '--bypass-hooks'] + if commit_queue_mode >= 2: + logging.info('Sending the CL to the CQ...') + cmd.extend(['-o', 'label=Bot-Commit+1']) + cmd.extend(['-o', 'label=Commit-Queue+2']) + cmd.extend(['--send-mail', '--cc', ','.join(cc_recipients)]) + elif commit_queue_mode >= 1: + logging.info('Starting CQ dry run...') + cmd.extend(['-o', 'label=Commit-Queue+1']) + extra_env = { + 'EDITOR': 'true', + 'SKIP_GCE_AUTH_FOR_GIT': '1', + } + stdout, stderr = _RunCommand(cmd, extra_env=extra_env) + logging.debug( + 'Output from "git cl upload":\nstdout:\n%s\n\nstderr:\n%s', + stdout, + stderr, + ) def GetRollRevisionRanges(opts, libyuv_deps): - current_cr_rev = libyuv_deps['vars']['chromium_revision'] - new_cr_rev = opts.revision - if not new_cr_rev: - stdout, _ = _RunCommand(['git', 'ls-remote', CHROMIUM_SRC_URL, 'HEAD']) - head_rev = stdout.strip().split('\t')[0] - logging.info('No revision specified. Using HEAD: %s', head_rev) - new_cr_rev = head_rev + current_cr_rev = libyuv_deps['vars']['chromium_revision'] + new_cr_rev = opts.revision + if not new_cr_rev: + stdout, _ = _RunCommand(['git', 'ls-remote', CHROMIUM_SRC_URL, 'HEAD']) + head_rev = stdout.strip().split('\t')[0] + logging.info('No revision specified. Using HEAD: %s', head_rev) + new_cr_rev = head_rev - return ChromiumRevisionUpdate(current_cr_rev, new_cr_rev) + return ChromiumRevisionUpdate(current_cr_rev, new_cr_rev) def main(): - p = argparse.ArgumentParser() - p.add_argument('--clean', - action='store_true', - default=False, - help='Removes any previous local roll branch.') - p.add_argument('-r', - '--revision', - help=('Chromium Git revision to roll to. Defaults to the ' - 'Chromium HEAD revision if omitted.')) - p.add_argument('--dry-run', - action='store_true', - default=False, - help=('Calculate changes and modify DEPS, but don\'t create ' - 'any local branch, commit, upload CL or send any ' - 'tryjobs.')) - p.add_argument('-i', - '--ignore-unclean-workdir', - action='store_true', - default=False, - help=('Ignore if the current branch is not main or if there ' - 'are uncommitted changes (default: %(default)s).')) - grp = p.add_mutually_exclusive_group() - grp.add_argument('--skip-cq', - action='store_true', - default=False, - help='Skip sending the CL to the CQ (default: %(default)s)') - grp.add_argument('--cq-over', - type=int, - default=1, - help=('Commit queue dry run if the revision difference ' - 'is below this number (default: %(default)s)')) - p.add_argument('-v', - '--verbose', - action='store_true', - default=False, - help='Be extra verbose in printing of log messages.') - opts = p.parse_args() + p = argparse.ArgumentParser() + p.add_argument( + '--clean', + action='store_true', + default=False, + help='Removes any previous local roll branch.', + ) + p.add_argument( + '-r', + '--revision', + help=( + 'Chromium Git revision to roll to. Defaults to the ' + 'Chromium HEAD revision if omitted.' + ), + ) + p.add_argument( + '--dry-run', + action='store_true', + default=False, + help=( + "Calculate changes and modify DEPS, but don't create " + 'any local branch, commit, upload CL or send any ' + 'tryjobs.' + ), + ) + p.add_argument( + '-i', + '--ignore-unclean-workdir', + action='store_true', + default=False, + help=( + 'Ignore if the current branch is not main or if there ' + 'are uncommitted changes (default: %(default)s).' + ), + ) + grp = p.add_mutually_exclusive_group() + grp.add_argument( + '--skip-cq', + action='store_true', + default=False, + help='Skip sending the CL to the CQ (default: %(default)s)', + ) + grp.add_argument( + '--cq-over', + type=int, + default=1, + help=( + 'Commit queue dry run if the revision difference ' + 'is below this number (default: %(default)s)' + ), + ) + p.add_argument( + '-v', + '--verbose', + action='store_true', + default=False, + help='Be extra verbose in printing of log messages.', + ) + opts = p.parse_args() - if opts.verbose: - logging.basicConfig(level=logging.DEBUG) - else: - logging.basicConfig(level=logging.INFO) + if opts.verbose: + logging.basicConfig(level=logging.DEBUG) + else: + logging.basicConfig(level=logging.INFO) - if not opts.ignore_unclean_workdir and not _IsTreeClean(): - logging.error('Please clean your local checkout first.') - return 1 + if not opts.ignore_unclean_workdir and not _IsTreeClean(): + logging.error('Please clean your local checkout first.') + return 1 - if opts.clean: - _RemovePreviousRollBranch(opts.dry_run) + if opts.clean: + _RemovePreviousRollBranch(opts.dry_run) - if not opts.ignore_unclean_workdir: - _EnsureUpdatedMainBranch(opts.dry_run) + if not opts.ignore_unclean_workdir: + _EnsureUpdatedMainBranch(opts.dry_run) - deps_filename = os.path.join(CHECKOUT_SRC_DIR, 'DEPS') - libyuv_deps = ParseLocalDepsFile(deps_filename) + deps_filename = os.path.join(CHECKOUT_SRC_DIR, 'DEPS') + libyuv_deps = ParseLocalDepsFile(deps_filename) - rev_update = GetRollRevisionRanges(opts, libyuv_deps) + rev_update = GetRollRevisionRanges(opts, libyuv_deps) - current_commit_pos = ParseCommitPosition( - ReadRemoteCrCommit(rev_update.current_chromium_rev)) - new_commit_pos = ParseCommitPosition( - ReadRemoteCrCommit(rev_update.new_chromium_rev)) + current_commit_pos = ParseCommitPosition( + ReadRemoteCrCommit(rev_update.current_chromium_rev) + ) + new_commit_pos = ParseCommitPosition( + ReadRemoteCrCommit(rev_update.new_chromium_rev) + ) - new_cr_content = ReadRemoteCrFile('DEPS', rev_update.new_chromium_rev) - new_cr_deps = ParseDepsDict(new_cr_content) - changed_deps = CalculateChangedDeps(libyuv_deps, new_cr_deps) - # Discard other deps, assumed to be chromium-only dependencies. - new_generated_android_deps, _ = FindAddedDeps(libyuv_deps, new_cr_deps) - removed_generated_android_deps, other_deps = FindRemovedDeps( - libyuv_deps, new_cr_deps) - if other_deps: - raise RollError('LibYUV DEPS entries are missing from Chromium: %s.\n' - 'Remove them or add them to either ' - 'LIBYUV_ONLY_DEPS or DONT_AUTOROLL_THESE.' % other_deps) - clang_change = CalculateChangedClang(rev_update.new_chromium_rev) - commit_msg = GenerateCommitMessage( - rev_update, - current_commit_pos, - new_commit_pos, - changed_deps, - added_deps_paths=new_generated_android_deps, - removed_deps_paths=removed_generated_android_deps, - clang_change=clang_change) - logging.debug('Commit message:\n%s', commit_msg) + new_cr_content = ReadRemoteCrFile('DEPS', rev_update.new_chromium_rev) + new_cr_deps = ParseDepsDict(new_cr_content) + changed_deps = CalculateChangedDeps(libyuv_deps, new_cr_deps) + # Discard other deps, assumed to be chromium-only dependencies. + new_generated_android_deps, _ = FindAddedDeps(libyuv_deps, new_cr_deps) + removed_generated_android_deps, other_deps = FindRemovedDeps( + libyuv_deps, new_cr_deps + ) + if other_deps: + raise RollError( + 'LibYUV DEPS entries are missing from Chromium: %s.\n' + 'Remove them or add them to either ' + 'LIBYUV_ONLY_DEPS or DONT_AUTOROLL_THESE.' % other_deps + ) + clang_change = CalculateChangedClang(rev_update.new_chromium_rev) + commit_msg = GenerateCommitMessage( + rev_update, + current_commit_pos, + new_commit_pos, + changed_deps, + added_deps_paths=new_generated_android_deps, + removed_deps_paths=removed_generated_android_deps, + clang_change=clang_change, + ) + logging.debug('Commit message:\n%s', commit_msg) - _CreateRollBranch(opts.dry_run) - if not opts.dry_run: - UpdateDepsFile(deps_filename, rev_update, changed_deps, new_cr_content) - if _IsTreeClean(): - logging.info("No DEPS changes detected, skipping CL creation.") - else: - _LocalCommit(commit_msg, opts.dry_run) - commit_queue_mode = ChooseCQMode(opts.skip_cq, opts.cq_over, - current_commit_pos, new_commit_pos) - logging.info('Uploading CL...') + _CreateRollBranch(opts.dry_run) if not opts.dry_run: - _UploadCL(commit_queue_mode, _GetCcRecipients(changed_deps)) - return 0 + UpdateDepsFile(deps_filename, rev_update, changed_deps, new_cr_content) + if _IsTreeClean(): + logging.info('No DEPS changes detected, skipping CL creation.') + else: + _LocalCommit(commit_msg, opts.dry_run) + commit_queue_mode = ChooseCQMode( + opts.skip_cq, opts.cq_over, current_commit_pos, new_commit_pos + ) + logging.info('Uploading CL...') + if not opts.dry_run: + _UploadCL(commit_queue_mode, _GetCcRecipients(changed_deps)) + return 0 if __name__ == '__main__': - sys.exit(main()) + sys.exit(main()) diff --git a/tools_libyuv/autoroller/unittests/roll_deps_test.py b/tools_libyuv/autoroller/unittests/roll_deps_test.py index af86bdd58..bd38c9409 100755 --- a/tools_libyuv/autoroller/unittests/roll_deps_test.py +++ b/tools_libyuv/autoroller/unittests/roll_deps_test.py @@ -16,23 +16,22 @@ import tempfile import unittest import roll_deps -from roll_deps import CalculateChangedDeps, GetMatchingDepsEntries, \ - ParseDepsDict, ParseLocalDepsFile, UpdateDepsFile +from roll_deps import CalculateChangedDeps, GetMatchingDepsEntries, ParseDepsDict, ParseLocalDepsFile, UpdateDepsFile # pylint: disable=line-too-long SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) PARENT_DIR = os.path.join(SCRIPT_DIR, os.pardir) sys.path.append(PARENT_DIR) TEST_DATA_VARS = { - 'chromium_git': 'https://chromium.googlesource.com', - 'chromium_revision': '1b9c098a08e40114e44b6c1ec33ddf95c40b901d', + 'chromium_git': 'https://chromium.googlesource.com', + 'chromium_revision': '1b9c098a08e40114e44b6c1ec33ddf95c40b901d', } DEPS_ENTRIES = { - 'src/build': 'https://build.com', - 'src/buildtools': 'https://buildtools.com', - 'src/testing/gtest': 'https://gtest.com', - 'src/testing/gmock': 'https://gmock.com', + 'src/build': 'https://build.com', + 'src/buildtools': 'https://buildtools.com', + 'src/testing/gtest': 'https://gtest.com', + 'src/testing/gmock': 'https://gmock.com', } BUILD_OLD_REV = '52f7afeca991d96d68cf0507e20dbdd5b845691f' @@ -42,107 +41,124 @@ BUILDTOOLS_NEW_REV = '55ad626b08ef971fd82a62b7abb325359542952b' class TestError(Exception): - pass + pass -class FakeCmd(): - def __init__(self): - self.expectations = [] +class FakeCmd: - def add_expectation(self, *args, **kwargs): - returns = kwargs.pop('_returns', None) - self.expectations.append((args, kwargs, returns)) + def __init__(self): + self.expectations = [] - def __call__(self, *args, **kwargs): - if not self.expectations: - raise TestError('Got unexpected\n%s\n%s' % (args, kwargs)) - exp_args, exp_kwargs, exp_returns = self.expectations.pop(0) - if args != exp_args or kwargs != exp_kwargs: - message = 'Expected:\n args: %s\n kwargs: %s\n' % (exp_args, exp_kwargs) - message += 'Got:\n args: %s\n kwargs: %s\n' % (args, kwargs) - raise TestError(message) - return exp_returns + def add_expectation(self, *args, **kwargs): + returns = kwargs.pop('_returns', None) + self.expectations.append((args, kwargs, returns)) + + def __call__(self, *args, **kwargs): + if not self.expectations: + raise TestError('Got unexpected\n%s\n%s' % (args, kwargs)) + exp_args, exp_kwargs, exp_returns = self.expectations.pop(0) + if args != exp_args or kwargs != exp_kwargs: + message = 'Expected:\n args: %s\n kwargs: %s\n' % ( + exp_args, + exp_kwargs, + ) + message += 'Got:\n args: %s\n kwargs: %s\n' % (args, kwargs) + raise TestError(message) + return exp_returns class TestRollChromiumRevision(unittest.TestCase): - def setUp(self): - self._output_dir = tempfile.mkdtemp() - for test_file in glob.glob(os.path.join(SCRIPT_DIR, 'testdata', '*')): - shutil.copy(test_file, self._output_dir) - self._libyuv_depsfile = os.path.join(self._output_dir, 'DEPS') - self._old_cr_depsfile = os.path.join(self._output_dir, 'DEPS.chromium.old') - self._new_cr_depsfile = os.path.join(self._output_dir, 'DEPS.chromium.new') - self.fake = FakeCmd() - self.old_RunCommand = getattr(roll_deps, '_RunCommand') - setattr(roll_deps, '_RunCommand', self.fake) + def setUp(self): + self._output_dir = tempfile.mkdtemp() + for test_file in glob.glob(os.path.join(SCRIPT_DIR, 'testdata', '*')): + shutil.copy(test_file, self._output_dir) + self._libyuv_depsfile = os.path.join(self._output_dir, 'DEPS') + self._old_cr_depsfile = os.path.join( + self._output_dir, 'DEPS.chromium.old' + ) + self._new_cr_depsfile = os.path.join( + self._output_dir, 'DEPS.chromium.new' + ) - def tearDown(self): - shutil.rmtree(self._output_dir, ignore_errors=True) - self.assertEqual(self.fake.expectations, []) - setattr(roll_deps, '_RunCommand', self.old_RunCommand) + self.fake = FakeCmd() + self.old_RunCommand = getattr(roll_deps, '_RunCommand') + setattr(roll_deps, '_RunCommand', self.fake) - def testVarLookup(self): - local_scope = {'foo': 'wrong', 'vars': {'foo': 'bar'}} - lookup = roll_deps.VarLookup(local_scope) - self.assertEqual(lookup('foo'), 'bar') + def tearDown(self): + shutil.rmtree(self._output_dir, ignore_errors=True) + self.assertEqual(self.fake.expectations, []) + setattr(roll_deps, '_RunCommand', self.old_RunCommand) - def testUpdateDepsFile(self): - new_rev = 'aaaaabbbbbcccccdddddeeeeefffff0000011111' + def testVarLookup(self): + local_scope = {'foo': 'wrong', 'vars': {'foo': 'bar'}} + lookup = roll_deps.VarLookup(local_scope) + self.assertEqual(lookup('foo'), 'bar') - current_rev = TEST_DATA_VARS['chromium_revision'] - UpdateDepsFile(self._libyuv_depsfile, current_rev, new_rev, []) - with open(self._libyuv_depsfile, 'r') as deps_file: - deps_contents = deps_file.read() - self.assertTrue(new_rev in deps_contents, - 'Failed to find %s in\n%s' % (new_rev, deps_contents)) + def testUpdateDepsFile(self): + new_rev = 'aaaaabbbbbcccccdddddeeeeefffff0000011111' - def testParseDepsDict(self): - with open(self._libyuv_depsfile, 'r') as deps_file: - deps_contents = deps_file.read() - local_scope = ParseDepsDict(deps_contents) - vars_dict = local_scope['vars'] + current_rev = TEST_DATA_VARS['chromium_revision'] + UpdateDepsFile(self._libyuv_depsfile, current_rev, new_rev, []) + with open(self._libyuv_depsfile, 'r') as deps_file: + deps_contents = deps_file.read() + self.assertTrue( + new_rev in deps_contents, + 'Failed to find %s in\n%s' % (new_rev, deps_contents), + ) - def assertVar(variable_name): - self.assertEqual(vars_dict[variable_name], TEST_DATA_VARS[variable_name]) - assertVar('chromium_git') - assertVar('chromium_revision') - self.assertEqual(len(local_scope['deps']), 3) + def testParseDepsDict(self): + with open(self._libyuv_depsfile, 'r') as deps_file: + deps_contents = deps_file.read() + local_scope = ParseDepsDict(deps_contents) + vars_dict = local_scope['vars'] - def testGetMatchingDepsEntriesReturnsPathInSimpleCase(self): - entries = GetMatchingDepsEntries(DEPS_ENTRIES, 'src/testing/gtest') - self.assertEqual(len(entries), 1) - self.assertEqual(entries[0], DEPS_ENTRIES['src/testing/gtest']) + def assertVar(variable_name): + self.assertEqual( + vars_dict[variable_name], TEST_DATA_VARS[variable_name] + ) - def testGetMatchingDepsEntriesHandlesSimilarStartingPaths(self): - entries = GetMatchingDepsEntries(DEPS_ENTRIES, 'src/testing') - self.assertEqual(len(entries), 2) + assertVar('chromium_git') + assertVar('chromium_revision') + self.assertEqual(len(local_scope['deps']), 3) - def testGetMatchingDepsEntriesHandlesTwoPathsWithIdenticalFirstParts(self): - entries = GetMatchingDepsEntries(DEPS_ENTRIES, 'src/build') - self.assertEqual(len(entries), 1) - self.assertEqual(entries[0], DEPS_ENTRIES['src/build']) + def testGetMatchingDepsEntriesReturnsPathInSimpleCase(self): + entries = GetMatchingDepsEntries(DEPS_ENTRIES, 'src/testing/gtest') + self.assertEqual(len(entries), 1) + self.assertEqual(entries[0], DEPS_ENTRIES['src/testing/gtest']) - def testCalculateChangedDeps(self): - _SetupGitLsRemoteCall(self.fake, - 'https://chromium.googlesource.com/chromium/src/build', BUILD_NEW_REV) - libyuv_deps = ParseLocalDepsFile(self._libyuv_depsfile) - new_cr_deps = ParseLocalDepsFile(self._new_cr_depsfile) - changed_deps = CalculateChangedDeps(libyuv_deps, new_cr_deps) - self.assertEqual(len(changed_deps), 2) - self.assertEqual(changed_deps[0].path, 'src/build') - self.assertEqual(changed_deps[0].current_rev, BUILD_OLD_REV) - self.assertEqual(changed_deps[0].new_rev, BUILD_NEW_REV) + def testGetMatchingDepsEntriesHandlesSimilarStartingPaths(self): + entries = GetMatchingDepsEntries(DEPS_ENTRIES, 'src/testing') + self.assertEqual(len(entries), 2) - self.assertEqual(changed_deps[1].path, 'src/buildtools') - self.assertEqual(changed_deps[1].current_rev, BUILDTOOLS_OLD_REV) - self.assertEqual(changed_deps[1].new_rev, BUILDTOOLS_NEW_REV) + def testGetMatchingDepsEntriesHandlesTwoPathsWithIdenticalFirstParts(self): + entries = GetMatchingDepsEntries(DEPS_ENTRIES, 'src/build') + self.assertEqual(len(entries), 1) + self.assertEqual(entries[0], DEPS_ENTRIES['src/build']) + + def testCalculateChangedDeps(self): + _SetupGitLsRemoteCall( + self.fake, + 'https://chromium.googlesource.com/chromium/src/build', + BUILD_NEW_REV, + ) + libyuv_deps = ParseLocalDepsFile(self._libyuv_depsfile) + new_cr_deps = ParseLocalDepsFile(self._new_cr_depsfile) + changed_deps = CalculateChangedDeps(libyuv_deps, new_cr_deps) + self.assertEqual(len(changed_deps), 2) + self.assertEqual(changed_deps[0].path, 'src/build') + self.assertEqual(changed_deps[0].current_rev, BUILD_OLD_REV) + self.assertEqual(changed_deps[0].new_rev, BUILD_NEW_REV) + + self.assertEqual(changed_deps[1].path, 'src/buildtools') + self.assertEqual(changed_deps[1].current_rev, BUILDTOOLS_OLD_REV) + self.assertEqual(changed_deps[1].new_rev, BUILDTOOLS_NEW_REV) def _SetupGitLsRemoteCall(cmd_fake, url, revision): - cmd = ['git', 'ls-remote', url, revision] - cmd_fake.add_expectation(cmd, _returns=(revision, None)) + cmd = ['git', 'ls-remote', url, revision] + cmd_fake.add_expectation(cmd, _returns=(revision, None)) if __name__ == '__main__': - unittest.main() + unittest.main() diff --git a/tools_libyuv/get_landmines.py b/tools_libyuv/get_landmines.py index 8b33483e6..ee0dbecd7 100755 --- a/tools_libyuv/get_landmines.py +++ b/tools_libyuv/get_landmines.py @@ -9,31 +9,30 @@ # be found in the AUTHORS file in the root of the source tree. """ -This file emits the list of reasons why a particular build needs to be clobbered -(or a list of 'landmines'). +This file emits the list of reasons why a particular build needs to be +clobbered (or a list of 'landmines'). """ import sys def print_landmines(): - """ - ALL LANDMINES ARE EMITTED FROM HERE. - """ - # DO NOT add landmines as part of a regular CL. Landmines are a last-effort - # bandaid fix if a CL that got landed has a build dependency bug and all bots - # need to be cleaned up. If you're writing a new CL that causes build - # dependency problems, fix the dependency problems instead of adding a - # landmine. - # See the Chromium version in src/build/get_landmines.py for usage examples. - print('Clobber to remove GYP artifacts after switching bots to GN.') - print('Another try to remove GYP artifacts after switching bots to GN.') + """ALL LANDMINES ARE EMITTED FROM HERE.""" + # DO NOT add landmines as part of a regular CL. Landmines are a last-effort + # bandaid fix if a CL that got landed has a build dependency bug and all + # bots need to be cleaned up. If you're writing a new CL that causes build + # dependency problems, fix the dependency problems instead of adding a + # landmine. + # See the Chromium version in src/build/get_landmines.py for usage + # examples. + print('Clobber to remove GYP artifacts after switching bots to GN.') + print('Another try to remove GYP artifacts after switching bots to GN.') def main(): - print_landmines() - return 0 + print_landmines() + return 0 if __name__ == '__main__': - sys.exit(main()) + sys.exit(main())