From 77ba1546f2b1d9e5066e0efc4d7a204f71e4fb8c Mon Sep 17 00:00:00 2001 From: "kjellander@google.com" Date: Tue, 23 Sep 2014 11:13:29 +0000 Subject: [PATCH] Make Libyuv work with Chromium Git checkouts This is very similar to the changes in https://code.google.com/p/webrtc/source/detail?r=6938 TESTED=gclient sync and runhooks on Mac and Linux + building successfully. R=fbarchard@chromium.org Review URL: https://webrtc-codereview.appspot.com/24619004 git-svn-id: http://libyuv.googlecode.com/svn/trunk@1089 16f28f9a-4ce2-e073-06de-1de4eb20be90 --- .gitignore | 22 ++ DEPS | 206 +--------------- PRESUBMIT.py | 45 ++++ chromium/.gclient | 19 ++ chromium/README | 5 + gyp_libyuv | 17 +- setup_links.py | 462 ++++++++++++++++++++++++++++++++++++ sync_chromium.py | 105 ++++++++ tools/sanitizer_options.gyp | 59 ----- 9 files changed, 682 insertions(+), 258 deletions(-) create mode 100644 .gitignore create mode 100755 PRESUBMIT.py create mode 100644 chromium/.gclient create mode 100644 chromium/README create mode 100755 setup_links.py create mode 100755 sync_chromium.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..9928fea19 --- /dev/null +++ b/.gitignore @@ -0,0 +1,22 @@ +.gn +build +buildtools +chromium/.gclient_entries +chromium/.last_sync_chromium +chromium/src/ +google_apis +links +net +out/ +testing +third_party/ +tools/android +tools/clang +tools/find_depot_tools.py +tools/gn +tools/gyp +tools/memory +tools/python +tools/valgrind +tools/win + diff --git a/DEPS b/DEPS index ecdaea987..657002541 100644 --- a/DEPS +++ b/DEPS @@ -1,5 +1,3 @@ -use_relative_paths = True - vars = { "libyuv_trunk" : "https://libyuv.googlecode.com/svn/trunk", @@ -8,208 +6,24 @@ vars = { "root_dir": "trunk", "extra_gyp_flag": "-Dextra_gyp_flag=0", - # Use this googlecode_url variable only if there is an internal mirror for it. - # If you do not know, use the full path while defining your new deps entry. - "googlecode_url": "http://%s.googlecode.com/svn", - "chromium_trunk" : "http://src.chromium.org/svn/trunk", - # chrome://version/ for revision of canary Chrome. - # http://chromium-status.appspot.com/lkgr is a last known good revision. - "chromium_revision": "291168", -} - -# NOTE: Prefer revision numbers to tags for svn deps. Use http rather than -# https; the latter can cause problems for users behind proxies. -deps = { - "../chromium_deps": - File(Var("chromium_trunk") + "/src/DEPS@" + Var("chromium_revision")), - - "../chromium_gn": - File(Var("chromium_trunk") + "/src/.gn@" + Var("chromium_revision")), - - "build": - Var("chromium_trunk") + "/src/build@" + Var("chromium_revision"), - - "buildtools": - From("chromium_deps", "src/buildtools"), - - # Needed by common.gypi. - "google_apis/build": - Var("chromium_trunk") + "/src/google_apis/build@" + Var("chromium_revision"), - - "testing": - Var("chromium_trunk") + "/src/testing@" + Var("chromium_revision"), - - "testing/gtest": - From("chromium_deps", "src/testing/gtest"), - - "tools/clang": - Var("chromium_trunk") + "/src/tools/clang@" + Var("chromium_revision"), - - "tools/gn": - Var("chromium_trunk") + "/src/tools/gn@" + Var("chromium_revision"), - - "tools/gyp": - From("chromium_deps", "src/tools/gyp"), - - "tools/memory": - Var("chromium_trunk") + "/src/tools/memory@" + Var("chromium_revision"), - - "tools/python": - Var("chromium_trunk") + "/src/tools/python@" + Var("chromium_revision"), - - "tools/valgrind": - Var("chromium_trunk") + "/src/tools/valgrind@" + Var("chromium_revision"), - - # Needed by build/common.gypi. - "tools/win/supalink": - Var("chromium_trunk") + "/src/tools/win/supalink@" + Var("chromium_revision"), - - "third_party/binutils": - Var("chromium_trunk") + "/src/third_party/binutils@" + Var("chromium_revision"), - - "third_party/libc++": - Var("chromium_trunk") + "/src/third_party/libc++@" + Var("chromium_revision"), - - "third_party/libc++/trunk": - From("chromium_deps", "src/third_party/libc++/trunk"), - - "third_party/libc++abi": - Var("chromium_trunk") + "/src/third_party/libc++abi@" + Var("chromium_revision"), - - "third_party/libc++abi/trunk": - From("chromium_deps", "src/third_party/libc++abi/trunk"), - - "third_party/libjpeg_turbo": - From("chromium_deps", "src/third_party/libjpeg_turbo"), - - # Yasm assember required for libjpeg_turbo - "third_party/yasm": - Var("chromium_trunk") + "/src/third_party/yasm@" + Var("chromium_revision"), - - "third_party/yasm/source/patched-yasm": - Var("chromium_trunk") + "/deps/third_party/yasm/patched-yasm@" + Var("chromium_revision"), -} - -deps_os = { - "win": { - # Use WebRTC's, stripped down, version of Cygwin (required by GYP). - "third_party/cygwin": - (Var("googlecode_url") % "webrtc") + "/deps/third_party/cygwin@2672", - - # Used by libjpeg-turbo. - # TODO(fbarchard): Remove binaries and run yasm from build folder. - "third_party/yasm/binaries": - Var("chromium_trunk") + "/deps/third_party/yasm/binaries@" + Var("chromium_revision"), - "third_party/yasm": None, - - "tools/find_depot_tools": - File(Var("chromium_trunk") + "/src/tools/find_depot_tools.py@" + Var("chromium_revision")), - }, - "android": { - "third_party/android_tools": - From("chromium_deps", "src/third_party/android_tools"), - - "third_party/libjpeg": - Var("chromium_trunk") + "/src/third_party/libjpeg@" + Var("chromium_revision"), - }, - "ios": { - # NSS, for SSLClientSocketNSS. - "third_party/nss": - From("chromium_deps", "src/third_party/nss"), - - "net/third_party/nss": - Var("chromium_trunk") + "/src/net/third_party/nss@" + Var("chromium_revision"), - - # class-dump utility to generate header files for undocumented SDKs. - "testing/iossim/third_party/class-dump": - From("chromium_deps", "src/testing/iossim/third_party/class-dump"), - - # Helper for running under the simulator. - "testing/iossim": - Var("chromium_trunk") + "/src/testing/iossim@" + Var("chromium_revision"), - }, + # Roll the Chromium Git hash to pick up newer versions of all the + # dependencies and tools linked to in setup_links.py. + "chromium_revision": "6455c698e51af65f57a8fe83547296218a5a7251", } hooks = [ { - # Copy .gn from temporary place (../chromium_gn) to root_dir. - "name": "copy .gn", + # Clone chromium and its deps. + "name": "sync chromium", "pattern": ".", - "action": ["python", Var("root_dir") + "/build/cp.py", - Var("root_dir") + "/../chromium_gn/.gn", - Var("root_dir")], - }, - # Pull GN binaries. This needs to be before running GYP below. - { - "name": "gn_win", - "pattern": ".", - "action": [ "download_from_google_storage", - "--no_resume", - "--platform=win32", - "--no_auth", - "--bucket", "chromium-gn", - "-s", Var("root_dir") + "/buildtools/win/gn.exe.sha1", - ], + "action": ["python", "-u", Var("root_dir") + "/sync_chromium.py", + "--target-revision", Var("chromium_revision")], }, { - "name": "gn_mac", + # Create links to shared dependencies in Chromium. + "name": "setup_links", "pattern": ".", - "action": [ "download_from_google_storage", - "--no_resume", - "--platform=darwin", - "--no_auth", - "--bucket", "chromium-gn", - "-s", Var("root_dir") + "/buildtools/mac/gn.sha1", - ], - }, - { - "name": "gn_linux", - "pattern": ".", - "action": [ "download_from_google_storage", - "--no_resume", - "--platform=linux*", - "--no_auth", - "--bucket", "chromium-gn", - "-s", Var("root_dir") + "/buildtools/linux64/gn.sha1", - ], - }, - { - "name": "gn_linux32", - "pattern": ".", - "action": [ "download_from_google_storage", - "--no_resume", - "--platform=linux*", - "--no_auth", - "--bucket", "chromium-gn", - "-s", Var("root_dir") + "/buildtools/linux32/gn.sha1", - ], - }, - { - # Remove GN binaries from tools/gn/bin that aren't used anymore. - # TODO(kjellander) remove after the end of July, 2014. - "name": "remove_old_gn_binaries", - "pattern": ".", - "action": ["python", Var("root_dir") + "/tools/gn/bin/rm_binaries.py"], - }, - { - # Pull clang on mac. If nothing changed, or on non-mac platforms, this takes - # zero seconds to run. If something changed, it downloads a prebuilt clang. - "pattern": ".", - "action": ["python", Var("root_dir") + "/tools/clang/scripts/update.py", - "--if-needed"], - }, - { - # Update the Windows toolchain if necessary. - "name": "win_toolchain", - "pattern": ".", - "action": ["python", Var("root_dir") + "/download_vs_toolchain.py", - "update"], - }, - { - # Pull binutils for gold. - "name": "binutils", - "pattern": ".", - "action": ["python", Var("root_dir") + "/third_party/binutils/download.py"], + "action": ["python", Var("root_dir") + "/setup_links.py"], }, { # A change to a .gyp, .gypi, or to GYP itself should run the generator. diff --git a/PRESUBMIT.py b/PRESUBMIT.py new file mode 100755 index 000000000..80ec30043 --- /dev/null +++ b/PRESUBMIT.py @@ -0,0 +1,45 @@ +# Copyright 2014 The LibYuv Project Authors. All rights reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import re +import sys + + +def GetDefaultTryConfigs(bots=None): + """Returns a list of ('bot', set(['tests']), optionally filtered by [bots]. + + For WebRTC purposes, we always return an empty list of tests, since we want + to run all tests by default on all our trybots. + """ + return { 'tryserver.libyuv': dict((bot, []) for bot in bots)} + + +# pylint: disable=W0613 +def GetPreferredTryMasters(project, change): + files = change.LocalPaths() + bots = [ + 'win', + 'win_rel', + 'win_x64_rel', + 'mac', + 'mac_rel', + 'mac_x64_rel', + 'ios', + 'ios_rel', + 'mac_asan', + 'linux', + 'linux_rel', + 'linux_memcheck', + 'linux_tsan2', + 'linux_asan', + 'android', + 'android_rel', + ] + if not files or all(re.search(r'[\\/]OWNERS$', f) for f in files): + return {} + return GetDefaultTryConfigs(bots) diff --git a/chromium/.gclient b/chromium/.gclient new file mode 100644 index 000000000..1ff06aaee --- /dev/null +++ b/chromium/.gclient @@ -0,0 +1,19 @@ +solutions = [{ + 'name': 'src', + 'url': 'https://chromium.googlesource.com/chromium/src.git', + 'deps_file': '.DEPS.git', + 'managed': False, + 'custom_deps': { + # Skip syncing some large dependencies Libyuv will never need. + 'src/chrome/tools/test/reference_build/chrome_linux': None, + 'src/chrome/tools/test/reference_build/chrome_mac': None, + 'src/chrome/tools/test/reference_build/chrome_win': None, + 'src/native_client': None, + 'src/third_party/ffmpeg': None, + 'src/third_party/WebKit': None, + 'src/v8': None, + }, + 'safesync_url': '' +}] + +cache_dir = None diff --git a/chromium/README b/chromium/README new file mode 100644 index 000000000..127f4b520 --- /dev/null +++ b/chromium/README @@ -0,0 +1,5 @@ +This .gclient file is used to do download a copy of Chromium. +Libyuv uses the Chromium build toolchain and a number of shared +dependencies by creating symlinks to folders in this checkout, +using the ../setup_links.py script. + diff --git a/gyp_libyuv b/gyp_libyuv index 64d426ec0..645d3ad45 100755 --- a/gyp_libyuv +++ b/gyp_libyuv @@ -19,7 +19,6 @@ import sys checkout_root = os.path.dirname(os.path.realpath(__file__)) sys.path.insert(0, os.path.join(checkout_root, 'build')) -sys.path.insert(0, os.path.join(checkout_root, 'tools', 'find_depot_tools')) import gyp_chromium import gyp_helper import vs_toolchain @@ -27,6 +26,13 @@ import vs_toolchain sys.path.insert(0, os.path.join(checkout_root, 'tools', 'gyp', 'pylib')) import gyp +def GetSupplementalFiles(): + """Returns a list of the supplemental files that are included in all GYP + sources.""" + # Can't use the one in gyp_chromium since the directory location of the root + # is different. + return glob.glob(os.path.join(checkout_root, '*', 'supplement.gypi')) + if __name__ == '__main__': args = sys.argv[1:] @@ -41,7 +47,10 @@ if __name__ == '__main__': # If we didn't get a file, assume 'all.gyp' in the root of the checkout. if not gyp_file_specified: - args.append(os.path.join(checkout_root, 'all.gyp')) + # Because of a bug in gyp, simply adding the abspath to all.gyp doesn't + # work, but chdir'ing and adding the relative path does. Spooky :/ + os.chdir(checkout_root) + args.append('all.gyp') # There shouldn't be a circular dependency relationship between .gyp files, args.append('--no-circular-check') @@ -50,7 +59,9 @@ if __name__ == '__main__': if not os.environ.get('GYP_GENERATORS'): os.environ['GYP_GENERATORS'] = 'ninja' - vs2013_runtime_dll_dirs = vs_toolchain.SetEnvironmentAndGetRuntimeDllDirs() + vs2013_runtime_dll_dirs = None + if int(os.environ.get('DEPOT_TOOLS_WIN_TOOLCHAIN', '1')): + vs2013_runtime_dll_dirs = vs_toolchain.SetEnvironmentAndGetRuntimeDllDirs() # Enforce gyp syntax checking. This adds about 20% execution time. args.append('--check') diff --git a/setup_links.py b/setup_links.py new file mode 100755 index 000000000..867924d46 --- /dev/null +++ b/setup_links.py @@ -0,0 +1,462 @@ +#!/usr/bin/env python +# Copyright 2014 The LibYuv Project Authors. All rights reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +"""Setup links to a Chromium checkout for Libyuv. + +Libyuv shares a lot of dependencies and build tools with Chromium. +To do this, many of the paths of a Chromium checkout is emulated by creating +symlinks to files and directories. This script handles the setup of symlinks to +achieve this. + +It's a modified copy of the similar script that lives in WebRTC. +It also handles cleanup of the legacy Subversion-based approach that was used +before Chrome switched over their master repo from Subversion to Git. +""" + + +import ctypes +import errno +import logging +import optparse +import os +import shelve +import shutil +import subprocess +import sys +import textwrap + + +DIRECTORIES = [ + 'build', + 'buildtools', + 'google_apis', # Needed by build/common.gypi. + 'net', + 'testing', + 'third_party/android_testrunner', + 'third_party/android_tools', + 'third_party/binutils', + 'third_party/libc++', + 'third_party/libc++abi', + 'third_party/libjpeg', + 'third_party/libjpeg_turbo', + 'third_party/llvm-build', + 'third_party/nss', + 'third_party/yasm', + 'tools/android', + 'tools/clang', + 'tools/gn', + 'tools/gyp', + 'tools/memory', + 'tools/python', + 'tools/valgrind', + 'tools/win', +] + +FILES = { + '.gn': None, + 'tools/find_depot_tools.py': None, + 'third_party/BUILD.gn': None, +} + +CHROMIUM_CHECKOUT = os.path.join('chromium', 'src') +LINKS_DB = 'links' + +# Version management to make future upgrades/downgrades easier to support. +SCHEMA_VERSION = 1 + + +def query_yes_no(question, default=False): + """Ask a yes/no question via raw_input() and return their answer. + + Modified from http://stackoverflow.com/a/3041990. + """ + prompt = " [%s/%%s]: " + prompt = prompt % ('Y' if default is True else 'y') + prompt = prompt % ('N' if default is False else 'n') + + if default is None: + default = 'INVALID' + + while True: + sys.stdout.write(question + prompt) + choice = raw_input().lower() + if choice == '' and default != 'INVALID': + return default + + if 'yes'.startswith(choice): + return True + elif 'no'.startswith(choice): + return False + + print "Please respond with 'yes' or 'no' (or 'y' or 'n')." + + +# Actions +class Action(object): + def __init__(self, dangerous): + self.dangerous = dangerous + + def announce(self, planning): + """Log a description of this action. + + Args: + planning - True iff we're in the planning stage, False if we're in the + doit stage. + """ + pass + + def doit(self, links_db): + """Execute the action, recording what we did to links_db, if necessary.""" + pass + + +class Remove(Action): + def __init__(self, path, dangerous): + super(Remove, self).__init__(dangerous) + self._priority = 0 + self._path = path + + def announce(self, planning): + log = logging.warn + filesystem_type = 'file' + if not self.dangerous: + log = logging.info + filesystem_type = 'link' + if planning: + log('Planning to remove %s: %s', filesystem_type, self._path) + else: + log('Removing %s: %s', filesystem_type, self._path) + + def doit(self, _links_db): + os.remove(self._path) + + +class Rmtree(Action): + def __init__(self, path): + super(Rmtree, self).__init__(dangerous=True) + self._priority = 0 + self._path = path + + def announce(self, planning): + if planning: + logging.warn('Planning to remove directory: %s', self._path) + else: + logging.warn('Removing directory: %s', self._path) + + def doit(self, _links_db): + if sys.platform.startswith('win'): + # shutil.rmtree() doesn't work on Windows if any of the directories are + # read-only, which svn repositories are. + subprocess.check_call(['rd', '/q', '/s', self._path], shell=True) + else: + shutil.rmtree(self._path) + + +class Makedirs(Action): + def __init__(self, path): + super(Makedirs, self).__init__(dangerous=False) + self._priority = 1 + self._path = path + + def doit(self, _links_db): + try: + os.makedirs(self._path) + except OSError as e: + if e.errno != errno.EEXIST: + raise + + +class Symlink(Action): + def __init__(self, source_path, link_path): + super(Symlink, self).__init__(dangerous=False) + self._priority = 2 + self._source_path = source_path + self._link_path = link_path + + def announce(self, planning): + if planning: + logging.info( + 'Planning to create link from %s to %s', self._link_path, + self._source_path) + else: + logging.debug( + 'Linking from %s to %s', self._link_path, self._source_path) + + def doit(self, links_db): + # Files not in the root directory need relative path calculation. + # On Windows, use absolute paths instead since NTFS doesn't seem to support + # relative paths for symlinks. + if sys.platform.startswith('win'): + source_path = os.path.abspath(self._source_path) + else: + if os.path.dirname(self._link_path) != self._link_path: + source_path = os.path.relpath(self._source_path, + os.path.dirname(self._link_path)) + + os.symlink(source_path, os.path.abspath(self._link_path)) + links_db[self._source_path] = self._link_path + + +class LinkError(IOError): + """Failed to create a link.""" + pass + + +# Handles symlink creation on the different platforms. +if sys.platform.startswith('win'): + def symlink(source_path, link_path): + flag = 1 if os.path.isdir(source_path) else 0 + if not ctypes.windll.kernel32.CreateSymbolicLinkW( + unicode(link_path), unicode(source_path), flag): + raise OSError('Failed to create symlink to %s. Notice that only NTFS ' + 'version 5.0 and up has all the needed APIs for ' + 'creating symlinks.' % source_path) + os.symlink = symlink + + +class LibyuvLinkSetup(): + def __init__(self, links_db, force=False, dry_run=False, prompt=False): + self._force = force + self._dry_run = dry_run + self._prompt = prompt + self._links_db = links_db + + def CreateLinks(self, on_bot): + logging.debug('CreateLinks') + # First, make a plan of action + actions = [] + + for source_path, link_path in FILES.iteritems(): + actions += self._ActionForPath( + source_path, link_path, check_fn=os.path.isfile, check_msg='files') + for source_dir in DIRECTORIES: + actions += self._ActionForPath( + source_dir, None, check_fn=os.path.isdir, + check_msg='directories') + + actions.sort() + + if self._dry_run: + for action in actions: + action.announce(planning=True) + logging.info('Not doing anything because dry-run was specified.') + sys.exit(0) + + if any(a.dangerous for a in actions): + logging.warn('Dangerous actions:') + for action in (a for a in actions if a.dangerous): + action.announce(planning=True) + print + + if not self._force: + logging.error(textwrap.dedent("""\ + @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ + A C T I O N R E Q I R E D + @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ + + Because chromium/src is transitioning to Git (from SVN), we needed to + change the way that the Libyuv standalone checkout works. Instead of + individually syncing subdirectories of Chromium in SVN, we're now + syncing Chromium (and all of its DEPS, as defined by its own DEPS file), + into the `chromium/src` directory. + + As such, all Chromium directories which are currently pulled by DEPS are + now replaced with a symlink into the full Chromium checkout. + + To avoid disrupting developers, we've chosen to not delete your + directories forcibly, in case you have some work in progress in one of + them :). + + ACTION REQUIRED: + Before running `gclient sync|runhooks` again, you must run: + %s%s --force + + Which will replace all directories which now must be symlinks, after + prompting with a summary of the work-to-be-done. + """), 'python ' if sys.platform.startswith('win') else '', sys.argv[0]) + sys.exit(1) + elif self._prompt: + if not query_yes_no('Would you like to perform the above plan?'): + sys.exit(1) + + for action in actions: + action.announce(planning=False) + action.doit(self._links_db) + + if not on_bot and self._force: + logging.info('Completed!\n\nNow run `gclient sync|runhooks` again to ' + 'let the remaining hooks (that probably were interrupted) ' + 'execute.') + + def CleanupLinks(self): + logging.debug('CleanupLinks') + for source, link_path in self._links_db.iteritems(): + if source == 'SCHEMA_VERSION': + continue + if os.path.islink(link_path) or sys.platform.startswith('win'): + # os.path.islink() always returns false on Windows + # See http://bugs.python.org/issue13143. + logging.debug('Removing link to %s at %s', source, link_path) + if not self._dry_run: + if os.path.exists(link_path): + if sys.platform.startswith('win') and os.path.isdir(link_path): + subprocess.check_call(['rmdir', '/q', link_path], shell=True) + else: + os.remove(link_path) + del self._links_db[source] + + @staticmethod + def _ActionForPath(source_path, link_path=None, check_fn=None, + check_msg=None): + """Create zero or more Actions to link to a file or directory. + + This will be a symlink on POSIX platforms. On Windows this requires + that NTFS is version 5.0 or higher (Vista or newer). + + Args: + source_path: Path relative to the Chromium checkout root. + For readability, the path may contain slashes, which will + automatically be converted to the right path delimiter on Windows. + link_path: The location for the link to create. If omitted it will be the + same path as source_path. + check_fn: A function returning true if the type of filesystem object is + correct for the attempted call. Otherwise an error message with + check_msg will be printed. + check_msg: String used to inform the user of an invalid attempt to create + a file. + Returns: + A list of Action objects. + """ + def fix_separators(path): + if sys.platform.startswith('win'): + return path.replace(os.altsep, os.sep) + else: + return path + + assert check_fn + assert check_msg + link_path = link_path or source_path + link_path = fix_separators(link_path) + + source_path = fix_separators(source_path) + source_path = os.path.join(CHROMIUM_CHECKOUT, source_path) + if os.path.exists(source_path) and not check_fn: + raise LinkError('_LinkChromiumPath can only be used to link to %s: ' + 'Tried to link to: %s' % (check_msg, source_path)) + + if not os.path.exists(source_path): + logging.debug('Silently ignoring missing source: %s. This is to avoid ' + 'errors on platform-specific dependencies.', source_path) + return [] + + actions = [] + + if os.path.exists(link_path) or os.path.islink(link_path): + if os.path.islink(link_path): + actions.append(Remove(link_path, dangerous=False)) + elif os.path.isfile(link_path): + actions.append(Remove(link_path, dangerous=True)) + elif os.path.isdir(link_path): + actions.append(Rmtree(link_path)) + else: + raise LinkError('Don\'t know how to plan: %s' % link_path) + + # Create parent directories to the target link if needed. + target_parent_dirs = os.path.dirname(link_path) + if (target_parent_dirs and + target_parent_dirs != link_path and + not os.path.exists(target_parent_dirs)): + actions.append(Makedirs(target_parent_dirs)) + + actions.append(Symlink(source_path, link_path)) + + return actions + +def _initialize_database(filename): + links_database = shelve.open(filename) + + # Wipe the database if this version of the script ends up looking at a + # newer (future) version of the links db, just to be sure. + version = links_database.get('SCHEMA_VERSION') + if version and version != SCHEMA_VERSION: + logging.info('Found database with schema version %s while this script only ' + 'supports %s. Wiping previous database contents.', version, + SCHEMA_VERSION) + links_database.clear() + links_database['SCHEMA_VERSION'] = SCHEMA_VERSION + return links_database + + +def main(): + on_bot = os.environ.get('CHROME_HEADLESS') == '1' + + parser = optparse.OptionParser() + parser.add_option('-d', '--dry-run', action='store_true', default=False, + help='Print what would be done, but don\'t perform any ' + 'operations. This will automatically set logging to ' + 'verbose.') + parser.add_option('-c', '--clean-only', action='store_true', default=False, + help='Only clean previously created links, don\'t create ' + 'new ones. This will automatically set logging to ' + 'verbose.') + parser.add_option('-f', '--force', action='store_true', default=on_bot, + help='Force link creation. CAUTION: This deletes existing ' + 'folders and files in the locations where links are ' + 'about to be created.') + parser.add_option('-n', '--no-prompt', action='store_false', dest='prompt', + default=(not on_bot), + help='Prompt if we\'re planning to do a dangerous action') + parser.add_option('-v', '--verbose', action='store_const', + const=logging.DEBUG, default=logging.INFO, + help='Print verbose output for debugging.') + options, _ = parser.parse_args() + + if options.dry_run or options.force or options.clean_only: + options.verbose = logging.DEBUG + logging.basicConfig(format='%(message)s', level=options.verbose) + + # Work from the root directory of the checkout. + script_dir = os.path.dirname(os.path.abspath(__file__)) + os.chdir(script_dir) + + if sys.platform.startswith('win'): + def is_admin(): + try: + return os.getuid() == 0 + except AttributeError: + return ctypes.windll.shell32.IsUserAnAdmin() != 0 + if not is_admin(): + logging.error('On Windows, you now need to have administrator ' + 'privileges for the shell running %s (or ' + '`gclient sync|runhooks`).\nPlease start another command ' + 'prompt as Administrator and try again.' % sys.argv[0]) + return 1 + + if not os.path.exists(CHROMIUM_CHECKOUT): + logging.error('Cannot find a Chromium checkout at %s. Did you run "gclient ' + 'sync" before running this script?', CHROMIUM_CHECKOUT) + return 2 + + links_database = _initialize_database(LINKS_DB) + try: + symlink_creator = LibyuvLinkSetup(links_database, options.force, + options.dry_run, options.prompt) + symlink_creator.CleanupLinks() + if not options.clean_only: + symlink_creator.CreateLinks(on_bot) + except LinkError as e: + print >> sys.stderr, e.message + return 3 + finally: + links_database.close() + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/sync_chromium.py b/sync_chromium.py new file mode 100755 index 000000000..0702ef00d --- /dev/null +++ b/sync_chromium.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python +# Copyright 2014 The LibYuv Project Authors. All rights reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import argparse +import os +import subprocess +import sys + +# Bump this whenever the algorithm changes and you need bots/devs to re-sync, +# ignoring the .last_sync_chromium file +SCRIPT_VERSION = 1 + +ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) + + +def get_target_os_list(): + try: + main_gclient = os.path.join(os.path.dirname(ROOT_DIR), '.gclient') + config_dict = {} + with open(main_gclient, 'rb') as deps_content: + exec(deps_content, config_dict) + return ','.join(config_dict.get('target_os', [])) + except Exception as e: + print >> sys.stderr, "error while parsing .gclient:", e + + +def main(): + CR_DIR = os.path.join(ROOT_DIR, 'chromium') + + p = argparse.ArgumentParser() + p.add_argument('--target-revision', required=True, + help='The target chromium git revision [REQUIRED]') + p.add_argument('--chromium-dir', default=CR_DIR, + help=('The path to the chromium directory to sync ' + '(default: %(default)r)')) + opts = p.parse_args() + opts.chromium_dir = os.path.abspath(opts.chromium_dir) + + target_os_list = get_target_os_list() + + # Do a quick check to see if we were successful last time to make runhooks + # sooper fast. + flag_file = os.path.join(opts.chromium_dir, '.last_sync_chromium') + flag_file_content = '\n'.join([ + str(SCRIPT_VERSION), + opts.target_revision, + repr(target_os_list), + ]) + if os.path.exists(flag_file): + with open(flag_file, 'r') as f: + if f.read() == flag_file_content: + print "Chromium already up to date:", opts.target_revision + return 0 + os.unlink(flag_file) + + env = os.environ.copy() + env['GYP_CHROMIUM_NO_ACTION'] = '1' + gclient_cmd = 'gclient.bat' if sys.platform.startswith('win') else 'gclient' + args = [ + gclient_cmd, 'sync', '--force', '--revision', 'src@'+opts.target_revision + ] + + if os.environ.get('CHROME_HEADLESS') == '1': + args.append('-vvv') + + if sys.platform.startswith('win'): + cache_path = os.path.join(os.path.splitdrive(ROOT_DIR)[0] + os.path.sep, + 'b', 'git-cache') + else: + cache_path = '/b/git-cache' + + gclientfile = os.path.join(opts.chromium_dir, '.gclient') + with open(gclientfile, 'rb') as spec: + spec = spec.read().splitlines() + spec[-1] = 'cache_dir = %r' % (cache_path,) + with open(gclientfile + '.bot', 'wb') as f: + f.write('\n'.join(spec)) + + args += [ + '--gclientfile', '.gclient.bot', + '--delete_unversioned_trees', '--reset', '--upstream' + ] + else: + args.append('--no-history') + + if target_os_list: + args += ['--deps=' + target_os_list] + + print 'Running "%s" in %s' % (' '.join(args), opts.chromium_dir) + ret = subprocess.call(args, cwd=opts.chromium_dir, env=env) + if ret == 0: + with open(flag_file, 'wb') as f: + f.write(flag_file_content) + + return ret + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tools/sanitizer_options.gyp b/tools/sanitizer_options.gyp index ea453d356..e69de29bb 100644 --- a/tools/sanitizer_options.gyp +++ b/tools/sanitizer_options.gyp @@ -1,59 +0,0 @@ -# Copyright 2014 The LibYuv Project Authors. All rights reserved. -# -# Use of this source code is governed by a BSD-style license -# that can be found in the LICENSE file in the root of the source -# tree. An additional intellectual property rights grant can be found -# in the file PATENTS. All contributing project authors may -# be found in the AUTHORS file in the root of the source tree. - -# This is a similar target to the one in Chromium's base.gyp. It's needed to get -# the same sanitizer settings as Chromium uses (i.e. ASan, LSan, TSan...). -{ - 'targets': [ - { - 'target_name': 'sanitizer_options', - 'type': 'static_library', - 'toolsets': ['host', 'target'], - 'variables': { - # Every target is going to depend on sanitizer_options, so allow - # this one to depend on itself. - 'prune_self_dependency': 1, - # Do not let 'none' targets depend on this one, they don't need to. - 'link_dependency': 1, - }, - 'sources': [ - 'sanitizer_options/sanitizer_options.cc', - ], - 'include_dirs': [ - '<(DEPTH)', - ], - # Some targets may want to opt-out from ASan, TSan and MSan and link - # without the corresponding runtime libraries. We drop the libc++ - # dependency and omit the compiler flags to avoid bringing instrumented - # code to those targets. - 'conditions': [ - ['use_custom_libcxx==1', { - 'dependencies!': [ - '<(DEPTH)/third_party/libc++/libc++.gyp:libcxx_proxy', - ], - }], - ['tsan==1', { - 'sources': [ - 'tsan_suppressions/tsan_suppressions.cc', - ], - }], - ], - 'cflags!': [ - '-fsanitize=address', - '-fsanitize=thread', - '-fsanitize=memory', - '-fsanitize-memory-track-origins', - ], - 'direct_dependent_settings': { - 'ldflags': [ - '-Wl,-u_sanitizer_options_link_helper', - ], - }, - }, - ], # targets -}