Update PRESUBMIT, cleanup_links and autoroller to py3

First pass, will continue in followup CL's.

Bug: libyuv:917
Change-Id: I1c1cad4ebb4e37225611fd11d420e12e3a7d45b5
Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/3488006
Reviewed-by: Mirko Bonadei <mbonadei@chromium.org>
Commit-Queue: Christoffer Jansson <jansson@chromium.org>

PRESUBMIT.py

@@ -6,6 +6,8 @@
 # in the file PATENTS. All contributing project authors may
 # be found in the AUTHORS file in the root of the source tree.
 
+# Runs PRESUBMIT.py in py3 mode by git cl presubmit.
+USE_PYTHON3 = True
 
 def _CommonChecks(input_api, output_api):
   """Checks common to both upload and commit."""
@@ -26,7 +28,8 @@ def _CommonChecks(input_api, output_api):
                          'E0611',  # No package y in x
                          'W0232',  # Class has no __init__ method
                         ],
-      pylintrc='pylintrc'))
+      pylintrc='pylintrc',
+      version='2.7'))
   return results
 
 

cleanup_links.py

@@ -1,4 +1,5 @@
-#!/usr/bin/env python
+#!/usr/bin/env vpython3
+
 # Copyright 2017 The LibYuv Project Authors. All rights reserved.
 #
 # Use of this source code is governed by a BSD-style license
@@ -18,8 +19,8 @@ landing that change, this script cleans up any old symlinks, avoiding annoying
 manual cleanup needed in order to complete gclient sync.
 """
 
+import argparse
 import logging
-import optparse
 import os
 import shelve
 import subprocess
@@ -32,14 +33,14 @@ LINKS_DB = 'links'
 # Version management to make future upgrades/downgrades easier to support.
 SCHEMA_VERSION = 1
 
-class WebRTCLinkSetup(object):
+class WebRTCLinkSetup():
   def __init__(self, links_db, dry_run=False):
     self._dry_run = dry_run
     self._links_db = links_db
 
   def CleanupLinks(self):
     logging.debug('CleanupLinks')
-    for source, link_path  in self._links_db.iteritems():
+    for source, link_path  in self._links_db.tems():
       if source == 'SCHEMA_VERSION':
         continue
       if os.path.islink(link_path) or sys.platform.startswith('win'):
@@ -71,15 +72,15 @@ def _initialize_database(filename):
 
 
 def main():
-  parser = optparse.OptionParser()
+  p = argparse.ArgumentParser()
-  parser.add_option('-d', '--dry-run', action='store_true', default=False,
+  p.add_argument('-d', '--dry-run', action='store_true', default=False,
-                    help='Print what would be done, but don\'t perform any '
+                 help='Print what would be done, but don\'t perform any '
-                         'operations. This will automatically set logging to '
+                      'operations. This will automatically set logging to '
-                         'verbose.')
+                      'verbose.')
-  parser.add_option('-v', '--verbose', action='store_const',
+  p.add_argument('-v', '--verbose', action='store_const',
-                    const=logging.DEBUG, default=logging.INFO,
+                 const=logging.DEBUG, default=logging.INFO,
-                    help='Print verbose output for debugging.')
+                 help='Print verbose output for debugging.')
-  options, _ = parser.parse_args()
+  options = p.parse_args()
 
   if options.dry_run:
     options.verbose = logging.DEBUG

tools_libyuv/autoroller/roll_deps.py

@@ -1,4 +1,5 @@
-#!/usr/bin/env python
+#!/usr/bin/env vpython3
+
 # Copyright 2017 The LibYuv Project Authors. All rights reserved.
 #
 # Use of this source code is governed by a BSD-style license
@@ -11,7 +12,6 @@
 # https://webrtc.googlesource.com/src/+/master/tools_webrtc/autoroller/roll_deps.py
 # customized for libyuv.
 
-
 """Script to automatically roll dependencies in the libyuv DEPS file."""
 
 import argparse
@@ -22,7 +22,7 @@ import os
 import re
 import subprocess
 import sys
-import urllib2
+import urllib.request
 
 
 # Skip these dependencies (list without solution name prefix).
@@ -78,7 +78,7 @@ def ParseDepsDict(deps_content):
 
 def ParseLocalDepsFile(filename):
   with open(filename, 'rb') as f:
-    deps_content = f.read()
+    deps_content = f.read().decode('utf-8')
   return ParseDepsDict(deps_content)
 
 
@@ -98,7 +98,7 @@ def ParseCommitPosition(commit_message):
 
 
 def _RunCommand(command, working_dir=None, ignore_exit_code=False,
-                extra_env=None):
+                extra_env=None, input_data=None):
   """Runs a command and returns the output from that command.
 
   If the command fails (exit code != 0), the function will exit the process.
@@ -113,12 +113,14 @@ def _RunCommand(command, working_dir=None, ignore_exit_code=False,
     assert all(isinstance(value, str) for value in extra_env.values())
     logging.debug('extra env: %s', extra_env)
     env.update(extra_env)
-  p = subprocess.Popen(command, stdout=subprocess.PIPE,
+  p = subprocess.Popen(command,
-                       stderr=subprocess.PIPE, env=env,
+                       stdin=subprocess.PIPE,
-                       cwd=working_dir, universal_newlines=True)
+                       stdout=subprocess.PIPE,
-  std_output = p.stdout.read()
+                       stderr=subprocess.PIPE,
-  err_output = p.stderr.read()
+                       env=env,
-  p.wait()
+                       cwd=working_dir,
+                       universal_newlines=True)
+  std_output, err_output = p.communicate(input_data)
   p.stdout.close()
   p.stderr.close()
   if not ignore_exit_code and p.returncode != 0:
@@ -154,7 +156,7 @@ def _ReadGitilesContent(url):
   # Download and decode BASE64 content until
   # https://code.google.com/p/gitiles/issues/detail?id=7 is fixed.
   base64_content = ReadUrlContent(url + '?format=TEXT')
-  return base64.b64decode(base64_content[0])
+  return base64.b64decode(base64_content[0]).decode('utf-8')
 
 
 def ReadRemoteCrFile(path_below_src, revision):
@@ -170,7 +172,7 @@ def ReadRemoteCrCommit(revision):
 
 def ReadUrlContent(url):
   """Connect to a remote host and read the contents. Returns a list of lines."""
-  conn = urllib2.urlopen(url)
+  conn = urllib.request.urlopen(url)
   try:
     return conn.readlines()
   except IOError as e:
@@ -193,7 +195,7 @@ def GetMatchingDepsEntries(depsentry_dict, dir_path):
     A list of DepsEntry objects.
   """
   result = []
-  for path, depsentry in depsentry_dict.iteritems():
+  for path, depsentry in depsentry_dict.items():
     if path == dir_path:
       result.append(depsentry)
     else:
@@ -203,26 +205,24 @@ def GetMatchingDepsEntries(depsentry_dict, dir_path):
         result.append(depsentry)
   return result
 
-
 def BuildDepsentryDict(deps_dict):
-  """Builds a dict of paths to DepsEntry objects from a raw parsed deps dict."""
+  """Builds a dict of paths to DepsEntry objects from a raw deps dict."""
   result = {}
+
   def AddDepsEntries(deps_subdict):
-    for path, deps_url_spec in deps_subdict.iteritems():
+    for path, deps_url_spec in deps_subdict.items():
-      # The deps url is either an URL and a condition, or just the URL.
       if isinstance(deps_url_spec, dict):
         if deps_url_spec.get('dep_type') == 'cipd':
           continue
         deps_url = deps_url_spec['url']
       else:
         deps_url = deps_url_spec
-
+      if not path in result:
-      if not result.has_key(path):
         url, revision = deps_url.split('@') if deps_url else (None, None)
         result[path] = DepsEntry(path, url, revision)
 
   AddDepsEntries(deps_dict['deps'])
-  for deps_os in ['win', 'mac', 'unix', 'android', 'ios', 'unix']:
+  for deps_os in ['win', 'mac', 'linux', 'android', 'ios', 'unix']:
     AddDepsEntries(deps_dict.get('deps_os', {}).get(deps_os, {}))
   return result
 
@@ -245,7 +245,7 @@ def CalculateChangedDeps(libyuv_deps, new_cr_deps):
   result = []
   libyuv_entries = BuildDepsentryDict(libyuv_deps)
   new_cr_entries = BuildDepsentryDict(new_cr_deps)
-  for path, libyuv_deps_entry in libyuv_entries.iteritems():
+  for path, libyuv_deps_entry in libyuv_entries.items():
     if path in DONT_AUTOROLL_THESE:
       continue
     cr_deps_entry = new_cr_entries.get(path)
@@ -277,7 +277,7 @@ def CalculateChangedClang(new_cr_rev):
         return match.group(1)
     raise RollError('Could not parse Clang revision from:\n' + '\n'.join('  ' + l for l in lines))
 
-  with open(CLANG_UPDATE_SCRIPT_LOCAL_PATH, 'rb') as f:
+  with open(CLANG_UPDATE_SCRIPT_LOCAL_PATH, 'r') as f:
     current_lines = f.readlines()
   current_rev = GetClangRev(current_lines)
 
@@ -335,10 +335,10 @@ def UpdateDepsFile(deps_filename, old_cr_revision, new_cr_revision,
 
   # Update the chromium_revision variable.
   with open(deps_filename, 'rb') as deps_file:
-    deps_content = deps_file.read()
+    deps_content = deps_file.read().decode('utf-8')
   deps_content = deps_content.replace(old_cr_revision, new_cr_revision)
   with open(deps_filename, 'wb') as deps_file:
-    deps_file.write(deps_content)
+    deps_file.write(deps_content.encode('utf-8'))
 
   # Update each individual DEPS entry.
   for dep in changed_deps:

tools_libyuv/autoroller/unittests/roll_deps_test.py

@@ -1,4 +1,5 @@
-#!/usr/bin/env python
+#!/usr/bin/env vpython3
+
 # Copyright 2017 The LibYuv Project Authors. All rights reserved.
 #
 # Use of this source code is governed by a BSD-style license
@@ -14,15 +15,13 @@ import sys
 import tempfile
 import unittest
 
+import roll_deps
+from roll_deps import CalculateChangedDeps, GetMatchingDepsEntries, \
+  ParseDepsDict, ParseLocalDepsFile, UpdateDepsFile
 
 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
 PARENT_DIR = os.path.join(SCRIPT_DIR, os.pardir)
 sys.path.append(PARENT_DIR)
-import roll_deps  # pylint: disable=wrong-import-position
-from roll_deps import CalculateChangedDeps, GetMatchingDepsEntries, \
-  ParseDepsDict, ParseLocalDepsFile, \
-  UpdateDepsFile  # pylint: disable=wrong-import-position
-
 
 TEST_DATA_VARS = {
   'chromium_git': 'https://chromium.googlesource.com',
@@ -46,7 +45,7 @@ class TestError(Exception):
   pass
 
 
-class FakeCmd(object):
+class FakeCmd():
   def __init__(self):
     self.expectations = []
 
@@ -86,43 +85,43 @@ class TestRollChromiumRevision(unittest.TestCase):
   def testVarLookup(self):
     local_scope = {'foo': 'wrong', 'vars': {'foo': 'bar'}}
     lookup = roll_deps.VarLookup(local_scope)
-    self.assertEquals(lookup('foo'), 'bar')
+    self.assertEqual(lookup('foo'), 'bar')
 
   def testUpdateDepsFile(self):
     new_rev = 'aaaaabbbbbcccccdddddeeeeefffff0000011111'
 
     current_rev = TEST_DATA_VARS['chromium_revision']
     UpdateDepsFile(self._libyuv_depsfile, current_rev, new_rev, [])
-    with open(self._libyuv_depsfile) as deps_file:
+    with open(self._libyuv_depsfile, 'r') as deps_file:
       deps_contents = deps_file.read()
       self.assertTrue(new_rev in deps_contents,
                       'Failed to find %s in\n%s' % (new_rev, deps_contents))
 
   def testParseDepsDict(self):
-    with open(self._libyuv_depsfile) as deps_file:
+    with open(self._libyuv_depsfile, 'r') as deps_file:
       deps_contents = deps_file.read()
     local_scope = ParseDepsDict(deps_contents)
     vars_dict = local_scope['vars']
 
     def assertVar(variable_name):
-      self.assertEquals(vars_dict[variable_name], TEST_DATA_VARS[variable_name])
+      self.assertEqual(vars_dict[variable_name], TEST_DATA_VARS[variable_name])
     assertVar('chromium_git')
     assertVar('chromium_revision')
-    self.assertEquals(len(local_scope['deps']), 3)
+    self.assertEqual(len(local_scope['deps']), 3)
 
   def testGetMatchingDepsEntriesReturnsPathInSimpleCase(self):
     entries = GetMatchingDepsEntries(DEPS_ENTRIES, 'src/testing/gtest')
-    self.assertEquals(len(entries), 1)
+    self.assertEqual(len(entries), 1)
-    self.assertEquals(entries[0], DEPS_ENTRIES['src/testing/gtest'])
+    self.assertEqual(entries[0], DEPS_ENTRIES['src/testing/gtest'])
 
   def testGetMatchingDepsEntriesHandlesSimilarStartingPaths(self):
     entries = GetMatchingDepsEntries(DEPS_ENTRIES, 'src/testing')
-    self.assertEquals(len(entries), 2)
+    self.assertEqual(len(entries), 2)
 
   def testGetMatchingDepsEntriesHandlesTwoPathsWithIdenticalFirstParts(self):
     entries = GetMatchingDepsEntries(DEPS_ENTRIES, 'src/build')
-    self.assertEquals(len(entries), 1)
+    self.assertEqual(len(entries), 1)
-    self.assertEquals(entries[0], DEPS_ENTRIES['src/build'])
+    self.assertEqual(entries[0], DEPS_ENTRIES['src/build'])
 
   def testCalculateChangedDeps(self):
     _SetupGitLsRemoteCall(self.fake,
@@ -130,14 +129,14 @@ class TestRollChromiumRevision(unittest.TestCase):
     libyuv_deps = ParseLocalDepsFile(self._libyuv_depsfile)
     new_cr_deps = ParseLocalDepsFile(self._new_cr_depsfile)
     changed_deps = CalculateChangedDeps(libyuv_deps, new_cr_deps)
-    self.assertEquals(len(changed_deps), 2)
+    self.assertEqual(len(changed_deps), 2)
-    self.assertEquals(changed_deps[0].path, 'src/build')
+    self.assertEqual(changed_deps[0].path, 'src/build')
-    self.assertEquals(changed_deps[0].current_rev, BUILD_OLD_REV)
+    self.assertEqual(changed_deps[0].current_rev, BUILD_OLD_REV)
-    self.assertEquals(changed_deps[0].new_rev, BUILD_NEW_REV)
+    self.assertEqual(changed_deps[0].new_rev, BUILD_NEW_REV)
 
-    self.assertEquals(changed_deps[1].path, 'src/buildtools')
+    self.assertEqual(changed_deps[1].path, 'src/buildtools')
-    self.assertEquals(changed_deps[1].current_rev, BUILDTOOLS_OLD_REV)
+    self.assertEqual(changed_deps[1].current_rev, BUILDTOOLS_OLD_REV)
-    self.assertEquals(changed_deps[1].new_rev, BUILDTOOLS_NEW_REV)
+    self.assertEqual(changed_deps[1].new_rev, BUILDTOOLS_NEW_REV)
 
 
 def _SetupGitLsRemoteCall(cmd_fake, url, revision):

tools_libyuv/get_landmines.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+
 # Copyright 2016 The LibYuv Project Authors. All rights reserved.
 #
 # Use of this source code is governed by a BSD-style license