Update generate_buildbot_json to Python 3.
Also updates the unit tests and coverage tests, and manage.py.
Bug: 1208869
Change-Id: I7e5d33aa87f2865e7a98d78dc10c135779dde46e
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2892631
Commit-Queue: Jamie Madill <[email protected]>
Reviewed-by: Dirk Pranke <[email protected]>
Reviewed-by: Brian Sheedy <[email protected]>
Cr-Commit-Position: refs/heads/master@{#885155}
diff --git a/testing/buildbot/generate_buildbot_json.py b/testing/buildbot/generate_buildbot_json.py
index 90072db..377273fd 100755
--- a/testing/buildbot/generate_buildbot_json.py
+++ b/testing/buildbot/generate_buildbot_json.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# Copyright 2016 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
@@ -12,6 +12,7 @@
import collections
import copy
import difflib
+import functools
import glob
import itertools
import json
@@ -50,13 +51,17 @@
raise NotImplementedError()
+def custom_cmp(a, b):
+ return int(a > b) - int(a < b)
+
+
def cmp_tests(a, b):
# Prefer to compare based on the "test" key.
- val = cmp(a['test'], b['test'])
+ val = custom_cmp(a['test'], b['test'])
if val != 0:
return val
if 'name' in a and 'name' in b:
- return cmp(a['name'], b['name']) # pragma: no cover
+ return custom_cmp(a['name'], b['name']) # pragma: no cover
if 'name' not in a and 'name' not in b:
return 0 # pragma: no cover
# Prefer to put variants of the same test after the first one.
@@ -74,7 +79,7 @@
def generate(self, waterfall, tester_name, tester_config, input_tests):
isolated_scripts = []
- for test_name, test_config in sorted(input_tests.iteritems()):
+ for test_name, test_config in sorted(input_tests.items()):
test = self.bb_gen.generate_gpu_telemetry_test(
waterfall, tester_name, tester_config, test_name, test_config,
self._is_android_webview)
@@ -97,7 +102,7 @@
# key (see gles2_conform_d3d9_test and similar variants). Avoid
# losing the order by avoiding coalescing the dictionaries into one.
gtests = []
- for test_name, test_config in sorted(input_tests.iteritems()):
+ for test_name, test_config in sorted(input_tests.items()):
# Variants allow more than one definition for a given test, and is defined
# in array format from resolve_variants().
if not isinstance(test_config, list):
@@ -112,7 +117,7 @@
return gtests
def sort(self, tests):
- return sorted(tests, cmp=cmp_tests)
+ return sorted(tests, key=functools.cmp_to_key(cmp_tests))
class IsolatedScriptTestGenerator(BaseGenerator):
@@ -121,7 +126,7 @@
def generate(self, waterfall, tester_name, tester_config, input_tests):
isolated_scripts = []
- for test_name, test_config in sorted(input_tests.iteritems()):
+ for test_name, test_config in sorted(input_tests.items()):
# Variants allow more than one definition for a given test, and is defined
# in array format from resolve_variants().
if not isinstance(test_config, list):
@@ -144,7 +149,7 @@
def generate(self, waterfall, tester_name, tester_config, input_tests):
scripts = []
- for test_name, test_config in sorted(input_tests.iteritems()):
+ for test_name, test_config in sorted(input_tests.items()):
test = self.bb_gen.generate_script_test(
waterfall, tester_name, tester_config, test_name, test_config)
if test:
@@ -161,7 +166,7 @@
def generate(self, waterfall, tester_name, tester_config, input_tests):
scripts = []
- for test_name, test_config in sorted(input_tests.iteritems()):
+ for test_name, test_config in sorted(input_tests.items()):
test = self.bb_gen.generate_junit_test(
waterfall, tester_name, tester_config, test_name, test_config)
if test:
@@ -178,7 +183,7 @@
def generate(self, waterfall, tester_name, tester_config, input_tests):
scripts = []
- for test_name, test_config in sorted(input_tests.iteritems()):
+ for test_name, test_config in sorted(input_tests.items()):
for config in test_config:
test = self.bb_gen.generate_skylab_test(waterfall, tester_name,
tester_config, test_name,
@@ -357,7 +362,7 @@
def print_line(self, line):
# Exists so that tests can mock
- print line # pragma: no cover
+ print(line) # pragma: no cover
def read_file(self, relative_path):
with open(self.generate_abs_file_path(relative_path)) as fp:
@@ -365,7 +370,7 @@
def write_file(self, relative_path, contents):
with open(self.generate_abs_file_path(relative_path), 'wb') as fp:
- fp.write(contents)
+ fp.write(contents.encode('utf-8'))
def pyl_file_path(self, filename):
if self.args and self.args.pyl_files_dir:
@@ -545,7 +550,7 @@
# swarming dimension_sets that we have to merge. It will fail
# to merge / override 'args' arrays which are different
# length.
- for idx in xrange(len(b[key])):
+ for idx in range(len(b[key])):
try:
a[key][idx] = self.dictionary_merge(a[key][idx], b[key][idx],
path + [str(key), str(idx)],
@@ -637,7 +642,7 @@
del swarming_dict['hard_timeout'] # pragma: no cover
if not swarming_dict.get('can_use_on_swarming_builders', False):
# Remove all other keys.
- for k in swarming_dict.keys(): # pragma: no cover
+ for k in list(swarming_dict): # pragma: no cover
if k != 'can_use_on_swarming_builders': # pragma: no cover
del swarming_dict[k] # pragma: no cover
@@ -667,12 +672,12 @@
replacements = self.get_test_replacements(
test, test_name, tester_name) or {}
valid_replacement_keys = ['args', 'non_precommit_args', 'precommit_args']
- for key, replacement_dict in replacements.iteritems():
+ for key, replacement_dict in replacements.items():
if key not in valid_replacement_keys:
raise BBGenErr(
'Given replacement key %s for %s on %s is not in the list of valid '
'keys %s' % (key, test_name, tester_name, valid_replacement_keys))
- for replacement_key, replacement_val in replacement_dict.iteritems():
+ for replacement_key, replacement_val in replacement_dict.items():
found_key = False
for i, test_key in enumerate(test.get(key, [])):
# Handle both the key/value being replaced being defined as two
@@ -1006,7 +1011,7 @@
other_suites = self.test_suites.get(other_test_type, {})
basic_suites = self.test_suites.get('basic_suites', {})
- for suite, suite_def in target_suites.iteritems():
+ for suite, suite_def in target_suites.items():
if suite in basic_suites:
raise BBGenErr('%s names may not duplicate basic test suite names '
'(error found while processsing %s)'
@@ -1031,13 +1036,13 @@
new_test_suites = {}
test_types = ['basic_suites', 'compound_suites', 'matrix_compound_suites']
for category in test_types:
- for name, value in self.test_suites.get(category, {}).iteritems():
+ for name, value in self.test_suites.get(category, {}).items():
new_test_suites[name] = value
self.test_suites = new_test_suites
def resolve_test_id_prefixes(self):
- for suite in self.test_suites['basic_suites'].itervalues():
- for key, test in suite.iteritems():
+ for suite in self.test_suites['basic_suites'].values():
+ for key, test in suite.items():
assert isinstance(test, dict)
# This assumes the recipe logic which prefers 'test' to 'isolate_name'
@@ -1074,7 +1079,7 @@
# referenced by compound suites exist.
basic_suites = self.test_suites.get('basic_suites')
- for name, value in compound_suites.iteritems():
+ for name, value in compound_suites.items():
# Resolve this to a dictionary.
full_suite = {}
for entry in value:
@@ -1103,7 +1108,7 @@
# Each test in a basic test suite will have a definition per variant.
test_suite = {}
- for test_name, test_config in basic_test_definition.iteritems():
+ for test_name, test_config in basic_test_definition.items():
definitions = []
for variant in variants:
# Unpack the variant from variants.pyl if it's string based.
@@ -1164,10 +1169,10 @@
# referenced by matrix suites exist.
basic_suites = self.test_suites.get('basic_suites')
- for test_name, matrix_config in matrix_compound_suites.iteritems():
+ for test_name, matrix_config in matrix_compound_suites.items():
full_suite = {}
- for test_suite, mtx_test_suite_config in matrix_config.iteritems():
+ for test_suite, mtx_test_suite_config in matrix_config.items():
basic_test_def = copy.deepcopy(basic_suites[test_suite])
if 'variants' in mtx_test_suite_config:
@@ -1180,8 +1185,8 @@
def link_waterfalls_to_test_suites(self):
for waterfall in self.waterfalls:
- for tester_name, tester in waterfall['machines'].iteritems():
- for suite, value in tester.get('test_suites', {}).iteritems():
+ for tester_name, tester in waterfall['machines'].items():
+ for suite, value in tester.get('test_suites', {}).items():
if not value in self.test_suites:
# Hard / impossible to cover this in the unit test.
raise self.unknown_test_suite(
@@ -1370,9 +1375,8 @@
A dictionary mapping builders to test specs
"""
return {
- name: self.get_tests_for_config(waterfall, name, config)
- for name, config
- in waterfall['machines'].iteritems()
+ name: self.get_tests_for_config(waterfall, name, config)
+ for name, config in waterfall['machines'].items()
}
def get_tests_for_config(self, waterfall, name, config):
@@ -1385,7 +1389,7 @@
if 'additional_compile_targets' in config:
tests['additional_compile_targets'] = config[
'additional_compile_targets']
- for test_type, input_tests in config.get('test_suites', {}).iteritems():
+ for test_type, input_tests in config.get('test_suites', {}).items():
if test_type not in generator_map:
raise self.unknown_test_suite_type(
test_type, name, waterfall['name']) # pragma: no cover
@@ -1581,8 +1585,8 @@
suites_seen = set()
generator_map = self.get_test_generator_map()
for waterfall in self.waterfalls:
- for bot_name, tester in waterfall['machines'].iteritems():
- for suite_type, suite in tester.get('test_suites', {}).iteritems():
+ for bot_name, tester in waterfall['machines'].items():
+ for suite_type, suite in tester.get('test_suites', {}).items():
if suite_type not in generator_map:
raise self.unknown_test_suite_type(suite_type, bot_name,
waterfall['name'])
@@ -1607,17 +1611,17 @@
all_bots = set()
missing_bots = set()
for waterfall in self.waterfalls:
- for bot_name, tester in waterfall['machines'].iteritems():
+ for bot_name, tester in waterfall['machines'].items():
all_bots.add(bot_name)
# In order to disambiguate between bots with the same name on
# different waterfalls, support has been added to various
# exceptions for concatenating the waterfall name after the bot
# name.
all_bots.add(bot_name + ' ' + waterfall['name'])
- for exception in self.exceptions.itervalues():
+ for exception in self.exceptions.values():
removals = (exception.get('remove_from', []) +
exception.get('remove_gtest_from', []) +
- exception.get('modifications', {}).keys())
+ list(exception.get('modifications', {}).keys()))
for removal in removals:
if removal not in all_bots:
missing_bots.add(removal)
@@ -1631,7 +1635,7 @@
seen_mixins = set()
for waterfall in self.waterfalls:
seen_mixins = seen_mixins.union(waterfall.get('mixins', set()))
- for bot_name, tester in waterfall['machines'].iteritems():
+ for bot_name, tester in waterfall['machines'].items():
seen_mixins = seen_mixins.union(tester.get('mixins', set()))
for suite in self.test_suites.values():
if isinstance(suite, list):
@@ -2090,7 +2094,7 @@
configuration, not grouped by their test suite.
"""
tests = {}
- for test_suite in test_suites.itervalues():
+ for test_suite in test_suites.values():
for test in test_suite:
test_info = test_suite[test]
test_name = test
