Generate buildbot json tests: Split mixin tests
This CL moves tests for the mixins feature into its own suite. It also
moves the test data inputs for those tests near this.
Change-Id: Id0e2c97ee59497e239231cc53e300d2f1f655b5c
Reviewed-on: https://chromium-review.googlesource.com/c/1303046
Reviewed-by: Kenneth Russell <[email protected]>
Commit-Queue: Stephen Martinis <[email protected]>
Cr-Commit-Position: refs/heads/master@{#603568}diff --git a/testing/buildbot/generate_buildbot_json_unittest.py b/testing/buildbot/generate_buildbot_json_unittest.py
index 09da7b92..d50b045 100755
--- a/testing/buildbot/generate_buildbot_json_unittest.py
+++ b/testing/buildbot/generate_buildbot_json_unittest.py
@@ -124,165 +124,6 @@
]
"""
-FOO_GTESTS_WATERFALL_MIXIN_WATERFALL = """\
-[
- {
- 'mixins': ['waterfall_mixin'],
- 'name': 'chromium.test',
- 'machines': {
- 'Fake Tester': {
- 'swarming': {},
- 'test_suites': {
- 'gtest_tests': 'foo_tests',
- },
- },
- },
- },
-]
-"""
-
-FOO_GTESTS_BUILDER_MIXIN_WATERFALL = """\
-[
- {
- 'name': 'chromium.test',
- 'machines': {
- 'Fake Tester': {
- 'mixins': ['builder_mixin'],
- 'swarming': {},
- 'test_suites': {
- 'gtest_tests': 'foo_tests',
- },
- },
- },
- },
-]
-"""
-
-FOO_GTESTS_BUILDER_MIXIN_NON_SWARMING_WATERFALL = """\
-[
- {
- 'name': 'chromium.test',
- 'machines': {
- 'Fake Tester': {
- 'mixins': ['random_mixin'],
- 'swarming': {},
- 'test_suites': {
- 'gtest_tests': 'foo_tests',
- },
- },
- },
- },
-]
-"""
-
-FOO_GTESTS_DIMENSIONS_MIXIN_WATERFALL = """\
-[
- {
- 'name': 'chromium.test',
- 'machines': {
- 'Fake Tester': {
- 'mixins': ['dimension_mixin'],
- 'swarming': {},
- 'test_suites': {
- 'gtest_tests': 'foo_tests',
- },
- },
- },
- },
-]
-"""
-
-FOO_GPU_TELEMETRY_TEST_DIMENSIONS_WATERFALL = """\
-[
- {
- 'name': 'chromium.test',
- 'machines': {
- 'Fake Tester': {
- 'mixins': ['dimension_mixin'],
- 'os_type': 'win',
- 'browser_config': 'release',
- 'test_suites': {
- 'gpu_telemetry_tests': 'foo_tests',
- },
- },
- },
- },
-]
-"""
-
-# Swarming mixins must be a list, a single string is not allowed.
-FOO_GTESTS_INVALID_LIST_MIXIN_WATERFALL = """\
-[
- {
- 'name': 'chromium.test',
- 'machines': {
- 'Fake Tester': {
- 'mixins': 'dimension_mixin',
- 'swarming': {},
- 'test_suites': {
- 'gtest_tests': 'foo_tests',
- },
- },
- },
- },
-]
-"""
-FOO_GTESTS_INVALID_NOTFOUND_MIXIN_WATERFALL = """\
-[
- {
- 'name': 'chromium.test',
- 'machines': {
- 'Fake Tester': {
- 'mixins': ['nonexistant'],
- 'swarming': {},
- 'test_suites': {
- 'gtest_tests': 'foo_tests',
- },
- },
- },
- },
-]
-"""
-
-FOO_GTESTS_TEST_MIXIN_WATERFALL = """\
-[
- {
- 'mixins': ['waterfall_mixin'],
- 'name': 'chromium.test',
- 'machines': {
- 'Fake Tester': {
- 'swarming': {},
- 'test_suites': {
- 'gtest_tests': 'foo_tests',
- },
- },
- },
- },
-]
-"""
-
-FOO_GTESTS_SORTING_MIXINS_WATERFALL = """\
-[
- {
- 'mixins': ['a_mixin', 'b_mixin', 'c_mixin'],
- 'name': 'chromium.test',
- 'machines': {
- 'Fake Tester': {
- 'swarming': {
- 'dimension_sets': [
- {
- 'kvm': '1',
- },
- ],
- },
- 'test_suites': {
- 'gtest_tests': 'foo_tests',
- },
- },
- },
- },
-]
-"""
FOO_LINUX_GTESTS_WATERFALL = """\
[
@@ -393,22 +234,6 @@
]
"""
-FOO_CTS_WATERFALL_MIXINS = """\
-[
- {
- 'name': 'chromium.test',
- 'machines': {
- 'Fake Tester': {
- 'mixins': ['test_mixin'],
- 'test_suites': {
- 'cts_tests': 'foo_cts_tests',
- },
- },
- },
- },
-]
-"""
-
FOO_INSTRUMENTATION_TEST_WATERFALL = """\
[
{
@@ -619,25 +444,6 @@
}
"""
-FOO_TEST_SUITE_WITH_MIXIN = """\
-{
- 'basic_suites': {
- 'foo_tests': {
- 'foo_test': {
- 'swarming': {
- 'dimension_sets': [
- {
- 'integrity': 'high',
- }
- ],
- 'expiration': 120,
- },
- 'mixins': ['test_mixin'],
- },
- },
- },
-}
-"""
FOO_SCRIPT_SUITE = """\
{
@@ -1393,152 +1199,6 @@
}
"""
-WATERFALL_MIXIN_WATERFALL_OUTPUT = """\
-{
- "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
- "AAAAA2 See generate_buildbot_json.py to make changes": {},
- "Fake Tester": {
- "gtest_tests": [
- {
- "swarming": {
- "can_use_on_swarming_builders": true,
- "dimension_sets": [
- {
- "integrity": "high"
- }
- ],
- "expiration": 120,
- "value": "waterfall"
- },
- "test": "foo_test"
- }
- ]
- }
-}
-"""
-
-WATERFALL_MIXIN_WATERFALL_EXCEPTION_OUTPUT = """\
-{
- "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
- "AAAAA2 See generate_buildbot_json.py to make changes": {},
- "Fake Tester": {
- "gtest_tests": [
- {
- "swarming": {
- "can_use_on_swarming_builders": true,
- "dimension_sets": [
- {
- "integrity": "high"
- }
- ],
- "expiration": 120,
- "value": "exception"
- },
- "test": "foo_test"
- }
- ]
- }
-}
-"""
-
-BUILDER_MIXIN_WATERFALL_OUTPUT = """\
-{
- "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
- "AAAAA2 See generate_buildbot_json.py to make changes": {},
- "Fake Tester": {
- "gtest_tests": [
- {
- "swarming": {
- "can_use_on_swarming_builders": true,
- "dimension_sets": [
- {
- "integrity": "high"
- }
- ],
- "expiration": 120,
- "value": "builder"
- },
- "test": "foo_test"
- }
- ]
- }
-}
-"""
-
-BUILDER_MIXIN_NON_SWARMING_WATERFALL_OUTPUT = """\
-{
- "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
- "AAAAA2 See generate_buildbot_json.py to make changes": {},
- "Fake Tester": {
- "gtest_tests": [
- {
- "swarming": {
- "can_use_on_swarming_builders": true,
- "dimension_sets": [
- {
- "integrity": "high"
- }
- ],
- "expiration": 120
- },
- "test": "foo_test",
- "value": "random"
- }
- ]
- }
-}
-"""
-
-TEST_MIXIN_WATERFALL_OUTPUT = """\
-{
- "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
- "AAAAA2 See generate_buildbot_json.py to make changes": {},
- "Fake Tester": {
- "gtest_tests": [
- {
- "swarming": {
- "can_use_on_swarming_builders": true,
- "dimension_sets": [
- {
- "integrity": "high",
- "kvm": "1"
- }
- ],
- "expiration": 120,
- "value": "test"
- },
- "test": "foo_test"
- }
- ]
- }
-}
-"""
-
-DIMENSIONS_MIXIN_WATERFALL_OUTPUT = """\
-{
- "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
- "AAAAA2 See generate_buildbot_json.py to make changes": {},
- "Fake Tester": {
- "gtest_tests": [
- {
- "swarming": {
- "can_use_on_swarming_builders": true,
- "dimension_sets": [
- {
- "iama": "mixin",
- "integrity": "high"
- }
- ],
- "expiration": 120,
- "value": "test"
- },
- "test": "foo_test"
- }
- ]
- }
-}
-"""
-
GPU_DIMENSIONS_WATERFALL_OUTPUT = """\
{
"AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
@@ -1591,77 +1251,6 @@
}
"""
-# These mixins are invalid; if passed to check_input_file_consistency, they will
-# fail. These are used for output file consistency checks.
-SWARMING_MIXINS = """\
-{
- 'builder_mixin': {
- 'swarming': {
- 'value': 'builder',
- },
- },
- 'dimension_mixin': {
- 'swarming': {
- 'dimensions': {
- 'iama': 'mixin',
- },
- },
- },
- 'random_mixin': {
- 'value': 'random',
- },
- 'test_mixin': {
- 'swarming': {
- 'value': 'test',
- },
- },
- 'waterfall_mixin': {
- 'swarming': {
- 'value': 'waterfall',
- },
- },
-}
-"""
-
-SWARMING_MIXINS_DUPLICATED = """\
-{
- 'builder_mixin': {
- 'value': 'builder',
- },
- 'builder_mixin': {
- 'value': 'builder',
- },
-}
-"""
-
-SWARMING_MIXINS_UNSORTED = """\
-{
- 'b_mixin': {
- 'b': 'b',
- },
- 'a_mixin': {
- 'a': 'a',
- },
- 'c_mixin': {
- 'c': 'c',
- },
-}
-"""
-
-SWARMING_MIXINS_SORTED = """\
-{
- 'a_mixin': {
- 'a': 'a',
- },
- 'b_mixin': {
- 'b': 'b',
- },
- 'c_mixin': {
- 'c': 'c',
- },
-}
-"""
-
TEST_SUITE_SORTING_WATERFALL = """
[
{
@@ -2247,145 +1836,6 @@
fbb.check_input_file_consistency(verbose=True)
self.assertFalse(fbb.printed_lines)
- def test_mixins_waterfall(self):
- fbb = FakeBBGen(FOO_GTESTS_WATERFALL_MIXIN_WATERFALL,
- FOO_TEST_SUITE,
- EMPTY_PYL_FILE,
- SWARMING_MIXINS,
- LUCI_MILO_CFG)
- fbb.files['chromium.test.json'] = WATERFALL_MIXIN_WATERFALL_OUTPUT
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
-
- def test_mixins_waterfall_exception_overrides(self):
- fbb = FakeBBGen(FOO_GTESTS_WATERFALL_MIXIN_WATERFALL,
- FOO_TEST_SUITE,
- SCRIPT_WITH_ARGS_SWARMING_EXCEPTIONS,
- SWARMING_MIXINS,
- LUCI_MILO_CFG)
- fbb.files['chromium.test.json'] = WATERFALL_MIXIN_WATERFALL_EXCEPTION_OUTPUT
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
-
- def test_mixins_builder(self):
- fbb = FakeBBGen(FOO_GTESTS_BUILDER_MIXIN_WATERFALL,
- FOO_TEST_SUITE,
- EMPTY_PYL_FILE,
- SWARMING_MIXINS,
- LUCI_MILO_CFG)
- fbb.files['chromium.test.json'] = BUILDER_MIXIN_WATERFALL_OUTPUT
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
-
- def test_mixins_builder_non_swarming(self):
- fbb = FakeBBGen(FOO_GTESTS_BUILDER_MIXIN_NON_SWARMING_WATERFALL,
- FOO_TEST_SUITE,
- EMPTY_PYL_FILE,
- SWARMING_MIXINS,
- LUCI_MILO_CFG)
- fbb.files['chromium.test.json'] = (
- BUILDER_MIXIN_NON_SWARMING_WATERFALL_OUTPUT)
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
-
- def test_mixins_test(self):
- fbb = FakeBBGen(FOO_GTESTS_WATERFALL,
- FOO_TEST_SUITE_WITH_MIXIN,
- EMPTY_PYL_FILE,
- SWARMING_MIXINS,
- LUCI_MILO_CFG)
- fbb.files['chromium.test.json'] = TEST_MIXIN_WATERFALL_OUTPUT
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
-
- def test_mixins_dimension(self):
- fbb = FakeBBGen(FOO_GTESTS_DIMENSIONS_MIXIN_WATERFALL,
- FOO_TEST_SUITE_WITH_MIXIN,
- EMPTY_PYL_FILE,
- SWARMING_MIXINS,
- LUCI_MILO_CFG)
- fbb.files['chromium.test.json'] = DIMENSIONS_MIXIN_WATERFALL_OUTPUT
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
-
- def test_mixins_dimension_gpu(self):
- fbb = FakeBBGen(FOO_GPU_TELEMETRY_TEST_DIMENSIONS_WATERFALL,
- FOO_TEST_SUITE_WITH_MIXIN,
- EMPTY_PYL_FILE,
- SWARMING_MIXINS,
- LUCI_MILO_CFG)
- fbb.files['chromium.test.json'] = GPU_DIMENSIONS_WATERFALL_OUTPUT
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
-
- def test_mixins_unreferenced(self):
- fbb = FakeBBGen(FOO_GTESTS_WATERFALL,
- FOO_TEST_SUITE_WITH_MIXIN,
- EMPTY_PYL_FILE,
- SWARMING_MIXINS,
- LUCI_MILO_CFG)
- with self.assertRaisesRegexp(generate_buildbot_json.BBGenErr,
- '.*mixins are unreferenced.*'):
- fbb.check_input_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
-
- def test_mixins_cts(self):
- fbb = FakeBBGen(FOO_CTS_WATERFALL_MIXINS,
- FOO_CTS_SUITE ,
- EMPTY_PYL_FILE,
- EMPTY_PYL_FILE,
- LUCI_MILO_CFG)
- fbb.files['chromium.test.json'] = CTS_OUTPUT
- fbb.check_input_file_consistency(verbose=True)
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
-
- def test_mixins_unused(self):
- fbb = FakeBBGen(FOO_GTESTS_INVALID_NOTFOUND_MIXIN_WATERFALL,
- FOO_TEST_SUITE_WITH_MIXIN,
- EMPTY_PYL_FILE,
- SWARMING_MIXINS,
- LUCI_MILO_CFG)
- fbb.files['chromium.test.json'] = DIMENSIONS_MIXIN_WATERFALL_OUTPUT
- with self.assertRaises(generate_buildbot_json.BBGenErr):
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
-
- def test_mixins_list(self):
- fbb = FakeBBGen(FOO_GTESTS_INVALID_LIST_MIXIN_WATERFALL,
- FOO_TEST_SUITE_WITH_MIXIN,
- EMPTY_PYL_FILE,
- SWARMING_MIXINS,
- LUCI_MILO_CFG)
- fbb.files['chromium.test.json'] = DIMENSIONS_MIXIN_WATERFALL_OUTPUT
- with self.assertRaises(generate_buildbot_json.BBGenErr):
- fbb.check_output_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
-
- def test_mixins_must_be_sorted(self):
- fbb = FakeBBGen(FOO_GTESTS_SORTING_MIXINS_WATERFALL,
- FOO_TEST_SUITE,
- EMPTY_PYL_FILE,
- SWARMING_MIXINS_SORTED,
- LUCI_MILO_CFG)
- fbb.check_input_file_consistency(verbose=True)
- self.assertFalse(fbb.printed_lines)
-
- fbb = FakeBBGen(FOO_GTESTS_SORTING_MIXINS_WATERFALL,
- FOO_TEST_SUITE,
- EMPTY_PYL_FILE,
- SWARMING_MIXINS_UNSORTED,
- LUCI_MILO_CFG)
- with self.assertRaises(generate_buildbot_json.BBGenErr):
- fbb.check_input_file_consistency(verbose=True)
- joined_lines = ' '.join(fbb.printed_lines)
- self.assertRegexpMatches(
- joined_lines, '.*\+._mixin.*')
- self.assertRegexpMatches(
- joined_lines, '.*\-._mixin.*')
- fbb.printed_lines = []
- self.assertFalse(fbb.printed_lines)
-
def test_waterfalls_must_be_sorted(self):
fbb = FakeBBGen(TEST_SUITE_SORTED_WATERFALL,
TEST_SUITE_SORTED,
@@ -2482,7 +1932,563 @@
fbb.printed_lines = []
self.assertFalse(fbb.printed_lines)
- def test_mixins_no_duplicate_keys(self):
+
+FOO_GTESTS_WATERFALL_MIXIN_WATERFALL = """\
+[
+ {
+ 'mixins': ['waterfall_mixin'],
+ 'name': 'chromium.test',
+ 'machines': {
+ 'Fake Tester': {
+ 'swarming': {},
+ 'test_suites': {
+ 'gtest_tests': 'foo_tests',
+ },
+ },
+ },
+ },
+]
+"""
+
+FOO_GTESTS_BUILDER_MIXIN_WATERFALL = """\
+[
+ {
+ 'name': 'chromium.test',
+ 'machines': {
+ 'Fake Tester': {
+ 'mixins': ['builder_mixin'],
+ 'swarming': {},
+ 'test_suites': {
+ 'gtest_tests': 'foo_tests',
+ },
+ },
+ },
+ },
+]
+"""
+
+FOO_GTESTS_BUILDER_MIXIN_NON_SWARMING_WATERFALL = """\
+[
+ {
+ 'name': 'chromium.test',
+ 'machines': {
+ 'Fake Tester': {
+ 'mixins': ['random_mixin'],
+ 'swarming': {},
+ 'test_suites': {
+ 'gtest_tests': 'foo_tests',
+ },
+ },
+ },
+ },
+]
+"""
+
+FOO_GTESTS_DIMENSIONS_MIXIN_WATERFALL = """\
+[
+ {
+ 'name': 'chromium.test',
+ 'machines': {
+ 'Fake Tester': {
+ 'mixins': ['dimension_mixin'],
+ 'swarming': {},
+ 'test_suites': {
+ 'gtest_tests': 'foo_tests',
+ },
+ },
+ },
+ },
+]
+"""
+
+FOO_GPU_TELEMETRY_TEST_DIMENSIONS_WATERFALL = """\
+[
+ {
+ 'name': 'chromium.test',
+ 'machines': {
+ 'Fake Tester': {
+ 'mixins': ['dimension_mixin'],
+ 'os_type': 'win',
+ 'browser_config': 'release',
+ 'test_suites': {
+ 'gpu_telemetry_tests': 'foo_tests',
+ },
+ },
+ },
+ },
+]
+"""
+
+# Swarming mixins must be a list, a single string is not allowed.
+FOO_GTESTS_INVALID_LIST_MIXIN_WATERFALL = """\
+[
+ {
+ 'name': 'chromium.test',
+ 'machines': {
+ 'Fake Tester': {
+ 'mixins': 'dimension_mixin',
+ 'swarming': {},
+ 'test_suites': {
+ 'gtest_tests': 'foo_tests',
+ },
+ },
+ },
+ },
+]
+"""
+FOO_GTESTS_INVALID_NOTFOUND_MIXIN_WATERFALL = """\
+[
+ {
+ 'name': 'chromium.test',
+ 'machines': {
+ 'Fake Tester': {
+ 'mixins': ['nonexistant'],
+ 'swarming': {},
+ 'test_suites': {
+ 'gtest_tests': 'foo_tests',
+ },
+ },
+ },
+ },
+]
+"""
+
+FOO_GTESTS_TEST_MIXIN_WATERFALL = """\
+[
+ {
+ 'mixins': ['waterfall_mixin'],
+ 'name': 'chromium.test',
+ 'machines': {
+ 'Fake Tester': {
+ 'swarming': {},
+ 'test_suites': {
+ 'gtest_tests': 'foo_tests',
+ },
+ },
+ },
+ },
+]
+"""
+
+FOO_GTESTS_SORTING_MIXINS_WATERFALL = """\
+[
+ {
+ 'mixins': ['a_mixin', 'b_mixin', 'c_mixin'],
+ 'name': 'chromium.test',
+ 'machines': {
+ 'Fake Tester': {
+ 'swarming': {
+ 'dimension_sets': [
+ {
+ 'kvm': '1',
+ },
+ ],
+ },
+ 'test_suites': {
+ 'gtest_tests': 'foo_tests',
+ },
+ },
+ },
+ },
+]
+"""
+
+FOO_TEST_SUITE_WITH_MIXIN = """\
+{
+ 'basic_suites': {
+ 'foo_tests': {
+ 'foo_test': {
+ 'swarming': {
+ 'dimension_sets': [
+ {
+ 'integrity': 'high',
+ }
+ ],
+ 'expiration': 120,
+ },
+ 'mixins': ['test_mixin'],
+ },
+ },
+ },
+}
+"""
+
+# These mixins are invalid; if passed to check_input_file_consistency, they will
+# fail. These are used for output file consistency checks.
+SWARMING_MIXINS = """\
+{
+ 'builder_mixin': {
+ 'swarming': {
+ 'value': 'builder',
+ },
+ },
+ 'dimension_mixin': {
+ 'swarming': {
+ 'dimensions': {
+ 'iama': 'mixin',
+ },
+ },
+ },
+ 'random_mixin': {
+ 'value': 'random',
+ },
+ 'test_mixin': {
+ 'swarming': {
+ 'value': 'test',
+ },
+ },
+ 'waterfall_mixin': {
+ 'swarming': {
+ 'value': 'waterfall',
+ },
+ },
+}
+"""
+
+SWARMING_MIXINS_DUPLICATED = """\
+{
+ 'builder_mixin': {
+ 'value': 'builder',
+ },
+ 'builder_mixin': {
+ 'value': 'builder',
+ },
+}
+"""
+
+SWARMING_MIXINS_UNSORTED = """\
+{
+ 'b_mixin': {
+ 'b': 'b',
+ },
+ 'a_mixin': {
+ 'a': 'a',
+ },
+ 'c_mixin': {
+ 'c': 'c',
+ },
+}
+"""
+
+SWARMING_MIXINS_SORTED = """\
+{
+ 'a_mixin': {
+ 'a': 'a',
+ },
+ 'b_mixin': {
+ 'b': 'b',
+ },
+ 'c_mixin': {
+ 'c': 'c',
+ },
+}
+"""
+
+FOO_CTS_WATERFALL_MIXINS = """\
+[
+ {
+ 'name': 'chromium.test',
+ 'machines': {
+ 'Fake Tester': {
+ 'mixins': ['test_mixin'],
+ 'test_suites': {
+ 'cts_tests': 'foo_cts_tests',
+ },
+ },
+ },
+ },
+]
+"""
+
+WATERFALL_MIXIN_WATERFALL_OUTPUT = """\
+{
+ "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
+ "AAAAA2 See generate_buildbot_json.py to make changes": {},
+ "Fake Tester": {
+ "gtest_tests": [
+ {
+ "swarming": {
+ "can_use_on_swarming_builders": true,
+ "dimension_sets": [
+ {
+ "integrity": "high"
+ }
+ ],
+ "expiration": 120,
+ "value": "waterfall"
+ },
+ "test": "foo_test"
+ }
+ ]
+ }
+}
+"""
+
+WATERFALL_MIXIN_WATERFALL_EXCEPTION_OUTPUT = """\
+{
+ "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
+ "AAAAA2 See generate_buildbot_json.py to make changes": {},
+ "Fake Tester": {
+ "gtest_tests": [
+ {
+ "swarming": {
+ "can_use_on_swarming_builders": true,
+ "dimension_sets": [
+ {
+ "integrity": "high"
+ }
+ ],
+ "expiration": 120,
+ "value": "exception"
+ },
+ "test": "foo_test"
+ }
+ ]
+ }
+}
+"""
+
+BUILDER_MIXIN_WATERFALL_OUTPUT = """\
+{
+ "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
+ "AAAAA2 See generate_buildbot_json.py to make changes": {},
+ "Fake Tester": {
+ "gtest_tests": [
+ {
+ "swarming": {
+ "can_use_on_swarming_builders": true,
+ "dimension_sets": [
+ {
+ "integrity": "high"
+ }
+ ],
+ "expiration": 120,
+ "value": "builder"
+ },
+ "test": "foo_test"
+ }
+ ]
+ }
+}
+"""
+
+BUILDER_MIXIN_NON_SWARMING_WATERFALL_OUTPUT = """\
+{
+ "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
+ "AAAAA2 See generate_buildbot_json.py to make changes": {},
+ "Fake Tester": {
+ "gtest_tests": [
+ {
+ "swarming": {
+ "can_use_on_swarming_builders": true,
+ "dimension_sets": [
+ {
+ "integrity": "high"
+ }
+ ],
+ "expiration": 120
+ },
+ "test": "foo_test",
+ "value": "random"
+ }
+ ]
+ }
+}
+"""
+
+TEST_MIXIN_WATERFALL_OUTPUT = """\
+{
+ "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
+ "AAAAA2 See generate_buildbot_json.py to make changes": {},
+ "Fake Tester": {
+ "gtest_tests": [
+ {
+ "swarming": {
+ "can_use_on_swarming_builders": true,
+ "dimension_sets": [
+ {
+ "integrity": "high",
+ "kvm": "1"
+ }
+ ],
+ "expiration": 120,
+ "value": "test"
+ },
+ "test": "foo_test"
+ }
+ ]
+ }
+}
+"""
+
+DIMENSIONS_MIXIN_WATERFALL_OUTPUT = """\
+{
+ "AAAAA1 AUTOGENERATED FILE DO NOT EDIT": {},
+ "AAAAA2 See generate_buildbot_json.py to make changes": {},
+ "Fake Tester": {
+ "gtest_tests": [
+ {
+ "swarming": {
+ "can_use_on_swarming_builders": true,
+ "dimension_sets": [
+ {
+ "iama": "mixin",
+ "integrity": "high"
+ }
+ ],
+ "expiration": 120,
+ "value": "test"
+ },
+ "test": "foo_test"
+ }
+ ]
+ }
+}
+"""
+
+class MixinTests(unittest.TestCase):
+ """Tests for the mixins feature."""
+ def test_mixins_must_be_sorted(self):
+ fbb = FakeBBGen(FOO_GTESTS_SORTING_MIXINS_WATERFALL,
+ FOO_TEST_SUITE,
+ EMPTY_PYL_FILE,
+ SWARMING_MIXINS_SORTED,
+ LUCI_MILO_CFG)
+ fbb.check_input_file_consistency(verbose=True)
+ self.assertFalse(fbb.printed_lines)
+
+ fbb = FakeBBGen(FOO_GTESTS_SORTING_MIXINS_WATERFALL,
+ FOO_TEST_SUITE,
+ EMPTY_PYL_FILE,
+ SWARMING_MIXINS_UNSORTED,
+ LUCI_MILO_CFG)
+ with self.assertRaises(generate_buildbot_json.BBGenErr):
+ fbb.check_input_file_consistency(verbose=True)
+ joined_lines = ' '.join(fbb.printed_lines)
+ self.assertRegexpMatches(
+ joined_lines, '.*\+._mixin.*')
+ self.assertRegexpMatches(
+ joined_lines, '.*\-._mixin.*')
+ fbb.printed_lines = []
+ self.assertFalse(fbb.printed_lines)
+
+ def test_waterfall(self):
+ fbb = FakeBBGen(FOO_GTESTS_WATERFALL_MIXIN_WATERFALL,
+ FOO_TEST_SUITE,
+ EMPTY_PYL_FILE,
+ SWARMING_MIXINS,
+ LUCI_MILO_CFG)
+ fbb.files['chromium.test.json'] = WATERFALL_MIXIN_WATERFALL_OUTPUT
+ fbb.check_output_file_consistency(verbose=True)
+ self.assertFalse(fbb.printed_lines)
+
+ def test_waterfall_exception_overrides(self):
+ fbb = FakeBBGen(FOO_GTESTS_WATERFALL_MIXIN_WATERFALL,
+ FOO_TEST_SUITE,
+ SCRIPT_WITH_ARGS_SWARMING_EXCEPTIONS,
+ SWARMING_MIXINS,
+ LUCI_MILO_CFG)
+ fbb.files['chromium.test.json'] = WATERFALL_MIXIN_WATERFALL_EXCEPTION_OUTPUT
+ fbb.check_output_file_consistency(verbose=True)
+ self.assertFalse(fbb.printed_lines)
+
+ def test_builder(self):
+ fbb = FakeBBGen(FOO_GTESTS_BUILDER_MIXIN_WATERFALL,
+ FOO_TEST_SUITE,
+ EMPTY_PYL_FILE,
+ SWARMING_MIXINS,
+ LUCI_MILO_CFG)
+ fbb.files['chromium.test.json'] = BUILDER_MIXIN_WATERFALL_OUTPUT
+ fbb.check_output_file_consistency(verbose=True)
+ self.assertFalse(fbb.printed_lines)
+
+ def test_builder_non_swarming(self):
+ fbb = FakeBBGen(FOO_GTESTS_BUILDER_MIXIN_NON_SWARMING_WATERFALL,
+ FOO_TEST_SUITE,
+ EMPTY_PYL_FILE,
+ SWARMING_MIXINS,
+ LUCI_MILO_CFG)
+ fbb.files['chromium.test.json'] = (
+ BUILDER_MIXIN_NON_SWARMING_WATERFALL_OUTPUT)
+ fbb.check_output_file_consistency(verbose=True)
+ self.assertFalse(fbb.printed_lines)
+
+ def test_test_suite(self):
+ fbb = FakeBBGen(FOO_GTESTS_WATERFALL,
+ FOO_TEST_SUITE_WITH_MIXIN,
+ EMPTY_PYL_FILE,
+ SWARMING_MIXINS,
+ LUCI_MILO_CFG)
+ fbb.files['chromium.test.json'] = TEST_MIXIN_WATERFALL_OUTPUT
+ fbb.check_output_file_consistency(verbose=True)
+ self.assertFalse(fbb.printed_lines)
+
+ def test_dimension(self):
+ fbb = FakeBBGen(FOO_GTESTS_DIMENSIONS_MIXIN_WATERFALL,
+ FOO_TEST_SUITE_WITH_MIXIN,
+ EMPTY_PYL_FILE,
+ SWARMING_MIXINS,
+ LUCI_MILO_CFG)
+ fbb.files['chromium.test.json'] = DIMENSIONS_MIXIN_WATERFALL_OUTPUT
+ fbb.check_output_file_consistency(verbose=True)
+ self.assertFalse(fbb.printed_lines)
+
+ def test_dimension_gpu(self):
+ fbb = FakeBBGen(FOO_GPU_TELEMETRY_TEST_DIMENSIONS_WATERFALL,
+ FOO_TEST_SUITE_WITH_MIXIN,
+ EMPTY_PYL_FILE,
+ SWARMING_MIXINS,
+ LUCI_MILO_CFG)
+ fbb.files['chromium.test.json'] = GPU_DIMENSIONS_WATERFALL_OUTPUT
+ fbb.check_output_file_consistency(verbose=True)
+ self.assertFalse(fbb.printed_lines)
+
+ def test_unreferenced(self):
+ fbb = FakeBBGen(FOO_GTESTS_WATERFALL,
+ FOO_TEST_SUITE_WITH_MIXIN,
+ EMPTY_PYL_FILE,
+ SWARMING_MIXINS,
+ LUCI_MILO_CFG)
+ with self.assertRaisesRegexp(generate_buildbot_json.BBGenErr,
+ '.*mixins are unreferenced.*'):
+ fbb.check_input_file_consistency(verbose=True)
+ self.assertFalse(fbb.printed_lines)
+
+ def test_cts(self):
+ fbb = FakeBBGen(FOO_CTS_WATERFALL_MIXINS,
+ FOO_CTS_SUITE ,
+ EMPTY_PYL_FILE,
+ EMPTY_PYL_FILE,
+ LUCI_MILO_CFG)
+ fbb.files['chromium.test.json'] = CTS_OUTPUT
+ fbb.check_input_file_consistency(verbose=True)
+ fbb.check_output_file_consistency(verbose=True)
+ self.assertFalse(fbb.printed_lines)
+
+ def test_unused(self):
+ fbb = FakeBBGen(FOO_GTESTS_INVALID_NOTFOUND_MIXIN_WATERFALL,
+ FOO_TEST_SUITE_WITH_MIXIN,
+ EMPTY_PYL_FILE,
+ SWARMING_MIXINS,
+ LUCI_MILO_CFG)
+ fbb.files['chromium.test.json'] = DIMENSIONS_MIXIN_WATERFALL_OUTPUT
+ with self.assertRaises(generate_buildbot_json.BBGenErr):
+ fbb.check_output_file_consistency(verbose=True)
+ self.assertFalse(fbb.printed_lines)
+
+ def test_list(self):
+ fbb = FakeBBGen(FOO_GTESTS_INVALID_LIST_MIXIN_WATERFALL,
+ FOO_TEST_SUITE_WITH_MIXIN,
+ EMPTY_PYL_FILE,
+ SWARMING_MIXINS,
+ LUCI_MILO_CFG)
+ fbb.files['chromium.test.json'] = DIMENSIONS_MIXIN_WATERFALL_OUTPUT
+ with self.assertRaises(generate_buildbot_json.BBGenErr):
+ fbb.check_output_file_consistency(verbose=True)
+ self.assertFalse(fbb.printed_lines)
+
+
+ def test_no_duplicate_keys(self):
fbb = FakeBBGen(FOO_GTESTS_BUILDER_MIXIN_WATERFALL,
FOO_TEST_SUITE,
EMPTY_PYL_FILE,
@@ -2525,6 +2531,5 @@
])
-
if __name__ == '__main__':
unittest.main()
