| #!/usr/bin/env python3 |
| # Copyright 2020 The Chromium Authors |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| import unittest |
| |
| # //testing/buildbot imports. |
| import buildbot_json_magic_substitutions as magic_substitutions |
| |
| |
| def CreateConfigWithPool(pool, device_type=None, board=None): |
| dims = { |
| 'name': 'test_name', |
| 'swarming': { |
| 'dimensions': { |
| 'pool': pool, |
| }, |
| }, |
| } |
| if device_type: |
| dims['swarming']['dimensions']['device_type'] = device_type |
| if board: |
| dims['swarming']['dimensions']['label-board'] = board |
| return dims |
| |
| |
| class AndroidDesktopTelemetryRemoteTest(unittest.TestCase): |
| |
| def testNonAndroid(self): |
| test_config = CreateConfigWithPool('chromium.tests', board='brya') |
| with self.assertRaises(AssertionError): |
| magic_substitutions.AndroidDesktopTelemetryRemote(test_config, None, |
| {'os_type': 'linux'}) |
| |
| def testNoBoard(self): |
| test_config = CreateConfigWithPool('chromium.tests') |
| self.assertEqual( |
| magic_substitutions.AndroidDesktopTelemetryRemote( |
| test_config, None, {'os_type': 'android'}), []) |
| |
| def testSuccess(self): |
| test_config = CreateConfigWithPool('chromium.tests', board='brya') |
| self.assertEqual( |
| magic_substitutions.AndroidDesktopTelemetryRemote( |
| test_config, None, {'os_type': 'android'}), [ |
| '--device=variable_lab_dut_hostname', |
| '--connect-to-device-over-network', |
| ]) |
| |
| |
| class ChromeOSTelemetryRemoteTest(unittest.TestCase): |
| |
| def testVirtualMachineSubstitutions(self): |
| test_config = CreateConfigWithPool('chromium.tests.cros.vm') |
| self.assertEqual( |
| magic_substitutions.ChromeOSTelemetryRemote(test_config, None, {}), [ |
| '--remote=127.0.0.1', |
| '--remote-ssh-port=9222', |
| ]) |
| |
| def testPhysicalHardwareSubstitutions(self): |
| test_config = CreateConfigWithPool('chromium.tests', device_type='eve') |
| self.assertEqual( |
| magic_substitutions.ChromeOSTelemetryRemote(test_config, None, {}), |
| ['--remote=variable_chromeos_device_hostname']) |
| |
| def testSkylabSubstitutions(self): |
| tester_config = {'browser_config': 'cros-chrome', 'use_swarming': False} |
| self.assertEqual( |
| magic_substitutions.ChromeOSTelemetryRemote({}, None, tester_config), |
| []) |
| |
| def testNoPool(self): |
| test_config = CreateConfigWithPool(None) |
| with self.assertRaisesRegex(RuntimeError, 'No pool *'): |
| magic_substitutions.ChromeOSTelemetryRemote(test_config, None, {}) |
| |
| def testUnknownPool(self): |
| test_config = CreateConfigWithPool('totally-legit-pool') |
| with self.assertRaisesRegex(RuntimeError, 'Unknown CrOS pool *'): |
| magic_substitutions.ChromeOSTelemetryRemote(test_config, None, {}) |
| |
| |
| class ChromeOSGtestFilterFileTest(unittest.TestCase): |
| def testVirtualMachineFile(self): |
| test_config = CreateConfigWithPool('chromium.tests.cros.vm') |
| self.assertEqual( |
| magic_substitutions.ChromeOSGtestFilterFile(test_config, None, {}), [ |
| '--test-launcher-filter-file=../../testing/buildbot/filters/' |
| 'chromeos.amd64-generic.test_name.filter', |
| ]) |
| |
| def testPhysicalHardwareFile(self): |
| test_config = CreateConfigWithPool('chromium.tests', device_type='eve') |
| self.assertEqual( |
| magic_substitutions.ChromeOSGtestFilterFile(test_config, None, {}), [ |
| '--test-launcher-filter-file=../../testing/buildbot/filters/' |
| 'chromeos.eve.test_name.filter', |
| ]) |
| |
| def testSkylab(self): |
| test_config = {'name': 'test_name', 'cros_board': 'eve'} |
| tester_config = {'browser_config': 'cros-chrome', 'use_swarming': False} |
| self.assertEqual( |
| magic_substitutions.ChromeOSGtestFilterFile(test_config, None, |
| tester_config), |
| [ |
| '--test-launcher-filter-file=../../testing/buildbot/filters/' |
| 'chromeos.eve.test_name.filter', |
| ]) |
| |
| def testSkylabWithVariant(self): |
| test_config = { |
| 'name': 'test_name SOME_VARIANT', |
| 'cros_board': 'eve', |
| 'variant_id': 'SOME_VARIANT', |
| } |
| tester_config = {'browser_config': 'cros-chrome', 'use_swarming': False} |
| self.assertEqual( |
| magic_substitutions.ChromeOSGtestFilterFile(test_config, None, |
| tester_config), |
| [ |
| '--test-launcher-filter-file=../../testing/buildbot/filters/' |
| 'chromeos.eve.test_name.filter', |
| ]) |
| |
| def testNoPool(self): |
| test_config = CreateConfigWithPool(None) |
| with self.assertRaisesRegex(RuntimeError, 'No pool *'): |
| magic_substitutions.ChromeOSGtestFilterFile(test_config, None, {}) |
| |
| def testUnknownPool(self): |
| test_config = CreateConfigWithPool('totally-legit-pool') |
| with self.assertRaisesRegex(RuntimeError, 'Unknown CrOS pool *'): |
| magic_substitutions.ChromeOSGtestFilterFile(test_config, None, {}) |
| |
| |
| def CreateConfigWithGpu(gpu): |
| return { |
| 'swarming': { |
| 'dimensions': { |
| 'gpu': gpu, |
| }, |
| }, |
| } |
| |
| |
| class GPUExpectedVendorId(unittest.TestCase): |
| def testSingleGpuSingleDimension(self): |
| test_config = CreateConfigWithGpu('vendor:device1-driver') |
| self.assertEqual( |
| magic_substitutions.GPUExpectedVendorId(test_config, None, {}), |
| ['--expected-vendor-id', 'vendor']) |
| |
| def testDoubleGpuSingleDimension(self): |
| test_config = CreateConfigWithGpu( |
| 'vendor:device1-driver|vendor:device2-driver') |
| self.assertEqual( |
| magic_substitutions.GPUExpectedVendorId(test_config, None, {}), |
| ['--expected-vendor-id', 'vendor']) |
| |
| def testDoubleGpuSingleDimensionDifferentVendors(self): |
| test_config = CreateConfigWithGpu( |
| 'vendor:device1-driver|vendor2:device2-driver') |
| with self.assertRaises(AssertionError): |
| magic_substitutions.GPUExpectedVendorId(test_config, None, {}) |
| |
| def testAppleSilicon(self): |
| test_config = CreateConfigWithGpu('apple:m1') |
| self.assertEqual( |
| magic_substitutions.GPUExpectedVendorId(test_config, None, {}), |
| ['--expected-vendor-id', '106b']) |
| |
| def testNoGpu(self): |
| test_config = { |
| 'swarming': { |
| 'dimensions': {}, |
| }, |
| } |
| self.assertEqual( |
| magic_substitutions.GPUExpectedVendorId(test_config, None, {}), |
| ['--expected-vendor-id', '0']) |
| |
| def testNoDimensions(self): |
| with self.assertRaises(AssertionError): |
| magic_substitutions.GPUExpectedVendorId({}, None, {}) |
| |
| def testAndroidDesktopKnownBoard(self): |
| test_config = { |
| 'name': 'test_name', |
| 'swarming': { |
| 'dimensions': { |
| 'label-board': 'brya', |
| }, |
| }, |
| } |
| tester_config = { |
| 'os_type': 'android', |
| } |
| self.assertEqual( |
| magic_substitutions.GPUExpectedVendorId(test_config, None, |
| tester_config), |
| ['--expected-vendor-id', '8086']) |
| |
| def testAndroidDesktopUnknownBoard(self): |
| test_config = { |
| 'name': 'test_name', |
| 'swarming': { |
| 'dimensions': { |
| 'label-board': 'fake_board', |
| }, |
| }, |
| } |
| tester_config = { |
| 'os_type': 'android', |
| } |
| self.assertEqual( |
| magic_substitutions.GPUExpectedVendorId(test_config, None, |
| tester_config), |
| ['--expected-vendor-id', '0']) |
| |
| def testSkylabKnownBoard(self): |
| test_config = { |
| 'name': 'test_name', |
| 'cros_board': 'volteer', |
| } |
| tester_config = { |
| 'browser_config': 'cros-chrome', |
| 'use_swarming': False, |
| } |
| self.assertEqual( |
| magic_substitutions.GPUExpectedVendorId(test_config, None, |
| tester_config), |
| ['--expected-vendor-id', '8086']) |
| |
| def testSkylabUnknownBoard(self): |
| test_config = { |
| 'name': 'test_name', |
| 'cros_board': 'fancy_new_board', |
| } |
| tester_config = { |
| 'browser_config': 'cros-chrome', |
| 'use_swarming': False, |
| } |
| self.assertEqual( |
| magic_substitutions.GPUExpectedVendorId(test_config, None, |
| tester_config), |
| ['--expected-vendor-id', '0']) |
| |
| |
| class GPUExpectedDeviceId(unittest.TestCase): |
| def assertDeviceIdCorrectness(self, retval, device_ids): |
| self.assertEqual(len(retval), 2 * len(device_ids)) |
| for i in range(0, len(retval), 2): |
| self.assertEqual(retval[i], '--expected-device-id') |
| for d in device_ids: |
| self.assertIn(d, retval) |
| |
| def testSingleGpuSingleDimension(self): |
| test_config = CreateConfigWithGpu('vendor:device1-driver') |
| self.assertDeviceIdCorrectness( |
| magic_substitutions.GPUExpectedDeviceId(test_config, None, {}), |
| ['device1']) |
| |
| def testDoubleGpuSingleDimension(self): |
| test_config = CreateConfigWithGpu( |
| 'vendor:device1-driver|vendor:device2-driver') |
| self.assertDeviceIdCorrectness( |
| magic_substitutions.GPUExpectedDeviceId(test_config, None, {}), |
| ['device1', 'device2']) |
| |
| def testAppleSilicon(self): |
| test_config = CreateConfigWithGpu('apple:m1') |
| self.assertDeviceIdCorrectness( |
| magic_substitutions.GPUExpectedDeviceId(test_config, None, {}), ['0']) |
| |
| def testNoGpu(self): |
| self.assertDeviceIdCorrectness( |
| magic_substitutions.GPUExpectedDeviceId( |
| {'swarming': { |
| 'dimensions': {} |
| }}, None, {}), ['0']) |
| |
| def testNoDimensions(self): |
| with self.assertRaises(AssertionError): |
| magic_substitutions.GPUExpectedDeviceId({}, None, {}) |
| |
| def testAndroidDesktopKnownBoard(self): |
| test_config = { |
| 'name': 'test_name', |
| 'swarming': { |
| 'dimensions': { |
| 'label-board': 'brya', |
| }, |
| }, |
| } |
| tester_config = { |
| 'os_type': 'android', |
| } |
| self.assertDeviceIdCorrectness( |
| magic_substitutions.GPUExpectedDeviceId(test_config, None, |
| tester_config), ['46a8']) |
| |
| def testAndroidDesktopUnknownBoard(self): |
| test_config = { |
| 'name': 'test_name', |
| 'swarming': { |
| 'dimensions': { |
| 'label-board': 'fake_board', |
| }, |
| }, |
| } |
| tester_config = { |
| 'os_type': 'android', |
| } |
| self.assertDeviceIdCorrectness( |
| magic_substitutions.GPUExpectedDeviceId(test_config, None, |
| tester_config), ['0']) |
| |
| def testSkylabKnownBoard(self): |
| test_config = { |
| 'name': 'test_name', |
| 'cros_board': 'volteer', |
| } |
| tester_config = { |
| 'browser_config': 'cros-chrome', |
| 'use_swarming': False, |
| } |
| self.assertDeviceIdCorrectness( |
| magic_substitutions.GPUExpectedDeviceId(test_config, None, |
| tester_config), ['9a49']) |
| |
| def testSkylabUnknownBoard(self): |
| test_config = { |
| 'name': 'test_name', |
| 'cros_board': 'fancy_new_board', |
| } |
| tester_config = { |
| 'browser_config': 'cros-chrome', |
| 'use_swarming': False, |
| } |
| self.assertDeviceIdCorrectness( |
| magic_substitutions.GPUExpectedDeviceId(test_config, None, |
| tester_config), ['0']) |
| |
| |
| class GPUParallelJobs(unittest.TestCase): |
| def testNoOsType(self): |
| test_config = CreateConfigWithGpu('vendor:device1-driver') |
| with self.assertRaises(AssertionError): |
| magic_substitutions.GPUParallelJobs(test_config, 'name', {}) |
| |
| def testParallelJobs(self): |
| test_config = CreateConfigWithGpu('vendor:device1-driver') |
| for os_type in ['lacros', 'linux', 'mac', 'win']: |
| retval = magic_substitutions.GPUParallelJobs(test_config, 'name', |
| {'os_type': os_type}) |
| self.assertEqual(retval, ['--jobs=4']) |
| |
| def testSerialJobs(self): |
| test_config = CreateConfigWithGpu('vendor:device1-driver') |
| for os_type in ['android', 'chromeos', 'fuchsia']: |
| retval = magic_substitutions.GPUParallelJobs(test_config, 'name', |
| {'os_type': os_type}) |
| self.assertEqual(retval, ['--jobs=1']) |
| |
| def testWebGPUCTSWindowsIntelSerialJobs(self): |
| intel_config = CreateConfigWithGpu('8086:device1-driver') |
| amd_config = CreateConfigWithGpu('1002:device1-driver') |
| |
| for gpu_config in [intel_config, amd_config]: |
| for name, telemetry_test_name in [('webgpu_cts', None), |
| (None, 'webgpu_cts')]: |
| is_intel = intel_config == gpu_config |
| c = gpu_config.copy() |
| if name: |
| c['name'] = name |
| if telemetry_test_name: |
| c['telemetry_test_name'] = telemetry_test_name |
| for os_type in ['lacros', 'linux', 'mac', 'win']: |
| retval = magic_substitutions.GPUParallelJobs(c, 'name', |
| {'os_type': os_type}) |
| if is_intel and os_type == 'win': |
| self.assertEqual(retval, ['--jobs=1']) |
| else: |
| self.assertEqual(retval, ['--jobs=4']) |
| |
| def testWebGLWindowsIntelParallelJobs(self): |
| intel_config = CreateConfigWithGpu('8086:device1-driver') |
| amd_config = CreateConfigWithGpu('1002:device1-driver') |
| for gpu_config in [intel_config, amd_config]: |
| for name, telemetry_test_name in [('webgl_conformance', None), |
| ('webgl1_conformance', None), |
| ('webgl2_conformance', None), |
| (None, 'webgl1_conformance'), |
| (None, 'webgl2_conformance')]: |
| is_intel = intel_config == gpu_config |
| c = gpu_config.copy() |
| if name: |
| c['name'] = name |
| if telemetry_test_name: |
| c['telemetry_test_name'] = telemetry_test_name |
| for os_type in ['lacros', 'linux', 'mac', 'win']: |
| retval = magic_substitutions.GPUParallelJobs(c, 'name', |
| {'os_type': os_type}) |
| if is_intel and os_type == 'win': |
| self.assertEqual(retval, ['--jobs=2']) |
| else: |
| self.assertEqual(retval, ['--jobs=4']) |
| |
| def testWebGLMacNvidiaParallelJobs(self): |
| amd_config = CreateConfigWithGpu('1002:device1-driver') |
| nvidia_config = CreateConfigWithGpu('10de:device1-driver') |
| |
| for gpu_config in [nvidia_config, amd_config]: |
| for name, telemetry_test_name in [('webgl1_conformance', None), |
| (None, 'webgl1_conformance')]: |
| is_nvidia = gpu_config == nvidia_config |
| c = gpu_config.copy() |
| if name: |
| c['name'] = name |
| if telemetry_test_name: |
| c['telemetry_test_name'] = telemetry_test_name |
| for os_type in ['lacros', 'linux', 'mac', 'win']: |
| retval = magic_substitutions.GPUParallelJobs(c, 'name', |
| {'os_type': os_type}) |
| if is_nvidia and os_type == 'mac': |
| self.assertEqual(retval, ['--jobs=3']) |
| else: |
| self.assertEqual(retval, ['--jobs=4']) |
| |
| def testPixelMacDebugParallelJobs(self): |
| gpu_config = CreateConfigWithGpu('1002:device1-driver') |
| for name, telemetry_test_name in [('pixel_skia_gold_test', None), |
| (None, 'pixel')]: |
| c = gpu_config.copy() |
| if name: |
| c['name'] = name |
| if telemetry_test_name: |
| c['telemetry_test_name'] = telemetry_test_name |
| for os_type in ['lacros', 'linux', 'mac', 'win']: |
| for tester_name in ('Name Debug', 'Name Dbg', 'name debug', 'name dbg'): |
| retval = magic_substitutions.GPUParallelJobs(c, tester_name, |
| {'os_type': os_type}) |
| if os_type == 'mac': |
| self.assertEqual(retval, ['--jobs=1']) |
| else: |
| self.assertEqual(retval, ['--jobs=4']) |
| # Double check that non-debug Mac pixel tests still get parallelized. |
| retval = magic_substitutions.GPUParallelJobs(c, 'name release', |
| {'os_type': 'mac'}) |
| self.assertEqual(retval, ['--jobs=4']) |
| |
| def testPixelMacNvidiaParallelJobs(self): |
| gpu_config = CreateConfigWithGpu('10de:device1-driver') |
| for name, telemetry_test_name in [('pixel_skia_gold_test', None), |
| (None, 'pixel')]: |
| c = gpu_config.copy() |
| if name: |
| c['name'] = name |
| if telemetry_test_name: |
| c['telemetry_test_name'] = telemetry_test_name |
| for os_type in ['lacros', 'linux', 'mac', 'win']: |
| retval = magic_substitutions.GPUParallelJobs(c, 'name', |
| {'os_type': os_type}) |
| if os_type == 'mac': |
| self.assertEqual(retval, ['--jobs=1']) |
| else: |
| self.assertEqual(retval, ['--jobs=4']) |
| |
| |
| def CreateConfigWithDeviceType(device_type): |
| return { |
| 'swarming': { |
| 'dimensions': { |
| 'device_type': device_type, |
| }, |
| }, |
| } |
| |
| |
| class GPUTelemetryNoRootForUnrootedDevices(unittest.TestCase): |
| def testNoOsType(self): |
| test_config = CreateConfigWithDeviceType('a13') |
| with self.assertRaises(AssertionError): |
| magic_substitutions.GPUTelemetryNoRootForUnrootedDevices( |
| test_config, None, {}) |
| |
| def testNonAndroidOs(self): |
| retval = magic_substitutions.GPUTelemetryNoRootForUnrootedDevices( |
| {}, None, {'os_type': 'linux'}) |
| self.assertEqual(retval, []) |
| |
| def testUnrootedDevices(self): |
| devices = ('a13', 'a13ve', 'a23', 'dm1q', 'devonn') |
| for d in devices: |
| test_config = CreateConfigWithDeviceType(d) |
| retval = magic_substitutions.GPUTelemetryNoRootForUnrootedDevices( |
| test_config, None, {'os_type': 'android'}) |
| self.assertEqual(retval, |
| ['--compatibility-mode=dont-require-rooted-device']) |
| |
| def testRootedDevices(self): |
| test_config = CreateConfigWithDeviceType('hammerhead') |
| retval = magic_substitutions.GPUTelemetryNoRootForUnrootedDevices( |
| test_config, None, {'os_type': 'android'}) |
| self.assertEqual(retval, []) |
| |
| |
| class GPUWebGLRuntimeFile(unittest.TestCase): |
| def testNoOsType(self): |
| test_config = {'telemetry_test_name': 'webgl1_conformance'} |
| with self.assertRaises(AssertionError): |
| magic_substitutions.GPUWebGLRuntimeFile(test_config, None, {}) |
| |
| def testNoSuite(self): |
| tester_config = {'os_type': 'linux'} |
| with self.assertRaises(AssertionError): |
| magic_substitutions.GPUWebGLRuntimeFile({}, None, tester_config) |
| |
| def testUnknownSuite(self): |
| test_config = {'telemetry_test_name': 'foo'} |
| tester_config = {'os_type': 'linux'} |
| with self.assertRaises(AssertionError): |
| magic_substitutions.GPUWebGLRuntimeFile(test_config, None, tester_config) |
| |
| def testKnownOsTypes(self): |
| for os_type in ('android', 'linux', 'mac', 'win'): |
| for suite in ('webgl1_conformance', 'webgl2_conformance'): |
| retval = magic_substitutions.GPUWebGLRuntimeFile( |
| {'telemetry_test_name': suite}, None, {'os_type': os_type}) |
| self.assertEqual(retval, [ |
| '--read-abbreviated-json-results-from=../../content/test/data/gpu/' |
| f'{suite}_{os_type}_runtimes.json' |
| ]) |
| |
| def testUnknownOsType(self): |
| for suite in ('webgl1_conformance', 'webgl2_conformance'): |
| retval = magic_substitutions.GPUWebGLRuntimeFile( |
| {'telemetry_test_name': suite}, None, {'os_type': 'foo'}) |
| self.assertEqual(retval, [ |
| '--read-abbreviated-json-results-from=../../content/test/data/gpu/' |
| f'{suite}_linux_runtimes.json' |
| ]) |
| |
| |
| if __name__ == '__main__': |
| unittest.main() |
