testing/buildbot/generate_buildbot_json_unittest.py

#!/usr/bin/env vpython3
# Copyright 2017 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Tests for generate_buildbot_json.py."""

import contextlib
import json
import os
import re
import unittest

import generate_buildbot_json
from pyfakefs import fake_filesystem_unittest

# pylint: disable=super-with-arguments

EMPTY_PYL_FILE = """\
{
}
"""

# Use this value to refer to the directory containing this code
# The tests use a fake filesystem and python filesystem calls are monkey-patched
# to use the fake filesystem, which affects abspath
THIS_DIR = os.path.dirname(os.path.abspath(__file__))


class TestCase(fake_filesystem_unittest.TestCase):
  def setUp(self):
    self.setUpPyfakefs()
    self.fs.cwd = THIS_DIR

    expectations_dir = os.path.join(THIS_DIR, 'unittest_expectations')
    self.output_dir = os.path.join(expectations_dir, self.id().split('.')[-1])

    # This allows reads into this directory to fallback to the real fs, which we
    # want so each test case's json can be checked-in.
    self.fs.add_real_directory(expectations_dir)

    self.set_args()

  def set_args(self, *args):
    self.args = generate_buildbot_json.BBJSONGenerator.parse_args((
        '--output-dir',
        self.output_dir,
    ) + args)

  def regen_test_json(self, fakebb):
    """Regenerates a unittest's json files.

    Useful when making sweeping changes to pyl inputs in many test cases. Can be
    used by simply inserting a call to this method in a unit test after
    initializing its FakeBBGen object.
    """
    contents = fakebb.generate_outputs()
    with fake_filesystem_unittest.Pause(self.fs):
      try:
        os.mkdir(self.output_dir)
      except FileExistsError:
        if not os.path.isdir(self.output_dir):
          raise

      fakebb.write_json_result(contents)


@contextlib.contextmanager
def dump_on_failure(fbb, dump=True):
  try:
    yield
  except:
    if dump:
      for l in fbb.printed_lines:
        print(l)
    raise

class FakeBBGen(generate_buildbot_json.BBJSONGenerator):
  def __init__(self,
               args,
               waterfalls,
               test_suites,
               luci_milo_cfg,
               project_pyl='{"validate_source_side_specs_have_builder": True}',
               exceptions=EMPTY_PYL_FILE,
               autoshard_exceptions=json.dumps({}),
               mixins=EMPTY_PYL_FILE,
               gn_isolate_map=EMPTY_PYL_FILE,
               variants=EMPTY_PYL_FILE):
    super(FakeBBGen, self).__init__(args)

    pyl_files_dir = args.pyl_files_dir or THIS_DIR
    infra_config_dir = args.infra_config_dir

    files = {
        args.waterfalls_pyl_path: waterfalls,
        args.test_suites_pyl_path: test_suites,
        args.test_suite_exceptions_pyl_path: exceptions,
        args.autoshard_exceptions_json_path: autoshard_exceptions,
        args.mixins_pyl_path: mixins,
        args.gn_isolate_map_pyl_path: gn_isolate_map,
        args.variants_pyl_path: variants,
        os.path.join(pyl_files_dir, 'gn_isolate_map2.pyl'):
        GPU_TELEMETRY_GN_ISOLATE_MAP,
        os.path.join(infra_config_dir, 'generated/project.pyl'): project_pyl,
        os.path.join(infra_config_dir, 'generated/luci/luci-milo.cfg'):
        luci_milo_cfg,
        os.path.join(infra_config_dir, 'generated/luci/luci-milo-dev.cfg'): '',
    }
    for path, content in files.items():
      if content is None:
        continue
      parent = os.path.abspath(os.path.dirname(path))
      if not os.path.exists(parent):
        os.makedirs(parent)
      with open(path, 'w') as f:
        f.write(content)

    self.printed_lines = []

  def print_line(self, line):
    self.printed_lines.append(line)

  # pragma pylint: disable=arguments-differ
  def check_output_file_consistency(self, verbose=False, dump=True):
    with dump_on_failure(self, dump=verbose and dump):
      super(FakeBBGen, self).check_output_file_consistency(verbose)
  # pragma pylint: enable=arguments-differ


FOO_GTESTS_WATERFALL = """\
[
  {
    'project': 'chromium',
    'bucket': 'ci',
    'name': 'chromium.test',
    'machines': {
      'Fake Tester': {
        'swarming': {
          'dimensions':{
            'kvm': '1',
            'os': 'Linux',
          },
        },
        'test_suites': {
          'gtest_tests': 'foo_tests',
        },
      },
    },
  },
]
"""

FOO_GTESTS_WITH_ENABLE_FEATURES_WATERFALL = """\
[
  {
    'project': 'chromium',
    'bucket': 'ci',
    'name': 'chromium.test',
    'machines': {
      'Fake Tester': {
        'swarming': {
          'dimensions': {
            'os': 'Linux',
          },
        },
        'test_suites': {
          'gtest_tests': 'foo_tests',
        },
        'args': [
          '--enable-features=Baz',
        ],
      },
    },
  },
]
"""

FOO_CHROMEOS_TRIGGER_SCRIPT_WATERFALL = """\
[
  {
    'project': 'chromium',
    'bucket': 'ci',
    'name': 'chromium.test',
    'machines': {
      'Fake Tester': {
        'swarming': {
          'dimensions': {
            "device_type": "foo_device",
          },
        },
        'test_suites': {
          'gtest_tests': 'foo_tests',
        },
        'os_type': 'chromeos',
      },
    },
  },
]
"""

FOO_LINUX_GTESTS_WATERFALL = """\
[
  {
    'project': 'chromium',
    'bucket': 'ci',
    'name': 'chromium.test',
    'machines': {
      'Fake Tester': {
        'os_type': 'linux',
        'swarming': {
          'dimensions': {
            'os': 'Linux',
          },
        },
        'test_suites': {
          'gtest_tests': 'foo_tests',
        },
      },
    },
  },
]
"""

COMPOSITION_GTEST_SUITE_WATERFALL = """\
[
  {
    'project': 'chromium',
    'bucket': 'ci',
    'name': 'chromium.test',
    'machines': {
      'Fake Tester': {
        'test_suites': {
          'gtest_tests': 'composition_tests',
        },
      },
    },
  },
]
"""

COMPOSITION_GTEST_SUITE_WITH_ARGS_WATERFALL = """\
[
  {
    'project': 'chromium',
    'bucket': 'ci',
    'name': 'chromium.test',
    'machines': {
      'Fake Tester': {
        'test_suites': {
          'gtest_tests': 'composition_tests',
        },
        'args': [
          '--this-is-an-argument',
        ],
      },
    },
  },
]
"""

FOO_ISOLATED_SCRIPTS_WATERFALL = """\
[
  {
    'project': 'chromium',
    'bucket': 'ci',
    'name': 'chromium.test',
    'machines': {
      'Fake Tester': {
        'test_suites': {
          'isolated_scripts': 'composition_tests',
        },
      },
    },
  },
]
"""

FOO_ISOLATED_SCRIPTS_WATERFALL_ANDROID = """\
[
  {
    'project': 'chromium',
    'bucket': 'ci',
    'name': 'chromium.test',
    'machines': {
      'Fake Tester': {
        'os_type': 'android',
        'test_suites': {
          'isolated_scripts': 'composition_tests',
        },
        'use_android_presentation': True,
      },
    },
  },
]
"""

FOO_SCRIPT_WATERFALL = """\
[
  {
    'project': 'chromium',
    'bucket': 'ci',
    'name': 'chromium.test',
    'machines': {
      'Fake Tester': {
        'test_suites': {
          'scripts': 'foo_scripts',
        },
      },
    },
  },
]
"""

FOO_SCRIPT_WATERFALL_MACHINE_FORBIDS_SCRIPT_TESTS = """\
[
  {
    'project': 'chromium',
    'bucket': 'ci',
    'name': 'chromium.test',
    'machines': {
      'Fake Tester': {
        'forbid_script_tests': True,
        'test_suites': {
          'scripts': 'foo_scripts',
        },
      },
    },
  },
]
"""

FOO_SCRIPT_WATERFALL_FORBID_SCRIPT_TESTS = """\
[
  {
    'project': 'chromium',
    'bucket': 'ci',
    'name': 'chromium.test',
    'forbid_script_tests': True,
    'machines': {
      'Fake Tester': {
        'test_suites': {
          'scripts': 'foo_scripts',
        },
      },
    },
  },
]
"""

FOO_JUNIT_WATERFALL = """\
[
  {
    'project': 'chromium',
    'bucket': 'ci',
    'name': 'chromium.test',
    'machines': {
      'Fake Tester': {
        'test_suites': {
          'junit_tests': 'composition_tests',
        },
      },
    },
  },
]
"""

FOO_GPU_TELEMETRY_TEST_WATERFALL = """\
[
  {
    'project': 'chromium',
    'bucket': 'ci',
    'name': 'chromium.test',
    'machines': {
      'Fake Tester': {
        'os_type': 'win',
        'browser_config': 'release',
        'swarming': {
          'dimensions': {
            'gpu': '10de:1cb3',
          },
        },
        'test_suites': {
          'gpu_telemetry_tests': 'composition_tests',
        },
      },
    },
  },
]
"""

FOO_GPU_TELEMETRY_TEST_WATERFALL_ANDROID = """\
[
  {
    'project': 'chromium',
    'bucket': 'ci',
    'name': 'chromium.test',
    'machines': {
      'Fake Tester': {
        'os_type': 'android',
        'browser_config': 'android-chromium',
        'swarming': {
          'dimensions': {
            'device_type': 'bullhead',
          },
        },
        'test_suites': {
          'gpu_telemetry_tests': 'composition_tests',
        },
      },
    },
  },
]
"""

FOO_GPU_TELEMETRY_TEST_WATERFALL_ANDROID_WEBVIEW = """\
[
  {
    'project': 'chromium',
    'bucket': 'ci',
    'name': 'chromium.test',
    'machines': {
      'Fake Tester': {
        'os_type': 'android',
        'browser_config': 'not-a-real-browser',
        'swarming': {
          'dimensions': {
            'device_type': 'bullhead',
          },
        },
        'test_suites': {
          'android_webview_gpu_telemetry_tests': 'composition_tests',
        },
      },
    },
  },
]
"""

FOO_GPU_TELEMETRY_TEST_WATERFALL_FUCHSIA = """\
[
  {
    'project': 'chromium',
    'bucket': 'ci',
    'name': 'chromium.test',
    'machines': {
      'Fake Tester': {
        'os_type': 'fuchsia',
        'browser_config': 'fuchsia-chrome',
        'swarming': {
          'dimensions': {
            'kvm': '1',
          },
        },
        'test_suites': {
          'gpu_telemetry_tests': 'composition_tests',
        },
      },
    },
  },
]
"""

FOO_GPU_TELEMETRY_TEST_WATERFALL_CAST_STREAMING = """\
[
  {
    'project': 'chromium',
    'bucket': 'ci',
    'name': 'chromium.test',
    'machines': {
      'Fake Tester': {
        'os_type': 'fuchsia',
        'browser_config': 'not-a-real-browser',
        'swarming': {
          'dimensions': {
            'kvm': '1',
          },
        },
        'test_suites': {
          'cast_streaming_tests': 'composition_tests',
        },
      },
    },
  },
]
"""

FOO_GPU_TELEMETRY_TEST_WATERFALL_SKYLAB = """\
[
  {
    'project': 'chromium',
    'bucket': 'ci',
    'name': 'chromium.test',
    'machines': {
      'Fake Tester': {
        'os_type': 'chromeos',
        'browser_config': 'cros-chrome',
        'use_swarming': False,
        'swarming': {
          'some_key': 'some_value',
        },
        'test_suites': {
          'skylab_gpu_telemetry_tests': 'composition_tests',
        },
      },
    },
  },
]
"""

NVIDIA_GPU_TELEMETRY_TEST_WATERFALL = """\
[
  {
    'project': 'chromium',
    'bucket': 'ci',
    'name': 'chromium.test',
    'machines': {
      'Fake Tester': {
        'os_type': 'win',
        'browser_config': 'release',
        'swarming': {
          'dimensions': {
            'gpu': '10de:1cb3-26.21.14.3102',
          },
        },
        'test_suites': {
          'gpu_telemetry_tests': 'composition_tests',
        },
      },
    },
  },
]
"""

INTEL_GPU_TELEMETRY_TEST_WATERFALL = """\
[
  {
    'project': 'chromium',
    'bucket': 'ci',
    'name': 'chromium.test',
    'machines': {
      'Fake Tester': {
        'os_type': 'win',
        'browser_config': 'release',
        'swarming': {
          'dimensions': {
            'gpu': '8086:5912-24.20.100.6286',
          },
        },
        'test_suites': {
          'gpu_telemetry_tests': 'composition_tests',
        },
      },
    },
  },
]
"""

INTEL_UHD_GPU_TELEMETRY_TEST_WATERFALL = """\
[
  {
    'project': 'chromium',
    'bucket': 'ci',
    'name': 'chromium.test',
    'machines': {
      'Fake Tester': {
        'os_type': 'win',
        'browser_config': 'release',
        'swarming': {
          'dimensions': {
            'gpu': '8086:3e92-24.20.100.6286',
          },
        },
        'test_suites': {
          'gpu_telemetry_tests': 'composition_tests',
        },
      },
    },
  },
]
"""

GPU_TELEMETRY_TEST_VARIANTS_WATERFALL = """\
[
  {
    'project': 'chromium',
    'bucket': 'ci',
    'name': 'chromium.test',
    'machines': {
      'Fake Tester': {
        'os_type': 'win',
        'browser_config': 'release',
        'swarming': {
          'dimensions': {
            'gpu': '8086:3e92-24.20.100.6286',
            'os': 'Linux',
          },
        },
        'test_suites': {
          'gpu_telemetry_tests': 'matrix_tests',
        },
      },
    },
  },
]
"""

UNKNOWN_TEST_SUITE_WATERFALL = """\
[
  {
    'project': 'chromium',
    'bucket': 'ci',
    'name': 'chromium.test',
    'machines': {
      'Fake Tester': {
        'test_suites': {
          'gtest_tests': 'baz_tests',
        },
      },
    },
  },
]
"""

UNKNOWN_TEST_SUITE_TYPE_WATERFALL = """\
[
  {
    'project': 'chromium',
    'bucket': 'ci',
    'name': 'chromium.test',
    'machines': {
      'Fake Tester': {
        'test_suites': {
          'gtest_tests': 'foo_tests',
          'foo_test_type': 'foo_tests',
        },
      },
    },
  },
]
"""

ANDROID_WATERFALL = """\
[
  {
    'project': 'chromium',
    'bucket': 'ci',
    'name': 'chromium.test',
    'machines': {
      'Android Builder': {
        'additional_compile_targets': [
          'bar_test',
        ],
      },
      'Fake Android K Tester': {
        'additional_compile_targets': [
          'bar_test',
        ],
        'swarming': {
          'dimensions': {
            'device_os': 'KTU84P',
            'device_type': 'hammerhead',
            'os': 'Android',
          },
        },
        'os_type': 'android',
        'skip_merge_script': True,
        'test_suites': {
          'gtest_tests': 'foo_tests',
        },
      },
      'Fake Android L Tester': {
        'swarming': {
          'dimensions': {
            'device_os': 'LMY41U',
            'device_os_type': 'user',
            'device_type': 'hammerhead',
            'os': 'Android',
          },
        },
        'os_type': 'android',
        'skip_merge_script': True,
        'test_suites': {
          'gtest_tests': 'foo_tests',
        },
      },
      'Fake Android M Tester': {
        'swarming': {
          'dimensions': {
            'device_os': 'MMB29Q',
            'device_type': 'bullhead',
            'os': 'Android',
          },
        },
        'os_type': 'android',
        'use_swarming': False,
        'test_suites': {
          'gtest_tests': 'foo_tests',
        },
      },
    },
  },
]
"""

UNKNOWN_BOT_GTESTS_WATERFALL = """\
[
  {
    'project': 'chromium',
    'bucket': 'ci',
    'name': 'chromium.test',
    'machines': {
      'Unknown Bot': {
        'test_suites': {
          'gtest_tests': 'foo_tests',
        },
      },
    },
  },
]
"""

MATRIX_GTEST_SUITE_WATERFALL = """\
[
  {
    'project': 'chromium',
    'bucket': 'ci',
    'name': 'chromium.test',
    'machines': {
      'Fake Tester': {
        'swarming': {
          'dimensions': {
            'os': 'Linux',
          },
        },
        'test_suites': {
          'gtest_tests': 'matrix_tests',
        },
      },
    },
  },
]
"""

MATRIX_GTEST_SUITE_WATERFALL_MIXINS = """\
[
  {
    'project': 'chromium',
    'bucket': 'ci',
    'name': 'chromium.test',
    'machines': {
      'Fake Tester': {
        'mixins': ['dimension_mixin'],
        'test_suites': {
          'gtest_tests': 'matrix_tests',
        },
      },
    },
  },
]
"""

FOO_TEST_SUITE = """\
{
  'basic_suites': {
    'foo_tests': {
      'foo_test': {
        'swarming': {
          'dimensions': {
            'integrity': 'high',
            'os': 'Linux',
          },
          'expiration': 120,
        },
      },
    },
  },
}
"""

FOO_TEST_SUITE_ANDROID_SWARMING = """\
{
  'basic_suites': {
    'foo_tests': {
      'foo_test': {
        'android_swarming': {
          'shards': 100,
        },
      },
    },
  },
}
"""

FOO_TEST_SUITE_NO_DIMENSIONS = """\
{
  'basic_suites': {
    'foo_tests': {
      'foo_test': {
      },
    },
  },
}
"""

FOO_TEST_SUITE_NOT_SORTED = """\
{
  'basic_suites': {
    'foo_tests': {
      'foo_test': {},
      'a_test': {},
    },
  },
}
"""

FOO_TEST_SUITE_WITH_ARGS = """\
{
  'basic_suites': {
    'foo_tests': {
      'foo_test': {
        'args': [
          '--c_arg',
        ],
      },
    },
  },
}
"""

FOO_TEST_SUITE_WITH_SWARMING_DIMENSION_SETS = """\
{
  'basic_suites': {
    'foo_tests': {
      'foo_test': {
        'swarming': {
          'dimension_sets': [
            {
              'foo': 'bar',
            },
          ],
        },
      },
    },
  },
}
"""

FOO_TEST_SUITE_WITH_NAME = """\
{
  'basic_suites': {
    'foo_tests': {
      'foo_test': {
        'name': 'bar_test',
      },
    },
  },
}
"""

FOO_TEST_SUITE_WITH_ISOLATE_NAME = """\
{
  'basic_suites': {
    'foo_tests': {
      'foo_test': {
        'isolate_name': 'bar_test',
      },
    },
  },
}
"""

FOO_TEST_SUITE_WITH_SWARMING_NAMED_CACHES = """\
{
  'basic_suites': {
    'foo_tests': {
      'foo_test': {
        'swarming': {
          'named_caches': [
            {
              'name': 'cache_in_test',
              'file': 'cache_in_test_file',
            },
          ],
        },
      },
    },
  },
}
"""

FOO_TEST_SUITE_WITH_LINUX_ARGS = """\
{
  'basic_suites': {
    'foo_tests': {
      'foo_test': {
        'linux_args': [
          '--no-xvfb',
        ],
      },
    },
  },
}
"""

FOO_TEST_SUITE_WITH_ENABLE_FEATURES = """\
{
  'basic_suites': {
    'foo_tests': {
      'foo_test': {
        'args': [
          '--enable-features=Foo,Bar',
        ],
      },
    },
  },
}
"""

FOO_TEST_SUITE_WITH_REMOVE_WATERFALL_MIXIN = """\
{
  'basic_suites': {
    'foo_tests': {
      'foo_test': {
        'remove_mixins': ['waterfall_mixin'],
        'swarming': {
          'dimensions': {
            'integrity': 'high',
          },
          'expiration': 120,
        },
      },
    },
  },
}
"""

FOO_TEST_SUITE_WITH_REMOVE_BUILDER_MIXIN = """\
{
  'basic_suites': {
    'foo_tests': {
      'foo_test': {
        'remove_mixins': ['builder_mixin'],
        'swarming': {
          'dimensions': {
            'integrity': 'high',
          },
          'expiration': 120,
        },
      },
    },
  },
}
"""

FOO_SCRIPT_SUITE = """\
{
  'basic_suites': {
    'foo_scripts': {
      'foo_test': {
        'script': 'foo.py',
      },
      'bar_test': {
        'script': 'bar.py',
      },
    },
  },
}
"""

GOOD_COMPOSITION_TEST_SUITES = """\
{
  'basic_suites': {
    'bar_tests': {
      'bar_test': {
        'swarming': {
          'dimensions': {
            'os': 'Linux',
          },
        },
      },
    },
    'foo_tests': {
      'foo_test': {
        'swarming': {
          'dimensions': {
            'os': 'Linux',
          },
        },
      },
    },
  },
  'compound_suites': {
    'composition_tests': [
      'foo_tests',
      'bar_tests',
    ],
  },
}
"""

BAD_COMPOSITION_TEST_SUITES = """\
{
  'basic_suites': {
    'bar_tests': {},
    'foo_tests': {},
  },
  'compound_suites': {
    'buggy_composition_tests': [
      'bar_tests',
    ],
    'composition_tests': [
      'foo_tests',
      'buggy_composition_tests',
    ],
  },
}
"""

CONFLICTING_COMPOSITION_TEST_SUITES = """\
{
  'basic_suites': {
    'bar_tests': {
      'baz_tests': {
        'args': [
          '--bar',
        ],
      }
    },
    'foo_tests': {
      'baz_tests': {
        'args': [
          '--foo',
        ],
      }
    },
  },
  'compound_suites': {
    'foobar_tests': [
      'foo_tests',
      'bar_tests',
    ],
  },
}
"""

DUPLICATES_COMPOSITION_TEST_SUITES = """\
{
  'basic_suites': {
    'bar_tests': {},
    'buggy_composition_tests': {},
    'foo_tests': {},
  },
  'compound_suites': {
    'bar_tests': [
      'foo_tests',
    ],
    'composition_tests': [
      'foo_tests',
      'buggy_composition_tests',
    ],
  },
}
"""

SCRIPT_SUITE = """\
{
  'basic_suites': {
    'foo_scripts': {
      'foo_test': {
        'script': 'foo.py',
      },
    },
  },
}
"""

UNREFED_TEST_SUITE = """\
{
  'basic_suites': {
    'bar_tests': {},
    'foo_tests': {},
  },
}
"""

REUSING_TEST_WITH_DIFFERENT_NAME = """\
{
  'basic_suites': {
    'foo_tests': {
      'foo_test': {},
      'variation_test': {
        'args': [
          '--variation',
        ],
        'test': 'foo_test',
      },
    },
  },
}
"""

COMPOSITION_SUITE_WITH_TELEMETRY_TEST_WITH_INVALID_NAME = """\
{
  'basic_suites': {
    'foo_tests': {
      'foo': {
        'telemetry_test_name': 'foo',
        'swarming': {
          'dimensions': {
            'os': 'Linux',
          },
        },
      },
    },
  },
  'compound_suites': {
    'composition_tests': [
      'foo_tests',
    ],
  },
}
"""

COMPOSITION_SUITE_WITH_TELEMETRY_TEST = """\
{
  'basic_suites': {
    'foo_tests': {
      'foo_tests': {
        'telemetry_test_name': 'foo',
        'swarming': {
          'dimensions': {
            'os': 'Linux',
          },
          'shards': 2,
        },
      },
    },
    'bar_tests': {
      'bar_test': {
        'swarming': {
          'dimensions': {
            'os': 'Linux',
          },
        },
      },
    },
  },
  'compound_suites': {
    'composition_tests': [
      'foo_tests',
      'bar_tests',
    ],
  },
}
"""

COMPOSITION_SUITE_WITH_GPU_ARGS = """\
{
  'basic_suites': {
    'foo_tests': {
      'foo_tests': {
        'telemetry_test_name': 'foo',
        'swarming': {
          'dimensions': {
            'os': 'Linux',
          },
        },
        'args': [
          '--gpu-vendor-id',
          '${gpu_vendor_id}',
          '--gpu-device-id',
          '${gpu_device_id}',
        ],
      },
    },
    'bar_tests': {
      'bar_test': {
        'swarming': {
          'dimensions': {
            'os': 'Linux',
          },
        },
      },
    },
  },
  'compound_suites': {
    'composition_tests': [
      'foo_tests',
      'bar_tests',
    ],
  },
}
"""

COMPOSITION_SUITE_WITH_PIXEL_AND_FILTER = """\
{
  'basic_suites': {
    'foo_tests': {
      'pixel': {
        'args': [
          '--git-revision=aaaa',
          '--test-filter=bar',
        ],
      },
    },
    'bar_tests': {
      'bar_test': {},
    },
  },
  'compound_suites': {
    'composition_tests': [
      'foo_tests',
      'bar_tests',
    ],
  },
}
"""

GTEST_AS_ISOLATED_SCRIPT_SUITE = """\
{
  'basic_suites': {
    'foo_tests': {
      'foo_test': {
        'script': 'foo.py',
        'use_isolated_scripts_api': True,
      },
    },
  },
}
"""

SCRIPT_WITH_ARGS_EXCEPTIONS = """\
{
  'foo_test': {
    'modifications': {
      'Fake Tester': {
        'args': ['--fake-arg'],
      },
    },
  },
}
"""

SCRIPT_WITH_ARGS_SWARMING_EXCEPTIONS = """\
{
  'foo_test': {
    'modifications': {
      'Fake Tester': {
        'swarming': {
          'value': 'exception',
        },
      },
    },
  },
}
"""

AUTOSHARD_EXCEPTIONS = json.dumps({
    'chromium.test': {
        'Fake Tester': {
            'foo_test': {
                'shards': 20,
            },
            'foo_test_apk': {
                'shards': 2,
            },
            'swarming_test a_variant': {
                'shards': 10,
            },
        }
    },
})

NO_BAR_TEST_EXCEPTIONS = """\
{
  'bar_test': {
    'remove_from': [
      'Fake Tester',
    ]
  }
}
"""

EMPTY_BAR_TEST_EXCEPTIONS = """\
{
  'bar_test': {
  }
}
"""

EXCEPTIONS_SORTED = """\
{
  'suite_c': {
    'modifications': {
      'Fake Tester': {
        'foo': 'bar',
      },
    },
  },
  'suite_d': {
    'modifications': {
      'Fake Tester': {
        'foo': 'baz',
      },
    },
  },
}
"""

EXCEPTIONS_UNSORTED = """\
{
  'suite_d': {
    'modifications': {
      'Fake Tester': {
        'foo': 'baz',
      },
    },
  },
  'suite_c': {
    'modifications': {
      'Fake Tester': {
        'foo': 'bar',
      },
    },
  },
}
"""

EXCEPTIONS_PER_TEST_UNSORTED = """\
{
  'suite_d': {
    'modifications': {
      'Other Tester': {
        'foo': 'baz',
      },
      'Fake Tester': {
        'foo': 'baz',
      },
    },
  },
}
"""

EXCEPTIONS_DUPS_REMOVE_FROM = """\
{
  'suite_d': {
    'remove_from': [
      'Fake Tester',
      'Fake Tester',
    ],
    'modifications': {
      'Fake Tester': {
        'foo': 'baz',
      },
    },
  },
}
"""

FOO_TEST_MODIFICATIONS = """\
{
  'foo_test': {
    'modifications': {
      'Fake Tester': {
        'args': [
          '--bar',
        ],
        'swarming': {
          'hard_timeout': 600,
        },
      },
    },
  }
}
"""

FOO_TEST_EXPLICIT_NONE_EXCEPTIONS = """\
{
  'foo_test': {
    'modifications': {
      'Fake Tester': {
        'swarming': {
          'dimensions': {
            'integrity': None,
          },
        },
      },
    },
  },
}
"""

NONEXISTENT_REMOVAL = """\
{
  'foo_test': {
    'remove_from': [
      'Nonexistent Tester',
    ]
  }
}
"""

NONEXISTENT_MODIFICATION = """\
{
  'foo_test': {
    'modifications': {
      'Nonexistent Tester': {
        'args': [],
      },
    },
  }
}
"""

LUCI_MILO_CFG = """\
consoles {
  builders {
    name: "buildbucket/luci.chromium.ci/Fake Tester"
  }
}
"""

LUCI_MILO_CFG_WATERFALL_SORTING = """\
consoles {
  builders {
    name: "buildbucket/luci.chromium.ci/Fake Tester"
    name: "buildbucket/luci.chromium.ci/Really Fake Tester"
  }
}
"""

TEST_SUITE_SORTING_WATERFALL = """
[
  {
    'project': 'chromium',
    'bucket': 'ci',
    'name': 'chromium.test',
    'machines': {
      'Fake Tester': {
        'test_suites': {
          'gtest_tests': 'suite_a',
          'scripts': 'suite_b',
        },
      },
    },
  },
]
"""

TEST_SUITE_SORTED_WATERFALL = """
[
  {
    'project': 'chromium',
    'bucket': 'ci',
    'name': 'chromium.test',
    'machines': {
      'Fake Tester': {
        'test_suites': {
          'gtest_tests': 'suite_a',
          'scripts': 'suite_b',
        },
      },
      'Really Fake Tester': {
        'test_suites': {
          'gtest_tests': 'suite_a',
          'scripts': 'suite_b',
        },
      },
    },
  },
  {
    'project': 'chromium',
    'bucket': 'ci',
    'name': 'chromium.zz.test',
    'machines': {
      'Fake Tester': {
        'test_suites': {
          'gtest_tests': 'suite_a',
          'scripts': 'suite_b',
        },
      },
      'Really Fake Tester': {
        'test_suites': {
          'gtest_tests': 'suite_a',
          'scripts': 'suite_b',
        },
      },
    },
  },
]
"""

TEST_SUITE_UNSORTED_WATERFALL_1 = """
[
  {
    'project': 'chromium',
    'bucket': 'ci',
    'name': 'chromium.zz.test',
    'machines': {
      'Fake Tester': {
        'test_suites': {
          'gtest_tests': 'suite_a',
          'scripts': 'suite_b',
        },
      },
      'Really Fake Tester': {
        'test_suites': {
          'gtest_tests': 'suite_a',
          'scripts': 'suite_b',
        },
      },
    },
  },
  {
    'project': 'chromium',
    'bucket': 'ci',
    'name': 'chromium.test',
    'machines': {
      'Fake Tester': {
        'test_suites': {
          'gtest_tests': 'suite_a',
          'scripts': 'suite_b',
        },
      },
      'Really Fake Tester': {
        'test_suites': {
          'gtest_tests': 'suite_a',
          'scripts': 'suite_b',
        },
      },
    },
  },
]
"""

TEST_SUITE_UNSORTED_WATERFALL_2 = """
[
  {
    'project': 'chromium',
    'bucket': 'ci',
    'name': 'chromium.test',
    'machines': {
      'Really Fake Tester': {
        'test_suites': {
          'gtest_tests': 'suite_a',
          'scripts': 'suite_b',
        },
      },
      'Fake Tester': {
        'test_suites': {
          'gtest_tests': 'suite_a',
          'scripts': 'suite_b',
        },
      },
    },
  },
  {
    'project': 'chromium',
    'bucket': 'ci',
    'name': 'chromium.zz.test',
    'machines': {
      'Fake Tester': {
        'test_suites': {
          'gtest_tests': 'suite_a',
          'scripts': 'suite_b',
        },
      },
      'Really Fake Tester': {
        'test_suites': {
          'gtest_tests': 'suite_a',
          'scripts': 'suite_b',
        },
      },
    },
  },
]
"""

# Note that the suites in basic_suites would be sorted after the suites in
# compound_suites. This is valid though, because each set of suites is sorted
# separately.
# suite_c is a 'gtest_tests' test
# suite_d is a 'scripts' test
TEST_SUITE_SORTED = """\
{
  'basic_suites': {
    'suite_c': {
      'suite_c': {},
    },
    'suite_d': {
      'script': {
        'script': 'suite_d.py',
      }
    },
  },
  'compound_suites': {
    'suite_a': [
      'suite_c',
    ],
    'suite_b': [
      'suite_d',
    ],
  },
}
"""

TEST_SUITE_UNSORTED_1 = """\
{
  'basic_suites': {
    'suite_d': {
      'a': 'b',
    },
    'suite_c': {
      'a': 'b',
    },
  },
  'compound_suites': {
    'suite_a': [
      'suite_c',
    ],
    'suite_b': [
      'suite_d',
    ],
  },
}
"""

TEST_SUITE_UNSORTED_2 = """\
{
  'basic_suites': {
    'suite_c': {
      'a': 'b',
    },
    'suite_d': {
      'a': 'b',
    },
  },
  'compound_suites': {
    'suite_b': [
      'suite_c',
    ],
    'suite_a': [
      'suite_d',
    ],
  },
}
"""
TEST_SUITE_UNSORTED_3 = """\
{
  'basic_suites': {
    'suite_d': {
      'a': 'b',
    },
    'suite_c': {
      'a': 'b',
    },
  },
  'compound_suites': {
    'suite_b': [
      'suite_c',
    ],
    'suite_a': [
      'suite_d',
    ],
  },
}
"""


TEST_SUITES_SYNTAX_ERROR = """\
{
  'basic_suites': {
    3: {
      'suite_c': {},
    },
  },
  'compound_suites': {},
}
"""

GN_ISOLATE_MAP="""\
{
  'foo_test': {
    'label': '//chrome/test:foo_test',
    'type': 'windowed_test_launcher',
  }
}
"""

GPU_TELEMETRY_GN_ISOLATE_MAP="""\
{
  'telemetry_gpu_integration_test': {
    'label': '//chrome/test:telemetry_gpu_integration_test',
    'type': 'script',
      }
}
"""

GPU_TELEMETRY_GN_ISOLATE_MAP_ANDROID = """\
{
  'telemetry_gpu_integration_test_android_chrome': {
    'label': '//chrome/test:telemetry_gpu_integration_test_android_chrome',
    'type': 'script',
      }
}
"""

GPU_TELEMETRY_GN_ISOLATE_MAP_ANDROID_WEBVIEW = """\
{
  'telemetry_gpu_integration_test_android_webview': {
    'label': '//chrome/test:telemetry_gpu_integration_test_android_webview',
    'type': 'script',
      }
}
"""

GPU_TELEMETRY_GN_ISOLATE_MAP_FUCHSIA = """\
{
  'telemetry_gpu_integration_test_fuchsia': {
    'label': '//chrome/test:telemetry_gpu_integration_test_fuchsia',
    'type': 'script',
      }
}
"""

GPU_TELEMETRY_GN_ISOLATE_MAP_CAST_STREAMING = """\
{
  'telemetry_gpu_integration_test_fuchsia': {
    'label': '//chrome/test:telemetry_gpu_integration_test_fuchsia',
    'type': 'script',
      }
}
"""

GN_ISOLATE_MAP_KEY_LABEL_MISMATCH="""\
{
  'foo_test': {
    'label': '//chrome/test:foo_test_tmp',
    'type': 'windowed_test_launcher',
  }
}
"""

GN_ISOLATE_MAP_USING_IMPLICIT_NAME="""\
{
  'foo_test': {
    'label': '//chrome/foo_test',
    'type': 'windowed_test_launcher',
  }
}
"""

class UnitTest(TestCase):

  def test_dimension_sets_causes_error(self):
    fbb = FakeBBGen(self.args, FOO_GTESTS_BUILDER_MIXIN_WATERFALL,
                    FOO_TEST_SUITE_WITH_SWARMING_DIMENSION_SETS, LUCI_MILO_CFG)
    fbb.check_input_file_consistency(verbose=True)
    with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
                                r'.*dimension_sets is no longer supported.*'):
      fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_name_causes_error(self):
    fbb = FakeBBGen(self.args, FOO_GTESTS_BUILDER_MIXIN_WATERFALL,
                    FOO_TEST_SUITE_WITH_NAME, LUCI_MILO_CFG)
    fbb.check_input_file_consistency(verbose=True)
    with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
                                r'.*\bname field is set\b.*'):
      fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_isolate_name_causes_error(self):
    fbb = FakeBBGen(self.args, FOO_GTESTS_BUILDER_MIXIN_WATERFALL,
                    FOO_TEST_SUITE_WITH_ISOLATE_NAME, LUCI_MILO_CFG)
    fbb.check_input_file_consistency(verbose=True)
    with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
                                r'.*\bisolate_name field is set\b.*'):
      fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_good_test_suites_are_ok(self):
    fbb = FakeBBGen(self.args, FOO_GTESTS_WATERFALL, FOO_TEST_SUITE,
                    LUCI_MILO_CFG)
    fbb.check_input_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_good_composition_test_suites_are_ok(self):
    fbb = FakeBBGen(self.args, COMPOSITION_GTEST_SUITE_WATERFALL,
                    GOOD_COMPOSITION_TEST_SUITES, LUCI_MILO_CFG)
    fbb.check_input_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_bad_composition_test_suites_are_caught(self):
    fbb = FakeBBGen(self.args, COMPOSITION_GTEST_SUITE_WATERFALL,
                    BAD_COMPOSITION_TEST_SUITES, LUCI_MILO_CFG)
    with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
                                'compound_suites may not refer to.*'):
      fbb.check_input_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_composition_test_suites_no_conflicts(self):
    fbb = FakeBBGen(self.args, COMPOSITION_GTEST_SUITE_WATERFALL,
                    CONFLICTING_COMPOSITION_TEST_SUITES, LUCI_MILO_CFG)
    with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
                                'Conflicting test definitions.*'):
      fbb.check_input_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_composition_test_suites_no_duplicate_names(self):
    fbb = FakeBBGen(self.args, COMPOSITION_GTEST_SUITE_WATERFALL,
                    DUPLICATES_COMPOSITION_TEST_SUITES, LUCI_MILO_CFG)
    with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
                                '.*may not duplicate basic test suite.*'):
      fbb.check_input_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_unknown_test_suites_are_caught(self):
    fbb = FakeBBGen(self.args, UNKNOWN_TEST_SUITE_WATERFALL, FOO_TEST_SUITE,
                    LUCI_MILO_CFG)
    with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
                                'Test suite baz_tests from machine.*'):
      fbb.check_input_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_unknown_test_suite_types_are_caught(self):
    fbb = FakeBBGen(self.args, UNKNOWN_TEST_SUITE_TYPE_WATERFALL,
                    FOO_TEST_SUITE, LUCI_MILO_CFG)
    with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
                                'Unknown test suite type foo_test_type.*'):
      fbb.check_input_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_unrefed_test_suite_caught(self):
    fbb = FakeBBGen(self.args, FOO_GTESTS_WATERFALL, UNREFED_TEST_SUITE,
                    LUCI_MILO_CFG)
    with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
                                '.*unreferenced.*bar_tests.*'):
      fbb.check_input_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_good_waterfall_output(self):
    fbb = FakeBBGen(self.args, COMPOSITION_GTEST_SUITE_WATERFALL,
                    GOOD_COMPOSITION_TEST_SUITES, LUCI_MILO_CFG)
    fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_reusing_gtest_targets(self):
    fbb = FakeBBGen(self.args,
                    FOO_GTESTS_WATERFALL,
                    REUSING_TEST_WITH_DIFFERENT_NAME,
                    LUCI_MILO_CFG,
                    gn_isolate_map=GN_ISOLATE_MAP)
    fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_load_multiple_isolate_map_files_with_duplicates(self):
    self.args.isolate_map_files = ['gn_isolate_map.pyl']
    fbb = FakeBBGen(self.args,
                    FOO_GTESTS_WATERFALL,
                    REUSING_TEST_WITH_DIFFERENT_NAME,
                    LUCI_MILO_CFG,
                    gn_isolate_map=GN_ISOLATE_MAP)
    with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
                                'Duplicate targets in isolate map files.*'):
      fbb.load_configuration_files()

  def test_load_multiple_isolate_map_files_without_duplicates(self):
    self.args.isolate_map_files = ['gn_isolate_map2.pyl']
    fbb = FakeBBGen(self.args,
                    FOO_GTESTS_WATERFALL,
                    REUSING_TEST_WITH_DIFFERENT_NAME,
                    LUCI_MILO_CFG,
                    gn_isolate_map=GN_ISOLATE_MAP)
    fbb.load_configuration_files()
    isolate_dict = {}
    isolate_map_1 = fbb.load_pyl_file('gn_isolate_map.pyl')
    isolate_map_2 = fbb.load_pyl_file('gn_isolate_map2.pyl')
    isolate_dict.update(isolate_map_1)
    isolate_dict.update(isolate_map_2)
    self.assertEqual(isolate_dict, fbb.gn_isolate_map)

  def test_gn_isolate_map_with_label_mismatch(self):
    fbb = FakeBBGen(self.args,
                    FOO_GTESTS_WATERFALL,
                    FOO_TEST_SUITE,
                    LUCI_MILO_CFG,
                    gn_isolate_map=GN_ISOLATE_MAP_KEY_LABEL_MISMATCH)
    with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
                                'key name.*foo_test.*label.*'
                                'foo_test_tmp.*'):
      fbb.check_input_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_gn_isolate_map_using_implicit_gn_name(self):
    fbb = FakeBBGen(self.args,
                    FOO_GTESTS_WATERFALL,
                    FOO_TEST_SUITE,
                    LUCI_MILO_CFG,
                    gn_isolate_map=GN_ISOLATE_MAP_USING_IMPLICIT_NAME)
    with self.assertRaisesRegex(
        generate_buildbot_json.BBGenErr,
        'Malformed.*//chrome/foo_test.*for key.*'
        'foo_test.*'):
      fbb.check_input_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_noop_exception_does_nothing(self):
    fbb = FakeBBGen(self.args,
                    COMPOSITION_GTEST_SUITE_WATERFALL,
                    GOOD_COMPOSITION_TEST_SUITES,
                    LUCI_MILO_CFG,
                    exceptions=EMPTY_BAR_TEST_EXCEPTIONS)
    fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_test_arg_merges(self):
    fbb = FakeBBGen(self.args,
                    FOO_GTESTS_WATERFALL,
                    FOO_TEST_SUITE_WITH_ARGS,
                    LUCI_MILO_CFG,
                    exceptions=FOO_TEST_MODIFICATIONS)
    fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_enable_features_arg_merges(self):
    fbb = FakeBBGen(self.args, FOO_GTESTS_WITH_ENABLE_FEATURES_WATERFALL,
                    FOO_TEST_SUITE_WITH_ENABLE_FEATURES, LUCI_MILO_CFG)
    fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_linux_args(self):
    fbb = FakeBBGen(self.args, FOO_LINUX_GTESTS_WATERFALL,
                    FOO_TEST_SUITE_WITH_LINUX_ARGS, LUCI_MILO_CFG)
    fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_test_filtering(self):
    fbb = FakeBBGen(self.args,
                    COMPOSITION_GTEST_SUITE_WATERFALL,
                    GOOD_COMPOSITION_TEST_SUITES,
                    LUCI_MILO_CFG,
                    exceptions=NO_BAR_TEST_EXCEPTIONS)
    fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_test_modifications(self):
    fbb = FakeBBGen(self.args,
                    FOO_GTESTS_WATERFALL,
                    FOO_TEST_SUITE,
                    LUCI_MILO_CFG,
                    exceptions=FOO_TEST_MODIFICATIONS)
    fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_test_with_explicit_none(self):
    fbb = FakeBBGen(self.args,
                    FOO_GTESTS_WATERFALL,
                    FOO_TEST_SUITE,
                    LUCI_MILO_CFG,
                    exceptions=FOO_TEST_EXPLICIT_NONE_EXCEPTIONS,
                    mixins=SWARMING_MIXINS)
    fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_isolated_script_tests(self):
    fbb = FakeBBGen(self.args,
                    FOO_ISOLATED_SCRIPTS_WATERFALL,
                    GOOD_COMPOSITION_TEST_SUITES,
                    LUCI_MILO_CFG,
                    exceptions=NO_BAR_TEST_EXCEPTIONS)
    fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_isolated_script_tests_android(self):
    fbb = FakeBBGen(self.args,
                    FOO_ISOLATED_SCRIPTS_WATERFALL_ANDROID,
                    GOOD_COMPOSITION_TEST_SUITES,
                    LUCI_MILO_CFG,
                    exceptions=NO_BAR_TEST_EXCEPTIONS)
    fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_script_with_args(self):
    fbb = FakeBBGen(self.args,
                    FOO_SCRIPT_WATERFALL,
                    SCRIPT_SUITE,
                    LUCI_MILO_CFG,
                    exceptions=SCRIPT_WITH_ARGS_EXCEPTIONS)
    fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_script(self):
    fbb = FakeBBGen(self.args,
                    FOO_SCRIPT_WATERFALL,
                    FOO_SCRIPT_SUITE,
                    LUCI_MILO_CFG,
                    exceptions=NO_BAR_TEST_EXCEPTIONS)
    fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_script_machine_forbids_scripts(self):
    fbb = FakeBBGen(self.args,
                    FOO_SCRIPT_WATERFALL_MACHINE_FORBIDS_SCRIPT_TESTS,
                    FOO_SCRIPT_SUITE,
                    LUCI_MILO_CFG,
                    exceptions=NO_BAR_TEST_EXCEPTIONS)
    with self.assertRaisesRegex(
        generate_buildbot_json.BBGenErr,
        'Attempted to generate a script test on tester.*'):
      fbb.check_output_file_consistency(verbose=True)

  def test_script_waterfall_forbids_scripts(self):
    fbb = FakeBBGen(self.args,
                    FOO_SCRIPT_WATERFALL_FORBID_SCRIPT_TESTS,
                    FOO_SCRIPT_SUITE,
                    LUCI_MILO_CFG,
                    exceptions=NO_BAR_TEST_EXCEPTIONS)
    with self.assertRaisesRegex(
        generate_buildbot_json.BBGenErr,
        'Attempted to generate a script test on tester.*'):
      fbb.check_output_file_consistency(verbose=True)

  def test_junit_tests(self):
    fbb = FakeBBGen(self.args,
                    FOO_JUNIT_WATERFALL,
                    GOOD_COMPOSITION_TEST_SUITES,
                    LUCI_MILO_CFG,
                    exceptions=NO_BAR_TEST_EXCEPTIONS)
    fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_gpu_telemetry_test_with_invalid_name(self):
    fbb = FakeBBGen(self.args,
                    FOO_GPU_TELEMETRY_TEST_WATERFALL,
                    COMPOSITION_SUITE_WITH_TELEMETRY_TEST_WITH_INVALID_NAME,
                    LUCI_MILO_CFG,
                    exceptions=NO_BAR_TEST_EXCEPTIONS,
                    gn_isolate_map=GPU_TELEMETRY_GN_ISOLATE_MAP)
    with self.assertRaisesRegex(
        generate_buildbot_json.BBGenErr,
        'telemetry test names must end with test or tests.*'):
      fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_gpu_telemetry_tests(self):
    fbb = FakeBBGen(self.args,
                    FOO_GPU_TELEMETRY_TEST_WATERFALL,
                    COMPOSITION_SUITE_WITH_TELEMETRY_TEST,
                    LUCI_MILO_CFG,
                    exceptions=NO_BAR_TEST_EXCEPTIONS,
                    gn_isolate_map=GPU_TELEMETRY_GN_ISOLATE_MAP)
    fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_gpu_telemetry_tests_android(self):
    fbb = FakeBBGen(self.args,
                    FOO_GPU_TELEMETRY_TEST_WATERFALL_ANDROID,
                    COMPOSITION_SUITE_WITH_TELEMETRY_TEST,
                    LUCI_MILO_CFG,
                    exceptions=NO_BAR_TEST_EXCEPTIONS,
                    gn_isolate_map=GPU_TELEMETRY_GN_ISOLATE_MAP_ANDROID)
    fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_gpu_telemetry_tests_android_webview(self):
    fbb = FakeBBGen(self.args,
                    FOO_GPU_TELEMETRY_TEST_WATERFALL_ANDROID_WEBVIEW,
                    COMPOSITION_SUITE_WITH_TELEMETRY_TEST,
                    LUCI_MILO_CFG,
                    exceptions=NO_BAR_TEST_EXCEPTIONS,
                    gn_isolate_map=GPU_TELEMETRY_GN_ISOLATE_MAP_ANDROID_WEBVIEW)
    fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_gpu_telemetry_tests_fuchsia(self):
    fbb = FakeBBGen(self.args,
                    FOO_GPU_TELEMETRY_TEST_WATERFALL_FUCHSIA,
                    COMPOSITION_SUITE_WITH_TELEMETRY_TEST,
                    LUCI_MILO_CFG,
                    exceptions=NO_BAR_TEST_EXCEPTIONS,
                    gn_isolate_map=GPU_TELEMETRY_GN_ISOLATE_MAP_FUCHSIA)
    fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_gpu_telemetry_tests_cast_streaming(self):
    fbb = FakeBBGen(self.args,
                    FOO_GPU_TELEMETRY_TEST_WATERFALL_CAST_STREAMING,
                    COMPOSITION_SUITE_WITH_TELEMETRY_TEST,
                    LUCI_MILO_CFG,
                    exceptions=NO_BAR_TEST_EXCEPTIONS,
                    gn_isolate_map=GPU_TELEMETRY_GN_ISOLATE_MAP_CAST_STREAMING)
    fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_gpu_telemetry_tests_skylab(self):
    fbb = FakeBBGen(self.args,
                    FOO_GPU_TELEMETRY_TEST_WATERFALL_SKYLAB,
                    COMPOSITION_SUITE_WITH_TELEMETRY_TEST,
                    LUCI_MILO_CFG,
                    exceptions=NO_BAR_TEST_EXCEPTIONS,
                    gn_isolate_map=GPU_TELEMETRY_GN_ISOLATE_MAP)
    fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_nvidia_gpu_telemetry_tests(self):
    fbb = FakeBBGen(self.args,
                    NVIDIA_GPU_TELEMETRY_TEST_WATERFALL,
                    COMPOSITION_SUITE_WITH_GPU_ARGS,
                    LUCI_MILO_CFG,
                    exceptions=NO_BAR_TEST_EXCEPTIONS,
                    gn_isolate_map=GPU_TELEMETRY_GN_ISOLATE_MAP)
    fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_intel_gpu_telemetry_tests(self):
    fbb = FakeBBGen(self.args,
                    INTEL_GPU_TELEMETRY_TEST_WATERFALL,
                    COMPOSITION_SUITE_WITH_GPU_ARGS,
                    LUCI_MILO_CFG,
                    exceptions=NO_BAR_TEST_EXCEPTIONS,
                    gn_isolate_map=GPU_TELEMETRY_GN_ISOLATE_MAP)
    fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_gpu_telemetry_tests_with_variants(self):
    fbb = FakeBBGen(self.args,
                    GPU_TELEMETRY_TEST_VARIANTS_WATERFALL,
                    MATRIX_COMPOUND_MIXED_VARIANTS_REF,
                    LUCI_MILO_CFG,
                    exceptions=NO_BAR_TEST_EXCEPTIONS,
                    gn_isolate_map=GPU_TELEMETRY_GN_ISOLATE_MAP,
                    variants=MULTI_VARIANTS_FILE)
    fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_intel_uhd_gpu_telemetry_tests(self):
    fbb = FakeBBGen(self.args,
                    INTEL_UHD_GPU_TELEMETRY_TEST_WATERFALL,
                    COMPOSITION_SUITE_WITH_GPU_ARGS,
                    LUCI_MILO_CFG,
                    exceptions=NO_BAR_TEST_EXCEPTIONS,
                    gn_isolate_map=GPU_TELEMETRY_GN_ISOLATE_MAP)
    fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_gtest_as_isolated_Script(self):
    fbb = FakeBBGen(self.args,
                    FOO_GTESTS_WATERFALL,
                    GTEST_AS_ISOLATED_SCRIPT_SUITE,
                    LUCI_MILO_CFG,
                    gn_isolate_map=GN_ISOLATE_MAP)
    fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_ungenerated_output_files_are_caught(self):
    fbb = FakeBBGen(self.args,
                    COMPOSITION_GTEST_SUITE_WATERFALL,
                    GOOD_COMPOSITION_TEST_SUITES,
                    LUCI_MILO_CFG,
                    exceptions=NO_BAR_TEST_EXCEPTIONS)
    with self.assertRaises(generate_buildbot_json.BBGenErr):
      fbb.check_output_file_consistency(verbose=True, dump=False)
    joined_lines = ' '.join(fbb.printed_lines)
    self.assertRegex(
        joined_lines, 'File chromium.test.json did not have the following'
        ' expected contents:.*')
    self.assertRegex(joined_lines, r'.*--- expected.*')
    self.assertRegex(joined_lines, r'.*\+\+\+ current.*')
    fbb.printed_lines = []
    self.assertFalse(fbb.printed_lines)

  def test_android_output_options(self):
    fbb = FakeBBGen(self.args, ANDROID_WATERFALL, FOO_TEST_SUITE, LUCI_MILO_CFG)
    fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_android_swarming(self):
    fbb = FakeBBGen(self.args, ANDROID_WATERFALL,
                    FOO_TEST_SUITE_ANDROID_SWARMING, LUCI_MILO_CFG)
    fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_nonexistent_removal_raises(self):
    fbb = FakeBBGen(self.args,
                    FOO_GTESTS_WATERFALL,
                    FOO_TEST_SUITE,
                    LUCI_MILO_CFG,
                    exceptions=NONEXISTENT_REMOVAL)
    with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
                                'The following nonexistent machines.*'):
      fbb.check_input_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_nonexistent_modification_raises(self):
    fbb = FakeBBGen(self.args,
                    FOO_GTESTS_WATERFALL,
                    FOO_TEST_SUITE,
                    LUCI_MILO_CFG,
                    exceptions=NONEXISTENT_MODIFICATION)
    with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
                                'The following nonexistent machines.*'):
      fbb.check_input_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_waterfall_args(self):
    fbb = FakeBBGen(self.args, COMPOSITION_GTEST_SUITE_WITH_ARGS_WATERFALL,
                    GOOD_COMPOSITION_TEST_SUITES, LUCI_MILO_CFG)
    fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_chromeos_trigger_script_output(self):
    fbb = FakeBBGen(self.args, FOO_CHROMEOS_TRIGGER_SCRIPT_WATERFALL,
                    FOO_TEST_SUITE, LUCI_MILO_CFG)
    fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_relative_pyl_file_dir(self):
    self.set_args('--pyl-files-dir=relative/path')
    fbb = FakeBBGen(self.args,
                    FOO_GTESTS_WATERFALL,
                    REUSING_TEST_WITH_DIFFERENT_NAME,
                    LUCI_MILO_CFG,
                    gn_isolate_map=GN_ISOLATE_MAP)
    fbb.check_input_file_consistency(verbose=True)
    fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_nonexistent_bot_raises(self):
    fbb = FakeBBGen(self.args, UNKNOWN_BOT_GTESTS_WATERFALL, FOO_TEST_SUITE,
                    LUCI_MILO_CFG)
    with self.assertRaises(generate_buildbot_json.BBGenErr):
      fbb.check_input_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_nonexistent_bot_raises_when_no_project_pyl_exists(self):
    fbb = FakeBBGen(self.args,
                    UNKNOWN_BOT_GTESTS_WATERFALL,
                    FOO_TEST_SUITE,
                    LUCI_MILO_CFG,
                    project_pyl=None)
    with self.assertRaises(generate_buildbot_json.BBGenErr):
      fbb.check_input_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_nonexistent_bot_does_not_raise_when_validation_disabled(self):
    fbb = FakeBBGen(
        self.args,
        UNKNOWN_BOT_GTESTS_WATERFALL,
        FOO_TEST_SUITE,
        LUCI_MILO_CFG,
        project_pyl='{"validate_source_side_specs_have_builder": False}')
    fbb.check_input_file_consistency(verbose=True)

  def test_waterfalls_must_be_sorted(self):
    fbb = FakeBBGen(self.args, TEST_SUITE_SORTED_WATERFALL, TEST_SUITE_SORTED,
                    LUCI_MILO_CFG_WATERFALL_SORTING)
    fbb.check_input_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

    fbb = FakeBBGen(self.args, TEST_SUITE_UNSORTED_WATERFALL_1,
                    TEST_SUITE_SORTED, LUCI_MILO_CFG_WATERFALL_SORTING)
    with self.assertRaisesRegex(
        generate_buildbot_json.BBGenErr,
        ('The following files have invalid keys: ' +
         re.escape(self.args.waterfalls_pyl_path)),
    ):
      fbb.check_input_file_consistency(verbose=True)
    joined_lines = '\n'.join(fbb.printed_lines)
    self.assertRegex(joined_lines, r'.*\+ chromium\..*test.*')
    self.assertRegex(joined_lines, r'.*\- chromium\..*test.*')
    fbb.printed_lines = []
    self.assertFalse(fbb.printed_lines)

    fbb = FakeBBGen(self.args, TEST_SUITE_UNSORTED_WATERFALL_2,
                    TEST_SUITE_SORTED, LUCI_MILO_CFG_WATERFALL_SORTING)
    with self.assertRaisesRegex(
        generate_buildbot_json.BBGenErr,
        ('The following files have invalid keys: ' +
         re.escape(self.args.waterfalls_pyl_path)),
    ):
      fbb.check_input_file_consistency(verbose=True)
    joined_lines = ' '.join(fbb.printed_lines)
    self.assertRegex(joined_lines, r'.*\+.*Fake Tester.*')
    self.assertRegex(joined_lines, r'.*\-.*Fake Tester.*')
    fbb.printed_lines = []
    self.assertFalse(fbb.printed_lines)

  def test_test_suite_exceptions_must_be_sorted(self):
    fbb = FakeBBGen(self.args,
                    TEST_SUITE_SORTING_WATERFALL,
                    TEST_SUITE_SORTED,
                    LUCI_MILO_CFG,
                    exceptions=EXCEPTIONS_SORTED)
    fbb.check_input_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

    fbb = FakeBBGen(self.args,
                    TEST_SUITE_SORTING_WATERFALL,
                    TEST_SUITE_SORTED,
                    LUCI_MILO_CFG,
                    exceptions=EXCEPTIONS_DUPS_REMOVE_FROM)
    with self.assertRaises(generate_buildbot_json.BBGenErr):
      fbb.check_input_file_consistency(verbose=True)
    joined_lines = ' '.join(fbb.printed_lines)
    self.assertRegex(joined_lines, r'.*\- Fake Tester.*')
    fbb.printed_lines = []
    self.assertFalse(fbb.printed_lines)

  def test_test_suite_exceptions_no_dups_remove_from(self):
    fbb = FakeBBGen(self.args,
                    TEST_SUITE_SORTING_WATERFALL,
                    TEST_SUITE_SORTED,
                    LUCI_MILO_CFG,
                    exceptions=EXCEPTIONS_SORTED)
    fbb.check_input_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

    fbb = FakeBBGen(self.args,
                    TEST_SUITE_SORTING_WATERFALL,
                    TEST_SUITE_SORTED,
                    LUCI_MILO_CFG,
                    exceptions=EXCEPTIONS_PER_TEST_UNSORTED)
    with self.assertRaises(generate_buildbot_json.BBGenErr):
      fbb.check_input_file_consistency(verbose=True)
    joined_lines = ' '.join(fbb.printed_lines)
    self.assertRegex(joined_lines, r'.*\+ Fake Tester.*')
    self.assertRegex(joined_lines, r'.*\- Fake Tester.*')
    fbb.printed_lines = []
    self.assertFalse(fbb.printed_lines)

  def test_test_suite_exceptions_per_test_must_be_sorted(self):
    fbb = FakeBBGen(self.args,
                    TEST_SUITE_SORTING_WATERFALL,
                    TEST_SUITE_SORTED,
                    LUCI_MILO_CFG,
                    exceptions=EXCEPTIONS_SORTED)
    fbb.check_input_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

    fbb = FakeBBGen(self.args,
                    TEST_SUITE_SORTING_WATERFALL,
                    TEST_SUITE_SORTED,
                    LUCI_MILO_CFG,
                    exceptions=EXCEPTIONS_UNSORTED)
    with self.assertRaises(generate_buildbot_json.BBGenErr):
      fbb.check_input_file_consistency(verbose=True)
    joined_lines = ' '.join(fbb.printed_lines)
    self.assertRegex(joined_lines, r'.*\+ suite_.*')
    self.assertRegex(joined_lines, r'.*\- suite_.*')
    fbb.printed_lines = []
    self.assertFalse(fbb.printed_lines)

  def test_test_suites_must_be_sorted(self):
    fbb = FakeBBGen(self.args, TEST_SUITE_SORTING_WATERFALL, TEST_SUITE_SORTED,
                    LUCI_MILO_CFG)
    fbb.check_input_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

    for unsorted in (
        TEST_SUITE_UNSORTED_1,
        TEST_SUITE_UNSORTED_2,
        TEST_SUITE_UNSORTED_3,
    ):
      fbb = FakeBBGen(self.args, TEST_SUITE_SORTING_WATERFALL, unsorted,
                      LUCI_MILO_CFG)
      with self.assertRaises(generate_buildbot_json.BBGenErr):
        fbb.check_input_file_consistency(verbose=True)
      joined_lines = ' '.join(fbb.printed_lines)
      self.assertRegex(joined_lines, r'.*\+ suite_.*')
      self.assertRegex(joined_lines, r'.*\- suite_.*')
      fbb.printed_lines = []
      self.assertFalse(fbb.printed_lines)


FOO_GTESTS_WATERFALL_MIXIN_WATERFALL = """\
[
  {
    'mixins': ['waterfall_mixin'],
    'project': 'chromium',
    'bucket': 'ci',
    'name': 'chromium.test',
    'machines': {
      'Fake Tester': {
        'swarming': {
          'dimensions': {
            'os': 'Linux',
          },
        },
        'test_suites': {
          'gtest_tests': 'foo_tests',
        },
      },
    },
  },
]
"""

FOO_GTESTS_BUILDER_MIXIN_WATERFALL = """\
[
  {
    'project': 'chromium',
    'bucket': 'ci',
    'name': 'chromium.test',
    'machines': {
      'Fake Tester': {
        'mixins': ['builder_mixin'],
        'swarming': {
          'dimensions': {
            'os': 'Linux',
          },
        },
        'test_suites': {
          'gtest_tests': 'foo_tests',
        },
      },
    },
  },
]
"""

FOO_LINUX_GTESTS_BUILDER_MIXIN_WATERFALL = """\
[
  {
    'project': 'chromium',
    'bucket': 'ci',
    'name': 'chromium.test',
    'machines': {
      'Fake Tester': {
        'os_type': 'linux',
        'mixins': ['builder_mixin'],
        'swarming': {
          'dimensions': {
            'os': 'Linux',
          },
        },
        'test_suites': {
          'gtest_tests': 'foo_tests',
        },
      },
    },
  },
]
"""

FOO_GTESTS_BUILDER_MIXIN_NON_SWARMING_WATERFALL = """\
[
  {
    'project': 'chromium',
    'bucket': 'ci',
    'name': 'chromium.test',
    'machines': {
      'Fake Tester': {
        'mixins': ['random_mixin'],
        'swarming': {},
        'test_suites': {
          'gtest_tests': 'foo_tests',
        },
      },
    },
  },
]
"""

FOO_GTESTS_DIMENSIONS_MIXIN_WATERFALL = """\
[
  {
    'project': 'chromium',
    'bucket': 'ci',
    'name': 'chromium.test',
    'machines': {
      'Fake Tester': {
        'mixins': ['dimension_mixin'],
        'swarming': {},
        'test_suites': {
          'gtest_tests': 'foo_tests',
        },
      },
    },
  },
]
"""

FOO_GPU_TELEMETRY_TEST_DIMENSIONS_WATERFALL = """\
[
  {
    'project': 'chromium',
    'bucket': 'ci',
    'name': 'chromium.test',
    'machines': {
      'Fake Tester': {
        'mixins': ['dimension_mixin'],
        'os_type': 'win',
        'browser_config': 'release',
        'test_suites': {
          'gpu_telemetry_tests': 'foo_tests',
        },
      },
    },
  },
]
"""

# Swarming mixins must be a list, a single string is not allowed.
FOO_GTESTS_INVALID_LIST_MIXIN_WATERFALL = """\
[
  {
    'project': 'chromium',
    'bucket': 'ci',
    'name': 'chromium.test',
    'machines': {
      'Fake Tester': {
        'mixins': 'dimension_mixin',
        'swarming': {},
        'test_suites': {
          'gtest_tests': 'foo_tests',
        },
      },
    },
  },
]
"""
FOO_GTESTS_INVALID_NOTFOUND_MIXIN_WATERFALL = """\
[
  {
    'project': 'chromium',
    'bucket': 'ci',
    'name': 'chromium.test',
    'machines': {
      'Fake Tester': {
        'mixins': ['nonexistant'],
        'swarming': {},
        'test_suites': {
          'gtest_tests': 'foo_tests',
        },
      },
    },
  },
]
"""

FOO_GTESTS_TEST_MIXIN_WATERFALL = """\
[
  {
    'project': 'chromium',
    'bucket': 'ci',
    'name': 'chromium.test',
    'mixins': ['waterfall_mixin'],
    'machines': {
      'Fake Tester': {
        'swarming': {},
        'test_suites': {
          'gtest_tests': 'foo_tests',
        },
      },
    },
  },
]
"""

FOO_GTESTS_SORTING_MIXINS_WATERFALL = """\
[
  {
    'mixins': ['a_mixin', 'b_mixin', 'c_mixin'],
    'project': 'chromium',
    'bucket': 'ci',
    'name': 'chromium.test',
    'machines': {
      'Fake Tester': {
        'swarming': {
          'dimensions': {
            'kvm': '1',
          },
        },
        'test_suites': {
          'gtest_tests': 'foo_tests',
        },
      },
    },
  },
]
"""

FOO_TEST_SUITE_WITH_MIXIN = """\
{
  'basic_suites': {
    'foo_tests': {
      'foo_test': {
        'swarming': {
          'dimensions': {
            'integrity': 'high',
            'os': 'Linux',
          },
          'expiration': 120,
        },
        'mixins': ['test_mixin'],
      },
    },
  },
}
"""

MIXIN_ARGS = """\
{
  'builder_mixin': {
    'args': [],
  },
}
"""

MIXIN_ARGS_NOT_LIST = """\
{
  'builder_mixin': {
    'args': 'I am not a list',
  },
}
"""

MIXIN_LINUX_ARGS = """\
{
  'builder_mixin': {
    'args': [ '--mixin-argument' ],
    'linux_args': [ '--linux-mixin-argument' ],
  },
}
"""

MIXIN_APPEND = """\
{
  'builder_mixin': {
    '$mixin_append': {
      'args': [ '--mixin-argument' ],
    },
  },
}
"""

# These mixins are invalid; if passed to check_input_file_consistency, they will
# fail. These are used for output file consistency checks.
SWARMING_MIXINS = """\
{
  'builder_mixin': {
    'swarming': {
      'value': 'builder',
    },
  },
  'dimension_mixin': {
    'swarming': {
      'dimensions': {
        'iama': 'mixin',
      },
    },
  },
  'random_mixin': {
    'value': 'random',
  },
  'test_mixin': {
    'swarming': {
      'value': 'test',
    },
  },
  'waterfall_mixin': {
    'swarming': {
      'value': 'waterfall',
    },
  },
}
"""

SWARMING_NAMED_CACHES = """\
{
  'builder_mixin': {
    'swarming': {
      'named_caches': [
        {
          'name': 'cache',
          'file': 'cache_file',
        },
      ],
    },
  },
}
"""

SWARMING_MIXINS_DUPLICATED = """\
{
  'builder_mixin': {
    'value': 'builder',
  },
  'builder_mixin': {
    'value': 'builder',
  },
}
"""

SWARMING_MIXINS_UNSORTED = """\
{
  'b_mixin': {
    'b': 'b',
  },
  'a_mixin': {
    'a': 'a',
  },
  'c_mixin': {
    'c': 'c',
  },
}
"""

SWARMING_MIXINS_SORTED = """\
{
  'a_mixin': {
    'a': 'a',
  },
  'b_mixin': {
    'b': 'b',
  },
  'c_mixin': {
    'c': 'c',
  },
}
"""


class MixinTests(TestCase):
  """Tests for the mixins feature."""
  def test_mixins_must_be_sorted(self):
    fbb = FakeBBGen(self.args,
                    FOO_GTESTS_SORTING_MIXINS_WATERFALL,
                    FOO_TEST_SUITE,
                    LUCI_MILO_CFG,
                    mixins=SWARMING_MIXINS_SORTED)
    fbb.check_input_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

    fbb = FakeBBGen(self.args,
                    FOO_GTESTS_SORTING_MIXINS_WATERFALL,
                    FOO_TEST_SUITE,
                    LUCI_MILO_CFG,
                    mixins=SWARMING_MIXINS_UNSORTED)
    with self.assertRaises(generate_buildbot_json.BBGenErr):
      fbb.check_input_file_consistency(verbose=True)
    joined_lines = '\n'.join(fbb.printed_lines)
    self.assertRegex(joined_lines, r'.*\+ ._mixin.*')
    self.assertRegex(joined_lines, r'.*\- ._mixin.*')
    fbb.printed_lines = []
    self.assertFalse(fbb.printed_lines)

  def test_waterfall(self):
    fbb = FakeBBGen(self.args,
                    FOO_GTESTS_WATERFALL_MIXIN_WATERFALL,
                    FOO_TEST_SUITE,
                    LUCI_MILO_CFG,
                    mixins=SWARMING_MIXINS)
    fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_waterfall_exception_overrides(self):
    fbb = FakeBBGen(self.args,
                    FOO_GTESTS_WATERFALL_MIXIN_WATERFALL,
                    FOO_TEST_SUITE,
                    LUCI_MILO_CFG,
                    exceptions=SCRIPT_WITH_ARGS_SWARMING_EXCEPTIONS,
                    mixins=SWARMING_MIXINS)
    fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_autoshard_exceptions(self):
    fbb = FakeBBGen(self.args,
                    FOO_GTESTS_WATERFALL_MIXIN_WATERFALL,
                    FOO_TEST_SUITE,
                    LUCI_MILO_CFG,
                    exceptions=SCRIPT_WITH_ARGS_SWARMING_EXCEPTIONS,
                    autoshard_exceptions=AUTOSHARD_EXCEPTIONS,
                    mixins=SWARMING_MIXINS)
    fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_autoshard_exceptions_variant_names(self):
    fbb = FakeBBGen(self.args,
                    MATRIX_GTEST_SUITE_WATERFALL,
                    MATRIX_COMPOUND_TEST_WITH_TEST_KEY,
                    LUCI_MILO_CFG,
                    autoshard_exceptions=AUTOSHARD_EXCEPTIONS,
                    variants=VARIANTS_FILE)
    fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_builder(self):
    fbb = FakeBBGen(self.args,
                    FOO_GTESTS_BUILDER_MIXIN_WATERFALL,
                    FOO_TEST_SUITE,
                    LUCI_MILO_CFG,
                    mixins=SWARMING_MIXINS)
    fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_builder_non_swarming(self):
    fbb = FakeBBGen(self.args,
                    FOO_GTESTS_BUILDER_MIXIN_NON_SWARMING_WATERFALL,
                    FOO_TEST_SUITE,
                    LUCI_MILO_CFG,
                    mixins=SWARMING_MIXINS)
    fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_test_suite(self):
    fbb = FakeBBGen(self.args,
                    FOO_GTESTS_WATERFALL,
                    FOO_TEST_SUITE_WITH_MIXIN,
                    LUCI_MILO_CFG,
                    mixins=SWARMING_MIXINS)
    fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_dimension(self):
    fbb = FakeBBGen(self.args,
                    FOO_GTESTS_DIMENSIONS_MIXIN_WATERFALL,
                    FOO_TEST_SUITE_WITH_MIXIN,
                    LUCI_MILO_CFG,
                    mixins=SWARMING_MIXINS)
    fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_dimension_gpu(self):
    fbb = FakeBBGen(self.args,
                    FOO_GPU_TELEMETRY_TEST_DIMENSIONS_WATERFALL,
                    FOO_TEST_SUITE_WITH_MIXIN,
                    LUCI_MILO_CFG,
                    mixins=SWARMING_MIXINS,
                    gn_isolate_map=GPU_TELEMETRY_GN_ISOLATE_MAP)
    fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_unreferenced(self):
    fbb = FakeBBGen(self.args,
                    FOO_GTESTS_WATERFALL,
                    FOO_TEST_SUITE_WITH_MIXIN,
                    LUCI_MILO_CFG,
                    mixins=SWARMING_MIXINS)
    with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
                                '.*mixins are unreferenced.*'):
      fbb.check_input_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_unused(self):
    fbb = FakeBBGen(self.args,
                    FOO_GTESTS_INVALID_NOTFOUND_MIXIN_WATERFALL,
                    FOO_TEST_SUITE_WITH_MIXIN,
                    LUCI_MILO_CFG,
                    mixins=SWARMING_MIXINS)
    with self.assertRaises(generate_buildbot_json.BBGenErr):
      fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_list(self):
    fbb = FakeBBGen(self.args,
                    FOO_GTESTS_INVALID_LIST_MIXIN_WATERFALL,
                    FOO_TEST_SUITE_WITH_MIXIN,
                    LUCI_MILO_CFG,
                    mixins=SWARMING_MIXINS)
    with self.assertRaises(generate_buildbot_json.BBGenErr):
      fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)


  def test_no_duplicate_keys(self):
    fbb = FakeBBGen(self.args,
                    FOO_GTESTS_BUILDER_MIXIN_WATERFALL,
                    FOO_TEST_SUITE,
                    LUCI_MILO_CFG,
                    mixins=SWARMING_MIXINS_DUPLICATED)
    with self.assertRaisesRegex(
        generate_buildbot_json.BBGenErr,
        ('The following files have invalid keys: ' +
         re.escape(self.args.mixins_pyl_path)),
    ):
      fbb.check_input_file_consistency(verbose=True)
    joined_lines = '\n'.join(fbb.printed_lines)
    self.assertRegex(joined_lines, r'.*\- builder_mixin')
    fbb.printed_lines = []
    self.assertFalse(fbb.printed_lines)

  def test_no_duplicate_keys_basic_test_suite(self):
    fbb = FakeBBGen(self.args, FOO_GTESTS_WATERFALL, FOO_TEST_SUITE_NOT_SORTED,
                    LUCI_MILO_CFG)
    with self.assertRaisesRegex(
        generate_buildbot_json.BBGenErr,
        ('The following files have invalid keys: ' +
         re.escape(self.args.test_suites_pyl_path)),
    ):
      fbb.check_input_file_consistency(verbose=True)
    joined_lines = '\n'.join(fbb.printed_lines)
    self.assertRegex(joined_lines, r'.*\- a_test')
    self.assertRegex(joined_lines, r'.*\+ a_test')
    fbb.printed_lines = []
    self.assertFalse(fbb.printed_lines)

  def test_type_assert_printing_help(self):
    fbb = FakeBBGen(self.args, FOO_GTESTS_WATERFALL, TEST_SUITES_SYNTAX_ERROR,
                    LUCI_MILO_CFG)
    with self.assertRaisesRegex(
        generate_buildbot_json.BBGenErr,
        f'Invalid \\.pyl file '
        f"'{re.escape(self.args.test_suites_pyl_path)}'.*",
    ):
      fbb.check_input_file_consistency(verbose=True)
    self.assertEqual(fbb.printed_lines, [
        f'== {self.args.test_suites_pyl_path} ==',
        '<snip>',
        '1 {',
        "2   'basic_suites': {",
        '--------------------------------------------------------------------'
        '------------',
        '3     3: {',
        '-------^------------------------------------------------------------'
        '------------',
        "4       'suite_c': {},",
        '5     },',
        '<snip>',
    ])

  def test_mixin_append(self):
    fbb = FakeBBGen(self.args,
                    FOO_GTESTS_BUILDER_MIXIN_WATERFALL,
                    FOO_TEST_SUITE_WITH_ARGS,
                    LUCI_MILO_CFG,
                    mixins=MIXIN_APPEND)
    with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
                                r'.*\$mixin_append is no longer supported.*'):
      fbb.check_input_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_swarming_named_caches(self):
    fbb = FakeBBGen(self.args,
                    FOO_GTESTS_BUILDER_MIXIN_WATERFALL,
                    FOO_TEST_SUITE_WITH_SWARMING_NAMED_CACHES,
                    LUCI_MILO_CFG,
                    mixins=SWARMING_NAMED_CACHES)
    fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_args_field_not_list(self):
    fbb = FakeBBGen(self.args,
                    FOO_GTESTS_BUILDER_MIXIN_WATERFALL,
                    FOO_TEST_SUITE_WITH_ARGS,
                    LUCI_MILO_CFG,
                    mixins=MIXIN_ARGS_NOT_LIST)
    with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
                                '"args" must be a list'):
      fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_args_field_merging(self):
    fbb = FakeBBGen(self.args,
                    FOO_GTESTS_BUILDER_MIXIN_WATERFALL,
                    FOO_TEST_SUITE_WITH_ARGS,
                    LUCI_MILO_CFG,
                    mixins=MIXIN_ARGS)
    fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_linux_args_field_merging(self):
    fbb = FakeBBGen(self.args,
                    FOO_LINUX_GTESTS_BUILDER_MIXIN_WATERFALL,
                    FOO_TEST_SUITE_WITH_ARGS,
                    LUCI_MILO_CFG,
                    mixins=MIXIN_LINUX_ARGS)
    fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_remove_mixin_test_remove_waterfall(self):
    fbb = FakeBBGen(self.args,
                    FOO_GTESTS_WATERFALL_MIXIN_WATERFALL,
                    FOO_TEST_SUITE_WITH_REMOVE_WATERFALL_MIXIN,
                    LUCI_MILO_CFG,
                    mixins=SWARMING_MIXINS)
    fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_remove_mixin_test_remove_builder(self):
    fbb = FakeBBGen(self.args,
                    FOO_GTESTS_BUILDER_MIXIN_WATERFALL,
                    FOO_TEST_SUITE_WITH_REMOVE_BUILDER_MIXIN,
                    LUCI_MILO_CFG,
                    mixins=SWARMING_MIXINS)
    fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

TEST_SUITE_WITH_PARAMS = """\
{
  'basic_suites': {
    'bar_tests': {
      'bar_test': {
        'args': ['--no-xvfb'],
        'swarming': {
          'dimensions': {
            'device_os': 'NMF26U'
          },
        },
        'should_retry_with_patch': False,
      },
      'bar_test_test': {
        'swarming': {
          'dimensions': {
            'kvm': '1'
          },
          'hard_timeout': 1000
        },
        'should_retry_with_patch': True
      }
    },
    'foo_tests': {
      'foo_test_empty': {},
      'foo_test': {
        'args': [
          '--jobs=1',
          '--verbose'
        ],
        'swarming': {
          'dimensions': {
            'device_os': 'MMB29Q'
          },
          'hard_timeout': 1800
        }
      },
      'pls': {
        'swarming': {
        },
      },
    },
  },
  'compound_suites': {
    'composition_tests': [
        'foo_tests',
        'bar_tests',
    ],
  },
}
"""
TEST_QUERY_BOTS_OUTPUT = {
    'Fake Android M Tester': {
        'gtest_tests': [{
            'name': 'foo_test',
            'test': 'foo_test',
        }]
    },
    'Fake Android L Tester': {
        'gtest_tests': [{
            'test':
            'foo_test',
            'args': [
                '--gs-results-bucket=chromium-result-details',
                '--recover-devices'
            ],
            'merge': {
                'script': '//testing/merge_scripts/standard_gtest_merge.py'
            },
            'name':
            'foo_test',
            'swarming': {
                'dimensions': {
                    'device_os': 'LMY41U',
                    'device_os_type': 'user',
                    'device_type': 'hammerhead',
                    'os': 'Android'
                },
            }
        }]
    },
    'Fake Android K Tester': {
        'additional_compile_targets': ['bar_test'],
        'gtest_tests': [{
            'test':
            'foo_test',
            'args': [
                '--gs-results-bucket=chromium-result-details',
                '--recover-devices'
            ],
            'merge': {
                'script': '//testing/merge_scripts/standard_gtest_merge.py'
            },
            'name':
            'foo_test',
            'swarming': {
                'dimensions': {
                    'device_os': 'KTU84P',
                    'device_os_type': 'userdebug',
                    'device_type': 'hammerhead',
                    'os': 'Android',
                },
            }
        }]
    },
    'Android Builder': {
        'additional_compile_targets': ['bar_test']
    }
}
TEST_QUERY_BOTS_TESTS_OUTPUT = {
    'Fake Android M Tester': [{
        'name': 'foo_test',
        'test': 'foo_test',
    }],
    'Fake Android L Tester': [{
        'test':
        'foo_test',
        'args':
        ['--gs-results-bucket=chromium-result-details', '--recover-devices'],
        'merge': {
            'script': '//testing/merge_scripts/standard_gtest_merge.py'
        },
        'name':
        'foo_test',
        'swarming': {
            'dimensions': {
                'device_os': 'LMY41U',
                'device_os_type': 'user',
                'device_type': 'hammerhead',
                'os': 'Android'
            },
        }
    }],
    'Android Builder': [],
    'Fake Android K Tester': [{
        'test':
        'foo_test',
        'args':
        ['--gs-results-bucket=chromium-result-details', '--recover-devices'],
        'merge': {
            'script': '//testing/merge_scripts/standard_gtest_merge.py'
        },
        'name':
        'foo_test',
        'swarming': {
            'dimensions': {
                'device_os': 'KTU84P',
                'device_os_type': 'userdebug',
                'device_type': 'hammerhead',
                'os': 'Android'
            },
        }
    }]
}

TEST_QUERY_BOT_OUTPUT = {
    'additional_compile_targets': ['bar_test'],
    'gtest_tests': [
        {
            'test':
            'foo_test',
            'args': [
                '--gs-results-bucket=chromium-result-details',
                '--recover-devices',
            ],
            'merge': {
                'script': '//testing/merge_scripts/standard_gtest_merge.py',
            },
            'name':
            'foo_test',
            'swarming': {
                'dimensions': {
                    'device_os': 'KTU84P',
                    'device_os_type': 'userdebug',
                    'device_type': 'hammerhead',
                    'os': 'Android',
                },
            },
        },
    ],
}
TEST_QUERY_BOT_TESTS_OUTPUT = [{
    'test':
    'foo_test',
    'args':
    ['--gs-results-bucket=chromium-result-details', '--recover-devices'],
    'merge': {
        'script': '//testing/merge_scripts/standard_gtest_merge.py'
    },
    'name':
    'foo_test',
    'swarming': {
        'dimensions': {
            'device_os': 'LMY41U',
            'device_os_type': 'user',
            'device_type': 'hammerhead',
            'os': 'Android'
        },
    }
}]

TEST_QUERY_TESTS_OUTPUT = {
    'bar_test': {
        'name': 'bar_test',
        'swarming': {
            'dimensions': {
                'os': 'Linux'
            },
        }
    },
    'foo_test': {
        'name': 'foo_test',
        'swarming': {
            'dimensions': {
                'os': 'Linux'
            },
        }
    }
}

TEST_QUERY_TESTS_MULTIPLE_PARAMS_OUTPUT = ['foo_test']

TEST_QUERY_TESTS_DIMENSION_PARAMS_OUTPUT = ['bar_test']

TEST_QUERY_TESTS_SWARMING_PARAMS_OUTPUT = ['bar_test_test']

TEST_QUERY_TESTS_PARAMS_OUTPUT = ['bar_test_test']

TEST_QUERY_TESTS_PARAMS_FALSE_OUTPUT = ['bar_test']

TEST_QUERY_TEST_OUTPUT = {
    'name': 'foo_test',
    'swarming': {
        'dimensions': {
            'os': 'Linux',
        },
    },
}

TEST_QUERY_TEST_BOTS_OUTPUT = [
    'Fake Android K Tester',
    'Fake Android L Tester',
    'Fake Android M Tester',
]

TEST_QUERY_TEST_BOTS_ISOLATED_SCRIPTS_OUTPUT = ['Fake Tester']

TEST_QUERY_TEST_BOTS_NO_BOTS_OUTPUT = []


class QueryTests(TestCase):
  """Tests for the query feature."""
  def test_query_bots(self):
    self.set_args('--query=bots')
    fbb = FakeBBGen(self.args,
                    ANDROID_WATERFALL,
                    GOOD_COMPOSITION_TEST_SUITES,
                    LUCI_MILO_CFG,
                    mixins=SWARMING_MIXINS_SORTED)
    fbb.query(fbb.args)
    query_json = json.loads(''.join(fbb.printed_lines))
    self.assertEqual(query_json, TEST_QUERY_BOTS_OUTPUT)

  def test_query_bots_invalid(self):
    self.set_args('--query=bots/blah/blah')
    fbb = FakeBBGen(self.args,
                    ANDROID_WATERFALL,
                    GOOD_COMPOSITION_TEST_SUITES,
                    LUCI_MILO_CFG,
                    mixins=SWARMING_MIXINS_SORTED)
    with self.assertRaises(SystemExit) as cm:
      fbb.query(fbb.args)
      self.assertEqual(cm.exception.code, 1)
    self.assertTrue(fbb.printed_lines)

  def test_query_bots_json(self):
    self.set_args('--query=bots', '--json=result.json')
    fbb = FakeBBGen(self.args,
                    ANDROID_WATERFALL,
                    GOOD_COMPOSITION_TEST_SUITES,
                    LUCI_MILO_CFG,
                    mixins=SWARMING_MIXINS_SORTED)
    fbb.query(fbb.args)
    self.assertFalse(fbb.printed_lines)

  def test_query_bots_tests(self):
    self.set_args('--query=bots/tests')
    fbb = FakeBBGen(self.args,
                    ANDROID_WATERFALL,
                    GOOD_COMPOSITION_TEST_SUITES,
                    LUCI_MILO_CFG,
                    mixins=SWARMING_MIXINS_SORTED)
    fbb.query(fbb.args)
    query_json = json.loads(''.join(fbb.printed_lines))
    self.assertEqual(query_json, TEST_QUERY_BOTS_TESTS_OUTPUT)

  def test_query_invalid_bots_tests(self):
    self.set_args('--query=bots/tdfjdk')
    fbb = FakeBBGen(self.args,
                    ANDROID_WATERFALL,
                    GOOD_COMPOSITION_TEST_SUITES,
                    LUCI_MILO_CFG,
                    mixins=SWARMING_MIXINS_SORTED)
    with self.assertRaises(SystemExit) as cm:
      fbb.query(fbb.args)
      self.assertEqual(cm.exception.code, 1)
    self.assertTrue(fbb.printed_lines)

  def test_query_bot(self):
    self.set_args('--query=bot/Fake Android K Tester')
    fbb = FakeBBGen(self.args,
                    ANDROID_WATERFALL,
                    GOOD_COMPOSITION_TEST_SUITES,
                    LUCI_MILO_CFG,
                    mixins=SWARMING_MIXINS_SORTED)
    fbb.query(fbb.args)
    query_json = json.loads(''.join(fbb.printed_lines))
    self.maxDiff = None  # pragma pylint: disable=attribute-defined-outside-init
    self.assertEqual(query_json, TEST_QUERY_BOT_OUTPUT)

  def test_query_bot_invalid_id(self):
    self.set_args('--query=bot/bot1')
    fbb = FakeBBGen(self.args,
                    ANDROID_WATERFALL,
                    GOOD_COMPOSITION_TEST_SUITES,
                    LUCI_MILO_CFG,
                    mixins=SWARMING_MIXINS_SORTED)
    with self.assertRaises(SystemExit) as cm:
      fbb.query(fbb.args)
      self.assertEqual(cm.exception.code, 1)
    self.assertTrue(fbb.printed_lines)

  def test_query_bot_invalid_query_too_many(self):
    self.set_args('--query=bot/Fake Android K Tester/blah/blah')
    fbb = FakeBBGen(self.args,
                    ANDROID_WATERFALL,
                    GOOD_COMPOSITION_TEST_SUITES,
                    LUCI_MILO_CFG,
                    mixins=SWARMING_MIXINS_SORTED)
    with self.assertRaises(SystemExit) as cm:
      fbb.query(fbb.args)
      self.assertEqual(cm.exception.code, 1)
    self.assertTrue(fbb.printed_lines)

  def test_query_bot_invalid_query_no_tests(self):
    self.set_args('--query=bot/Fake Android K Tester/blahs')
    fbb = FakeBBGen(self.args,
                    ANDROID_WATERFALL,
                    GOOD_COMPOSITION_TEST_SUITES,
                    LUCI_MILO_CFG,
                    mixins=SWARMING_MIXINS_SORTED)
    with self.assertRaises(SystemExit) as cm:
      fbb.query(fbb.args)
      self.assertEqual(cm.exception.code, 1)
    self.assertTrue(fbb.printed_lines)

  def test_query_bot_tests(self):
    self.set_args('--query=bot/Fake Android L Tester/tests')
    fbb = FakeBBGen(self.args,
                    ANDROID_WATERFALL,
                    GOOD_COMPOSITION_TEST_SUITES,
                    LUCI_MILO_CFG,
                    mixins=SWARMING_MIXINS_SORTED)
    fbb.query(fbb.args)
    query_json = json.loads(''.join(fbb.printed_lines))
    self.assertEqual(query_json, TEST_QUERY_BOT_TESTS_OUTPUT)

  def test_query_tests(self):
    self.set_args('--query=tests')
    fbb = FakeBBGen(self.args,
                    ANDROID_WATERFALL,
                    GOOD_COMPOSITION_TEST_SUITES,
                    LUCI_MILO_CFG,
                    mixins=SWARMING_MIXINS_SORTED)
    fbb.query(fbb.args)
    query_json = json.loads(''.join(fbb.printed_lines))
    self.assertEqual(query_json, TEST_QUERY_TESTS_OUTPUT)

  def test_query_tests_invalid(self):
    self.set_args('--query=tests/blah/blah')
    fbb = FakeBBGen(self.args,
                    ANDROID_WATERFALL,
                    GOOD_COMPOSITION_TEST_SUITES,
                    LUCI_MILO_CFG,
                    mixins=SWARMING_MIXINS_SORTED)
    with self.assertRaises(SystemExit) as cm:
      fbb.query(fbb.args)
      self.assertEqual(cm.exception.code, 1)
    self.assertTrue(fbb.printed_lines)

  def test_query_tests_multiple_params(self):
    self.set_args('--query=tests/--jobs=1&--verbose')
    fbb = FakeBBGen(self.args,
                    ANDROID_WATERFALL,
                    TEST_SUITE_WITH_PARAMS,
                    LUCI_MILO_CFG,
                    mixins=SWARMING_MIXINS_SORTED)
    fbb.query(fbb.args)
    query_json = json.loads(''.join(fbb.printed_lines))
    self.assertEqual(query_json, TEST_QUERY_TESTS_MULTIPLE_PARAMS_OUTPUT)

  def test_query_tests_invalid_params(self):
    self.set_args('--query=tests/device_os?')
    fbb = FakeBBGen(self.args,
                    ANDROID_WATERFALL,
                    TEST_SUITE_WITH_PARAMS,
                    LUCI_MILO_CFG,
                    mixins=SWARMING_MIXINS_SORTED)
    with self.assertRaises(SystemExit) as cm:
      fbb.query(fbb.args)
      self.assertEqual(cm.exception.code, 1)
    self.assertTrue(fbb.printed_lines)

  def test_query_tests_dimension_params(self):
    self.set_args('--query=tests/device_os:NMF26U')
    fbb = FakeBBGen(self.args,
                    ANDROID_WATERFALL,
                    TEST_SUITE_WITH_PARAMS,
                    LUCI_MILO_CFG,
                    mixins=SWARMING_MIXINS_SORTED)
    fbb.query(fbb.args)
    query_json = json.loads(''.join(fbb.printed_lines))
    self.assertEqual(query_json, TEST_QUERY_TESTS_DIMENSION_PARAMS_OUTPUT)

  def test_query_tests_swarming_params(self):
    self.set_args('--query=tests/hard_timeout:1000')
    fbb = FakeBBGen(self.args,
                    ANDROID_WATERFALL,
                    TEST_SUITE_WITH_PARAMS,
                    LUCI_MILO_CFG,
                    mixins=SWARMING_MIXINS_SORTED)
    fbb.query(fbb.args)
    query_json = json.loads(''.join(fbb.printed_lines))
    self.assertEqual(query_json, TEST_QUERY_TESTS_SWARMING_PARAMS_OUTPUT)

  def test_query_tests_params(self):
    self.set_args('--query=tests/should_retry_with_patch:true')
    fbb = FakeBBGen(self.args,
                    ANDROID_WATERFALL,
                    TEST_SUITE_WITH_PARAMS,
                    LUCI_MILO_CFG,
                    mixins=SWARMING_MIXINS_SORTED)
    fbb.query(fbb.args)
    query_json = json.loads(''.join(fbb.printed_lines))
    self.assertEqual(query_json, TEST_QUERY_TESTS_PARAMS_OUTPUT)

  def test_query_tests_params_false(self):
    self.set_args('--query=tests/should_retry_with_patch:false')
    fbb = FakeBBGen(self.args,
                    ANDROID_WATERFALL,
                    TEST_SUITE_WITH_PARAMS,
                    LUCI_MILO_CFG,
                    mixins=SWARMING_MIXINS_SORTED)
    fbb.query(fbb.args)
    query_json = json.loads(''.join(fbb.printed_lines))
    self.assertEqual(query_json, TEST_QUERY_TESTS_PARAMS_FALSE_OUTPUT)

  def test_query_test(self):
    self.set_args('--query=test/foo_test')
    fbb = FakeBBGen(self.args,
                    ANDROID_WATERFALL,
                    GOOD_COMPOSITION_TEST_SUITES,
                    LUCI_MILO_CFG,
                    mixins=SWARMING_MIXINS_SORTED)
    fbb.query(fbb.args)
    query_json = json.loads(''.join(fbb.printed_lines))
    self.assertEqual(query_json, TEST_QUERY_TEST_OUTPUT)

  def test_query_test_invalid_id(self):
    self.set_args('--query=test/foo_foo')
    fbb = FakeBBGen(self.args,
                    ANDROID_WATERFALL,
                    GOOD_COMPOSITION_TEST_SUITES,
                    LUCI_MILO_CFG,
                    mixins=SWARMING_MIXINS_SORTED)
    with self.assertRaises(SystemExit) as cm:
      fbb.query(fbb.args)
      self.assertEqual(cm.exception.code, 1)
    self.assertTrue(fbb.printed_lines)

  def test_query_test_invalid_length(self):
    self.set_args('--query=test/foo_tests/foo/foo')
    fbb = FakeBBGen(self.args,
                    ANDROID_WATERFALL,
                    GOOD_COMPOSITION_TEST_SUITES,
                    LUCI_MILO_CFG,
                    mixins=SWARMING_MIXINS_SORTED)
    with self.assertRaises(SystemExit) as cm:
      fbb.query(fbb.args)
      self.assertEqual(cm.exception.code, 1)
    self.assertTrue(fbb.printed_lines)

  def test_query_test_bots(self):
    self.set_args('--query=test/foo_test/bots')
    fbb = FakeBBGen(self.args,
                    ANDROID_WATERFALL,
                    GOOD_COMPOSITION_TEST_SUITES,
                    LUCI_MILO_CFG,
                    mixins=SWARMING_MIXINS_SORTED)
    fbb.query(fbb.args)
    query_json = json.loads(''.join(fbb.printed_lines))
    self.assertEqual(query_json, TEST_QUERY_TEST_BOTS_OUTPUT)

  def test_query_test_bots_isolated_scripts(self):
    self.set_args('--query=test/foo_test/bots')
    fbb = FakeBBGen(self.args,
                    FOO_ISOLATED_SCRIPTS_WATERFALL,
                    GOOD_COMPOSITION_TEST_SUITES,
                    LUCI_MILO_CFG,
                    mixins=SWARMING_MIXINS_SORTED)
    fbb.query(fbb.args)
    query_json = json.loads(''.join(fbb.printed_lines))
    self.assertEqual(query_json, TEST_QUERY_TEST_BOTS_ISOLATED_SCRIPTS_OUTPUT)

  def test_query_test_bots_invalid(self):
    self.set_args('--query=test/foo_tests/foo')
    fbb = FakeBBGen(self.args,
                    ANDROID_WATERFALL,
                    GOOD_COMPOSITION_TEST_SUITES,
                    LUCI_MILO_CFG,
                    mixins=SWARMING_MIXINS_SORTED)
    with self.assertRaises(SystemExit) as cm:
      fbb.query(fbb.args)
      self.assertEqual(cm.exception.code, 1)
    self.assertTrue(fbb.printed_lines)

  def test_query_test_bots_no_bots(self):
    self.set_args('--query=test/bar_tests/bots')
    fbb = FakeBBGen(self.args,
                    ANDROID_WATERFALL,
                    GOOD_COMPOSITION_TEST_SUITES,
                    LUCI_MILO_CFG,
                    mixins=SWARMING_MIXINS_SORTED)
    fbb.query(fbb.args)
    query_json = json.loads(''.join(fbb.printed_lines))
    self.assertEqual(query_json, TEST_QUERY_TEST_BOTS_NO_BOTS_OUTPUT)

  def test_query_invalid(self):
    self.set_args('--query=foo')
    fbb = FakeBBGen(self.args,
                    ANDROID_WATERFALL,
                    GOOD_COMPOSITION_TEST_SUITES,
                    LUCI_MILO_CFG,
                    mixins=SWARMING_MIXINS_SORTED)
    with self.assertRaises(SystemExit) as cm:
      fbb.query(fbb.args)
      self.assertEqual(cm.exception.code, 1)
    self.assertTrue(fbb.printed_lines)


FOO_TEST_SUITE_WITH_ENABLE_FEATURES_SEPARATE_ENTRIES = """\
{
  'basic_suites': {
    'foo_tests': {
      'foo_test': {
        'args': [
          '--enable-features',
          'Foo,Bar',
        ],
      },
    },
  },
}
"""


FOO_TEST_REPLACEMENTS_REMOVE_NO_VALUE = """\
{
  'foo_test': {
    'replacements': {
      'Fake Tester': {
        'args': {
          '--c_arg': None,
        },
      },
    },
  },
}
"""


FOO_TEST_REPLACEMENTS_REMOVE_VALUE = """\
{
  'foo_test': {
    'replacements': {
      'Fake Tester': {
        'args': {
          '--enable-features': None,
        },
      },
    },
  },
}
"""


FOO_TEST_REPLACEMENTS_REPLACE_VALUE = """\
{
  'foo_test': {
    'replacements': {
      'Fake Tester': {
        'args': {
          '--enable-features': 'Bar,Baz',
        },
      },
    },
  },
}
"""


FOO_TEST_REPLACEMENTS_INVALID_KEY = """\
{
  'foo_test': {
    'replacements': {
      'Fake Tester': {
        'invalid': {
          '--enable-features': 'Bar,Baz',
        },
      },
    },
  },
}
"""


class ReplacementTests(TestCase):
  """Tests for the arg replacement feature."""
  def test_replacement_valid_remove_no_value(self):
    fbb = FakeBBGen(self.args,
                    FOO_GTESTS_WATERFALL,
                    FOO_TEST_SUITE_WITH_ARGS,
                    LUCI_MILO_CFG,
                    exceptions=FOO_TEST_REPLACEMENTS_REMOVE_NO_VALUE)
    fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_replacement_valid_remove_value(self):
    fbb = FakeBBGen(self.args,
                    FOO_GTESTS_WATERFALL,
                    FOO_TEST_SUITE_WITH_ENABLE_FEATURES,
                    LUCI_MILO_CFG,
                    exceptions=FOO_TEST_REPLACEMENTS_REMOVE_VALUE)
    fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_replacement_valid_replace_value(self):
    fbb = FakeBBGen(self.args,
                    FOO_GTESTS_WATERFALL,
                    FOO_TEST_SUITE_WITH_ENABLE_FEATURES,
                    LUCI_MILO_CFG,
                    exceptions=FOO_TEST_REPLACEMENTS_REPLACE_VALUE)
    fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_replacement_valid_replace_value_separate_entries(self):
    fbb = FakeBBGen(self.args,
                    FOO_GTESTS_WATERFALL,
                    FOO_TEST_SUITE_WITH_ENABLE_FEATURES_SEPARATE_ENTRIES,
                    LUCI_MILO_CFG,
                    exceptions=FOO_TEST_REPLACEMENTS_REPLACE_VALUE)
    fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_replacement_invalid_key_not_valid(self):
    fbb = FakeBBGen(self.args,
                    FOO_GTESTS_WATERFALL,
                    FOO_TEST_SUITE,
                    LUCI_MILO_CFG,
                    exceptions=FOO_TEST_REPLACEMENTS_INVALID_KEY)
    with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
                                'Given replacement key *'):
      fbb.check_output_file_consistency(verbose=True)

  def test_replacement_invalid_key_not_found(self):
    fbb = FakeBBGen(self.args,
                    FOO_GTESTS_WATERFALL,
                    FOO_TEST_SUITE_WITH_ARGS,
                    LUCI_MILO_CFG,
                    exceptions=FOO_TEST_REPLACEMENTS_REPLACE_VALUE)
    with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
                                'Could not find *'):
      fbb.check_output_file_consistency(verbose=True)


FOO_TEST_SUITE_WITH_MAGIC_ARGS = """\
{
  'basic_suites': {
    'foo_tests': {
      'foo_test': {
        'args': [
          '$$MAGIC_SUBSTITUTION_TestOnlySubstitution',
        ],
      },
    },
  },
}
"""


FOO_TEST_SUITE_WITH_INVALID_MAGIC_ARGS = """\
{
  'basic_suites': {
    'foo_tests': {
      'foo_test': {
        'args': [
          '$$MAGIC_SUBSTITUTION_NotARealSubstitution',
        ],
      },
    },
  },
}
"""


class MagicSubstitutionTests(TestCase):
  """Tests for the magic substitution feature."""
  def test_valid_function(self):
    fbb = FakeBBGen(self.args, FOO_GTESTS_WATERFALL,
                    FOO_TEST_SUITE_WITH_MAGIC_ARGS, LUCI_MILO_CFG)
    fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_invalid_function(self):
    fbb = FakeBBGen(self.args, FOO_GTESTS_WATERFALL,
                    FOO_TEST_SUITE_WITH_INVALID_MAGIC_ARGS, LUCI_MILO_CFG)
    with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
                                'Magic substitution function *'):
      fbb.check_output_file_consistency(verbose=True)


# Matrix compound composition test suites

MATRIX_COMPOUND_EMPTY = """\
{
  'basic_suites': {
    'bar_tests': {
      'bar_test': {},
    },
    'foo_tests': {
      'foo_test': {},
    },
  },
  'matrix_compound_suites': {
    'matrix_tests': {
      'foo_tests': {},
      'bar_tests': {},
    },
  },
}
"""

MATRIX_COMPOUND_EMPTY_WITH_DESCRIPTION = """\
{
  'basic_suites': {
    'bar_tests': {
      'bar_test': {
        'description': 'This is a bar test',
      },
    },
    'foo_tests': {
      'foo_test': {},
    },
  },
  'matrix_compound_suites': {
    'matrix_tests': {
      'foo_tests': {},
      'bar_tests': {},
    },
  },
}
"""

MATRIX_COMPOUND_MISSING_IDENTIFIER = """\
{
  'basic_suites': {
    'foo_tests': {
      'foo_test': {},
    },
  },
  'matrix_compound_suites': {
    'matrix_tests': {
      'foo_tests': {
        'variants': [
          'missing-identifier',
        ],
      },
    },
  },
}
"""

VARIANTS_FILE_MISSING_IDENTIFIER = """\
{
  'missing-identifier': {}
}
"""

MATRIX_COMPOUND_EMPTY_IDENTIFIER = """\
{
  'basic_suites': {
    'foo_tests': {
      'foo_test': {},
    },
  },
  'matrix_compound_suites': {
    'matrix_tests': {
      'foo_tests': {
        'variants': [
          'empty-identifier',
        ],
      },
    },
  },
}
"""

EMPTY_IDENTIFIER_VARIANTS = """\
{
  'empty-identifier': {
    'identifier': '',
    'swarming': {
      'dimensions': {
        'foo': 'empty identifier not allowed',
      },
    },
  },
}
"""

MATRIX_COMPOUND_TRAILING_IDENTIFIER = """\
{
  'basic_suites': {
    'foo_tests': {
      'foo_test': {},
    },
  },
  'matrix_compound_suites': {
    'matrix_tests': {
      'foo_tests': {
        'variants': [
          'trailing-whitespace',
        ],
      },
    },
  },
}
"""

TRAILING_WHITESPACE_VARIANTS = """\
{
  'trailing-whitespace': {
    'identifier': 'id ',
    'swarming': {
      'dimensions': {
        'foo': 'trailing whitespace not allowed',
      },
    },
  },
}
"""

MATRIX_REF_NONEXISTENT = """\
{
  'basic_suites': {
    'foo_tests': {
      'foo_test': {},
    },
  },
  'matrix_compound_suites': {
    'matrix_tests': {
      'bar_test': {},
    },
  },
}
"""

MATRIX_COMPOUND_REF_COMPOSITION = """\
{
  'basic_suites': {
    'foo_tests': {
      'foo_test': {},
    },
  },
  'compound_suites': {
    'sample_composition': {
      'foo_tests': {},
    },
  },
  'matrix_compound_suites': {
    'matrix_tests': {
      'sample_composition': {},
    },
  },
}
"""

MATRIX_COMPOSITION_REF_MATRIX = """\
{
  'basic_suites': {
    'foo_tests': {
      'foo_test': {},
    },
  },
  'matrix_compound_suites': {
    'a_test': {
      'foo_tests': {},
    },
    'matrix_tests': {
      'a_test': {},
    },
  },
}
"""

MATRIX_COMPOUND_VARIANTS_MIXINS_MERGE = """\
{
  'basic_suites': {
    'foo_tests': {
      'set': {
        'mixins': [ 'test_mixin' ],
      },
    },
  },
  'matrix_compound_suites': {
    'matrix_tests': {
      'foo_tests': {
        'variants': [
          {
            'mixins': [ 'dimension_mixin' ],
          },
        ],
      },
    },
  },
}
"""

MATRIX_COMPOUND_VARIANTS_MIXINS = """\
{
  'basic_suites': {
    'foo_tests': {
      'set': {
        'mixins': [ 'test_mixin' ],
      },
    },
  },
  'matrix_compound_suites': {
    'matrix_tests': {
      'foo_tests': {
        'variants': [
          {
            'mixins': [
              'dimension_mixin',
              'waterfall_mixin',
              'builder_mixin',
              'random_mixin'
            ],
          },
        ],
      },
    },
  },
}
"""

MATRIX_COMPOUND_VARIANTS_MIXINS_REMOVE = """\
{
  'basic_suites': {
    'foo_tests': {
      'set': {
        'remove_mixins': ['builder_mixin'],
      },
    },
  },
  'matrix_compound_suites': {
    'matrix_tests': {
      'foo_tests': {
        'variants': [
          {
            'mixins': [ 'builder_mixin' ],
          }
        ],
      },
    },
  },
}
"""

MATRIX_COMPOUND_CONFLICTING_TEST_SUITES = """\
{
  'basic_suites': {
    'bar_tests': {
      'baz_tests': {
        'args': [
          '--bar',
        ],
      }
    },
    'foo_tests': {
      'baz_tests': {
        'args': [
          '--foo',
        ],
      }
    },
  },
  'matrix_compound_suites': {
    'matrix_tests': {
      'bar_tests': {
        'variants': [
          'a_variant',
        ],
      },
      'foo_tests': {
        'variants': [
          'a_variant',
        ]
      }
    },
  },
}
"""

MATRIX_COMPOUND_EMPTY_VARIANTS = """\
{
  'basic_suites': {
    'foo_tests': {
      'baz_tests': {
        'args': [
          '--foo',
        ],
      }
    },
  },
  'matrix_compound_suites': {
    'matrix_tests': {
      'foo_tests': {
        'variants': [],
      }
    },
  },
}
"""

MATRIX_COMPOUND_TARGETS_ARGS = """\
{
  'basic_suites': {
    'foo_tests': {
      'args_test': {
        'args': [
          '--iam'
        ],
      },
    }
  },
  'matrix_compound_suites': {
    'matrix_tests': {
      'foo_tests': {
        'variants': [
          'an-arg',
        ],
      },
    },
  },
}
"""

ARGS_VARIANTS_FILE = """\
{
  'an-arg': {
    'identifier': 'args',
    'args': [
      '--anarg',
    ],
  },
}
"""

MATRIX_COMPOUND_TARGETS_MIXINS = """\
{
  'basic_suites': {
    'foo_tests': {
      'mixins_test': {
        'mixins': [ 'test_mixin' ],
      },
    }
  },
  'matrix_compound_suites': {
    'matrix_tests': {
      'foo_tests': {
        'mixins': [ 'random_mixin' ],
        'variants': [
          'mixins',
        ],
      },
    },
  },
}
"""

MIXINS_VARIANTS_FILE = """\
{
  'mixins': {
    'identifier': 'mixins',
    'mixins': [ 'dimension_mixin' ],
  },
}
"""

MATRIX_COMPOUND_TARGETS_SWARMING = """\
{
  'basic_suites': {
    'foo_tests': {
      'swarming_test': {
        'swarming': {
          'foo': 'bar',
          'dimensions': {
            'foo': 'bar',
          },
        },
      },
    }
  },
  'matrix_compound_suites': {
    'matrix_tests': {
      'foo_tests': {
        'variants': [
          'swarming-variant',
        ],
      },
    },
  },
}
"""

SWARMING_VARIANTS_FILE = """\
{
  'swarming-variant': {
    'identifier': 'swarming',
    'swarming': {
      'a': 'b',
      'dimensions': {
        'hello': 'world',
      },
    },
  },
}
"""

MATRIX_COMPOUND_VARIANTS_REF = """\
{
  'basic_suites': {
    'foo_tests': {
      'swarming_test': {},
    }
  },
  'matrix_compound_suites': {
    'matrix_tests': {
      'foo_tests': {
        'variants': [
          'a_variant'
        ],
      },
    },
  },
}
"""

MATRIX_COMPOUND_VARIANTS_REF_WITH_DESCRIPTION = """\
{
  'basic_suites': {
    'foo_tests': {
      'swarming_test': {
        'description': 'This is a swarming test.'
      },
    }
  },
  'matrix_compound_suites': {
    'matrix_tests': {
      'foo_tests': {
        'variants': [
          'a_variant'
        ],
      },
    },
  },
}
"""

MATRIX_COMPOUND_TEST_WITH_TEST_KEY = """\
{
  'basic_suites': {
    'foo_tests': {
      'swarming_test': {
          'test': 'foo_test_apk'
      },
    }
  },
  'matrix_compound_suites': {
    'matrix_tests': {
      'foo_tests': {
        'variants': [
          'a_variant',
        ],
      },
    },
  },
}
"""

MATRIX_COMPOUND_MIXED_VARIANTS_REF = """\
{
  'basic_suites': {
    'foo_tests': {
      'swarming_test': {},
    }
  },
  'matrix_compound_suites': {
    'matrix_tests': {
      'foo_tests': {
        'variants': [
          'a_variant',
          'b_variant',
        ],
      },
    },
  },
}
"""

VARIANTS_FILE = """\
{
  'a_variant': {
    'args': [
      '--platform',
      'device',
      '--version',
      '1'
    ],
    'identifier': 'a_variant'
  }
}
"""

VARIANTS_FILE_WITH_DESCRIPTION = """\
{
  'a_variant': {
    'identifier': 'a_variant',
    'description': 'Variant description.'
  }
}
"""

MULTI_VARIANTS_FILE = """\
{
  'a_variant': {
    'args': [
      '--platform',
      'device',
      '--version',
      '1'
    ],
    'identifier': 'a_variant'
  },
  'b_variant': {
    'args': [
      '--platform',
      'sim',
      '--version',
      '2'
    ],
    'identifier': 'b_variant'
  }
}
"""

# # Dictionary composition test suite outputs

EMPTY_SKYLAB_TEST_EXCEPTIONS = """\
{
  'tast.foo OCTOPUS_TOT': {
    'remove_from': [
      'Fake Tester',
    ]
  },
  'tast.foo OCTOPUS_TOT-1': {
    'remove_from': [
      'Fake Tester',
    ]
  }
}
"""

MATRIX_SKYLAB_WATERFALL_WITH_NO_CROS_BOARD = """\
[
  {
    'project': 'chromium',
    'bucket': 'ci',
    'name': 'chromium.test',
    'machines': {
      'Fake Tester': {
        'test_suites': {
          'skylab_tests': 'cros_skylab_basic_x86',
        },
      },
    },
  },
]
"""

MATRIX_SKYLAB_WATERFALL = """\
[
  {
    'project': 'chromium',
    'bucket': 'ci',
    'name': 'chromium.test',
    'machines': {
      'Fake Tester': {
        'test_suites': {
          'skylab_tests': 'cros_skylab_basic_x86',
        },
        'cros_board': 'octopus',
        'cros_dut_pool': 'chromium',
        'run_cft': True,
      },
    },
  },
]
"""

MATRIX_SKYLAB_WATERFALL_WITH_BUILD_TARGET_VARIANT = """\
[
  {
    'project': 'chromium',
    'bucket': 'ci',
    'name': 'chromium.test',
    'machines': {
      'Fake Tester': {
        'test_suites': {
          'skylab_tests': 'cros_skylab_basic_x86',
        },
        'cros_board': 'octopus',
        'cros_build_target': 'octopus-arc-t',
        'cros_dut_pool': 'chromium',
        'run_cft': True,
      },
    },
  },
]
"""

MATRIX_COMPOUND_SKYLAB_REF = """\
{
  'basic_suites': {
    'autotest_suites': {
      'autotest_suite': {
      },
    },
    'cros_skylab_basic': {
      'benchmark_suite': {
        'benchmark': 'something',
      },
      'chrome_all_tast_tests': {
        'tast_expr': 'dummy expr',
        'timeout': 3600,
      },
      'gtest_suite': { },
      'lacros_all_tast_tests': {
        'tast_expr': 'lacros expr',
        'timeout': 3600,
      },
    },
  },
  'compound_suites': {},
  'matrix_compound_suites': {
    'cros_skylab_basic_x86': {
      'autotest_suites': {
        'variants': [
          'octopus-89-with-autotest-name',
        ],
      },
      'cros_skylab_basic': {
        'tast_expr': 'dummy expr',
        'variants': [
          'octopus-89',
          'octopus-88',
        ],
      },
    },
  },
}
"""

SKYLAB_VARIANTS = """\
{
  'octopus-89': {
    'skylab': {
      'cros_board': 'octopus',
      'cros_model': 'casta',
      'cros_chrome_version': '89.0.3234.0',
      'cros_img': 'octopus-release/R89-13655.0.0',
    },
    'enabled': True,
    'identifier': 'OCTOPUS_TOT',
  },
  'octopus-89-with-autotest-name': {
    'skylab': {
      'cros_chrome_version': '89.0.3234.0',
      'cros_img': 'octopus-release/R89-13655.0.0',
      'autotest_name': 'unique_autotest_name',
    },
    'enabled': True,
    'identifier': 'OCTOPUS_TOT',
  },
  'octopus-88': {
    'skylab': {
      'cros_chrome_version': '88.0.2324.0',
      'cros_img': 'octopus-release/R88-13597.23.0',
    },
    'enabled': True,
    'identifier': 'OCTOPUS_TOT-1',
  },
}
"""

SKYLAB_VARIANTS_WITH_BUILD_VARIANT = """\
{
  'octopus-89': {
    'skylab': {
      'cros_board': 'octopus',
      'cros_model': 'casta',
      'cros_chrome_version': '89.0.3234.0',
      'cros_img': 'octopus-arc-t-release/R89-13655.0.0',
    },
    'enabled': True,
    'identifier': 'OCTOPUS_TOT',
  },
  'octopus-89-with-autotest-name': {
    'skylab': {
      'cros_chrome_version': '89.0.3234.0',
      'cros_img': 'octopus-arc-t-release/R89-13655.0.0',
      'autotest_name': 'unique_autotest_name',
    },
    'enabled': True,
    'identifier': 'OCTOPUS_TOT',
  },
  'octopus-88': {
    'skylab': {
      'cros_chrome_version': '88.0.2324.0',
      'cros_img': 'octopus-arc-t-release/R88-13597.23.0',
    },
    'enabled': True,
    'identifier': 'OCTOPUS_TOT-1',
  },
}
"""

ENABLED_AND_DISABLED_MATRIX_COMPOUND_SKYLAB_REF = """\
{
  'basic_suites': {
    'cros_skylab_basic': {
      'tast.basic': {
        'tast_expr': 'dummy expr',
        'suite': 'tast.basic',
        'shard_level_retries_on_ctp': 2,
        'timeout': 3600,
      },
      'tast.foo': {
        'tast_expr': 'dummy expr',
        'suite': 'tast.foo',
        'timeout': 3600,
      },
    },
  },
  'compound_suites': {},
  'matrix_compound_suites': {
    'cros_skylab_basic_x86': {
      'cros_skylab_basic': {
        'variants': [
          'enabled',
          'disabled',
        ],
      },
    },
  },
}
"""

ENABLED_AND_DISABLED_VARIANTS = """\
{
  'enabled': {
    'skylab': {
      'cros_board': 'octopus',
      'cros_chrome_version': '89.0.3234.0',
      'cros_img': 'octopus-release/R89-13655.0.0',
    },
    'enabled': True,
    'identifier': 'OCTOPUS_TOT',
  },
  'disabled': {
    'skylab': {
      'cros_board': 'octopus',
      'cros_chrome_version': '88.0.2324.0',
      'cros_img': 'octopus-release/R88-13597.23.0',
    },
    'enabled': False,
    'identifier': 'OCTOPUS_TOT-1',
  },
}
"""


class MatrixCompositionTests(TestCase):

  def test_good_structure_no_configs(self):
    """
    Tests matrix compound test suite structure with no configs,
    no conflicts and no bad references
    """
    fbb = FakeBBGen(self.args, MATRIX_GTEST_SUITE_WATERFALL,
                    MATRIX_COMPOUND_EMPTY, LUCI_MILO_CFG)
    fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_good_structure_with_description(self):
    """
    Tests matrix compound test suite structure with description.
    """
    fbb = FakeBBGen(self.args, MATRIX_GTEST_SUITE_WATERFALL,
                    MATRIX_COMPOUND_EMPTY_WITH_DESCRIPTION, LUCI_MILO_CFG)
    fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_missing_identifier(self):
    """
    Variant is missing an identifier
    """
    fbb = FakeBBGen(self.args,
                    MATRIX_GTEST_SUITE_WATERFALL,
                    MATRIX_COMPOUND_MISSING_IDENTIFIER,
                    LUCI_MILO_CFG,
                    variants=VARIANTS_FILE_MISSING_IDENTIFIER)
    with self.assertRaisesRegex(
        generate_buildbot_json.BBGenErr,
        'Missing required identifier field in matrix compound suite*'):
      fbb.check_output_file_consistency(verbose=True)

  def test_empty_identifier(self):
    """
    Variant identifier is empty.
    """
    fbb = FakeBBGen(self.args,
                    MATRIX_GTEST_SUITE_WATERFALL,
                    MATRIX_COMPOUND_EMPTY_IDENTIFIER,
                    LUCI_MILO_CFG,
                    variants=EMPTY_IDENTIFIER_VARIANTS)
    with self.assertRaisesRegex(
        generate_buildbot_json.BBGenErr,
        'Identifier field can not be "" in matrix compound suite*'):
      fbb.check_output_file_consistency(verbose=True)

  def test_trailing_identifier(self):
    """
    Variant identifier has trailing whitespace.
    """
    fbb = FakeBBGen(self.args,
                    MATRIX_GTEST_SUITE_WATERFALL,
                    MATRIX_COMPOUND_TRAILING_IDENTIFIER,
                    LUCI_MILO_CFG,
                    variants=TRAILING_WHITESPACE_VARIANTS)
    with self.assertRaisesRegex(
        generate_buildbot_json.BBGenErr,
        'Identifier field can not have leading and trailing whitespace in'
        ' matrix compound suite*'):
      fbb.check_output_file_consistency(verbose=True)

  def test_noexistent_ref(self):
    """
    Test referencing a non-existent basic test suite
    """
    fbb = FakeBBGen(self.args, MATRIX_GTEST_SUITE_WATERFALL,
                    MATRIX_REF_NONEXISTENT, LUCI_MILO_CFG)
    with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
                                'Unable to find reference to *'):
      fbb.check_output_file_consistency(verbose=True)

  def test_ref_to_composition(self):
    """
    Test referencing another composition test suite
    """
    fbb = FakeBBGen(self.args, MATRIX_GTEST_SUITE_WATERFALL,
                    MATRIX_COMPOUND_REF_COMPOSITION, LUCI_MILO_CFG)
    with self.assertRaisesRegex(
        generate_buildbot_json.BBGenErr,
        'matrix_compound_suites may not refer to other *'):
      fbb.check_output_file_consistency(verbose=True)

  def test_ref_to_matrix(self):
    """
    Test referencing another matrix test suite
    """
    fbb = FakeBBGen(self.args, MATRIX_GTEST_SUITE_WATERFALL,
                    MATRIX_COMPOSITION_REF_MATRIX, LUCI_MILO_CFG)
    with self.assertRaisesRegex(
        generate_buildbot_json.BBGenErr,
        'matrix_compound_suites may not refer to other *'):
      fbb.check_output_file_consistency(verbose=True)

  def test_conflicting_names(self):
    fbb = FakeBBGen(self.args,
                    MATRIX_GTEST_SUITE_WATERFALL,
                    MATRIX_COMPOUND_CONFLICTING_TEST_SUITES,
                    LUCI_MILO_CFG,
                    variants=VARIANTS_FILE)
    with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
                                'Conflicting test definitions.*'):
      fbb.check_input_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_empty_variants(self):
    fbb = FakeBBGen(self.args,
                    MATRIX_GTEST_SUITE_WATERFALL,
                    MATRIX_COMPOUND_EMPTY_VARIANTS,
                    LUCI_MILO_CFG,
                    mixins=SWARMING_MIXINS)
    fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_variants_swarming_dict_merge_args(self):
    """
    Test targets with swarming dictionary defined by both basic and matrix
    """
    fbb = FakeBBGen(self.args,
                    MATRIX_GTEST_SUITE_WATERFALL,
                    MATRIX_COMPOUND_TARGETS_ARGS,
                    LUCI_MILO_CFG,
                    mixins=SWARMING_MIXINS,
                    variants=ARGS_VARIANTS_FILE)
    fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_variants_swarming_dict_merge_mixins(self):
    """
    Test targets with swarming dictionary defined by both basic and matrix
    """
    fbb = FakeBBGen(self.args,
                    MATRIX_GTEST_SUITE_WATERFALL,
                    MATRIX_COMPOUND_TARGETS_MIXINS,
                    LUCI_MILO_CFG,
                    mixins=SWARMING_MIXINS,
                    variants=MIXINS_VARIANTS_FILE)
    fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_variants_swarming_dict_swarming(self):
    """
    Test targets with swarming dictionary defined by both basic and matrix
    """
    fbb = FakeBBGen(self.args,
                    MATRIX_GTEST_SUITE_WATERFALL,
                    MATRIX_COMPOUND_TARGETS_SWARMING,
                    LUCI_MILO_CFG,
                    variants=SWARMING_VARIANTS_FILE)
    fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_variant_test_suite_with_test_key(self):
    """
    Test targets in matrix compound test suites with variants
    """
    fbb = FakeBBGen(self.args,
                    MATRIX_GTEST_SUITE_WATERFALL,
                    MATRIX_COMPOUND_TEST_WITH_TEST_KEY,
                    LUCI_MILO_CFG,
                    variants=VARIANTS_FILE)
    fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_variants_with_description(self):
    """Test variants with description field"""
    fbb = FakeBBGen(self.args,
                    MATRIX_GTEST_SUITE_WATERFALL,
                    MATRIX_COMPOUND_VARIANTS_REF,
                    LUCI_MILO_CFG,
                    variants=VARIANTS_FILE_WITH_DESCRIPTION)
    fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_both_test_suite_and_variants_with_description(self):
    """Test both test suite and variants with description field"""
    fbb = FakeBBGen(self.args,
                    MATRIX_GTEST_SUITE_WATERFALL,
                    MATRIX_COMPOUND_VARIANTS_REF_WITH_DESCRIPTION,
                    LUCI_MILO_CFG,
                    variants=VARIANTS_FILE_WITH_DESCRIPTION)
    fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_variants_pyl_ref(self):
    """Test targets with variants string ref"""
    fbb = FakeBBGen(self.args,
                    MATRIX_GTEST_SUITE_WATERFALL,
                    MATRIX_COMPOUND_VARIANTS_REF,
                    LUCI_MILO_CFG,
                    variants=VARIANTS_FILE)
    fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_variants_pyl_no_ref(self):
    """Test targets with variants string ref, not defined in variants.pyl"""
    fbb = FakeBBGen(self.args, MATRIX_GTEST_SUITE_WATERFALL,
                    MATRIX_COMPOUND_VARIANTS_REF, LUCI_MILO_CFG)
    with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
                                'Missing variant definition for *'):
      fbb.check_output_file_consistency(verbose=True)

  def test_variants_pyl_all_unreferenced(self):
    """Test targets with variants in variants.pyl, unreferenced in tests"""
    fbb = FakeBBGen(self.args,
                    MATRIX_GTEST_SUITE_WATERFALL,
                    MATRIX_COMPOUND_VARIANTS_REF,
                    LUCI_MILO_CFG,
                    variants=MULTI_VARIANTS_FILE)
    with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
                                'The following variants were unreferenced *'):
      fbb.check_input_file_consistency(verbose=True)

  def test_good_skylab_matrix_with_variants(self):
    fbb = FakeBBGen(self.args,
                    MATRIX_SKYLAB_WATERFALL,
                    MATRIX_COMPOUND_SKYLAB_REF,
                    LUCI_MILO_CFG,
                    exceptions=EMPTY_SKYLAB_TEST_EXCEPTIONS,
                    variants=SKYLAB_VARIANTS)
    fbb.check_input_file_consistency(verbose=True)
    fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_good_skylab_matrix_with_build_target_variant_and_variants(self):
    fbb = FakeBBGen(self.args,
                    MATRIX_SKYLAB_WATERFALL_WITH_BUILD_TARGET_VARIANT,
                    MATRIX_COMPOUND_SKYLAB_REF,
                    LUCI_MILO_CFG,
                    exceptions=EMPTY_SKYLAB_TEST_EXCEPTIONS,
                    variants=SKYLAB_VARIANTS_WITH_BUILD_VARIANT)
    fbb.check_input_file_consistency(verbose=True)
    fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_enabled_and_disabled_skylab_matrix_with_variants(self):
    """Test with disabled variants"""
    fbb = FakeBBGen(self.args,
                    MATRIX_SKYLAB_WATERFALL,
                    ENABLED_AND_DISABLED_MATRIX_COMPOUND_SKYLAB_REF,
                    LUCI_MILO_CFG,
                    exceptions=EMPTY_SKYLAB_TEST_EXCEPTIONS,
                    variants=ENABLED_AND_DISABLED_VARIANTS)
    # some skylab test variant is disabled; the corresponding skylab tests
    # is not generated.
    fbb.check_input_file_consistency(verbose=True)
    fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_invalid_skylab_matrix_with_variants(self):
    fbb = FakeBBGen(self.args,
                    MATRIX_SKYLAB_WATERFALL_WITH_NO_CROS_BOARD,
                    MATRIX_COMPOUND_SKYLAB_REF,
                    LUCI_MILO_CFG,
                    exceptions=EMPTY_SKYLAB_TEST_EXCEPTIONS,
                    variants=SKYLAB_VARIANTS)
    with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
                                'skylab tests must specify cros_board.'):
      fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)


MAC_TEST_SUITE = """\
{
  'basic_suites': {
    'foo_tests': {
      'foo_test': {
      },
    },
  },
}
"""

NO_DIMENSIONS_GTESTS_WATERFALL = """\
[
  {
    'project': 'chromium',
    'bucket': 'ci',
    'name': 'chromium.test',
    'machines': {
      'Mac': {
        'swarming': {},
        'test_suites': {
          'gtest_tests': 'foo_tests',
        },
      },
    },
  },
]
"""

NO_OS_GTESTS_WATERFALL = """\
[
  {
    'project': 'chromium',
    'bucket': 'ci',
    'name': 'chromium.test',
    'machines': {
      'Mac': {
        'swarming': {
          'dimensions': {
            'foo': 'bar',
          },
        },
        'test_suites': {
          'gtest_tests': 'foo_tests',
        },
      },
    },
  },
]
"""

MAC_ISOLATED_SCRIPTS_WATERFALL = """\
[
  {
    'project': 'chromium',
    'bucket': 'ci',
    'name': 'chromium.test',
    'machines': {
      'Mac': {
        'swarming': {
          'dimensions': {
            'os': 'Mac',
          },
        },
        'test_suites': {
          'isolated_scripts': 'foo_tests',
        },
      },
    },
  },
]
"""

MAC_LUCI_MILO_CFG = """\
consoles {
  builders {
    name: "buildbucket/luci.chromium.ci/Mac"
  }
}
"""


class SwarmingTests(TestCase):
  def test_builder_with_no_dimension_fails(self):
    fbb = FakeBBGen(self.args, NO_DIMENSIONS_GTESTS_WATERFALL, MAC_TEST_SUITE,
                    MAC_LUCI_MILO_CFG)
    fbb.check_input_file_consistency(verbose=True)
    with self.assertRaisesRegex(
        generate_buildbot_json.BBGenErr,
        'dimensions must be specified for all swarmed tests'):
      fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_builder_with_no_os_dimension_fails(self):
    fbb = FakeBBGen(self.args, NO_OS_GTESTS_WATERFALL, MAC_TEST_SUITE,
                    MAC_LUCI_MILO_CFG)
    fbb.check_input_file_consistency(verbose=True)
    with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
                                'os must be specified for all swarmed tests'):
      fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)

  def test_mac_builder_with_no_cpu_dimension_in_isolated_script_fails(self):
    fbb = FakeBBGen(self.args, MAC_ISOLATED_SCRIPTS_WATERFALL, MAC_TEST_SUITE,
                    MAC_LUCI_MILO_CFG)
    fbb.check_input_file_consistency(verbose=True)
    with self.assertRaisesRegex(generate_buildbot_json.BBGenErr,
                                'cpu must be specified for mac'):
      fbb.check_output_file_consistency(verbose=True)
    self.assertFalse(fbb.printed_lines)


if __name__ == '__main__':
  unittest.main()