| 1 | #!/usr/bin/env python
|
|---|
| 2 | #
|
|---|
| 3 | # Copyright 2005 Google Inc. All Rights Reserved.
|
|---|
| 4 | #
|
|---|
| 5 | # Redistribution and use in source and binary forms, with or without
|
|---|
| 6 | # modification, are permitted provided that the following conditions are
|
|---|
| 7 | # met:
|
|---|
| 8 | #
|
|---|
| 9 | # * Redistributions of source code must retain the above copyright
|
|---|
| 10 | # notice, this list of conditions and the following disclaimer.
|
|---|
| 11 | # * Redistributions in binary form must reproduce the above
|
|---|
| 12 | # copyright notice, this list of conditions and the following disclaimer
|
|---|
| 13 | # in the documentation and/or other materials provided with the
|
|---|
| 14 | # distribution.
|
|---|
| 15 | # * Neither the name of Google Inc. nor the names of its
|
|---|
| 16 | # contributors may be used to endorse or promote products derived from
|
|---|
| 17 | # this software without specific prior written permission.
|
|---|
| 18 | #
|
|---|
| 19 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
|---|
| 20 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
|---|
| 21 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
|---|
| 22 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
|---|
| 23 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
|---|
| 24 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
|---|
| 25 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|---|
| 26 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|---|
| 27 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|---|
| 28 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|---|
| 29 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|---|
| 30 |
|
|---|
| 31 | """Unit test for Google Test test filters.
|
|---|
| 32 |
|
|---|
| 33 | A user can specify which test(s) in a Google Test program to run via either
|
|---|
| 34 | the GTEST_FILTER environment variable or the --gtest_filter flag.
|
|---|
| 35 | This script tests such functionality by invoking
|
|---|
| 36 | gtest_filter_unittest_ (a program written with Google Test) with different
|
|---|
| 37 | environments and command line flags.
|
|---|
| 38 |
|
|---|
| 39 | Note that test sharding may also influence which tests are filtered. Therefore,
|
|---|
| 40 | we test that here also.
|
|---|
| 41 | """
|
|---|
| 42 |
|
|---|
| 43 | __author__ = 'wan@google.com (Zhanyong Wan)'
|
|---|
| 44 |
|
|---|
| 45 | import os
|
|---|
| 46 | import re
|
|---|
| 47 | import sets
|
|---|
| 48 | import sys
|
|---|
| 49 |
|
|---|
| 50 | import gtest_test_utils
|
|---|
| 51 |
|
|---|
| 52 | # Constants.
|
|---|
| 53 |
|
|---|
| 54 | # Checks if this platform can pass empty environment variables to child
|
|---|
| 55 | # processes. We set an env variable to an empty string and invoke a python
|
|---|
| 56 | # script in a subprocess to print whether the variable is STILL in
|
|---|
| 57 | # os.environ. We then use 'eval' to parse the child's output so that an
|
|---|
| 58 | # exception is thrown if the input is anything other than 'True' nor 'False'.
|
|---|
| 59 | os.environ['EMPTY_VAR'] = ''
|
|---|
| 60 | child = gtest_test_utils.Subprocess(
|
|---|
| 61 | [sys.executable, '-c', 'import os; print \'EMPTY_VAR\' in os.environ'])
|
|---|
| 62 | CAN_PASS_EMPTY_ENV = eval(child.output)
|
|---|
| 63 |
|
|---|
| 64 |
|
|---|
| 65 | # Check if this platform can unset environment variables in child processes.
|
|---|
| 66 | # We set an env variable to a non-empty string, unset it, and invoke
|
|---|
| 67 | # a python script in a subprocess to print whether the variable
|
|---|
| 68 | # is NO LONGER in os.environ.
|
|---|
| 69 | # We use 'eval' to parse the child's output so that an exception
|
|---|
| 70 | # is thrown if the input is neither 'True' nor 'False'.
|
|---|
| 71 | os.environ['UNSET_VAR'] = 'X'
|
|---|
| 72 | del os.environ['UNSET_VAR']
|
|---|
| 73 | child = gtest_test_utils.Subprocess(
|
|---|
| 74 | [sys.executable, '-c', 'import os; print \'UNSET_VAR\' not in os.environ'])
|
|---|
| 75 | CAN_UNSET_ENV = eval(child.output)
|
|---|
| 76 |
|
|---|
| 77 |
|
|---|
| 78 | # Checks if we should test with an empty filter. This doesn't
|
|---|
| 79 | # make sense on platforms that cannot pass empty env variables (Win32)
|
|---|
| 80 | # and on platforms that cannot unset variables (since we cannot tell
|
|---|
| 81 | # the difference between "" and NULL -- Borland and Solaris < 5.10)
|
|---|
| 82 | CAN_TEST_EMPTY_FILTER = (CAN_PASS_EMPTY_ENV and CAN_UNSET_ENV)
|
|---|
| 83 |
|
|---|
| 84 |
|
|---|
| 85 | # The environment variable for specifying the test filters.
|
|---|
| 86 | FILTER_ENV_VAR = 'GTEST_FILTER'
|
|---|
| 87 |
|
|---|
| 88 | # The environment variables for test sharding.
|
|---|
| 89 | TOTAL_SHARDS_ENV_VAR = 'GTEST_TOTAL_SHARDS'
|
|---|
| 90 | SHARD_INDEX_ENV_VAR = 'GTEST_SHARD_INDEX'
|
|---|
| 91 | SHARD_STATUS_FILE_ENV_VAR = 'GTEST_SHARD_STATUS_FILE'
|
|---|
| 92 |
|
|---|
| 93 | # The command line flag for specifying the test filters.
|
|---|
| 94 | FILTER_FLAG = 'gtest_filter'
|
|---|
| 95 |
|
|---|
| 96 | # The command line flag for including disabled tests.
|
|---|
| 97 | ALSO_RUN_DISABED_TESTS_FLAG = 'gtest_also_run_disabled_tests'
|
|---|
| 98 |
|
|---|
| 99 | # Command to run the gtest_filter_unittest_ program.
|
|---|
| 100 | COMMAND = gtest_test_utils.GetTestExecutablePath('gtest_filter_unittest_')
|
|---|
| 101 |
|
|---|
| 102 | # Regex for determining whether parameterized tests are enabled in the binary.
|
|---|
| 103 | PARAM_TEST_REGEX = re.compile(r'/ParamTest')
|
|---|
| 104 |
|
|---|
| 105 | # Regex for parsing test case names from Google Test's output.
|
|---|
| 106 | TEST_CASE_REGEX = re.compile(r'^\[\-+\] \d+ tests? from (\w+(/\w+)?)')
|
|---|
| 107 |
|
|---|
| 108 | # Regex for parsing test names from Google Test's output.
|
|---|
| 109 | TEST_REGEX = re.compile(r'^\[\s*RUN\s*\].*\.(\w+(/\w+)?)')
|
|---|
| 110 |
|
|---|
| 111 | # The command line flag to tell Google Test to output the list of tests it
|
|---|
| 112 | # will run.
|
|---|
| 113 | LIST_TESTS_FLAG = '--gtest_list_tests'
|
|---|
| 114 |
|
|---|
| 115 | # Indicates whether Google Test supports death tests.
|
|---|
| 116 | SUPPORTS_DEATH_TESTS = 'HasDeathTest' in gtest_test_utils.Subprocess(
|
|---|
| 117 | [COMMAND, LIST_TESTS_FLAG]).output
|
|---|
| 118 |
|
|---|
| 119 | # Full names of all tests in gtest_filter_unittests_.
|
|---|
| 120 | PARAM_TESTS = [
|
|---|
| 121 | 'SeqP/ParamTest.TestX/0',
|
|---|
| 122 | 'SeqP/ParamTest.TestX/1',
|
|---|
| 123 | 'SeqP/ParamTest.TestY/0',
|
|---|
| 124 | 'SeqP/ParamTest.TestY/1',
|
|---|
| 125 | 'SeqQ/ParamTest.TestX/0',
|
|---|
| 126 | 'SeqQ/ParamTest.TestX/1',
|
|---|
| 127 | 'SeqQ/ParamTest.TestY/0',
|
|---|
| 128 | 'SeqQ/ParamTest.TestY/1',
|
|---|
| 129 | ]
|
|---|
| 130 |
|
|---|
| 131 | DISABLED_TESTS = [
|
|---|
| 132 | 'BarTest.DISABLED_TestFour',
|
|---|
| 133 | 'BarTest.DISABLED_TestFive',
|
|---|
| 134 | 'BazTest.DISABLED_TestC',
|
|---|
| 135 | 'DISABLED_FoobarTest.Test1',
|
|---|
| 136 | 'DISABLED_FoobarTest.DISABLED_Test2',
|
|---|
| 137 | 'DISABLED_FoobarbazTest.TestA',
|
|---|
| 138 | ]
|
|---|
| 139 |
|
|---|
| 140 | if SUPPORTS_DEATH_TESTS:
|
|---|
| 141 | DEATH_TESTS = [
|
|---|
| 142 | 'HasDeathTest.Test1',
|
|---|
| 143 | 'HasDeathTest.Test2',
|
|---|
| 144 | ]
|
|---|
| 145 | else:
|
|---|
| 146 | DEATH_TESTS = []
|
|---|
| 147 |
|
|---|
| 148 | # All the non-disabled tests.
|
|---|
| 149 | ACTIVE_TESTS = [
|
|---|
| 150 | 'FooTest.Abc',
|
|---|
| 151 | 'FooTest.Xyz',
|
|---|
| 152 |
|
|---|
| 153 | 'BarTest.TestOne',
|
|---|
| 154 | 'BarTest.TestTwo',
|
|---|
| 155 | 'BarTest.TestThree',
|
|---|
| 156 |
|
|---|
| 157 | 'BazTest.TestOne',
|
|---|
| 158 | 'BazTest.TestA',
|
|---|
| 159 | 'BazTest.TestB',
|
|---|
| 160 | ] + DEATH_TESTS + PARAM_TESTS
|
|---|
| 161 |
|
|---|
| 162 | param_tests_present = None
|
|---|
| 163 |
|
|---|
| 164 | # Utilities.
|
|---|
| 165 |
|
|---|
| 166 | environ = os.environ.copy()
|
|---|
| 167 |
|
|---|
| 168 |
|
|---|
| 169 | def SetEnvVar(env_var, value):
|
|---|
| 170 | """Sets the env variable to 'value'; unsets it when 'value' is None."""
|
|---|
| 171 |
|
|---|
| 172 | if value is not None:
|
|---|
| 173 | environ[env_var] = value
|
|---|
| 174 | elif env_var in environ:
|
|---|
| 175 | del environ[env_var]
|
|---|
| 176 |
|
|---|
| 177 |
|
|---|
| 178 | def RunAndReturnOutput(args = None):
|
|---|
| 179 | """Runs the test program and returns its output."""
|
|---|
| 180 |
|
|---|
| 181 | return gtest_test_utils.Subprocess([COMMAND] + (args or []),
|
|---|
| 182 | env=environ).output
|
|---|
| 183 |
|
|---|
| 184 |
|
|---|
| 185 | def RunAndExtractTestList(args = None):
|
|---|
| 186 | """Runs the test program and returns its exit code and a list of tests run."""
|
|---|
| 187 |
|
|---|
| 188 | p = gtest_test_utils.Subprocess([COMMAND] + (args or []), env=environ)
|
|---|
| 189 | tests_run = []
|
|---|
| 190 | test_case = ''
|
|---|
| 191 | test = ''
|
|---|
| 192 | for line in p.output.split('\n'):
|
|---|
| 193 | match = TEST_CASE_REGEX.match(line)
|
|---|
| 194 | if match is not None:
|
|---|
| 195 | test_case = match.group(1)
|
|---|
| 196 | else:
|
|---|
| 197 | match = TEST_REGEX.match(line)
|
|---|
| 198 | if match is not None:
|
|---|
| 199 | test = match.group(1)
|
|---|
| 200 | tests_run.append(test_case + '.' + test)
|
|---|
| 201 | return (tests_run, p.exit_code)
|
|---|
| 202 |
|
|---|
| 203 |
|
|---|
| 204 | def InvokeWithModifiedEnv(extra_env, function, *args, **kwargs):
|
|---|
| 205 | """Runs the given function and arguments in a modified environment."""
|
|---|
| 206 | try:
|
|---|
| 207 | original_env = environ.copy()
|
|---|
| 208 | environ.update(extra_env)
|
|---|
| 209 | return function(*args, **kwargs)
|
|---|
| 210 | finally:
|
|---|
| 211 | environ.clear()
|
|---|
| 212 | environ.update(original_env)
|
|---|
| 213 |
|
|---|
| 214 |
|
|---|
| 215 | def RunWithSharding(total_shards, shard_index, command):
|
|---|
| 216 | """Runs a test program shard and returns exit code and a list of tests run."""
|
|---|
| 217 |
|
|---|
| 218 | extra_env = {SHARD_INDEX_ENV_VAR: str(shard_index),
|
|---|
| 219 | TOTAL_SHARDS_ENV_VAR: str(total_shards)}
|
|---|
| 220 | return InvokeWithModifiedEnv(extra_env, RunAndExtractTestList, command)
|
|---|
| 221 |
|
|---|
| 222 | # The unit test.
|
|---|
| 223 |
|
|---|
| 224 |
|
|---|
| 225 | class GTestFilterUnitTest(gtest_test_utils.TestCase):
|
|---|
| 226 | """Tests the env variable or the command line flag to filter tests."""
|
|---|
| 227 |
|
|---|
| 228 | # Utilities.
|
|---|
| 229 |
|
|---|
| 230 | def AssertSetEqual(self, lhs, rhs):
|
|---|
| 231 | """Asserts that two sets are equal."""
|
|---|
| 232 |
|
|---|
| 233 | for elem in lhs:
|
|---|
| 234 | self.assert_(elem in rhs, '%s in %s' % (elem, rhs))
|
|---|
| 235 |
|
|---|
| 236 | for elem in rhs:
|
|---|
| 237 | self.assert_(elem in lhs, '%s in %s' % (elem, lhs))
|
|---|
| 238 |
|
|---|
| 239 | def AssertPartitionIsValid(self, set_var, list_of_sets):
|
|---|
| 240 | """Asserts that list_of_sets is a valid partition of set_var."""
|
|---|
| 241 |
|
|---|
| 242 | full_partition = []
|
|---|
| 243 | for slice_var in list_of_sets:
|
|---|
| 244 | full_partition.extend(slice_var)
|
|---|
| 245 | self.assertEqual(len(set_var), len(full_partition))
|
|---|
| 246 | self.assertEqual(sets.Set(set_var), sets.Set(full_partition))
|
|---|
| 247 |
|
|---|
| 248 | def AdjustForParameterizedTests(self, tests_to_run):
|
|---|
| 249 | """Adjust tests_to_run in case value parameterized tests are disabled."""
|
|---|
| 250 |
|
|---|
| 251 | global param_tests_present
|
|---|
| 252 | if not param_tests_present:
|
|---|
| 253 | return list(sets.Set(tests_to_run) - sets.Set(PARAM_TESTS))
|
|---|
| 254 | else:
|
|---|
| 255 | return tests_to_run
|
|---|
| 256 |
|
|---|
| 257 | def RunAndVerify(self, gtest_filter, tests_to_run):
|
|---|
| 258 | """Checks that the binary runs correct set of tests for a given filter."""
|
|---|
| 259 |
|
|---|
| 260 | tests_to_run = self.AdjustForParameterizedTests(tests_to_run)
|
|---|
| 261 |
|
|---|
| 262 | # First, tests using the environment variable.
|
|---|
| 263 |
|
|---|
| 264 | # Windows removes empty variables from the environment when passing it
|
|---|
| 265 | # to a new process. This means it is impossible to pass an empty filter
|
|---|
| 266 | # into a process using the environment variable. However, we can still
|
|---|
| 267 | # test the case when the variable is not supplied (i.e., gtest_filter is
|
|---|
| 268 | # None).
|
|---|
| 269 | # pylint: disable-msg=C6403
|
|---|
| 270 | if CAN_TEST_EMPTY_FILTER or gtest_filter != '':
|
|---|
| 271 | SetEnvVar(FILTER_ENV_VAR, gtest_filter)
|
|---|
| 272 | tests_run = RunAndExtractTestList()[0]
|
|---|
| 273 | SetEnvVar(FILTER_ENV_VAR, None)
|
|---|
| 274 | self.AssertSetEqual(tests_run, tests_to_run)
|
|---|
| 275 | # pylint: enable-msg=C6403
|
|---|
| 276 |
|
|---|
| 277 | # Next, tests using the command line flag.
|
|---|
| 278 |
|
|---|
| 279 | if gtest_filter is None:
|
|---|
| 280 | args = []
|
|---|
| 281 | else:
|
|---|
| 282 | args = ['--%s=%s' % (FILTER_FLAG, gtest_filter)]
|
|---|
| 283 |
|
|---|
| 284 | tests_run = RunAndExtractTestList(args)[0]
|
|---|
| 285 | self.AssertSetEqual(tests_run, tests_to_run)
|
|---|
| 286 |
|
|---|
| 287 | def RunAndVerifyWithSharding(self, gtest_filter, total_shards, tests_to_run,
|
|---|
| 288 | args=None, check_exit_0=False):
|
|---|
| 289 | """Checks that binary runs correct tests for the given filter and shard.
|
|---|
| 290 |
|
|---|
| 291 | Runs all shards of gtest_filter_unittest_ with the given filter, and
|
|---|
| 292 | verifies that the right set of tests were run. The union of tests run
|
|---|
| 293 | on each shard should be identical to tests_to_run, without duplicates.
|
|---|
| 294 |
|
|---|
| 295 | Args:
|
|---|
| 296 | gtest_filter: A filter to apply to the tests.
|
|---|
| 297 | total_shards: A total number of shards to split test run into.
|
|---|
| 298 | tests_to_run: A set of tests expected to run.
|
|---|
| 299 | args : Arguments to pass to the to the test binary.
|
|---|
| 300 | check_exit_0: When set to a true value, make sure that all shards
|
|---|
| 301 | return 0.
|
|---|
| 302 | """
|
|---|
| 303 |
|
|---|
| 304 | tests_to_run = self.AdjustForParameterizedTests(tests_to_run)
|
|---|
| 305 |
|
|---|
| 306 | # Windows removes empty variables from the environment when passing it
|
|---|
| 307 | # to a new process. This means it is impossible to pass an empty filter
|
|---|
| 308 | # into a process using the environment variable. However, we can still
|
|---|
| 309 | # test the case when the variable is not supplied (i.e., gtest_filter is
|
|---|
| 310 | # None).
|
|---|
| 311 | # pylint: disable-msg=C6403
|
|---|
| 312 | if CAN_TEST_EMPTY_FILTER or gtest_filter != '':
|
|---|
| 313 | SetEnvVar(FILTER_ENV_VAR, gtest_filter)
|
|---|
| 314 | partition = []
|
|---|
| 315 | for i in range(0, total_shards):
|
|---|
| 316 | (tests_run, exit_code) = RunWithSharding(total_shards, i, args)
|
|---|
| 317 | if check_exit_0:
|
|---|
| 318 | self.assertEqual(0, exit_code)
|
|---|
| 319 | partition.append(tests_run)
|
|---|
| 320 |
|
|---|
| 321 | self.AssertPartitionIsValid(tests_to_run, partition)
|
|---|
| 322 | SetEnvVar(FILTER_ENV_VAR, None)
|
|---|
| 323 | # pylint: enable-msg=C6403
|
|---|
| 324 |
|
|---|
| 325 | def RunAndVerifyAllowingDisabled(self, gtest_filter, tests_to_run):
|
|---|
| 326 | """Checks that the binary runs correct set of tests for the given filter.
|
|---|
| 327 |
|
|---|
| 328 | Runs gtest_filter_unittest_ with the given filter, and enables
|
|---|
| 329 | disabled tests. Verifies that the right set of tests were run.
|
|---|
| 330 |
|
|---|
| 331 | Args:
|
|---|
| 332 | gtest_filter: A filter to apply to the tests.
|
|---|
| 333 | tests_to_run: A set of tests expected to run.
|
|---|
| 334 | """
|
|---|
| 335 |
|
|---|
| 336 | tests_to_run = self.AdjustForParameterizedTests(tests_to_run)
|
|---|
| 337 |
|
|---|
| 338 | # Construct the command line.
|
|---|
| 339 | args = ['--%s' % ALSO_RUN_DISABED_TESTS_FLAG]
|
|---|
| 340 | if gtest_filter is not None:
|
|---|
| 341 | args.append('--%s=%s' % (FILTER_FLAG, gtest_filter))
|
|---|
| 342 |
|
|---|
| 343 | tests_run = RunAndExtractTestList(args)[0]
|
|---|
| 344 | self.AssertSetEqual(tests_run, tests_to_run)
|
|---|
| 345 |
|
|---|
| 346 | def setUp(self):
|
|---|
| 347 | """Sets up test case.
|
|---|
| 348 |
|
|---|
| 349 | Determines whether value-parameterized tests are enabled in the binary and
|
|---|
| 350 | sets the flags accordingly.
|
|---|
| 351 | """
|
|---|
| 352 |
|
|---|
| 353 | global param_tests_present
|
|---|
| 354 | if param_tests_present is None:
|
|---|
| 355 | param_tests_present = PARAM_TEST_REGEX.search(
|
|---|
| 356 | RunAndReturnOutput()) is not None
|
|---|
| 357 |
|
|---|
| 358 | def testDefaultBehavior(self):
|
|---|
| 359 | """Tests the behavior of not specifying the filter."""
|
|---|
| 360 |
|
|---|
| 361 | self.RunAndVerify(None, ACTIVE_TESTS)
|
|---|
| 362 |
|
|---|
| 363 | def testDefaultBehaviorWithShards(self):
|
|---|
| 364 | """Tests the behavior without the filter, with sharding enabled."""
|
|---|
| 365 |
|
|---|
| 366 | self.RunAndVerifyWithSharding(None, 1, ACTIVE_TESTS)
|
|---|
| 367 | self.RunAndVerifyWithSharding(None, 2, ACTIVE_TESTS)
|
|---|
| 368 | self.RunAndVerifyWithSharding(None, len(ACTIVE_TESTS) - 1, ACTIVE_TESTS)
|
|---|
| 369 | self.RunAndVerifyWithSharding(None, len(ACTIVE_TESTS), ACTIVE_TESTS)
|
|---|
| 370 | self.RunAndVerifyWithSharding(None, len(ACTIVE_TESTS) + 1, ACTIVE_TESTS)
|
|---|
| 371 |
|
|---|
| 372 | def testEmptyFilter(self):
|
|---|
| 373 | """Tests an empty filter."""
|
|---|
| 374 |
|
|---|
| 375 | self.RunAndVerify('', [])
|
|---|
| 376 | self.RunAndVerifyWithSharding('', 1, [])
|
|---|
| 377 | self.RunAndVerifyWithSharding('', 2, [])
|
|---|
| 378 |
|
|---|
| 379 | def testBadFilter(self):
|
|---|
| 380 | """Tests a filter that matches nothing."""
|
|---|
| 381 |
|
|---|
| 382 | self.RunAndVerify('BadFilter', [])
|
|---|
| 383 | self.RunAndVerifyAllowingDisabled('BadFilter', [])
|
|---|
| 384 |
|
|---|
| 385 | def testFullName(self):
|
|---|
| 386 | """Tests filtering by full name."""
|
|---|
| 387 |
|
|---|
| 388 | self.RunAndVerify('FooTest.Xyz', ['FooTest.Xyz'])
|
|---|
| 389 | self.RunAndVerifyAllowingDisabled('FooTest.Xyz', ['FooTest.Xyz'])
|
|---|
| 390 | self.RunAndVerifyWithSharding('FooTest.Xyz', 5, ['FooTest.Xyz'])
|
|---|
| 391 |
|
|---|
| 392 | def testUniversalFilters(self):
|
|---|
| 393 | """Tests filters that match everything."""
|
|---|
| 394 |
|
|---|
| 395 | self.RunAndVerify('*', ACTIVE_TESTS)
|
|---|
| 396 | self.RunAndVerify('*.*', ACTIVE_TESTS)
|
|---|
| 397 | self.RunAndVerifyWithSharding('*.*', len(ACTIVE_TESTS) - 3, ACTIVE_TESTS)
|
|---|
| 398 | self.RunAndVerifyAllowingDisabled('*', ACTIVE_TESTS + DISABLED_TESTS)
|
|---|
| 399 | self.RunAndVerifyAllowingDisabled('*.*', ACTIVE_TESTS + DISABLED_TESTS)
|
|---|
| 400 |
|
|---|
| 401 | def testFilterByTestCase(self):
|
|---|
| 402 | """Tests filtering by test case name."""
|
|---|
| 403 |
|
|---|
| 404 | self.RunAndVerify('FooTest.*', ['FooTest.Abc', 'FooTest.Xyz'])
|
|---|
| 405 |
|
|---|
| 406 | BAZ_TESTS = ['BazTest.TestOne', 'BazTest.TestA', 'BazTest.TestB']
|
|---|
| 407 | self.RunAndVerify('BazTest.*', BAZ_TESTS)
|
|---|
| 408 | self.RunAndVerifyAllowingDisabled('BazTest.*',
|
|---|
| 409 | BAZ_TESTS + ['BazTest.DISABLED_TestC'])
|
|---|
| 410 |
|
|---|
| 411 | def testFilterByTest(self):
|
|---|
| 412 | """Tests filtering by test name."""
|
|---|
| 413 |
|
|---|
| 414 | self.RunAndVerify('*.TestOne', ['BarTest.TestOne', 'BazTest.TestOne'])
|
|---|
| 415 |
|
|---|
| 416 | def testFilterDisabledTests(self):
|
|---|
| 417 | """Select only the disabled tests to run."""
|
|---|
| 418 |
|
|---|
| 419 | self.RunAndVerify('DISABLED_FoobarTest.Test1', [])
|
|---|
| 420 | self.RunAndVerifyAllowingDisabled('DISABLED_FoobarTest.Test1',
|
|---|
| 421 | ['DISABLED_FoobarTest.Test1'])
|
|---|
| 422 |
|
|---|
| 423 | self.RunAndVerify('*DISABLED_*', [])
|
|---|
| 424 | self.RunAndVerifyAllowingDisabled('*DISABLED_*', DISABLED_TESTS)
|
|---|
| 425 |
|
|---|
| 426 | self.RunAndVerify('*.DISABLED_*', [])
|
|---|
| 427 | self.RunAndVerifyAllowingDisabled('*.DISABLED_*', [
|
|---|
| 428 | 'BarTest.DISABLED_TestFour',
|
|---|
| 429 | 'BarTest.DISABLED_TestFive',
|
|---|
| 430 | 'BazTest.DISABLED_TestC',
|
|---|
| 431 | 'DISABLED_FoobarTest.DISABLED_Test2',
|
|---|
| 432 | ])
|
|---|
| 433 |
|
|---|
| 434 | self.RunAndVerify('DISABLED_*', [])
|
|---|
| 435 | self.RunAndVerifyAllowingDisabled('DISABLED_*', [
|
|---|
| 436 | 'DISABLED_FoobarTest.Test1',
|
|---|
| 437 | 'DISABLED_FoobarTest.DISABLED_Test2',
|
|---|
| 438 | 'DISABLED_FoobarbazTest.TestA',
|
|---|
| 439 | ])
|
|---|
| 440 |
|
|---|
| 441 | def testWildcardInTestCaseName(self):
|
|---|
| 442 | """Tests using wildcard in the test case name."""
|
|---|
| 443 |
|
|---|
| 444 | self.RunAndVerify('*a*.*', [
|
|---|
| 445 | 'BarTest.TestOne',
|
|---|
| 446 | 'BarTest.TestTwo',
|
|---|
| 447 | 'BarTest.TestThree',
|
|---|
| 448 |
|
|---|
| 449 | 'BazTest.TestOne',
|
|---|
| 450 | 'BazTest.TestA',
|
|---|
| 451 | 'BazTest.TestB', ] + DEATH_TESTS + PARAM_TESTS)
|
|---|
| 452 |
|
|---|
| 453 | def testWildcardInTestName(self):
|
|---|
| 454 | """Tests using wildcard in the test name."""
|
|---|
| 455 |
|
|---|
| 456 | self.RunAndVerify('*.*A*', ['FooTest.Abc', 'BazTest.TestA'])
|
|---|
| 457 |
|
|---|
| 458 | def testFilterWithoutDot(self):
|
|---|
| 459 | """Tests a filter that has no '.' in it."""
|
|---|
| 460 |
|
|---|
| 461 | self.RunAndVerify('*z*', [
|
|---|
| 462 | 'FooTest.Xyz',
|
|---|
| 463 |
|
|---|
| 464 | 'BazTest.TestOne',
|
|---|
| 465 | 'BazTest.TestA',
|
|---|
| 466 | 'BazTest.TestB',
|
|---|
| 467 | ])
|
|---|
| 468 |
|
|---|
| 469 | def testTwoPatterns(self):
|
|---|
| 470 | """Tests filters that consist of two patterns."""
|
|---|
| 471 |
|
|---|
| 472 | self.RunAndVerify('Foo*.*:*A*', [
|
|---|
| 473 | 'FooTest.Abc',
|
|---|
| 474 | 'FooTest.Xyz',
|
|---|
| 475 |
|
|---|
| 476 | 'BazTest.TestA',
|
|---|
| 477 | ])
|
|---|
| 478 |
|
|---|
| 479 | # An empty pattern + a non-empty one
|
|---|
| 480 | self.RunAndVerify(':*A*', ['FooTest.Abc', 'BazTest.TestA'])
|
|---|
| 481 |
|
|---|
| 482 | def testThreePatterns(self):
|
|---|
| 483 | """Tests filters that consist of three patterns."""
|
|---|
| 484 |
|
|---|
| 485 | self.RunAndVerify('*oo*:*A*:*One', [
|
|---|
| 486 | 'FooTest.Abc',
|
|---|
| 487 | 'FooTest.Xyz',
|
|---|
| 488 |
|
|---|
| 489 | 'BarTest.TestOne',
|
|---|
| 490 |
|
|---|
| 491 | 'BazTest.TestOne',
|
|---|
| 492 | 'BazTest.TestA',
|
|---|
| 493 | ])
|
|---|
| 494 |
|
|---|
| 495 | # The 2nd pattern is empty.
|
|---|
| 496 | self.RunAndVerify('*oo*::*One', [
|
|---|
| 497 | 'FooTest.Abc',
|
|---|
| 498 | 'FooTest.Xyz',
|
|---|
| 499 |
|
|---|
| 500 | 'BarTest.TestOne',
|
|---|
| 501 |
|
|---|
| 502 | 'BazTest.TestOne',
|
|---|
| 503 | ])
|
|---|
| 504 |
|
|---|
| 505 | # The last 2 patterns are empty.
|
|---|
| 506 | self.RunAndVerify('*oo*::', [
|
|---|
| 507 | 'FooTest.Abc',
|
|---|
| 508 | 'FooTest.Xyz',
|
|---|
| 509 | ])
|
|---|
| 510 |
|
|---|
| 511 | def testNegativeFilters(self):
|
|---|
| 512 | self.RunAndVerify('*-BazTest.TestOne', [
|
|---|
| 513 | 'FooTest.Abc',
|
|---|
| 514 | 'FooTest.Xyz',
|
|---|
| 515 |
|
|---|
| 516 | 'BarTest.TestOne',
|
|---|
| 517 | 'BarTest.TestTwo',
|
|---|
| 518 | 'BarTest.TestThree',
|
|---|
| 519 |
|
|---|
| 520 | 'BazTest.TestA',
|
|---|
| 521 | 'BazTest.TestB',
|
|---|
| 522 | ] + DEATH_TESTS + PARAM_TESTS)
|
|---|
| 523 |
|
|---|
| 524 | self.RunAndVerify('*-FooTest.Abc:BazTest.*', [
|
|---|
| 525 | 'FooTest.Xyz',
|
|---|
| 526 |
|
|---|
| 527 | 'BarTest.TestOne',
|
|---|
| 528 | 'BarTest.TestTwo',
|
|---|
| 529 | 'BarTest.TestThree',
|
|---|
| 530 | ] + DEATH_TESTS + PARAM_TESTS)
|
|---|
| 531 |
|
|---|
| 532 | self.RunAndVerify('BarTest.*-BarTest.TestOne', [
|
|---|
| 533 | 'BarTest.TestTwo',
|
|---|
| 534 | 'BarTest.TestThree',
|
|---|
| 535 | ])
|
|---|
| 536 |
|
|---|
| 537 | # Tests without leading '*'.
|
|---|
| 538 | self.RunAndVerify('-FooTest.Abc:FooTest.Xyz:BazTest.*', [
|
|---|
| 539 | 'BarTest.TestOne',
|
|---|
| 540 | 'BarTest.TestTwo',
|
|---|
| 541 | 'BarTest.TestThree',
|
|---|
| 542 | ] + DEATH_TESTS + PARAM_TESTS)
|
|---|
| 543 |
|
|---|
| 544 | # Value parameterized tests.
|
|---|
| 545 | self.RunAndVerify('*/*', PARAM_TESTS)
|
|---|
| 546 |
|
|---|
| 547 | # Value parameterized tests filtering by the sequence name.
|
|---|
| 548 | self.RunAndVerify('SeqP/*', [
|
|---|
| 549 | 'SeqP/ParamTest.TestX/0',
|
|---|
| 550 | 'SeqP/ParamTest.TestX/1',
|
|---|
| 551 | 'SeqP/ParamTest.TestY/0',
|
|---|
| 552 | 'SeqP/ParamTest.TestY/1',
|
|---|
| 553 | ])
|
|---|
| 554 |
|
|---|
| 555 | # Value parameterized tests filtering by the test name.
|
|---|
| 556 | self.RunAndVerify('*/0', [
|
|---|
| 557 | 'SeqP/ParamTest.TestX/0',
|
|---|
| 558 | 'SeqP/ParamTest.TestY/0',
|
|---|
| 559 | 'SeqQ/ParamTest.TestX/0',
|
|---|
| 560 | 'SeqQ/ParamTest.TestY/0',
|
|---|
| 561 | ])
|
|---|
| 562 |
|
|---|
| 563 | def testFlagOverridesEnvVar(self):
|
|---|
| 564 | """Tests that the filter flag overrides the filtering env. variable."""
|
|---|
| 565 |
|
|---|
| 566 | SetEnvVar(FILTER_ENV_VAR, 'Foo*')
|
|---|
| 567 | args = ['--%s=%s' % (FILTER_FLAG, '*One')]
|
|---|
| 568 | tests_run = RunAndExtractTestList(args)[0]
|
|---|
| 569 | SetEnvVar(FILTER_ENV_VAR, None)
|
|---|
| 570 |
|
|---|
| 571 | self.AssertSetEqual(tests_run, ['BarTest.TestOne', 'BazTest.TestOne'])
|
|---|
| 572 |
|
|---|
| 573 | def testShardStatusFileIsCreated(self):
|
|---|
| 574 | """Tests that the shard file is created if specified in the environment."""
|
|---|
| 575 |
|
|---|
| 576 | shard_status_file = os.path.join(gtest_test_utils.GetTempDir(),
|
|---|
| 577 | 'shard_status_file')
|
|---|
| 578 | self.assert_(not os.path.exists(shard_status_file))
|
|---|
| 579 |
|
|---|
| 580 | extra_env = {SHARD_STATUS_FILE_ENV_VAR: shard_status_file}
|
|---|
| 581 | try:
|
|---|
| 582 | InvokeWithModifiedEnv(extra_env, RunAndReturnOutput)
|
|---|
| 583 | finally:
|
|---|
| 584 | self.assert_(os.path.exists(shard_status_file))
|
|---|
| 585 | os.remove(shard_status_file)
|
|---|
| 586 |
|
|---|
| 587 | def testShardStatusFileIsCreatedWithListTests(self):
|
|---|
| 588 | """Tests that the shard file is created with the "list_tests" flag."""
|
|---|
| 589 |
|
|---|
| 590 | shard_status_file = os.path.join(gtest_test_utils.GetTempDir(),
|
|---|
| 591 | 'shard_status_file2')
|
|---|
| 592 | self.assert_(not os.path.exists(shard_status_file))
|
|---|
| 593 |
|
|---|
| 594 | extra_env = {SHARD_STATUS_FILE_ENV_VAR: shard_status_file}
|
|---|
| 595 | try:
|
|---|
| 596 | output = InvokeWithModifiedEnv(extra_env,
|
|---|
| 597 | RunAndReturnOutput,
|
|---|
| 598 | [LIST_TESTS_FLAG])
|
|---|
| 599 | finally:
|
|---|
| 600 | # This assertion ensures that Google Test enumerated the tests as
|
|---|
| 601 | # opposed to running them.
|
|---|
| 602 | self.assert_('[==========]' not in output,
|
|---|
| 603 | 'Unexpected output during test enumeration.\n'
|
|---|
| 604 | 'Please ensure that LIST_TESTS_FLAG is assigned the\n'
|
|---|
| 605 | 'correct flag value for listing Google Test tests.')
|
|---|
| 606 |
|
|---|
| 607 | self.assert_(os.path.exists(shard_status_file))
|
|---|
| 608 | os.remove(shard_status_file)
|
|---|
| 609 |
|
|---|
| 610 | if SUPPORTS_DEATH_TESTS:
|
|---|
| 611 | def testShardingWorksWithDeathTests(self):
|
|---|
| 612 | """Tests integration with death tests and sharding."""
|
|---|
| 613 |
|
|---|
| 614 | gtest_filter = 'HasDeathTest.*:SeqP/*'
|
|---|
| 615 | expected_tests = [
|
|---|
| 616 | 'HasDeathTest.Test1',
|
|---|
| 617 | 'HasDeathTest.Test2',
|
|---|
| 618 |
|
|---|
| 619 | 'SeqP/ParamTest.TestX/0',
|
|---|
| 620 | 'SeqP/ParamTest.TestX/1',
|
|---|
| 621 | 'SeqP/ParamTest.TestY/0',
|
|---|
| 622 | 'SeqP/ParamTest.TestY/1',
|
|---|
| 623 | ]
|
|---|
| 624 |
|
|---|
| 625 | for flag in ['--gtest_death_test_style=threadsafe',
|
|---|
| 626 | '--gtest_death_test_style=fast']:
|
|---|
| 627 | self.RunAndVerifyWithSharding(gtest_filter, 3, expected_tests,
|
|---|
| 628 | check_exit_0=True, args=[flag])
|
|---|
| 629 | self.RunAndVerifyWithSharding(gtest_filter, 5, expected_tests,
|
|---|
| 630 | check_exit_0=True, args=[flag])
|
|---|
| 631 |
|
|---|
| 632 | if __name__ == '__main__':
|
|---|
| 633 | gtest_test_utils.Main()
|
|---|
