Tools/ChangeLog

2011-11-04  Ryosuke Niwa  <rniwa@webkit.org>

        Parse reftest.list and extract types of ref tests

        https://bugs.webkit.org/show_bug.cgi?id=66837

        Reviewed by NOBODY (OOPS!).

        Added a facility to parse reftest.list and pass it to TestInput objects to Manager.

        Modified test_files to consider reftest.list as a test file so that port.tests can find

        reftest.list files in test directories. These reftest.list entries are removed by

        Manager._parse_all_reftest_lists, called by Manager.collect_tests, as it collects test

        and reference file mapping.

        * Scripts/webkitpy/layout_tests/controllers/manager.py:

        * Scripts/webkitpy/layout_tests/controllers/manager_unittest.py: Added a test.

        * Scripts/webkitpy/layout_tests/models/test_input.py:

        * Scripts/webkitpy/layout_tests/port/test.py:

        * Scripts/webkitpy/layout_tests/port/test_files.py:

        * Scripts/webkitpy/layout_tests/port/test_files_unittest.py: Added test cases.

2011-11-04  Eric Seidel  <eric@webkit.org>

        Upgrade to the latest Mechanize

99343

Tools/Scripts/webkitpy/layout_tests/controllers/manager.py

@@class Manager(object):

        paths = self._strip_test_dir_prefixes(args)

        if self._options.test_list:

            paths += self._strip_test_dir_prefixes(read_test_files(self._fs, self._options.test_list, self._port.TEST_PATH_SEPARATOR))

329

329330 self._test_files = self._port.tests(paths)

        self._parse_all_reftest_lists()

    def _parse_all_reftest_lists(self):

        """Parse all reftests.list in tests' directories and populate self._reftests

        Args:

        tests: list of tests for which references may be parsed

        """

        self._reftests = {}

        test_dirs = set()

        reftest_lists = []

        for path in self._test_files:

            if 'reftest.list' == self._fs.basename(path):

                reftest_lists.append(path)

                reftest_list_abs_path = self._fs.join(self._port.layout_tests_dir(), path)

                self._parse_reftest_list(self._fs.read_text_file(reftest_list_abs_path), self._fs.dirname(path))

        for reftest_list in reftest_lists:

            self._test_files.remove(reftest_list)

        # Don't treat reference files as test files

        for expectation_type, ref_file_path in self._reftests.values():

            if ref_file_path in self._test_files:

                self._test_files.remove(ref_file_path)

    def _parse_reftest_list(self, reftest_list, test_dir):

        for line in reftest_list.split('\n'):

            split_line = line.split()

            if len(split_line) < 3:

                continue

            expectation_type, test_file, ref_file = split_line

            ref_file_path = self._fs.join(test_dir, ref_file)

            self._reftests[self._fs.join(test_dir, test_file)] = (expectation_type, ref_file_path)

    def _strip_test_dir_prefixes(self, paths):

        return [self._strip_test_dir_prefix(path) for path in paths if path]

@@class Manager(object):

        """Returns the appropriate TestInput object for the file. Mostly this

        is used for looking up the timeout value (in ms) to use for the given

        test."""

        if test_file in self._reftests:

            is_mismatch = self._reftests[test_file][0] == '!='

            ref_file = self._reftests[test_file][1]

        else:

            is_mismatch = False

            ref_file = None

547588 if self._test_is_slow(test_file):

548 return TestInput(test_file, self._options.slow_time_out_ms)

549 return TestInput(test_file, self._options.time_out_ms)

589 return TestInput(test_file, self._options.slow_time_out_ms, ref_file, is_mismatch)

590 return TestInput(test_file, self._options.time_out_ms, ref_file, is_mismatch)

    def _test_requires_lock(self, test_file):

        """Return True if the test needs to be locked when

99285

Tools/Scripts/webkitpy/layout_tests/controllers/manager_unittest.py

@@class ManagerTest(unittest.TestCase):

            manager = get_manager_with_tests(['http\\tests\\mime'])

            self.assertTrue(manager.needs_servers())

    def test_parse_reftest_list(self):

        manager = Manager(port=MockHost().port_factory.get('test-win-xp', None), options=MockOptions(), printer=Mock())

        manager._reftests = {}

        manager._parse_reftest_list("""== test.html test-ref.html

!= test.html test-noref.html""", 'foo/')

        self.assertEquals(manager._reftests, {'foo/test.html': ('==', 'foo/test-ref.html'), 'foo/test.html': ('!=', 'foo/test-noref.html')})

    def test_collect_tests_with_single_reftest(self):

        options, args = run_webkit_tests.parse_args(['--platform=test', '--print=nothing', 'reftests/foo/test.html'])

        manager = Manager(port=MockHost().port_factory.get('test'), options=options, printer=Mock())

        manager.collect_tests(args)

        self.assertTrue('reftests/foo/reftest.list' not in manager._test_files)

        self.assertTrue('reftests/foo/test.html' in manager._test_files)

        self.assertEqual(manager._reftests['reftests/foo/test.html'], ('==', 'reftests/foo/test-ref.html'))

    def test_collect_tests_with_reftests(self):

        options, args = run_webkit_tests.parse_args(['--platform=test', '--print=nothing', 'reftests/foo'])

        manager = Manager(port=MockHost().port_factory.get('test'), options=options, printer=Mock())

        manager.collect_tests(args)

        self.assertTrue('reftests/foo/reftest.list' not in manager._test_files)

        self.assertTrue('reftests/foo/test.html' in manager._test_files)

        self.assertTrue('reftests/foo/test-ref.html' not in manager._test_files)

        self.assertTrue('reftests/foo/test.html' in manager._reftests)

        self.assertTrue('reftests/foo/test-ref.html' not in manager._reftests)

        self.assertEqual(manager._reftests['reftests/foo/test.html'], ('==', 'reftests/foo/test-ref.html'))

    def test_collect_tests_with_reftests_at_parentdir(self):

        options, args = run_webkit_tests.parse_args(['--platform=test', '--print=nothing', 'reftests'])

        manager = Manager(port=MockHost().port_factory.get('test'), options=options, printer=Mock())

        manager.collect_tests(args)

        self.assertTrue('reftests/foo/reftest.list' not in manager._test_files)

        self.assertTrue('reftests/foo/test.html' in manager._test_files)

        self.assertTrue('reftests/foo/test-ref.html' not in manager._test_files)

        self.assertTrue('reftests/foo/test.html' in manager._reftests)

        self.assertTrue('reftests/foo/test-ref.html' not in manager._reftests)

        self.assertEqual(manager._reftests['reftests/foo/test.html'], ('==', 'reftests/foo/test-ref.html'))

class NaturalCompareTest(unittest.TestCase):

    def assert_cmp(self, x, y, result):

99285

Tools/Scripts/webkitpy/layout_tests/models/test_input.py

class TestInput:

    """Groups information about a test for easy passing of data."""

35 def __init__(self, test_name, timeout):

35 def __init__(self, test_name, timeout, ref_file=None, is_mismatch=False):

        """Holds the input parameters for a test.

        Args:

          test: name of test (not an absolute path!)

@@class TestInput:

          """

        self.test_name = test_name

        self.timeout = timeout

43 self.ref_file = ref_file

44 self.is_mismatch = is_mismatch

    def __repr__(self):

        return "TestInput('%s', %d)" % (self.test_name, self.timeout)

99285

Tools/Scripts/webkitpy/layout_tests/port/test.py

@@layer at (0,0) size 800x34

    tests.add_reftest('failures/unexpected/reftest.html', 'failures/unexpected/reftest-expected.html', same_image=False)

    tests.add_reftest('failures/unexpected/mismatch.html', 'failures/unexpected/mismatch-expected-mismatch.html', same_image=True)

    # FIXME: Add a reftest which crashes.

188 tests.add('reftests/foo/test.html')

189 tests.add('reftests/foo/test-ref.html')

    tests.add('websocket/tests/passes/text.html')

@@def unit_test_filesystem(files=None):

    test_list = unit_test_list()

    files = files or {}

214 def add_file(files, test, suffix, contents):

216 def add_test_file(files, test, suffix, contents):

215217 dirname = test.name[0:test.name.rfind('/')]

216218 base = test.base

217 path = LAYOUT_TEST_DIR + '/' + dirname + '/' + base + suffix

218 files[path] = contents

        add_file(files, dirname + '/' + base + suffix, contents)

    def add_file(files, file_name, contents):

        files[LAYOUT_TEST_DIR + '/' + file_name] = contents

    # Add each test and the expected output, if any.

    for test in test_list.tests.values():

222 add_file(files, test, '.html', '')

226 add_test_file(files, test, '.html', '')

        if test.is_reftest:

            continue

        if test.actual_audio:

226 add_file(files, test, '-expected.wav', test.expected_audio)

230 add_test_file(files, test, '-expected.wav', test.expected_audio)

227231 continue

228232

229 add_file(files, test, '-expected.txt', test.expected_text)

230 add_file(files, test, '-expected.png', test.expected_image)

233 add_test_file(files, test, '-expected.txt', test.expected_text)

234 add_test_file(files, test, '-expected.png', test.expected_image)

    # Add the test_expectations file.

@@WONTFIX SKIP : failures/expected/keyboar

WONTFIX SKIP : failures/expected/exception.html = CRASH

"""

    add_file(files, 'reftests/foo/reftest.list', """

== test.html test-ref.html

""")

    # FIXME: This test was only being ignored because of missing a leading '/'.

    # Fixing the typo causes several tests to assert, so disabling the test entirely.

    # Add in a file should be ignored by test_files.find().

99285

Tools/Scripts/webkitpy/layout_tests/port/test_files.py

@@def normalized_find(filesystem, paths):

    # FIXME: I'm not sure there's much point in this being a set. A list would probably be faster.

    test_files = set()

    for path in paths_to_walk:

91 files = filesystem.files_under(path, _skipped_directories, _is_test_file)

91 files = filesystem.files_under(path, _skipped_directories, _is_test_file_or_reftest_list)

9292 test_files.update(set(files))

        if filesystem.isfile(path):

            test_dir = filesystem.dirname(path)

            reftest_list = filesystem.join(test_dir, 'reftest.list')

            if reftest_list not in test_files and filesystem.exists(reftest_list):

                test_files.add(reftest_list)

    gather_time = time.time() - gather_start_time

    _log.debug("Test gathering took %f seconds" % gather_time)

@@def is_reference_html_file(filename):

    return filename.endswith('-expected.html') or filename.endswith('-expected-mismatch.html')

110 def _is_test_file(filesystem, dirname, filename):

111 return _has_supported_extension(filesystem, filename) and not is_reference_html_file(filename)

115def _is_test_file_or_reftest_list(filesystem, dirname, filename):

116 return (_has_supported_extension(filesystem, filename) and not is_reference_html_file(filename)) or filename == 'reftest.list'

99285

Tools/Scripts/webkitpy/layout_tests/port/test_files_unittest.py

@@class TestFilesTest(unittest.TestCase):

        tests = test_files.find(port, ['userscripts/resources'])

        self.assertEqual(tests, set([]))

62 def test_is_test_file(self):

    def test_find_with_reftests(self):

        port = test.TestPort()

        tests = test_files.find(port, ['reftests/foo/test.html'])

        self.assertEqual(tests, set([port.layout_tests_dir() + '/reftests/foo/reftest.list', port.layout_tests_dir() + '/reftests/foo/test.html']))

    def test_is_test_file_or_reftest_list(self):

6368 port = test.TestPort()

6469 fs = port._filesystem

        self.assertTrue(test_files._is_test_file(fs, '', 'foo.html'))

        self.assertTrue(test_files._is_test_file(fs, '', 'foo.shtml'))

        self.assertFalse(test_files._is_test_file(fs, '', 'foo.png'))

        self.assertFalse(test_files._is_test_file(fs, '', 'foo-expected.html'))

        self.assertFalse(test_files._is_test_file(fs, '', 'foo-expected-mismatch.html'))

        self.assertTrue(test_files._is_test_file_or_reftest_list(fs, '', 'foo.html'))

        self.assertTrue(test_files._is_test_file_or_reftest_list(fs, '', 'foo.shtml'))

        self.assertTrue(test_files._is_test_file_or_reftest_list(fs, '', 'reftest.list'))

        self.assertFalse(test_files._is_test_file_or_reftest_list(fs, '', 'foo.png'))

        self.assertFalse(test_files._is_test_file_or_reftest_list(fs, '', 'foo-expected.html'))

        self.assertFalse(test_files._is_test_file_or_reftest_list(fs, '', 'foo-expected-mismatch.html'))

        self.assertFalse(test_files._is_test_file_or_reftest_list(fs, '', 'some.list'))

class MockWinFileSystem(object):

99285