Attachment #113744 for bug #66837

View Details Raw Unified Return to Bug 66837

| Differences between



2011-11-04  Ryosuke Niwa  <rniwa@webkit.org>

        Parse reftest.list and extract types of ref tests
        https://bugs.webkit.org/show_bug.cgi?id=66837

        Reviewed by NOBODY (OOPS!).

        Added a facility to parse reftest.list and pass it to TestInput objects to Manager.

        Modified test_files to consider reftest.list as a test file so that port.tests can find
        reftest.list files in test directories. These reftest.list entries are removed by
        Manager._parse_all_reftest_lists, called by Manager.collect_tests, as it collects test
        and reference file mapping.

        * Scripts/webkitpy/layout_tests/controllers/manager.py:
        * Scripts/webkitpy/layout_tests/controllers/manager_unittest.py: Added a test.
        * Scripts/webkitpy/layout_tests/models/test_input.py:
        * Scripts/webkitpy/layout_tests/port/test.py:
        * Scripts/webkitpy/layout_tests/port/test_files.py:
        * Scripts/webkitpy/layout_tests/port/test_files_unittest.py: Added test cases.

2011-11-04  Eric Seidel  <eric@webkit.org>

        Upgrade to the latest Mechanize



        paths = self._strip_test_dir_prefixes(args)
        if self._options.test_list:
            paths += self._strip_test_dir_prefixes(read_test_files(self._fs, self._options.test_list, self._port.TEST_PATH_SEPARATOR))

        self._test_files = self._port.tests(paths)
        self._parse_all_reftest_lists()

    def _parse_all_reftest_lists(self):
        """Parse all reftests.list in tests' directories and populate self._reftests

        Args:
        tests: list of tests for which references may be parsed
        """

        self._reftests = {}
        test_dirs = set()
        reftest_lists = []
        for path in self._test_files:
            if 'reftest.list' == self._fs.basename(path):
                reftest_lists.append(path)
                reftest_list_abs_path = self._fs.join(self._port.layout_tests_dir(), path)
                self._parse_reftest_list(self._fs.read_text_file(reftest_list_abs_path), self._fs.dirname(path))

        for reftest_list in reftest_lists:
            self._test_files.remove(reftest_list)

        # Don't treat reference files as test files
        for expectation_type, ref_file_path in self._reftests.values():
            if ref_file_path in self._test_files:
                self._test_files.remove(ref_file_path)

    def _parse_reftest_list(self, reftest_list, test_dir):
        for line in reftest_list.split('\n'):
            split_line = line.split()
            if len(split_line) < 3:
                continue
            expectation_type, test_file, ref_file = split_line
            ref_file_path = self._fs.join(test_dir, ref_file)
            self._reftests[self._fs.join(test_dir, test_file)] = (expectation_type, ref_file_path)

    def _strip_test_dir_prefixes(self, paths):
        return [self._strip_test_dir_prefix(path) for path in paths if path]

        """Returns the appropriate TestInput object for the file. Mostly this
        is used for looking up the timeout value (in ms) to use for the given
        test."""
        if test_file in self._reftests:
            is_mismatch = self._reftests[test_file][0] == '!='
            ref_file = self._reftests[test_file][1]
        else:
            is_mismatch = False
            ref_file = None
        if self._test_is_slow(test_file):
            return TestInput(test_file, self._options.slow_time_out_ms, ref_file, is_mismatch)
        return TestInput(test_file, self._options.time_out_ms, ref_file, is_mismatch)

    def _test_requires_lock(self, test_file):
        """Return True if the test needs to be locked when



            manager = get_manager_with_tests(['http\\tests\\mime'])
            self.assertTrue(manager.needs_servers())

    def test_parse_reftest_list(self):
        manager = Manager(port=MockHost().port_factory.get('test-win-xp', None), options=MockOptions(), printer=Mock())
        manager._reftests = {}
        manager._parse_reftest_list("""== test.html test-ref.html

!= test.html test-noref.html""", 'foo/')
        self.assertEquals(manager._reftests, {'foo/test.html': ('==', 'foo/test-ref.html'), 'foo/test.html': ('!=', 'foo/test-noref.html')})

    def test_collect_tests_with_single_reftest(self):
        options, args = run_webkit_tests.parse_args(['--platform=test', '--print=nothing', 'reftests/foo/test.html'])
        manager = Manager(port=MockHost().port_factory.get('test'), options=options, printer=Mock())
        manager.collect_tests(args)
        self.assertTrue('reftests/foo/reftest.list' not in manager._test_files)
        self.assertTrue('reftests/foo/test.html' in manager._test_files)
        self.assertEqual(manager._reftests['reftests/foo/test.html'], ('==', 'reftests/foo/test-ref.html'))

    def test_collect_tests_with_reftests(self):
        options, args = run_webkit_tests.parse_args(['--platform=test', '--print=nothing', 'reftests/foo'])
        manager = Manager(port=MockHost().port_factory.get('test'), options=options, printer=Mock())
        manager.collect_tests(args)
        self.assertTrue('reftests/foo/reftest.list' not in manager._test_files)
        self.assertTrue('reftests/foo/test.html' in manager._test_files)
        self.assertTrue('reftests/foo/test-ref.html' not in manager._test_files)
        self.assertTrue('reftests/foo/test.html' in manager._reftests)
        self.assertTrue('reftests/foo/test-ref.html' not in manager._reftests)
        self.assertEqual(manager._reftests['reftests/foo/test.html'], ('==', 'reftests/foo/test-ref.html'))

    def test_collect_tests_with_reftests_at_parentdir(self):
        options, args = run_webkit_tests.parse_args(['--platform=test', '--print=nothing', 'reftests'])
        manager = Manager(port=MockHost().port_factory.get('test'), options=options, printer=Mock())
        manager.collect_tests(args)
        self.assertTrue('reftests/foo/reftest.list' not in manager._test_files)
        self.assertTrue('reftests/foo/test.html' in manager._test_files)
        self.assertTrue('reftests/foo/test-ref.html' not in manager._test_files)
        self.assertTrue('reftests/foo/test.html' in manager._reftests)
        self.assertTrue('reftests/foo/test-ref.html' not in manager._reftests)
        self.assertEqual(manager._reftests['reftests/foo/test.html'], ('==', 'reftests/foo/test-ref.html'))

class NaturalCompareTest(unittest.TestCase):
    def assert_cmp(self, x, y, result):

Lines 32-38 Tools/Scripts/webkitpy/layout_tests/models/test_input.py_sec1
- Tools/Scripts/webkitpy/layout_tests/models/test_input.py -1 / +3 lines
32	class TestInput:	32	class TestInput:
33	"""Groups information about a test for easy passing of data."""	33	"""Groups information about a test for easy passing of data."""
34		34
35	def __init__(self, test_name, timeout):	35	def __init__(self, test_name, timeout, ref_file=None, is_mismatch=False):
36	"""Holds the input parameters for a test.	36	"""Holds the input parameters for a test.
37	Args:	37	Args:
38	test: name of test (not an absolute path!)	38	test: name of test (not an absolute path!)
Lines 40-45 class TestInput: Tools/Scripts/webkitpy/layout_tests/models/test_input.py_sec2
40	"""	40	"""
41	self.test_name = test_name	41	self.test_name = test_name
42	self.timeout = timeout	42	self.timeout = timeout
		43	self.ref_file = ref_file
		44	self.is_mismatch = is_mismatch
43		45
44	def __repr__(self):	46	def __repr__(self):
45	return "TestInput('%s', %d)" % (self.test_name, self.timeout)	47	return "TestInput('%s', %d)" % (self.test_name, self.timeout)

Lines 185-190 layer at (0,0) size 800x34 Tools/Scripts/webkitpy/layout_tests/port/test.py_sec1
- Tools/Scripts/webkitpy/layout_tests/port/test.py -7 / +15 lines
185	tests.add_reftest('failures/unexpected/reftest.html', 'failures/unexpected/reftest-expected.html', same_image=False)	185	tests.add_reftest('failures/unexpected/reftest.html', 'failures/unexpected/reftest-expected.html', same_image=False)
186	tests.add_reftest('failures/unexpected/mismatch.html', 'failures/unexpected/mismatch-expected-mismatch.html', same_image=True)	186	tests.add_reftest('failures/unexpected/mismatch.html', 'failures/unexpected/mismatch-expected-mismatch.html', same_image=True)
187	# FIXME: Add a reftest which crashes.	187	# FIXME: Add a reftest which crashes.
		188	tests.add('reftests/foo/test.html')
		189	tests.add('reftests/foo/test-ref.html')
188		190
189	tests.add('websocket/tests/passes/text.html')	191	tests.add('websocket/tests/passes/text.html')
190		192
Lines 211-233 def unit_test_filesystem(files=None): Tools/Scripts/webkitpy/layout_tests/port/test.py_sec2
211	test_list = unit_test_list()	213	test_list = unit_test_list()
212	files = files or {}	214	files = files or {}
213		215
214	def add_file(files, test, suffix, contents):	216	def add_test_file(files, test, suffix, contents):
215	dirname = test.name[0:test.name.rfind('/')]	217	dirname = test.name[0:test.name.rfind('/')]
216	base = test.base	218	base = test.base
217	path = LAYOUT_TEST_DIR + '/' + dirname + '/' + base + suffix	219	add_file(files, dirname + '/' + base + suffix, contents)
218	files[path] = contents	220
		221	def add_file(files, file_name, contents):
		222	files[LAYOUT_TEST_DIR + '/' + file_name] = contents
219		223
220	# Add each test and the expected output, if any.	224	# Add each test and the expected output, if any.
221	for test in test_list.tests.values():	225	for test in test_list.tests.values():
222	add_file(files, test, '.html', '')	226	add_test_file(files, test, '.html', '')
223	if test.is_reftest:	227	if test.is_reftest:
224	continue	228	continue
225	if test.actual_audio:	229	if test.actual_audio:
226	add_file(files, test, '-expected.wav', test.expected_audio)	230	add_test_file(files, test, '-expected.wav', test.expected_audio)
227	continue	231	continue
228		232
229	add_file(files, test, '-expected.txt', test.expected_text)	233	add_test_file(files, test, '-expected.txt', test.expected_text)
230	add_file(files, test, '-expected.png', test.expected_image)	234	add_test_file(files, test, '-expected.png', test.expected_image)
231		235
232		236
233	# Add the test_expectations file.	237	# Add the test_expectations file.
Lines 253-258 WONTFIX SKIP : failures/expected/keyboar Tools/Scripts/webkitpy/layout_tests/port/test.py_sec3
253	WONTFIX SKIP : failures/expected/exception.html = CRASH	257	WONTFIX SKIP : failures/expected/exception.html = CRASH
254	"""	258	"""
255		259
		260	add_file(files, 'reftests/foo/reftest.list', """
		261	== test.html test-ref.html
		262	""")
		263
256	# FIXME: This test was only being ignored because of missing a leading '/'.	264	# FIXME: This test was only being ignored because of missing a leading '/'.
257	# Fixing the typo causes several tests to assert, so disabling the test entirely.	265	# Fixing the typo causes several tests to assert, so disabling the test entirely.
258	# Add in a file should be ignored by test_files.find().	266	# Add in a file should be ignored by test_files.find().

Lines 88-95 def normalized_find(filesystem, paths): Tools/Scripts/webkitpy/layout_tests/port/test_files.py_sec1
- Tools/Scripts/webkitpy/layout_tests/port/test_files.py -3 / +8 lines
88	# FIXME: I'm not sure there's much point in this being a set. A list would probably be faster.	88	# FIXME: I'm not sure there's much point in this being a set. A list would probably be faster.
89	test_files = set()	89	test_files = set()
90	for path in paths_to_walk:	90	for path in paths_to_walk:
91	files = filesystem.files_under(path, _skipped_directories, _is_test_file)	91	files = filesystem.files_under(path, _skipped_directories, _is_test_file_or_reftest_list)
92	test_files.update(set(files))	92	test_files.update(set(files))
		93	if filesystem.isfile(path):
		94	test_dir = filesystem.dirname(path)
		95	reftest_list = filesystem.join(test_dir, 'reftest.list')
		96	if reftest_list not in test_files and filesystem.exists(reftest_list):
		97	test_files.add(reftest_list)
93		98
94	gather_time = time.time() - gather_start_time	99	gather_time = time.time() - gather_start_time
95	_log.debug("Test gathering took %f seconds" % gather_time)	100	_log.debug("Test gathering took %f seconds" % gather_time)
Lines 107-111 def is_reference_html_file(filename): Tools/Scripts/webkitpy/layout_tests/port/test_files.py_sec2
107	return filename.endswith('-expected.html') or filename.endswith('-expected-mismatch.html')	112	return filename.endswith('-expected.html') or filename.endswith('-expected-mismatch.html')
108		113
109		114
110	def _is_test_file(filesystem, dirname, filename):	115	def _is_test_file_or_reftest_list(filesystem, dirname, filename):
111	return _has_supported_extension(filesystem, filename) and not is_reference_html_file(filename)	116	return (_has_supported_extension(filesystem, filename) and not is_reference_html_file(filename)) or filename == 'reftest.list'

Lines 59-72 class TestFilesTest(unittest.TestCase): Tools/Scripts/webkitpy/layout_tests/port/test_files_unittest.py_sec1
- Tools/Scripts/webkitpy/layout_tests/port/test_files_unittest.py -6 / +13 lines
59	tests = test_files.find(port, ['userscripts/resources'])	59	tests = test_files.find(port, ['userscripts/resources'])
60	self.assertEqual(tests, set([]))	60	self.assertEqual(tests, set([]))
61		61
62	def test_is_test_file(self):	62	def test_find_with_reftests(self):
		63	port = test.TestPort()
		64	tests = test_files.find(port, ['reftests/foo/test.html'])
		65	self.assertEqual(tests, set([port.layout_tests_dir() + '/reftests/foo/reftest.list', port.layout_tests_dir() + '/reftests/foo/test.html']))
		66
		67	def test_is_test_file_or_reftest_list(self):
63	port = test.TestPort()	68	port = test.TestPort()
64	fs = port._filesystem	69	fs = port._filesystem
65	self.assertTrue(test_files._is_test_file(fs, '', 'foo.html'))	70	self.assertTrue(test_files._is_test_file_or_reftest_list(fs, '', 'foo.html'))
66	self.assertTrue(test_files._is_test_file(fs, '', 'foo.shtml'))	71	self.assertTrue(test_files._is_test_file_or_reftest_list(fs, '', 'foo.shtml'))
67	self.assertFalse(test_files._is_test_file(fs, '', 'foo.png'))	72	self.assertTrue(test_files._is_test_file_or_reftest_list(fs, '', 'reftest.list'))
68	self.assertFalse(test_files._is_test_file(fs, '', 'foo-expected.html'))	73	self.assertFalse(test_files._is_test_file_or_reftest_list(fs, '', 'foo.png'))
69	self.assertFalse(test_files._is_test_file(fs, '', 'foo-expected-mismatch.html'))	74	self.assertFalse(test_files._is_test_file_or_reftest_list(fs, '', 'foo-expected.html'))
		75	self.assertFalse(test_files._is_test_file_or_reftest_list(fs, '', 'foo-expected-mismatch.html'))
		76	self.assertFalse(test_files._is_test_file_or_reftest_list(fs, '', 'some.list'))
70		77
71		78
72	class MockWinFileSystem(object):	79	class MockWinFileSystem(object):

Return to Bug 66837

Lines 326-332 class Manager(object): Tools/Scripts/webkitpy/layout_tests/controllers/manager.py_sec1
- Tools/Scripts/webkitpy/layout_tests/controllers/manager.py -2 / +43 lines
326	paths = self._strip_test_dir_prefixes(args)	326	paths = self._strip_test_dir_prefixes(args)
327	if self._options.test_list:	327	if self._options.test_list:
328	paths += self._strip_test_dir_prefixes(read_test_files(self._fs, self._options.test_list, self._port.TEST_PATH_SEPARATOR))	328	paths += self._strip_test_dir_prefixes(read_test_files(self._fs, self._options.test_list, self._port.TEST_PATH_SEPARATOR))
		329
329	self._test_files = self._port.tests(paths)	330	self._test_files = self._port.tests(paths)
		331	self._parse_all_reftest_lists()
		332
		333	def _parse_all_reftest_lists(self):
		334	"""Parse all reftests.list in tests' directories and populate self._reftests
		335
		336	Args:
		337	tests: list of tests for which references may be parsed
		338	"""
		339
		340	self._reftests = {}
		341	test_dirs = set()
		342	reftest_lists = []
		343	for path in self._test_files:
		344	if 'reftest.list' == self._fs.basename(path):
		345	reftest_lists.append(path)
		346	reftest_list_abs_path = self._fs.join(self._port.layout_tests_dir(), path)
		347	self._parse_reftest_list(self._fs.read_text_file(reftest_list_abs_path), self._fs.dirname(path))
		348
		349	for reftest_list in reftest_lists:
		350	self._test_files.remove(reftest_list)
		351
		352	# Don't treat reference files as test files
		353	for expectation_type, ref_file_path in self._reftests.values():
		354	if ref_file_path in self._test_files:
		355	self._test_files.remove(ref_file_path)
		356
		357	def _parse_reftest_list(self, reftest_list, test_dir):
		358	for line in reftest_list.split('\n'):
		359	split_line = line.split()
		360	if len(split_line) < 3:
		361	continue
		362	expectation_type, test_file, ref_file = split_line
		363	ref_file_path = self._fs.join(test_dir, ref_file)
		364	self._reftests[self._fs.join(test_dir, test_file)] = (expectation_type, ref_file_path)
330		365
331	def _strip_test_dir_prefixes(self, paths):	366	def _strip_test_dir_prefixes(self, paths):
332	return [self._strip_test_dir_prefix(path) for path in paths if path]	367	return [self._strip_test_dir_prefix(path) for path in paths if path]
Lines 544-552 class Manager(object): Tools/Scripts/webkitpy/layout_tests/controllers/manager.py_sec2
544	"""Returns the appropriate TestInput object for the file. Mostly this	579	"""Returns the appropriate TestInput object for the file. Mostly this
545	is used for looking up the timeout value (in ms) to use for the given	580	is used for looking up the timeout value (in ms) to use for the given
546	test."""	581	test."""
		582	if test_file in self._reftests:
		583	is_mismatch = self._reftests[test_file][0] == '!='
		584	ref_file = self._reftests[test_file][1]
		585	else:
		586	is_mismatch = False
		587	ref_file = None
547	if self._test_is_slow(test_file):	588	if self._test_is_slow(test_file):
548	return TestInput(test_file, self._options.slow_time_out_ms)	589	return TestInput(test_file, self._options.slow_time_out_ms, ref_file, is_mismatch)
549	return TestInput(test_file, self._options.time_out_ms)	590	return TestInput(test_file, self._options.time_out_ms, ref_file, is_mismatch)
550		591
551	def _test_requires_lock(self, test_file):	592	def _test_requires_lock(self, test_file):
552	"""Return True if the test needs to be locked when	593	"""Return True if the test needs to be locked when

Lines 1-3 Tools/ChangeLog_sec1
- Tools/ChangeLog +21 lines
		1	2011-11-04 Ryosuke Niwa <rniwa@webkit.org>
		2
		3	Parse reftest.list and extract types of ref tests
		4	https://bugs.webkit.org/show_bug.cgi?id=66837
		5
		6	Reviewed by NOBODY (OOPS!).
		7
		8	Added a facility to parse reftest.list and pass it to TestInput objects to Manager.
		9
		10	Modified test_files to consider reftest.list as a test file so that port.tests can find
		11	reftest.list files in test directories. These reftest.list entries are removed by
		12	Manager._parse_all_reftest_lists, called by Manager.collect_tests, as it collects test
		13	and reference file mapping.
		14
		15	* Scripts/webkitpy/layout_tests/controllers/manager.py:
		16	* Scripts/webkitpy/layout_tests/controllers/manager_unittest.py: Added a test.
		17	* Scripts/webkitpy/layout_tests/models/test_input.py:
		18	* Scripts/webkitpy/layout_tests/port/test.py:
		19	* Scripts/webkitpy/layout_tests/port/test_files.py:
		20	* Scripts/webkitpy/layout_tests/port/test_files_unittest.py: Added test cases.
		21
1	2011-11-04 Eric Seidel <eric@webkit.org>	22	2011-11-04 Eric Seidel <eric@webkit.org>
2		23
3	Upgrade to the latest Mechanize	24	Upgrade to the latest Mechanize

Lines 263-268 class ManagerTest(unittest.TestCase): Tools/Scripts/webkitpy/layout_tests/controllers/manager_unittest.py_sec1
- Tools/Scripts/webkitpy/layout_tests/controllers/manager_unittest.py +37 lines
263	manager = get_manager_with_tests(['http\\tests\\mime'])	263	manager = get_manager_with_tests(['http\\tests\\mime'])
264	self.assertTrue(manager.needs_servers())	264	self.assertTrue(manager.needs_servers())
265		265
		266	def test_parse_reftest_list(self):
		267	manager = Manager(port=MockHost().port_factory.get('test-win-xp', None), options=MockOptions(), printer=Mock())
		268	manager._reftests = {}
		269	manager._parse_reftest_list("""== test.html test-ref.html
		270
		271	!= test.html test-noref.html""", 'foo/')
		272	self.assertEquals(manager._reftests, {'foo/test.html': ('==', 'foo/test-ref.html'), 'foo/test.html': ('!=', 'foo/test-noref.html')})
		273
		274	def test_collect_tests_with_single_reftest(self):
		275	options, args = run_webkit_tests.parse_args(['--platform=test', '--print=nothing', 'reftests/foo/test.html'])
		276	manager = Manager(port=MockHost().port_factory.get('test'), options=options, printer=Mock())
		277	manager.collect_tests(args)
		278	self.assertTrue('reftests/foo/reftest.list' not in manager._test_files)
		279	self.assertTrue('reftests/foo/test.html' in manager._test_files)
		280	self.assertEqual(manager._reftests['reftests/foo/test.html'], ('==', 'reftests/foo/test-ref.html'))
		281
		282	def test_collect_tests_with_reftests(self):
		283	options, args = run_webkit_tests.parse_args(['--platform=test', '--print=nothing', 'reftests/foo'])
		284	manager = Manager(port=MockHost().port_factory.get('test'), options=options, printer=Mock())
		285	manager.collect_tests(args)
		286	self.assertTrue('reftests/foo/reftest.list' not in manager._test_files)
		287	self.assertTrue('reftests/foo/test.html' in manager._test_files)
		288	self.assertTrue('reftests/foo/test-ref.html' not in manager._test_files)
		289	self.assertTrue('reftests/foo/test.html' in manager._reftests)
		290	self.assertTrue('reftests/foo/test-ref.html' not in manager._reftests)
		291	self.assertEqual(manager._reftests['reftests/foo/test.html'], ('==', 'reftests/foo/test-ref.html'))
		292
		293	def test_collect_tests_with_reftests_at_parentdir(self):
		294	options, args = run_webkit_tests.parse_args(['--platform=test', '--print=nothing', 'reftests'])
		295	manager = Manager(port=MockHost().port_factory.get('test'), options=options, printer=Mock())
		296	manager.collect_tests(args)
		297	self.assertTrue('reftests/foo/reftest.list' not in manager._test_files)
		298	self.assertTrue('reftests/foo/test.html' in manager._test_files)
		299	self.assertTrue('reftests/foo/test-ref.html' not in manager._test_files)
		300	self.assertTrue('reftests/foo/test.html' in manager._reftests)
		301	self.assertTrue('reftests/foo/test-ref.html' not in manager._reftests)
		302	self.assertEqual(manager._reftests['reftests/foo/test.html'], ('==', 'reftests/foo/test-ref.html'))
266		303
267	class NaturalCompareTest(unittest.TestCase):	304	class NaturalCompareTest(unittest.TestCase):
268	def assert_cmp(self, x, y, result):	305	def assert_cmp(self, x, y, result):