Tools/ChangeLog

 12011-12-01 Ryosuke Niwa <rniwa@webkit.org>
 2
 3 Parse reftest.list and extract types of ref tests
 4 https://bugs.webkit.org/show_bug.cgi?id=66837
 5
 6 Reviewed by NOBODY (OOPS!).
 7
 8 Add support for reftest.list to base port.
 9
 10 * Scripts/webkitpy/common/find_files.py:
 11 (find):
 12 * Scripts/webkitpy/layout_tests/controllers/single_test_runner.py:
 13 (SingleTestRunner.__init__):
 14 * Scripts/webkitpy/layout_tests/port/base.py:
 15 (Port.__init__): Initialize self._reftest_list. It's a dictionary mapping from a test directory
 16 to a dictionary of {test path: ("==" or "!=", reference file path)}
 17 (Port._get_reftest_list): Added; calls test_file.parse_reftest_list to fill self._reftest_list.
 18 (Port._reference_file_for): Added; obtains the reference file name given a test name.
 19 (Port.is_reftest): Added; Calls _reference_file_for.
 20 (Port.reftest_expected_filename): Calls _reference_file_for.
 21 (Port.reftest_expected_mismatch_filename): Ditto.
 22 (Port.find_test_files):
 23 (is_reference_html_file): Treat any file that starts with ref- or notref- or ends with
 24 -expected, -expected-mismach, -ref, or -notref as a reference file.
 25 (_is_test_file):
 26 (_parse_reftest_list): Added.
 27 * Scripts/webkitpy/layout_tests/port/base_unittest.py:
 28 (PortTest.test_is_test_file):
 29 (PortTest.test_parse_reftest_list):
 30 * Scripts/webkitpy/layout_tests/port/dryrun.py:
 31 (DryrunDriver.run_test):
 32 * Scripts/webkitpy/layout_tests/port/test.py:
 33 (unit_test_filesystem.add_test_file):
 34 (unit_test_filesystem.add_file):
 35 (unit_test_filesystem):
 36 * Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py:
 37 (get_tests_run.RecordingTestDriver.run_test):
 38 (MainTest.test_unexpected_failures):
 39 (MainTest.test_missing_and_unexpected_results):
 40 (EndToEndTest.test_end_to_end):
 41 * Scripts/webkitpy/to_be_moved/rebaseline_chromium_webkit_tests.py:
 42 (Rebaseliner._compile_rebaselining_tests):
 43
1442011-12-01 Sheriff Bot <webkit.review.bot@gmail.com>
245
346 Unreviewed, rolling out r101711.
101725

Tools/Scripts/webkitpy/common/find_files.py

@@def find(filesystem, base_dir, paths=Non
6262 everything under the base_dir.
6363 """
6464
65  global _skipped_directories
6665 paths = paths or ['*']
6766 skipped_directories = skipped_directories or set(['.svn', '_svn'])
6867 return _normalized_find(filesystem, _normalize(filesystem, base_dir, paths), skipped_directories, file_filter)
101725

Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_integrationtest.py

@@from __future__ import with_statement
3636import codecs
3737import itertools
3838import logging
39 import os
4039import Queue
 40import re
4141import sys
4242import thread
4343import time

@@def get_tests_run(extra_args=None, tests
166166 # In case of reftest, one test calls the driver's run_test() twice.
167167 # We should not add a reference html used by reftests to tests unless include_reference_html parameter
168168 # is explicitly given.
169  if include_reference_html or not is_reference_html_file(test_input.test_name):
 169 filesystem = self._port.host.filesystem
 170 dirname, filename = filesystem.split(test_name)
 171 if include_reference_html or not is_reference_html_file(filesystem, dirname, filename):
170172 self._current_test_batch.append(test_name)
171173 return TestDriver.run_test(self, test_input)
172174

@@class MainTest(unittest.TestCase):
426428
427429 # Update this magic number if you add an unexpected test to webkitpy.layout_tests.port.test
428430 # FIXME: It's nice to have a routine in port/test.py that returns this number.
429  unexpected_tests_count = 7
 431 unexpected_tests_count = 8
430432
431433 self.assertEqual(res, unexpected_tests_count)
432434 self.assertFalse(out.empty())

@@class MainTest(unittest.TestCase):
448450 expected_token = '"unexpected":{"text-image-checksum.html":{"expected":"PASS","actual":"TEXT"},"missing_text.html":{"expected":"PASS","is_missing_text":true,"actual":"MISSING"}'
449451 json_string = fs.read_text_file('/tmp/layout-test-results/full_results.json')
450452 self.assertTrue(json_string.find(expected_token) != -1)
451  self.assertTrue(json_string.find('"num_regression":1') == -1)
452  self.assertTrue(json_string.find('"num_flaky":1') == -1)
 453 self.assertTrue(json_string.find('"num_regressions":1') != -1)
 454 self.assertTrue(json_string.find('"num_flaky":0') != -1)
 455 self.assertTrue(json_string.find('"num_missing":1') != -1)
 456
 457 def test_missing_and_unexpected_results(self):
 458 # Test that we update expectations in place. If the expectation
 459 # is missing, update the expected generic location.
 460 fs = unit_test_filesystem()
 461 res, out, err, _ = logging_run(['--no-show-results', 'reftests/foo/'], tests_included=True, filesystem=fs, record_results=True)
 462 file_list = fs.written_files.keys()
 463 file_list.remove('/tmp/layout-test-results/tests_run0.txt')
 464 self.assertEquals(res, 1)
 465 json_string = fs.read_text_file('/tmp/layout-test-results/full_results.json')
 466 self.assertTrue(json_string.find('"unlistedtest.html":{"expected":"PASS","is_missing_text":true,"actual":"MISSING","is_missing_image":true}') != -1)
 467 self.assertTrue(json_string.find('"num_regressions":1') != -1)
 468 self.assertTrue(json_string.find('"num_flaky":0') != -1)
453469 self.assertTrue(json_string.find('"num_missing":1') != -1)
454470
455471 def test_missing_and_unexpected_results_with_custom_exit_code(self):

@@class EndToEndTest(unittest.TestCase):
762778 fs = unit_test_filesystem()
763779 res, out, err, user = logging_run(record_results=True, tests_included=True, filesystem=fs)
764780
765  # Seven tests should fail, so the return code should be 7.
766  self.assertEquals(res, 7)
 781 # Update this magic number if you add an unexpected test to webkitpy.layout_tests.port.test
 782 # FIXME: It's nice to have a routine in port/test.py that returns this number.
 783 unexpected_tests_count = 8
 784
 785 self.assertEquals(res, unexpected_tests_count)
767786 results = self.parse_full_results(fs.files['/tmp/layout-test-results/full_results.json'])
768787
769788 # Check to ensure we're passing back image diff %age correctly.
101725

Tools/Scripts/webkitpy/layout_tests/controllers/single_test_runner.py

@@class SingleTestRunner:
6868 return
6969
7070 reftest_expected_filename = port.reftest_expected_filename(self._test_name)
71  if fs.exists(reftest_expected_filename):
 71 if reftest_expected_filename and fs.exists(reftest_expected_filename):
7272 self._is_reftest = True
7373 self._reference_filename = reftest_expected_filename
7474
7575 reftest_expected_mismatch_filename = port.reftest_expected_mismatch_filename(self._test_name)
76  if fs.exists(reftest_expected_mismatch_filename):
 76 if reftest_expected_mismatch_filename and fs.exists(reftest_expected_mismatch_filename):
7777 if self._is_reftest:
78  _log.error('It is not allowed that one test file has both'
79  ' expected.html file and expected-mismatch.html file'
80  ' at the same time. Please remove either %s or %s.',
81  reftest_expected_filename, reftest_expected_mismatch_filename)
 78 _log.error('One test file cannot have both match and mismatch references. Please remove either %s or %s',
 79 reftest_expected_filename, reftest_expected_mismatch_filename)
8280 else:
8381 self._is_reftest = True
8482 self._is_mismatch_reftest = True
101725

Tools/Scripts/webkitpy/layout_tests/port/base.py

@@import cgi
3636import difflib
3737import errno
3838import os
 39import re
3940
4041from webkitpy.common.memoized import memoized
4142

@@class Port(object):
147148
148149 self.set_option_default('configuration', self.default_configuration())
149150 self._test_configuration = None
 151 self._reftest_list = {}
150152 self._multiprocessing_is_available = (multiprocessing is not None)
151153 self._results_directory = None
152154

@@class Port(object):
424426 text = self._filesystem.read_binary_file(baseline_path)
425427 return text.replace("\r\n", "\n")
426428
 429 def _get_reftest_list(self, test_name):
 430 dirname = self._filesystem.join(self.layout_tests_dir(), self._filesystem.dirname(test_name))
 431 if dirname not in self._reftest_list:
 432 self._reftest_list[dirname] = _parse_reftest_list(self._filesystem, dirname)
 433 return self._reftest_list[dirname]
 434
 435 def _reference_file_for(self, test_name, expectation):
 436 reftest_list = self._get_reftest_list(test_name)
 437 if not reftest_list:
 438 if expectation == '==':
 439 return self.expected_filename(test_name, '.html')
 440 else:
 441 return self.expected_filename(test_name, '-mismatch.html')
 442
 443 filename = self._filesystem.join(self.layout_tests_dir(), test_name)
 444 if filename not in reftest_list or reftest_list[filename][0] != expectation:
 445 return None
 446 return reftest_list[filename][1]
 447
 448 def is_reftest(self, test_name):
 449 reftest_list = self._get_reftest_list(test_name)
 450 if not reftest_list:
 451 has_expected = self._filesystem.exists(self.expected_filename(test_name, '.html'))
 452 return has_expected or self._filesystem.exists(self.expected_filename(test_name, '-mismatch.html'))
 453 filename = self._filesystem.join(self.layout_tests_dir(), test_name)
 454 return filename in reftest_list
 455
427456 def reftest_expected_filename(self, test_name):
428457 """Return the filename of reference we expect the test matches."""
429  return self.expected_filename(test_name, '.html')
 458 return self._reference_file_for(test_name, '==')
430459
431460 def reftest_expected_mismatch_filename(self, test_name):
432461 """Return the filename of reference we don't expect the test matches."""
433  return self.expected_filename(test_name, '-mismatch.html')
 462 return self._reference_file_for(test_name, '!=')
434463
435464 def test_to_uri(self, test_name):
436465 """Convert a test name to a URI."""

@@class Port(object):
466495
467496 def find_test_files(self, paths):
468497 # When collecting test cases, skip these directories
469  skipped_directories = set(['.svn', '_svn', 'resources', 'script-tests'])
 498 skipped_directories = set(['.svn', '_svn', 'resources', 'script-tests', 'reference', 'reftest'])
470499 return find_files.find(self.filesystem, self.layout_tests_dir(), paths, skipped_directories, _is_test_file)
471500
472501 def test_dirs(self):

@@class Port(object):
10021031
10031032
10041033# When collecting test cases, we include any file with these extensions.
1005 _supported_file_extensions = set(['.html', '.shtml', '.xml', '.xhtml', '.pl',
1006  '.htm', '.php', '.svg', '.mht'])
 1034_supported_file_extensions = set(['.html', '.shtml', '.xml', '.xhtml', '.pl', '.htm', '.php', '.svg', '.mht'])
10071035
10081036
1009 def is_reference_html_file(filename):
1010  return filename.endswith('-expected.html') or filename.endswith('-expected-mismatch.html')
 1037def is_reference_html_file(filesystem, dirname, filename):
 1038 if filename.startswith('ref-') or filename.endswith('notref-'):
 1039 return True
 1040 filename_wihout_ext, unused = filesystem.splitext(filename)
 1041 for suffix in ['-expected', '-expected-mismatch', '-ref', '-notref']:
 1042 if filename_wihout_ext.endswith(suffix):
 1043 return True
 1044 return False
10111045
10121046
10131047def _has_supported_extension(filesystem, filename):

@@def _has_supported_extension(filesystem,
10171051
10181052
10191053def _is_test_file(filesystem, dirname, filename):
1020  return _has_supported_extension(filesystem, filename) and not is_reference_html_file(filename)
 1054 return _has_supported_extension(filesystem, filename) and not is_reference_html_file(filesystem, dirname, filename)
 1055
 1056
 1057def _parse_reftest_list(filesystem, test_dirpath):
 1058 reftest_list_path = filesystem.join(test_dirpath, 'reftest.list')
 1059 if not filesystem.isfile(reftest_list_path):
 1060 return None
 1061 reftest_list_file = filesystem.read_text_file(reftest_list_path)
 1062
 1063 parsed_list = dict()
 1064 for line in reftest_list_file.split('\n'):
 1065 line = re.sub('#.+$', '', line)
 1066 split_line = line.split()
 1067 if len(split_line) < 3:
 1068 continue
 1069 expectation_type, test_file, ref_file = split_line
 1070 parsed_list[filesystem.join(test_dirpath, test_file)] = (expectation_type, filesystem.join(test_dirpath, ref_file))
 1071 return parsed_list
101725

Tools/Scripts/webkitpy/layout_tests/port/base_unittest.py

@@from webkitpy.common.host_mock import Mo
4343
4444from webkitpy.layout_tests.port import Port, Driver, DriverOutput
4545from webkitpy.layout_tests.port.base import _is_test_file
 46from webkitpy.layout_tests.port.base import _parse_reftest_list
4647from webkitpy.layout_tests.port.test import TestPort
4748
4849import config

@@class PortTest(unittest.TestCase):
306307 filesystem = MockFileSystem()
307308 self.assertTrue(_is_test_file(filesystem, '', 'foo.html'))
308309 self.assertTrue(_is_test_file(filesystem, '', 'foo.shtml'))
 310 self.assertTrue(_is_test_file(filesystem, '', 'test-ref-test.html'))
309311 self.assertFalse(_is_test_file(filesystem, '', 'foo.png'))
310312 self.assertFalse(_is_test_file(filesystem, '', 'foo-expected.html'))
311313 self.assertFalse(_is_test_file(filesystem, '', 'foo-expected-mismatch.html'))
 314 self.assertFalse(_is_test_file(filesystem, '', 'foo-ref.html'))
 315 self.assertFalse(_is_test_file(filesystem, '', 'foo-notref.html'))
 316 self.assertFalse(_is_test_file(filesystem, '', 'foo-notref.xht'))
 317 self.assertFalse(_is_test_file(filesystem, '', 'foo-ref.xhtml'))
 318 self.assertFalse(_is_test_file(filesystem, '', 'ref-foo.html'))
 319 self.assertFalse(_is_test_file(filesystem, '', 'notref-foo.xhr'))
312320
 321 def test_parse_reftest_list(self):
 322 port = TestPort()
 323 port.host.filesystem.files['bar/reftest.list'] = "\n".join(["== test.html test-ref.html",
 324 "",
 325 "# some comment",
 326 "!= test-2.html test-notref.html # more comments",
 327 "== test-3.html test-ref.html"])
 328
 329 reftest_list = _parse_reftest_list(port.host.filesystem, 'bar')
 330 self.assertEqual(reftest_list, {'bar/test.html': ('==', 'bar/test-ref.html'),
 331 'bar/test-2.html': ('!=', 'bar/test-notref.html'),
 332 'bar/test-3.html': ('==', 'bar/test-ref.html')})
313333
314334class VirtualTest(unittest.TestCase):
315335 """Tests that various methods expected to be virtual are."""
101725

Tools/Scripts/webkitpy/layout_tests/port/dryrun.py

@@class DryrunDriver(Driver):
112112 def run_test(self, driver_input):
113113 start_time = time.time()
114114 fs = self._port._filesystem
115  if (fs.exists(self._port.reftest_expected_filename(driver_input.test_name)) or
116  fs.exists(self._port.reftest_expected_mismatch_filename(driver_input.test_name)) or
117  driver_input.test_name.endswith('-expected.html')):
 115 if (self._port.is_reftest(driver_input.test_name) or driver_input.test_name.endswith('-expected.html')):
118116 text = 'test-text'
119117 image = 'test-image'
120118 checksum = 'test-checksum'
101725

Tools/Scripts/webkitpy/layout_tests/port/test.py

@@layer at (0,0) size 800x34
187187 tests.add_reftest('failures/unexpected/reftest.html', 'failures/unexpected/reftest-expected.html', same_image=False)
188188 tests.add_reftest('failures/unexpected/mismatch.html', 'failures/unexpected/mismatch-expected-mismatch.html', same_image=True)
189189 # FIXME: Add a reftest which crashes.
 190 tests.add('reftests/foo/test.html')
 191 tests.add('reftests/foo/test-ref.html')
 192
 193 # The following files shouldn't be treated as reftests
 194 tests.add_reftest('reftests/foo/unlistedtest.html', 'reftests/foo/unlistedtest-expected.html', same_image=True)
 195 tests.add('reftests/foo/reference/bar/common.html')
 196 tests.add('reftests/foo/reftest/bar/shared.html')
190197
191198 tests.add('websocket/tests/passes/text.html')
192199

@@def unit_test_filesystem(files=None):
213220 test_list = unit_test_list()
214221 files = files or {}
215222
216  def add_file(files, test, suffix, contents):
 223 def add_test_file(files, test, suffix, contents):
217224 dirname = test.name[0:test.name.rfind('/')]
218225 base = test.base
219  path = LAYOUT_TEST_DIR + '/' + dirname + '/' + base + suffix
220  files[path] = contents
 226 add_file(files, dirname + '/' + base + suffix, contents)
 227
 228 def add_file(files, file_name, contents):
 229 files[LAYOUT_TEST_DIR + '/' + file_name] = contents
221230
222231 # Add each test and the expected output, if any.
223232 for test in test_list.tests.values():
224  add_file(files, test, '.html', '')
 233 add_test_file(files, test, '.html', '')
225234 if test.is_reftest:
226235 continue
227236 if test.actual_audio:
228  add_file(files, test, '-expected.wav', test.expected_audio)
 237 add_test_file(files, test, '-expected.wav', test.expected_audio)
229238 continue
230239
231  add_file(files, test, '-expected.txt', test.expected_text)
232  add_file(files, test, '-expected.png', test.expected_image)
 240 add_test_file(files, test, '-expected.txt', test.expected_text)
 241 add_test_file(files, test, '-expected.png', test.expected_image)
233242
234243
235244 # Add the test_expectations file.

@@WONTFIX SKIP : failures/expected/keyboar
255264WONTFIX SKIP : failures/expected/exception.html = CRASH
256265"""
257266
 267 add_file(files, 'reftests/foo/reftest.list', """
 268== test.html test-ref.html
 269""")
 270
258271 # FIXME: This test was only being ignored because of missing a leading '/'.
259272 # Fixing the typo causes several tests to assert, so disabling the test entirely.
260273 # Add in a file should be ignored by port.find_test_files().
101725

Tools/Scripts/webkitpy/to_be_moved/rebaseline_chromium_webkit_tests.py

@@class Rebaseliner(object):
251251
252252 fs = self._target_port._filesystem
253253 for test in self._rebaselining_tests:
254  test_abspath = self._target_port.abspath_for_test(test)
255  if (fs.exists(self._target_port.reftest_expected_filename(test_abspath)) or
256  fs.exists(self._target_port.reftest_expected_mismatch_filename(test_abspath))):
 254 if self._target_port.is_reftest(test):
257255 _log.error('%s seems to be a reftest. We can not rebase for reftests.', test)
258256 self._rebaselining_tests = set()
259257 return False
101725