import io import json import os import html5lib import pytest from selenium import webdriver from wptserver import WPTServer ENC = 'utf8' HERE = os.path.dirname(os.path.abspath(__file__)) WPT_ROOT = os.path.normpath(os.path.join(HERE, '..', '..')) HARNESS = os.path.join(HERE, 'harness.html') def pytest_addoption(parser): parser.addoption("--binary", action="store", default=None, help="path to browser binary") def pytest_collect_file(path, parent): if path.ext.lower() == '.html': return HTMLItem(str(path), parent) def pytest_configure(config): config.driver = webdriver.Firefox(firefox_binary=config.getoption("--binary")) config.server = WPTServer(WPT_ROOT) config.server.start() config.add_cleanup(lambda: config.server.stop()) config.add_cleanup(lambda: config.driver.quit()) class HTMLItem(pytest.Item, pytest.Collector): def __init__(self, filename, parent): self.filename = filename with io.open(filename, encoding=ENC) as f: markup = f.read() parsed = html5lib.parse(markup, namespaceHTMLElements=False) name = None self.expected = None for element in parsed.getiterator(): if not name and element.tag == 'title': name = element.text continue if element.attrib.get('id') == 'expected': self.expected = json.loads(unicode(element.text)) continue if not name: raise ValueError('No name found in file: %s' % filename) super(HTMLItem, self).__init__(name, parent) def reportinfo(self): return self.fspath, None, self.filename def repr_failure(self, excinfo): return pytest.Collector.repr_failure(self, excinfo) def runtest(self): driver = self.session.config.driver server = self.session.config.server driver.get(server.url(HARNESS)) actual = driver.execute_async_script('runTest("%s", "foo", arguments[0])' % server.url(str(self.filename))) # Test object ordering is not guaranteed. This weak assertion verifies # that the indices are unique and sequential indices = [test_obj.get('index') for test_obj in actual['tests']] self._assert_sequence(indices) summarized = {} summarized[u'summarized_status'] = self._summarize_status(actual['status']) summarized[u'summarized_tests'] = [ self._summarize_test(test) for test in actual['tests']] summarized[u'summarized_tests'].sort(key=lambda test_obj: test_obj.get('name')) summarized[u'type'] = actual['type'] if not self.expected: assert summarized[u'summarized_status'][u'status_string'] == u'OK', summarized[u'summarized_status'][u'message'] for test in summarized[u'summarized_tests']: msg = "%s\n%s:\n%s" % (test[u'name'], test[u'message'], test[u'stack']) assert test[u'status_string'] == u'PASS', msg else: assert summarized == self.expected @staticmethod def _assert_sequence(nums): assert nums == range(1, nums[-1] + 1) @staticmethod def _scrub_stack(test_obj): copy = dict(test_obj) assert 'stack' in copy if copy['stack'] is not None: copy['stack'] = u'(implementation-defined)' return copy @staticmethod def _expand_status(status_obj): for key, value in [item for item in status_obj.items()]: # In "status" and "test" objects, the "status" value enum # definitions are interspersed with properties for unrelated # metadata. The following condition is a best-effort attempt to # ignore non-enum properties. if key != key.upper() or not isinstance(value, int): continue del status_obj[key] if status_obj['status'] == value: status_obj[u'status_string'] = key del status_obj['status'] return status_obj @staticmethod def _summarize_test(test_obj): del test_obj['index'] assert 'phase' in test_obj assert 'phases' in test_obj assert 'COMPLETE' in test_obj['phases'] assert test_obj['phase'] == test_obj['phases']['COMPLETE'] del test_obj['phases'] del test_obj['phase'] return HTMLItem._expand_status(HTMLItem._scrub_stack(test_obj)) @staticmethod def _summarize_status(status_obj): return HTMLItem._expand_status(HTMLItem._scrub_stack(status_obj))