wpt: Create a base class for Servo process executors (#31115)

This change creates a base class for the Servo process executors, to
handle shared functionality. The only thing that hasn't moved there yet
is the actual process execution, which can happen in a followup change.

The main motivation behind this change is consistently handling
`on_evironment_change` which is used to handle changes to the `prefs`
value stored in `__dir__.ini`. Inherited `prefs` (thos in `__dir__.ini`)
aren't passed when creating tests for reference HTML. This change takes
a similar appraoch to Gecko, which just listens to
`on_environment_change` to note when these prefs change.
This commit is contained in:
Martin Robinson 2024-01-18 17:12:09 +01:00 committed by GitHub
parent c641c589e5
commit fdfeb3ed44
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
49 changed files with 95 additions and 233 deletions

View file

@ -1,2 +0,0 @@
[flexbox_columns-flexitems-2.html]
expected: FAIL

View file

@ -1,2 +0,0 @@
[flexbox_columns-flexitems.html]
expected: FAIL

View file

@ -481333,7 +481333,7 @@
[]
],
"executorservo.py": [
"3250c74ff4397d229e370a8c317b0e2a269dc9a8",
"dd1ea1a3b36951d3e124447343120ebd107908d4",
[]
],
"executorservodriver.py": [

View file

@ -1,2 +0,0 @@
[flexbox_absolute-atomic.html]
expected: FAIL

View file

@ -1,2 +0,0 @@
[align-items-baseline-overflow-non-visible.html]
expected: FAIL

View file

@ -1,2 +0,0 @@
[dynamic-baseline-change-nested.html]
expected: FAIL

View file

@ -1,2 +0,0 @@
[dynamic-bsize-change.html]
expected: FAIL

View file

@ -1,2 +0,0 @@
[dynamic-isize-change-001.html]
expected: FAIL

View file

@ -1,2 +0,0 @@
[flex-item-vertical-align.html]
expected: FAIL

View file

@ -1,2 +0,0 @@
[flex-order.html]
expected: FAIL

View file

@ -1,2 +0,0 @@
[flexbox-align-self-baseline-horiz-004.xhtml]
expected: FAIL

View file

@ -1,2 +0,0 @@
[flexbox-align-self-horiz-005.xhtml]
expected: FAIL

View file

@ -1,2 +0,0 @@
[flexbox-baseline-nested-001.html]
expected: FAIL

View file

@ -1,2 +0,0 @@
[flexbox-collapsed-item-horiz-001.html]
expected: FAIL

View file

@ -1,2 +0,0 @@
[flexbox-dyn-resize-001.html]
expected: FAIL

View file

@ -1,2 +0,0 @@
[flexbox-flex-basis-content-001a.html]
expected: FAIL

View file

@ -1,2 +0,0 @@
[flexbox-flex-basis-content-001b.html]
expected: FAIL

View file

@ -1,2 +0,0 @@
[flexbox-flex-basis-content-003a.html]
expected: FAIL

View file

@ -1,2 +0,0 @@
[flexbox-flex-basis-content-003b.html]
expected: FAIL

View file

@ -1,2 +0,0 @@
[flexbox-flex-basis-content-004a.html]
expected: FAIL

View file

@ -1,2 +0,0 @@
[flexbox-flex-basis-content-004b.html]
expected: FAIL

View file

@ -1,2 +0,0 @@
[flexbox-gap-position-absolute.html]
expected: FAIL

View file

@ -1,2 +0,0 @@
[flexbox-margin-auto-horiz-002.xhtml]
expected: FAIL

View file

@ -1,2 +0,0 @@
[flexbox-mbp-horiz-004.xhtml]
expected: FAIL

View file

@ -1,2 +0,0 @@
[flexbox-overflow-horiz-002.html]
expected: FAIL

View file

@ -1,2 +0,0 @@
[flexbox-overflow-horiz-003.html]
expected: FAIL

View file

@ -1,2 +0,0 @@
[flexbox-overflow-horiz-005.html]
expected: FAIL

View file

@ -1,2 +0,0 @@
[flexbox-overflow-vert-005.html]
expected: FAIL

View file

@ -1,3 +0,0 @@
[flexbox-root-node-001a.html]
expected: FAIL

View file

@ -1,3 +0,0 @@
[flexbox-root-node-001b.html]
expected: FAIL

View file

@ -1,2 +0,0 @@
[flexbox-table-fixup-001.xhtml]
expected: FAIL

View file

@ -1,2 +0,0 @@
[flexbox-whitespace-handling-002.xhtml]
expected: FAIL

View file

@ -1,2 +0,0 @@
[flexbox-with-pseudo-elements-001.html]
expected: FAIL

View file

@ -1,2 +0,0 @@
[flexbox-with-pseudo-elements-002.html]
expected: FAIL

View file

@ -1,2 +0,0 @@
[flexbox-with-pseudo-elements-003.html]
expected: FAIL

View file

@ -1,2 +0,0 @@
[flexbox_justifycontent-end-rtl.html]
expected: FAIL

View file

@ -1,2 +0,0 @@
[flexbox_justifycontent-end.html]
expected: FAIL

View file

@ -1,2 +0,0 @@
[flexbox_object.html]
expected: FAIL

View file

@ -1,2 +0,0 @@
[gap-003-ltr.html]
expected: FAIL

View file

@ -1,2 +0,0 @@
[gap-003-rtl.html]
expected: FAIL

View file

@ -1,2 +0,0 @@
[gap-008-ltr.html]
expected: FAIL

View file

@ -1,2 +0,0 @@
[gap-009-ltr.html]
expected: FAIL

View file

@ -1,2 +0,0 @@
[nested-orthogonal-flexbox-relayout.html]
expected: FAIL

View file

@ -1,2 +0,0 @@
[overflow-area-001.html]
expected: FAIL

View file

@ -1,2 +0,0 @@
[overflow-area-002.html]
expected: FAIL

View file

@ -1,2 +0,0 @@
[percentage-widths-001.html]
expected: FAIL

View file

@ -1,2 +0,0 @@
[table-item-flex-percentage-min-width.html]
expected: FAIL

View file

@ -1,2 +0,0 @@
[table-item-flex-percentage-width.html]
expected: FAIL

View file

@ -27,53 +27,24 @@ pytestrunner = None
webdriver = None
def write_hosts_file(config):
hosts_fd, hosts_path = tempfile.mkstemp()
with os.fdopen(hosts_fd, "w") as f:
f.write(make_hosts_file(config, "127.0.0.1"))
return hosts_path
def build_servo_command(test, test_url_func, browser, binary, pause_after_test, debug_info,
extra_args=None, debug_opts="replace-surrogates"):
args = [
"--hard-fail", "-u", "Servo/wptrunner",
# See https://github.com/servo/servo/issues/30080.
# For some reason rustls does not like the certificate generated by the WPT tooling.
"--ignore-certificate-errors",
"-z", test_url_func(test),
]
if debug_opts:
args += ["-Z", debug_opts]
for stylesheet in browser.user_stylesheets:
args += ["--user-stylesheet", stylesheet]
for pref, value in test.environment.get('prefs', {}).items():
args += ["--pref", f"{pref}={value}"]
if browser.ca_certificate_path:
args += ["--certificate-path", browser.ca_certificate_path]
if extra_args:
args += extra_args
args += browser.binary_args
debug_args, command = browser_command(binary, args, debug_info)
if pause_after_test:
command.remove("-z")
return debug_args + command
class ServoTestharnessExecutor(ProcessTestExecutor):
convert_result = testharness_result_converter
def __init__(self, logger, browser, server_config, timeout_multiplier=1, debug_info=None,
pause_after_test=False, **kwargs):
class ServoExecutor(ProcessTestExecutor):
def __init__(self, logger, browser, server_config, timeout_multiplier, debug_info,
pause_after_test, reftest_screenshot="unexpected"):
ProcessTestExecutor.__init__(self, logger, browser, server_config,
timeout_multiplier=timeout_multiplier,
debug_info=debug_info)
debug_info=debug_info,
reftest_screenshot=reftest_screenshot)
self.pause_after_test = pause_after_test
self.result_data = None
self.result_flag = None
self.environment = {}
self.protocol = ConnectionlessProtocol(self, browser)
self.hosts_path = write_hosts_file(server_config)
hosts_fd, self.hosts_path = tempfile.mkstemp()
with os.fdopen(hosts_fd, "w") as f:
f.write(make_hosts_file(server_config, "127.0.0.1"))
self.env_for_tests = os.environ.copy()
self.env_for_tests["HOST_FILE"] = self.hosts_path
self.env_for_tests["RUST_BACKTRACE"] = "1"
def teardown(self):
try:
@ -82,32 +53,70 @@ class ServoTestharnessExecutor(ProcessTestExecutor):
pass
ProcessTestExecutor.teardown(self)
def on_environment_change(self, new_environment):
self.environment = new_environment
return super().on_environment_change(new_environment)
def on_output(self, line):
line = line.decode("utf8", "replace")
if self.interactive:
print(line)
else:
self.logger.process_output(self.proc.pid, line, " ".join(self.command), self.test.url)
def build_servo_command(self, test, extra_args=None, debug_opts="replace-surrogates"):
args = [
"--hard-fail", "-u", "Servo/wptrunner",
# See https://github.com/servo/servo/issues/30080.
# For some reason rustls does not like the certificate generated by the WPT tooling.
"--ignore-certificate-errors",
"-z", self.test_url(test),
]
if debug_opts:
args += ["-Z", debug_opts]
for stylesheet in self.browser.user_stylesheets:
args += ["--user-stylesheet", stylesheet]
for pref, value in self.environment.get('prefs', {}).items():
args += ["--pref", f"{pref}={value}"]
if self.browser.ca_certificate_path:
args += ["--certificate-path", self.browser.ca_certificate_path]
if extra_args:
args += extra_args
args += self.browser.binary_args
debug_args, command = browser_command(self.binary, args, self.debug_info)
if self.pause_after_test:
command.remove("-z")
return debug_args + command
class ServoTestharnessExecutor(ServoExecutor):
convert_result = testharness_result_converter
def __init__(self, logger, browser, server_config, timeout_multiplier=1, debug_info=None,
pause_after_test=False, **kwargs):
ServoExecutor.__init__(self, logger, browser, server_config,
timeout_multiplier=timeout_multiplier,
debug_info=debug_info,
pause_after_test=pause_after_test)
self.result_data = None
self.result_flag = None
def do_test(self, test):
self.test = test
self.result_data = None
self.result_flag = threading.Event()
self.command = build_servo_command(test,
self.test_url,
self.browser,
self.binary,
self.pause_after_test,
self.debug_info)
env = os.environ.copy()
env["HOST_FILE"] = self.hosts_path
env["RUST_BACKTRACE"] = "1"
self.command = self.build_servo_command(test)
if not self.interactive:
self.proc = ProcessHandler(self.command,
processOutputLine=[self.on_output],
onFinish=self.on_finish,
env=env,
env=self.env_for_tests,
storeOutput=False)
self.proc.run()
else:
self.proc = subprocess.Popen(self.command, env=env)
self.proc = subprocess.Popen(self.command, env=self.env_for_tests)
try:
timeout = test.timeout * self.timeout_multiplier
@ -132,7 +141,6 @@ class ServoTestharnessExecutor(ProcessTestExecutor):
else:
result = (test.result_cls("TIMEOUT", None), [])
if proc_is_running:
if self.pause_after_test:
self.logger.info("Pausing until the browser exits")
@ -147,17 +155,12 @@ class ServoTestharnessExecutor(ProcessTestExecutor):
def on_output(self, line):
prefix = "ALERT: RESULT: "
line = line.decode("utf8", "replace")
if line.startswith(prefix):
self.result_data = json.loads(line[len(prefix):])
decoded_line = line.decode("utf8", "replace")
if decoded_line.startswith(prefix):
self.result_data = json.loads(decoded_line[len(prefix):])
self.result_flag.set()
else:
if self.interactive:
print(line)
else:
self.logger.process_output(self.proc.pid,
line,
" ".join(self.command), self.test.url)
ServoExecutor.on_output(self, line)
def on_finish(self):
self.result_flag.set()
@ -179,37 +182,32 @@ class TempFilename:
pass
class ServoRefTestExecutor(ProcessTestExecutor):
class ServoRefTestExecutor(ServoExecutor):
convert_result = reftest_result_converter
def __init__(self, logger, browser, server_config, binary=None, timeout_multiplier=1,
screenshot_cache=None, debug_info=None, pause_after_test=False,
reftest_screenshot="unexpected", **kwargs):
ProcessTestExecutor.__init__(self,
logger,
browser,
server_config,
timeout_multiplier=timeout_multiplier,
debug_info=debug_info,
reftest_screenshot=reftest_screenshot)
ServoExecutor.__init__(self,
logger,
browser,
server_config,
timeout_multiplier=timeout_multiplier,
debug_info=debug_info,
reftest_screenshot=reftest_screenshot,
pause_after_test=pause_after_test)
self.protocol = ConnectionlessProtocol(self, browser)
self.screenshot_cache = screenshot_cache
self.reftest_screenshot = reftest_screenshot
self.implementation = RefTestImplementation(self)
self.tempdir = tempfile.mkdtemp()
self.hosts_path = write_hosts_file(server_config)
def reset(self):
self.implementation.reset()
def teardown(self):
try:
os.unlink(self.hosts_path)
except OSError:
pass
os.rmdir(self.tempdir)
ProcessTestExecutor.teardown(self)
ServoExecutor.teardown(self)
def screenshot(self, test, viewport_size, dpi, page_ranges):
with TempFilename(self.tempdir) as output_path:
@ -221,24 +219,12 @@ class ServoRefTestExecutor(ProcessTestExecutor):
if dpi:
extra_args += ["--device-pixel-ratio", dpi]
self.command = build_servo_command(test,
self.test_url,
self.browser,
self.binary,
False,
self.debug_info,
extra_args,
debug_opts)
env = os.environ.copy()
env["HOST_FILE"] = self.hosts_path
env["RUST_BACKTRACE"] = "1"
self.command = self.build_servo_command(test, extra_args, debug_opts)
if not self.interactive:
self.proc = ProcessHandler(self.command,
processOutputLine=[self.on_output],
env=env)
env=self.env_for_tests)
try:
self.proc.run()
@ -248,8 +234,7 @@ class ServoRefTestExecutor(ProcessTestExecutor):
self.proc.kill()
raise
else:
self.proc = subprocess.Popen(self.command,
env=env)
self.proc = subprocess.Popen(self.command, env=self.env_for_tests)
try:
rv = self.proc.wait()
except KeyboardInterrupt:
@ -276,20 +261,11 @@ class ServoRefTestExecutor(ProcessTestExecutor):
return self.convert_result(test, result)
def on_output(self, line):
line = line.decode("utf8", "replace")
if self.interactive:
print(line)
else:
self.logger.process_output(self.proc.pid,
line,
" ".join(self.command), self.test.url)
class ServoTimedRunner(TimedRunner):
def run_func(self):
try:
self.result = True, self.func(self.protocol, self.url, self.timeout)
self.result = (True, self.func(self.protocol, self.url, self.timeout))
except Exception as e:
message = getattr(e, "message", "")
if message:
@ -303,23 +279,22 @@ class ServoTimedRunner(TimedRunner):
pass
class ServoCrashtestExecutor(ProcessTestExecutor):
class ServoCrashtestExecutor(ServoExecutor):
convert_result = crashtest_result_converter
def __init__(self, logger, browser, server_config, binary=None, timeout_multiplier=1,
screenshot_cache=None, debug_info=None, pause_after_test=False,
**kwargs):
ProcessTestExecutor.__init__(self,
logger,
browser,
server_config,
timeout_multiplier=timeout_multiplier,
debug_info=debug_info)
ServoExecutor.__init__(self,
logger,
browser,
server_config,
timeout_multiplier=timeout_multiplier,
debug_info=debug_info,
pause_after_test=pause_after_test)
self.pause_after_test = pause_after_test
self.protocol = ConnectionlessProtocol(self, browser)
self.tempdir = tempfile.mkdtemp()
self.hosts_path = write_hosts_file(server_config)
def do_test(self, test):
timeout = (test.timeout * self.timeout_multiplier if self.debug_info is None
@ -340,36 +315,19 @@ class ServoCrashtestExecutor(ProcessTestExecutor):
return (test.result_cls(*data), [])
def do_crashtest(self, protocol, url, timeout):
env = os.environ.copy()
env["HOST_FILE"] = self.hosts_path
env["RUST_BACKTRACE"] = "1"
self.command = build_servo_command(self.test,
self.test_url,
self.browser,
self.binary,
False,
self.debug_info,
extra_args=["-x"])
self.command = self.build_servo_command(self.test, extra_args=["-x"])
if not self.interactive:
self.proc = ProcessHandler(self.command,
env=env,
env=self.env_for_tests,
processOutputLine=[self.on_output],
storeOutput=False)
self.proc.run()
else:
self.proc = subprocess.Popen(self.command, env=env)
self.proc = subprocess.Popen(self.command, env=self.env_for_tests)
self.proc.wait()
if self.proc.poll() >= 0:
return {"status": "PASS", "message": None}
return {"status": "CRASH", "message": None}
def on_output(self, line):
line = line.decode("utf8", "replace")
self.logger.process_output(self.proc.pid,
line,
" ".join(self.command), self.test.url)