Upgrade wptrunner (tests/wpt/harness) to latest version

tests/wpt/harness/docs/usage.rst

@@ -154,11 +154,11 @@ metadata files in a subdirectory of the current directory named ``meta``.
 Output
 ------
 
-wptrunner uses the :py:mod:`mozlog.structured` package for output. This
+wptrunner uses the :py:mod:`mozlog` package for output. This
 structures events such as test results or log messages as JSON objects
 that can then be fed to other tools for interpretation. More details
 about the message format are given in the
-:py:mod:`mozlog.structured` documentation.
+:py:mod:`mozlog` documentation.
 
 By default the raw JSON messages are dumped to stdout. This is
 convenient for piping into other tools, but not ideal for humans

tests/wpt/harness/requirements.txt

@@ -1,4 +1,4 @@
 html5lib >= 0.99
 mozinfo >= 0.7
-mozlog >= 2.8
+mozlog >= 3.0
 mozdebug >= 0.1

tests/wpt/harness/test/test.cfg.example

@@ -8,9 +8,13 @@ ssl-type=none
 # prefs-root=/path/to/gecko-src/testing/profiles/
 
 # [servo]
-# binary=/path/to/servo-src/components/servo/target/servo
+# binary=/path/to/servo-src/target/release/servo
+# exclude=testharness # Because it needs a special testharness.js
+
+# [servodriver]
+# binary=/path/to/servo-src/target/release/servo
 # exclude=testharness # Because it needs a special testharness.js
 
 # [chrome]
 # binary=/path/to/chrome
-# webdriver-binary=/path/to/chromedriver
+# webdriver-binary=/path/to/chromedriver

tests/wpt/harness/test/test.py

@@ -8,9 +8,9 @@ import threading
 import time
 from StringIO import StringIO
 
-from mozlog.structured import structuredlog, reader
-from mozlog.structured.handlers import BaseHandler, StreamHandler, StatusHandler
-from mozlog.structured.formatters import MachFormatter
+from mozlog import structuredlog, reader
+from mozlog.handlers import BaseHandler, StreamHandler, StatusHandler
+from mozlog.formatters import MachFormatter
 from wptrunner import wptcommandline, wptrunner
 
 here = os.path.abspath(os.path.dirname(__file__))

tests/wpt/harness/wptrunner/browsers/firefox.py

@@ -27,7 +27,8 @@ __wptrunner__ = {"product": "firefox",
                               "reftest": "MarionetteRefTestExecutor"},
                  "browser_kwargs": "browser_kwargs",
                  "executor_kwargs": "executor_kwargs",
-                 "env_options": "env_options"}
+                 "env_options": "env_options",
+                 "run_info_extras": "run_info_extras"}
 
 
 def check_args(**kwargs):
@@ -43,7 +44,8 @@ def browser_kwargs(**kwargs):
             "symbols_path": kwargs["symbols_path"],
             "stackwalk_binary": kwargs["stackwalk_binary"],
             "certutil_binary": kwargs["certutil_binary"],
-            "ca_certificate_path": kwargs["ssl_env"].ca_cert_path()}
+            "ca_certificate_path": kwargs["ssl_env"].ca_cert_path(),
+            "e10s": kwargs["gecko_e10s"]}
 
 
 def executor_kwargs(test_type, server_config, cache_manager, run_info_data,
@@ -63,13 +65,15 @@ def env_options():
             "certificate_domain": "web-platform.test",
             "supports_debugger": True}
 
+def run_info_extras(**kwargs):
+    return {"e10s": kwargs["gecko_e10s"]}
 
 class FirefoxBrowser(Browser):
     used_ports = set()
 
     def __init__(self, logger, binary, prefs_root, debug_info=None,
                  symbols_path=None, stackwalk_binary=None, certutil_binary=None,
-                 ca_certificate_path=None):
+                 ca_certificate_path=None, e10s=False):
         Browser.__init__(self, logger)
         self.binary = binary
         self.prefs_root = prefs_root
@@ -81,6 +85,7 @@ class FirefoxBrowser(Browser):
         self.stackwalk_binary = stackwalk_binary
         self.ca_certificate_path = ca_certificate_path
         self.certutil_binary = certutil_binary
+        self.e10s = e10s
 
     def start(self):
         self.marionette_port = get_free_port(2828, exclude=self.used_ports)
@@ -99,6 +104,8 @@ class FirefoxBrowser(Browser):
                                       "marionette.defaultPrefs.port": self.marionette_port,
                                       "dom.disable_open_during_load": False,
                                       "network.dns.localDomains": ",".join(hostnames)})
+        if self.e10s:
+            self.profile.set_preferences({"browser.tabs.remote.autostart": True})
 
         if self.ca_certificate_path is not None:
             self.setup_ssl()

tests/wpt/harness/wptrunner/environment.py

@@ -10,7 +10,7 @@ import socket
 import sys
 import time
 
-from mozlog.structured import get_default_logger, handlers
+from mozlog import get_default_logger, handlers
 
 from wptlogging import LogLevelRewriter
 
@@ -80,25 +80,6 @@ class TestEnvironmentError(Exception):
     pass
 
 
-class StaticHandler(object):
-    def __init__(self, path, format_args, content_type, **headers):
-        with open(path) as f:
-            self.data = f.read() % format_args
-
-        self.resp_headers = [("Content-Type", content_type)]
-        for k, v in headers.iteritems():
-            resp_headers.append((k.replace("_", "-"), v))
-
-        self.handler = serve.handlers.handler(self.handle_request)
-
-    def handle_request(self, request, response):
-        return self.resp_headers, self.data
-
-    def __call__(self, request, response):
-        rv = self.handler(request, response)
-        return rv
-
-
 class TestEnvironment(object):
     def __init__(self, test_paths, ssl_env, pause_after_test, debug_info, options):
         """Context manager that owns the test environment i.e. the http and
@@ -114,29 +95,30 @@ class TestEnvironment(object):
         self.options = options if options is not None else {}
 
         self.cache_manager = multiprocessing.Manager()
-        self.routes = self.get_routes()
+        self.stash = serve.stash.StashServer()
 
 
     def __enter__(self):
+        self.stash.__enter__()
         self.ssl_env.__enter__()
         self.cache_manager.__enter__()
         self.setup_server_logging()
         self.config = self.load_config()
         serve.set_computed_defaults(self.config)
         self.external_config, self.servers = serve.start(self.config, self.ssl_env,
-                                                         self.routes)
+                                                         self.get_routes())
         if self.options.get("supports_debugger") and self.debug_info and self.debug_info.interactive:
             self.ignore_interrupts()
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         self.process_interrupts()
-        self.cache_manager.__exit__(exc_type, exc_val, exc_tb)
-        self.ssl_env.__exit__(exc_type, exc_val, exc_tb)
-
         for scheme, servers in self.servers.iteritems():
             for port, server in servers:
                 server.kill()
+        self.cache_manager.__exit__(exc_type, exc_val, exc_tb)
+        self.ssl_env.__exit__(exc_type, exc_val, exc_tb)
+        self.stash.__exit__()
 
     def ignore_interrupts(self):
         signal.signal(signal.SIGINT, signal.SIG_IGN)
@@ -193,40 +175,25 @@ class TestEnvironment(object):
             pass
 
     def get_routes(self):
-        routes = serve.default_routes()
+        route_builder = serve.RoutesBuilder()
+
         for path, format_args, content_type, route in [
                 ("testharness_runner.html", {}, "text/html", "/testharness_runner.html"),
                 (self.options.get("testharnessreport", "testharnessreport.js"),
                  {"output": self.pause_after_test}, "text/javascript",
                  "/resources/testharnessreport.js")]:
-            handler = StaticHandler(os.path.join(here, path), format_args, content_type)
-            routes.insert(0, (b"GET", str(route), handler))
+            path = os.path.normpath(os.path.join(here, path))
+            route_builder.add_static(path, format_args, content_type, route)
 
-        for url, paths in self.test_paths.iteritems():
-            if url == "/":
+        for url_base, paths in self.test_paths.iteritems():
+            if url_base == "/":
                 continue
-
-            path = paths["tests_path"]
-            url = "/%s/" % url.strip("/")
-
-            for (method,
-                 suffix,
-                 handler_cls) in [(b"*",
-                                   b"*.py",
-                                   serve.handlers.PythonScriptHandler),
-                                  (b"GET",
-                                   "*.asis",
-                                   serve.handlers.AsIsHandler),
-                                  (b"GET",
-                                   "*",
-                                   serve.handlers.FileHandler)]:
-                route = (method, b"%s%s" % (str(url), str(suffix)), handler_cls(path, url_base=url))
-                routes.insert(-3, route)
+            route_builder.add_mount_point(url_base, paths["tests_path"])
 
         if "/" not in self.test_paths:
-            routes = routes[:-3]
+            del route_builder.mountpoint_routes["/"]
 
-        return routes
+        return route_builder.get_routes()
 
     def ensure_started(self):
         # Pause for a while to ensure that the server has a chance to start

tests/wpt/harness/wptrunner/executors/base.py

@@ -55,12 +55,14 @@ class TestharnessResultConverter(object):
 
     def __call__(self, test, result):
         """Convert a JSON result into a (TestResult, [SubtestResult]) tuple"""
-        assert result["test"] == test.url, ("Got results from %s, expected %s" %
-                                            (result["test"], test.url))
-        harness_result = test.result_cls(self.harness_codes[result["status"]], result["message"])
+        result_url, status, message, stack, subtest_results = result
+        assert result_url == test.url, ("Got results from %s, expected %s" %
+                                      (result_url, test.url))
+        harness_result = test.result_cls(self.harness_codes[status], message)
         return (harness_result,
-                [test.subtest_result_cls(subtest["name"], self.test_codes[subtest["status"]],
-                                         subtest["message"], subtest.get("stack", None)) for subtest in result["tests"]])
+                [test.subtest_result_cls(name, self.test_codes[status], message, stack)
+                 for name, status, message, stack in subtest_results])
+
 testharness_result_converter = TestharnessResultConverter()
 
 

tests/wpt/harness/wptrunner/executors/executormarionette.py

@@ -107,6 +107,12 @@ class MarionetteProtocol(Protocol):
         return True
 
     def after_connect(self):
+        # Turn off debug-level logging by default since this is so verbose
+        with self.marionette.using_context("chrome"):
+            self.marionette.execute_script("""
+              Components.utils.import("resource://gre/modules/Log.jsm");
+              Log.repository.getLogger("Marionette").level = Log.Level.Info;
+            """)
         self.load_runner("http")
 
     def load_runner(self, protocol):

tests/wpt/harness/wptrunner/executors/executorservo.py

@@ -65,6 +65,8 @@ class ServoTestharnessExecutor(ProcessTestExecutor):
         args = ["--cpu", "--hard-fail", "-u", "Servo/wptrunner", "-z", self.test_url(test)]
         for stylesheet in self.browser.user_stylesheets:
             args += ["--user-stylesheet", stylesheet]
+        for pref in test.environment.get('prefs', {}):
+            args += ["--pref", pref]
         debug_args, command = browser_command(self.binary, args, self.debug_info)
 
         self.command = command
@@ -104,7 +106,6 @@ class ServoTestharnessExecutor(ProcessTestExecutor):
 
             if self.result_flag.is_set():
                 if self.result_data is not None:
-                    self.result_data["test"] = test.url
                     result = self.convert_result(test, self.result_data)
                 else:
                     self.proc.wait()
@@ -190,26 +191,44 @@ class ServoRefTestExecutor(ProcessTestExecutor):
         full_url = self.test_url(test)
 
         with TempFilename(self.tempdir) as output_path:
-            self.command = [self.binary, "--cpu", "--hard-fail", "--exit",
-                            "-u", "Servo/wptrunner", "-Z", "disable-text-aa",
-                            "--output=%s" % output_path, full_url]
+            debug_args, command = browser_command(
+                self.binary,
+                ["--cpu", "--hard-fail", "--exit", "-u", "Servo/wptrunner",
+                 "-Z", "disable-text-aa", "--output=%s" % output_path, full_url],
+                self.debug_info)
+
             for stylesheet in self.browser.user_stylesheets:
-                self.command += ["--user-stylesheet", stylesheet]
+                command += ["--user-stylesheet", stylesheet]
+
+            for pref in test.environment.get('prefs', {}):
+                command += ["--pref", pref]
+
+            self.command = debug_args + command
 
             env = os.environ.copy()
             env["HOST_FILE"] = self.hosts_path
 
-            self.proc = ProcessHandler(self.command,
-                                       processOutputLine=[self.on_output],
-                                       env=env)
+            if not self.interactive:
+                self.proc = ProcessHandler(self.command,
+                                           processOutputLine=[self.on_output],
+                                           env=env)
 
-            try:
-                self.proc.run()
-                timeout = test.timeout * self.timeout_multiplier + 5
-                rv = self.proc.wait(timeout=timeout)
-            except KeyboardInterrupt:
-                self.proc.kill()
-                raise
+
+                try:
+                    self.proc.run()
+                    timeout = test.timeout * self.timeout_multiplier + 5
+                    rv = self.proc.wait(timeout=timeout)
+                except KeyboardInterrupt:
+                    self.proc.kill()
+                    raise
+            else:
+                self.proc = subprocess.Popen(self.command,
+                                             env=env)
+                try:
+                    rv = self.proc.wait()
+                except KeyboardInterrupt:
+                    self.proc.kill()
+                    raise
 
             if rv is None:
                 self.proc.kill()

tests/wpt/harness/wptrunner/executors/testharness_marionette.js

@@ -13,11 +13,16 @@ window.wrappedJSObject.addEventListener("message", function listener(event) {
     clearTimeout(timer);
     var tests = event.data.tests;
     var status = event.data.status;
-    marionetteScriptFinished({test:"%(url)s",
-                              tests: tests,
-                              status: status.status,
-                              message: status.message,
-                              stack: status.stack});
+
+    var subtest_results = tests.map(function(x) {
+        return [x.name, x.status, x.message, x.stack]
+    });
+
+    marionetteScriptFinished(["%(url)s",
+                              status.status,
+                              status.message,
+                              status.stack,
+                              subtest_results]);
 }, false);
 
 window.wrappedJSObject.win = window.open("%(abs_url)s", "%(window_id)s");

tests/wpt/harness/wptrunner/executors/testharness_webdriver.js

@@ -8,12 +8,17 @@ window.timeout_multiplier = %(timeout_multiplier)d;
 window.addEventListener("message", function(event) {
   var tests = event.data[0];
   var status = event.data[1];
+
+  var subtest_results = tests.map(function(x) {
+      return [x.name, x.status, x.message, x.stack]
+  });
+
   clearTimeout(timer);
-  callback({test:"%(url)s",
-            tests: tests,
-            status: status.status,
-            message: status.message,
-            stack: status.stack});
+  callback(["%(url)s",
+            status.status,
+            status.message,
+            status.stack,
+            subtest_results]);
 }, false);
 
 window.win = window.open("%(abs_url)s", "%(window_id)s");

tests/wpt/harness/wptrunner/manifestinclude.py

@@ -57,7 +57,7 @@ class IncludeManifest(ManifestItem):
             try:
                 skip_value = self.get("skip", {"test_type": test.item_type}).lower()
                 assert skip_value in ("true", "false")
-                return False if skip_value == "true" else True
+                return skip_value != "true"
             except KeyError:
                 if node.parent is not None:
                     node = node.parent
@@ -107,6 +107,7 @@ class IncludeManifest(ManifestItem):
             if component not in node.child_map:
                 new_node = IncludeManifest(DataNode(component))
                 node.append(new_node)
+                new_node.set("skip", node.get("skip", {}))
 
             node = node.child_map[component]
 

tests/wpt/harness/wptrunner/manifestupdate.py

@@ -329,7 +329,7 @@ def group_conditionals(values):
 
     properties = set(item[0] for item in by_property.iterkeys())
 
-    prop_order = ["debug", "os", "version", "processor", "bits"]
+    prop_order = ["debug", "e10s", "os", "version", "processor", "bits"]
     include_props = []
 
     for prop in prop_order:
@@ -356,7 +356,7 @@ def make_expr(prop_set, status):
 
     assert len(prop_set) > 0
 
-    no_value_props = set(["debug"])
+    no_value_props = set(["debug", "e10s"])
 
     expressions = []
     for prop, value in prop_set:

tests/wpt/harness/wptrunner/metadata.py

@@ -10,8 +10,8 @@ import types
 import uuid
 from collections import defaultdict
 
-from mozlog.structured import reader
-from mozlog.structured import structuredlog
+from mozlog import reader
+from mozlog import structuredlog
 
 import expected
 import manifestupdate

tests/wpt/harness/wptrunner/products.py

@@ -43,6 +43,8 @@ def load_product(config, product):
     browser_kwargs = getattr(module, data["browser_kwargs"])
     executor_kwargs = getattr(module, data["executor_kwargs"])
     env_options = getattr(module, data["env_options"])()
+    run_info_extras = (getattr(module, data["run_info_extras"])
+                       if "run_info_extras" in data else lambda **kwargs:{})
 
     executor_classes = {}
     for test_type, cls_name in data["executor"].iteritems():
@@ -52,4 +54,4 @@ def load_product(config, product):
     return (check_args,
             browser_cls, browser_kwargs,
             executor_classes, executor_kwargs,
-            env_options)
+            env_options, run_info_extras)

tests/wpt/harness/wptrunner/reduce.py

@@ -10,7 +10,7 @@ from collections import defaultdict
 import wptrunner
 import wpttest
 
-from mozlog.structured import commandline, reader
+from mozlog import commandline, reader
 
 logger = None
 

tests/wpt/harness/wptrunner/testharnessreport-servo.js

@@ -7,12 +7,14 @@ var props = {output:%(output)d};
 setup(props);
 
 add_completion_callback(function (tests, harness_status) {
-    alert("RESULT: " + JSON.stringify({
-        tests: tests.map(function(t) {
-            return { name: t.name, status: t.status, message: t.message, stack: t.stack}
+    var id = location.pathname + location.search + location.hash;
+    alert("RESULT: " + JSON.stringify([
+        id,
+        harness_status.status,
+        harness_status.message,
+        harness_status.stack,
+        tests.map(function(t) {
+            return [t.name, t.status, t.message, t.stack]
         }),
-        status: harness_status.status,
-        message: harness_status.message,
-        stack: harness_status.stack,
-    }));
+    ]));
 });

tests/wpt/harness/wptrunner/testharnessreport-servodriver.js

@@ -6,15 +6,15 @@ setup({output:%(output)d});
 
 add_completion_callback(function() {
     add_completion_callback(function (tests, status) {
-        var test_results = tests.map(function(x) {
-            return {name:x.name, status:x.status, message:x.message, stack:x.stack}
+        var subtest_results = tests.map(function(x) {
+            return [x.name, x.status, x.message, x.stack]
         });
         var id = location.pathname + location.search + location.hash;
-        var results = JSON.stringify({test: id,
-                                      tests:test_results,
-                                      status: status.status,
-                                      message: status.message,
-                                      stack: status.stack});
+        var results = JSON.stringify([id,
+                                      status.status,
+                                      status.message,
+                                      status.stack,
+                                      subtest_results]);
         (function done() {
             if (window.__wd_results_callback__) {
                 clearTimeout(__wd_results_timer__);

tests/wpt/harness/wptrunner/testharnessreport.js

@@ -15,16 +15,3 @@ if (window.opener && window.opener.explicit_timeout) {
 }
 
 setup(props);
-add_completion_callback(function() {
-    add_completion_callback(function(tests, status) {
-        var harness_status = {
-            "status": status.status,
-            "message": status.message,
-            "stack": status.stack
-        };
-        var test_results = tests.map(function(x) {
-            return {name:x.name, status:x.status, message:x.message, stack:x.stack}
-        });
-        window.opener.postMessage([test_results, harness_status], "*");
-    })
-});

tests/wpt/harness/wptrunner/testloader.py

@@ -496,7 +496,7 @@ class TestLoader(object):
     def iter_tests(self):
         manifest_items = []
 
-        for manifest in self.manifests.keys():
+        for manifest in sorted(self.manifests.keys(), key=lambda x:x.url_base):
             manifest_iter = iterfilter(self.manifest_filters,
                                        manifest.itertypes(*self.test_types))
             manifest_items.extend(manifest_iter)

tests/wpt/harness/wptrunner/testrunner.py

@@ -11,7 +11,7 @@ import traceback
 from Queue import Empty
 from multiprocessing import Process, current_process, Queue
 
-from mozlog.structured import structuredlog
+from mozlog import structuredlog
 
 # Special value used as a sentinal in various commands
 Stop = object()

tests/wpt/harness/wptrunner/tests/test_update.py

@@ -6,7 +6,7 @@ import unittest
 import StringIO
 
 from .. import metadata, manifestupdate
-from mozlog.structured import structuredlog, handlers, formatters
+from mozlog import structuredlog, handlers, formatters
 
 
 class TestExpectedUpdater(unittest.TestCase):

tests/wpt/harness/wptrunner/update/sync.py

@@ -124,24 +124,23 @@ class GetSyncTargetCommit(Step):
 class LoadManifest(Step):
     """Load the test manifest"""
 
-    provides = ["test_manifest"]
+    provides = ["manifest_path", "test_manifest", "old_manifest"]
 
     def create(self, state):
-        state.test_manifest = testloader.ManifestLoader(state.tests_path).load_manifest(
-            state.tests_path, state.metadata_path,
-        )
+        from manifest import manifest
+        state.manifest_path = os.path.join(state.metadata_path, "MANIFEST.json")
+        # Conservatively always rebuild the manifest when doing a sync
+        state.old_manifest = manifest.load(state.tests_path, state.manifest_path)
+        state.test_manifest = manifest.Manifest(None, "/")
 
 
 class UpdateManifest(Step):
     """Update the manifest to match the tests in the sync tree checkout"""
 
-    provides = ["initial_rev"]
     def create(self, state):
         from manifest import manifest, update
-        test_manifest = state.test_manifest
-        state.initial_rev = test_manifest.rev
-        update.update(state.sync["path"], "/", test_manifest)
-        manifest.write(test_manifest, os.path.join(state.metadata_path, "MANIFEST.json"))
+        update.update(state.sync["path"], "/", state.test_manifest)
+        manifest.write(state.test_manifest, state.manifest_path)
 
 
 class CopyWorkTree(Step):

tests/wpt/harness/wptrunner/vcs.py

@@ -5,7 +5,7 @@
 import subprocess
 from functools import partial
 
-from mozlog.structured import get_default_logger
+from mozlog import get_default_logger
 
 logger = None
 

tests/wpt/harness/wptrunner/wptcommandline.py

@@ -35,7 +35,7 @@ def require_arg(kwargs, name, value_func=None):
 
 
 def create_parser(product_choices=None):
-    from mozlog.structured import commandline
+    from mozlog import commandline
 
     import products
 
@@ -154,6 +154,8 @@ def create_parser(product_choices=None):
     gecko_group = parser.add_argument_group("Gecko-specific")
     gecko_group.add_argument("--prefs-root", dest="prefs_root", action="store", type=abs_path,
                              help="Path to the folder containing browser prefs")
+    gecko_group.add_argument("--e10s", dest="gecko_e10s", action="store_true",
+                             help="Path to the folder containing browser prefs")
 
     b2g_group = parser.add_argument_group("B2G-specific")
     b2g_group.add_argument("--b2g-no-backup", action="store_true", default=False,
@@ -290,7 +292,7 @@ def check_args(kwargs):
             kwargs["debugger"] = mozdebug.get_default_debugger_name()
         debug_info = mozdebug.get_debugger_info(kwargs["debugger"],
                                                 kwargs["debugger_args"])
-        if debug_info.interactive:
+        if debug_info and debug_info.interactive:
             if kwargs["processes"] != 1:
                 kwargs["processes"] = 1
             kwargs["no_capture_stdio"] = True

tests/wpt/harness/wptrunner/wptlogging.py

@@ -8,7 +8,7 @@ import threading
 from StringIO import StringIO
 from multiprocessing import Queue
 
-from mozlog.structured import commandline, stdadapter
+from mozlog import commandline, stdadapter
 
 def setup(args, defaults):
     logger = commandline.setup_logging("web-platform-tests", args, defaults)

tests/wpt/harness/wptrunner/wptrunner.py

@@ -40,8 +40,12 @@ def setup_logging(*args, **kwargs):
     global logger
     logger = wptlogging.setup(*args, **kwargs)
 
-def get_loader(test_paths, product, ssl_env, debug=None, **kwargs):
-    run_info = wpttest.get_run_info(kwargs["run_info"], product, debug=debug)
+def get_loader(test_paths, product, ssl_env, debug=None, run_info_extras=None, **kwargs):
+    if run_info_extras is None:
+        run_info_extras = {}
+
+    run_info = wpttest.get_run_info(kwargs["run_info"], product, debug=debug,
+                                    extras=run_info_extras)
 
     test_manifests = testloader.ManifestLoader(test_paths, force_manifest_update=kwargs["manifest_update"]).load()
 
@@ -111,17 +115,21 @@ def run_tests(config, test_paths, product, **kwargs):
         (check_args,
          browser_cls, get_browser_kwargs,
          executor_classes, get_executor_kwargs,
-         env_options) = products.load_product(config, product)
+         env_options, run_info_extras) = products.load_product(config, product)
 
         ssl_env = env.ssl_env(logger, **kwargs)
 
         check_args(**kwargs)
 
         if "test_loader" in kwargs:
-            run_info = wpttest.get_run_info(kwargs["run_info"], product, debug=None)
+            run_info = wpttest.get_run_info(kwargs["run_info"], product, debug=None,
+                                            extras=run_info_extras(**kwargs))
             test_loader = kwargs["test_loader"]
         else:
-            run_info, test_loader = get_loader(test_paths, product, ssl_env,
+            run_info, test_loader = get_loader(test_paths,
+                                               product,
+                                               ssl_env,
+                                               run_info_extras=run_info_extras(**kwargs),
                                                **kwargs)
 
         if kwargs["run_by_dir"] is False:

tests/wpt/harness/wptrunner/wpttest.py

@@ -57,7 +57,7 @@ def get_run_info(metadata_root, product, **kwargs):
 
 
 class RunInfo(dict):
-    def __init__(self, metadata_root, product, debug):
+    def __init__(self, metadata_root, product, debug, extras=None):
         self._update_mozinfo(metadata_root)
         self.update(mozinfo.info)
         self["product"] = product
@@ -66,6 +66,8 @@ class RunInfo(dict):
         elif "debug" not in self:
             # Default to release
             self["debug"] = False
+        if extras is not None:
+            self.update(extras)
 
     def _update_mozinfo(self, metadata_root):
         """Add extra build information from a mozinfo.json file in a parent