Update web-platform-tests to revision 10168e9a5d44efbc6e7d416d1d454eb9c9f1396c

tests/wpt/web-platform-tests/tools/lint/lint.py

@@ -722,7 +722,8 @@ def changed_files(wpt_root):
 
 def lint_paths(kwargs, wpt_root):
     if kwargs.get("paths"):
-        paths = kwargs["paths"]
+        r = os.path.realpath(wpt_root)
+        paths = [os.path.relpath(os.path.realpath(x), r) for x in kwargs["paths"]]
     elif kwargs["all"]:
         paths = list(all_filesystem_paths(wpt_root))
     else:

tests/wpt/web-platform-tests/tools/lint/tests/test_lint.py

@@ -390,7 +390,10 @@ def test_main_with_args():
         sys.argv = ['./lint', 'a', 'b', 'c']
         with _mock_lint('lint', return_value=True) as m:
             lint_mod.main(**vars(create_parser().parse_args()))
-            m.assert_called_once_with(repo_root, ['a', 'b', 'c'], "normal")
+            m.assert_called_once_with(repo_root,
+                                      [os.path.relpath(os.path.join(os.getcwd(), x), repo_root)
+                                       for x in ['a', 'b', 'c']],
+                                      "normal")
     finally:
         sys.argv = orig_argv
 

tests/wpt/web-platform-tests/tools/pytest.ini

@@ -1,2 +1,3 @@
 [pytest]
 norecursedirs = .* {arch} *.egg html5lib third_party pywebsocket six wpt wptrunner
+xfail_strict=true

tests/wpt/web-platform-tests/tools/tox.ini

@@ -16,7 +16,7 @@ deps =
   pytest-catchlog
 
 commands =
-  pytest --cov
+  pytest --cov {posargs}
   flake8
 
 passenv =

tests/wpt/web-platform-tests/tools/wpt/tests/test_wpt.py

@@ -1,7 +1,10 @@
+import errno
 import os
 import shutil
 import socket
 import subprocess
+import sys
+import tempfile
 import time
 import urllib2
 
@@ -10,8 +13,42 @@ import pytest
 from tools.wpt import wpt
 
 
-pytestmark = pytest.mark.skipif(os.name == "nt",
-                                reason="Tests currently don't work on Windows for path reasons")
+def is_port_8000_in_use():
+    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    try:
+        s.bind(("127.0.0.1", 8000))
+    except socket.error as e:
+        if e.errno == errno.EADDRINUSE:
+            return True
+        else:
+            raise e
+    finally:
+        s.close()
+    return False
+
+
+@pytest.fixture(scope="module")
+def manifest_dir():
+    def update_manifest():
+        with pytest.raises(SystemExit) as excinfo:
+            wpt.main(argv=["manifest", "--no-download", "--path", os.path.join(path, "MANIFEST.json")])
+        assert excinfo.value.code == 0
+
+    if os.environ.get('TRAVIS') == "true":
+        path = "~/meta"
+        update_manifest()
+        yield path
+    else:
+        try:
+            path = tempfile.mkdtemp()
+            old_path = os.path.join(wpt.localpaths.repo_root, "MANIFEST.json")
+            if os.path.exists(os.path.join(wpt.localpaths.repo_root, "MANIFEST.json")):
+                shutil.copyfile(old_path, os.path.join(path, "MANIFEST.json"))
+            update_manifest()
+            yield path
+        finally:
+            shutil.rmtree(path)
+
 
 def test_missing():
     with pytest.raises(SystemExit):
@@ -27,11 +64,17 @@ def test_help():
 
 
 @pytest.mark.slow
-@pytest.mark.system_dependent
 @pytest.mark.remote_network
-def test_run_firefox():
+@pytest.mark.xfail(sys.platform == "darwin",
+                   reason="https://github.com/w3c/web-platform-tests/issues/9090")
+@pytest.mark.xfail(sys.platform == "win32",
+                   reason="Tests currently don't work on Windows for path reasons")
+def test_run_firefox(manifest_dir):
     # TODO: It seems like there's a bug in argparse that makes this argument order required
     # should try to work around that
+    if is_port_8000_in_use():
+        pytest.skip("port 8000 already in use")
+
     os.environ["MOZ_HEADLESS"] = "1"
     try:
         fx_path = os.path.join(wpt.localpaths.repo_root, "_venv", "firefox")
@@ -39,7 +82,7 @@ def test_run_firefox():
             shutil.rmtree(fx_path)
         with pytest.raises(SystemExit) as excinfo:
             wpt.main(argv=["run", "--no-pause", "--install-browser", "--yes",
-                           "--metadata", "~/meta/",
+                           "--metadata", manifest_dir,
                            "firefox", "/dom/nodes/Element-tagName.html"])
         assert os.path.exists(fx_path)
         shutil.rmtree(fx_path)
@@ -49,17 +92,23 @@ def test_run_firefox():
 
 
 @pytest.mark.slow
-@pytest.mark.system_dependent
-def test_run_chrome():
+@pytest.mark.xfail(sys.platform == "win32",
+                   reason="Tests currently don't work on Windows for path reasons")
+def test_run_chrome(manifest_dir):
+    if is_port_8000_in_use():
+        pytest.skip("port 8000 already in use")
+
     with pytest.raises(SystemExit) as excinfo:
         wpt.main(argv=["run", "--yes", "--no-pause", "--binary-arg", "headless",
-                       "--metadata", "~/meta/",
+                       "--metadata", manifest_dir,
                        "chrome", "/dom/nodes/Element-tagName.html"])
     assert excinfo.value.code == 0
 
 
 @pytest.mark.slow
 @pytest.mark.remote_network
+@pytest.mark.xfail(sys.platform == "win32",
+                   reason="Tests currently don't work on Windows for path reasons")
 def test_install_chromedriver():
     chromedriver_path = os.path.join(wpt.localpaths.repo_root, "_venv", "bin", "chromedriver")
     if os.path.exists(chromedriver_path):
@@ -73,6 +122,10 @@ def test_install_chromedriver():
 
 @pytest.mark.slow
 @pytest.mark.remote_network
+@pytest.mark.xfail(sys.platform == "darwin",
+                   reason="https://github.com/w3c/web-platform-tests/issues/9090")
+@pytest.mark.xfail(sys.platform == "win32",
+                   reason="Tests currently don't work on Windows for path reasons")
 def test_install_firefox():
     fx_path = os.path.join(wpt.localpaths.repo_root, "_venv", "firefox")
     if os.path.exists(fx_path):
@@ -84,6 +137,8 @@ def test_install_firefox():
     shutil.rmtree(fx_path)
 
 
+@pytest.mark.xfail(sys.platform == "win32",
+                   reason="Tests currently don't work on Windows for path reasons")
 def test_files_changed(capsys):
     commit = "9047ac1d9f51b1e9faa4f9fad9c47d109609ab09"
     with pytest.raises(SystemExit) as excinfo:
@@ -120,27 +175,26 @@ def test_files_changed_ignore_rules():
 
 
 @pytest.mark.slow  # this updates the manifest
-@pytest.mark.system_dependent
-def test_tests_affected(capsys):
+@pytest.mark.xfail(sys.platform == "win32",
+                   reason="Tests currently don't work on Windows for path reasons")
+def test_tests_affected(capsys, manifest_dir):
     # This doesn't really work properly for random commits because we test the files in
     # the current working directory for references to the changed files, not the ones at
     # that specific commit. But we can at least test it returns something sensible
     commit = "9047ac1d9f51b1e9faa4f9fad9c47d109609ab09"
     with pytest.raises(SystemExit) as excinfo:
-        wpt.main(argv=["tests-affected", "--metadata", "~/meta/", "%s~..%s" % (commit, commit)])
+        wpt.main(argv=["tests-affected", "--metadata", manifest_dir, "%s~..%s" % (commit, commit)])
     assert excinfo.value.code == 0
     out, err = capsys.readouterr()
     assert "html/browsers/offline/appcache/workers/appcache-worker.html" in out
 
 
 @pytest.mark.slow
-@pytest.mark.system_dependent
+@pytest.mark.xfail(sys.platform == "win32",
+                   reason="Tests currently don't work on Windows for path reasons")
 def test_serve():
-    def test():
-        s = socket.socket()
-        s.connect(("127.0.0.1", 8000))
-    with pytest.raises(socket.error):
-        test()
+    if is_port_8000_in_use():
+        pytest.skip("port 8000 already in use")
 
     p = subprocess.Popen([os.path.join(wpt.localpaths.repo_root, "wpt"), "serve"],
                          preexec_fn=os.setsid)
@@ -148,8 +202,10 @@ def test_serve():
     start = time.time()
     try:
         while True:
+            if p.poll() is not None:
+                assert False, "server not running"
             if time.time() - start > 60:
-                assert False
+                assert False, "server did not start responding within 60s"
             try:
                 resp = urllib2.urlopen("http://web-platform.test:8000")
                 print resp

tests/wpt/web-platform-tests/tools/wptrunner/README.rst

@@ -197,7 +197,7 @@ language e.g.::
 
   if debug and (platform == "linux" or platform == "osx"): FAIL
 
-For test expectations the avaliable variables are those in the
+For test expectations the available variables are those in the
 `run_info` which for desktop are `version`, `os`, `bits`, `processor`,
 `debug` and `product`.
 

tests/wpt/web-platform-tests/tools/wptrunner/requirements_sauce.txt

@@ -1,2 +1,3 @@
 mozprocess >= 0.19
 selenium >= 3.3.0
+requests

tests/wpt/web-platform-tests/tools/wptrunner/tox.ini

@@ -2,19 +2,21 @@
 xfail_strict=true
 
 [tox]
-envlist = {py27,pypy}-{base,chrome,firefox,servo},py27-flake8
+envlist = {py27,pypy}-{base,chrome,firefox,sauce,servo},py27-flake8
 
 [testenv]
 deps =
      pytest>=2.9
      pytest-cov
      pytest-xdist
+     mock
      -r{toxinidir}/requirements.txt
      chrome: -r{toxinidir}/requirements_chrome.txt
      firefox: -r{toxinidir}/requirements_firefox.txt
+     sauce: -r{toxinidir}/requirements_sauce.txt
      servo: -r{toxinidir}/requirements_servo.txt
 
-commands = pytest --cov
+commands = pytest {posargs:--cov}
 
 [testenv:py27-flake8]
 # flake8 versions should be kept in sync across tools/tox.ini, tools/wpt/tox.ini, and tools/wptrunner/tox.ini

tests/wpt/web-platform-tests/tools/wptrunner/wptrunner/browsers/sauce.py

@@ -156,10 +156,28 @@ class SauceConnect():
             "web-platform.test",
             "*.web-platform.test"
         ])
-        while not os.path.exists('./sauce_is_ready') and not self.sc_process.poll():
-            time.sleep(5)
 
-        if self.sc_process.returncode is not None and self.sc_process.returncode > 0:
+        # Timeout config vars
+        each_sleep_secs = 1
+        max_wait = 30
+        kill_wait = 5
+
+        tot_wait = 0
+        while not os.path.exists('./sauce_is_ready') and self.sc_process.poll() is None:
+            if tot_wait >= max_wait:
+                self.sc_process.terminate()
+                while self.sc_process.poll() is None:
+                    time.sleep(each_sleep_secs)
+                    tot_wait += each_sleep_secs
+                    if tot_wait >= (max_wait + kill_wait):
+                        self.sc_process.kill()
+                        break
+                raise SauceException("Sauce Connect Proxy was not ready after %d seconds" % tot_wait)
+
+            time.sleep(each_sleep_secs)
+            tot_wait += each_sleep_secs
+
+        if self.sc_process.returncode is not None:
             raise SauceException("Unable to start Sauce Connect Proxy. Process exited with code %s", self.sc_process.returncode)
 
     def __exit__(self, exc_type, exc_val, exc_tb):

tests/wpt/web-platform-tests/tools/wptrunner/wptrunner/executors/executormarionette.py

@@ -96,7 +96,7 @@ class MarionetteProtocol(Protocol):
     def teardown(self):
         try:
             self.marionette._request_in_app_shutdown()
-            self.marionette.delete_session(send_request=False, reset_session_id=True)
+            self.marionette.delete_session(send_request=False)
         except Exception:
             # This is typically because the session never started
             pass
@@ -131,13 +131,14 @@ class MarionetteProtocol(Protocol):
         self.logger.debug("Loading %s" % url)
         self.runner_handle = self.marionette.current_window_handle
         try:
-            self.marionette.navigate(url)
+            self.dismiss_alert(lambda: self.marionette.navigate(url))
         except Exception as e:
             self.logger.critical(
                 "Loading initial page %s failed. Ensure that the "
                 "there are no other programs bound to this port and "
                 "that your firewall rules or network setup does not "
                 "prevent access.\e%s" % (url, traceback.format_exc(e)))
+            raise
         self.marionette.execute_script(
             "document.title = '%s'" % threading.current_thread().name.replace("'", '"'))
 
@@ -157,6 +158,7 @@ class MarionetteProtocol(Protocol):
 
         for handle in handles:
             try:
+                self.dismiss_alert(lambda: self.marionette.switch_to_window(handle))
                 self.marionette.switch_to_window(handle)
                 self.marionette.close()
             except errors.NoSuchWindowException:
@@ -168,6 +170,19 @@ class MarionetteProtocol(Protocol):
         if runner_handle != self.runner_handle:
             self.load_runner(protocol)
 
+    def dismiss_alert(self, f):
+        while True:
+            try:
+                f()
+            except errors.UnexpectedAlertOpen:
+                alert = self.marionette.switch_to_alert()
+                try:
+                    alert.dismiss()
+                except errors.NoAlertPresentException:
+                    pass
+            else:
+                break
+
     def wait(self):
         try:
             socket_timeout = self.marionette.client.socket_timeout

tests/wpt/web-platform-tests/tools/wptrunner/wptrunner/executors/pytestrunner/runner.py

@@ -52,12 +52,13 @@ def run(path, server_config, session_config, timeout=0):
     with TemporaryDirectory() as cache:
         try:
             pytest.main(["--strict",  # turn warnings into errors
-                         "--verbose",  # show each individual subtest
+                         "-vv",  # show each individual subtest and full failure logs
                          "--capture", "no",  # enable stdout/stderr from tests
                          "--basetemp", cache,  # temporary directory
                          "--showlocals",  # display contents of variables in local scope
                          "-p", "no:mozlog",  # use the WPT result recorder
                          "-p", "no:cacheprovider",  # disable state preservation across invocations
+                         "-o=console_output_style=classic",  # disable test progress bar
                          path],
                         plugins=[harness, subtests])
         except Exception as e:

tests/wpt/web-platform-tests/tools/wptrunner/wptrunner/metadata.py

@@ -65,9 +65,9 @@ def update_expected(test_paths, serve_root, log_file_names,
                 for test in tree.iterchildren():
                     for subtest in test.iterchildren():
                         if subtest.new_disabled:
-                            print os.path.dirname(subtest.root.test_path) + "/" + subtest.name
+                            print "disabled: %s" % os.path.dirname(subtest.root.test_path) + "/" + subtest.name
                     if test.new_disabled:
-                        print test.root.test_path
+                        print "disabled: %s" % test.root.test_path
 
     results_changed = [item.test_path for item in expected_map.itervalues() if item.modified]
 

tests/wpt/web-platform-tests/tools/wptrunner/wptrunner/tests/browsers/test_sauce.py

@@ -0,0 +1,84 @@
+import sys
+from os.path import join, dirname
+
+import mock
+import pytest
+
+sys.path.insert(0, join(dirname(__file__), "..", "..", ".."))
+
+sauce = pytest.importorskip("wptrunner.browsers.sauce")
+
+
+def test_sauceconnect_success():
+    with mock.patch.object(sauce.SauceConnect, "upload_prerun_exec"),\
+            mock.patch.object(sauce.subprocess, "Popen") as Popen,\
+            mock.patch.object(sauce.os.path, "exists") as exists:
+        # Act as if it's still running
+        Popen.return_value.poll.return_value = None
+        Popen.return_value.returncode = None
+        # Act as if file created
+        exists.return_value = True
+
+        sauce_connect = sauce.SauceConnect(
+            sauce_user="aaa",
+            sauce_key="bbb",
+            sauce_tunnel_id="ccc",
+            sauce_connect_binary="ddd")
+
+        sauce_connect.__enter__(None)
+
+
+@pytest.mark.parametrize("readyfile,returncode", [
+    (True, 0),
+    (True, 1),
+    (True, 2),
+    (False, 0),
+    (False, 1),
+    (False, 2),
+])
+def test_sauceconnect_failure_exit(readyfile, returncode):
+    with mock.patch.object(sauce.SauceConnect, "upload_prerun_exec"),\
+            mock.patch.object(sauce.subprocess, "Popen") as Popen,\
+            mock.patch.object(sauce.os.path, "exists") as exists,\
+            mock.patch.object(sauce.time, "sleep") as sleep:
+        Popen.return_value.poll.return_value = returncode
+        Popen.return_value.returncode = returncode
+        exists.return_value = readyfile
+
+        sauce_connect = sauce.SauceConnect(
+            sauce_user="aaa",
+            sauce_key="bbb",
+            sauce_tunnel_id="ccc",
+            sauce_connect_binary="ddd")
+
+        with pytest.raises(sauce.SauceException):
+            sauce_connect.__enter__(None)
+
+        # Given we appear to exit immediately with these mocks, sleep shouldn't be called
+        sleep.assert_not_called()
+
+
+def test_sauceconnect_failure_never_ready():
+    with mock.patch.object(sauce.SauceConnect, "upload_prerun_exec"),\
+            mock.patch.object(sauce.subprocess, "Popen") as Popen,\
+            mock.patch.object(sauce.os.path, "exists") as exists,\
+            mock.patch.object(sauce.time, "sleep") as sleep:
+        Popen.return_value.poll.return_value = None
+        Popen.return_value.returncode = None
+        exists.return_value = False
+
+        sauce_connect = sauce.SauceConnect(
+            sauce_user="aaa",
+            sauce_key="bbb",
+            sauce_tunnel_id="ccc",
+            sauce_connect_binary="ddd")
+
+        with pytest.raises(sauce.SauceException):
+            sauce_connect.__enter__(None)
+
+        # We should sleep while waiting for it to create the readyfile
+        sleep.assert_called()
+
+        # Check we actually kill it after termination fails
+        Popen.return_value.terminate.assert_called()
+        Popen.return_value.kill.assert_called()