Update web-platform-tests to revision b7a8b84debb42268ea95a45bdad8f727d1facdf7

tests/wpt/web-platform-tests/tools/ci/azure/affected_tests.yml

@@ -16,7 +16,7 @@ steps:
 - template: install_safari.yml
 - template: update_hosts.yml
 - template: update_manifest.yml
-- script: no_proxy='*' ./wpt run --yes --no-pause --no-fail-on-unexpected --no-restart-on-unexpected --affected ${{ parameters.affectedRange }} --log-wptreport $(Build.ArtifactStagingDirectory)/wpt_report.json --channel preview safari
+- script: no_proxy='*' ./wpt run --yes --no-pause --no-fail-on-unexpected --no-restart-on-unexpected --affected ${{ parameters.affectedRange }} --log-wptreport $(Build.ArtifactStagingDirectory)/wpt_report.json --log-wptscreenshot $(Build.ArtifactStagingDirectory)/wpt_screenshot.txt --channel preview safari
   displayName: 'Run tests'
 - task: PublishBuildArtifacts@1
   displayName: 'Publish results'

tests/wpt/web-platform-tests/tools/ci/azure/fyi_hook.yml

@@ -12,7 +12,7 @@ jobs:
   pool:
     vmImage: 'ubuntu-16.04'
   steps:
-  - script: curl -s -S -d "artifact=${{ parameters.artifactName }}" -X POST https://wpt.fyi/api/checks/azure/$(Build.BuildId)
+  - script: curl -f -s -S -d "artifact=${{ parameters.artifactName }}" -X POST https://wpt.fyi/api/checks/azure/$(Build.BuildId)
     displayName: 'Invoke wpt.fyi hook'
-  - script: curl -s -S -d "artifact=${{ parameters.artifactName }}" -X POST https://staging.wpt.fyi/api/checks/azure/$(Build.BuildId)
+  - script: curl -f -s -S -d "artifact=${{ parameters.artifactName }}" -X POST https://staging.wpt.fyi/api/checks/azure/$(Build.BuildId)
     displayName: 'Invoke staging.wpt.fyi hook'

tests/wpt/web-platform-tests/tools/ci/before_install.sh

@@ -1,6 +1,9 @@
 #!/bin/bash
 set -e
 
+export GITHUB_PULL_REQUEST=$TRAVIS_PULL_REQUEST
+export GITHUB_BRANCH=$TRAVIS_BRANCH
+
 if [[ $RUN_JOB -eq 1 ]] || ./wpt test-jobs --includes $JOB; then
     export RUN_JOB=1
     git submodule update --init --recursive 1>&2

tests/wpt/web-platform-tests/tools/ci/ci_resources_unittest.sh

@@ -5,13 +5,9 @@ SCRIPT_DIR=$(cd $(dirname "$0") && pwd -P)
 WPT_ROOT=$SCRIPT_DIR/../..
 cd $WPT_ROOT
 
-source tools/ci/lib.sh
-
 main() {
-    hosts_fixup
-
     cd $WPT_ROOT
-    pip install -U tox
+    pip install --user -U tox
     ./wpt install firefox browser --destination $HOME
     ./wpt install firefox webdriver --destination $HOME/firefox
     export PATH=$HOME/firefox:$PATH

tests/wpt/web-platform-tests/tools/ci/ci_tools_unittest.sh

@@ -19,7 +19,7 @@ run_applicable_tox () {
 }
 
 if ./wpt test-jobs --includes tools_unittest; then
-    pip install -U tox codecov
+    pip install --user -U tox codecov
     cd tools
     run_applicable_tox
     cd $WPT_ROOT

tests/wpt/web-platform-tests/tools/ci/ci_wpt.sh

@@ -5,13 +5,9 @@ SCRIPT_DIR=$(cd $(dirname "$0") && pwd -P)
 WPT_ROOT=$SCRIPT_DIR/../..
 cd $WPT_ROOT
 
-source tools/ci/lib.sh
-
 main() {
     git fetch --quiet --unshallow https://github.com/web-platform-tests/wpt.git +refs/heads/*:refs/remotes/origin/*
-    hosts_fixup
-    install_chrome unstable
-    pip install -U tox codecov
+    pip install --user -U tox codecov
     cd tools/wpt
     tox
 }

tests/wpt/web-platform-tests/tools/ci/ci_wptrunner_infrastructure.sh

@@ -5,7 +5,9 @@ SCRIPT_DIR=$(cd $(dirname "$0") && pwd -P)
 WPT_ROOT=$SCRIPT_DIR/../..
 cd $WPT_ROOT
 
-source tools/ci/lib.sh
+add_wpt_hosts() {
+    ./wpt make-hosts-file | sudo tee -a /etc/hosts
+}
 
 test_infrastructure() {
     local ARGS="";
@@ -21,13 +23,8 @@ main() {
     PRODUCTS=( "firefox" "chrome" )
     ./wpt manifest --rebuild -p ~/meta/MANIFEST.json
     for PRODUCT in "${PRODUCTS[@]}"; do
-        if [ "$PRODUCT" != "firefox" ]; then
-            # Firefox is expected to work using pref settings for DNS
-            # Don't adjust the hostnames in that case to ensure this keeps working
-            hosts_fixup
-        fi
         if [[ "$PRODUCT" == "chrome" ]]; then
-            install_chrome unstable
+            add_wpt_hosts
             test_infrastructure "--binary=$(which google-chrome-unstable)"
         else
             test_infrastructure

tests/wpt/web-platform-tests/tools/ci/lib.sh

@@ -1,35 +0,0 @@
-hosts_fixup() {
-    echo "travis_fold:start:hosts_fixup"
-    echo "Rewriting hosts file"
-    echo "## /etc/hosts ##"
-    cat /etc/hosts
-    sudo sed -i 's/^::1\s*localhost/::1/' /etc/hosts
-    ./wpt make-hosts-file | sudo tee -a /etc/hosts
-    echo "== /etc/hosts =="
-    cat /etc/hosts
-    echo "----------------"
-    echo "travis_fold:end:hosts_fixup"
-}
-
-install_chrome() {
-    channel=$1
-    deb_archive=google-chrome-${channel}_current_amd64.deb
-    wget -q https://dl.google.com/linux/direct/$deb_archive
-
-    # If the environment provides an installation of Google Chrome, the
-    # existing binary may take precedence over the one introduced in this
-    # script. Remove any previously-existing "alternatives" prior to
-    # installation in order to ensure that the new binary is installed as
-    # intended.
-    if sudo update-alternatives --list google-chrome; then
-        sudo update-alternatives --remove-all google-chrome
-    fi
-
-    # Installation will fail in cases where the package has unmet dependencies.
-    # When this occurs, attempt to use the system package manager to fetch the
-    # required packages and retry.
-    if ! sudo dpkg --install $deb_archive; then
-      sudo apt-get install --fix-broken
-      sudo dpkg --install $deb_archive
-    fi
-}

tests/wpt/web-platform-tests/tools/ci/run_tc.py

@@ -0,0 +1,261 @@
+#!/usr/bin/env python
+
+"""Wrapper script for running jobs in TaskCluster
+
+This is intended for running test jobs in TaskCluster. The script
+takes a two positional arguments which are the name of the test job
+and the script to actually run.
+
+The name of the test job is used to determine whether the script should be run
+for this push (this is in lieu of having a proper decision task). There are
+several ways that the script can be scheduled to run
+
+1. The output of wpt test-jobs includes the job name
+2. The job name is included in a job declaration (see below)
+3. The string "all" is included in the job declaration
+4. The job name is set to "all"
+
+A job declaration is a line appearing in the pull request body (for
+pull requests) or first commit message (for pushes) of the form:
+
+tc-jobs: job1,job2,[...]
+
+In addition, there are a number of keyword arguments used to set options for the
+environment in which the jobs run. Documentation for these is in the command help.
+
+As well as running the script, the script sets two environment variables;
+GITHUB_BRANCH which is the branch that the commits will merge into (if it's a PR)
+or the branch that the commits are on (if it's a push), and GITHUB_PULL_REQUEST
+which is the string "false" if the event triggering this job wasn't a pull request
+or the pull request number if it was. The semantics of these variables are chosen
+to match the corresponding TRAVIS_* variables.
+
+Note: for local testing in the Docker image the script ought to still work, but
+full functionality requires that the TASK_EVENT environment variable is set to
+the serialization of a GitHub event payload.
+"""
+
+import argparse
+import json
+import os
+import re
+import subprocess
+import sys
+try:
+    from urllib2 import urlopen
+except ImportError:
+    # Python 3 case
+    from urllib.request import urlopen
+
+
+root = os.path.abspath(
+    os.path.join(os.path.dirname(__file__),
+                 os.pardir,
+                 os.pardir))
+
+
+def run(cmd, return_stdout=False, **kwargs):
+    print(" ".join(cmd))
+    if return_stdout:
+        f = subprocess.check_output
+    else:
+        f = subprocess.check_call
+    return f(cmd, **kwargs)
+
+
+def start(cmd):
+    print(" ".join(cmd))
+    subprocess.Popen(cmd)
+
+
+def get_parser():
+    p = argparse.ArgumentParser()
+    p.add_argument("--oom-killer",
+                   action="store_true",
+                   default=False,
+                   help="Run userspace OOM killer")
+    p.add_argument("--hosts",
+                   dest="hosts_file",
+                   action="store_true",
+                   default=True,
+                   help="Setup wpt entries in hosts file")
+    p.add_argument("--no-hosts",
+                   dest="hosts_file",
+                   action="store_false",
+                   help="Don't setup wpt entries in hosts file")
+    p.add_argument("--browser",
+                   action="append",
+                   default=[],
+                   help="Browsers that will be used in the job")
+    p.add_argument("--channel",
+                   default=None,
+                   choices=["experimental", "dev", "nightly", "beta", "stable"],
+                   help="Chrome browser channel")
+    p.add_argument("--xvfb",
+                   action="store_true",
+                   help="Start xvfb")
+    p.add_argument("--checkout",
+                   help="Revision to checkout before starting job")
+    p.add_argument("job",
+                   help="Name of the job associated with the current event")
+    p.add_argument("script",
+                   help="Script to run for the job")
+    p.add_argument("script_args",
+                   nargs=argparse.REMAINDER,
+                   help="Additional arguments to pass to the script")
+    return p
+
+
+def start_userspace_oom_killer():
+    # Start userspace OOM killer: https://github.com/rfjakob/earlyoom
+    # It will report memory usage every minute and prefer to kill browsers.
+    start(["sudo", "earlyoom", "-p", "-r", "60" "--prefer=(chrome|firefox)", "--avoid=python"])
+
+
+def make_hosts_file():
+    subprocess.check_call(["sudo", "sh", "-c", "./wpt make-hosts-file >> /etc/hosts"])
+
+
+def checkout_revision(rev):
+    subprocess.check_call(["git", "checkout", "-q", rev])
+
+
+def install_chrome(channel):
+    if channel in ("experimental", "dev", "nightly"):
+        deb_archive = "google-chrome-unstable_current_amd64.deb"
+    elif channel == "beta":
+        deb_archive = "google-chrome-beta_current_amd64.deb"
+    elif channel == "stable":
+        deb_archive = "google-chrome-stable_current_amd64.deb"
+    else:
+        raise ValueError("Unrecognized release channel: %s" % channel)
+
+    dest = os.path.join("/tmp", deb_archive)
+    resp = urlopen("https://dl.google.com/linux/direct/%s" % deb_archive)
+    with open(dest, "w") as f:
+        f.write(resp.read())
+
+    subprocess.check_call(["sudo", "apt-get", "-qqy", "update"])
+    subprocess.check_call(["sudo", "gdebi", "-n", "/tmp/%s" % deb_archive])
+
+
+def start_xvfb():
+    start(["sudo", "Xvfb", os.environ["DISPLAY"], "-screen", "0",
+           "%sx%sx%s" % (os.environ["SCREEN_WIDTH"],
+                         os.environ["SCREEN_HEIGHT"],
+                         os.environ["SCREEN_DEPTH"])])
+    start(["sudo", "fluxbox", "-display", os.environ["DISPLAY"]])
+
+
+def get_extra_jobs(event):
+    body = None
+    jobs = set()
+    if "commits" in event:
+        body = event["commits"][0]["message"]
+    elif "pull_request" in event:
+        body = event["pull_request"]["body"]
+
+    if not body:
+        return jobs
+
+    regexp = re.compile(r"\s*tc-jobs:(.*)$")
+
+    for line in body.splitlines():
+        m = regexp.match(line)
+        if m:
+            items = m.group(1)
+            for item in items.split(","):
+                jobs.add(item.strip())
+            break
+    return jobs
+
+
+def set_variables(event):
+    # Set some variables that we use to get the commits on the current branch
+    ref_prefix = "refs/heads/"
+    pull_request = "false"
+    branch = None
+    if "pull_request" in event:
+        pull_request = str(event["pull_request"]["number"])
+        # Note that this is the branch that a PR will merge to,
+        # not the branch name for the PR
+        branch = event["pull_request"]["base"]["ref"]
+    elif "ref" in event:
+        branch = event["ref"]
+        if branch.startswith(ref_prefix):
+            branch = branch[len(ref_prefix):]
+
+    os.environ["GITHUB_PULL_REQUEST"] = pull_request
+    if branch:
+        os.environ["GITHUB_BRANCH"] = branch
+
+
+def include_job(job):
+    jobs_str = run([os.path.join(root, "wpt"),
+                    "test-jobs"], return_stdout=True)
+    print(jobs_str)
+    return job in set(jobs_str.splitlines())
+
+
+def setup_environment(args):
+    if args.hosts_file:
+        make_hosts_file()
+
+    if "chrome" in args.browser:
+        assert args.channel is not None
+        install_chrome(args.channel)
+
+    if args.xvfb:
+        start_xvfb()
+
+    if args.oom_killer:
+        start_userspace_oom_killer()
+
+    if args.checkout:
+        checkout_revision(args.checkout)
+
+
+def main():
+    args = get_parser().parse_args()
+    try:
+        event = json.loads(os.environ["TASK_EVENT"])
+    except KeyError:
+        # For example under local testing
+        event = {}
+
+    if event:
+        set_variables(event)
+
+    if os.environ.get("GITHUB_BRANCH"):
+        # Ensure that the remote base branch exists
+        # TODO: move this somewhere earlier in the task
+        run(["git", "fetch", "origin", "%s:%s" % (os.environ["GITHUB_BRANCH"],
+                                                  os.environ["GITHUB_BRANCH"])])
+
+    extra_jobs = get_extra_jobs(event)
+
+    job = args.job
+
+    run_if = [(lambda: job == "all", "job set to 'all'"),
+              (lambda:"all" in extra_jobs, "Manually specified jobs includes 'all'"),
+              (lambda:job in extra_jobs, "Manually specified jobs includes '%s'" % job),
+              (lambda:include_job(job), "CI required jobs includes '%s'" % job)]
+
+    for fn, msg in run_if:
+        if fn():
+            print(msg)
+            break
+    else:
+        print("Job not scheduled for this push")
+        return
+
+    # Run the job
+    setup_environment(args)
+    os.chdir(root)
+    cmd = [args.script] + args.script_args
+    print(cmd)
+    sys.exit(subprocess.call(cmd))
+
+
+if __name__ == "__main__":
+    main()

tests/wpt/web-platform-tests/tools/ci/start.sh

@@ -1,31 +1 @@
-# This script is designed to be sourced from tools/docker/start.sh
-
-# Start userspace OOM killer: https://github.com/rfjakob/earlyoom
-# It will report memory usage every minute and prefer to kill browsers.
-sudo earlyoom -p -r 60 --prefer '(chrome|firefox)' --avoid 'python' &
-
-sudo sh -c './wpt make-hosts-file >> /etc/hosts'
-
-if [[ $BROWSER == "chrome" ]] || [[ "$BROWSER" == all ]]
-then
-    # Install Chrome dev
-    if [[ "$CHANNEL" == "dev" ]] || [[ "$CHANNEL" == "nightly" ]]
-    then
-       deb_archive=google-chrome-unstable_current_amd64.deb
-    elif [[ "$CHANNEL" == "beta" ]]
-    then
-        deb_archive=google-chrome-beta_current_amd64.deb
-    elif [[ "$CHANNEL" == "stable" ]]
-    then
-        deb_archive=google-chrome-stable_current_amd64.deb
-    else
-        echo Unrecognized release channel: $CHANNEL >&2
-        exit 1
-    fi
-    wget -O /tmp/$deb_archive https://dl.google.com/linux/direct/$deb_archive
-
-    sudo apt-get -qqy update && sudo gdebi -n /tmp/$deb_archive
-fi
-
-sudo Xvfb $DISPLAY -screen 0 ${SCREEN_WIDTH}x${SCREEN_HEIGHT}x${SCREEN_DEPTH} &
-sudo fluxbox -display $DISPLAY &
+# Contents of this script superceeded by tools/ci/run_tc.py

tests/wpt/web-platform-tests/tools/ci/tests/test_run_tc.py

@@ -0,0 +1,33 @@
+import pytest
+
+from six import iteritems
+
+from tools.ci import run_tc
+
+
+@pytest.mark.parametrize("msg,expected", [
+    ("Some initial line\n\ntc-jobs:foo,bar", set(["foo", "bar"])),
+    ("Some initial line\n\ntc-jobs:foo, bar", set(["foo", "bar"])),
+    ("tc-jobs:foo, bar   \nbaz", set(["foo", "bar"])),
+    ("tc-jobs:all", set(["all"])),
+    ("", set()),
+    ("tc-jobs:foo\ntc-jobs:bar", set(["foo"]))])
+@pytest.mark.parametrize("event", [
+    {"commits": [{"message": "<message>"}]},
+    {"pull_request": {"body": "<message>"}}
+])
+def test_extra_jobs_pr(msg, expected, event):
+    def sub(obj):
+        """Copy obj, except if it's a string with the value <message>
+        replace it with the value of the msg argument"""
+        if isinstance(obj, dict):
+            return {key: sub(value) for (key, value) in iteritems(obj)}
+        elif isinstance(obj, list):
+            return [sub(value) for value in obj]
+        elif obj == "<message>":
+            return msg
+        return obj
+
+    event = sub(event)
+
+    assert run_tc.get_extra_jobs(event) == expected