Update web-platform-tests to revision 58eb04cecbbec2e18531ab440225e38944a9c444

tests/wpt/web-platform-tests/css/tools/travis/build.py

@@ -0,0 +1,223 @@
+import itertools
+import os
+import shutil
+import subprocess
+import sys
+
+import vcs
+
+here = os.path.abspath(os.path.dirname(__file__))
+
+source_dir = os.path.join(here, "..", "..")
+
+remote_built = "https://github.com/jgraham/css-test-built.git"
+built_dir = os.path.join(here, "css-test-built")
+
+local_files = ["manifest", "serve", "serve.py", ".gitmodules", "tools", "resources",
+               "config.default.json"]
+
+def get_hgsubstate():
+    state = {}
+    with open(os.path.join(source_dir, ".hgsubstate"), "r") as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            revision, path = line.split(" ", 1)
+            state[path] = revision
+    return state
+
+def fetch_submodules():
+    hg = vcs.hg
+    orig_dir = os.getcwd()
+    state = get_hgsubstate()
+    for tool in ["apiclient", "w3ctestlib"]:
+        dest_dir = os.path.join(source_dir, "tools", tool)
+        repo_path = "tools/" + tool
+        if os.path.exists(os.path.join(dest_dir, ".hg")):
+            try:
+                os.chdir(dest_dir)
+                if repo_path in state:
+                    rev = state[repo_path]
+                    try:
+                        hg("update", rev, log_error=False)
+                    except subprocess.CalledProcessError:
+                        hg("pull")
+                        hg("update", rev)
+                else:
+                    hg("pull")
+                    hg("update")
+            finally:
+                os.chdir(orig_dir)
+        else:
+            hg("clone", ("https://hg.csswg.org/dev/%s" % tool), dest_dir)
+            try:
+                os.chdir(dest_dir)
+                if repo_path in state:
+                    hg("update", state[repo_path])
+                else:
+                    hg("update")
+            finally:
+                os.chdir(orig_dir)
+
+def update_dist():
+    if not os.path.exists(built_dir) or not vcs.is_git_root(built_dir):
+        git = vcs.git
+        git("clone", "--depth", "1", remote_built, built_dir)
+    else:
+        git = vcs.bind_to_repo(vcs.git, built_dir)
+        git("fetch")
+        if "origin/master" in git("branch", "-a"):
+            git("checkout", "master")
+            git("merge", "--ff-only", "origin/master")
+
+    git = vcs.bind_to_repo(vcs.git, built_dir)
+    git("config", "user.email", "CssBuildBot@users.noreply.github.com")
+    git("config", "user.name", "CSS Build Bot")
+    git("submodule", "update", "--init", "--recursive")
+
+def setup_virtualenv():
+    virtualenv_path = os.path.join(here, "_virtualenv")
+
+    if not os.path.exists(virtualenv_path):
+        subprocess.check_call(["virtualenv", virtualenv_path])
+
+    activate_path = os.path.join(virtualenv_path, "bin", "activate_this.py")
+
+    execfile(activate_path, dict(__file__=activate_path))
+
+    subprocess.check_call(["pip", "-q", "install", "mercurial"])
+    subprocess.check_call(["pip", "-q", "install", "html5lib==0.9999999"])
+    subprocess.check_call(["pip", "-q", "install", "lxml"])
+    subprocess.check_call(["pip", "-q", "install", "Template-Python"])
+
+
+def update_to_changeset(changeset):
+    git = vcs.bind_to_repo(vcs.git, source_dir)
+    git("checkout", changeset)
+    apply_build_system_fixes()
+
+def apply_build_system_fixes():
+    fixes = [
+        "c017547f65e07bdd889736524d47824d032ba2e8",
+        "cb4a737a88aa7e2f4e54383c57ffa2dfae093dcf",
+        "ec540343a3e729644c8178dbcf6d063dca20d49f",
+    ]
+    git = vcs.bind_to_repo(vcs.git, source_dir)
+    for fix in fixes:
+        git("cherry-pick", "--keep-redundant-commits", fix)
+
+def build_tests():
+    subprocess.check_call(["python", os.path.join(source_dir, "tools", "build.py")],
+                           cwd=source_dir)
+
+def remove_current_files():
+    for node in os.listdir(built_dir):
+        if node.startswith(".git"):
+            continue
+
+        if node in ("resources", "tools"):
+            continue
+
+        path = os.path.join(built_dir, node)
+        if os.path.isdir(path):
+            shutil.rmtree(path)
+        else:
+            os.remove(path)
+
+def copy_files():
+    dist_path = os.path.join(source_dir, "dist")
+    for node in os.listdir(dist_path):
+        src_path = os.path.join(dist_path, node)
+        dest_path = os.path.join(built_dir, node)
+        if os.path.isdir(src_path):
+            shutil.copytree(src_path, dest_path)
+        else:
+            shutil.copy2(src_path, dest_path)
+
+def update_git():
+    git = vcs.bind_to_repo(vcs.git, built_dir)
+    git("add", ".")
+
+def add_changeset(changeset):
+    git = vcs.bind_to_repo(vcs.git, built_dir)
+
+    dest_path = os.path.join(built_dir, "source_rev")
+    with open(dest_path, "w") as f:
+        f.write(changeset)
+    git("add", os.path.relpath(dest_path, built_dir))
+
+def commit(changeset):
+    git = vcs.git
+
+    msg = git("log", "-r", changeset, "-n", "1", "--pretty=%B", repo=source_dir)
+    msg = "%s\n\nBuild from revision %s" % (msg, changeset)
+
+    git("commit", "-m", msg, repo=built_dir)
+
+def get_new_commits():
+    git = vcs.bind_to_repo(vcs.git, source_dir)
+    commit_path = os.path.join(built_dir, "source_rev")
+    with open(commit_path) as f:
+        prev_commit = f.read().strip()
+
+    if git("rev-parse", "--revs-only", prev_commit).strip() != prev_commit:
+        # we don't have prev_commit in current tree, so let's just do what's new
+        commit_range = os.environ['TRAVIS_COMMIT_RANGE']
+        assert (os.environ["TRAVIS_PULL_REQUEST"] != "false" or
+                os.environ["TRAVIS_BRANCH"] != "master")
+    else:
+        merge_base = git("merge-base", prev_commit, os.environ['TRAVIS_COMMIT']).strip()
+        commit_range = "%s..%s" % (merge_base, os.environ['TRAVIS_COMMIT'])
+
+    commits = git("log", "--pretty=%H", "-r", commit_range).strip()
+    if not commits:
+        return []
+    return reversed(commits.split("\n"))
+
+def maybe_push():
+    if os.environ["TRAVIS_PULL_REQUEST"] != "false":
+        return
+
+    if os.environ["TRAVIS_BRANCH"] != "master":
+        return
+
+    git = vcs.bind_to_repo(vcs.git, built_dir)
+
+    out = "https://%s@github.com/jgraham/css-test-built.git" % os.environ["TOKEN"]
+    git("remote", "add", "out", out, quiet=True)
+
+    for i in range(2):
+        try:
+            git("push", "out", "HEAD:master")
+        except subprocess.CalledProcessError:
+            if i == 0:
+                git("fetch", "origin")
+                git("rebase", "origin/master")
+        else:
+            return
+
+    raise Exception("Push failed")
+
+def main():
+    setup_virtualenv()
+    fetch_submodules()
+    update_dist()
+    changesets = list(get_new_commits())
+    print >> sys.stderr, "Building %d changesets:" % len(changesets)
+    print >> sys.stderr, "\n".join(changesets)
+    if len(changesets) > 50:
+        raise Exception("Building more than 50 changesets, giving up")
+
+    for changeset in changesets:
+        update_to_changeset(changeset)
+        remove_current_files()
+        build_tests()
+        copy_files()
+        update_git()
+        add_changeset(changeset)
+        commit(changeset)
+    maybe_push()
+
+if __name__ == "__main__":
+    main()