Fix WPT sync and simplify the update scripts (#29999)

Much of the code used to import WPT tests from upstream has been moved
to the WPT repository itself, so this can be reused. In addition,
simplify the workflows by merging the entire process into mach and also
directly into the GitHub workflow. This should fix WPT imports after
combining compilation of layout and legacy layout.

Signed-off-by: Martin Robinson <mrobinson@igalia.com>
This commit is contained in:
Martin Robinson 2023-07-17 15:55:57 +02:00 committed by GitHub
parent 4c8b47adbb
commit da5b861b3c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 124 additions and 541 deletions

View file

@ -1,59 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at https://mozilla.org/MPL/2.0/.
import os
import sys
from .tree import GitTree, GeckoCommit
from wptrunner.update import setup_logging, WPTUpdate # noqa: F401
from wptrunner.update.base import Step, StepRunner, exit_unclean # noqa: F401
from wptrunner.update.update import LoadConfig, SyncFromUpstream, UpdateMetadata # noqa: F401
from wptrunner import wptcommandline # noqa: F401
class LoadTrees(Step):
"""Load gecko tree and sync tree containing web-platform-tests"""
provides = ["local_tree", "sync_tree"]
def create(self, state):
if os.path.exists(state.sync["path"]):
sync_tree = GitTree(root=state.sync["path"])
else:
sync_tree = None
assert GitTree.is_type()
state.update({"local_tree": GitTree(commit_cls=GeckoCommit),
"sync_tree": sync_tree})
class UpdateRunner(StepRunner):
"""Overall runner for updating web-platform-tests in Gecko."""
steps = [LoadConfig,
LoadTrees,
SyncFromUpstream,
UpdateMetadata]
def run_update(**kwargs):
logger = setup_logging(kwargs, {"mach": sys.stdout})
updater = WPTUpdate(logger, runner_cls=UpdateRunner, **kwargs)
return updater.run() != exit_unclean
def create_parser():
parser = wptcommandline.create_parser_update()
parser.add_argument("--layout-2020", "--with-layout-2020", default=False, action="store_true",
help="Use expected results for the 2020 layout engine")
parser.add_argument("--layout-2013", "--with-layout-2013", default=True, action="store_true",
help="Use expected results for the 2013 layout engine")
return parser
def check_args(kwargs):
wptcommandline.set_from_config(kwargs)
if hasattr(wptcommandline, 'check_paths'):
wptcommandline.check_paths(kwargs)
return kwargs

View file

@ -1,201 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at https://mozilla.org/MPL/2.0/.
from distutils.spawn import find_executable
import re
import subprocess
import sys
import tempfile
from wptrunner import update as wptupdate
from wptrunner.update.tree import Commit, CommitMessage, get_unique_name
class GitTree(wptupdate.tree.GitTree):
def __init__(self, *args, **kwargs):
"""Extension of the basic GitTree with extra methods for
transfering patches"""
commit_cls = kwargs.pop("commit_cls", Commit)
wptupdate.tree.GitTree.__init__(self, *args, **kwargs)
self.commit_cls = commit_cls
def create_branch(self, name, ref=None):
"""Create a named branch,
:param name: String representing the branch name.
:param ref: None to use current HEAD or rev that the branch should point to"""
args = []
if ref is not None:
if hasattr(ref, "sha1"):
ref = ref.sha1
args.append(ref)
self.git("branch", name, *args)
def commits_by_message(self, message, path=None):
"""List of commits with messages containing a given string.
:param message: The string that must be contained in the message.
:param path: Path to a file or directory the commit touches
"""
args = ["--pretty=format:%H", "--reverse", "-z", "--grep=%s" % message]
if path is not None:
args.append("--")
args.append(path)
data = self.git("log", *args)
return [self.commit_cls(self, sha1) for sha1 in data.split("\0")]
def log(self, base_commit=None, path=None):
"""List commits touching a certian path from a given base commit.
:base_param commit: Commit object for the base commit from which to log
:param path: Path that the commits must touch
"""
args = ["--pretty=format:%H", "--reverse", "-z", "--no-merges"]
if base_commit is not None:
args.append("%s.." % base_commit.sha1)
if path is not None:
args.append("--")
args.append(path)
data = self.git("log", *args)
return [self.commit_cls(self, sha1) for sha1 in data.split("\0") if sha1]
def import_patch(self, patch, strip_count):
"""Import a patch file into the tree and commit it
:param patch: a Patch object containing the patch to import
"""
with tempfile.NamedTemporaryFile() as f:
f.write(patch.diff)
f.flush()
f.seek(0)
self.git("apply", "--index", f.name, "-p", str(strip_count))
self.git("commit", "-m", patch.message.text, "--author=%s" % patch.full_author)
def rebase(self, ref, continue_rebase=False):
"""Rebase the current branch onto another commit.
:param ref: A Commit object for the commit to rebase onto
:param continue_rebase: Continue an in-progress rebase"""
if continue_rebase:
args = ["--continue"]
else:
if hasattr(ref, "sha1"):
ref = ref.sha1
args = [ref]
self.git("rebase", *args)
def push(self, remote, local_ref, remote_ref, force=False):
"""Push local changes to a remote.
:param remote: URL of the remote to push to
:param local_ref: Local branch to push
:param remote_ref: Name of the remote branch to push to
:param force: Do a force push
"""
args = []
if force:
args.append("-f")
args.extend([remote, "%s:%s" % (local_ref, remote_ref)])
self.git("push", *args)
def unique_branch_name(self, prefix):
"""Get an unused branch name in the local tree
:param prefix: Prefix to use at the start of the branch name"""
branches = [ref[len("refs/heads/"):] for sha1, ref in self.list_refs()
if ref.startswith("refs/heads/")]
return get_unique_name(branches, prefix)
class Patch(object):
def __init__(self, author, email, message, merge_message, diff):
self.author = author
self.email = email
self.merge_message = merge_message
if isinstance(message, CommitMessage):
self.message = message
else:
self.message = GeckoCommitMessage(message)
self.diff = diff
def __repr__(self):
return "<Patch (%s)>" % self.message.full_summary
@property
def full_author(self):
return "%s <%s>" % (self.author, self.email)
@property
def empty(self):
return bool(self.diff.strip())
class GeckoCommitMessage(CommitMessage):
"""Commit message following the Gecko conventions for identifying bug number
and reviewer"""
# c.f. http://hg.mozilla.org/hgcustom/version-control-tools/file/tip/hghooks/mozhghooks/commit-message.py
# which has the regexps that are actually enforced by the VCS hooks. These are
# slightly different because we need to parse out specific parts of the message rather
# than just enforce a general pattern.
_bug_re = re.compile(r"^Bug (\d+)[^\w]*(?:Part \d+[^\w]*)?(.*?)\s*(?:r=(\w*))?$",
re.IGNORECASE)
_merge_re = re.compile(r"^Auto merge of #(\d+) - [^:]+:[^,]+, r=(.+)$", re.IGNORECASE)
_backout_re = re.compile(r"^(?:Back(?:ing|ed)\s+out)|Backout|(?:Revert|(?:ed|ing))",
re.IGNORECASE)
_backout_sha1_re = re.compile(r"(?:\s|\:)(0-9a-f){12}")
def _parse_message(self):
CommitMessage._parse_message(self)
if self._backout_re.match(self.full_summary):
self.backouts = self._backout_re.findall(self.full_summary)
else:
self.backouts = []
m = self._merge_re.match(self.full_summary)
if m is not None:
self.bug, self.reviewer = m.groups()
self.summary = self.full_summary
else:
m = self._bug_re.match(self.full_summary)
if m is not None:
self.bug, self.summary, self.reviewer = m.groups()
else:
self.bug, self.summary, self.reviewer = None, self.full_summary, None
class GeckoCommit(Commit):
msg_cls = GeckoCommitMessage
def __init__(self, tree, sha1, is_merge=False):
Commit.__init__(self, tree, sha1)
if not is_merge:
args = ["-c", sha1]
try:
merge_rev = self.git("when-merged", *args).strip()
except subprocess.CalledProcessError as exn:
if not find_executable('git-when-merged'):
print('Please add the `when-merged` git command to your PATH '
'(https://github.com/mhagger/git-when-merged/).')
sys.exit(1)
raise exn
self.merge = GeckoCommit(tree, merge_rev, True)
def export_patch(self, path=None):
"""Convert a commit in the tree to a Patch with the bug number and
reviewer stripped from the message"""
args = ["--binary", self.sha1]
if path is not None:
args.append("--")
args.append(path)
diff = self.git("show", *args)
merge_message = self.merge.message if self.merge else None
return Patch(self.author, self.email, self.message, merge_message, diff)

View file

@ -4,26 +4,130 @@
# pylint: disable=missing-docstring
import os
import subprocess
import shutil
import sys
from wptrunner.update import setup_logging, WPTUpdate # noqa: F401
from wptrunner.update.base import exit_unclean # noqa: F401
from wptrunner import wptcommandline # noqa: F401
from . import WPT_PATH, update_args_for_legacy_layout
from . import importer
from . import manifestupdate
TEST_ROOT = os.path.join(WPT_PATH, 'tests')
META_ROOTS = [
os.path.join(WPT_PATH, 'meta'),
os.path.join(WPT_PATH, 'meta-legacy')
]
def set_if_none(args: dict, key: str, value):
if key not in args or args[key] is None:
args[key] = value
def do_sync(**kwargs) -> int:
last_commit = subprocess.check_output(["git", "log", "-1"])
# Commits should always be authored by the GitHub Actions bot.
os.environ["GIT_AUTHOR_NAME"] = "Servo WPT Sync"
os.environ["GIT_AUTHOR_EMAIL"] = "josh+wptsync@joshmatthews.net"
os.environ["GIT_COMMITTER_NAME"] = os.environ['GIT_AUTHOR_NAME']
os.environ["GIT_COMMITTER_EMAIL"] = os.environ['GIT_AUTHOR_EMAIL']
print("Updating WPT from upstream...")
run_update(**kwargs)
if last_commit == subprocess.check_output(["git", "log", "-1"]):
return 255
# Update the manifest twice to reach a fixed state
# (https://github.com/servo/servo/issues/22275).
print("Updating test manifests...")
manifestupdate.update(check_clean=False)
manifestupdate.update(check_clean=False)
remove_unused_metadata()
if subprocess.check_call(["git", "commit", "-a", "--amend", "--no-edit", "-q"]) != 0:
print("Ammending commit failed. Bailing out.")
return 1
return 0
def update_tests(**kwargs):
def remove_unused_metadata():
print("Removing unused results...")
unused_files = []
unused_dirs = []
for meta_root in META_ROOTS:
for base_dir, dir_names, files in os.walk(meta_root):
# Skip recursing into any directories that were previously found to be missing.
if base_dir in unused_dirs:
# Skip processing any subdirectories of known missing directories.
unused_dirs += [os.path.join(base_dir, x) for x in dir_names]
continue
for dir_name in dir_names:
dir_path = os.path.join(base_dir, dir_name)
# Skip any known directories that are meta-metadata.
if dir_name == '.cache':
unused_dirs.append(dir_path)
continue
# Turn tests/wpt/meta/foo into tests/wpt/tests/foo.
test_dir = os.path.join(TEST_ROOT, os.path.relpath(dir_path, meta_root))
if not os.path.exists(test_dir):
unused_dirs.append(dir_path)
for fname in files:
# Skip any known files that are meta-metadata.
if not fname.endswith(".ini") or fname == '__dir__.ini':
continue
# Turn tests/wpt/meta/foo/bar.html.ini into tests/wpt/tests/foo/bar.html.
test_file = os.path.join(
TEST_ROOT, os.path.relpath(base_dir, meta_root), fname[:-4])
if not os.path.exists(test_file):
unused_files.append(os.path.join(base_dir, fname))
for file in unused_files:
print(f" - {file}")
os.remove(file)
for directory in unused_dirs:
print(f" - {directory}")
shutil.rmtree(directory)
def update_tests(**kwargs) -> int:
def set_if_none(args: dict, key: str, value):
if key not in args or args[key] is None:
args[key] = value
set_if_none(kwargs, "product", "servo")
set_if_none(kwargs, "config", os.path.join(WPT_PATH, "config.ini"))
kwargs["store_state"] = False
importer.check_args(kwargs)
wptcommandline.set_from_config(kwargs)
if hasattr(wptcommandline, 'check_paths'):
wptcommandline.check_paths(kwargs)
update_args_for_legacy_layout(kwargs)
return 1 if not importer.run_update(**kwargs) else 0
if kwargs.get('sync', False):
return do_sync(**kwargs)
return 0 if run_update(**kwargs) else 1
def create_parser(**kwargs):
return importer.create_parser()
def run_update(**kwargs) -> bool:
"""Run the update process returning True if the process is successful."""
logger = setup_logging(kwargs, {"mach": sys.stdout})
return WPTUpdate(logger, **kwargs).run() != exit_unclean
def create_parser(**_kwargs):
parser = wptcommandline.create_parser_update()
parser.add_argument("--layout-2020", "--with-layout-2020", default=False, action="store_true",
help="Use expected results for the 2020 layout engine")
parser.add_argument("--layout-2013", "--with-layout-2013", default=True, action="store_true",
help="Use expected results for the 2013 layout engine")
return parser