diff --git a/.github/workflows/linux-wpt.yml b/.github/workflows/linux-wpt.yml index 0570fae701b..c1336ed2ef0 100644 --- a/.github/workflows/linux-wpt.yml +++ b/.github/workflows/linux-wpt.yml @@ -41,10 +41,10 @@ jobs: wget http://mirrors.kernel.org/ubuntu/pool/main/libf/libffi/libffi6_3.2.1-8_amd64.deb sudo apt install ./libffi6_3.2.1-8_amd64.deb python3 ./mach bootstrap-gstreamer - - name: Fetch upstream changes before testing + - name: Sync from upstream WPT if: ${{ inputs.wpt == 'sync' }} run: | - ./etc/ci/update-wpt-checkout fetch-upstream-changes + ./mach update-wpt --sync --patch - name: Run tests if: ${{ inputs.wpt != 'sync' }} run: | diff --git a/.github/workflows/pull-request-wpt-export.yml b/.github/workflows/pull-request-wpt-export.yml index f81a9b0c408..b9b34a902f2 100644 --- a/.github/workflows/pull-request-wpt-export.yml +++ b/.github/workflows/pull-request-wpt-export.yml @@ -1,4 +1,4 @@ -name: Pull request (WPT export) +name: WPT Export on: pull_request_target: types: ['opened', 'synchronize', 'reopened', 'edited', 'closed'] diff --git a/.github/workflows/scheduled-wpt-import.yml b/.github/workflows/scheduled-wpt-import.yml index ad2be05e715..2d50192ceea 100644 --- a/.github/workflows/scheduled-wpt-import.yml +++ b/.github/workflows/scheduled-wpt-import.yml @@ -1,4 +1,4 @@ -name: Scheduled WPT import +name: WPT Import on: schedule: @@ -46,16 +46,16 @@ jobs: git fetch --unshallow upstream - name: Fetch upstream changes before syncing run: | - ./etc/ci/update-wpt-checkout fetch-upstream-changes - - name: Run WPT Update - env: - MAX_CHUNK_ID: 20 - WPT_SYNC_TOKEN: ${{ secrets.WPT_SYNC_TOKEN }} + ./mach update-wpt --sync --patch + - name: Amend commit with test results run: | export CURRENT_DATE=$(date +"%d-%m-%Y") echo $CURRENT_DATE echo "CURRENT_DATE=$CURRENT_DATE" >> $GITHUB_ENV - ./etc/ci/wpt-scheduled-update.sh + ./mach update-wpt wpt-logs-linux-layout-2013/test-wpt.*.log + ./mach update-wpt --layout-2020 wpt-logs-linux-layout-2020/test-wpt.*.log + git add tests/wpt/meta tests/wpt/meta-legacy-layout + git commit -a --amend --no-edit - name: Push changes uses: ad-m/github-push-action@master with: @@ -68,9 +68,8 @@ jobs: BODY=$(cat < or the MIT license -# , at your -# option. This file may not be copied, modified, or distributed -# except according to those terms. - -# For all directories and ini files under the WPT metadata directory, -# check whether there is a match directory/test file in the vendored WPT -# test collection. If there is not, the test result file is leftover since -# the original test was moved/renamed/deleted and no longer serves any -# purpose. - -import os - -test_root = os.path.join('tests', 'wpt', 'tests') -meta_root = os.path.join('tests', 'wpt', 'meta') - -missing_dirs = [] - -for base_dir, dir_names, files in os.walk(meta_root): - # Skip recursing into any directories that were previously found to be missing. - if base_dir in missing_dirs: - # Skip processing any subdirectories of known missing directories. - missing_dirs += map(lambda x: os.path.join(base_dir, x), dir_names) - continue - - for dir_name in dir_names: - meta_dir = os.path.join(base_dir, dir_name) - - # Skip any known directories that are meta-metadata. - if dir_name == '.cache': - missing_dirs += [meta_dir] - continue - - # Turn tests/wpt/meta/foo into tests/wpt/tests/foo. - test_dir = os.path.join(test_root, os.path.relpath(meta_dir, meta_root)) - if not os.path.exists(test_dir): - missing_dirs += [meta_dir] - print(meta_dir) - - for fname in files: - # Skip any known files that are meta-metadata. - if fname in ['__dir__.ini', 'MANIFEST.json', 'mozilla-sync']: - continue - - # Turn tests/wpt/meta/foo/bar.html.ini into tests/wpt/tests/foo/bar.html. - test_dir = os.path.join(test_root, os.path.relpath(base_dir, meta_root)) - test_file = os.path.join(test_dir, fname) - if not os.path.exists(os.path.splitext(test_file)[0]): - print(os.path.join(base_dir, fname)) diff --git a/python/wpt/importer/__init__.py b/python/wpt/importer/__init__.py deleted file mode 100644 index 789ee2b0a42..00000000000 --- a/python/wpt/importer/__init__.py +++ /dev/null @@ -1,59 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at https://mozilla.org/MPL/2.0/. - -import os -import sys - -from .tree import GitTree, GeckoCommit - -from wptrunner.update import setup_logging, WPTUpdate # noqa: F401 -from wptrunner.update.base import Step, StepRunner, exit_unclean # noqa: F401 -from wptrunner.update.update import LoadConfig, SyncFromUpstream, UpdateMetadata # noqa: F401 -from wptrunner import wptcommandline # noqa: F401 - - -class LoadTrees(Step): - """Load gecko tree and sync tree containing web-platform-tests""" - - provides = ["local_tree", "sync_tree"] - - def create(self, state): - if os.path.exists(state.sync["path"]): - sync_tree = GitTree(root=state.sync["path"]) - else: - sync_tree = None - - assert GitTree.is_type() - state.update({"local_tree": GitTree(commit_cls=GeckoCommit), - "sync_tree": sync_tree}) - - -class UpdateRunner(StepRunner): - """Overall runner for updating web-platform-tests in Gecko.""" - steps = [LoadConfig, - LoadTrees, - SyncFromUpstream, - UpdateMetadata] - - -def run_update(**kwargs): - logger = setup_logging(kwargs, {"mach": sys.stdout}) - updater = WPTUpdate(logger, runner_cls=UpdateRunner, **kwargs) - return updater.run() != exit_unclean - - -def create_parser(): - parser = wptcommandline.create_parser_update() - parser.add_argument("--layout-2020", "--with-layout-2020", default=False, action="store_true", - help="Use expected results for the 2020 layout engine") - parser.add_argument("--layout-2013", "--with-layout-2013", default=True, action="store_true", - help="Use expected results for the 2013 layout engine") - return parser - - -def check_args(kwargs): - wptcommandline.set_from_config(kwargs) - if hasattr(wptcommandline, 'check_paths'): - wptcommandline.check_paths(kwargs) - return kwargs diff --git a/python/wpt/importer/tree.py b/python/wpt/importer/tree.py deleted file mode 100644 index ec38e9b481e..00000000000 --- a/python/wpt/importer/tree.py +++ /dev/null @@ -1,201 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at https://mozilla.org/MPL/2.0/. - -from distutils.spawn import find_executable -import re -import subprocess -import sys -import tempfile - -from wptrunner import update as wptupdate -from wptrunner.update.tree import Commit, CommitMessage, get_unique_name - - -class GitTree(wptupdate.tree.GitTree): - def __init__(self, *args, **kwargs): - """Extension of the basic GitTree with extra methods for - transfering patches""" - commit_cls = kwargs.pop("commit_cls", Commit) - wptupdate.tree.GitTree.__init__(self, *args, **kwargs) - self.commit_cls = commit_cls - - def create_branch(self, name, ref=None): - """Create a named branch, - - :param name: String representing the branch name. - :param ref: None to use current HEAD or rev that the branch should point to""" - - args = [] - if ref is not None: - if hasattr(ref, "sha1"): - ref = ref.sha1 - args.append(ref) - self.git("branch", name, *args) - - def commits_by_message(self, message, path=None): - """List of commits with messages containing a given string. - - :param message: The string that must be contained in the message. - :param path: Path to a file or directory the commit touches - """ - args = ["--pretty=format:%H", "--reverse", "-z", "--grep=%s" % message] - if path is not None: - args.append("--") - args.append(path) - data = self.git("log", *args) - return [self.commit_cls(self, sha1) for sha1 in data.split("\0")] - - def log(self, base_commit=None, path=None): - """List commits touching a certian path from a given base commit. - - :base_param commit: Commit object for the base commit from which to log - :param path: Path that the commits must touch - """ - args = ["--pretty=format:%H", "--reverse", "-z", "--no-merges"] - if base_commit is not None: - args.append("%s.." % base_commit.sha1) - if path is not None: - args.append("--") - args.append(path) - data = self.git("log", *args) - return [self.commit_cls(self, sha1) for sha1 in data.split("\0") if sha1] - - def import_patch(self, patch, strip_count): - """Import a patch file into the tree and commit it - - :param patch: a Patch object containing the patch to import - """ - - with tempfile.NamedTemporaryFile() as f: - f.write(patch.diff) - f.flush() - f.seek(0) - self.git("apply", "--index", f.name, "-p", str(strip_count)) - self.git("commit", "-m", patch.message.text, "--author=%s" % patch.full_author) - - def rebase(self, ref, continue_rebase=False): - """Rebase the current branch onto another commit. - - :param ref: A Commit object for the commit to rebase onto - :param continue_rebase: Continue an in-progress rebase""" - if continue_rebase: - args = ["--continue"] - else: - if hasattr(ref, "sha1"): - ref = ref.sha1 - args = [ref] - self.git("rebase", *args) - - def push(self, remote, local_ref, remote_ref, force=False): - """Push local changes to a remote. - - :param remote: URL of the remote to push to - :param local_ref: Local branch to push - :param remote_ref: Name of the remote branch to push to - :param force: Do a force push - """ - args = [] - if force: - args.append("-f") - args.extend([remote, "%s:%s" % (local_ref, remote_ref)]) - self.git("push", *args) - - def unique_branch_name(self, prefix): - """Get an unused branch name in the local tree - - :param prefix: Prefix to use at the start of the branch name""" - branches = [ref[len("refs/heads/"):] for sha1, ref in self.list_refs() - if ref.startswith("refs/heads/")] - return get_unique_name(branches, prefix) - - -class Patch(object): - def __init__(self, author, email, message, merge_message, diff): - self.author = author - self.email = email - self.merge_message = merge_message - if isinstance(message, CommitMessage): - self.message = message - else: - self.message = GeckoCommitMessage(message) - self.diff = diff - - def __repr__(self): - return "" % self.message.full_summary - - @property - def full_author(self): - return "%s <%s>" % (self.author, self.email) - - @property - def empty(self): - return bool(self.diff.strip()) - - -class GeckoCommitMessage(CommitMessage): - """Commit message following the Gecko conventions for identifying bug number - and reviewer""" - - # c.f. http://hg.mozilla.org/hgcustom/version-control-tools/file/tip/hghooks/mozhghooks/commit-message.py - # which has the regexps that are actually enforced by the VCS hooks. These are - # slightly different because we need to parse out specific parts of the message rather - # than just enforce a general pattern. - - _bug_re = re.compile(r"^Bug (\d+)[^\w]*(?:Part \d+[^\w]*)?(.*?)\s*(?:r=(\w*))?$", - re.IGNORECASE) - _merge_re = re.compile(r"^Auto merge of #(\d+) - [^:]+:[^,]+, r=(.+)$", re.IGNORECASE) - - _backout_re = re.compile(r"^(?:Back(?:ing|ed)\s+out)|Backout|(?:Revert|(?:ed|ing))", - re.IGNORECASE) - _backout_sha1_re = re.compile(r"(?:\s|\:)(0-9a-f){12}") - - def _parse_message(self): - CommitMessage._parse_message(self) - - if self._backout_re.match(self.full_summary): - self.backouts = self._backout_re.findall(self.full_summary) - else: - self.backouts = [] - - m = self._merge_re.match(self.full_summary) - if m is not None: - self.bug, self.reviewer = m.groups() - self.summary = self.full_summary - else: - m = self._bug_re.match(self.full_summary) - if m is not None: - self.bug, self.summary, self.reviewer = m.groups() - else: - self.bug, self.summary, self.reviewer = None, self.full_summary, None - - -class GeckoCommit(Commit): - msg_cls = GeckoCommitMessage - - def __init__(self, tree, sha1, is_merge=False): - Commit.__init__(self, tree, sha1) - if not is_merge: - args = ["-c", sha1] - try: - merge_rev = self.git("when-merged", *args).strip() - except subprocess.CalledProcessError as exn: - if not find_executable('git-when-merged'): - print('Please add the `when-merged` git command to your PATH ' - '(https://github.com/mhagger/git-when-merged/).') - sys.exit(1) - raise exn - self.merge = GeckoCommit(tree, merge_rev, True) - - def export_patch(self, path=None): - """Convert a commit in the tree to a Patch with the bug number and - reviewer stripped from the message""" - args = ["--binary", self.sha1] - if path is not None: - args.append("--") - args.append(path) - - diff = self.git("show", *args) - - merge_message = self.merge.message if self.merge else None - return Patch(self.author, self.email, self.message, merge_message, diff) diff --git a/python/wpt/update.py b/python/wpt/update.py index a31598c09fb..70b52b2acc3 100644 --- a/python/wpt/update.py +++ b/python/wpt/update.py @@ -4,26 +4,130 @@ # pylint: disable=missing-docstring import os +import subprocess +import shutil +import sys + +from wptrunner.update import setup_logging, WPTUpdate # noqa: F401 +from wptrunner.update.base import exit_unclean # noqa: F401 +from wptrunner import wptcommandline # noqa: F401 from . import WPT_PATH, update_args_for_legacy_layout -from . import importer +from . import manifestupdate + +TEST_ROOT = os.path.join(WPT_PATH, 'tests') +META_ROOTS = [ + os.path.join(WPT_PATH, 'meta'), + os.path.join(WPT_PATH, 'meta-legacy') +] -def set_if_none(args: dict, key: str, value): - if key not in args or args[key] is None: - args[key] = value +def do_sync(**kwargs) -> int: + last_commit = subprocess.check_output(["git", "log", "-1"]) + + # Commits should always be authored by the GitHub Actions bot. + os.environ["GIT_AUTHOR_NAME"] = "Servo WPT Sync" + os.environ["GIT_AUTHOR_EMAIL"] = "josh+wptsync@joshmatthews.net" + os.environ["GIT_COMMITTER_NAME"] = os.environ['GIT_AUTHOR_NAME'] + os.environ["GIT_COMMITTER_EMAIL"] = os.environ['GIT_AUTHOR_EMAIL'] + + print("Updating WPT from upstream...") + run_update(**kwargs) + + if last_commit == subprocess.check_output(["git", "log", "-1"]): + return 255 + + # Update the manifest twice to reach a fixed state + # (https://github.com/servo/servo/issues/22275). + print("Updating test manifests...") + manifestupdate.update(check_clean=False) + manifestupdate.update(check_clean=False) + + remove_unused_metadata() + + if subprocess.check_call(["git", "commit", "-a", "--amend", "--no-edit", "-q"]) != 0: + print("Ammending commit failed. Bailing out.") + return 1 + + return 0 -def update_tests(**kwargs): +def remove_unused_metadata(): + print("Removing unused results...") + unused_files = [] + unused_dirs = [] + + for meta_root in META_ROOTS: + for base_dir, dir_names, files in os.walk(meta_root): + # Skip recursing into any directories that were previously found to be missing. + if base_dir in unused_dirs: + # Skip processing any subdirectories of known missing directories. + unused_dirs += [os.path.join(base_dir, x) for x in dir_names] + continue + + for dir_name in dir_names: + dir_path = os.path.join(base_dir, dir_name) + + # Skip any known directories that are meta-metadata. + if dir_name == '.cache': + unused_dirs.append(dir_path) + continue + + # Turn tests/wpt/meta/foo into tests/wpt/tests/foo. + test_dir = os.path.join(TEST_ROOT, os.path.relpath(dir_path, meta_root)) + if not os.path.exists(test_dir): + unused_dirs.append(dir_path) + + for fname in files: + # Skip any known files that are meta-metadata. + if not fname.endswith(".ini") or fname == '__dir__.ini': + continue + + # Turn tests/wpt/meta/foo/bar.html.ini into tests/wpt/tests/foo/bar.html. + test_file = os.path.join( + TEST_ROOT, os.path.relpath(base_dir, meta_root), fname[:-4]) + + if not os.path.exists(test_file): + unused_files.append(os.path.join(base_dir, fname)) + + for file in unused_files: + print(f" - {file}") + os.remove(file) + for directory in unused_dirs: + print(f" - {directory}") + shutil.rmtree(directory) + + +def update_tests(**kwargs) -> int: + def set_if_none(args: dict, key: str, value): + if key not in args or args[key] is None: + args[key] = value + set_if_none(kwargs, "product", "servo") set_if_none(kwargs, "config", os.path.join(WPT_PATH, "config.ini")) kwargs["store_state"] = False - importer.check_args(kwargs) + wptcommandline.set_from_config(kwargs) + if hasattr(wptcommandline, 'check_paths'): + wptcommandline.check_paths(kwargs) update_args_for_legacy_layout(kwargs) - return 1 if not importer.run_update(**kwargs) else 0 + if kwargs.get('sync', False): + return do_sync(**kwargs) + + return 0 if run_update(**kwargs) else 1 -def create_parser(**kwargs): - return importer.create_parser() +def run_update(**kwargs) -> bool: + """Run the update process returning True if the process is successful.""" + logger = setup_logging(kwargs, {"mach": sys.stdout}) + return WPTUpdate(logger, **kwargs).run() != exit_unclean + + +def create_parser(**_kwargs): + parser = wptcommandline.create_parser_update() + parser.add_argument("--layout-2020", "--with-layout-2020", default=False, action="store_true", + help="Use expected results for the 2020 layout engine") + parser.add_argument("--layout-2013", "--with-layout-2013", default=True, action="store_true", + help="Use expected results for the 2013 layout engine") + return parser