servo/python/wpt/manifestupdate.py

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at https://mozilla.org/MPL/2.0/.

from wptrunner.wptcommandline import TestRoot
from typing import Mapping, Any
import argparse
from argparse import ArgumentParser
import os
import sys
import tempfile
from collections import defaultdict
from six import iterkeys, iteritems

from . import SERVO_ROOT, WPT_PATH
from mozlog import commandline

# This must happen after importing from "." since it adds WPT
# tools to the Python system path.
import manifest as wptmanifest

from wptrunner.wptcommandline import get_test_paths, set_from_config
from wptrunner import wptlogging


def create_parser() -> ArgumentParser:
    p = argparse.ArgumentParser()
    p.add_argument(
        "--check-clean", action="store_true", help="Check that updating the manifest doesn't lead to any changes"
    )
    p.add_argument("--rebuild", action="store_true", help="Rebuild the manifest from scratch")
    commandline.add_logging_group(p)

    return p


def update(
    check_clean: bool = True,
    rebuild: bool = False,
    logger: Any = None,
    **kwargs: Any,
) -> int:
    if not logger:
        logger = wptlogging.setup(kwargs, {"mach": sys.stdout})
    kwargs = {
        "config": os.path.join(WPT_PATH, "config.ini"),
        "product": "servo",
        "manifest_path": os.path.join(WPT_PATH, "meta"),
        "tests_root": None,
        "metadata_root": None,
    }

    set_from_config(kwargs)
    config = kwargs["config"]
    test_paths = get_test_paths(config)

    if check_clean:
        return _check_clean(logger, test_paths)

    return _update(logger, test_paths, rebuild)


def _update(logger: Any, test_paths: Mapping[str, TestRoot], rebuild: bool) -> int:
    for url_base, paths in iteritems(test_paths):
        manifest_path = os.path.join(paths.metadata_path, "MANIFEST.json")
        cache_subdir = os.path.relpath(os.path.dirname(manifest_path), os.path.dirname(__file__))
        wptmanifest.manifest.load_and_update(
            paths.tests_path,
            manifest_path,
            url_base,
            working_copy=True,
            rebuild=rebuild,
            cache_root=os.path.join(SERVO_ROOT, ".wpt", cache_subdir),
        )
    return 0


def _check_clean(logger: Any, test_paths: Mapping[str, TestRoot]) -> int:
    manifests_by_path = {}
    rv = 0
    for url_base, paths in iteritems(test_paths):
        tests_path = paths.tests_path
        manifest_path = os.path.join(paths.metadata_path, "MANIFEST.json")

        old_manifest = wptmanifest.manifest.load_and_update(
            tests_path, manifest_path, url_base, working_copy=False, update=False, write_manifest=False
        )

        # Even if no cache is specified, one will be used automatically by the
        # VCS integration. Create a brand new cache every time to ensure that
        # the VCS integration always thinks that any file modifications in the
        # working directory are new and interesting.
        cache_root = tempfile.mkdtemp()
        new_manifest = wptmanifest.manifest.load_and_update(
            tests_path,
            manifest_path,
            url_base,
            working_copy=True,
            update=True,
            cache_root=cache_root,
            write_manifest=False,
            allow_cached=False,
        )

        manifests_by_path[manifest_path] = (old_manifest, new_manifest)

    for manifest_path, (old_manifest, new_manifest) in iteritems(manifests_by_path):
        if not diff_manifests(logger, manifest_path, old_manifest, new_manifest):
            logger.error("Manifest %s is outdated, use |./mach update-manifest| to fix." % manifest_path)
            rv = 1

    return rv


def diff_manifests(logger: Any, manifest_path: Any, old_manifest: Any, new_manifest: Any) -> bool:
    """Lint the differences between old and new versions of a
    manifest. Differences are considered significant (and so produce
    lint errors) if they produce a meaningful difference in the actual
    tests run.

    :param logger: mozlog logger to use for output
    :param manifest_path: Path to the manifest being linted
    :param old_manifest: Manifest object representing the initial manifest
    :param new_manifest: Manifest object representing the updated manifest
    """
    logger.info("Diffing old and new manifests %s" % manifest_path)
    old_items, new_items = defaultdict(set), defaultdict(set)
    for manifest, items in [(old_manifest, old_items), (new_manifest, new_items)]:
        for test_type, path, tests in manifest:
            for test in tests:
                test_id = [test.id]
                if hasattr(test, "script_metadata"):
                    if test.script_metadata is not None:
                        test_id.extend(tuple(item) for item in test.script_metadata)
                if hasattr(test, "references"):
                    test_id.extend(tuple(item) for item in test.references)
                test_id = tuple(test_id)
                items[path].add((test_type, test_id))

    old_paths = set(iterkeys(old_items))
    new_paths = set(iterkeys(new_items))

    added_paths = new_paths - old_paths
    deleted_paths = old_paths - new_paths

    common_paths = new_paths & old_paths

    clean = True

    for path in added_paths:
        clean = False
        log_error(logger, manifest_path, "%s in source but not in manifest." % path)
    for path in deleted_paths:
        clean = False
        log_error(logger, manifest_path, "%s in manifest but removed from source." % path)

    for path in common_paths:
        old_tests = old_items[path]
        new_tests = new_items[path]
        added_tests = new_tests - old_tests
        removed_tests = old_tests - new_tests
        if added_tests or removed_tests:
            clean = False
            log_error(logger, manifest_path, "%s changed test types or metadata" % path)

    if clean:
        # Manifest currently has some list vs tuple inconsistencies that break
        # a simple equality comparison.
        old_paths = old_manifest.to_json()["items"]
        new_paths = new_manifest.to_json()["items"]
        if old_paths != new_paths:
            logger.warning("Manifest %s contains correct tests but file hashes changed." % manifest_path)
            clean = False

    return clean


def log_error(logger: Any, manifest_path: Any, msg: str) -> None:
    logger.lint_error(path=manifest_path, message=msg, lineno=0, source="", linter="wpt-manifest")