servo/decision-task.py

# coding: utf8

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.


import os.path
from decisionlib import DecisionTask


def main():
    decision = DecisionTask(
        project_name="Servo",  # Used in task names
        route_prefix="project.servo.servo",
        worker_type="servo-docker-worker",
    )
    create_task = decision.create_task_with_in_tree_dockerfile

    # FIXME: remove this before merging in servo/servo
    os.environ["GIT_URL"] = "https://github.com/SimonSapin/servo"
    os.environ["GIT_REF"] = "refs/heads/taskcluster-experiments-20180918"
    os.environ["GIT_SHA"] = "605d74c59b6de7ae2b535d42fde40405a96b67e0"
    decision.docker_image_cache_expiry = "1 week"
    decision.route_prefix = "project.servo.servo-taskcluster-experiments"
    # ~


    # https://docs.taskcluster.net/docs/reference/workers/docker-worker/docs/caches
    cache_scopes = [
        "docker-worker:cache:cargo-*",
    ]
    build_caches = {
        "cargo-registry-cache": "/root/.cargo/registry",
        "cargo-git-cache": "/root/.cargo/git",
        "cargo-rustup": "/root/.rustup",
        "cargo-sccache": "/root/.cache/sccache",
    }
    build_artifacts_expiry = "1 week"
    build_env = {
        "RUST_BACKTRACE": "1",
        "RUSTFLAGS": "-Dwarnings",
        "CARGO_INCREMENTAL": "0",
        "SCCACHE_IDLE_TIMEOUT": "1200",
        "CCACHE": "sccache",
        "RUSTC_WRAPPER": "sccache",
    }
    build_kwargs = {
        "max_run_time_minutes": 60,
        "dockerfile": "build-x86_64-linux.dockerfile",
        "env": build_env,
        "scopes": cache_scopes,
        "cache": build_caches,
    }
    run_kwargs = {
        "max_run_time_minutes": 60,
        "dockerfile": "run-x86_64-linux.dockerfile",
    }

    create_task(
        task_name="Linux x86_64: tidy + dev build + unit tests",
        script="""
            ./mach test-tidy --no-progress --all
            ./mach build --dev
            ./mach test-unit
            ./mach package --dev
            ./mach test-tidy --no-progress --self-test
            ./etc/memory_reports_over_time.py --test
            ./etc/ci/lockfile_changed.sh
            ./etc/ci/check_no_panic.sh
        """,
        **build_kwargs
    )

    release_build_task = create_task(
        task_name="Linux x86_64: release build",
        script="""
            ./mach build --release --with-debug-assertions -p servo
            ./etc/ci/lockfile_changed.sh
            gzip target/release/servo
        """,
        artifacts=[
            ("/repo/target/release/servo.gz", build_artifacts_expiry),
        ],
        **build_kwargs
    )

    fetch_build = """
        mkdir -p target/release
        curl \
            "https://queue.taskcluster.net/v1/task/${BUILD_TASK_ID}/artifacts/public/servo.gz" \
            --retry 5 \
            --connect-timeout 10 \
            --location \
            | gunzip > target/release/servo
    """

    total_chunks = 4
    for i in range(total_chunks):
        chunk = i + 1
        create_task(
            task_name="Linux x86_64: WPT chunk %s / %s" % (chunk, total_chunks),
            script=fetch_build + """
                mkdir -p target/release
                curl \
                    "https://queue.taskcluster.net/v1/task/${BUILD_TASK_ID}/artifacts/public/servo.gz" \
                    --retry 5 \
                    --connect-timeout 10 \
                    --location \
                    | gunzip > target/release/servo

                ./mach test-wpt-failure
                ./mach test-wpt \
                    --release \
                    --processes 24 \
                    --total-chunks "$TOTAL_CHUNKS" \
                    --this-chunk "$THIS_CHUNK" \
                    --log-raw test-wpt.log \
                    --log-errorsummary wpt-errorsummary.log \
                    --always-succeed
                ./mach filter-intermittents\
                    wpt-errorsummary.log \
                    --log-intermittents intermittents.log \
                    --log-filteredsummary filtered-wpt-errorsummary.log \
                    --tracker-api default \
                    --reporter-api default
            """,
            env={
                "TOTAL_CHUNKS": total_chunks,
                "THIS_CHUNK": chunk,
                "BUILD_TASK_ID": release_build_task,
            },
            **run_kwargs
        )

    create_task(
        task_name="Linux x86_64: WPT extra",
        script=fetch_build + """
            ./mach test-wpt-failure
            ./mach test-wpt --release --binary-arg=--multiprocess --processes 24 \
                --log-raw test-wpt-mp.log \
                --log-errorsummary wpt-mp-errorsummary.log \
                eventsource
        """,
        env={
            "BUILD_TASK_ID": release_build_task,
        },
        **run_kwargs
    )


if __name__ == "__main__":
    os.chdir(os.path.join(".", os.path.dirname(__file__)))
    main()