From 659597281b77142246c172e12c07c70f3da9386e Mon Sep 17 00:00:00 2001 From: Martin Robinson Date: Mon, 9 Jan 2023 13:17:55 +0100 Subject: [PATCH] Add a script to upstream WPT changes via a GitHub Action This is a modified version of the webhook found at servo/upstream_wpt_webhook and deployed via SaltStack. It is updated to use modern Python and to assume that GitHub Actions will fetch the appropriate source code locally before the script is run. Fixes #29206. Fixes #23798. Signed-off-by: Martin Robinson --- .../workflows/test-upstream-wpt-changes.yml | 18 + .github/workflows/upstream-wpt-changes.yml | 34 + etc/ci/upstream-wpt-changes/README | 0 .../upstream-wpt-changes/requirements-dev.txt | 4 + etc/ci/upstream-wpt-changes/requirements.txt | 1 + etc/ci/upstream-wpt-changes/test.py | 621 ++++++++++++++++++ etc/ci/upstream-wpt-changes/tests/18746.diff | 10 + etc/ci/upstream-wpt-changes/tests/closed.json | 476 ++++++++++++++ .../tests/does-not-apply-cleanly.diff | 9 + etc/ci/upstream-wpt-changes/tests/edited.json | 514 +++++++++++++++ etc/ci/upstream-wpt-changes/tests/merged.json | 515 +++++++++++++++ .../tests/move-into-wpt.diff | 4 + .../tests/move-out-of-wpt.diff | 4 + .../upstream-wpt-changes/tests/non-wpt.diff | 10 + .../tests/opened-with-no-sync-signal.json | 440 +++++++++++++ etc/ci/upstream-wpt-changes/tests/opened.json | 440 +++++++++++++ .../tests/servo-mock/README.md | 1 + .../mozilla/tests/mozilla/mozilla-test.html | 3 + .../tests/servo-mock/tests/wpt/something.py | 10 + .../wpt/web-platform-tests/css/css-test.html | 3 + .../css/out-of-sync-test.html | 3 + .../fetch/api/redirect/redirect-location.html | 16 + .../tests/wpt/web-platform-tests/test.html | 3 + .../tests/synchronize-multiple.json | 499 ++++++++++++++ .../tests/synchronize.json | 499 ++++++++++++++ .../tests/wpt-mock/css/css-test.html | 3 + .../fetch/api/redirect/redirect-location.html | 16 + .../tests/wpt-mock/test.html | 3 + etc/ci/upstream-wpt-changes/tests/wpt.diff | 9 + .../upstream-wpt-changes.py | 43 ++ .../wptupstreamer/__init__.py | 268 ++++++++ .../wptupstreamer/common.py | 53 ++ .../wptupstreamer/github.py | 166 +++++ .../wptupstreamer/step.py | 300 +++++++++ 34 files changed, 4998 insertions(+) create mode 100644 .github/workflows/test-upstream-wpt-changes.yml create mode 100644 .github/workflows/upstream-wpt-changes.yml create mode 100644 etc/ci/upstream-wpt-changes/README create mode 100644 etc/ci/upstream-wpt-changes/requirements-dev.txt create mode 100644 etc/ci/upstream-wpt-changes/requirements.txt create mode 100644 etc/ci/upstream-wpt-changes/test.py create mode 100644 etc/ci/upstream-wpt-changes/tests/18746.diff create mode 100644 etc/ci/upstream-wpt-changes/tests/closed.json create mode 100644 etc/ci/upstream-wpt-changes/tests/does-not-apply-cleanly.diff create mode 100644 etc/ci/upstream-wpt-changes/tests/edited.json create mode 100644 etc/ci/upstream-wpt-changes/tests/merged.json create mode 100644 etc/ci/upstream-wpt-changes/tests/move-into-wpt.diff create mode 100644 etc/ci/upstream-wpt-changes/tests/move-out-of-wpt.diff create mode 100644 etc/ci/upstream-wpt-changes/tests/non-wpt.diff create mode 100644 etc/ci/upstream-wpt-changes/tests/opened-with-no-sync-signal.json create mode 100644 etc/ci/upstream-wpt-changes/tests/opened.json create mode 100644 etc/ci/upstream-wpt-changes/tests/servo-mock/README.md create mode 100644 etc/ci/upstream-wpt-changes/tests/servo-mock/tests/wpt/mozilla/tests/mozilla/mozilla-test.html create mode 100644 etc/ci/upstream-wpt-changes/tests/servo-mock/tests/wpt/something.py create mode 100644 etc/ci/upstream-wpt-changes/tests/servo-mock/tests/wpt/web-platform-tests/css/css-test.html create mode 100644 etc/ci/upstream-wpt-changes/tests/servo-mock/tests/wpt/web-platform-tests/css/out-of-sync-test.html create mode 100644 etc/ci/upstream-wpt-changes/tests/servo-mock/tests/wpt/web-platform-tests/fetch/api/redirect/redirect-location.html create mode 100644 etc/ci/upstream-wpt-changes/tests/servo-mock/tests/wpt/web-platform-tests/test.html create mode 100644 etc/ci/upstream-wpt-changes/tests/synchronize-multiple.json create mode 100644 etc/ci/upstream-wpt-changes/tests/synchronize.json create mode 100644 etc/ci/upstream-wpt-changes/tests/wpt-mock/css/css-test.html create mode 100644 etc/ci/upstream-wpt-changes/tests/wpt-mock/fetch/api/redirect/redirect-location.html create mode 100644 etc/ci/upstream-wpt-changes/tests/wpt-mock/test.html create mode 100644 etc/ci/upstream-wpt-changes/tests/wpt.diff create mode 100755 etc/ci/upstream-wpt-changes/upstream-wpt-changes.py create mode 100644 etc/ci/upstream-wpt-changes/wptupstreamer/__init__.py create mode 100644 etc/ci/upstream-wpt-changes/wptupstreamer/common.py create mode 100644 etc/ci/upstream-wpt-changes/wptupstreamer/github.py create mode 100644 etc/ci/upstream-wpt-changes/wptupstreamer/step.py diff --git a/.github/workflows/test-upstream-wpt-changes.yml b/.github/workflows/test-upstream-wpt-changes.yml new file mode 100644 index 00000000000..4eb006829b9 --- /dev/null +++ b/.github/workflows/test-upstream-wpt-changes.yml @@ -0,0 +1,18 @@ +name: Test changes to upstream-wpt-changes +on: + pull_request: + branches: ["**"] + paths: ["etc/ci/upstream-wpt-changes/**"] + +jobs: + test: + name: Test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Install dependencies + run: | + python3 -m pip install --upgrade -r etc/ci/upstream-wpt-changes/requirements-dev.txt + - name: Running tests + run: | + python3 etc/ci/upstream-wpt-changes/test.py diff --git a/.github/workflows/upstream-wpt-changes.yml b/.github/workflows/upstream-wpt-changes.yml new file mode 100644 index 00000000000..c03fd8f0b35 --- /dev/null +++ b/.github/workflows/upstream-wpt-changes.yml @@ -0,0 +1,34 @@ +# Disabled until the previous bot is turned off. +#name: Upstream Web Platform Test changes +#on: +# pull_request: +# types: ['opened', 'synchronize', 'reopened', 'edited', 'closed'] +# +#jobs: +# upstream: +# runs-on: ubuntu-latest +# steps: +# - name: Calculate PR fetch depth +# run: echo "PR_FETCH_DEPTH=$(( ${{ github.event.pull_request.commits }} + 1 ))" >> "${GITHUB_ENV}" +# - name: Check out shallow servo PR +# run: | +# mkdir servo +# cd servo +# git init -b main +# git remote add origin ${{ github.event.repository.clone_url}} +# git fetch origin pull/${{ github.event.pull_request.number}}/head:pr --depth ${{ env.PR_FETCH_DEPTH }} +# git checkout pr +# - name: Check out wpt +# uses: actions/checkout@v3 +# with: +# path: wpt +# repository: 'web-platform-tests/wpt' +# token: ${{ secrets.WPT_UPSTREAM_TOKEN }} +# - name: Install requirements +# run: pip install -r servo/etc/ci/upstream-wpt-changes/requirements.txt +# - name: Process pull request +# run: servo/etc/ci/upstream-wpt-changes/upstream-wpt-changes.py +# env: +# GITHUB_CONTEXT: ${{ toJson(github) }} +# GITHUB_TOKEN: ${{ secrets.WPT_UPSTREAM_TOKEN }} +# diff --git a/etc/ci/upstream-wpt-changes/README b/etc/ci/upstream-wpt-changes/README new file mode 100644 index 00000000000..e69de29bb2d diff --git a/etc/ci/upstream-wpt-changes/requirements-dev.txt b/etc/ci/upstream-wpt-changes/requirements-dev.txt new file mode 100644 index 00000000000..d60a3d3dad9 --- /dev/null +++ b/etc/ci/upstream-wpt-changes/requirements-dev.txt @@ -0,0 +1,4 @@ +-r requirements.txt + +flask +types-requests \ No newline at end of file diff --git a/etc/ci/upstream-wpt-changes/requirements.txt b/etc/ci/upstream-wpt-changes/requirements.txt new file mode 100644 index 00000000000..663bd1f6a2a --- /dev/null +++ b/etc/ci/upstream-wpt-changes/requirements.txt @@ -0,0 +1 @@ +requests \ No newline at end of file diff --git a/etc/ci/upstream-wpt-changes/test.py b/etc/ci/upstream-wpt-changes/test.py new file mode 100644 index 00000000000..67d83603805 --- /dev/null +++ b/etc/ci/upstream-wpt-changes/test.py @@ -0,0 +1,621 @@ +# Copyright 2023 The Servo Project Developers. See the COPYRIGHT +# file at the top-level directory of this distribution. +# +# Licensed under the Apache License, Version 2.0 or the MIT license +# , at your +# option. This file may not be copied, modified, or distributed +# except according to those terms. + +# pylint: disable=broad-except +# pylint: disable=dangerous-default-value +# pylint: disable=global-statement +# pylint: disable=line-too-long +# pylint: disable=missing-docstring +# pylint: disable=protected-access + +# This allows using types that are defined later in the file. +from __future__ import annotations + +import dataclasses +import json +import locale +import logging +import os +import shutil +import subprocess +import sys +import tempfile +import threading +import time +import unittest + +from functools import partial +from typing import Any, Optional, Tuple +from wsgiref.simple_server import WSGIRequestHandler, make_server + +import flask +import flask.cli +import requests +from wptupstreamer import SyncRun, WPTSync +from wptupstreamer.step import CreateOrUpdateBranchForPRStep + +TESTS_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "tests") +SYNC: Optional[WPTSync] = None +TMP_DIR: Optional[str] = None +PORT = 9000 + +if "-v" in sys.argv or "--verbose" in sys.argv: + logging.getLogger().level = logging.DEBUG + + +@dataclasses.dataclass +class MockPullRequest(): + head: str + number: int + state: str = "open" + + +class MockGitHubAPIServer(): + def __init__(self, port: int): + self.port = port + self.disable_logging() + self.app = flask.Flask(__name__) + self.pulls: list[MockPullRequest] = [] + + class NoLoggingHandler(WSGIRequestHandler): + def log_message(self, *args): + pass + if logging.getLogger().level == logging.DEBUG: + handler = WSGIRequestHandler + else: + handler = NoLoggingHandler + + self.server = make_server('localhost', self.port, self.app, handler_class=handler) + self.start_server_thread() + + def disable_logging(self): + flask.cli.show_server_banner = lambda *args: None + logging.getLogger("werkzeug").disabled = True + logging.getLogger('werkzeug').setLevel(logging.CRITICAL) + + def start(self): + self.thread.start() + + # Wait for the server to be started. + while True: + try: + response = requests.get(f'http://localhost:{self.port}/ping', timeout=1) + assert response.status_code == 200 + assert response.text == 'pong' + break + except Exception: + time.sleep(0.1) + + def reset_server_state_with_pull_requests(self, pulls: list[MockPullRequest]): + response = requests.get( + f'http://localhost:{self.port}/reset-mock-github', + json=[dataclasses.asdict(pull_request) for pull_request in pulls], + timeout=1 + ) + assert response.status_code == 200 + assert response.text == 'πŸ‘' + + def shutdown(self): + self.server.shutdown() + self.thread.join() + + def start_server_thread(self): + # pylint: disable=unused-argument + self.thread = threading.Thread(target=self.server.serve_forever, daemon=True) + self.thread.start() + + @self.app.route("/ping") + def ping(): + return ('pong', 200) + + @self.app.route("/reset-mock-github") + def reset_server(): + self.pulls = [ + MockPullRequest(pull_request['head'], + pull_request['number'], + pull_request['state']) + for pull_request in flask.request.json] + return ('πŸ‘', 200) + + @self.app.route("/repos///pulls//merge", methods=['PUT']) + def merge_pull_request(org, repo, number): + for pull_request in self.pulls: + if pull_request.number == number: + pull_request.state = 'closed' + return ('', 204) + return ('', 404) + + @self.app.route("/repos///pulls", methods=['GET']) + def get_pulls(org, repo): + for pull_request in self.pulls: + if pull_request.head == flask.request.args["head"]: + return json.dumps([{"number": pull_request.number}]) + return json.dumps([]) + + @self.app.route("/repos///pulls", methods=['POST']) + def create_pull_request(org, repo): + new_pr_number = len(self.pulls) + 1 + self.pulls.append(MockPullRequest( + flask.request.json["head"], + new_pr_number, + "open" + )) + return {"number": new_pr_number} + + @self.app.route("/repos///pulls/", methods=['PATCH']) + def update_pull_request(org, repo, number): + for pull_request in self.pulls: + if pull_request.number == number: + if 'state' in flask.request.json: + pull_request.state = flask.request.json['state'] + return ('', 204) + return ('', 404) + + @self.app.route("/repos///issues//labels", methods=['GET', 'POST']) + @self.app.route("/repos///issues//labels/