mirror of
https://github.com/servo/servo.git
synced 2025-07-01 04:23:39 +01:00
133 lines
3.7 KiB
Python
133 lines
3.7 KiB
Python
from __future__ import absolute_import
|
|
|
|
import argparse
|
|
import gzip
|
|
import json
|
|
import io
|
|
import os
|
|
from datetime import datetime, timedelta
|
|
|
|
from six.moves.urllib.request import urlopen
|
|
|
|
from .vcs import Git
|
|
|
|
from . import log
|
|
|
|
here = os.path.dirname(__file__)
|
|
|
|
wpt_root = os.path.abspath(os.path.join(here, os.pardir, os.pardir))
|
|
logger = log.get_logger()
|
|
|
|
|
|
def abs_path(path):
|
|
return os.path.abspath(os.path.expanduser(path))
|
|
|
|
|
|
def should_download(manifest_path, rebuild_time=timedelta(days=5)):
|
|
if not os.path.exists(manifest_path):
|
|
return True
|
|
mtime = datetime.fromtimestamp(os.path.getmtime(manifest_path))
|
|
if mtime < datetime.now() - rebuild_time:
|
|
return True
|
|
logger.info("Skipping manifest download because existing file is recent")
|
|
return False
|
|
|
|
|
|
def git_commits(repo_root):
|
|
git = Git.get_func(repo_root)
|
|
return [item for item in git("log", "--format=%H", "-n50").split("\n") if item]
|
|
|
|
|
|
def github_url(commits):
|
|
try:
|
|
resp = urlopen("https://api.github.com/repos/web-platform-tests/wpt/releases")
|
|
except Exception:
|
|
return None
|
|
|
|
if resp.code != 200:
|
|
return None
|
|
|
|
try:
|
|
releases = json.load(resp.fp)
|
|
except ValueError:
|
|
logger.warning("Response was not valid JSON")
|
|
return None
|
|
|
|
fallback = None
|
|
for release in releases:
|
|
for commit in commits:
|
|
for item in release["assets"]:
|
|
if item["name"] == "MANIFEST-%s.json.gz" % commit:
|
|
return item["browser_download_url"]
|
|
elif item["name"] == "MANIFEST.json.gz" and not fallback:
|
|
fallback = item["browser_download_url"]
|
|
if fallback:
|
|
logger.info("Can't find a commit-specific manifest so just using the most recent one")
|
|
return fallback
|
|
|
|
|
|
def download_manifest(manifest_path, commits_func, url_func, force=False):
|
|
if not force and not should_download(manifest_path):
|
|
return False
|
|
|
|
commits = commits_func()
|
|
|
|
url = url_func(commits)
|
|
if not url:
|
|
logger.warning("No generated manifest found")
|
|
return False
|
|
|
|
logger.info("Downloading manifest from %s" % url)
|
|
try:
|
|
resp = urlopen(url)
|
|
except Exception:
|
|
logger.warning("Downloading pregenerated manifest failed")
|
|
return False
|
|
|
|
if resp.code != 200:
|
|
logger.warning("Downloading pregenerated manifest failed; got HTTP status %d" %
|
|
resp.code)
|
|
return False
|
|
|
|
gzf = gzip.GzipFile(fileobj=io.BytesIO(resp.read()))
|
|
|
|
try:
|
|
decompressed = gzf.read()
|
|
except IOError:
|
|
logger.warning("Failed to decompress downloaded file")
|
|
return False
|
|
|
|
try:
|
|
with open(manifest_path, "w") as f:
|
|
f.write(decompressed)
|
|
except Exception:
|
|
logger.warning("Failed to write manifest")
|
|
return False
|
|
logger.info("Manifest downloaded")
|
|
return True
|
|
|
|
|
|
def create_parser():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument(
|
|
"-p", "--path", type=abs_path, help="Path to manifest file.")
|
|
parser.add_argument(
|
|
"--tests-root", type=abs_path, default=wpt_root, help="Path to root of tests.")
|
|
parser.add_argument(
|
|
"--force", action="store_true",
|
|
help="Always download, even if the existing manifest is recent")
|
|
return parser
|
|
|
|
|
|
def download_from_github(path, tests_root, force=False):
|
|
return download_manifest(path, lambda: git_commits(tests_root), github_url,
|
|
force=force)
|
|
|
|
|
|
def run(**kwargs):
|
|
if kwargs["path"] is None:
|
|
path = os.path.join(kwargs["tests_root"], "MANIFEST.json")
|
|
else:
|
|
path = kwargs["path"]
|
|
success = download_from_github(path, kwargs["tests_root"], kwargs["force"])
|
|
return 0 if success else 1
|