Cargoify servo

python/servo/bootstrap_commands.py

@@ -0,0 +1,153 @@
+from __future__ import print_function, unicode_literals
+
+import os
+import os.path as path
+import shutil
+import subprocess
+import sys
+import tarfile
+import urllib
+
+from mach.decorators import (
+    CommandArgument,
+    CommandProvider,
+    Command,
+)
+
+from servo.command_base import CommandBase, cd
+
+def host_triple():
+    os_type = subprocess.check_output(["uname", "-s"]).strip().lower()
+    if os_type == "linux":
+        os_type = "unknown-linux-gnu"
+    elif os_type == "darwin":
+        os_type = "apple-darwin"
+    elif os_type == "android":
+        os_type == "linux-androideabi"
+    else:
+        os_type == "unknown"
+
+    cpu_type = subprocess.check_output(["uname", "-m"]).strip().lower()
+    if cpu_type in ["i386", "i486", "i686", "i768", "x86"]:
+        cpu_type = "i686"
+    elif cpu_type in ["x86_64", "x86-64", "x64", "amd64"]:
+        cpu_type = "x86_64"
+    elif cpu_type == "arm":
+        cpu_type = "arm"
+    else:
+        cpu_type = "unknown"
+
+    return "%s-%s" % (cpu_type, os_type)
+
+def download(desc, src, dst):
+    recved = [0]
+    def report(count, bsize, fsize):
+        recved[0] += bsize
+        pct = recved[0] * 100.0 / fsize
+        print("\rDownloading %s: %5.1f%%" % (desc, pct), end="")
+        sys.stdout.flush()
+
+    urllib.urlretrieve(src, dst, report)
+    print()
+
+def extract(src, dst, movedir=None):
+    tarfile.open(src).extractall(dst)
+
+    if movedir:
+        for f in os.listdir(movedir):
+            frm = path.join(movedir, f)
+            to = path.join(dst, f)
+            os.rename(frm, to)
+        os.rmdir(movedir)
+
+    os.remove(src)
+
+@CommandProvider
+class MachCommands(CommandBase):
+    @Command('env',
+             description='Print environment setup commands',
+             category='bootstrap')
+    def env(self):
+        env = self.build_env()
+        print("export PATH=%s" % env["PATH"])
+        if sys.platform == "darwin":
+            print("export DYLD_LIBRARY_PATH=%s" % env["DYLD_LIBRARY_PATH"])
+        else:
+            print("export LD_LIBRARY_PATH=%s" % env["LD_LIBRARY_PATH"])
+
+    @Command('bootstrap-rust',
+             description='Download the Rust compiler snapshot',
+             category='bootstrap')
+    @CommandArgument('--force', '-f',
+                     action='store_true',
+                     help='Force download even if a snapshot already exists')
+    def bootstrap_rustc(self, force=False):
+        rust_dir = path.join(self.context.topdir, "rust")
+        if not force and path.exists(path.join(rust_dir, "bin", "rustc")):
+            print("Snapshot Rust compiler already downloaded.", end=" ")
+            print("Use |bootstrap_rust --force| to download again.")
+            return
+
+        if path.isdir(rust_dir):
+            shutil.rmtree(rust_dir)
+        os.mkdir(rust_dir)
+
+        snapshot_hash = open(path.join(self.context.topdir, "rust-snapshot-hash")).read().strip()
+        snapshot_path = "%s-%s.tar.gz" % (snapshot_hash, host_triple())
+        snapshot_url = "https://servo-rust.s3.amazonaws.com/%s" % snapshot_path
+        tgz_file = path.join(rust_dir, path.basename(snapshot_path))
+
+        download("Rust snapshot", snapshot_url, tgz_file)
+
+        print("Extracting Rust snapshot...")
+        snap_dir = path.join(rust_dir,
+                             path.basename(tgz_file).replace(".tar.gz", ""))
+        extract(tgz_file, rust_dir, movedir=snap_dir)
+        print("Snapshot Rust ready.")
+
+    @Command('bootstrap-cargo',
+             description='Download the Cargo build tool',
+             category='bootstrap')
+    @CommandArgument('--force', '-f',
+                     action='store_true',
+                     help='Force download even if cargo already exists')
+    def bootstrap_cargo(self, force=False):
+        cargo_dir = path.join(self.context.topdir, "cargo")
+        if not force and path.exists(path.join(cargo_dir, "bin", "cargo")):
+            print("Cargo already downloaded.", end=" ")
+            print("Use |bootstrap_cargo --force| to download again.")
+            return
+
+        if path.isdir(cargo_dir):
+            shutil.rmtree(cargo_dir)
+        os.mkdir(cargo_dir)
+
+        tgz_file = "cargo-nightly-%s.tar.gz" % host_triple()
+        nightly_url = "http://static.rust-lang.org/cargo-dist/%s" % tgz_file
+
+        download("Cargo nightly", nightly_url, tgz_file)
+
+        print("Extracting Cargo nightly...")
+        nightly_dir = path.join(cargo_dir,
+                                path.basename(tgz_file).replace(".tar.gz", ""))
+        extract(tgz_file, cargo_dir, movedir=nightly_dir)
+        print("Cargo ready.")
+
+    @Command('update-submodules',
+             description='Update submodules',
+             category='bootstrap')
+    def update_submodules(self):
+        submodules = subprocess.check_output(["git", "submodule", "status"])
+        for line in submodules.split('\n'):
+            components = line.strip().split(' ')
+            if len(components) > 1:
+                module_path = components[1]
+                if path.exists(module_path):
+                    with cd(module_path):
+                        output = subprocess.check_output(["git", "status", "--porcelain"])
+                        if len(output) != 0:
+                            print("error: submodule %s is not clean" % module_path)
+                            print("\nClean the submodule and try again.")
+                            return 1
+        subprocess.check_call(["git", "submodule", "--quiet", "sync", "--recursive"])
+        subprocess.check_call(["git", "submodule", "update", "--init", "--recursive"])

python/servo/build_commands.py

@@ -0,0 +1,85 @@
+from __future__ import print_function, unicode_literals
+
+import json
+import os
+import os.path as path
+import shutil
+import subprocess
+import sys
+import tarfile
+from time import time
+import urllib
+
+from mach.registrar import Registrar
+from mach.decorators import (
+    CommandArgument,
+    CommandProvider,
+    Command,
+)
+
+from servo.command_base import CommandBase, cd
+
+@CommandProvider
+class MachCommands(CommandBase):
+    @Command('build',
+             description='Build Servo',
+             category='build')
+    @CommandArgument('--target', '-t',
+                     default=None,
+                     help='Cross compile for given target platform')
+    @CommandArgument('--release', '-r',
+                     action='store_true',
+                     help='Build in release mode')
+    @CommandArgument('--jobs', '-j',
+                     default=None,
+                     help='Number of jobs to run in parallel')
+    def build(self, target, release=False, jobs=None):
+        self.ensure_bootstrapped()
+
+        opts = []
+        if release:
+            opts += ["--release"]
+        if jobs is not None:
+            opts += ["-j", jobs]
+
+        build_start = time()
+        subprocess.check_call(["cargo", "build"] + opts, env=self.build_env())
+        elapsed = time() - build_start
+
+        print("Build completed in %0.2fs" % elapsed)
+
+    @Command('build-cef',
+             description='Build the Chromium Embedding Framework library',
+             category='build')
+    @CommandArgument('--jobs', '-j',
+                     default=None,
+                     help='Number of jobs to run in parallel')
+    def build_cef(self, jobs=None):
+        self.ensure_bootstrapped()
+
+        ret = None
+        opts = []
+        if jobs is not None:
+            opts += ["-j", jobs]
+
+        build_start = time()
+        with cd(path.join("ports", "cef")):
+            ret = subprocess.call(["cargo", "build"], env=self.build_env())
+        elapsed = time() - build_start
+
+        print("CEF build completed in %0.2fs" % elapsed)
+
+        return ret
+
+    @Command('build-tests',
+             description='Build the Servo test suites',
+             category='build')
+    @CommandArgument('--jobs', '-j',
+                     default=None,
+                     help='Number of jobs to run in parallel')
+    def build_tests(self, jobs=None):
+        self.ensure_bootstrapped()
+        opts = []
+        if jobs is not None:
+            opts += ["-j", jobs]
+        subprocess.check_call(["cargo", "test", "--no-run"], env=self.build_env())

python/servo/command_base.py

@@ -0,0 +1,94 @@
+import os
+from os import path
+import subprocess
+import sys
+import toml
+
+from mach.registrar import Registrar
+
+class cd:
+    """Context manager for changing the current working directory"""
+    def __init__(self, newPath):
+        self.newPath = newPath
+
+    def __enter__(self):
+        self.savedPath = os.getcwd()
+        os.chdir(self.newPath)
+
+    def __exit__(self, etype, value, traceback):
+        os.chdir(self.savedPath)
+
+class CommandBase(object):
+    """Base class for mach command providers.
+
+    This mostly handles configuration management, such as .servobuild."""
+
+    def __init__(self, context):
+        self.context = context
+
+        if not hasattr(self.context, "bootstrapped"):
+            self.context.bootstrapped = False
+
+        config_path = path.join(context.topdir, ".servobuild")
+        if path.exists(config_path):
+            self.config = toml.loads(open(config_path).read())
+        else:
+            self.config = {}
+
+        # Handle missing/default items
+        self.config.setdefault("tools", {})
+        self.config["tools"].setdefault("system-rust", False)
+        self.config["tools"].setdefault("system-cargo", False)
+        self.config["tools"].setdefault("rust-root", "")
+        self.config["tools"].setdefault("cargo-root", "")
+        if not self.config["tools"]["system-rust"]:
+            self.config["tools"]["rust-root"] = path.join(context.topdir, "rust")
+        if not self.config["tools"]["system-cargo"]:
+            self.config["tools"]["cargo-root"] = path.join(context.topdir, "cargo")
+
+    def build_env(self):
+        """Return an extended environment dictionary."""
+        env = os.environ.copy()
+        extra_path = []
+        extra_lib = []
+        if not self.config["tools"]["system-rust"] or self.config["tools"]["rust-root"]:
+            extra_path += [path.join(self.config["tools"]["rust-root"], "bin")]
+            extra_lib += [path.join(self.config["tools"]["rust-root"], "lib")]
+        if not self.config["tools"]["system-cargo"] or self.config["tools"]["cargo-root"]:
+            extra_path += [path.join(self.config["tools"]["cargo-root"], "bin")]
+
+        if extra_path:
+            env["PATH"] = "%s%s%s" % (os.pathsep.join(extra_path), os.pathsep, env["PATH"])
+        if extra_lib:
+            if sys.platform == "darwin":
+                env["DYLD_LIBRARY_PATH"] = "%s%s%s" % \
+                                           (os.pathsep.join(extra_lib),
+                                            os.pathsep,
+                                            env.get("DYLD_LIBRARY_PATH", ""))
+            else:
+                env["LD_LIBRARY_PATH"] = "%s%s%s" % \
+                                         (os.pathsep.join(extra_lib),
+                                          os.pathsep,
+                                          env.get("LD_LIBRARY_PATH", ""))
+
+        return env
+
+    def ensure_bootstrapped(self):
+        if self.context.bootstrapped: return
+
+        submodules = subprocess.check_output(["git", "submodule", "status"])
+        for line in submodules.split('\n'):
+            components = line.strip().split(' ')
+            if len(components) > 1 and components[0].startswith('-'):
+                module_path = components[1]
+                subprocess.check_call(["git", "submodule", "update",
+                                       "--init", "--recursive", "--", module_path])
+
+        if not self.config["tools"]["system-rust"] and \
+           not path.exists(path.join(self.context.topdir, "rust", "bin", "rustc")):
+            Registrar.dispatch("bootstrap-rust", context=self.context)
+        if not self.config["tools"]["system-cargo"] and \
+           not path.exists(path.join(self.context.topdir, "cargo", "bin", "cargo")):
+            Registrar.dispatch("bootstrap-cargo", context=self.context)
+
+        self.context.bootstrapped = True

python/servo/devenv_commands.py

@@ -0,0 +1,32 @@
+from __future__ import print_function, unicode_literals
+
+import json
+import os
+import os.path as path
+import shutil
+import subprocess
+import sys
+import tarfile
+from time import time
+import urllib
+
+from mach.registrar import Registrar
+from mach.decorators import (
+    CommandArgument,
+    CommandProvider,
+    Command,
+)
+
+from servo.command_base import CommandBase
+
+@CommandProvider
+class MachCommands(CommandBase):
+    @Command('cargo',
+             description='Run Cargo',
+             category='devenv',
+             allow_all_args=True)
+    @CommandArgument('params', default=None, nargs='...',
+                     help="Command-line arguments to be passed through to Cervo")
+    def run(self, params):
+        return subprocess.call(["cargo"] + params,
+                               env=self.build_env())

python/servo/post_build_commands.py

@@ -0,0 +1,44 @@
+from __future__ import print_function, unicode_literals
+
+import json
+import os
+import os.path as path
+import shutil
+import subprocess
+import sys
+import tarfile
+from time import time
+import urllib
+
+from mach.registrar import Registrar
+from mach.decorators import (
+    CommandArgument,
+    CommandProvider,
+    Command,
+)
+
+from servo.command_base import CommandBase
+
+@CommandProvider
+class MachCommands(CommandBase):
+    @Command('run',
+             description='Run Servo',
+             category='post-build',
+             allow_all_args=True)
+    @CommandArgument('params', default=None, nargs='...',
+                     help="Command-line arguments to be passed through to Servo")
+    def run(self, params):
+        subprocess.check_call([path.join("target", "servo")] + params,
+                              env=self.build_env())
+
+    @Command('doc',
+             description='Generate documentation',
+             category='post-build',
+             allow_all_args=True)
+    @CommandArgument('params', default=None, nargs='...',
+                     help="Command-line arguments to be passed through to cargo doc")
+    def doc(self, params):
+        self.ensure_bootstrapped()
+        return subprocess.call(["cargo", "doc"] + params,
+                               env=self.build_env())
+

python/servo/testing_commands.py

@@ -0,0 +1,122 @@
+from __future__ import print_function, unicode_literals
+
+import json
+import os
+import os.path as path
+import shutil
+import subprocess
+import sys
+import tarfile
+from time import time
+import urllib
+
+from mach.registrar import Registrar
+from mach.decorators import (
+    CommandArgument,
+    CommandProvider,
+    Command,
+)
+
+from servo.command_base import CommandBase
+import tidy
+
+@CommandProvider
+class MachCommands(CommandBase):
+    def __init__(self, context):
+        CommandBase.__init__(self, context)
+        if not hasattr(self.context, "built_tests"):
+            self.context.built_tests = False
+
+    def ensure_built_tests(self):
+        if self.context.built_tests: return
+        Registrar.dispatch('build-tests', context=self.context)
+        self.context.built_tests = True
+
+    def find_test(self, prefix):
+        candidates = [f for f in os.listdir(path.join(self.context.topdir, "target"))
+                      if f.startswith(prefix + "-")]
+        if candidates:
+            return path.join(self.context.topdir, "target", candidates[0])
+        return None
+
+    def run_test(self, prefix, args=[]):
+        t = self.find_test(prefix)
+        if t:
+            return subprocess.call([t] + args, env=self.build_env())
+
+    @Command('test',
+             description='Run all Servo tests',
+             category='testing')
+    def test(self):
+        test_start = time()
+        for t in ["tidy", "unit", "ref", "content", "wpt"]:
+            Registrar.dispatch("test-%s" % t, context=self.context)
+        elapsed = time() - test_start
+
+        print("Tests completed in %0.2fs" % elapsed)
+
+    @Command('test-unit',
+             description='Run libservo unit tests',
+             category='testing')
+    def test_unit(self):
+        self.ensure_bootstrapped()
+        self.ensure_built_tests()
+        return self.run_test("servo")
+
+    @Command('test-ref',
+             description='Run the reference tests',
+             category='testing')
+    @CommandArgument('--kind', '-k', default=None)
+    def test_ref(self, kind=None):
+        self.ensure_bootstrapped()
+        self.ensure_built_tests()
+
+        kinds = ["cpu", "gpu"] if kind is None else [kind]
+        test_path = path.join(self.context.topdir, "tests", "ref")
+        error = False
+
+        test_start = time()
+        for k in kinds:
+            print("Running %s reftests..." % k)
+            ret = self.run_test("reftest", [k, test_path])
+            error = error or ret != 0
+        elapsed = time() - test_start
+
+        print("Reference tests completed in %0.2fs" % elapsed)
+
+        if error: return 1
+
+    @Command('test-content',
+             description='Run the content tests',
+             category='testing')
+    def test_content(self):
+        self.ensure_bootstrapped()
+        self.ensure_built_tests()
+
+        test_path = path.join(self.context.topdir, "tests", "content")
+        test_start = time()
+        ret = self.run_test("contenttest", ["--source-dir=%s" % test_path])
+        elapsed = time() - test_start
+
+        print("Content tests completed in %0.2fs" % elapsed)
+        return ret
+
+    @Command('test-tidy',
+             description='Run the source code tidiness check',
+             category='testing')
+    def test_tidy(self):
+        errors = 0
+        for p in ["src", "components"]:
+            ret = tidy.scan(path.join(self.context.topdir, p))
+            if ret != 0: errors = 1
+        return errors
+
+    @Command('test-wpt',
+             description='Run the web platform tests',
+             category='testing',
+             allow_all_args=True)
+    @CommandArgument('params', default=None, nargs='...',
+                     help="Command-line arguments to be passed through to wpt/run.sh")
+    def test_wpt(self, params):
+        return subprocess.call(["bash", path.join("tests", "wpt", "run.sh")] + params,
+                               env=self.build_env())