win32: mach and build command fixes

- Add SERVO_USE_NIGHTLY_RUST env var to use the latest rust/cargo nightly snapshot
- Fix up looking for cargo binary (in cargo/bin/cargo, not bin/cargo)
- Fix up win32 executable checking (use .exe suffix)
- fix up win32 PATH handling (subprocess must use shell=True for PATH change to be honored)

python/servo/bootstrap_commands.py

@@ -28,7 +28,7 @@ from mach.decorators import (
     Command,
 )
 
-from servo.command_base import CommandBase, cd, host_triple
+from servo.command_base import CommandBase, cd, host_triple, use_nightly_rust, check_call, BIN_SUFFIX
 
 
 def download(desc, src, writer):
@@ -111,7 +111,7 @@ class MachCommands(CommandBase):
     def bootstrap_rustc(self, force=False):
         rust_dir = path.join(
             self.context.sharedir, "rust", self.rust_path())
-        if not force and path.exists(path.join(rust_dir, "rustc", "bin", "rustc")):
+        if not force and path.exists(path.join(rust_dir, "rustc", "bin", "rustc" + BIN_SUFFIX)):
             print("Rust compiler already downloaded.", end=" ")
             print("Use |bootstrap-rust --force| to download again.")
             return
@@ -203,7 +203,7 @@ class MachCommands(CommandBase):
     def bootstrap_cargo(self, force=False):
         cargo_dir = path.join(self.context.sharedir, "cargo",
                               self.cargo_build_id())
-        if not force and path.exists(path.join(cargo_dir, "bin", "cargo")):
+        if not force and path.exists(path.join(cargo_dir, "cargo", "bin", "cargo" + BIN_SUFFIX)):
             print("Cargo already downloaded.", end=" ")
             print("Use |bootstrap-cargo --force| to download again.")
             return
@@ -289,9 +289,9 @@ class MachCommands(CommandBase):
                                   % module_path)
                             print("\nClean the submodule and try again.")
                             return 1
-        subprocess.check_call(
+        check_call(
             ["git", "submodule", "--quiet", "sync", "--recursive"])
-        subprocess.check_call(
+        check_call(
             ["git", "submodule", "update", "--init", "--recursive"])
 
     @Command('clean-nightlies',

python/servo/build_commands.py

@@ -11,7 +11,6 @@ from __future__ import print_function, unicode_literals
 
 import os
 import os.path as path
-import subprocess
 import sys
 import shutil
 
@@ -23,7 +22,7 @@ from mach.decorators import (
     Command,
 )
 
-from servo.command_base import CommandBase, cd
+from servo.command_base import CommandBase, cd, call
 
 
 def is_headless_build():
@@ -123,14 +122,6 @@ def notify(title, text):
             print("[Warning] Could not generate notification! %s" % extra, file=sys.stderr)
 
 
-def call(*args, **kwargs):
-    """Wrap `subprocess.call`, printing the command if verbose=True."""
-    verbose = kwargs.pop('verbose', False)
-    if verbose:
-        print(' '.join(args[0]))
-    return subprocess.call(*args, **kwargs)
-
-
 @CommandProvider
 class MachCommands(CommandBase):
     @Command('build',

python/servo/command_base.py

@@ -16,6 +16,10 @@ import toml
 
 from mach.registrar import Registrar
 
+BIN_SUFFIX = ""
+if sys.platform == "win32":
+    BIN_SUFFIX = ".exe"
+
 
 @contextlib.contextmanager
 def cd(new_path):
@@ -36,6 +40,8 @@ def host_triple():
         os_type = "apple-darwin"
     elif os_type == "android":
         os_type = "linux-androideabi"
+    elif os_type.startswith("mingw64_nt-"):
+        os_type = "pc-windows-gnu"
     else:
         os_type = "unknown"
 
@@ -52,6 +58,37 @@ def host_triple():
     return "%s-%s" % (cpu_type, os_type)
 
 
+def use_nightly_rust():
+    envvar = os.environ.get("SERVO_USE_NIGHTLY_RUST")
+    if envvar:
+        return envvar != "0"
+    return False
+
+
+def call(*args, **kwargs):
+    """Wrap `subprocess.call`, printing the command if verbose=True."""
+    verbose = kwargs.pop('verbose', False)
+    if verbose:
+        print(' '.join(args[0]))
+    if sys.platform == "win32":
+        # we have to use shell=True in order to get PATH handling
+        # when looking for the binary on Windows
+        return subprocess.call(*args, shell=True, **kwargs)
+    return subprocess.call(*args, **kwargs)
+
+
+def check_call(*args, **kwargs):
+    """Wrap `subprocess.check_call`, printing the command if verbose=True."""
+    verbose = kwargs.pop('verbose', False)
+    if verbose:
+        print(' '.join(args[0]))
+    if sys.platform == "win32":
+        # we have to use shell=True in order to get PATH handling
+        # when looking for the binary on Windows
+        return subprocess.check_call(*args, shell=True, **kwargs)
+    return subprocess.check_call(*args, **kwargs)
+
+
 class CommandBase(object):
     """Base class for mach command providers.
 
@@ -333,13 +370,13 @@ class CommandBase(object):
 
         if not self.config["tools"]["system-rust"] and \
            not path.exists(path.join(
-                self.config["tools"]["rust-root"], "rustc", "bin", "rustc")):
+                self.config["tools"]["rust-root"], "rustc", "bin", "rustc" + BIN_SUFFIX)):
             print("looking for rustc at %s" % path.join(
-                self.config["tools"]["rust-root"], "rustc", "bin", "rustc"))
+                self.config["tools"]["rust-root"], "rustc", "bin", "rustc" + BIN_SUFFIX))
             Registrar.dispatch("bootstrap-rust", context=self.context)
         if not self.config["tools"]["system-cargo"] and \
            not path.exists(path.join(
-                self.config["tools"]["cargo-root"], "cargo", "bin", "cargo")):
+                self.config["tools"]["cargo-root"], "cargo", "bin", "cargo" + BIN_SUFFIX)):
             Registrar.dispatch("bootstrap-cargo", context=self.context)
 
         self.context.bootstrapped = True

python/servo/devenv_commands.py

@@ -10,7 +10,6 @@
 from __future__ import print_function, unicode_literals
 from os import path, getcwd, listdir
 
-import subprocess
 import sys
 
 from mach.decorators import (
@@ -19,7 +18,7 @@ from mach.decorators import (
     Command,
 )
 
-from servo.command_base import CommandBase, cd
+from servo.command_base import CommandBase, cd, call
 
 
 @CommandProvider
@@ -36,10 +35,8 @@ class MachCommands(CommandBase):
 
         if self.context.topdir == getcwd():
             with cd(path.join('components', 'servo')):
-                return subprocess.call(
+                return call(["cargo"] + params, env=self.build_env())
-                    ["cargo"] + params, env=self.build_env())
+        return call(['cargo'] + params, env=self.build_env())
-        return subprocess.call(['cargo'] + params,
-                               env=self.build_env())
 
     @Command('cargo-update',
              description='Same as update-cargo',
@@ -89,7 +86,7 @@ class MachCommands(CommandBase):
         for cargo_path in cargo_paths:
             with cd(cargo_path):
                 print(cargo_path)
-                subprocess.call(["cargo", "update"] + params,
+                call(["cargo", "update"] + params,
                      env=self.build_env())
 
     @Command('clippy',
@@ -111,7 +108,7 @@ class MachCommands(CommandBase):
     def rustc(self, params):
         if params is None:
             params = []
-        return subprocess.call(["rustc"] + params, env=self.build_env())
+        return call(["rustc"] + params, env=self.build_env())
 
     @Command('rust-root',
              description='Print the path to the root of the Rust compiler',
@@ -140,7 +137,7 @@ class MachCommands(CommandBase):
         root_dirs_abs = [path.join(self.context.topdir, s) for s in root_dirs]
         # Absolute paths for all directories to be considered
         grep_paths = root_dirs_abs + tests_dirs_abs
-        return subprocess.call(
+        return call(
             ["git"] + ["grep"] + params + ['--'] + grep_paths + [':(exclude)*.min.js'],
             env=self.build_env())
 
@@ -149,14 +146,14 @@ class MachCommands(CommandBase):
              category='devenv')
     def upgrade_wpt_runner(self):
         with cd(path.join(self.context.topdir, 'tests', 'wpt', 'harness')):
-            code = subprocess.call(["git", "init"], env=self.build_env())
+            code = call(["git", "init"], env=self.build_env())
             if code:
                 return code
-            subprocess.call(
+            call(
                 ["git", "remote", "add", "upstream", "https://github.com/w3c/wptrunner.git"], env=self.build_env())
-            code = subprocess.call(["git", "fetch", "upstream"], env=self.build_env())
+            code = call(["git", "fetch", "upstream"], env=self.build_env())
             if code:
                 return code
-            code = subprocess.call(["git", "reset", '--', "hard", "remotes/upstream/master"], env=self.build_env())
+            code = call(["git", "reset", '--', "hard", "remotes/upstream/master"], env=self.build_env())
             if code:
                 return code

python/servo/post_build_commands.py

@@ -22,7 +22,7 @@ from mach.decorators import (
     Command,
 )
 
-from servo.command_base import CommandBase, cd
+from servo.command_base import CommandBase, cd, call, check_call
 
 
 def read_file(filename, if_exists=False):
@@ -114,7 +114,7 @@ class PostBuildCommands(CommandBase):
             args = args + params
 
         try:
-            subprocess.check_call(args, env=env)
+            check_call(args, env=env)
         except subprocess.CalledProcessError as e:
             print("Servo exited with return value %d" % e.returncode)
             return e.returncode
@@ -142,7 +142,7 @@ class PostBuildCommands(CommandBase):
         servo_cmd = [self.get_binary_path(release, dev)] + params
         rr_cmd = ['rr', '--fatal-errors', 'record']
         try:
-            subprocess.check_call(rr_cmd + servo_cmd)
+            check_call(rr_cmd + servo_cmd)
         except OSError as e:
             if e.errno == 2:
                 print("rr binary can't be found!")
@@ -154,7 +154,7 @@ class PostBuildCommands(CommandBase):
              category='post-build')
     def rr_replay(self):
         try:
-            subprocess.check_call(['rr', '--fatal-errors', 'replay'])
+            check_call(['rr', '--fatal-errors', 'replay'])
         except OSError as e:
             if e.errno == 2:
                 print("rr binary can't be found!")
@@ -191,7 +191,7 @@ class PostBuildCommands(CommandBase):
                     else:
                         copy2(full_name, destination)
 
-        return subprocess.call(["cargo", "doc"] + params,
+        return call(["cargo", "doc"] + params,
                     env=self.build_env(), cwd=self.servo_crate())
 
     @Command('browse-doc',

python/servo/testing_commands.py

@@ -26,7 +26,7 @@ from mach.decorators import (
     Command,
 )
 
-from servo.command_base import CommandBase
+from servo.command_base import CommandBase, call, check_call
 from wptrunner import wptcommandline
 from update import updatecommandline
 import tidy
@@ -78,7 +78,7 @@ class MachCommands(CommandBase):
     def run_test(self, prefix, args=[], release=False):
         t = self.find_test(prefix, release=release)
         if t:
-            return subprocess.call([t] + args, env=self.build_env())
+            return call([t] + args, env=self.build_env())
 
     @Command('test',
              description='Run all Servo tests',
@@ -203,7 +203,7 @@ class MachCommands(CommandBase):
         for crate in packages:
             args += ["-p", "%s_tests" % crate]
         args += test_patterns
-        result = subprocess.call(args, env=self.build_env(), cwd=self.servo_crate())
+        result = call(args, env=self.build_env(), cwd=self.servo_crate())
         if result != 0:
             return result
 
@@ -237,7 +237,7 @@ class MachCommands(CommandBase):
              category='testing')
     def test_wpt_failure(self):
         self.ensure_bootstrapped()
-        return not subprocess.call([
+        return not call([
             "bash",
             path.join("tests", "wpt", "run.sh"),
             "--no-pause-after-test",
@@ -395,17 +395,17 @@ class MachCommands(CommandBase):
 
         # Clone the jQuery repository if it doesn't exist
         if not os.path.isdir(jquery_dir):
-            subprocess.check_call(
+            check_call(
                 ["git", "clone", "-b", "servo", "--depth", "1", "https://github.com/servo/jquery", jquery_dir])
 
         # Run pull in case the jQuery repo was updated since last test run
-        subprocess.check_call(
+        check_call(
             ["git", "-C", jquery_dir, "pull"])
 
         # Check that a release servo build exists
         bin_path = path.abspath(self.get_binary_path(release, dev))
 
-        return subprocess.check_call(
+        return check_call(
             [run_file, cmd, bin_path, base_dir])
 
     def dromaeo_test_runner(self, tests, release, dev):
@@ -416,21 +416,21 @@ class MachCommands(CommandBase):
 
         # Clone the Dromaeo repository if it doesn't exist
         if not os.path.isdir(dromaeo_dir):
-            subprocess.check_call(
+            check_call(
                 ["git", "clone", "-b", "servo", "--depth", "1", "https://github.com/notriddle/dromaeo", dromaeo_dir])
 
         # Run pull in case the Dromaeo repo was updated since last test run
-        subprocess.check_call(
+        check_call(
             ["git", "-C", dromaeo_dir, "pull"])
 
         # Compile test suite
-        subprocess.check_call(
+        check_call(
             ["make", "-C", dromaeo_dir, "web"])
 
         # Check that a release servo build exists
         bin_path = path.abspath(self.get_binary_path(release, dev))
 
-        return subprocess.check_call(
+        return check_call(
             [run_file, "|".join(tests), bin_path, base_dir])