Automatically clean old nightlies

Add a new `-keep` flag to `./mach clean-nightlies` to control how many
nightlies to keep, which keeps the n most recent nightlies found in git.

Use this to clean old nightlies at the start of each build,
currently keeping 3 nightlies at a time.

etc/ci/buildbot_steps.yml

@@ -1,4 +1,5 @@
 mac-rel-wpt1:
+  - ./mach clean-nightlies --keep 3 --force
   - ./mach build --release
   - ./mach test-wpt-failure
   - ./mach test-wpt --release --processes 8 --total-chunks 2 --this-chunk 1 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log --always-succeed
@@ -9,12 +10,14 @@ mac-rel-wpt1:
   - bash ./etc/ci/manifest_changed.sh
 
 mac-rel-wpt2:
+  - ./mach clean-nightlies --keep 3 --force
   - ./mach build --release
   - ./mach test-wpt --release --processes 8 --total-chunks 2 --this-chunk 2 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log --always-succeed
   - ./mach filter-intermittents wpt-errorsummary.log --log-intermittents intermittents.log --log-filteredsummary filtered-wpt-errorsummary.log --use-tracker
   - ./mach build-geckolib --release
 
 mac-dev-unit:
+  - ./mach clean-nightlies --keep 3 --force
   - env SERVO_RUSTC_LLVM_ASSERTIONS=1 ./mach build --dev
   - env SERVO_RUSTC_LLVM_ASSERTIONS=1 ./mach test-unit
   - env SERVO_RUSTC_LLVM_ASSERTIONS=1 ./mach package --dev
@@ -24,6 +27,7 @@ mac-dev-unit:
   - bash ./etc/ci/manifest_changed.sh
 
 mac-rel-css:
+  - ./mach clean-nightlies --keep 3 --force
   - ./mach build --release
   - ./mach test-css --release --processes 4 --log-raw test-css.log --log-errorsummary css-errorsummary.log --always-succeed
   - ./mach filter-intermittents css-errorsummary.log --log-intermittents intermittents.log --log-filteredsummary filtered-css-errorsummary.log --use-tracker
@@ -31,20 +35,24 @@ mac-rel-css:
   - bash ./etc/ci/manifest_changed.sh
 
 mac-nightly:
+  - ./mach clean-nightlies --keep 3 --force
   - ./mach build --release
   - ./mach package --release
   - ./etc/ci/upload_nightly.sh mac
   - ./etc/ci/upload_nightly.sh macbrew
 
 linux-rel-intermittent:
+  - ./mach clean-nightlies --keep 3 --force
   - ./mach build --release
   - ./etc/ci/check_intermittents.sh --log-raw intermittents.log
 
 mac-rel-intermittent:
+  - ./mach clean-nightlies --keep 3 --force
   - ./mach build --release
   - ./etc/ci/check_intermittents.sh --log-raw intermittents.log
 
 linux-dev:
+  - ./mach clean-nightlies --keep 3 --force
   - ./mach test-tidy --no-progress --all
   - ./mach test-tidy --no-progress --self-test
   - env SERVO_RUSTC_LLVM_ASSERTIONS=1 ./mach build --dev
@@ -59,6 +67,7 @@ linux-dev:
   - bash ./etc/ci/check_no_panic.sh
 
 linux-rel-wpt:
+  - ./mach clean-nightlies --keep 3 --force
   - ./mach build --release --with-debug-assertions
   - ./mach test-wpt-failure
   - ./mach test-wpt --release --processes 24 --log-raw test-wpt.log --log-errorsummary wpt-errorsummary.log --always-succeed
@@ -66,6 +75,7 @@ linux-rel-wpt:
   - ./mach test-wpt --release --binary-arg=--multiprocess --processes 24 --log-raw test-wpt-mp.log --log-errorsummary wpt-mp-errorsummary.log eventsource
 
 linux-rel-css:
+  - ./mach clean-nightlies --keep 3 --force
   - ./mach build --release --with-debug-assertions
   - ./mach test-css --release --processes 16 --log-raw test-css.log --log-errorsummary css-errorsummary.log --always-succeed
   - ./mach filter-intermittents css-errorsummary.log --log-intermittents intermittents.log --log-filteredsummary filtered-css-errorsummary.log --use-tracker
@@ -76,11 +86,13 @@ linux-rel-css:
   - bash ./etc/ci/manifest_changed.sh
 
 linux-nightly:
+  - ./mach clean-nightlies --keep 3 --force
   - ./mach build --release
   - ./mach package --release
   - ./etc/ci/upload_nightly.sh linux
 
 android:
+  - ./mach clean-nightlies --keep 3 --force
   - env SERVO_RUSTC_LLVM_ASSERTIONS=1 ./mach build --android --dev
   - env SERVO_RUSTC_LLVM_ASSERTIONS=1 ./mach package --android --dev
   - bash ./etc/ci/lockfile_changed.sh
@@ -88,27 +100,32 @@ android:
   - python ./etc/ci/check_dynamic_symbols.py
 
 android-nightly:
+  - ./mach clean-nightlies --keep 3 --force
   - env SERVO_RUSTC_LLVM_ASSERTIONS=1 ./mach build --android --release
   - env SERVO_RUSTC_LLVM_ASSERTIONS=1 ./mach package --android --release
   - ./etc/ci/upload_nightly.sh android
 
 arm32:
+  - ./mach clean-nightlies --keep 3 --force
   - env SERVO_RUSTC_LLVM_ASSERTIONS=1 ./mach build --rel --target=arm-unknown-linux-gnueabihf
   - bash ./etc/ci/lockfile_changed.sh
   - bash ./etc/ci/manifest_changed.sh
 
 arm64:
+  - ./mach clean-nightlies --keep 3 --force
   - env SERVO_RUSTC_LLVM_ASSERTIONS=1 ./mach build --rel --target=aarch64-unknown-linux-gnu
   - bash ./etc/ci/lockfile_changed.sh
   - bash ./etc/ci/manifest_changed.sh
 
 windows-msvc-dev:
+  - mach.bat clean-nightlies --keep 3 --force
   - mach.bat build --dev
   - mach.bat test-unit
   - mach.bat package --dev
   - mach.bat build-geckolib
 
 windows-msvc-nightly:
+  - mach.bat clean-nightlies --keep 3 --force
   - mach.bat build --release
   - mach.bat package --release
   - bash -l ./etc/ci/upload_nightly.sh windows-msvc

python/servo/bootstrap_commands.py

@@ -15,6 +15,7 @@ import os
 import os.path as path
 import re
 import shutil
+import subprocess
 import sys
 import urllib2
 
@@ -26,7 +27,7 @@ from mach.decorators import (
 
 import servo.bootstrap as bootstrap
 from servo.command_base import CommandBase, BIN_SUFFIX
-from servo.util import download_bytes, download_file, extract, host_triple
+from servo.util import delete, download_bytes, download_file, extract, host_triple
 
 
 @CommandProvider
@@ -285,26 +286,56 @@ class MachCommands(CommandBase):
     @CommandArgument('--force', '-f',
                      action='store_true',
                      help='Actually remove stuff')
-    def clean_nightlies(self, force=False):
+    @CommandArgument('--keep',
-        rust_current = self.rust_path().split('/')[0]
+                     default='1',
+                     help='Keep up to this many most recent nightlies')
+    def clean_nightlies(self, force=False, keep=None):
+        rust_current = self.rust_version()
         cargo_current = self.cargo_build_id()
-        print("Current Rust version: " + rust_current)
+        print("Current Rust version: {}".format(rust_current))
-        print("Current Cargo version: " + cargo_current)
+        print("Current Cargo version: {}".format(cargo_current))
+        to_keep = {
+            'rust': set(),
+            'cargo': set(),
+        }
+        if int(keep) == 1:
+            # Optimize keep=1 case to not invoke git
+            to_keep['rust'].add(rust_current)
+            to_keep['cargo'].add(cargo_current)
+        else:
+            for tool in ["rust", "cargo"]:
+                commit_file = '{}-commit-hash'.format(tool)
+                cmd = subprocess.Popen(
+                    ['git', 'log', '--pretty=format:%H', '-n', keep, commit_file],
+                    stdout=subprocess.PIPE,
+                    universal_newlines=True
+                )
+                stdout, _ = cmd.communicate()
+                for commit in stdout.splitlines():
+                    cmd = subprocess.Popen(
+                        ['git', 'show', '{}:{}'.format(commit, commit_file)],
+                        stdout=subprocess.PIPE,
+                        universal_newlines=True
+                    )
+                    commit_hash, _ = cmd.communicate()
+                    to_keep[tool].add(commit_hash.rstrip())
+
         removing_anything = False
-        for current, base in [(rust_current, "rust"), (cargo_current, "cargo")]:
+        for tool in ["rust", "cargo"]:
-            base = path.join(self.context.sharedir, base)
+            base = path.join(self.context.sharedir, tool)
             for name in os.listdir(base):
-                if name != current:
+                # We append `-alt` if LLVM assertions aren't enabled,
+                # so use just the commit hash itself.
+                # This may occasionally leave an extra nightly behind
+                # but won't remove too many nightlies.
+                if name.partition('-')[0] not in to_keep[tool]:
                     removing_anything = True
-                    name = path.join(base, name)
+                    full_path = path.join(base, name)
                     if force:
-                        print("Removing " + name)
+                        print("Removing {}".format(full_path))
-                        if os.path.isdir(name):
+                        delete(full_path)
-                            shutil.rmtree(name)
                     else:
-                            os.remove(name)
+                        print("Would remove {}".format(full_path))
-                    else:
-                        print("Would remove " + name)
         if not removing_anything:
             print("Nothing to remove.")
         elif not force: