Add %include support to dockerfiles

decision-task.py

@@ -48,7 +48,7 @@ def main():
     }
     build_kwargs = {
         "max_run_time_minutes": 60,
-        "dockerfile": "build-x86_64-linux.dockerfile",
+        "dockerfile": dockerfile_path("build"),
         "env": build_env,
         "scopes": cache_scopes,
         "cache": build_caches,
@@ -108,7 +108,7 @@ def main():
             env=dict(**env or {}, BUILD_TASK_ID=release_build_task),
             dependencies=[release_build_task],
             max_run_time_minutes=60,
-            dockerfile="run-x86_64-linux.dockerfile",
+            dockerfile=dockerfile_path("run"),
             **kwargs
         )
 
@@ -156,6 +156,9 @@ def main():
         )
 
 
+def dockerfile_path(name):
+    return os.path.join(os.path.dirname(__file__), "docker", name + ".dockerfile")
+
+
 if __name__ == "__main__":
-    os.chdir(os.path.join(".", os.path.dirname(__file__)))
     main()

decisionlib.py

@@ -80,8 +80,7 @@ class DecisionTask:
         return task_id
 
     def find_or_build_docker_image(self, dockerfile):
-        with open(dockerfile, "rb") as f:
-            dockerfile_contents = f.read()
+        dockerfile_contents = expand_dockerfile(dockerfile)
         digest = hashlib.sha256(dockerfile_contents).hexdigest()
 
         return self.find_or_create_task(
@@ -210,5 +209,19 @@ def image_name(dockerfile):
         return basename
 
 
+def expand_dockerfile(dockerfile):
+    with open(dockerfile, "rb") as f:
+        dockerfile_contents = f.read()
+
+    include_marker = b"% include"
+    if not dockerfile_contents.startswith(include_marker):
+        return dockerfile_contents
+
+    include_line, _, rest = dockerfile_contents.partition(b"\n")
+    included = include_line[len(include_marker):].strip().decode("utf8")
+    path = os.path.join(os.path.dirname(dockerfile), included)
+    return b"\n".join([expand_dockerfile(path), rest])
+
+
 def deindent(string):
     return re.sub("\n +", " \n ", string)

docker/base.dockerfile

@@ -2,7 +2,7 @@ FROM ubuntu:bionic-20180821
 
 ENV \
     #
-    # The 'tzdata' APT package waits for user input on install by default
+    # Some APT packages like 'tzdata' wait for user input on install by default.
     # https://stackoverflow.com/questions/44331836/apt-get-install-tzdata-noninteractive
     DEBIAN_FRONTEND=noninteractive
 
@@ -18,12 +18,6 @@ RUN \
         python2.7 \
         virtualenv \
         #
-        # Fetching build artifacts
-        curl \
-        #
-        # Servo’s runtime dependencies
-        libgl1 \
-        libssl1.0.0 \
-        libdbus-1-3 \
-        libgstreamer-plugins-bad1.0-0 \
-        gstreamer1.0-plugins-good
+        # Installing rustup and sccache (build dockerfile) or fetching build artifacts (run tasks)
+        curl
+

docker/build.dockerfile

@@ -1,31 +1,7 @@
-FROM ubuntu:bionic-20180821
-
-ENV \
-    #
-    # Use rustup’s 'cargo' and 'rustc'
-    PATH="/root/.cargo/bin:${PATH}" \
-    #
-    # SpiderMonkey’s build system fails if $SHELL is unset
-    SHELL=/bin/dash \
-    #
-    # The 'tzdata' APT package waits for user input on install by default
-    # https://stackoverflow.com/questions/44331836/apt-get-install-tzdata-noninteractive
-    DEBIAN_FRONTEND=noninteractive
+% include base.dockerfile
 
 RUN \
-    apt-get update -q && \
     apt-get install -qy --no-install-recommends \
-        #
-        # Cloning the repository
-        git \
-        ca-certificates \
-        #
-        # Installing rustup
-        curl \
-        #
-        # Running mach
-        python2.7 \
-        virtualenv \
         #
         # Multiple C/C++ dependencies built from source
         g++ \
@@ -41,11 +17,8 @@ RUN \
         # Bindgen (for SpiderMonkey bindings)
         clang \
         #
-        # gstreamer
-        libglib2.0-dev \
-        libgstreamer-plugins-base1.0-dev \
+        # GStreamer
         libgstreamer-plugins-bad1.0-dev \
-        libgstreamer1.0-dev \
         #
         # OpenSSL
         libssl1.0-dev \
@@ -68,4 +41,3 @@ RUN \
         https://github.com/mozilla/sccache/releases/download/0.2.7/sccache-0.2.7-x86_64-unknown-linux-musl.tar.gz \
         | tar -xz --strip-components=1 -C /usr/local/bin/ \
             sccache-0.2.7-x86_64-unknown-linux-musl/sccache
-

docker/run.dockerfile

@@ -0,0 +1,9 @@
+% include base.dockerfile
+
+# Servo’s runtime dependencies
+RUN apt-get install -qy --no-install-recommends \
+    libgl1 \
+    libssl1.0.0 \
+    libdbus-1-3 \
+    libgstreamer-plugins-bad1.0-0 \
+    gstreamer1.0-plugins-good