From 6ced45fd6126b214d299d10c7f68626800c79050 Mon Sep 17 00:00:00 2001
From: Simon Sapin <simon.sapin@exyr.org>
Date: Fri, 21 Sep 2018 12:26:39 +0200
Subject: [PATCH] Add %include support to dockerfiles

---
 decision-task.py                              |  9 ++++--
 decisionlib.py                                | 17 ++++++++--
 .../base.dockerfile                           | 14 +++-----
 .../build.dockerfile                          | 32 ++-----------------
 docker/run.dockerfile                         |  9 ++++++
 5 files changed, 36 insertions(+), 45 deletions(-)
 rename run-x86_64-linux.dockerfile => docker/base.dockerfile (57%)
 rename build-x86_64-linux.dockerfile => docker/build.dockerfile (55%)
 create mode 100644 docker/run.dockerfile

diff --git a/decision-task.py b/decision-task.py
index c4e12723c5c..cca8f3a16f5 100644
--- a/decision-task.py
+++ b/decision-task.py
@@ -48,7 +48,7 @@ def main():
     }
     build_kwargs = {
         "max_run_time_minutes": 60,
-        "dockerfile": "build-x86_64-linux.dockerfile",
+        "dockerfile": dockerfile_path("build"),
         "env": build_env,
         "scopes": cache_scopes,
         "cache": build_caches,
@@ -108,7 +108,7 @@ def main():
             env=dict(**env or {}, BUILD_TASK_ID=release_build_task),
             dependencies=[release_build_task],
             max_run_time_minutes=60,
-            dockerfile="run-x86_64-linux.dockerfile",
+            dockerfile=dockerfile_path("run"),
             **kwargs
         )
 
@@ -156,6 +156,9 @@ def main():
         )
 
 
+def dockerfile_path(name):
+    return os.path.join(os.path.dirname(__file__), "docker", name + ".dockerfile")
+
+
 if __name__ == "__main__":
-    os.chdir(os.path.join(".", os.path.dirname(__file__)))
     main()
diff --git a/decisionlib.py b/decisionlib.py
index 5b0c58ed855..055d0c8da46 100644
--- a/decisionlib.py
+++ b/decisionlib.py
@@ -80,8 +80,7 @@ class DecisionTask:
         return task_id
 
     def find_or_build_docker_image(self, dockerfile):
-        with open(dockerfile, "rb") as f:
-            dockerfile_contents = f.read()
+        dockerfile_contents = expand_dockerfile(dockerfile)
         digest = hashlib.sha256(dockerfile_contents).hexdigest()
 
         return self.find_or_create_task(
@@ -210,5 +209,19 @@ def image_name(dockerfile):
         return basename
 
 
+def expand_dockerfile(dockerfile):
+    with open(dockerfile, "rb") as f:
+        dockerfile_contents = f.read()
+
+    include_marker = b"% include"
+    if not dockerfile_contents.startswith(include_marker):
+        return dockerfile_contents
+
+    include_line, _, rest = dockerfile_contents.partition(b"\n")
+    included = include_line[len(include_marker):].strip().decode("utf8")
+    path = os.path.join(os.path.dirname(dockerfile), included)
+    return b"\n".join([expand_dockerfile(path), rest])
+
+
 def deindent(string):
     return re.sub("\n +", " \n ", string)
diff --git a/run-x86_64-linux.dockerfile b/docker/base.dockerfile
similarity index 57%
rename from run-x86_64-linux.dockerfile
rename to docker/base.dockerfile
index 538471f467d..891469e8df4 100644
--- a/run-x86_64-linux.dockerfile
+++ b/docker/base.dockerfile
@@ -2,7 +2,7 @@ FROM ubuntu:bionic-20180821
 
 ENV \
     #
-    # The 'tzdata' APT package waits for user input on install by default
+    # Some APT packages like 'tzdata' wait for user input on install by default.
     # https://stackoverflow.com/questions/44331836/apt-get-install-tzdata-noninteractive
     DEBIAN_FRONTEND=noninteractive
 
@@ -18,12 +18,6 @@ RUN \
         python2.7 \
         virtualenv \
         #
-        # Fetching build artifacts
-        curl \
-        #
-        # Servo’s runtime dependencies
-        libgl1 \
-        libssl1.0.0 \
-        libdbus-1-3 \
-        libgstreamer-plugins-bad1.0-0 \
-        gstreamer1.0-plugins-good
+        # Installing rustup and sccache (build dockerfile) or fetching build artifacts (run tasks)
+        curl
+
diff --git a/build-x86_64-linux.dockerfile b/docker/build.dockerfile
similarity index 55%
rename from build-x86_64-linux.dockerfile
rename to docker/build.dockerfile
index 7336724b725..8b7b3f75f6f 100644
--- a/build-x86_64-linux.dockerfile
+++ b/docker/build.dockerfile
@@ -1,31 +1,7 @@
-FROM ubuntu:bionic-20180821
-
-ENV \
-    #
-    # Use rustup’s 'cargo' and 'rustc'
-    PATH="/root/.cargo/bin:${PATH}" \
-    #
-    # SpiderMonkey’s build system fails if $SHELL is unset
-    SHELL=/bin/dash \
-    #
-    # The 'tzdata' APT package waits for user input on install by default
-    # https://stackoverflow.com/questions/44331836/apt-get-install-tzdata-noninteractive
-    DEBIAN_FRONTEND=noninteractive
+% include base.dockerfile
 
 RUN \
-    apt-get update -q && \
     apt-get install -qy --no-install-recommends \
-        #
-        # Cloning the repository
-        git \
-        ca-certificates \
-        #
-        # Installing rustup
-        curl \
-        #
-        # Running mach
-        python2.7 \
-        virtualenv \
         #
         # Multiple C/C++ dependencies built from source
         g++ \
@@ -41,11 +17,8 @@ RUN \
         # Bindgen (for SpiderMonkey bindings)
         clang \
         #
-        # gstreamer
-        libglib2.0-dev \
-        libgstreamer-plugins-base1.0-dev \
+        # GStreamer
         libgstreamer-plugins-bad1.0-dev \
-        libgstreamer1.0-dev \
         #
         # OpenSSL
         libssl1.0-dev \
@@ -68,4 +41,3 @@ RUN \
         https://github.com/mozilla/sccache/releases/download/0.2.7/sccache-0.2.7-x86_64-unknown-linux-musl.tar.gz \
         | tar -xz --strip-components=1 -C /usr/local/bin/ \
             sccache-0.2.7-x86_64-unknown-linux-musl/sccache
-
diff --git a/docker/run.dockerfile b/docker/run.dockerfile
new file mode 100644
index 00000000000..12275024c32
--- /dev/null
+++ b/docker/run.dockerfile
@@ -0,0 +1,9 @@
+% include base.dockerfile
+
+# Servo’s runtime dependencies
+RUN apt-get install -qy --no-install-recommends \
+    libgl1 \
+    libssl1.0.0 \
+    libdbus-1-3 \
+    libgstreamer-plugins-bad1.0-0 \
+    gstreamer1.0-plugins-good