Create a util.py Python module for common functions

Extracting these functions helps avoid circular dependencies,
and make them easier to find/reuse.

python/servo/bootstrap_commands.py

@@ -8,7 +8,6 @@
 # except according to those terms.
 
 from __future__ import print_function, unicode_literals
-from socket import error as socket_error
 
 import base64
 import json
@@ -17,9 +16,6 @@ import os.path as path
 import re
 import shutil
 import sys
-import StringIO
-import tarfile
-import zipfile
 import urllib2
 
 from mach.decorators import (
@@ -28,93 +24,8 @@ from mach.decorators import (
     Command,
 )
 
-from servo.command_base import CommandBase, host_triple, BIN_SUFFIX
-
-
-def download(desc, src, writer, start_byte=0):
-    if start_byte:
-        print("Resuming download of %s..." % desc)
-    else:
-        print("Downloading %s..." % desc)
-    dumb = (os.environ.get("TERM") == "dumb") or (not sys.stdout.isatty())
-
-    try:
-        req = urllib2.Request(src)
-        if start_byte:
-            req = urllib2.Request(src, headers={'Range': 'bytes={}-'.format(start_byte)})
-        resp = urllib2.urlopen(req)
-
-        fsize = None
-        if resp.info().getheader('Content-Length'):
-            fsize = int(resp.info().getheader('Content-Length').strip()) + start_byte
-
-        recved = start_byte
-        chunk_size = 8192
-
-        while True:
-            chunk = resp.read(chunk_size)
-            if not chunk:
-                break
-            recved += len(chunk)
-            if not dumb:
-                if fsize is not None:
-                    pct = recved * 100.0 / fsize
-                    print("\rDownloading %s: %5.1f%%" % (desc, pct), end="")
-
-                sys.stdout.flush()
-            writer.write(chunk)
-
-        if not dumb:
-            print()
-    except urllib2.HTTPError, e:
-        print("Download failed (%d): %s - %s" % (e.code, e.reason, src))
-        if e.code == 403:
-            print("No Rust compiler binary available for this platform. "
-                  "Please see https://github.com/servo/servo/#prerequisites")
-        sys.exit(1)
-    except urllib2.URLError, e:
-        print("Error downloading Rust compiler: %s. The failing URL was: %s" % (e.reason, src))
-        sys.exit(1)
-    except socket_error, e:
-        print("Looks like there's a connectivity issue, check your Internet connection. %s" % (e))
-        sys.exit(1)
-    except KeyboardInterrupt:
-        writer.flush()
-        raise
-
-
-def download_file(desc, src, dst):
-    tmp_path = dst + ".part"
-    try:
-        start_byte = os.path.getsize(tmp_path)
-        with open(tmp_path, 'ab') as fd:
-            download(desc, src, fd, start_byte=start_byte)
-    except os.error:
-        with open(tmp_path, 'wb') as fd:
-            download(desc, src, fd)
-    os.rename(tmp_path, dst)
-
-
-def download_bytes(desc, src):
-    content_writer = StringIO.StringIO()
-    download(desc, src, content_writer)
-    return content_writer.getvalue()
-
-
-def extract(src, dst, movedir=None):
-    if src.endswith(".zip"):
-        zipfile.ZipFile(src).extractall(dst)
-    else:
-        tarfile.open(src).extractall(dst)
-
-    if movedir:
-        for f in os.listdir(movedir):
-            frm = path.join(movedir, f)
-            to = path.join(dst, f)
-            os.rename(frm, to)
-        os.rmdir(movedir)
-
-    os.remove(src)
+from servo.command_base import CommandBase, BIN_SUFFIX
+from servo.util import download_bytes, download_file, extract, host_triple
 
 
 @CommandProvider

python/servo/bootstrapper/windows_msvc.py

@@ -3,10 +3,10 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import os
-import sys
 import shutil
 from distutils import spawn
 
+from servo.util import extract, download_file
 from base import BaseBootstrapper
 from packages import WINDOWS_MSVC as deps
 
@@ -21,7 +21,6 @@ class WindowsMsvcBootstrapper(BaseBootstrapper):
         self.install_system_packages()
 
     def install_system_packages(self, packages=deps):
-        from servo.bootstrap_commands import extract, download_file
 
         deps_dir = os.path.join(self.context.sharedir, "msvc-dependencies")
         deps_url = "https://servo-rust.s3.amazonaws.com/msvc-deps/"

python/servo/build_commands.py

@@ -24,7 +24,8 @@ from mach.decorators import (
     Command,
 )
 
-from servo.command_base import CommandBase, cd, call, BIN_SUFFIX, host_triple, find_dep_path_newest
+from servo.command_base import CommandBase, cd, call, BIN_SUFFIX, find_dep_path_newest
+from servo.util import host_triple
 
 
 def format_duration(seconds):

python/servo/command_base.py

@@ -18,11 +18,12 @@ import subprocess
 from subprocess import PIPE
 import sys
 import tarfile
-import platform
 
 import toml
 
 from mach.registrar import Registrar
+from servo.util import host_triple
+
 
 BIN_SUFFIX = ".exe" if sys.platform == "win32" else ""
 
@@ -107,51 +108,6 @@ def archive_deterministically(dir_to_archive, dest_archive, prepend_path=None):
         os.rename(temp_file, dest_archive)
 
 
-def host_triple():
-    os_type = platform.system().lower()
-    if os_type == "linux":
-        os_type = "unknown-linux-gnu"
-    elif os_type == "darwin":
-        os_type = "apple-darwin"
-    elif os_type == "android":
-        os_type = "linux-androideabi"
-    elif os_type == "windows":
-        # If we are in a Visual Studio environment, use msvc
-        if os.getenv("PLATFORM") is not None:
-            os_type = "pc-windows-msvc"
-        elif os.getenv("MSYSTEM") is not None:
-            os_type = "pc-windows-gnu"
-        else:
-            os_type = "unknown"
-    elif os_type.startswith("mingw64_nt-") or os_type.startswith("cygwin_nt-"):
-        os_type = "pc-windows-gnu"
-    elif os_type == "freebsd":
-        os_type = "unknown-freebsd"
-    else:
-        os_type = "unknown"
-
-    cpu_type = platform.machine().lower()
-    if os_type.endswith("-msvc"):
-        # vcvars*.bat should set it properly
-        platform_env = os.environ.get("PLATFORM")
-        if platform_env == "X86":
-            cpu_type = "i686"
-        elif platform_env == "X64":
-            cpu_type = "x86_64"
-        else:
-            cpu_type = "unknown"
-    elif cpu_type in ["i386", "i486", "i686", "i768", "x86"]:
-        cpu_type = "i686"
-    elif cpu_type in ["x86_64", "x86-64", "x64", "amd64"]:
-        cpu_type = "x86_64"
-    elif cpu_type == "arm":
-        cpu_type = "arm"
-    else:
-        cpu_type = "unknown"
-
-    return "%s-%s" % (cpu_type, os_type)
-
-
 def normalize_env(env):
     # There is a bug in subprocess where it doesn't like unicode types in
     # environment variables. Here, ensure all unicode are converted to

python/servo/testing_commands.py

@@ -30,8 +30,9 @@ from mach.decorators import (
 
 from servo.command_base import (
     BuildNotFound, CommandBase,
-    call, cd, check_call, host_triple, set_osmesa_env,
+    call, cd, check_call, set_osmesa_env,
 )
+from servo.util import host_triple
 
 from wptrunner import wptcommandline
 from update import updatecommandline

python/servo/util.py

@@ -0,0 +1,151 @@
+# Copyright 2013 The Servo Project Developers. See the COPYRIGHT
+# file at the top-level directory of this distribution.
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import os
+import os.path as path
+import platform
+import sys
+from socket import error as socket_error
+import StringIO
+import tarfile
+import zipfile
+import urllib2
+
+
+def host_triple():
+    os_type = platform.system().lower()
+    if os_type == "linux":
+        os_type = "unknown-linux-gnu"
+    elif os_type == "darwin":
+        os_type = "apple-darwin"
+    elif os_type == "android":
+        os_type = "linux-androideabi"
+    elif os_type == "windows":
+        # If we are in a Visual Studio environment, use msvc
+        if os.getenv("PLATFORM") is not None:
+            os_type = "pc-windows-msvc"
+        elif os.getenv("MSYSTEM") is not None:
+            os_type = "pc-windows-gnu"
+        else:
+            os_type = "unknown"
+    elif os_type.startswith("mingw64_nt-") or os_type.startswith("cygwin_nt-"):
+        os_type = "pc-windows-gnu"
+    elif os_type == "freebsd":
+        os_type = "unknown-freebsd"
+    else:
+        os_type = "unknown"
+
+    cpu_type = platform.machine().lower()
+    if os_type.endswith("-msvc"):
+        # vcvars*.bat should set it properly
+        platform_env = os.environ.get("PLATFORM")
+        if platform_env == "X86":
+            cpu_type = "i686"
+        elif platform_env == "X64":
+            cpu_type = "x86_64"
+        else:
+            cpu_type = "unknown"
+    elif cpu_type in ["i386", "i486", "i686", "i768", "x86"]:
+        cpu_type = "i686"
+    elif cpu_type in ["x86_64", "x86-64", "x64", "amd64"]:
+        cpu_type = "x86_64"
+    elif cpu_type == "arm":
+        cpu_type = "arm"
+    else:
+        cpu_type = "unknown"
+
+    return "{}-{}".format(cpu_type, os_type)
+
+
+def download(desc, src, writer, start_byte=0):
+    if start_byte:
+        print("Resuming download of %s..." % desc)
+    else:
+        print("Downloading %s..." % desc)
+    dumb = (os.environ.get("TERM") == "dumb") or (not sys.stdout.isatty())
+
+    try:
+        req = urllib2.Request(src)
+        if start_byte:
+            req = urllib2.Request(src, headers={'Range': 'bytes={}-'.format(start_byte)})
+        resp = urllib2.urlopen(req)
+
+        fsize = None
+        if resp.info().getheader('Content-Length'):
+            fsize = int(resp.info().getheader('Content-Length').strip()) + start_byte
+
+        recved = start_byte
+        chunk_size = 8192
+
+        while True:
+            chunk = resp.read(chunk_size)
+            if not chunk:
+                break
+            recved += len(chunk)
+            if not dumb:
+                if fsize is not None:
+                    pct = recved * 100.0 / fsize
+                    print("\rDownloading %s: %5.1f%%" % (desc, pct), end="")
+
+                sys.stdout.flush()
+            writer.write(chunk)
+
+        if not dumb:
+            print()
+    except urllib2.HTTPError, e:
+        print("Download failed (%d): %s - %s" % (e.code, e.reason, src))
+        if e.code == 403:
+            print("No Rust compiler binary available for this platform. "
+                  "Please see https://github.com/servo/servo/#prerequisites")
+        sys.exit(1)
+    except urllib2.URLError, e:
+        print("Error downloading Rust compiler: %s. The failing URL was: %s" % (e.reason, src))
+        sys.exit(1)
+    except socket_error, e:
+        print("Looks like there's a connectivity issue, check your Internet connection. %s" % (e))
+        sys.exit(1)
+    except KeyboardInterrupt:
+        writer.flush()
+        raise
+
+
+def download_bytes(desc, src):
+    content_writer = StringIO.StringIO()
+    download(desc, src, content_writer)
+    return content_writer.getvalue()
+
+
+def download_file(desc, src, dst):
+    tmp_path = dst + ".part"
+    try:
+        start_byte = path.getsize(tmp_path)
+        with open(tmp_path, 'ab') as fd:
+            download(desc, src, fd, start_byte=start_byte)
+    except os.error:
+        with open(tmp_path, 'wb') as fd:
+            download(desc, src, fd)
+    os.rename(tmp_path, dst)
+
+
+def extract(src, dst, movedir=None):
+    if src.endswith(".zip"):
+        zipfile.ZipFile(src).extractall(dst)
+    else:
+        tarfile.open(src).extractall(dst)
+
+    if movedir:
+        for f in os.listdir(movedir):
+            frm = path.join(movedir, f)
+            to = path.join(dst, f)
+            os.rename(frm, to)
+        os.rmdir(movedir)
+
+    os.remove(src)