diff --git a/.gitignore b/.gitignore index 2c335f4bdbf..f778f67ea83 100644 --- a/.gitignore +++ b/.gitignore @@ -10,7 +10,7 @@ /ports/android/libs /ports/android/local.properties /ports/android/obj -/python/_virtualenv +/python/_virtualenv* /python/tidy/servo_tidy.egg-info /tests/wpt/sync *~ diff --git a/etc/ci/clean_build_artifacts.sh b/etc/ci/clean_build_artifacts.sh index cacaf96caae..f6206485ca4 100755 --- a/etc/ci/clean_build_artifacts.sh +++ b/etc/ci/clean_build_artifacts.sh @@ -9,4 +9,4 @@ set -o nounset set -o pipefail rm -rf target/ -rm -rf python/_virtualenv/ +rm -rf python/_virtualenv*/ diff --git a/etc/taskcluster/decision_task.py b/etc/taskcluster/decision_task.py index 21789d2442b..7dbc9202005 100644 --- a/etc/taskcluster/decision_task.py +++ b/etc/taskcluster/decision_task.py @@ -205,13 +205,13 @@ def linux_tidy_unit(): .with_treeherder("Linux x64", "Tidy+Unit") .with_script(""" ./mach test-tidy --no-progress --all - ./mach build --dev - ./mach test-unit - ./mach package --dev - ./mach build --dev --features canvas2d-raqote - ./mach build --dev --features layout-2020 - ./mach build --dev --libsimpleservo - ./mach build --dev -p servo-gst-plugin + python3 ./mach build --dev + python3 ./mach test-unit + python3 ./mach package --dev + python3 ./mach build --dev --features canvas2d-raqote + python3 ./mach build --dev --features layout-2020 + python3 ./mach build --dev --libsimpleservo + python3 ./mach build --dev -p servo-gst-plugin ./mach test-tidy --no-progress --self-test ./etc/memory_reports_over_time.py --test diff --git a/etc/taskcluster/docker/base.dockerfile b/etc/taskcluster/docker/base.dockerfile index abfa98fda41..568741185cf 100644 --- a/etc/taskcluster/docker/base.dockerfile +++ b/etc/taskcluster/docker/base.dockerfile @@ -14,11 +14,17 @@ RUN \ git \ ca-certificates \ # - # Running mach + # Running mach with Python 2 python \ python-pip \ python-dev \ # + # Running mach with Python 3 + python3 \ + python3-pip \ + python3-dev \ + virtualenv \ + # # Compiling C modules when installing Python packages in a virtualenv gcc \ # diff --git a/mach b/mach index c92653e1af8..317374d6c27 100755 --- a/mach +++ b/mach @@ -18,8 +18,8 @@ import sys # Check for the current python version as some users (especially on archlinux) # may not have python 2 installed and their /bin/python binary symlinked to # python 3. -if sys.version_info >= (3, 0): - print("mach does not support python 3, please install python 2") +if sys.version_info >= (3, 0) and sys.version_info < (3, 5): + print("mach does not support python 3 (< 3.5), please install python 2 or python 3 (>= 3.5)") sys.exit(1) diff --git a/python/mach_bootstrap.py b/python/mach_bootstrap.py index 2c83c6492e7..049dff4b7e7 100644 --- a/python/mach_bootstrap.py +++ b/python/mach_bootstrap.py @@ -104,13 +104,22 @@ def _process_exec(args): if process.returncode: print('"%s" failed with error code %d:' % ('" "'.join(args), process.returncode)) + if sys.version_info >= (3, 0): + stdout = sys.stdout.buffer + else: + stdout = sys.stdout + print('Output:') out.seek(0) - shutil.copyfileobj(out, sys.stdout) + stdout.flush() + shutil.copyfileobj(out, stdout) + stdout.flush() print('Error:') err.seek(0) - shutil.copyfileobj(err, sys.stdout) + stdout.flush() + shutil.copyfileobj(err, stdout) + stdout.flush() sys.exit(1) @@ -145,7 +154,7 @@ def wptserve_path(is_firefox, topdir, *paths): def _activate_virtualenv(topdir, is_firefox): - virtualenv_path = os.path.join(topdir, "python", "_virtualenv") + virtualenv_path = os.path.join(topdir, "python", "_virtualenv%d.%d" % (sys.version_info[0], sys.version_info[1])) check_exec_path = lambda path: path.startswith(virtualenv_path) python = sys.executable # If there was no python, mach wouldn't have run at all! if not python: @@ -256,7 +265,8 @@ def bootstrap(topdir): # We don't support paths with Unicode characters for now # https://github.com/servo/servo/issues/10002 try: - topdir.decode('ascii') + # Trick to support both python2 and python3 + topdir.encode().decode('ascii') except UnicodeDecodeError: print('Cannot run mach in a path with Unicode characters.') print('Current path:', topdir) @@ -269,10 +279,10 @@ def bootstrap(topdir): print('Current path:', topdir) sys.exit(1) - # Ensure we are running Python 2.7+. We put this check here so we generate a + # Ensure we are running Python 2.7+ or Python 3.5+. We put this check here so we generate a # user-friendly error message rather than a cryptic stack trace on module import. - if not (3, 0) > sys.version_info >= (2, 7): - print('Python 2.7 or above (but not Python 3) is required to run mach.') + if sys.version_info < (2, 7) or (sys.version_info >= (3, 0) and sys.version_info < (3, 5)): + print('Python2 (>=2.7) or Python3 (>=3.5) is required to run mach.') print('You are running Python', platform.python_version()) sys.exit(1) diff --git a/python/requirements.txt b/python/requirements.txt index e0e4bb40cc0..79c2374f24e 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -3,7 +3,7 @@ blessings == 1.6 distro == 1.4 -mach == 0.6.0 +mach == 1.0.0 mozdebug == 0.1 mozinfo == 0.8 mozlog == 3.6 diff --git a/python/servo/command_base.py b/python/servo/command_base.py index 7473b78844d..764afc9a15a 100644 --- a/python/servo/command_base.py +++ b/python/servo/command_base.py @@ -27,6 +27,7 @@ import six import sys import tarfile import zipfile +import functools from xml.etree.ElementTree import XML from servo.util import download_file import six.moves.urllib as urllib @@ -103,15 +104,15 @@ def archive_deterministically(dir_to_archive, dest_archive, prepend_path=None): # Sort file entries with the fixed locale with setlocale('C'): - file_list.sort(cmp=locale.strcoll) + file_list.sort(key=functools.cmp_to_key(locale.strcoll)) # Use a temporary file and atomic rename to avoid partially-formed # packaging (in case of exceptional situations like running out of disk space). # TODO do this in a temporary folder after #11983 is fixed temp_file = '{}.temp~'.format(dest_archive) - with os.fdopen(os.open(temp_file, os.O_WRONLY | os.O_CREAT, 0o644), 'w') as out_file: + with os.fdopen(os.open(temp_file, os.O_WRONLY | os.O_CREAT, 0o644), 'wb') as out_file: if dest_archive.endswith('.zip'): - with zipfile.ZipFile(temp_file, 'w', zipfile.ZIP_DEFLATED) as zip_file: + with zipfile.ZipFile(out_file, 'w', zipfile.ZIP_DEFLATED) as zip_file: for entry in file_list: arcname = entry if prepend_path is not None: @@ -257,7 +258,7 @@ def gstreamer_root(target, env, topdir=None): return env.get(gst_env) elif os.path.exists(path.join(gst_default_path, "bin", "ffi-7.dll")): return gst_default_path - elif sys.platform == "linux2": + elif is_linux(): return path.join(topdir, "support", "linux", "gstreamer", "gst") return None @@ -579,7 +580,7 @@ class CommandBase(object): if "x86_64" not in effective_target or "android" in effective_target: # We don't build gstreamer for non-x86_64 / android yet return False - if sys.platform == "linux2" or is_windows(): + if is_linux() or is_windows(): if path.isdir(gstreamer_root(effective_target, env, self.get_top_dir())): return True else: @@ -689,7 +690,7 @@ install them, let us know by filing a bug!") extra_lib = [libpath] + extra_lib append_to_path_env(path.join(libpath, "pkgconfig"), env, "PKG_CONFIG_PATH") - if sys.platform == "linux2": + if is_linux(): distrib, version, _ = distro.linux_distribution() distrib = six.ensure_str(distrib) version = six.ensure_str(version) diff --git a/python/tidy/servo_tidy/tidy.py b/python/tidy/servo_tidy/tidy.py index d2840cc884c..a3c1c0a9a01 100644 --- a/python/tidy/servo_tidy/tidy.py +++ b/python/tidy/servo_tidy/tidy.py @@ -10,6 +10,7 @@ from __future__ import print_function import fnmatch +import glob import imp import itertools import json @@ -19,6 +20,7 @@ import subprocess import sys import colorama +import six import toml import voluptuous import yaml @@ -116,7 +118,7 @@ def is_iter_empty(iterator): def normilize_paths(paths): - if isinstance(paths, basestring): + if isinstance(paths, six.string_types): return os.path.join(*paths.split('/')) else: return [os.path.join(*path.split('/')) for path in paths] @@ -909,7 +911,7 @@ def check_config_file(config_file, print_text=True, no_wpt=False): exclude = config_content.get("ignore", {}) # Check for invalid listed ignored directories - exclude_dirs = exclude.get("directories", []) + exclude_dirs = [d for p in exclude.get("directories", []) for d in (glob.glob(p) or [p])] skip_dirs = ["./target", "./tests"] invalid_dirs = [d for d in exclude_dirs if not os.path.isdir(d) and not any(s in d for s in skip_dirs)] @@ -971,7 +973,8 @@ def check_config_file(config_file, print_text=True, no_wpt=False): def parse_config(config_file): exclude = config_file.get("ignore", {}) # Add list of ignored directories to config - config["ignore"]["directories"] += normilize_paths(exclude.get("directories", [])) + ignored_directories = [d for p in exclude.get("directories", []) for d in (glob.glob(p) or [p])] + config["ignore"]["directories"] += normilize_paths(ignored_directories) # Add list of ignored files to config config["ignore"]["files"] += normilize_paths(exclude.get("files", [])) # Add list of ignored packages to config diff --git a/servo-tidy.toml b/servo-tidy.toml index 71f9fb08e36..c35e180c260 100644 --- a/servo-tidy.toml +++ b/servo-tidy.toml @@ -91,7 +91,7 @@ directories = [ "./python/tidy/servo_tidy_tests", "./components/script/dom/bindings/codegen/parser", "./components/script/dom/bindings/codegen/ply", - "./python/_virtualenv", + "./python/_virtualenv*", "./components/hashglobe/src", # Generated and upstream code combined with our own. Could use cleanup "./target",