From 00cf7452ef2bcb272dafb09a711301a2ef280ae2 Mon Sep 17 00:00:00 2001 From: marmeladema Date: Tue, 22 Oct 2019 11:59:10 +0100 Subject: [PATCH 01/12] Generate per python version virtualenv --- etc/ci/clean_build_artifacts.sh | 2 +- python/mach_bootstrap.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/ci/clean_build_artifacts.sh b/etc/ci/clean_build_artifacts.sh index cacaf96caae..f6206485ca4 100755 --- a/etc/ci/clean_build_artifacts.sh +++ b/etc/ci/clean_build_artifacts.sh @@ -9,4 +9,4 @@ set -o nounset set -o pipefail rm -rf target/ -rm -rf python/_virtualenv/ +rm -rf python/_virtualenv*/ diff --git a/python/mach_bootstrap.py b/python/mach_bootstrap.py index 2c83c6492e7..80b71a2bfeb 100644 --- a/python/mach_bootstrap.py +++ b/python/mach_bootstrap.py @@ -145,7 +145,7 @@ def wptserve_path(is_firefox, topdir, *paths): def _activate_virtualenv(topdir, is_firefox): - virtualenv_path = os.path.join(topdir, "python", "_virtualenv") + virtualenv_path = os.path.join(topdir, "python", "_virtualenv%d.%d" % (sys.version_info[0], sys.version_info[1])) check_exec_path = lambda path: path.startswith(virtualenv_path) python = sys.executable # If there was no python, mach wouldn't have run at all! if not python: From 76ff22f5d0ac7b36230ac3d7e9599167b0f6349a Mon Sep 17 00:00:00 2001 From: marmeladema Date: Tue, 22 Oct 2019 22:21:16 +0100 Subject: [PATCH 02/12] Allow to specify glob pattern for ignored directories in servo-tidy.toml --- python/tidy/servo_tidy/tidy.py | 6 ++++-- servo-tidy.toml | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/python/tidy/servo_tidy/tidy.py b/python/tidy/servo_tidy/tidy.py index d2840cc884c..899bb60e775 100644 --- a/python/tidy/servo_tidy/tidy.py +++ b/python/tidy/servo_tidy/tidy.py @@ -10,6 +10,7 @@ from __future__ import print_function import fnmatch +import glob import imp import itertools import json @@ -909,7 +910,7 @@ def check_config_file(config_file, print_text=True, no_wpt=False): exclude = config_content.get("ignore", {}) # Check for invalid listed ignored directories - exclude_dirs = exclude.get("directories", []) + exclude_dirs = [d for p in exclude.get("directories", []) for d in (glob.glob(p) or [p])] skip_dirs = ["./target", "./tests"] invalid_dirs = [d for d in exclude_dirs if not os.path.isdir(d) and not any(s in d for s in skip_dirs)] @@ -971,7 +972,8 @@ def check_config_file(config_file, print_text=True, no_wpt=False): def parse_config(config_file): exclude = config_file.get("ignore", {}) # Add list of ignored directories to config - config["ignore"]["directories"] += normilize_paths(exclude.get("directories", [])) + ignored_directories = [d for p in exclude.get("directories", []) for d in (glob.glob(p) or [p])] + config["ignore"]["directories"] += normilize_paths(ignored_directories) # Add list of ignored files to config config["ignore"]["files"] += normilize_paths(exclude.get("files", [])) # Add list of ignored packages to config diff --git a/servo-tidy.toml b/servo-tidy.toml index 71f9fb08e36..c35e180c260 100644 --- a/servo-tidy.toml +++ b/servo-tidy.toml @@ -91,7 +91,7 @@ directories = [ "./python/tidy/servo_tidy_tests", "./components/script/dom/bindings/codegen/parser", "./components/script/dom/bindings/codegen/ply", - "./python/_virtualenv", + "./python/_virtualenv*", "./components/hashglobe/src", # Generated and upstream code combined with our own. Could use cleanup "./target", From 01c1a6c4c98c946e925f02ea779849ee5a86a886 Mon Sep 17 00:00:00 2001 From: marmeladema Date: Tue, 22 Oct 2019 22:50:40 +0100 Subject: [PATCH 03/12] Update mach to version 1.0.0 --- python/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/requirements.txt b/python/requirements.txt index e0e4bb40cc0..79c2374f24e 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -3,7 +3,7 @@ blessings == 1.6 distro == 1.4 -mach == 0.6.0 +mach == 1.0.0 mozdebug == 0.1 mozinfo == 0.8 mozlog == 3.6 From 67c2c115c00e0073e2e13d29becf14363ed6f6b9 Mon Sep 17 00:00:00 2001 From: marmeladema Date: Tue, 22 Oct 2019 23:01:35 +0100 Subject: [PATCH 04/12] Allow to run mach with Python3 (>=3.5) --- mach | 4 ++-- python/mach_bootstrap.py | 9 +++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/mach b/mach index c92653e1af8..317374d6c27 100755 --- a/mach +++ b/mach @@ -18,8 +18,8 @@ import sys # Check for the current python version as some users (especially on archlinux) # may not have python 2 installed and their /bin/python binary symlinked to # python 3. -if sys.version_info >= (3, 0): - print("mach does not support python 3, please install python 2") +if sys.version_info >= (3, 0) and sys.version_info < (3, 5): + print("mach does not support python 3 (< 3.5), please install python 2 or python 3 (>= 3.5)") sys.exit(1) diff --git a/python/mach_bootstrap.py b/python/mach_bootstrap.py index 80b71a2bfeb..5dd1077c923 100644 --- a/python/mach_bootstrap.py +++ b/python/mach_bootstrap.py @@ -256,7 +256,8 @@ def bootstrap(topdir): # We don't support paths with Unicode characters for now # https://github.com/servo/servo/issues/10002 try: - topdir.decode('ascii') + # Trick to support both python2 and python3 + topdir.encode().decode('ascii') except UnicodeDecodeError: print('Cannot run mach in a path with Unicode characters.') print('Current path:', topdir) @@ -269,10 +270,10 @@ def bootstrap(topdir): print('Current path:', topdir) sys.exit(1) - # Ensure we are running Python 2.7+. We put this check here so we generate a + # Ensure we are running Python 2.7+ or Python 3.5+. We put this check here so we generate a # user-friendly error message rather than a cryptic stack trace on module import. - if not (3, 0) > sys.version_info >= (2, 7): - print('Python 2.7 or above (but not Python 3) is required to run mach.') + if sys.version_info < (2, 7) or (sys.version_info >= (3, 0) and sys.version_info < (3, 5)): + print('Python2 (>=2.7) or Python3 (>=3.5) is required to run mach.') print('You are running Python', platform.python_version()) sys.exit(1) From 3f48f2863cf21e3584c908d653e15521aae3ed5c Mon Sep 17 00:00:00 2001 From: marmeladema Date: Mon, 28 Oct 2019 23:51:10 +0000 Subject: [PATCH 05/12] Convert cmp= to key= in sort method call for Python3 compatibility --- python/servo/command_base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/servo/command_base.py b/python/servo/command_base.py index 7473b78844d..7184010442c 100644 --- a/python/servo/command_base.py +++ b/python/servo/command_base.py @@ -27,6 +27,7 @@ import six import sys import tarfile import zipfile +import functools from xml.etree.ElementTree import XML from servo.util import download_file import six.moves.urllib as urllib @@ -103,7 +104,7 @@ def archive_deterministically(dir_to_archive, dest_archive, prepend_path=None): # Sort file entries with the fixed locale with setlocale('C'): - file_list.sort(cmp=locale.strcoll) + file_list.sort(key=functools.cmp_to_key(locale.strcoll)) # Use a temporary file and atomic rename to avoid partially-formed # packaging (in case of exceptional situations like running out of disk space). From 267eaad4ebcc26337956ae298babb98694554a1c Mon Sep 17 00:00:00 2001 From: marmeladema Date: Mon, 28 Oct 2019 23:51:42 +0000 Subject: [PATCH 06/12] Properly open file as 'wb' in archive_deterministically --- python/servo/command_base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/servo/command_base.py b/python/servo/command_base.py index 7184010442c..9dfa6795d12 100644 --- a/python/servo/command_base.py +++ b/python/servo/command_base.py @@ -110,9 +110,9 @@ def archive_deterministically(dir_to_archive, dest_archive, prepend_path=None): # packaging (in case of exceptional situations like running out of disk space). # TODO do this in a temporary folder after #11983 is fixed temp_file = '{}.temp~'.format(dest_archive) - with os.fdopen(os.open(temp_file, os.O_WRONLY | os.O_CREAT, 0o644), 'w') as out_file: + with os.fdopen(os.open(temp_file, os.O_WRONLY | os.O_CREAT, 0o644), 'wb') as out_file: if dest_archive.endswith('.zip'): - with zipfile.ZipFile(temp_file, 'w', zipfile.ZIP_DEFLATED) as zip_file: + with zipfile.ZipFile(out_file, 'w', zipfile.ZIP_DEFLATED) as zip_file: for entry in file_list: arcname = entry if prepend_path is not None: From 176e63284e60208a4cc627174f63f7a033d67976 Mon Sep 17 00:00:00 2001 From: marmeladema Date: Tue, 29 Oct 2019 00:01:53 +0000 Subject: [PATCH 07/12] Replace usage of basestring by six.string_types for Python3 compatibility --- python/tidy/servo_tidy/tidy.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/tidy/servo_tidy/tidy.py b/python/tidy/servo_tidy/tidy.py index 899bb60e775..a3c1c0a9a01 100644 --- a/python/tidy/servo_tidy/tidy.py +++ b/python/tidy/servo_tidy/tidy.py @@ -20,6 +20,7 @@ import subprocess import sys import colorama +import six import toml import voluptuous import yaml @@ -117,7 +118,7 @@ def is_iter_empty(iterator): def normilize_paths(paths): - if isinstance(paths, basestring): + if isinstance(paths, six.string_types): return os.path.join(*paths.split('/')) else: return [os.path.join(*path.split('/')) for path in paths] From 64e9c0ee230fc0ff9a64a123bb8d612933d87e45 Mon Sep 17 00:00:00 2001 From: marmeladema Date: Tue, 29 Oct 2019 00:02:27 +0000 Subject: [PATCH 08/12] taskcluster: change some mach commands to use python3 under linux --- etc/taskcluster/decision_task.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/etc/taskcluster/decision_task.py b/etc/taskcluster/decision_task.py index 7853a84c2f6..b3fee637ce6 100644 --- a/etc/taskcluster/decision_task.py +++ b/etc/taskcluster/decision_task.py @@ -205,13 +205,13 @@ def linux_tidy_unit(): .with_treeherder("Linux x64", "Tidy+Unit") .with_script(""" ./mach test-tidy --no-progress --all - ./mach build --dev - ./mach test-unit - ./mach package --dev - ./mach build --dev --features canvas2d-raqote - ./mach build --dev --features layout-2020 - ./mach build --dev --libsimpleservo - ./mach build --dev -p servo-gst-plugin + python3 ./mach build --dev + python3 ./mach test-unit + python3 ./mach package --dev + python3 ./mach build --dev --features canvas2d-raqote + python3 ./mach build --dev --features layout-2020 + python3 ./mach build --dev --libsimpleservo + python3 ./mach build --dev -p servo-gst-plugin ./mach test-tidy --no-progress --self-test ./etc/memory_reports_over_time.py --test From 2bacf69d1588f9f06744e02b0ec0f4e4e60097da Mon Sep 17 00:00:00 2001 From: Simon Sapin Date: Thu, 14 Nov 2019 17:12:06 +0100 Subject: [PATCH 09/12] Taskcluster: install packages for Python 3 in Docker tasks --- etc/taskcluster/docker/base.dockerfile | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/etc/taskcluster/docker/base.dockerfile b/etc/taskcluster/docker/base.dockerfile index abfa98fda41..568741185cf 100644 --- a/etc/taskcluster/docker/base.dockerfile +++ b/etc/taskcluster/docker/base.dockerfile @@ -14,11 +14,17 @@ RUN \ git \ ca-certificates \ # - # Running mach + # Running mach with Python 2 python \ python-pip \ python-dev \ # + # Running mach with Python 3 + python3 \ + python3-pip \ + python3-dev \ + virtualenv \ + # # Compiling C modules when installing Python packages in a virtualenv gcc \ # From 63d5d8a4b607f291804f6fae62c1ec0433cc0051 Mon Sep 17 00:00:00 2001 From: Simon Sapin Date: Thu, 14 Nov 2019 17:15:28 +0100 Subject: [PATCH 10/12] .gitignore all virtualenvs --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 2c335f4bdbf..f778f67ea83 100644 --- a/.gitignore +++ b/.gitignore @@ -10,7 +10,7 @@ /ports/android/libs /ports/android/local.properties /ports/android/obj -/python/_virtualenv +/python/_virtualenv* /python/tidy/servo_tidy.egg-info /tests/wpt/sync *~ From 17233f2dd298a1ed6933e45870c3cb6878456cef Mon Sep 17 00:00:00 2001 From: marmeladema Date: Mon, 9 Dec 2019 21:17:34 +0000 Subject: [PATCH 11/12] Write subprocess stdout/stderr logs to raw stdout buffer when using Python3 --- python/mach_bootstrap.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/python/mach_bootstrap.py b/python/mach_bootstrap.py index 5dd1077c923..049dff4b7e7 100644 --- a/python/mach_bootstrap.py +++ b/python/mach_bootstrap.py @@ -104,13 +104,22 @@ def _process_exec(args): if process.returncode: print('"%s" failed with error code %d:' % ('" "'.join(args), process.returncode)) + if sys.version_info >= (3, 0): + stdout = sys.stdout.buffer + else: + stdout = sys.stdout + print('Output:') out.seek(0) - shutil.copyfileobj(out, sys.stdout) + stdout.flush() + shutil.copyfileobj(out, stdout) + stdout.flush() print('Error:') err.seek(0) - shutil.copyfileobj(err, sys.stdout) + stdout.flush() + shutil.copyfileobj(err, stdout) + stdout.flush() sys.exit(1) From 02c55e2ecaa110120ac228c59a8778b19ef494d5 Mon Sep 17 00:00:00 2001 From: marmeladema Date: Mon, 9 Dec 2019 23:01:34 +0000 Subject: [PATCH 12/12] Use is_linux to check for current platform instead of relying on sys.platform --- python/servo/command_base.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/servo/command_base.py b/python/servo/command_base.py index 9dfa6795d12..764afc9a15a 100644 --- a/python/servo/command_base.py +++ b/python/servo/command_base.py @@ -258,7 +258,7 @@ def gstreamer_root(target, env, topdir=None): return env.get(gst_env) elif os.path.exists(path.join(gst_default_path, "bin", "ffi-7.dll")): return gst_default_path - elif sys.platform == "linux2": + elif is_linux(): return path.join(topdir, "support", "linux", "gstreamer", "gst") return None @@ -580,7 +580,7 @@ class CommandBase(object): if "x86_64" not in effective_target or "android" in effective_target: # We don't build gstreamer for non-x86_64 / android yet return False - if sys.platform == "linux2" or is_windows(): + if is_linux() or is_windows(): if path.isdir(gstreamer_root(effective_target, env, self.get_top_dir())): return True else: @@ -690,7 +690,7 @@ install them, let us know by filing a bug!") extra_lib = [libpath] + extra_lib append_to_path_env(path.join(libpath, "pkgconfig"), env, "PKG_CONFIG_PATH") - if sys.platform == "linux2": + if is_linux(): distrib, version, _ = distro.linux_distribution() distrib = six.ensure_str(distrib) version = six.ensure_str(version)