Auto merge of #24575 - marmeladema:issue-23607/compat, r=SimonSapin

Enable some mach commands to be run with python3

This change finally enable the following commands to be run with python3:
* `build`
* `test-unit`
* `package`

As previously explained, `test-tidy` will require more work in the wpt repository directly. Maybe `test-tidy --no-wpt` is achievable relatively quickly though.

For possible remaining bits that might need to be worked on, see https://github.com/servo/servo/issues/23607

---
<!-- Thank you for contributing to Servo! Please replace each `[ ]` by `[X]` when the step is complete, and replace `___` with appropriate data: -->
- [x] `./mach build -d` does not report any errors
- [x] `./mach test-tidy` does not report any errors

<!-- Either: -->
- [x] There are tests for these changes

<!-- Also, please make sure that "Allow edits from maintainers" checkbox is checked, so that we can help you if you get stuck somewhere along the way.-->

<!-- Pull requests that do not address these steps are welcome, but they will require additional verification as part of the review process. -->

.gitignore

@@ -10,7 +10,7 @@
 /ports/android/libs
 /ports/android/local.properties
 /ports/android/obj
-/python/_virtualenv
+/python/_virtualenv*
 /python/tidy/servo_tidy.egg-info
 /tests/wpt/sync
 *~

etc/ci/clean_build_artifacts.sh

@@ -9,4 +9,4 @@ set -o nounset
 set -o pipefail
 
 rm -rf target/
-rm -rf python/_virtualenv/
+rm -rf python/_virtualenv*/

etc/taskcluster/decision_task.py

@@ -205,13 +205,13 @@ def linux_tidy_unit():
         .with_treeherder("Linux x64", "Tidy+Unit")
         .with_script("""
             ./mach test-tidy --no-progress --all
-            ./mach build --dev
+            python3 ./mach build --dev
-            ./mach test-unit
+            python3 ./mach test-unit
-            ./mach package --dev
+            python3 ./mach package --dev
-            ./mach build --dev --features canvas2d-raqote
+            python3 ./mach build --dev --features canvas2d-raqote
-            ./mach build --dev --features layout-2020
+            python3 ./mach build --dev --features layout-2020
-            ./mach build --dev --libsimpleservo
+            python3 ./mach build --dev --libsimpleservo
-            ./mach build --dev -p servo-gst-plugin
+            python3 ./mach build --dev -p servo-gst-plugin
             ./mach test-tidy --no-progress --self-test
 
             ./etc/memory_reports_over_time.py --test

etc/taskcluster/docker/base.dockerfile

@@ -14,11 +14,17 @@ RUN \
         git \
         ca-certificates \
         #
-        # Running mach
+        # Running mach with Python 2
         python \
         python-pip \
         python-dev \
         #
+        # Running mach with Python 3
+        python3 \
+        python3-pip \
+        python3-dev \
+        virtualenv \
+        #
         # Compiling C modules when installing Python packages in a virtualenv
         gcc \
         #

mach

@@ -18,8 +18,8 @@ import sys
 # Check for the current python version as some users (especially on archlinux)
 # may not have python 2 installed and their /bin/python binary symlinked to
 # python 3.
-if sys.version_info >= (3, 0):
+if sys.version_info >= (3, 0) and sys.version_info < (3, 5):
-    print("mach does not support python 3, please install python 2")
+    print("mach does not support python 3 (< 3.5), please install python 2 or python 3 (>= 3.5)")
     sys.exit(1)
 
 

python/mach_bootstrap.py

@@ -104,13 +104,22 @@ def _process_exec(args):
             if process.returncode:
                 print('"%s" failed with error code %d:' % ('" "'.join(args), process.returncode))
 
+                if sys.version_info >= (3, 0):
+                    stdout = sys.stdout.buffer
+                else:
+                    stdout = sys.stdout
+
                 print('Output:')
                 out.seek(0)
-                shutil.copyfileobj(out, sys.stdout)
+                stdout.flush()
+                shutil.copyfileobj(out, stdout)
+                stdout.flush()
 
                 print('Error:')
                 err.seek(0)
-                shutil.copyfileobj(err, sys.stdout)
+                stdout.flush()
+                shutil.copyfileobj(err, stdout)
+                stdout.flush()
 
                 sys.exit(1)
 
@@ -145,7 +154,7 @@ def wptserve_path(is_firefox, topdir, *paths):
 
 
 def _activate_virtualenv(topdir, is_firefox):
-    virtualenv_path = os.path.join(topdir, "python", "_virtualenv")
+    virtualenv_path = os.path.join(topdir, "python", "_virtualenv%d.%d" % (sys.version_info[0], sys.version_info[1]))
     check_exec_path = lambda path: path.startswith(virtualenv_path)
     python = sys.executable   # If there was no python, mach wouldn't have run at all!
     if not python:
@@ -256,7 +265,8 @@ def bootstrap(topdir):
     # We don't support paths with Unicode characters for now
     # https://github.com/servo/servo/issues/10002
     try:
-        topdir.decode('ascii')
+        # Trick to support both python2 and python3
+        topdir.encode().decode('ascii')
     except UnicodeDecodeError:
         print('Cannot run mach in a path with Unicode characters.')
         print('Current path:', topdir)
@@ -269,10 +279,10 @@ def bootstrap(topdir):
         print('Current path:', topdir)
         sys.exit(1)
 
-    # Ensure we are running Python 2.7+. We put this check here so we generate a
+    # Ensure we are running Python 2.7+ or Python 3.5+. We put this check here so we generate a
     # user-friendly error message rather than a cryptic stack trace on module import.
-    if not (3, 0) > sys.version_info >= (2, 7):
+    if sys.version_info < (2, 7) or (sys.version_info >= (3, 0) and sys.version_info < (3, 5)):
-        print('Python 2.7 or above (but not Python 3) is required to run mach.')
+        print('Python2 (>=2.7) or Python3 (>=3.5) is required to run mach.')
         print('You are running Python', platform.python_version())
         sys.exit(1)
 

python/requirements.txt

@@ -3,7 +3,7 @@
 
 blessings == 1.6
 distro == 1.4
-mach == 0.6.0
+mach == 1.0.0
 mozdebug == 0.1
 mozinfo == 0.8
 mozlog == 3.6

python/servo/command_base.py

@@ -27,6 +27,7 @@ import six
 import sys
 import tarfile
 import zipfile
+import functools
 from xml.etree.ElementTree import XML
 from servo.util import download_file
 import six.moves.urllib as urllib
@@ -103,15 +104,15 @@ def archive_deterministically(dir_to_archive, dest_archive, prepend_path=None):
 
         # Sort file entries with the fixed locale
         with setlocale('C'):
-            file_list.sort(cmp=locale.strcoll)
+            file_list.sort(key=functools.cmp_to_key(locale.strcoll))
 
         # Use a temporary file and atomic rename to avoid partially-formed
         # packaging (in case of exceptional situations like running out of disk space).
         # TODO do this in a temporary folder after #11983 is fixed
         temp_file = '{}.temp~'.format(dest_archive)
-        with os.fdopen(os.open(temp_file, os.O_WRONLY | os.O_CREAT, 0o644), 'w') as out_file:
+        with os.fdopen(os.open(temp_file, os.O_WRONLY | os.O_CREAT, 0o644), 'wb') as out_file:
             if dest_archive.endswith('.zip'):
-                with zipfile.ZipFile(temp_file, 'w', zipfile.ZIP_DEFLATED) as zip_file:
+                with zipfile.ZipFile(out_file, 'w', zipfile.ZIP_DEFLATED) as zip_file:
                     for entry in file_list:
                         arcname = entry
                         if prepend_path is not None:
@@ -257,7 +258,7 @@ def gstreamer_root(target, env, topdir=None):
             return env.get(gst_env)
         elif os.path.exists(path.join(gst_default_path, "bin", "ffi-7.dll")):
             return gst_default_path
-    elif sys.platform == "linux2":
+    elif is_linux():
         return path.join(topdir, "support", "linux", "gstreamer", "gst")
     return None
 
@@ -579,7 +580,7 @@ class CommandBase(object):
         if "x86_64" not in effective_target or "android" in effective_target:
             # We don't build gstreamer for non-x86_64 / android yet
             return False
-        if sys.platform == "linux2" or is_windows():
+        if is_linux() or is_windows():
             if path.isdir(gstreamer_root(effective_target, env, self.get_top_dir())):
                 return True
             else:
@@ -689,7 +690,7 @@ install them, let us know by filing a bug!")
             extra_lib = [libpath] + extra_lib
             append_to_path_env(path.join(libpath, "pkgconfig"), env, "PKG_CONFIG_PATH")
 
-        if sys.platform == "linux2":
+        if is_linux():
             distrib, version, _ = distro.linux_distribution()
             distrib = six.ensure_str(distrib)
             version = six.ensure_str(version)

python/tidy/servo_tidy/tidy.py

@@ -10,6 +10,7 @@
 from __future__ import print_function
 
 import fnmatch
+import glob
 import imp
 import itertools
 import json
@@ -19,6 +20,7 @@ import subprocess
 import sys
 
 import colorama
+import six
 import toml
 import voluptuous
 import yaml
@@ -116,7 +118,7 @@ def is_iter_empty(iterator):
 
 
 def normilize_paths(paths):
-    if isinstance(paths, basestring):
+    if isinstance(paths, six.string_types):
         return os.path.join(*paths.split('/'))
     else:
         return [os.path.join(*path.split('/')) for path in paths]
@@ -909,7 +911,7 @@ def check_config_file(config_file, print_text=True, no_wpt=False):
     exclude = config_content.get("ignore", {})
 
     # Check for invalid listed ignored directories
-    exclude_dirs = exclude.get("directories", [])
+    exclude_dirs = [d for p in exclude.get("directories", []) for d in (glob.glob(p) or [p])]
     skip_dirs = ["./target", "./tests"]
     invalid_dirs = [d for d in exclude_dirs if not os.path.isdir(d) and not any(s in d for s in skip_dirs)]
 
@@ -971,7 +973,8 @@ def check_config_file(config_file, print_text=True, no_wpt=False):
 def parse_config(config_file):
     exclude = config_file.get("ignore", {})
     # Add list of ignored directories to config
-    config["ignore"]["directories"] += normilize_paths(exclude.get("directories", []))
+    ignored_directories = [d for p in exclude.get("directories", []) for d in (glob.glob(p) or [p])]
+    config["ignore"]["directories"] += normilize_paths(ignored_directories)
     # Add list of ignored files to config
     config["ignore"]["files"] += normilize_paths(exclude.get("files", []))
     # Add list of ignored packages to config

servo-tidy.toml

@@ -91,7 +91,7 @@ directories = [
   "./python/tidy/servo_tidy_tests",
   "./components/script/dom/bindings/codegen/parser",
   "./components/script/dom/bindings/codegen/ply",
-  "./python/_virtualenv",
+  "./python/_virtualenv*",
   "./components/hashglobe/src",
   # Generated and upstream code combined with our own. Could use cleanup
   "./target",