From 3381f2a70442aa6a6c31a0bc4a4c3601299631f5 Mon Sep 17 00:00:00 2001 From: Jonathan Schwender <55576758+jschwe@users.noreply.github.com> Date: Mon, 17 Jun 2024 13:27:23 +0200 Subject: [PATCH] Add OpenHarmony support to mach and CI (#32507) * Add ohos to mach Signed-off-by: Jonathan Schwender * Add OpenHarmony build to CI * Rename ohos sdk action I decided to rename the upstream ohos sdk action to setup-ohos-sdk, making it clearer that is a github action repository. Signed-off-by: Jonathan Schwender * Remove commented line Signed-off-by: Jonathan Schwender --------- Signed-off-by: Jonathan Schwender Signed-off-by: Jonathan Schwender --- .github/workflows/dispatch-workflow.yml | 7 ++ .github/workflows/main.yml | 8 ++ .github/workflows/ohos.yml | 72 ++++++++++++ python/servo/command_base.py | 145 ++++++++++++++++++++++++ python/servo/try_parser.py | 15 ++- servobuild.example | 5 + 6 files changed, 250 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/ohos.yml diff --git a/.github/workflows/dispatch-workflow.yml b/.github/workflows/dispatch-workflow.yml index 89671f5656c..10da49baf05 100644 --- a/.github/workflows/dispatch-workflow.yml +++ b/.github/workflows/dispatch-workflow.yml @@ -57,3 +57,10 @@ jobs: secrets: inherit with: profile: ${{ inputs.profile }} + + ohos: + if: ${{ inputs.workflow == 'ohos' }} + name: OpenHarmony + uses: ./.github/workflows/ohos.yml + with: + profile: ${{ inputs.profile }} diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 20ec16e5ea4..84073d1c3a8 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -46,6 +46,13 @@ jobs: profile: "release" secrets: inherit + build-ohos: + name: OpenHarmony + if: ${{ github.event_name != 'pull_request' }} + uses: ./.github/workflows/ohos.yml + with: + profile: "release" + build-result: name: Result runs-on: ubuntu-latest @@ -56,6 +63,7 @@ jobs: - "build-mac" - "build-linux" - "build-android" + - "build-ohos" steps: - name: Merge build timings uses: actions/upload-artifact/merge@v4 diff --git a/.github/workflows/ohos.yml b/.github/workflows/ohos.yml new file mode 100644 index 00000000000..c5722bb9f40 --- /dev/null +++ b/.github/workflows/ohos.yml @@ -0,0 +1,72 @@ +name: OpenHarmony +on: + workflow_call: + inputs: + profile: + required: false + default: "release" + type: string + workflow_dispatch: + inputs: + profile: + required: false + default: "release" + type: choice + description: "Cargo build profile" + options: [ "release", "debug", "production"] + +env: + RUST_BACKTRACE: 1 + SHELL: /bin/bash + SCCACHE_GHA_ENABLED: "true" + RUSTC_WRAPPER: "sccache" + CCACHE: "sccache" + CARGO_INCREMENTAL: 0 + +jobs: + build: + name: OpenHarmony Build + runs-on: ubuntu-22.04 + strategy: + matrix: + arch: ['aarch64-unknown-linux-ohos'] + steps: + - uses: actions/checkout@v4 + if: github.event_name != 'issue_comment' && github.event_name != 'pull_request_target' + with: + fetch-depth: 2 + # This is necessary to checkout the pull request if this run was triggered + # via an `issue_comment` action on a pull request. + - uses: actions/checkout@v4 + if: github.event_name == 'issue_comment' || github.event_name == 'pull_request_target' + with: + ref: refs/pull/${{ github.event.issue.number || github.event.number }}/head + fetch-depth: 2 + - name: Run sccache-cache + uses: mozilla-actions/sccache-action@v0.0.4 + - name: Install taplo + uses: baptiste0928/cargo-install@v3 + with: + crate: taplo-cli + locked: true + - name: Bootstrap Python + run: python3 -m pip install --upgrade pip virtualenv + - name: Bootstrap dependencies + run: sudo apt update && python3 ./mach bootstrap + - name: Setup OpenHarmony SDK + id: setup_sdk + uses: openharmony-rs/setup-ohos-sdk@v0.1 + with: + version: "4.1" + - name: Build (arch ${{ matrix.arch }} profile ${{ inputs.profile }}) + env: + OHOS_SDK_NATIVE: ${{ steps.setup_sdk.outputs.ohos_sdk_native }} + run: | + python3 ./mach build --locked --target ${{ matrix.arch }} --${{ inputs.profile }} + cp -r target/cargo-timings target/cargo-timings-ohos-${{ matrix.arch }} + - name: Archive build timing + uses: actions/upload-artifact@v4 + with: + name: cargo-timings-ohos-${{ matrix.arch }} + # Using a wildcard here ensures that the archive includes the path. + path: target/cargo-timings-* diff --git a/python/servo/command_base.py b/python/servo/command_base.py index 2fd48512eb0..74ee2f475b4 100644 --- a/python/servo/command_base.py +++ b/python/servo/command_base.py @@ -10,6 +10,9 @@ from __future__ import annotations import contextlib +import errno +import json +import pathlib from enum import Enum from typing import Dict, List, Optional import functools @@ -32,6 +35,7 @@ from glob import glob from os import path from subprocess import PIPE from xml.etree.ElementTree import XML +from packaging.version import parse as parse_version import toml @@ -310,6 +314,9 @@ class CommandBase(object): self.config["android"].setdefault("ndk", "") self.config["android"].setdefault("toolchain", "") + self.config.setdefault("ohos", {}) + self.config["ohos"].setdefault("ndk", "") + # Set default android target self.setup_configuration_for_android_target("armv7-linux-androideabi") @@ -343,6 +350,9 @@ class CommandBase(object): elif target: base_path = path.join(base_path, target) + if target is not None and "-ohos" in target: + return path.join(base_path, build_type.directory_name(), "libservoshell.so") + binary_name = f"servo{servo.platform.get().executable_suffix()}" binary_path = path.join(base_path, build_type.directory_name(), binary_name) @@ -524,6 +534,7 @@ class CommandBase(object): env["LSAN_OPTIONS"] = f"{env.get('LSAN_OPTIONS', '')}:suppressions={ASAN_LEAK_SUPPRESSION_FILE}" self.build_android_env_if_needed(env) + self.build_ohos_env_if_needed(env) return env @@ -658,6 +669,134 @@ class CommandBase(object): env['PKG_CONFIG_SYSROOT_DIR'] = path.join(llvm_toolchain, 'sysroot') + def build_ohos_env_if_needed(self, env: Dict[str, str]): + if not (self.cross_compile_target and self.cross_compile_target.endswith('-ohos')): + return + + # Paths to OpenHarmony SDK and build tools: + # Note: `OHOS_SDK_NATIVE` is the CMake variable name the `hvigor` build-system + # uses for the native directory of the SDK, so we use the same name to be consistent. + if "OHOS_SDK_NATIVE" not in env and self.config["ohos"]["ndk"]: + env["OHOS_SDK_NATIVE"] = self.config["ohos"]["ndk"] + + if "OHOS_SDK_NATIVE" not in env: + print("Please set the OHOS_SDK_NATIVE environment variable to the location of the `native` directory " + "in the OpenHarmony SDK.") + sys.exit(1) + + ndk_root = pathlib.Path(env["OHOS_SDK_NATIVE"]) + + if not ndk_root.is_dir(): + print(f"OHOS_SDK_NATIVE is not set to a valid directory: `{ndk_root}`") + sys.exit(1) + + ndk_root = ndk_root.resolve() + package_info = ndk_root.joinpath("oh-uni-package.json") + try: + with open(package_info) as meta_file: + meta = json.load(meta_file) + ohos_api_version = int(meta['apiVersion']) + ohos_sdk_version = parse_version(meta['version']) + if ohos_sdk_version < parse_version('4.0'): + print("Warning: mach build currently assumes at least the OpenHarmony 4.0 SDK is used.") + print(f"Info: The OpenHarmony SDK {ohos_sdk_version} is targeting API-level {ohos_api_version}") + except Exception as e: + print(f"Failed to read metadata information from {package_info}") + print(f"Exception: {e}") + + # The OpenHarmony SDK for Windows hosts currently does not contain a libclang shared library, + # which is required by `bindgen` (see issue + # https://gitee.com/openharmony/third_party_llvm-project/issues/I8H50W). Using upstream `clang` is currently + # also not easily possible, since `libcxx` support still needs to be upstreamed ( + # https://github.com/llvm/llvm-project/pull/73114). + os_type = platform.system().lower() + if os_type not in ["linux", "darwin"]: + raise Exception("OpenHarmony builds are currently only supported on Linux and macOS Hosts.") + + llvm_toolchain = ndk_root.joinpath("llvm") + llvm_bin = llvm_toolchain.joinpath("bin") + ohos_sysroot = ndk_root.joinpath("sysroot") + if not (llvm_toolchain.is_dir() and llvm_bin.is_dir()): + print(f"Expected to find `llvm` and `llvm/bin` folder under $OHOS_SDK_NATIVE at `{llvm_toolchain}`") + sys.exit(1) + if not ohos_sysroot.is_dir(): + print(f"Could not find OpenHarmony sysroot in {ndk_root}") + sys.exit(1) + + # Note: We don't use the `-clang` wrappers on purpose, since + # a) the OH 4.0 SDK does not have them yet AND + # b) the wrappers in the newer SDKs are bash scripts, which can cause problems + # on windows, depending on how the wrapper is called. + # Instead, we ensure that all the necessary flags for the c-compiler are set + # via environment variables such as `TARGET_CFLAGS`. + def to_sdk_llvm_bin(prog: str): + if is_windows(): + prog = prog + '.exe' + llvm_prog = llvm_bin.joinpath(prog) + if not llvm_prog.is_file(): + raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), llvm_prog) + return str(llvm_bin.joinpath(prog)) + + # CC and CXX should already be set to appropriate host compilers by `build_env()` + env['HOST_CC'] = env['CC'] + env['HOST_CXX'] = env['CXX'] + env['TARGET_AR'] = to_sdk_llvm_bin("llvm-ar") + env['TARGET_RANLIB'] = to_sdk_llvm_bin("llvm-ranlib") + env['TARGET_READELF'] = to_sdk_llvm_bin("llvm-readelf") + env['TARGET_OBJCOPY'] = to_sdk_llvm_bin("llvm-objcopy") + env['TARGET_STRIP'] = to_sdk_llvm_bin("llvm-strip") + + rust_target_triple = str(self.cross_compile_target).replace('-', '_') + ndk_clang = to_sdk_llvm_bin("clang") + ndk_clangxx = to_sdk_llvm_bin("clang++") + env[f'CC_{rust_target_triple}'] = ndk_clang + env[f'CXX_{rust_target_triple}'] = ndk_clangxx + # The clang target name is different from the LLVM target name + clang_target_triple = str(self.cross_compile_target).replace('-unknown-', '-') + clang_target_triple_underscore = clang_target_triple.replace('-', '_') + env[f'CC_{clang_target_triple_underscore}'] = ndk_clang + env[f'CXX_{clang_target_triple_underscore}'] = ndk_clangxx + # rustc linker + env[f'CARGO_TARGET_{rust_target_triple.upper()}_LINKER'] = ndk_clang + # We could also use a cross-compile wrapper + env["RUSTFLAGS"] += f' -Clink-arg=--target={clang_target_triple}' + env["RUSTFLAGS"] += f' -Clink-arg=--sysroot={ohos_sysroot}' + + env['HOST_CFLAGS'] = '' + env['HOST_CXXFLAGS'] = '' + ohos_cflags = ['-D__MUSL__', f' --target={clang_target_triple}', f' --sysroot={ohos_sysroot}'] + if clang_target_triple.startswith('armv7-'): + ohos_cflags.extend(['-march=armv7-a', '-mfloat-abi=softfp', '-mtune=generic-armv7-a', '-mthumb']) + ohos_cflags_str = " ".join(ohos_cflags) + env['TARGET_CFLAGS'] = ohos_cflags_str + env['TARGET_CPPFLAGS'] = '-D__MUSL__' + env['TARGET_CXXFLAGS'] = ohos_cflags_str + + # CMake related flags + cmake_toolchain_file = ndk_root.joinpath("build", "cmake", "ohos.toolchain.cmake") + if cmake_toolchain_file.is_file(): + env[f'CMAKE_TOOLCHAIN_FILE_{rust_target_triple}'] = str(cmake_toolchain_file) + else: + print( + f"Warning: Failed to find the OpenHarmony CMake Toolchain file - Expected it at {cmake_toolchain_file}") + env[f'CMAKE_C_COMPILER_{rust_target_triple}'] = ndk_clang + env[f'CMAKE_CXX_COMPILER_{rust_target_triple}'] = ndk_clangxx + + # pkg-config + pkg_config_path = '{}:{}'.format(str(ohos_sysroot.joinpath("usr", "lib", "pkgconfig")), + str(ohos_sysroot.joinpath("usr", "share", "pkgconfig"))) + env[f'PKG_CONFIG_SYSROOT_DIR_{rust_target_triple}'] = str(ohos_sysroot) + env[f'PKG_CONFIG_PATH_{rust_target_triple}'] = pkg_config_path + + # bindgen / libclang-sys + env["LIBCLANG_PATH"] = path.join(llvm_toolchain, "lib") + env["CLANG_PATH"] = ndk_clangxx + env[f'CXXSTDLIB_{clang_target_triple_underscore}'] = "c++" + bindgen_extra_clangs_args_var = f'BINDGEN_EXTRA_CLANG_ARGS_{rust_target_triple}' + bindgen_extra_clangs_args = env.get(bindgen_extra_clangs_args_var, "") + bindgen_extra_clangs_args = bindgen_extra_clangs_args + " " + ohos_cflags_str + env[bindgen_extra_clangs_args_var] = bindgen_extra_clangs_args + @staticmethod def common_command_arguments(build_configuration=False, build_type=False): decorators = [] @@ -867,6 +1006,12 @@ class CommandBase(object): args += ["--target", target_override] elif self.cross_compile_target: args += ["--target", self.cross_compile_target] + # The same would apply to android once we merge the jniapi into servoshell + if '-ohos' in self.cross_compile_target: + # Note: in practice `cargo rustc` should just be used unconditionally. + assert command != 'build', "For Android / OpenHarmony `cargo rustc` must be used instead of cargo build" + if command == 'rustc': + args += ["--lib", "--crate-type=cdylib"] if "-p" not in cargo_args: # We're building specific package, that may not have features features = list(self.features) diff --git a/python/servo/try_parser.py b/python/servo/try_parser.py index 56ce4c3f2ad..ec1bbcc87bd 100644 --- a/python/servo/try_parser.py +++ b/python/servo/try_parser.py @@ -46,6 +46,7 @@ class Workflow(str, Enum): MACOS = "macos" WINDOWS = "windows" ANDROID = "android" + OHOS = "ohos" @dataclass @@ -95,6 +96,8 @@ def handle_preset(s: str) -> Optional[JobConfig]: return JobConfig("MacOS WPT", Workflow.MACOS, wpt_layout=Layout.layout2020) elif s == "android": return JobConfig("Android", Workflow.ANDROID) + elif s in ["ohos", "openharmony"]: + return JobConfig("OpenHarmony", Workflow.OHOS) elif s == "webgpu": return JobConfig("WebGPU CTS", Workflow.LINUX, wpt_layout=Layout.layout2020, # reftests are mode for new layout @@ -135,7 +138,7 @@ class Config(object): self.fail_fast = True continue # skip over keyword if word == "full": - words.extend(["linux-wpt", "macos", "windows", "android"]) + words.extend(["linux-wpt", "macos", "windows", "android", "ohos"]) continue # skip over keyword job = handle_preset(word) @@ -211,6 +214,14 @@ class TestParser(unittest.TestCase): "profile": "release", "unit_tests": False, "wpt_tests_to_run": "" + }, + { + "name": "OpenHarmony", + "workflow": "ohos", + "wpt_layout": "none", + "profile": "release", + "unit_tests": False, + "wpt_tests_to_run": "" } ]}) @@ -248,7 +259,7 @@ class TestParser(unittest.TestCase): self.assertEqual(a, JobConfig("Linux", Workflow.LINUX, unit_tests=True)) def test_full(self): - self.assertDictEqual(json.loads(Config("linux-wpt macos windows android").to_json()), + self.assertDictEqual(json.loads(Config("linux-wpt macos windows android ohos").to_json()), json.loads(Config("").to_json())) diff --git a/servobuild.example b/servobuild.example index d0d930ec202..bc8219b6673 100644 --- a/servobuild.example +++ b/servobuild.example @@ -59,3 +59,8 @@ media-stack = "auto" # Defaults to the value of $ANDROID_SDK_ROOT, $ANDROID_NDK_ROOT respectively #sdk = "/opt/android-sdk" #ndk = "/opt/android-ndk" + +# OpenHarmony +[ohos] +# Defaults to the value of $OHOS_SDK_NATIVE +#ndk = "/path/to/ohos-sdk//native"