From 31b3f5ea79cf47ec838aa46071445355b3b5f4d0 Mon Sep 17 00:00:00 2001 From: Anonymous Maarten Date: Sat, 3 Aug 2024 21:00:46 +0200 Subject: [PATCH] ci: add GDK --- .github/actions/setup-gdk-desktop/action.yml | 82 +++++ .github/workflows/create-test-plan.py | 20 +- .github/workflows/generic.yml | 5 + .gitignore | 1 + build-scripts/setup-gdk-desktop.py | 303 +++++++++++++++++++ 5 files changed, 407 insertions(+), 4 deletions(-) create mode 100644 .github/actions/setup-gdk-desktop/action.yml create mode 100644 build-scripts/setup-gdk-desktop.py diff --git a/.github/actions/setup-gdk-desktop/action.yml b/.github/actions/setup-gdk-desktop/action.yml new file mode 100644 index 000000000..10427ace3 --- /dev/null +++ b/.github/actions/setup-gdk-desktop/action.yml @@ -0,0 +1,82 @@ +name: 'Setup GDK (Game Development Kit) for Windows Desktop' +description: 'Download GDK and install into MSBuild' +inputs: + # Keep edition and ref in sync! + edition: + description: 'GDK edition' + default: '240601' # YYMMUU (Year Month Update) + ref: + description: 'Git reference' + default: 'June_2024_Update_1' + folder: + description: 'Folder where to create Directory.Build.props' + required: true + default: '${{ github.workspace }}' +runs: + using: 'composite' + steps: + - uses: actions/setup-python@main + with: + python-version: 3.x + - name: 'Calculate variables' + id: calc + shell: pwsh + run: | + $vs_folder=@(vswhere -latest -property installationPath) + echo "vs-folder=${vs_folder}" >> $Env:GITHUB_OUTPUT + + echo "gdk-path=${{ runner.temp }}\GDK-${{ inputs.edition }}" >> $Env:GITHUB_OUTPUT + + echo "cache-key=gdk-${{ inputs.ref }}-${{ inputs.edition }}" >> $Env:GITHUB_OUTPUT + - name: 'Restore cached GDK' + id: cache-restore + uses: actions/cache/restore@v4 + with: + path: '${{ steps.calc.outputs.gdk-path }}' + key: ${{ steps.calc.outputs.cache-key }} + - name: 'Download GDK' + if: ${{ !steps.cache-restore.outputs.cache-hit }} + shell: pwsh + run: | + python build-scripts/setup-gdk-desktop.py ` + --download ` + --temp-folder "${{ runner.temp }}" ` + --gdk-path="${{ steps.calc.outputs.gdk-path }}" ` + --ref-edition "${{ inputs.ref }},${{ inputs.edition }}" ` + --vs-folder="${{ steps.calc.outputs.vs-folder }}" ` + --no-user-props + - name: 'Extract GDK' + if: ${{ !steps.cache-restore.outputs.cache-hit }} + shell: pwsh + run: | + python build-scripts/setup-gdk-desktop.py ` + --extract ` + --ref-edition "${{ inputs.ref }},${{ inputs.edition }}" ` + --temp-folder "${{ runner.temp }}" ` + --gdk-path="${{ steps.calc.outputs.gdk-path }}" ` + --vs-folder="${{ steps.calc.outputs.vs-folder }}" ` + --no-user-props + - name: 'Cache GDK' + if: ${{ !steps.cache-restore.outputs.cache-hit }} + uses: actions/cache/save@v4 + with: + path: '${{ steps.calc.outputs.gdk-path }}' + key: ${{ steps.calc.outputs.cache-key }} + - name: 'Copy MSBuild files into GDK' + shell: pwsh + run: | + python build-scripts/setup-gdk-desktop.py ` + --ref-edition "${{ inputs.ref }},${{ inputs.edition }}" ` + --gdk-path="${{ steps.calc.outputs.gdk-path }}" ` + --vs-folder="${{ steps.calc.outputs.vs-folder }}" ` + --copy-msbuild ` + --no-user-props + - name: 'Write user props' + shell: pwsh + run: | + python build-scripts/setup-gdk-desktop.py ` + --ref-edition "${{ inputs.ref }},${{ inputs.edition }}" ` + --temp-folder "${{ runner.temp }}" ` + --vs-folder="${{ steps.calc.outputs.vs-folder }}" ` + --gdk-path="${{ steps.calc.outputs.gdk-path }}" ` + "--props-folder=${{ inputs.folder }}" diff --git a/.github/workflows/create-test-plan.py b/.github/workflows/create-test-plan.py index c51df8349..c6d4f0aa8 100755 --- a/.github/workflows/create-test-plan.py +++ b/.github/workflows/create-test-plan.py @@ -95,6 +95,7 @@ class JobSpec: msvc_arch: Optional[MsvcArch] = None clang_cl: bool = False uwp: bool = False + gdk: bool = False vita_gles: Optional[VitaGLES] = None @@ -111,13 +112,14 @@ JOB_SPECS = { "msvc-arm32": JobSpec(name="Windows (MSVC, ARM)", os=JobOs.WindowsLatest, platform=SdlPlatform.Msvc, artifact="SDL-VC-arm32", msvc_arch=MsvcArch.Arm32, ), "msvc-arm64": JobSpec(name="Windows (MSVC, ARM64)", os=JobOs.WindowsLatest, platform=SdlPlatform.Msvc, artifact="SDL-VC-arm64", msvc_arch=MsvcArch.Arm64, ), "msvc-uwp-x64": JobSpec(name="UWP (MSVC, x64)", os=JobOs.WindowsLatest, platform=SdlPlatform.Msvc, artifact="SDL-VC-UWP", msvc_arch=MsvcArch.X64, msvc_project="VisualC-WinRT/SDL-UWP.sln", uwp=True, ), + "msvc-gdk-x64": JobSpec(name="GDK (MSVC, x64)", os=JobOs.WindowsLatest, platform=SdlPlatform.Msvc, artifact="SDL-VC-GDK", msvc_arch=MsvcArch.X64, msvc_project="VisualC-GDK/SDL.sln", gdk=True, no_cmake=True, ), "ubuntu-20.04": JobSpec(name="Ubuntu 20.04", os=JobOs.Ubuntu20_04, platform=SdlPlatform.Linux, artifact="SDL-ubuntu20.04", ), "ubuntu-22.04": JobSpec(name="Ubuntu 22.04", os=JobOs.Ubuntu22_04, platform=SdlPlatform.Linux, artifact="SDL-ubuntu22.04", ), "ubuntu-intel-icx": JobSpec(name="Ubuntu 20.04 (Intel oneAPI)", os=JobOs.Ubuntu20_04, platform=SdlPlatform.Linux, artifact="SDL-ubuntu20.04-oneapi", intel=IntelCompiler.Icx, ), "ubuntu-intel-icc": JobSpec(name="Ubuntu 20.04 (Intel Compiler)", os=JobOs.Ubuntu20_04, platform=SdlPlatform.Linux, artifact="SDL-ubuntu20.04-icc", intel=IntelCompiler.Icc, ), "macos-framework-x64": JobSpec(name="MacOS (Framework) (x86_64)", os=JobOs.Macos12, platform=SdlPlatform.MacOS, artifact="SDL-macos-framework", apple_framework=True, apple_archs={AppleArch.Aarch64, AppleArch.X86_64, }, ), "macos-framework-arm64": JobSpec(name="MacOS (Framework) (arm64)", os=JobOs.MacosLatest, platform=SdlPlatform.MacOS, artifact=None, apple_framework=True, apple_archs={AppleArch.Aarch64, AppleArch.X86_64, }, ), - "amcos-gnu-arm64": JobSpec(name="MacOS (GNU prefix)", os=JobOs.MacosLatest, platform=SdlPlatform.MacOS, artifact="SDL-macos-arm64-gnu", apple_framework=False, apple_archs={AppleArch.Aarch64, }, ), + "macos-gnu-arm64": JobSpec(name="MacOS (GNU prefix)", os=JobOs.MacosLatest, platform=SdlPlatform.MacOS, artifact="SDL-macos-arm64-gnu", apple_framework=False, apple_archs={AppleArch.Aarch64, }, ), "android-cmake": JobSpec(name="Android (CMake)", os=JobOs.UbuntuLatest, platform=SdlPlatform.Android, artifact="SDL-android-arm64", android_abi="arm64-v8a", android_arch="aarch64", android_platform=23, ), "android-cmake-lean": JobSpec(name="Android (CMake, lean)", os=JobOs.UbuntuLatest, platform=SdlPlatform.Android, artifact="SDL-lean-android-arm64", android_abi="arm64-v8a", android_arch="aarch64", android_platform=23, lean=True, ), "android-mk": JobSpec(name="Android (Android.mk)", os=JobOs.UbuntuLatest, platform=SdlPlatform.Android, artifact=None, no_cmake=True, android_mk=True, ), @@ -191,6 +193,7 @@ class JobDetails: setup_libusb_arch: str = "" xcode_sdk: str = "" cpactions: bool = False + setup_gdk_folder: str = "" cpactions_os: str = "" cpactions_version: str = "" cpactions_arch: str = "" @@ -254,6 +257,7 @@ class JobDetails: "cpactions-setup-cmd": self.cpactions_setup_cmd, "cpactions-install-cmd": self.cpactions_install_cmd, "setup-vita-gles-type": self.setup_vita_gles_type, + "setup-gdk-folder": self.setup_gdk_folder, } return {k: v for k, v in data.items() if v != ""} @@ -314,7 +318,7 @@ def spec_to_job(spec: JobSpec) -> JobDetails: )) match spec.platform: case SdlPlatform.Msvc: - job.setup_ninja = True + job.setup_ninja = not spec.gdk job.clang_tidy = False # complains about \threadsafety: "unknown command tag name [clang-diagnostic-documentation-unknown-command]" job.msvc_project = spec.msvc_project if spec.msvc_project else "" job.test_pkg_config = False @@ -344,11 +348,14 @@ def spec_to_job(spec: JobSpec) -> JobDetails: if spec.msvc_project: match spec.msvc_arch: case MsvcArch.X86: - job.msvc_project_flags.append("-p:Platform=Win32") + msvc_platform = "Win32" case MsvcArch.X64: - job.msvc_project_flags.append("-p:Platform=x64") + msvc_platform = "x64" case _: raise ValueError(f"Unsupported vcxproj architecture (arch={spec.msvc_arch})") + if spec.gdk: + msvc_platform = f"Gaming.Desktop.{msvc_platform}" + job.msvc_project_flags.append(f"-p:Platform={msvc_platform}") match spec.msvc_arch: case MsvcArch.X86: job.msvc_vcvars = "x64_x86" @@ -367,6 +374,8 @@ def spec_to_job(spec: JobSpec) -> JobDetails: "-DCMAKE_SYSTEM_VERSION=10.0", )) job.msvc_project_flags.append("-p:WindowsTargetPlatformVersion=10.0.17763.0") + elif spec.gdk: + job.setup_gdk_folder = "VisualC-GDK" else: match spec.msvc_arch: case MsvcArch.X86: @@ -611,6 +620,9 @@ def spec_to_job(spec: JobSpec) -> JobDetails: if fpic is not None: job.cmake_arguments.append(f"-DCMAKE_POSITION_INDEPENDENT_CODE={tf(fpic)}") + if job.no_cmake: + job.cmake_arguments = [] + return job diff --git a/.github/workflows/generic.yml b/.github/workflows/generic.yml index 9a05ff16d..2798ec7cd 100644 --- a/.github/workflows/generic.yml +++ b/.github/workflows/generic.yml @@ -68,6 +68,11 @@ jobs: if: ${{ matrix.platform.platform == 'msvc' }} with: arch: ${{ matrix.platform.msvc-vcvars }} + - name: 'Set up Windows GDK Desktop' + uses: ./.github/actions/setup-gdk-desktop + if: ${{ matrix.platform.setup-gdk-folder != '' }} + with: + folder: '${{ matrix.platform.setup-gdk-folder }}' - name: 'Set up LoongArch64 toolchain' uses: ./.github/actions/setup-loongarch64-toolchain id: setup-loongarch64-toolchain diff --git a/.gitignore b/.gitignore index ef5cc7137..5326a62c3 100644 --- a/.gitignore +++ b/.gitignore @@ -83,6 +83,7 @@ VisualC/tests/testyuv/testyuv.bmp VisualC-GDK/**/Layout src/render/direct3d12/D3D12_*_One.h src/render/direct3d12/D3D12_*_Series.h +Directory.Build.props # for Android android-project/local.properties diff --git a/build-scripts/setup-gdk-desktop.py b/build-scripts/setup-gdk-desktop.py new file mode 100644 index 000000000..d2309a0e3 --- /dev/null +++ b/build-scripts/setup-gdk-desktop.py @@ -0,0 +1,303 @@ +#!/usr/bin/env python + +import argparse +import functools +import logging +import os +from pathlib import Path +import re +import shutil +import subprocess +import tempfile +import textwrap +import urllib.request +import zipfile + +# Update both variables when updating the GDK +GIT_REF = "June_2024_Update_1" +GDK_EDITION = "240601" # YYMMUU + +logger = logging.getLogger(__name__) + +class GdDesktopConfigurator: + def __init__(self, gdk_path, arch, vs_folder, vs_version=None, vs_toolset=None, temp_folder=None, git_ref=None, gdk_edition=None): + self.git_ref = git_ref or GIT_REF + self.gdk_edition = gdk_edition or GDK_EDITION + self.gdk_path = gdk_path + self.temp_folder = temp_folder or Path(tempfile.gettempdir()) + self.dl_archive_path = Path(self.temp_folder) / f"{ self.git_ref }.zip" + self.gdk_extract_path = Path(self.temp_folder) / f"GDK-{ self.git_ref }" + self.arch = arch + self.vs_folder = vs_folder + self._vs_version = vs_version + self._vs_toolset = vs_toolset + + def download_archive(self) -> None: + gdk_url = f"https://github.com/microsoft/GDK/archive/refs/tags/{ GIT_REF }.zip" + logger.info("Downloading %s to %s", gdk_url, self.dl_archive_path) + urllib.request.urlretrieve(gdk_url, self.dl_archive_path) + assert self.dl_archive_path.is_file() + + def extract_zip_archive(self) -> None: + extract_path = self.gdk_extract_path.parent + assert self.dl_archive_path.is_file() + logger.info("Extracting %s to %s", self.dl_archive_path, extract_path) + with zipfile.ZipFile(self.dl_archive_path) as zf: + zf.extractall(extract_path) + assert self.gdk_extract_path.is_dir(), f"{self.gdk_extract_path} must exist" + + def extract_development_kit(self) -> None: + extract_dks_cmd = self.gdk_extract_path / "SetupScripts/ExtractXboxOneDKs.cmd" + assert extract_dks_cmd.is_file() + logger.info("Extracting GDK Development Kit: running %s", extract_dks_cmd) + cmd = ["cmd.exe", "/C", str(extract_dks_cmd), str(self.gdk_extract_path), str(self.gdk_path)] + logger.debug("Running %r", cmd) + subprocess.check_call(cmd) + + def detect_vs_version(self) -> str: + vs_regex = re.compile("VS([0-9]{4})") + supported_vs_versions = [] + for p in self.gaming_grdk_build_path.iterdir(): + if not p.is_dir(): + continue + if m := vs_regex.match(p.name): + supported_vs_versions.append(m.group(1)) + logger.info(f"Supported Visual Studio versions: {supported_vs_versions}") + vs_versions = set(self.vs_folder.parts).intersection(set(supported_vs_versions)) + if not vs_versions: + raise RuntimeError("Visual Studio version is incompatible") + if len(vs_versions) > 1: + raise RuntimeError(f"Too many compatible VS versions found ({vs_versions})") + vs_version = vs_versions.pop() + logger.info(f"Used Visual Studio version: {vs_version}") + return vs_version + + def detect_vs_toolset(self) -> str: + toolset_paths = [] + for ts_path in self.gdk_toolset_parent_path.iterdir(): + if not ts_path.is_dir(): + continue + ms_props = ts_path / "Microsoft.Cpp.props" + if not ms_props.is_file(): + continue + toolset_paths.append(ts_path.name) + logger.info("Detected Visual Studio toolsets: %s", toolset_paths) + assert toolset_paths, "Have we detected at least one toolset?" + + def toolset_number(toolset: str) -> int: + if m:= re.match("[^0-9]*([0-9]+).*", toolset): + return int(m.group(1)) + return -9 + + return max(toolset_paths, key=toolset_number) + + @property + def vs_version(self) -> str: + if self._vs_version is None: + self._vs_version = self.detect_vs_version() + return self._vs_version + + @property + def vs_toolset(self) -> str: + if self._vs_toolset is None: + self._vs_toolset = self.detect_vs_toolset() + return self._vs_toolset + + @staticmethod + def copy_files_and_merge_into(srcdir: Path, dstdir: Path) -> None: + logger.info(f"Copy {srcdir} to {dstdir}") + for root, _, files in os.walk(srcdir): + dest_root = dstdir / Path(root).relative_to(srcdir) + if not dest_root.is_dir(): + dest_root.mkdir() + for file in files: + srcfile = Path(root) / file + dstfile = dest_root / file + shutil.copy(srcfile, dstfile) + + def copy_msbuild(self) -> None: + vc_toolset_parent_path = self.vs_folder / "MSBuild/Microsoft/VC" + if 1: + logger.info(f"Detected compatible Visual Studio version: {self.vs_version}") + srcdir = vc_toolset_parent_path + dstdir = self.gdk_toolset_parent_path + assert srcdir.is_dir(), "Source directory must exist" + assert dstdir.is_dir(), "Destination directory must exist" + + self.copy_files_and_merge_into(srcdir=srcdir, dstdir=dstdir) + + @property + def game_dk_path(self) -> Path: + return self.gdk_path / "Microsoft GDK" + + @property + def game_dk_latest_path(self) -> Path: + return self.game_dk_path / self.gdk_edition + + @property + def windows_sdk_path(self) -> Path: + return self.gdk_path / "Windows Kits/10" + + @property + def gaming_grdk_build_path(self) -> Path: + return self.game_dk_latest_path / "GRDK" + + @property + def gdk_toolset_parent_path(self) -> Path: + return self.gaming_grdk_build_path / f"VS{self.vs_version}/flatDeployment/MSBuild/Microsoft/VC" + + @property + def env(self) -> dict[str, str]: + game_dk = self.game_dk_path + game_dk_latest = self.game_dk_latest_path + windows_sdk_dir = self.windows_sdk_path + gaming_grdk_build = self.gaming_grdk_build_path + + return { + "GRDKEDITION": f"{self.gdk_edition}", + "GameDK": f"{game_dk}\\", + "GameDKLatest": f"{ game_dk_latest }\\", + "WindowsSdkDir": f"{ windows_sdk_dir }\\", + "GamingGRDKBuild": f"{ gaming_grdk_build }\\", + "VSInstallDir": f"{ self.vs_folder }\\", + } + + def create_user_props(self, path: Path) -> None: + vc_targets_path = self.gaming_grdk_build_path / f"VS{ self.vs_version }/flatDeployment/MSBuild/Microsoft/VC/{ self.vs_toolset }" + vc_targets_path16 = self.gaming_grdk_build_path / f"VS2019/flatDeployment/MSBuild/Microsoft/VC/{ self.vs_toolset }" + vc_targets_path17 = self.gaming_grdk_build_path / f"VS2022/flatDeployment/MSBuild/Microsoft/VC/{ self.vs_toolset }" + additional_include_directories = ";".join(str(p) for p in self.gdk_include_paths) + additional_library_directories = ";".join(str(p) for p in self.gdk_library_paths) + durango_xdk_install_path = self.gdk_path / "Microsoft GDK" + with path.open("w") as f: + f.write(textwrap.dedent(f"""\ + + + + { vc_targets_path }\\ + { vc_targets_path16 }\\ + { vc_targets_path17 }\\ + { self.gaming_grdk_build_path }\\ + Gaming.Desktop.x64 + Debug + { self.gdk_edition } + { durango_xdk_install_path } + + $(DurangoXdkInstallPath)\\{self.gdk_edition}\\GRDK\\VS2019\\flatDeployment\\MSBuild\\Microsoft\\VC\\{self.vs_toolset}\\Platforms\\$(Platform)\\ + $(DurangoXdkInstallPath)\\{self.gdk_edition}\\GRDK\\VS2019\\flatDeployment\\MSBuild\\Microsoft\\VC\\{self.vs_toolset}\\Platforms\\$(Platform)\\ + $(DurangoXdkInstallPath)\\{self.gdk_edition}\\GRDK\\VS2022\\flatDeployment\\MSBuild\\Microsoft\\VC\\{self.vs_toolset}\\Platforms\\$(Platform)\\ + $(DurangoXdkInstallPath)\\{self.gdk_edition}\\GRDK\\VS2022\\flatDeployment\\MSBuild\\Microsoft\\VC\\{self.vs_toolset}\\Platforms\\$(Platform)\\ + + true + true + true + + + + { additional_include_directories };%(AdditionalIncludeDirectories) + + + { additional_library_directories };%(AdditionalLibraryDirectories) + + + + """)) + + @property + def gdk_include_paths(self) -> list[Path]: + return [ + self.gaming_grdk_build_path / "gamekit/include", + ] + + @property + def gdk_library_paths(self) -> list[Path]: + return [ + self.gaming_grdk_build_path / f"gamekit/lib/{self.arch}", + ] + + @property + def gdk_binary_path(self) -> list[Path]: + return [ + self.gaming_grdk_build_path / "bin", + self.game_dk_path / "bin", + ] + + @property + def build_env(self) -> dict[str, str]: + gdk_include = ";".join(str(p) for p in self.gdk_include_paths) + gdk_lib = ";".join(str(p) for p in self.gdk_library_paths) + gdk_path = ";".join(str(p) for p in self.gdk_binary_path) + return { + "GDK_INCLUDE": gdk_include, + "GDK_LIB": gdk_lib, + "GDK_PATH": gdk_path, + } + + def print_env(self) -> None: + for k, v in self.env.items(): + print(f"set \"{k}={v}\"") + print() + for k, v in self.build_env.items(): + print(f"set \"{k}={v}\"") + print() + print(f"set \"PATH=%GDK_PATH%;%PATH%\"") + print(f"set \"LIB=%GDK_LIB%;%LIB%\"") + print(f"set \"INCLUDE=%GDK_INCLUDE%;%INCLUDE%\"") + + +def main(): + logging.basicConfig(level=logging.INFO) + parser = argparse.ArgumentParser(allow_abbrev=False) + parser.add_argument("--arch", choices=["amd64"], default="amd64", help="Architecture") + parser.add_argument("--download", action="store_true", help="Download GDK") + parser.add_argument("--extract", action="store_true", help="Extract downloaded GDK") + parser.add_argument("--copy-msbuild", action="store_true", help="Copy MSBuild files") + parser.add_argument("--temp-folder", help="Temporary folder where to download and extract GDK") + parser.add_argument("--gdk-path", required=True, type=Path, help="Folder where to store the GDK") + parser.add_argument("--ref-edition", type=str, help="Git ref and GDK edition separated by comma") + parser.add_argument("--vs-folder", required=True, type=Path, help="Installation folder of Visual Studio") + parser.add_argument("--vs-version", required=False, type=int, help="Visual Studio version") + parser.add_argument("--vs-toolset", required=False, help="Visual Studio toolset (e.g. v150)") + parser.add_argument("--props-folder", required=False, type=Path, default=Path(), help="Visual Studio toolset (e.g. v150)") + parser.add_argument("--no-user-props", required=False, dest="user_props", action="store_false", help="Don't ") + args = parser.parse_args() + + logging.basicConfig(level=logging.INFO) + + git_ref = None + gdk_edition = None + if args.ref_edition is not None: + git_ref, gdk_edition = args.ref_edition.split(",", 1) + try: + int(gdk_edition) + except ValueError: + parser.error("Edition should be an integer (YYMMUU) (Y=year M=month U=update)") + + configurator = GdDesktopConfigurator( + arch=args.arch, + git_ref=git_ref, + gdk_edition=gdk_edition, + vs_folder=args.vs_folder, + vs_version=args.vs_version, + vs_toolset=args.vs_toolset, + gdk_path=args.gdk_path, + temp_folder=args.temp_folder, + ) + + if args.download: + configurator.download_archive() + + if args.extract: + configurator.extract_zip_archive() + + configurator.extract_development_kit() + + if args.copy_msbuild: + configurator.copy_msbuild() + + if args.user_props: + configurator.print_env() + configurator.create_user_props(args.props_folder / "Directory.Build.props") + +if __name__ == "__main__": + raise SystemExit(main())