diff --git a/README.md b/README.md index cdf2c8e..89f8d38 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ For more details on CodeQL customization packs see the section [CodeQL customiza The CodeQL bundle application can be installed using `pip` with the command: ```bash -python3.11 -m pip install https://github.com/advanced-security/codeql-bundle/releases/download/v0.5.0/codeql_bundle-0.5.0-py3-none-any.whl +python3.11 -m pip install https://github.com/advanced-security/codeql-bundle/releases/download/v0.6.0/codeql_bundle-0.6.0-py3-none-any.whl ``` ## Usage @@ -37,8 +37,8 @@ with the command: codeql-bundle --bundle codeql-bundle-v2.26.1 --output codeql-custom-bundle.tar.gz --workspace --log INFO ``` -If the source bundle is the platform agnostic bundle then you can create platform specific bundles to reduce the size of the used bundle(s). -The following example creates platform specific bundles for all the currently supported platforms. +Targets include `linux64`, `linux-arm64`, `osx64`, and `win64`. Local +all-platform archives can build for multiple targets. ```bash codeql-bundle --bundle --output --workspace --log INFO -p linux64 -p osx64 -p win64 diff --git a/codeql_bundle/cli.py b/codeql_bundle/cli.py index d8d8c1b..7cebcf4 100644 --- a/codeql_bundle/cli.py +++ b/codeql_bundle/cli.py @@ -65,7 +65,9 @@ "-p", "--platform", multiple=True, - type=click.Choice(["linux64", "osx64", "win64"], case_sensitive=False), + type=click.Choice( + ["linux64", "linux-arm64", "osx64", "win64"], case_sensitive=False + ), help="Target platform for the bundle", ) @click.option( diff --git a/codeql_bundle/helpers/bundle.py b/codeql_bundle/helpers/bundle.py index 24f2975..6a2c290 100644 --- a/codeql_bundle/helpers/bundle.py +++ b/codeql_bundle/helpers/bundle.py @@ -199,11 +199,14 @@ class BundlePlatform(Enum): LINUX = 1 WINDOWS = 2 OSX = 3 + LINUX_ARM64 = 4 @staticmethod def from_string(platform: str) -> "BundlePlatform": if platform.lower() == "linux" or platform.lower() == "linux64": return BundlePlatform.LINUX + elif platform.lower() == "linux-arm64": + return BundlePlatform.LINUX_ARM64 elif platform.lower() == "windows" or platform.lower() == "win64": return BundlePlatform.WINDOWS elif platform.lower() == "osx" or platform.lower() == "osx64": @@ -214,6 +217,8 @@ def from_string(platform: str) -> "BundlePlatform": def __str__(self): if self == BundlePlatform.LINUX: return "linux64" + elif self == BundlePlatform.LINUX_ARM64: + return "linux-arm64" elif self == BundlePlatform.WINDOWS: return "win64" elif self == BundlePlatform.OSX: @@ -248,6 +253,12 @@ def supports_linux() -> set[BundlePlatform]: else: return set() + def supports_linux_arm64() -> set[BundlePlatform]: + if (self.bundle_path / "cpp" / "tools" / "linux-arm64").exists(): + return {BundlePlatform.LINUX_ARM64} + else: + return set() + def supports_macos() -> set[BundlePlatform]: if (self.bundle_path / "cpp" / "tools" / "osx64").exists(): return {BundlePlatform.OSX} @@ -261,14 +272,23 @@ def supports_windows() -> set[BundlePlatform]: return set() self.platforms: set[BundlePlatform] = ( - supports_linux() | supports_macos() | supports_windows() + supports_linux() + | supports_linux_arm64() + | supports_macos() + | supports_windows() ) current_system = platform.system() if not current_system in ["Linux", "Darwin", "Windows"]: raise BundleException(f"Unsupported system: {current_system}") - if current_system == "Linux" and BundlePlatform.LINUX not in self.platforms: - raise BundleException("Bundle doesn't support Linux!") + if current_system == "Linux": + current_linux = ( + BundlePlatform.LINUX_ARM64 + if platform.machine().lower() in {"aarch64", "arm64"} + else BundlePlatform.LINUX + ) + if current_linux not in self.platforms: + raise BundleException(f"Bundle doesn't support {current_linux}!") elif current_system == "Darwin" and BundlePlatform.OSX not in self.platforms: raise BundleException("Bundle doesn't support OSX!") elif ( @@ -788,25 +808,29 @@ def is_unsafe_path(basedir: Path, path: Path) -> bool: if platform.system() == "Windows": keytool = "tools/win64/java/bin/keytool.exe" elif platform.system() == "Linux": - keytool = "tools/linux64/java/bin/keytool" + linux = ( + "linux-arm64" + if platform.machine().lower() in {"aarch64", "arm64"} + else "linux64" + ) + keytool = f"tools/{linux}/java/bin/keytool" elif platform.system() == "Darwin": keytool = "tools/osx64/java/bin/keytool" else: raise BundleException(f"Unsupported platform {platform.system()}") keytool = self.bundle_path / keytool - if not keytool.exists(): + if "CodeQLBundleAdditionalCertificates" in config and not keytool.exists(): raise BundleException(f"Keytool {keytool} does not exist.") - keystores: list[str] = [ - "tools/win64/java/lib/security/cacerts", - "tools/linux64/java/lib/security/cacerts", - "tools/osx64/java/lib/security/cacerts", - "tools/osx64/java-aarch64/lib/security/cacerts", - ] + keystores = list( + (self.bundle_path / "tools").glob("*/java*/lib/security/cacerts") + ) # Add the certificates to the Java keystores if "CodeQLBundleAdditionalCertificates" in config: + if not keystores: + raise BundleException("The bundle contains no Java keystores.") for cert in config["CodeQLBundleAdditionalCertificates"]: src = workspace_path / Path(cert["Source"]) src = src.resolve() @@ -818,9 +842,6 @@ def is_unsafe_path(basedir: Path, path: Path) -> bool: raise BundleException(f"Certificate file {src} does not exist.") for keystore in keystores: - keystore = self.bundle_path / keystore - if not keystore.exists(): - raise BundleException(f"Keystore {keystore} does not exist.") logging.info(f"Adding certificate {src} to keystore {keystore}") subprocess.run( [ @@ -919,22 +940,36 @@ def get_nonplatform_tool_paths( """Get a list of paths to tools that are not for the specified platform relative to the root of a bundle.""" specialize_path: Optional[Callable[[Path], List[Path]]] = None linux64_subpaths = [Path("linux64"), Path("linux")] + linux_arm64_subpaths = [Path("linux-arm64")] osx64_subpaths = [Path("osx64"), Path("macos")] win64_subpaths = [Path("win64"), Path("windows")] if platform == BundlePlatform.LINUX: specialize_path = lambda p: [ p / subpath - for subpath in osx64_subpaths + win64_subpaths + for subpath in osx64_subpaths + + win64_subpaths + + linux_arm64_subpaths + ] + elif platform == BundlePlatform.LINUX_ARM64: + specialize_path = lambda p: [ + p / subpath + for subpath in osx64_subpaths + + win64_subpaths + + linux64_subpaths ] elif platform == BundlePlatform.WINDOWS: specialize_path = lambda p: [ p / subpath - for subpath in osx64_subpaths + linux64_subpaths + for subpath in osx64_subpaths + + linux64_subpaths + + linux_arm64_subpaths ] elif platform == BundlePlatform.OSX: specialize_path = lambda p: [ p / subpath - for subpath in linux64_subpaths + win64_subpaths + for subpath in linux64_subpaths + + win64_subpaths + + linux_arm64_subpaths ] else: raise BundleException(f"Unsupported platform {platform}.") @@ -960,10 +995,24 @@ def filter(tarinfo: tarfile.TarInfo) -> Optional[tarfile.TarInfo]: if platform == BundlePlatform.LINUX: exclusion_paths.append(Path("swift/qltest/osx64")) exclusion_paths.append(Path("swift/resource-dir/osx64")) + exclusion_paths.append(Path("swift/qltest/linux-arm64")) + exclusion_paths.append( + Path("swift/resource-dir/linux-arm64") + ) + + if platform == BundlePlatform.LINUX_ARM64: + exclusion_paths.append(Path("swift/qltest/osx64")) + exclusion_paths.append(Path("swift/resource-dir/osx64")) + exclusion_paths.append(Path("swift/qltest/linux64")) + exclusion_paths.append(Path("swift/resource-dir/linux64")) if platform == BundlePlatform.OSX: exclusion_paths.append(Path("swift/qltest/linux64")) exclusion_paths.append(Path("swift/resource-dir/linux64")) + exclusion_paths.append(Path("swift/qltest/linux-arm64")) + exclusion_paths.append( + Path("swift/resource-dir/linux-arm64") + ) tarfile_path_root = Path(tarfile_path.parts[0]) exclusion_paths = [ diff --git a/pyproject.toml b/pyproject.toml index 664a87d..2022d0a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "codeql-bundle" -version = "0.5.0" +version = "0.6.0" description = "Tool to create custom CodeQL bundles" authors = ["Remco Vermeulen "] readme = "README.md" diff --git a/tests/test_bundle.py b/tests/test_bundle.py new file mode 100644 index 0000000..a5eb0ea --- /dev/null +++ b/tests/test_bundle.py @@ -0,0 +1,44 @@ +from pathlib import Path +from tempfile import TemporaryDirectory +import tarfile +import unittest + +from codeql_bundle.helpers.bundle import BundlePlatform, CustomBundle + + +class BundleTests(unittest.TestCase): + def test_linux_bundles_keep_only_the_target_architecture(self) -> None: + with TemporaryDirectory() as directory: + root = Path(directory) + bundle = object.__new__(CustomBundle) + bundle.tmp_dir = None + bundle.bundle_path = root / "bundle" + bundle.languages = set() + bundle.platforms = { + BundlePlatform.LINUX, + BundlePlatform.LINUX_ARM64, + } + for platform in bundle.platforms: + path = bundle.bundle_path / f"tools/{platform}/tool" + path.parent.mkdir(parents=True, exist_ok=True) + path.touch() + + config = root / "qlt.conf.json" + config.write_text("{}") + bundle.add_files_and_certs(config, root) + bundle.bundle(root, bundle.platforms) + + for target, excluded in ( + (BundlePlatform.LINUX, BundlePlatform.LINUX_ARM64), + (BundlePlatform.LINUX_ARM64, BundlePlatform.LINUX), + ): + with tarfile.open( + root / f"codeql-bundle-{target}.tar.gz" + ) as archive: + names = archive.getnames() + self.assertIn(f"codeql/tools/{target}/tool", names) + self.assertNotIn(f"codeql/tools/{excluded}/tool", names) + + +if __name__ == "__main__": + unittest.main()