From 8f55c8e12e8ac20645de3e96bf168cc617e0eb26 Mon Sep 17 00:00:00 2001 From: Simon Marlow Date: Thu, 1 Oct 2026 16:51:15 +0100 Subject: [PATCH] Update the prelude to upstream facebook/buck2 as of 2484d70a6f8a Brings prelude/ from upstream 1469fe7461 (2026-08-14, the snapshot this was originally taken from) forward to 2484d70a6f8a (2026-09-30, the commit buck2 release 2026-10-01 was built from): 368 upstream commits touching prelude/. The old snapshot passed `eager_materialization_enabled` to `ctx.actions.run`, so it could not be loaded by any buck2 release newer than 2026-09-01. The new one loads with 2026-10-01 and `latest`, and no longer with 2026-09-01 (it uses `root_values`). Done as a three-way merge (base: upstream at 1469fe7461; ours: the old prelude/; theirs: upstream at 2484d70a6f8a). Upstream touched none of our Haskell changes (haskell/*, decls/haskell_*.bzl, toolchains/haskell.bzl, third-party/pkgconfig.bzl); the only file changed on both sides was toolchains/demo.bzl, which merged without conflicts. Everything else is taken from upstream unchanged. CI is moved to the 2026-10-01 release to match. Tested with buck2 2026-10-01: example/ (TemplateHaskell, .hsc, C++ FFI) builds and tests in dev, opt and prof modes. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/ci.yml | 19 +- prelude/BUCK | 7 + prelude/android/android.bzl | 2 + prelude/android/android_aar.bzl | 6 +- prelude/android/android_apk.bzl | 16 +- prelude/android/android_binary.bzl | 100 +- .../android_binary_native_library_rules.bzl | 589 ++++++- .../android_binary_resources_rules.bzl | 48 +- prelude/android/android_bundle.bzl | 55 +- .../android/android_instrumentation_apk.bzl | 39 +- .../android/android_instrumentation_test.bzl | 33 +- prelude/android/android_library.bzl | 56 +- prelude/android/android_manifest.bzl | 4 +- prelude/android/android_prebuilt_aar.bzl | 2 +- prelude/android/android_providers.bzl | 20 + prelude/android/android_resource.bzl | 7 +- prelude/android/android_toolchain.bzl | 14 +- prelude/android/apk_genrule.bzl | 42 +- prelude/android/bundletool_util.bzl | 2 +- prelude/android/dex_rules.bzl | 529 ++---- prelude/android/gen_aidl.bzl | 8 +- prelude/{java => android}/keystore.bzl | 2 +- prelude/android/native_build_commands.bzl | 124 ++ prelude/android/preprocess_java_classes.bzl | 11 +- prelude/android/r_dot_java.bzl | 20 +- prelude/android/relinker_linker_outputs.bzl | 48 +- prelude/android/robolectric_test.bzl | 53 +- prelude/apple/apple_binary.bzl | 65 +- prelude/apple/apple_bundle.bzl | 30 +- prelude/apple/apple_bundle_destination.bzl | 30 +- prelude/apple/apple_bundle_part.bzl | 186 ++- prelude/apple/apple_entitlements.bzl | 4 +- prelude/apple/apple_finalize_codesign.bzl | 10 +- prelude/apple/apple_framework_versions.bzl | 8 + prelude/apple/apple_library.bzl | 38 +- prelude/apple/apple_package.bzl | 18 +- prelude/apple/apple_package_types.bzl | 4 +- prelude/apple/apple_resource_types.bzl | 9 + prelude/apple/apple_rules_decls.bzl | 6 +- prelude/apple/apple_test.bzl | 23 +- prelude/apple/apple_toolchain_types.bzl | 5 +- prelude/apple/apple_tools.bzl | 5 +- prelude/apple/apple_utility.bzl | 12 +- .../apple/mockingbird/mockingbird_types.bzl | 12 +- prelude/apple/prebuilt_apple_framework.bzl | 21 + .../prebuilt_apple_xcframework_macro_impl.bzl | 2 +- prelude/apple/resource_groups.bzl | 17 +- .../apple/swift/apple_sdk_clang_module.bzl | 3 + .../apple/swift/apple_sdk_modules_utility.bzl | 74 +- .../apple/swift/apple_sdk_swift_module.bzl | 3 + prelude/apple/swift/swift_compilation.bzl | 122 +- prelude/apple/swift/swift_toolchain_types.bzl | 2 + prelude/apple/swift/swift_types.bzl | 6 + prelude/apple/tools/BUCK | 7 +- prelude/apple/tools/bundling/BUCK | 32 +- .../apple/tools/bundling/incremental_utils.py | 7 + .../tools/bundling/incremental_utils_test.py | 24 + prelude/apple/tools/bundling/logging_utils.py | 62 + prelude/apple/tools/bundling/main.py | 168 +- ..._main.py => provisioning_manifest_main.py} | 47 +- .../bundling/resolve_signing_context_main.py | 75 + .../apple/tools/bundling/signing_context.py | 138 +- .../tools/bundling/signing_context_data.py | 157 ++ .../bundling/signing_context_data_test.py | 219 +++ prelude/apple/tools/bundling/signing_info.py | 77 + .../apple/tools/bundling/signing_info_main.py | 44 + .../apple/tools/bundling/signing_info_test.py | 130 ++ prelude/apple/tools/code_signing/BUCK | 9 + .../tools/code_signing/codesign_bundle.py | 54 +- prelude/apple/tools/code_signing/identity.py | 26 + .../tools/code_signing/info_plist_metadata.py | 33 + .../provisioning_profile_metadata.py | 165 +- .../provisioning_profile_selection.py | 22 + .../apple/tools/code_signing/serialization.py | 67 + .../code_signing/signing_context_types.py | 143 ++ .../signing_context_types_test.py | 316 ++++ prelude/apple/tools/defs.bzl | 4 + .../tools/selective_debugging/scrubber.py | 4 +- prelude/apple/user/watch_transition.bzl | 10 +- .../swiftmodule_change_analysis.bzl | 125 ++ prelude/apple/versions.bzl | 1 + prelude/apple/xcassets_asset_symbols.bzl | 25 + prelude/cfg/modifier/cfg_constructor.bzl | 21 +- prelude/cfg/modifier/common.bzl | 10 +- prelude/cfg/modifier/name.bzl | 2 +- prelude/configurations/rules.bzl | 8 +- prelude/configurations/util.bzl | 19 +- prelude/cxx/anon_link.bzl | 301 ++-- prelude/cxx/archive.bzl | 4 +- prelude/cxx/argsfiles.bzl | 7 + prelude/cxx/bitcode.bzl | 2 +- prelude/cxx/comp_db.bzl | 3 +- prelude/cxx/compile.bzl | 875 +++++++--- prelude/cxx/compile_types.bzl | 4 +- prelude/cxx/cuda.bzl | 486 ++++-- prelude/cxx/cxx.bzl | 87 +- prelude/cxx/cxx_bolt.bzl | 13 +- prelude/cxx/cxx_executable.bzl | 138 +- prelude/cxx/cxx_flags.bzl | 113 ++ prelude/cxx/cxx_library.bzl | 78 +- prelude/cxx/cxx_library_utility.bzl | 11 +- prelude/cxx/cxx_toolchain.bzl | 24 +- prelude/cxx/cxx_toolchain_types.bzl | 31 +- prelude/cxx/cxx_types.bzl | 15 +- prelude/cxx/dist_lto/darwin/dist_lto.bzl | 23 +- prelude/cxx/dist_lto/dist_lto.bzl | 18 +- .../dist_lto/tools/dist_lto_planner_gnu.py | 7 +- prelude/cxx/dwp.bzl | 3 + prelude/cxx/groups.bzl | 7 +- prelude/cxx/groups_types.bzl | 7 + prelude/cxx/headers.bzl | 4 + prelude/cxx/link.bzl | 29 +- prelude/cxx/link_groups.bzl | 9 + prelude/cxx/link_types.bzl | 7 +- prelude/cxx/preprocessor.bzl | 73 +- prelude/cxx/shared_library_interface.bzl | 3 +- prelude/cxx/symbols.bzl | 3 +- prelude/cxx/user/cxx_toolchain_override.bzl | 22 +- prelude/cxx/windows_resource.bzl | 8 +- prelude/decls/android_rules.bzl | 51 +- prelude/decls/common.bzl | 4 - prelude/decls/core_rules.bzl | 21 +- prelude/decls/cxx_common.bzl | 16 + prelude/decls/cxx_rules.bzl | 139 +- prelude/decls/erlang_rules.bzl | 41 +- prelude/decls/java_rules.bzl | 51 +- prelude/decls/jvm_common.bzl | 16 +- prelude/decls/kotlin_rules.bzl | 30 +- prelude/decls/python_rules.bzl | 5 + prelude/decls/rust_rules.bzl | 12 +- prelude/decls/toolchains_common.bzl | 6 +- .../common/include/buck_ct_records.hrl | 4 +- .../common/src/artifact_annotations.erl | 18 +- .../common_test/common/src/buck_ct_parser.erl | 2 +- .../common/src/buck_ct_provider.erl | 23 +- .../common_test/common/src/common_util.erl | 39 +- .../common/src/ct_error_printer.erl | 2 +- .../common_test/common/src/ct_stdout.erl | 5 +- .../{test_binary => common}/src/list_test.erl | 4 +- .../common/src/test_artifact_directory.erl | 23 +- .../common/test/ct_stdout_SUITE.erl | 8 +- .../cth_hooks/src/cth_skip_cases.erl | 136 ++ .../common_test/cth_hooks/src/cth_tpx.erl | 14 +- .../cth_hooks/src/cth_tpx_role.erl | 10 +- .../cth_hooks/src/cth_tpx_test_tree.erl | 61 +- .../test_binary/src/json_interfacer.erl | 38 +- .../test_binary/src/listing_interfacer.erl | 3 +- .../test_binary/src/test_binary.erl | 6 +- .../common_test/test_binary/src/test_info.erl | 3 +- .../test_binary/src/test_runner.erl | 138 +- .../common_test/test_cli_lib/src/test.erl | 4 +- .../test_cli_lib/test/test_cli_e2e_SUITE.erl | 12 +- .../test_exec/src/ct_daemon_core.erl | 12 +- .../test_exec/src/ct_daemon_hooks.erl | 5 +- .../test_exec/src/ct_daemon_node.erl | 8 +- .../test_exec/src/ct_daemon_runner.erl | 10 +- .../common_test/test_exec/src/ct_executor.erl | 12 +- .../test_exec/src/ct_executor_watchdog.erl | 4 +- .../common_test/test_exec/src/ct_runner.erl | 29 +- .../test_exec/src/epmd_manager.erl | 4 +- .../common_test/test_exec/src/test_exec.erl | 3 +- .../test/ct_daemon_capture_SUITE.erl | 6 +- .../test_exec/test/ct_executor_SUITE.erl | 6 +- prelude/erlang/elp.bxl | 14 + prelude/erlang/erlang_application.bzl | 8 +- prelude/erlang/erlang_error_handler.bzl | 12 +- prelude/erlang/erlang_erts.bzl | 140 ++ prelude/erlang/erlang_headers.bzl | 6 +- prelude/erlang/erlang_info.bzl | 21 +- prelude/erlang/erlang_otp_application.bzl | 39 +- prelude/erlang/erlang_release.bzl | 610 ++++--- prelude/erlang/erlang_tests.bzl | 34 +- prelude/erlang/erlang_toolchain.bzl | 67 +- prelude/erlang/erts/BUCK | 11 + .../{toolchain => erts}/extract_from_otp.erl | 58 +- prelude/erlang/toolchain/BUCK | 1 - prelude/erlang/toolchain/README.md | 125 -- .../erlang/toolchain/boot_script_builder.erl | 8 +- .../erlang/toolchain/generate_otp_versions.py | 245 --- prelude/erlang/toolchain/otp_versions.bzl | 90 -- prelude/genrule.bzl | 25 +- prelude/genrule_local_labels.bzl | 6 + prelude/genrule_prefer_local_labels.bzl | 2 +- prelude/go/go_test.bzl | 8 +- prelude/go/link.bzl | 5 +- prelude/go/package_builder.bzl | 12 +- prelude/go/tools/gobuckify/lib/buck.go | 87 +- prelude/go/tools/gobuckify/lib/buck_test.go | 289 ++-- prelude/go/tools/gobuckify/lib/golist.go | 79 +- prelude/go/tools/gobuckify/lib/render.go | 41 +- prelude/go/tools/gobuckify/lib/render_test.go | 157 +- .../go/tools/gopackagesdriver/driver/buck.go | 46 +- .../gopackagesdriver/driver/buck_test.go | 113 +- .../tools/gopackagesdriver/driver/driver.go | 2 +- .../gopackagesdriver/driver/query_bxl.go | 69 +- prelude/graphql/graphql.bzl | 1 + prelude/java/dex.bzl | 15 +- prelude/java/gwt_binary.bzl | 4 +- prelude/java/java.bzl | 2 - prelude/java/java_binary.bzl | 4 +- prelude/java/java_library.bzl | 33 +- prelude/java/java_providers.bzl | 97 +- prelude/java/java_test.bzl | 10 +- prelude/java/java_toolchain.bzl | 1 + prelude/java/javacd_jar_creator.bzl | 50 +- prelude/js/js_bundle.bzl | 1 + prelude/js/js_bundle_genrule.bzl | 9 +- prelude/js/js_library.bzl | 12 +- prelude/js/js_providers.bzl | 7 +- prelude/js/js_utils.bzl | 8 +- prelude/jvm/cd_jar_creator_util.bzl | 136 +- prelude/kotlin/kotlin_library.bzl | 54 +- prelude/kotlin/kotlin_test.bzl | 27 +- prelude/kotlin/kotlin_utils.bzl | 27 + prelude/kotlin/kotlincd_jar_creator.bzl | 109 +- prelude/linking/add_elf_sections.bzl | 108 ++ prelude/linking/generated_build_info.bzl | 330 ++++ prelude/linking/link_groups.bzl | 15 +- prelude/linking/link_info.bzl | 70 +- prelude/linking/linkable_graph.bzl | 10 + prelude/linking/shared_libraries.bzl | 19 +- prelude/linking/stamp_build_info.bzl | 68 +- prelude/platforms/apple/platforms.bzl | 2 +- prelude/python/compile.bzl | 16 +- prelude/python/interface.bzl | 6 +- prelude/python/internal_tools.bzl | 4 + prelude/python/linking/link_helper.bzl | 28 +- prelude/python/linking/native.bzl | 74 +- prelude/python/make_py_package.bzl | 41 +- prelude/python/prebuilt_python_library.bzl | 15 +- prelude/python/python.bzl | 28 +- prelude/python/python_binary.bzl | 50 +- prelude/python/python_library.bzl | 62 +- prelude/python/sourcedb/pyrefly.bxl | 2 +- prelude/python/toolchain.bzl | 5 +- prelude/python/tools/BUCK | 17 + prelude/python/tools/__test_main__.py | 1 + prelude/python/tools/compile.py | 2 +- prelude/python/tools/empty_typing_result.json | 1 + prelude/python/tools/gen_bytecode_bundle.py | 145 ++ .../tools/make_py_package_manifest_module.py | 39 +- .../python/tools/make_py_package_modules.py | 4 +- .../tools/tests/gen_bytecode_bundle_test.py | 299 ++++ .../make_py_package_manifest_module_test.py | 86 + .../type_check_result_to_validation_test.py | 10 + prelude/python/tools/tests/wheel_test.py | 33 + .../tools/type_check_result_to_validation.py | 2 +- prelude/python/tools/wheel.py | 10 +- prelude/python/transitions.bzl | 41 +- prelude/python/typing.bzl | 11 +- prelude/resources.bzl | 5 + prelude/rules.bzl | 15 +- prelude/rules_impl.bzl | 37 +- prelude/rust/build.bzl | 817 ++++++---- prelude/rust/build_params.bzl | 11 + .../rust/buildscript/buildscript_platform.bzl | 1 + prelude/rust/cargo_package.bzl | 59 +- prelude/rust/context.bzl | 46 +- prelude/rust/extern.bzl | 11 +- prelude/rust/outputs.bzl | 11 +- prelude/rust/rust_binary.bzl | 184 ++- prelude/rust/rust_library.bzl | 55 +- prelude/rust/rust_toolchain.bzl | 16 + prelude/rust/tools/extract_link_action.py | 167 +- prelude/rust/tools/rustc_action.py | 92 +- prelude/target_stats/target_stats.bzl | 129 +- prelude/target_stats/target_stats_config.bzl | 6 +- prelude/target_stats/target_stats_tools.bzl | 11 +- prelude/target_stats/target_stats_types.bzl | 22 +- prelude/toolchains/android.bzl | 41 +- .../buck/android/AdbExecutionContext.java | 10 - .../com/facebook/buck/android/AdbHelper.java | 128 +- .../src/com/facebook/buck/android/aapt/BUCK | 44 +- .../facebook/buck/android/aapt/MiniAapt.java | 34 +- .../buck/android/aapt/RDotTxtEntry.java | 75 +- .../aapt/ResourceSourceMapExecutableMain.kt | 246 +++ .../android/aapt/ResourceSourceMapFormat.kt | 121 ++ .../dex/D8OutputOption.java} | 18 +- .../facebook/buck/android/dex/D8Utils.java | 14 +- .../android/dex/MultiDexExecutableMain.java | 13 +- ...SecondaryDexCompressionExecutableMain.java | 33 +- .../android/exopackage/AndroidDevice.java | 12 +- .../android/exopackage/AndroidDeviceImpl.kt | 63 +- .../exopackage/ExopackageInstaller.java | 359 +++-- .../com/facebook/buck/android/resources/BUCK | 2 - .../resources/ExoResourcesRewriter.java | 81 +- .../ExoResourcesRewriterExecutableMain.java | 48 +- .../resources/MergeAndroidResources.java | 123 +- .../MergeAndroidResourcesExecutableMain.java | 10 - .../resources/MergeAssetsExecutableMain.java | 4 - .../android/resources/MergeAssetsUtils.java | 42 +- .../android/resources/ResTablePackage.java | 38 +- .../buck/android/resources/ResTableType.java | 75 +- .../resources/ResourceProcessingConfig.java | 36 - .../buck/android/resources/ResourcesXml.java | 38 +- .../resources/UsedResourcesFinder.java | 11 +- .../buck/cd/resources/proto/common.proto | 3 +- .../buck/cd/resources/proto/javacd.proto | 58 +- .../buck/cd/resources/proto/kotlincd.proto | 16 +- .../buck/installer/InstallCommand.java | 15 + .../buck/installer/InstallerService.java | 9 +- .../installer/android/AdbHelperFactory.java | 75 + .../installer/android/AndroidArtifacts.java | 304 ++-- .../installer/android/AndroidInstall.java | 115 +- .../android/AndroidInstallApkOptions.java | 4 - .../android/AndroidInstallException.kt | 8 + .../android/AndroidInstallerManager.java | 573 ++++--- .../installer/android/InstallMetrics.java | 242 +++ .../buck/installer/android/InstallState.java | 76 + .../installer/android/StreamedPushes.java | 107 ++ .../buck/installer/proto/install.proto | 4 + .../facebook/buck/jvm/cd/DepFileUtils.java | 16 +- .../buck/jvm/cd/command/BaseJarCommand.kt | 79 +- .../buck/jvm/cd/command/PostBuildParams.kt | 2 - .../facebook/buck/jvm/cd/command/kotlin/BUCK | 1 - .../cd/command/kotlin/BuildKotlinCommand.kt | 1 + .../cd/command/kotlin/KotlinExtraParams.kt | 1 - .../java/CompilerOutputPathsSerializer.java | 5 +- .../java/JarParametersSerializer.java | 3 +- .../java/LogLevelSerializer.java | 54 - .../java/ResolvedJavacOptionsSerializer.java | 2 +- ...solvedJavacPluginPropertiesSerializer.java | 13 +- .../java/ResolvedJavacSerializer.java | 48 - .../AnnotationProcessingToolSerializer.java | 46 - .../kotlin/KotlinExtraParamsSerializer.java | 15 +- .../jvm/java/BaseJavacToJarStepFactory.java | 1 + .../jvm/java/DaemonJavacToJarStepFactory.java | 2 + .../facebook/buck/jvm/java/JavaExtraParams.kt | 14 + .../com/facebook/buck/jvm/java/JavaPaths.java | 26 + .../com/facebook/buck/jvm/java/JavacStep.java | 23 +- .../src/com/facebook/buck/jvm/java/abi/BUCK | 3 +- .../javacd/main/JavaCDCommand.java | 4 +- .../stepsbuilder/javacd/main/JavaCDMain.java | 5 +- .../buck/jvm/kotlin/ClasspathUtils.java | 3 +- .../kotlin/DaemonKotlincToJarStepFactory.java | 11 +- .../buck/jvm/kotlin/JavacStepsBuilder.java | 5 +- .../buck/jvm/kotlin/KaptStepsBuilder.java | 6 - .../buck/jvm/kotlin/KosabiConfig.java | 3 + .../facebook/buck/jvm/kotlin/KotlincStep.java | 106 +- .../buck/jvm/kotlin/KspStepsBuilder.java | 91 +- .../jvm/kotlin/buildtools/BuckKotlinLogger.kt | 8 + .../cd/analytics/KotlinCDLoggingContext.kt | 3 + .../logger/KotlinCDLoggerAnalytics.kt | 1 + .../logger/model/KotlinCDLogEntry.kt | 9 + .../kotlin/cd/workertool/KotlinCDCommand.java | 2 +- .../kosabi/common/FullTypeQualifier.kt | 2 +- .../kosabi/common/NonConventionalImports.kt | 5 +- .../kosabi/common/stub/model/KFunStub.kt | 2 + .../common/stub/render/StubBytecodeRender.kt | 16 +- .../kosabi/common/stub/render/StubRender.kt | 28 +- .../kosabi/k2-jvm-abi-gen/BUCK | 8 +- .../kotlin/K2JvmAbiCommandLineProcessor.kt | 14 + .../main/kotlin/K2JvmAbiConfigurationKeys.kt | 3 + .../K2JvmAbiFirAnalysisHandlerExtension.kt | 386 ++++- .../compilercompat/2.2.0/IrNonAbiStripper.kt | 222 ++- .../compilercompat/2.3.0/IrNonAbiStripper.kt | 286 ++-- .../compose/ClassStabilityTransformer.kt | 159 +- .../kotlin/compose/ComposableTypeRewriter.kt | 94 +- .../kotlin/compose/ComposerParamInjector.kt | 105 +- .../parcelize/ParcelableCreatorEmulator.kt | 97 ++ .../ParcelizeAbiEmulationExtension.kt | 24 + .../main/kotlin/pipeline/AbiGenPipeline.kt | 1421 ++++++++++++++++- .../main/kotlin/pipeline/AbiGenRepairLog.kt | 201 +++ .../compilerplugins/kosabi/stubsgen/BUCK | 6 +- .../kosabi/stubsgen/cli/StubsGenCli.kt | 1 + .../generators/AnnotationStubsGenerator.kt | 82 +- .../ClassLevelFunctionStubsGenerator.kt | 6 +- .../stubsgen/generators/GenerationContext.kt | 127 +- .../generators/GenericStubsGenerator.kt | 26 +- .../generators/InnerClassStubsGenerator.kt | 2 +- .../generators/InterfaceStubsGenerator.kt | 92 +- .../generators/PrimitiveStubsGenerator.kt | 21 +- .../SamePackageClassStubsGenerator.kt | 34 +- .../StubsCodegenCommandLineProcessor.kt | 0 .../StubsCodegenCompilerPluginRegistrar.kt | 0 ...ubsCodegenK2FirAnalysisHandlerExtension.kt | 33 +- ...otlin.compiler.plugin.CommandLineProcessor | 0 ...in.compiler.plugin.CompilerPluginRegistrar | 0 .../facebook/buck/jvm/kotlin/ksp/Factory.kt | 2 + .../buck/jvm/kotlin/ksp/Ksp2NoOpDetector.kt | 97 ++ .../facebook/buck/jvm/kotlin/ksp/Ksp2Step.kt | 55 +- .../src/com/facebook/buck/testrunner/BUCK | 12 +- .../BasePerTestCoverageRunListener.kt | 13 +- .../facebook/buck/testrunner/JUnitRunner.java | 78 +- .../testrunner/PerTestCoverageExtension.kt | 25 + .../PerTestCoverageExtensionManager.kt | 112 ++ .../PerTestJUnitCoverageRunListener.kt | 25 + .../buck/util/environment/Platform.java | 29 +- .../facebook/buck/util/zip/ZipScrubber.java | 8 - .../facebook/buck/android/AdbHelperTest.java | 9 +- .../buck/android/TestAdbExecutionContext.java | 8 +- .../buck/android/TestAndroidDevice.java | 5 - .../test/com/facebook/buck/android/aapt/BUCK | 15 +- .../buck/android/aapt/MiniAaptTest.java | 15 +- .../buck/android/aapt/RDotTxtEntryUtil.java | 45 +- .../ResourceSourceMapExecutableMainTest.kt | 205 +++ .../exopackage/AndroidDeviceImplTest.kt | 72 +- .../exopackage/ExopackageShardingTest.java | 196 +++ .../resources/ExoResourcesRewriterTest.java | 72 - .../buck/installer/InstallerServiceTest.java | 5 + .../android/AndroidArtifactsTest.java | 242 +-- .../installer/android/AndroidInstallTest.java | 31 + .../com/facebook/buck/installer/android/BUCK | 1 + .../android/BuildExopackageInfoTest.java | 130 ++ .../installer/android/InstallMetricsTest.java | 314 ++++ .../installer/android/StreamedPushesTest.java | 111 ++ .../facebook/buck/jvm/cd/DepFileUtilsTest.kt | 16 +- .../buck/jvm/cd/serialization/kotlin/BUCK | 2 + .../kotlin/CompilerCommandDefaultsTest.java | 227 +++ .../facebook/buck/jvm/java/JavacStepTest.java | 70 + .../buck/jvm/java/abi/StubJarTest.java | 169 +- .../test/com/facebook/buck/jvm/kotlin/BUCK | 14 +- .../buck/jvm/kotlin/JarsActionMetadataTest.kt | 6 +- .../buck/jvm/kotlin/KotlincStepTest.java | 67 + .../buck/jvm/kotlin/Ksp2NoOpDetectorTest.kt | 187 +++ .../facebook/buck/jvm/kotlin/Ksp2StepTest.kt | 57 + .../buck/jvm/kotlin/KspAdapterBaseFixture.kt | 20 + .../buck/jvm/kotlin/KspStepsBuilderTest.java | 114 ++ .../logger/KotlinCDLoggerAnalyticsTest.kt | 24 + .../buck/jvm/kotlin/testutil/compiler/BUCK | 1 + .../test/com/facebook/buck/testrunner/BUCK | 13 + .../buck/testrunner/JUnitRunnerTest.kt | 80 + .../PerTestJUnitCoverageRunListenerTest.kt | 100 ++ prelude/toolchains/android/third-party/BUCK | 91 +- prelude/toolchains/android/tools/protobuf.bzl | 1 + prelude/toolchains/conan/defs.bzl | 2 +- prelude/toolchains/cxx.bzl | 3 +- prelude/toolchains/cxx/zig/defs.bzl | 1 - prelude/toolchains/demo.bzl | 21 +- prelude/toolchains/erlang.bzl | 8 + prelude/toolchains/go/go_toolchain.bzl | 64 +- prelude/toolchains/java.bzl | 31 +- prelude/toolchains/kotlin.bzl | 4 +- prelude/toolchains/ocaml.bzl | 23 +- prelude/toolchains/python.bzl | 59 +- .../constraint_override_registry.bzl | 24 + prelude/transitions/constraint_overrides.bzl | 20 +- prelude/utils/argfile.bzl | 25 + prelude/xplugins/debug_artifacts.bzl | 52 +- prelude/xplugins/types.bzl | 6 + prelude/xplugins/utils.bzl | 102 -- 441 files changed, 18631 insertions(+), 7115 deletions(-) rename prelude/{java => android}/keystore.bzl (93%) create mode 100644 prelude/android/native_build_commands.bzl create mode 100644 prelude/apple/tools/bundling/logging_utils.py rename prelude/apple/tools/bundling/{signing_context_main.py => provisioning_manifest_main.py} (59%) create mode 100644 prelude/apple/tools/bundling/resolve_signing_context_main.py create mode 100644 prelude/apple/tools/bundling/signing_context_data.py create mode 100644 prelude/apple/tools/bundling/signing_context_data_test.py create mode 100644 prelude/apple/tools/bundling/signing_info.py create mode 100644 prelude/apple/tools/bundling/signing_info_main.py create mode 100644 prelude/apple/tools/bundling/signing_info_test.py create mode 100644 prelude/apple/tools/code_signing/serialization.py create mode 100644 prelude/apple/tools/code_signing/signing_context_types.py create mode 100644 prelude/apple/tools/code_signing/signing_context_types_test.py create mode 100644 prelude/apple/validation/swiftmodule_change_analysis.bzl create mode 100644 prelude/apple/xcassets_asset_symbols.bzl create mode 100644 prelude/cxx/cxx_flags.bzl rename prelude/erlang/common_test/{test_binary => common}/src/list_test.erl (100%) create mode 100644 prelude/erlang/common_test/cth_hooks/src/cth_skip_cases.erl create mode 100644 prelude/erlang/erlang_erts.bzl create mode 100644 prelude/erlang/erts/BUCK rename prelude/erlang/{toolchain => erts}/extract_from_otp.erl (75%) delete mode 100644 prelude/erlang/toolchain/README.md delete mode 100755 prelude/erlang/toolchain/generate_otp_versions.py delete mode 100644 prelude/erlang/toolchain/otp_versions.bzl create mode 100644 prelude/linking/add_elf_sections.bzl create mode 100644 prelude/linking/generated_build_info.bzl create mode 100644 prelude/python/tools/empty_typing_result.json create mode 100644 prelude/python/tools/gen_bytecode_bundle.py create mode 100644 prelude/python/tools/tests/gen_bytecode_bundle_test.py create mode 100644 prelude/python/tools/tests/make_py_package_manifest_module_test.py create mode 100644 prelude/toolchains/android/src/com/facebook/buck/android/aapt/ResourceSourceMapExecutableMain.kt create mode 100644 prelude/toolchains/android/src/com/facebook/buck/android/aapt/ResourceSourceMapFormat.kt rename prelude/toolchains/android/src/com/facebook/buck/{jvm/cd/command/kotlin/AnnotationProcessingTool.kt => android/dex/D8OutputOption.java} (50%) delete mode 100644 prelude/toolchains/android/src/com/facebook/buck/android/resources/ResourceProcessingConfig.java create mode 100644 prelude/toolchains/android/src/com/facebook/buck/installer/android/AdbHelperFactory.java create mode 100644 prelude/toolchains/android/src/com/facebook/buck/installer/android/InstallMetrics.java create mode 100644 prelude/toolchains/android/src/com/facebook/buck/installer/android/InstallState.java create mode 100644 prelude/toolchains/android/src/com/facebook/buck/installer/android/StreamedPushes.java delete mode 100644 prelude/toolchains/android/src/com/facebook/buck/jvm/cd/serialization/java/LogLevelSerializer.java delete mode 100644 prelude/toolchains/android/src/com/facebook/buck/jvm/cd/serialization/java/ResolvedJavacSerializer.java delete mode 100644 prelude/toolchains/android/src/com/facebook/buck/jvm/cd/serialization/kotlin/AnnotationProcessingToolSerializer.java create mode 100644 prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/k2-jvm-abi-gen/src/main/kotlin/parcelize/ParcelableCreatorEmulator.kt create mode 100644 prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/k2-jvm-abi-gen/src/main/kotlin/parcelize/ParcelizeAbiEmulationExtension.kt create mode 100644 prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/k2-jvm-abi-gen/src/main/kotlin/pipeline/AbiGenRepairLog.kt rename prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/plugin/{stubsgen_k2 => }/StubsCodegenCommandLineProcessor.kt (100%) rename prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/plugin/{stubsgen_k2 => }/StubsCodegenCompilerPluginRegistrar.kt (100%) rename prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/plugin/{stubsgen_k2 => }/StubsCodegenK2FirAnalysisHandlerExtension.kt (65%) rename prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/plugin/{stubsgen_k2 => }/resources/META-INF/services/org.jetbrains.kotlin.compiler.plugin.CommandLineProcessor (100%) rename prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/plugin/{stubsgen_k2 => }/resources/META-INF/services/org.jetbrains.kotlin.compiler.plugin.CompilerPluginRegistrar (100%) create mode 100644 prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/ksp/Ksp2NoOpDetector.kt create mode 100644 prelude/toolchains/android/src/com/facebook/buck/testrunner/PerTestCoverageExtension.kt create mode 100644 prelude/toolchains/android/src/com/facebook/buck/testrunner/PerTestCoverageExtensionManager.kt create mode 100644 prelude/toolchains/android/test/com/facebook/buck/android/aapt/ResourceSourceMapExecutableMainTest.kt create mode 100644 prelude/toolchains/android/test/com/facebook/buck/android/exopackage/ExopackageShardingTest.java create mode 100644 prelude/toolchains/android/test/com/facebook/buck/installer/android/AndroidInstallTest.java create mode 100644 prelude/toolchains/android/test/com/facebook/buck/installer/android/BuildExopackageInfoTest.java create mode 100644 prelude/toolchains/android/test/com/facebook/buck/installer/android/InstallMetricsTest.java create mode 100644 prelude/toolchains/android/test/com/facebook/buck/installer/android/StreamedPushesTest.java create mode 100644 prelude/toolchains/android/test/com/facebook/buck/jvm/cd/serialization/kotlin/CompilerCommandDefaultsTest.java create mode 100644 prelude/toolchains/android/test/com/facebook/buck/jvm/kotlin/KotlincStepTest.java create mode 100644 prelude/toolchains/android/test/com/facebook/buck/jvm/kotlin/Ksp2NoOpDetectorTest.kt create mode 100644 prelude/toolchains/android/test/com/facebook/buck/jvm/kotlin/Ksp2StepTest.kt create mode 100644 prelude/toolchains/android/test/com/facebook/buck/jvm/kotlin/KspAdapterBaseFixture.kt create mode 100644 prelude/toolchains/android/test/com/facebook/buck/jvm/kotlin/KspStepsBuilderTest.java create mode 100644 prelude/toolchains/android/test/com/facebook/buck/testrunner/JUnitRunnerTest.kt create mode 100644 prelude/transitions/constraint_override_registry.bzl diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 31ecccc..21f9059 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,18 +20,17 @@ jobs: ghc-version: ${{ matrix.ghc-version }} cabal-version: '3.14.2.0' - # Pinned, not `latest`: the `latest` tag is a rolling nightly, and - # buck2 core's builtin API drifts against whatever prelude/ snapshot - # is checked into this repo - confirmed directly, `latest` (as of - # 2026-09-16, built from the 2026-09-15 nightly) fails to even - # *load* prelude//cxx/link.bzl ("Unexpected parameter named - # `eager_materialization_enabled`"), while the 2026-09-01 dated - # release still builds and tests this repo's example/ cleanly in - # both dev and opt mode. Bump this tag deliberately, together with - # prelude/, rather than letting it silently float. + # Pinned, not `latest`: buck2 core's builtin API drifts against + # whatever prelude/ snapshot is checked into this repo (e.g. the + # previous snapshot passed `eager_materialization_enabled` to + # `ctx.actions.run`, which releases after 2026-09-01 reject, while + # this one needs releases from 2026-10-01 on). prelude/ is upstream's + # as of facebook/buck2@2484d70a6f8a (the commit this release was cut + # from) plus this repo's changes - bump this tag deliberately, + # together with prelude/, rather than letting it silently float. - name: Install buck2 run: | - curl -fL https://github.com/facebook/buck2/releases/download/2026-09-01/buck2-x86_64-unknown-linux-gnu.zst -o /tmp/buck2.zst + curl -fL https://github.com/facebook/buck2/releases/download/2026-10-01/buck2-x86_64-unknown-linux-gnu.zst -o /tmp/buck2.zst zstd -d /tmp/buck2.zst -o /usr/local/bin/buck2 chmod +x /usr/local/bin/buck2 buck2 --version diff --git a/prelude/BUCK b/prelude/BUCK index fd69f01..0276b6c 100644 --- a/prelude/BUCK +++ b/prelude/BUCK @@ -1,4 +1,5 @@ load("@prelude//:none.bzl", "none_rule") +load("@prelude//transitions:constraint_override_registry.bzl", "constraint_override_refs", "constraint_override_registry") load("@prelude//utils:source_listing.bzl", "source_listing") load(":native.bzl", prelude = "native") @@ -60,3 +61,9 @@ none_rule( name = "none", visibility = ["PUBLIC"], ) + +constraint_override_registry( + name = "constraint_override_registry", + refs = constraint_override_refs(), + visibility = ["PUBLIC"], +) diff --git a/prelude/android/android.bzl b/prelude/android/android.bzl index 935d812..784c82d 100644 --- a/prelude/android/android.bzl +++ b/prelude/android/android.bzl @@ -18,6 +18,7 @@ load(":android_prebuilt_aar.bzl", "android_prebuilt_aar_impl") load(":android_resource.bzl", "android_resource_impl") load(":apk_genrule.bzl", "apk_genrule_impl") load(":gen_aidl.bzl", "gen_aidl_impl") +load(":keystore.bzl", "keystore_impl") load(":prebuilt_native_library.bzl", "prebuilt_native_library_impl") load(":robolectric_test.bzl", "robolectric_test_impl") load(":voltron.bzl", "android_app_modularity_impl") @@ -36,6 +37,7 @@ implemented_rules = { "android_resource": android_resource_impl, "apk_genrule": apk_genrule_impl, "gen_aidl": gen_aidl_impl, + "keystore": keystore_impl, "prebuilt_native_library": prebuilt_native_library_impl, "robolectric_test": robolectric_test_impl, } diff --git a/prelude/android/android_aar.bzl b/prelude/android/android_aar.bzl index 84a8edd..0d3308a 100644 --- a/prelude/android/android_aar.bzl +++ b/prelude/android/android_aar.bzl @@ -23,6 +23,7 @@ load("@prelude//android:cpu_filters.bzl", "CPU_FILTER_FOR_DEFAULT_PLATFORM", "CP load("@prelude//android:util.bzl", "create_enhancement_context") load("@prelude//java:java_providers.bzl", "create_java_packaging_dep", "get_all_java_packaging_deps", "get_all_java_packaging_deps_from_packaging_infos") load("@prelude//java:java_toolchain.bzl", "JavaToolchainInfo") +load("@prelude//target_stats:target_stats.bzl", "target_stats_aggregate_providers_and_subtargets") load("@prelude//utils:argfile.bzl", "argfile") load("@prelude//utils:utils.bzl", "flatten") @@ -196,4 +197,7 @@ def android_aar_impl(ctx: AnalysisContext) -> list[Provider]: ctx.actions.run(create_aar_cmd, category = "create_aar") - return [DefaultInfo(default_outputs = [aar], sub_targets = enhancement_ctx.get_sub_targets() | sub_targets)] + target_stats_providers, target_stats_subtargets = target_stats_aggregate_providers_and_subtargets(ctx, deps = deps) + sub_targets.update(target_stats_subtargets) + + return [DefaultInfo(default_outputs = [aar], sub_targets = enhancement_ctx.get_sub_targets() | sub_targets)] + target_stats_providers diff --git a/prelude/android/android_apk.bzl b/prelude/android/android_apk.bzl index 4b3aeed..e4ffda6 100644 --- a/prelude/android/android_apk.bzl +++ b/prelude/android/android_apk.bzl @@ -18,14 +18,15 @@ load( "AndroidBinaryResourcesInfo", "DexFilesInfo", "ExopackageInfo", + "KeystoreInfo", ) load("@prelude//android:android_toolchain.bzl", "AndroidToolchainInfo") load("@prelude//android:util.bzl", "package_validators_decorator") load("@prelude//java:class_to_srcs.bzl", "merge_class_to_source_map_from_jar") -load("@prelude//java:java_providers.bzl", "KeystoreInfo") load("@prelude//java:java_toolchain.bzl", "JavaToolchainInfo") load("@prelude//java/utils:java_more_utils.bzl", "get_path_separator_for_exec_os") load("@prelude//java/utils:java_utils.bzl", "get_class_to_source_map_info") +load("@prelude//target_stats:target_stats.bzl", "target_stats_aggregate_providers_and_subtargets") load("@prelude//utils:argfile.bzl", "argfile") load("@prelude//utils:utils.bzl", "flatten") @@ -107,6 +108,13 @@ def android_apk_impl(ctx: AnalysisContext) -> list[Provider]: ) ] + # ctx.attrs.deps is split-transitioned here (one Dependency per ABI). + target_stats_providers, target_stats_subtargets = target_stats_aggregate_providers_and_subtargets( + ctx, + deps = android_binary_info.deps_by_platform[android_binary_info.primary_platform], + ) + sub_targets.update(target_stats_subtargets) + providers = [ AndroidApkInfo( apk = output_apk, @@ -156,7 +164,10 @@ def android_apk_impl(ctx: AnalysisContext) -> list[Provider]: }, ), class_to_srcs, - ] + ] + target_stats_providers + + if android_binary_info.preprocessed_java_classes_info: + providers.append(android_binary_info.preprocessed_java_classes_info) # Expose the exopackage secondary-dex dir so android_instrumentation_test can push it to the device. if exopackage_info != None and exopackage_info.secondary_dex_info != None: @@ -314,7 +325,6 @@ def get_install_config(apex_mode: bool) -> dict[str, typing.Any]: "adb_restart_on_failure": read_root_config("adb", "adb_restart_on_failure", "true"), "apex_mode": apex_mode, "multi_install_mode": read_root_config("adb", "multi_install_mode", "false"), - "skip_install_metadata": read_root_config("adb", "skip_install_metadata", "false"), "staged_install_mode": read_root_config("adb", "staged_install_mode", None), } diff --git a/prelude/android/android_binary.bzl b/prelude/android/android_binary.bzl index 7d179fd..e3e849b 100644 --- a/prelude/android/android_binary.bzl +++ b/prelude/android/android_binary.bzl @@ -14,6 +14,7 @@ load( "AndroidBinaryNativeLibsInfo", # @unused Used as type "AndroidBinaryResourcesInfo", # @unused Used as type "AndroidBuildConfigInfo", # @unused Used as type + "AndroidPreprocessedJavaClassesInfo", # @unused Used as type "BuildConfigField", "DexFilesInfo", "merge_android_packageable_info", @@ -27,8 +28,6 @@ load("@prelude//android:exopackage.bzl", "get_exopackage_flags") load("@prelude//android:preprocess_java_classes.bzl", "get_preprocessed_java_classes") load("@prelude//android:util.bzl", "create_enhancement_context") load("@prelude//android:voltron.bzl", "get_target_to_module_mapping") -load("@prelude//java:dex.bzl", "get_dex_produced_from_java_library") -load("@prelude//java:dex_toolchain.bzl", "DexToolchainInfo") load( "@prelude//java:java_providers.bzl", "JavaPackagingDep", # @unused Used as type @@ -40,18 +39,6 @@ load( load("@prelude//java:proguard.bzl", "get_proguard_output") load("@prelude//utils:expect.bzl", "expect") -def _preprocessed_jars_args(jars: list[Artifact]): - return cmd_args(jars) - -# Holds the preprocessed jar set in a single node so that each pre_dex action can reference one -# shared projection. Handing the raw list to every action instead retains a copy of it per action, -# which is quadratic in the number of jars. -PreprocessedJarsTSet = transitive_set( - args_projections = { - "jars": _preprocessed_jars_args, - }, -) - AndroidBinaryInfo = record( sub_targets = dict, java_packaging_deps = list[JavaPackagingDep], @@ -61,12 +48,14 @@ AndroidBinaryInfo = record( native_library_info = AndroidBinaryNativeLibsInfo, resources_info = AndroidBinaryResourcesInfo, materialized_artifacts = list[Artifact], + preprocessed_java_classes_info = AndroidPreprocessedJavaClassesInfo | None, validation_outputs = list[Artifact], ) def get_binary_info(ctx: AnalysisContext, use_proto_format: bool) -> AndroidBinaryInfo: sub_targets = {} materialized_artifacts = [] + preprocessed_java_classes_info = None deps_by_platform = get_deps_by_platform(ctx) primary_platform = CPU_FILTER_FOR_PRIMARY_PLATFORM if CPU_FILTER_FOR_PRIMARY_PLATFORM in deps_by_platform else CPU_FILTER_FOR_DEFAULT_PLATFORM @@ -88,8 +77,6 @@ def get_binary_info(ctx: AnalysisContext, use_proto_format: bool) -> AndroidBina ) should_pre_dex = not ctx.attrs.disable_pre_dex and not has_proguard_config and not ctx.attrs.preprocess_java_classes_bash - preprocess_predex_merge = ctx.attrs.preprocess_java_classes_bash and not has_proguard_config and not ctx.attrs.disable_pre_dex - enhancement_ctx = create_enhancement_context(ctx) if target_to_module_mapping_file: enhancement_ctx.debug_output("module.mapping", target_to_module_mapping_file) @@ -132,7 +119,7 @@ def get_binary_info(ctx: AnalysisContext, use_proto_format: bool) -> AndroidBina use_proto_format = use_proto_format, referenced_resources_lists = referenced_resources_lists, manifest_entries = ctx.attrs.manifest_entries, - generate_strings_and_ids_separately = should_pre_dex or preprocess_predex_merge, + generate_strings_and_ids_separately = should_pre_dex, aapt2_preferred_density = ctx.attrs.aapt2_preferred_density, ) sub_targets["manifest"] = [DefaultInfo(default_output = resources_info.manifest)] @@ -157,7 +144,7 @@ def get_binary_info(ctx: AnalysisContext, use_proto_format: bool) -> AndroidBina default_output = r_dot_java_info.source_zipped, ), ] - validation_outputs = [] + validation_outputs = list(native_library_info.validation_outputs) dex_java_packaging_deps = [ packaging_dep for packaging_dep in java_packaging_deps if packaging_dep.dex and packaging_dep.dex.dex.owner.raw_target() not in no_dx_target_labels ] @@ -192,9 +179,9 @@ def get_binary_info(ctx: AnalysisContext, use_proto_format: bool) -> AndroidBina ) if ctx.attrs.preprocess_java_classes_bash: - jars_to_owners, materialized_artifacts_dir = get_preprocessed_java_classes(enhancement_ctx, jars_to_owners) - if materialized_artifacts_dir: - materialized_artifacts.append(materialized_artifacts_dir) + jars_to_owners, preprocessed_java_classes_info = get_preprocessed_java_classes(enhancement_ctx, jars_to_owners) + if preprocessed_java_classes_info: + materialized_artifacts.append(preprocessed_java_classes_info.materialized_artifacts_dir) if has_proguard_config: additional_proguard_configs = ( [resources_info.proguard_config_file] if not ctx.attrs.ignore_aapt_proguard_config and resources_info.proguard_config_file else [] @@ -236,75 +223,7 @@ def get_binary_info(ctx: AnalysisContext, use_proto_format: bool) -> AndroidBina else: proguard_output = None - if preprocess_predex_merge: - dex_toolchain = ctx.attrs._dex_toolchain[DexToolchainInfo] - preprocessed_jars = list(jars_to_owners.keys()) - - # R.java jars are byte-light but field-heavy (one field per resource id) and are - # re-dexed from scratch here. Dexing them with the default weight factor packs the - # whole R.java jar into a single secondary dex, overflowing the 64K field-reference - # limit. Preserve the r_dot_java_weight_factor that compiled_r_dot_java_deps applies - # on the non-preprocessed path so R.java spreads across secondary dexes here too. - r_dot_java_jar_basenames = [dep.jar.basename for dep in compiled_r_dot_java_deps] - - # Every jar desugars against the whole preprocessed set, so the classpath is identical - # for all of them. Share one projection and one classpath file rather than rebuilding - # both per jar. - desugar_deps = ctx.actions.tset(PreprocessedJarsTSet, value = preprocessed_jars).project_as_args("jars") - desugar_deps_file = ctx.actions.write( - "preprocessed_desugar_deps_file.txt", - desugar_deps, - has_content_based_path = True, - ) - pre_dexed_libs = [] - pre_dexed_artifacts_to_owners = {} - for jar in preprocessed_jars: - weight_factor = 1 - for r_dot_java_jar_basename in r_dot_java_jar_basenames: - if jar.basename.endswith(r_dot_java_jar_basename): - weight_factor = android_toolchain.r_dot_java_weight_factor * 2 - break - pre_dexed_lib = get_dex_produced_from_java_library( - ctx, - dex_toolchain = dex_toolchain, - jar_to_dex = jar, - needs_desugar = True, - desugar_deps = desugar_deps, - weight_factor = weight_factor, - desugar_deps_file = desugar_deps_file, - ) - pre_dexed_libs.append(pre_dexed_lib) - if pre_dexed_lib.dex: - pre_dexed_artifacts_to_owners[pre_dexed_lib.dex] = jars_to_owners[jar] - if ctx.attrs.use_split_dex: - multidex_min_api = ctx.attrs.multidex_min_api - if multidex_min_api == None: - multidex_min_api = getattr(ctx.attrs, "_dex_min_sdk_version", None) - if multidex_min_api == None: - multidex_min_api = ctx.attrs.min_sdk_version - if multidex_min_api != None and int(multidex_min_api) > 21 and not ctx.attrs.primary_dex_patterns: - dex_files_info = get_multi_dex( - ctx, - android_toolchain, - pre_dexed_artifacts_to_owners, - ctx.attrs.primary_dex_patterns, - apk_module_graph_file = target_to_module_mapping_file, - enable_bootstrap_dexes = ctx.attrs.enable_bootstrap_dexes, - multidex_min_api = str(multidex_min_api), - pre_dexed_inputs = True, - ) - else: - dex_files_info = merge_to_split_dex( - ctx, - android_toolchain, - pre_dexed_libs, - get_split_dex_merge_config(ctx, android_toolchain), - target_to_module_mapping_file, - enable_bootstrap_dexes = ctx.attrs.enable_bootstrap_dexes, - ) - else: - dex_files_info = merge_to_single_dex(ctx, android_toolchain, pre_dexed_libs) - elif ctx.attrs.use_split_dex: + if ctx.attrs.use_split_dex: dex_files_info = get_multi_dex( ctx, android_toolchain, @@ -344,6 +263,7 @@ def get_binary_info(ctx: AnalysisContext, use_proto_format: bool) -> AndroidBina native_library_info = native_library_info, resources_info = resources_info, materialized_artifacts = materialized_artifacts, + preprocessed_java_classes_info = preprocessed_java_classes_info, validation_outputs = validation_outputs, ) diff --git a/prelude/android/android_binary_native_library_rules.bzl b/prelude/android/android_binary_native_library_rules.bzl index a5b0685..792f063 100644 --- a/prelude/android/android_binary_native_library_rules.bzl +++ b/prelude/android/android_binary_native_library_rules.bzl @@ -16,6 +16,14 @@ load( ) load("@prelude//android:android_toolchain.bzl", "AndroidToolchainInfo") load("@prelude//android:cpu_filters.bzl", "CPU_FILTER_FOR_PRIMARY_PLATFORM", "CPU_FILTER_TO_ABI_DIRECTORY") +load( + "@prelude//android:native_build_commands.bzl", + "EMIT_NATIVE_BUILD_COMMANDS", + "GATORADE_PHASE_SUBTARGETS", + "NATIVE_BUILD_COMMAND_KINDS", + "native_build_command_entry", + "record_link_command", +) load("@prelude//android:relinker_linker_outputs.bzl", "get_extra_relinker_args") load("@prelude//android:util.bzl", "EnhancementContext", "merge_extra_linker_args") load("@prelude//android:voltron.bzl", "ROOT_MODULE", "all_targets_in_root_module", "get_apk_module_graph_info", "is_root_module") @@ -44,11 +52,13 @@ load("@prelude//java:java_library.bzl", "compile_to_jar") # @unused load("@prelude//linking:execution_preference.bzl", "LinkExecutionPreference", "get_action_execution_attributes") load( "@prelude//linking:link_info.bzl", + "ArchiveLinkable", "LibOutputStyle", "LinkArgs", "LinkInfo", "LinkOrdering", "LinkedObject", + "ObjectsLinkable", "SharedLibLinkable", "get_lib_output_style", "set_link_info_link_whole", @@ -80,6 +90,8 @@ load("@prelude//utils:expect.bzl", "expect") load("@prelude//utils:graph_utils.bzl", "post_order_traversal", "pre_order_traversal", "rust_matching_topological_traversal") load("@prelude//utils:utils.bzl", "dedupe_by_value") +_GATORADE_PHASE_ORDER = ["early", "middle", "late"] + # Native libraries on Android are built for a particular Application Binary Interface (ABI). We # package native libraries for one (or more, for multi-arch builds) ABIs into an Android APK. # @@ -114,6 +126,73 @@ def _merged_lib_provided_by_apk_under_test(merged_lib, shared_libraries_to_exclu return False return True +def _register_gatorade_phase_evidence( + enhance_ctx: EnhancementContext, + platforms: list[str], + native_library_merge_dir: Artifact | None, + middle_gatorade_products: Artifact | None, + unstripped_native_libraries_files: Artifact, + relinked_libs_output: Artifact | None, + defer_relink: bool, +) -> None: + ctx = enhance_ctx.ctx + configured = getattr(ctx.attrs, "gatorade_phases", []) + configured_phases = [phase for phase in _GATORADE_PHASE_ORDER if phase in configured] + phase_outputs = {} + phase_dependencies = {} + if "early" in configured_phases and native_library_merge_dir != None: + phase_outputs["early"] = native_library_merge_dir + phase_dependencies["early"] = "merge_sequence" + if "middle" in configured_phases and middle_gatorade_products != None: + phase_outputs["middle"] = middle_gatorade_products + phase_dependencies["middle"] = "middle_products" + if "late" in configured_phases: + phase_outputs["late"] = relinked_libs_output if defer_relink else unstripped_native_libraries_files + phase_dependencies["late"] = "relinked_libraries" if defer_relink else "final_unstripped_libraries" + if not phase_outputs: + return + + evidence = ctx.actions.declare_output( + "gatorade_phase_evidence.json", + has_content_based_path = False, + ) + # Preserve the associated inputs so building this manifest also materializes + # the phase products it certifies. + evidence_inputs = ctx.actions.write_json( + evidence, + { + "configured_phases": configured_phases, + "defer_relink": defer_relink, + "engaged_phases": [phase for phase in _GATORADE_PHASE_ORDER if phase in phase_outputs], + "phase_dependencies": phase_dependencies, + "phase_outputs": phase_outputs, + "platforms": sorted(platforms), + "schema_version": 1, + "target": str(ctx.label.raw_target()), + }, + pretty = True, + with_inputs = True, + ) + enhance_ctx.debug_output( + "gatorade_phase_evidence", + evidence, + other_outputs = [evidence_inputs], + ) + +def _materialize_middle_gatorade_products( + ctx: AnalysisContext, + output, + products_by_platform: dict[str, Artifact], + relinked_libraries_by_platform: dict[str, dict[str, SharedLibrary]], +) -> None: + entries = {} + for platform in sorted(products_by_platform): + entries["{}/products".format(platform)] = products_by_platform[platform] + relinked_libraries = relinked_libraries_by_platform[platform] + for soname in sorted(relinked_libraries): + entries["{}/relinked/{}".format(platform, soname)] = relinked_libraries[soname].lib.output + ctx.actions.symlinked_dir(output, entries) + def get_android_binary_native_library_info( enhance_ctx: EnhancementContext, android_packageable_info: AndroidPackageableInfo, @@ -153,6 +232,22 @@ def get_android_binary_native_library_info( enhance_ctx.debug_output("native_libs", ctx.actions.write("native_libs", [], has_content_based_path = False)) enhance_ctx.debug_output("linker_argsfiles", ctx.actions.symlinked_dir("linker_argsfiles", {}, has_content_based_path = False)) enhance_ctx.debug_output("linker_commands", ctx.actions.write("linker_commands", [], has_content_based_path = False)) + enhance_ctx.debug_output( + "native_build_commands", + ctx.actions.write_json("native_build_commands", [], has_content_based_path = False), + # Expose the same empty per-kind filter sub_targets as the main path so + # TARGET[native_build_commands][] resolves uniformly (to empty JSON) even on + # apps with no native libraries. + sub_targets = { + kind: [DefaultInfo(default_outputs = [ctx.actions.write_json("native_build_commands.{}.json".format(kind), [], has_content_based_path = False)])] + for kind in NATIVE_BUILD_COMMAND_KINDS + }, + ) + # The top-level Gatorade-phase sub-targets output that phase's produced artifacts (an empty + # dir here: an app with no native libraries runs no Gatorade phase), so they resolve + # uniformly. See the main path for the populated case. + for gatorade_phase in GATORADE_PHASE_SUBTARGETS: + enhance_ctx.debug_output(gatorade_phase, ctx.actions.symlinked_dir(gatorade_phase, {}, has_content_based_path = False)) enhance_ctx.debug_output("unstripped_native_libraries", ctx.actions.write("unstripped_native_libraries", [], has_content_based_path = False)) enhance_ctx.debug_output( "unstripped_native_libraries_json", ctx.actions.write_json("unstripped_native_libraries_json", {}, has_content_based_path = False) @@ -160,6 +255,7 @@ def get_android_binary_native_library_info( enhance_ctx.debug_output( "unstripped_native_libraries_files", ctx.actions.symlinked_dir("unstripped_native_libraries_files", {}, has_content_based_path = False) ) + enhance_ctx.debug_output("relinker_extra_outputs", ctx.actions.symlinked_dir("relinker_extra_outputs", {}, has_content_based_path = False)) return AndroidBinaryNativeLibsInfo( prebuilt_native_library_dirs = [], shared_libraries = [], @@ -169,6 +265,7 @@ def get_android_binary_native_library_info( non_root_module_native_lib_assets = [], generated_java_code = [], unstripped_shared_libraries = None, + validation_outputs = [], ) native_libs = ctx.actions.declare_output("native_libs_symlink", has_content_based_path = False) @@ -204,6 +301,20 @@ def get_android_binary_native_library_info( non_root_module_lib_assets, ] + # [native_build_commands] outputs. `native_build_commands` (full) + the per-kind filters are + # produced by a final combine action (see below) that merges two fragments: the "base" fragment + # (all commands known by the end of the outer dynamic lambda) and the late-Gatorade "codegen" + # fragment. The codegen commands (clang --codegen-library) are constructed inside a NESTED + # Gatorade dynamic that runs after the outer lambda, so they cannot be folded in by the outer + # lambda directly and instead flow through this fragment + combine. + native_build_commands = ctx.actions.declare_output("native_build_commands.json", has_content_based_path = False) + native_build_commands_by_kind = { + kind: ctx.actions.declare_output("native_build_commands.{}.json".format(kind), has_content_based_path = False) for kind in NATIVE_BUILD_COMMAND_KINDS + } + native_build_commands_base = ctx.actions.declare_output("native_build_commands.base.json", has_content_based_path = False) + native_build_commands_codegen = ctx.actions.declare_output("native_build_commands.late_gatorade_codegen.json", has_content_based_path = False) + dynamic_outputs.append(native_build_commands_base) + fake_input = ctx.actions.write("dynamic.trigger", "", has_content_based_path = False) # some cases don't actually need to use a dynamic_output, but it's simplest to consistently use it anyway. we need some fake input to allow that. @@ -228,14 +339,56 @@ def get_android_binary_native_library_info( linkable_nodes_by_platform = {} + # Analysis-scope [native_build_commands] entries (e.g. compute_mergemap) that run before the + # dynamic lambda. The lambda seeds its own list from a copy of this, then appends the + # in-lambda link/bolt commands. Entries created inside the frozen lambda cannot be appended + # here, so anything produced at analysis time is collected into this list instead. + native_build_command_entries = [] + + # TARGET[] Gatorade-phase product sub-targets. Each outputs the full set of artifacts that + # phase's gatorade invocation(s) produce, as a symlinked_dir (like [native_libs]). + # - early runs at analysis scope: early_gatorade_libraries fills this {relpath: artifact} map, + # and early_gatorade_products is built from it below. + # - middle reuses middle_gatorade_products, declared below and materialized inside the dynamic + # lambda when the middle phase runs. + # - late runs inside the nested Gatorade dynamics, so its dir is declared here, appended to + # dynamic_outputs, and bound inside (empty when the phase is off). + early_gatorade_product_mapping = {} + late_gatorade_products = ctx.actions.declare_output("late_gatorade_products", dir = True, has_content_based_path = False) + dynamic_outputs.append(late_gatorade_products) + has_native_merging = native_library_merge_sequence or native_library_merge_map enable_relinker = getattr(ctx.attrs, "enable_relinker", False) defer_relink = getattr(ctx.attrs, "defer_relink", False) and enable_relinker + middle_gatorade_products = None + if enable_relinker and "middle" in getattr(ctx.attrs, "gatorade_phases", []): + middle_gatorade_products = ctx.actions.declare_output( + "middle_gatorade_products", + dir = True, + has_content_based_path = False, + ) + dynamic_outputs.append(middle_gatorade_products) + + # The outer lambda owns the codegen fragment. With late Gatorade, the nested Gatorade dynamic + # (gatorade_libraries or gatorade_deferred_libs) binds it through the passed OutputArtifact; + # otherwise the outer lambda binds it with an empty list, so the combine always has both inputs. + # @oss-disable[end= ]: will_capture_late_gatorade_codegen = is_late_gatorade_enabled(ctx) + will_capture_late_gatorade_codegen = False # @oss-enable + dynamic_outputs.append(native_build_commands_codegen) + if has_native_merging or enable_relinker: native_merge_debug = ctx.actions.declare_output("native_merge_debug", dir = True, has_content_based_path = False) dynamic_outputs.append(native_merge_debug) + # Argsfiles referenced by [native_build_commands][compile] entries, attached as other_outputs + # below. Keyed by the Artifact, not short_path: short_path is only the extension-based filename + # (e.g. `.cpp.cxx_compile_argsfile`), identical across every library and platform. + compile_argsfiles = {} + + # EMIT_NATIVE_BUILD_COMMANDS also needs the node map (compile entries read each node's + # compile_cmds), so compile capture works on apps that neither merge nor relink. + if has_native_merging or enable_relinker or EMIT_NATIVE_BUILD_COMMANDS: # We serialize info about the linkable graph and the apk module mapping and pass that to an # external subcommand to apply a merge sequence algorithm and return us the merge mapping. for platform, deps in deps_by_platform.items(): @@ -246,6 +399,15 @@ def get_android_binary_native_library_info( linkables_debug = ctx.actions.write("linkables." + platform, list(graph_node_map.keys()), has_content_based_path = False) enhance_ctx.debug_output("linkables." + platform, linkables_debug) linkable_nodes_by_platform[platform] = graph_node_map + if EMIT_NATIVE_BUILD_COMMANDS: + for node in graph_node_map.values(): + for cc in node.compile_cmds: + compile_argsfiles[cc.cxx_compile_cmd.argsfile.file] = None + + jni_onload_check_report = None + if has_native_merging and ctx.attrs._android_toolchain[AndroidToolchainInfo].jni_onload_check: + jni_onload_check_report = ctx.actions.declare_output("jni_onload_check.json", has_content_based_path = False) + dynamic_outputs.append(jni_onload_check_report) relinked_libs_output = None relinked_libs_manifest = None @@ -264,6 +426,13 @@ def get_android_binary_native_library_info( unrelinked_libs_output = ctx.actions.declare_output("unrelinked_libs", dir = True) dynamic_outputs.append(unrelinked_libs_output) + # The `extra_relinker_outputs` of every relinked library, as + # ///, exposed as the [relinker_extra_outputs] sub-target so + # they can be built (and so put on disk) without knowing their paths. Always + # present; empty when nothing is configured or nothing is relinked. + relinker_extra_outputs = ctx.actions.declare_output("relinker_extra_outputs", dir = True, has_content_based_path = False) + dynamic_outputs.append(relinker_extra_outputs) + lib_outputs_by_platform = _declare_library_subtargets( ctx, dynamic_outputs, original_shared_libs_by_platform, native_library_merge_map, native_library_merge_sequence, enable_relinker ) @@ -289,8 +458,25 @@ def get_android_binary_native_library_info( native_library_merge_dir = ctx.actions.declare_output("merge_sequence_output", has_content_based_path = False) native_library_merge_map = native_library_merge_dir.project("merge.map") split_groups_map = native_library_merge_dir.project("split_groups.map") - mergemap_cmd.add(cmd_args(native_library_merge_dir.as_output(), format = "--output={}")) - ctx.actions.run(mergemap_cmd, category = "compute_mergemap", allow_cache_upload = True) + + # Record the mergemap command before adding the output as an `as_output()` (which + # cannot be re-serialized by write_json); reference the plain output artifact instead. + native_build_command_entries.append( + native_build_command_entry( + kind = "mergemap", + category = "compute_mergemap", + arch = None, + soname = None, + output = native_library_merge_dir.short_path, + argv = cmd_args(mergemap_cmd, cmd_args(native_library_merge_dir, format = "--output={}")), + ) + ) + + ctx.actions.run( + cmd_args(mergemap_cmd, cmd_args(native_library_merge_dir.as_output(), format = "--output={}")), + category = "compute_mergemap", + allow_cache_upload = True, + ) else: native_library_merge_dir = ctx.actions.declare_output("merge_sequence_output", dir = True, has_content_based_path = False) native_library_merge_map = native_library_merge_dir.project("merge.map") @@ -307,6 +493,8 @@ def get_android_binary_native_library_info( apk_module_graph_file, native_library_merge_non_asset_libs, native_library_merge_dir, + # @oss-disable[end= ]: native_build_command_entries, + # @oss-disable[end= ]: early_gatorade_product_mapping, ] # @oss-disable[end= ]: early_gatorade_libraries(*args) @@ -323,6 +511,11 @@ def get_android_binary_native_library_info( generated_java_code.append(mergemap_gencode_jar) def dynamic_native_libs_info(ctx: AnalysisContext, artifacts, outputs): + middle_gatorade_outputs_by_platform = {} + middle_gatorade_relinked_libraries_by_platform = {} + # Seed with the analysis-scope entries (frozen list, read-only copy), then append the + # in-lambda link/relink/merge/bolt commands as the pipeline produces them. + native_cmd_entries = list(native_build_command_entries) get_module_from_target = all_targets_in_root_module get_module_tdeps = all_targets_in_root_module get_calculated_module_deps = all_targets_in_root_module @@ -372,6 +565,7 @@ def get_android_binary_native_library_info( shared_object_targets = {} debug_info_by_platform = {} # dict[str, MergedLinkablesDebugInfo] merged_shared_libs_by_platform = {} # dict[str, dict[str, MergedSharedLibrary]] + jni_onload_check_inputs = [] if jni_onload_check_report else None for platform in original_shared_libs_by_platform: merged_shared_libs, debug_info = _get_merged_linkables_for_platform( ctx = ctx, @@ -384,6 +578,7 @@ def get_android_binary_native_library_info( merge_map = merge_map_by_platform[platform], merge_linker_args = native_library_merge_linker_args or {}, apk_module_graph = get_module_from_target, + jni_onload_check_inputs = jni_onload_check_inputs, ) debug_info_by_platform[platform] = debug_info merged_shared_libs_by_platform[platform] = merged_shared_libs @@ -418,6 +613,17 @@ def get_android_binary_native_library_info( ctx.actions.symlinked_dir(outputs[native_merge_debug], native_library_merge_debug_outputs) + if jni_onload_check_report: + # nm is the same across the android platforms in a binary, so + # any of them serves for a symbol-table read. + check_platform = list(original_shared_libs_by_platform.keys())[0] + _run_jni_onload_check( + ctx, + ctx.attrs._cxx_toolchain[check_platform][CxxToolchainInfo], + jni_onload_check_inputs, + outputs[jni_onload_check_report], + ) + # Merged libraries are assembled from the full linkable graph, so unlike the # non-merging path (which filters via get_default_shared_libs) they do not # otherwise honor shared_libraries_to_exclude. Drop any merged library whose @@ -443,9 +649,53 @@ def get_android_binary_native_library_info( else: final_shared_libs_by_platform = original_shared_libs_by_platform + relinked_libs_for_extra_outputs = {} + + # [native_build_commands][merge]: record every per-soname merge link here, once, before any + # relink reassigns final_shared_libs_by_platform. This is the single capture point for merge + # links across all paths (inline relink, deferred relink, and late-Gatorade), which is why + # the finalization step below never labels anything kind=merge. + if has_native_merging: + for platform, libs in final_shared_libs_by_platform.items(): + for soname, lib in libs.items(): + record_link_command(native_cmd_entries, "merge", platform, soname, lib.lib) + + # [native_build_commands][compile]: emit one entry per source in every linkable node that + # carries compile commands (populated only when `-c cxx.emit_native_build_commands=true`, so + # this whole block is skipped otherwise). Attribute each to the FINAL merged soname via the + # merge map (falling back to the node's own pre-merge soname), keeping the per-library soname + # in pre_merge_soname. The soname is best-effort: split groups, static-only nodes, and + # late-gatorade code motion make it approximate — consumers should cross-reference merge.map. + if EMIT_NATIVE_BUILD_COMMANDS: + for platform, node_map in linkable_nodes_by_platform.items(): + merged_by_label = merged_shared_lib_targets_by_platform.get(platform, {}) + for node_label, node in node_map.items(): + pre_merge_soname = node.default_soname + for cc in node.compile_cmds: + # A source compiled more than once in the same node (different flavors/flags) + # carries a non-None index; append it so those entries get distinct identifiers. + identifier = "{}/{}/{}".format(platform, node_label.raw_target(), cc.src.short_path) + if cc.index != None: + identifier += "/{}".format(cc.index) + native_cmd_entries.append( + native_build_command_entry( + kind = "compile", + category = cc.cxx_compile_cmd.category, + arch = platform, + soname = merged_by_label.get(node_label, pre_merge_soname), + pre_merge_soname = pre_merge_soname, + output = cc.src.short_path, + argv = cmd_args(cc.cxx_compile_cmd.base_compile_cmd, cc.cxx_compile_cmd.argsfile.cmd_form, cc.args), + argsfile = cc.cxx_compile_cmd.argsfile.file, + identifier = identifier, + ) + ) + if enable_relinker and not defer_relink: unrelinked_shared_libs_by_platform = final_shared_libs_by_platform - final_shared_libs_by_platform = _relink_for_native_libs(ctx, final_shared_libs_by_platform) + final_shared_libs_by_platform, middle_gatorade_outputs_by_platform = _relink_for_native_libs(ctx, final_shared_libs_by_platform, native_cmd_entries) + middle_gatorade_relinked_libraries_by_platform = final_shared_libs_by_platform + relinked_libs_for_extra_outputs = final_shared_libs_by_platform _link_library_subtargets( ctx, outputs, @@ -470,7 +720,9 @@ def get_android_binary_native_library_info( # The relinked libs are exposed as [relinked_libs] sub-target for the combine genrule. # A JSON manifest listing the / entries is produced alongside so that # the combine script knows exactly which libraries to replace without guessing. - relinked_libs_by_platform = _relink_for_native_libs(ctx, final_shared_libs_by_platform) + relinked_libs_by_platform, middle_gatorade_outputs_by_platform = _relink_for_native_libs(ctx, final_shared_libs_by_platform, native_cmd_entries) + middle_gatorade_relinked_libraries_by_platform = relinked_libs_by_platform + relinked_libs_for_extra_outputs = relinked_libs_by_platform if False: # @oss-enable # @oss-disable[end= ]: if is_late_gatorade_enabled(ctx): @@ -487,6 +739,9 @@ def get_android_binary_native_library_info( # @oss-disable[end= ]: relinked_libs_by_platform, # @oss-disable[end= ]: deferred_gatorade_output, # @oss-disable[end= ]: [outputs[relinked_libs_output], outputs[relinked_libs_manifest]], + # @oss-disable[end= ]: native_cmd_entries, + # @oss-disable[end= ]: outputs[native_build_commands_codegen], + # @oss-disable[end= ]: outputs[late_gatorade_products], # @oss-disable[end= ]: ) else: _write_native_libs_dir_and_manifest( @@ -513,8 +768,18 @@ def get_android_binary_native_library_info( stripped = False, ) - if ctx.attrs._android_toolchain[AndroidToolchainInfo].cross_module_native_deps_check: - # note: can only detect these if linkable_nodes_by_platform is created, ie. if using relinker or merging + if middle_gatorade_products != None: + _materialize_middle_gatorade_products( + ctx, + outputs[middle_gatorade_products], + middle_gatorade_outputs_by_platform, + middle_gatorade_relinked_libraries_by_platform, + ) + + if ctx.attrs._android_toolchain[AndroidToolchainInfo].cross_module_native_deps_check and (has_native_merging or enable_relinker): + # note: can only detect these if linkable_nodes_by_platform is created, ie. if using relinker or merging. + # EMIT_NATIVE_BUILD_COMMANDS also builds it on apps that do neither; those apps never ran + # this check, so it stays gated on merging/relinking. cross_module_link_errors = [] for linkable_nodes in linkable_nodes_by_platform.values(): for target, node in linkable_nodes.items(): @@ -538,6 +803,8 @@ def get_android_binary_native_library_info( + "\n".join(cross_module_link_errors) ) + _write_relinker_extra_outputs_dir(ctx, relinked_libs_for_extra_outputs, outputs[relinker_extra_outputs]) + native_lib_dynamic_outputs = { "linker_argsfiles": outputs[linker_argsfiles], "linker_commands": outputs[linker_commands], @@ -566,11 +833,22 @@ def get_android_binary_native_library_info( all_prebuilt_native_library_dirs, get_module_from_target, native_lib_dynamic_outputs, + # @oss-disable[end= ]: native_cmd_entries, + # @oss-disable[end= ]: outputs[native_build_commands_codegen], + # @oss-disable[end= ]: outputs[late_gatorade_products], ] # @oss-disable[end= ]: subtarget_shared_libs_by_platform = gatorade_libraries(*args) else: subtarget_shared_libs_by_platform = _post_native_lib_graph_finalization_steps( - ctx, final_shared_libs_by_platform, all_prebuilt_native_library_dirs, get_module_from_target, **native_lib_dynamic_outputs + ctx, + final_shared_libs_by_platform, + all_prebuilt_native_library_dirs, + get_module_from_target, + native_cmd_entries = native_cmd_entries, + # Only record the libs finalized here as plain kind=link when the app neither merges + # nor relinks; otherwise merge/relink links are already captured at their own sites. + record_final_link = not has_native_merging and not enable_relinker, + **native_lib_dynamic_outputs, ) # Subtargets can't be created or changed from within dynamic actions, so the individual @@ -588,7 +866,43 @@ def get_android_binary_native_library_info( native_merge_debug, ) + # Write the "base" fragment: every native-build command known by the end of this lambda. + # with_inputs = False embeds each argv/argsfile as text without materializing the produced + # .o/.so, matching the linker_commands writer. The final [native_build_commands] JSON and its + # per-kind filters are produced by the combine action below (which folds in the late-Gatorade + # codegen fragment). + ctx.actions.write_json(outputs[native_build_commands_base], native_cmd_entries, with_inputs = False) + + # On paths that don't capture late-Gatorade codegen, bind its fragment here (empty) so the + # combine action always has both inputs. On the capturing path the nested Gatorade dynamic + # binds it instead. + if not will_capture_late_gatorade_codegen: + ctx.actions.write_json(outputs[native_build_commands_codegen], [], with_inputs = False) + + # TARGET[late_gatorade] products are bound inside the nested Gatorade dynamic (which knows the + # output library graph); on every non-late path the outer lambda binds it empty here, mirroring + # the native_build_commands_codegen fragment's ownership. + if not will_capture_late_gatorade_codegen: + ctx.actions.symlinked_dir(outputs[late_gatorade_products], {}) + ctx.actions.dynamic_output(dynamic = dynamic_inputs, inputs = [], outputs = [o.as_output() for o in dynamic_outputs], f = dynamic_native_libs_info) + + # Combine the base fragment with the late-Gatorade codegen fragment into the final + # [native_build_commands] JSON and its per-kind filters. Both fragments were written with + # with_inputs = False, so their entries are already-rendered plain JSON; read them back and + # re-emit. This runs after the (possibly nested) writers of both fragments complete. + def _combine_native_build_commands(ctx: AnalysisContext, artifacts, outputs): + entries = artifacts[native_build_commands_base].read_json() + artifacts[native_build_commands_codegen].read_json() + ctx.actions.write_json(outputs[native_build_commands], entries) + for kind in NATIVE_BUILD_COMMAND_KINDS: + ctx.actions.write_json(outputs[native_build_commands_by_kind[kind]], [entry for entry in entries if entry["kind"] == kind]) + + ctx.actions.dynamic_output( + dynamic = [native_build_commands_base, native_build_commands_codegen], + inputs = [], + outputs = [native_build_commands.as_output()] + [native_build_commands_by_kind[kind].as_output() for kind in NATIVE_BUILD_COMMAND_KINDS], + f = _combine_native_build_commands, + ) combined_asset_libs = ctx.actions.declare_output("combined_asset_libs", dir = True, has_content_based_path = False) ctx.actions.run( cmd_args([ @@ -624,8 +938,53 @@ def get_android_binary_native_library_info( if native_merge_debug: enhance_ctx.debug_output("native_merge_debug", native_merge_debug) + if jni_onload_check_report: + # Buildable on its own so the check can be run, and its report read, + # without building the whole apk. + enhance_ctx.debug_output("jni_onload_check", jni_onload_check_report) + enhance_ctx.debug_output("linker_argsfiles", linker_argsfiles) enhance_ctx.debug_output("linker_commands", linker_commands) + # Materialize the argsfiles a consumer needs to expand an entry's `@argsfile` on disk: + # - link-family kinds (merge/link/relink/bolt) -> the linker_argsfiles symlinked_dir built in + # finalization from every final lib's linker_argsfile; + # - compile -> the per-source compile argsfiles collected at analysis time above. + # mergemap has no argsfile; Gatorade argsfiles are recorded by path only (see + # native_build_commands.bzl `argsfile` note), so those kinds get no extra outputs here. + compile_argsfiles_outputs = list(compile_argsfiles.keys()) + _kind_argsfile_outputs = {kind: [linker_argsfiles] for kind in ["merge", "link", "relink", "bolt"]} + _kind_argsfile_outputs["compile"] = compile_argsfiles_outputs + enhance_ctx.debug_output( + "native_build_commands", + native_build_commands, + other_outputs = [linker_argsfiles] + compile_argsfiles_outputs, + sub_targets = { + kind: [ + DefaultInfo( + default_outputs = [native_build_commands_by_kind[kind]], + other_outputs = _kind_argsfile_outputs.get(kind, []), + ) + ] + for kind in NATIVE_BUILD_COMMAND_KINDS + }, + ) + # TARGET[] Gatorade-phase product sub-targets: building one runs that phase and outputs + # the artifacts its gatorade invocation(s) produce (a symlinked_dir, like [native_libs]). Early's + # products were collected at analysis scope; middle/late were bound inside the dynamic lambda. + # middle_gatorade_products is only declared when the middle phase runs, so otherwise the + # sub-target resolves to an empty dir. + early_gatorade_products = ctx.actions.symlinked_dir("early_gatorade_products", early_gatorade_product_mapping, has_content_based_path = False) + gatorade_phase_products = { + "early_gatorade": early_gatorade_products, + "late_gatorade": late_gatorade_products, + "middle_gatorade": middle_gatorade_products or ctx.actions.symlinked_dir("middle_gatorade_products", {}, has_content_based_path = False), + } + expect( + sorted(gatorade_phase_products.keys()) == sorted(GATORADE_PHASE_SUBTARGETS), + "gatorade_phase_products must cover exactly GATORADE_PHASE_SUBTARGETS", + ) + for gatorade_phase, products in gatorade_phase_products.items(): + enhance_ctx.debug_output(gatorade_phase, products) enhance_ctx.debug_output("unstripped_native_libraries", unstripped_native_libraries, other_outputs = [unstripped_native_libraries_files]) enhance_ctx.debug_output("unstripped_native_libraries_json", unstripped_native_libraries_json, other_outputs = [unstripped_native_libraries_files]) enhance_ctx.debug_output("unstripped_native_libraries_files", unstripped_native_libraries_files) @@ -635,6 +994,17 @@ def get_android_binary_native_library_info( enhance_ctx.debug_output("relinked_libs_manifest", relinked_libs_manifest) if unrelinked_libs_output: enhance_ctx.debug_output("unrelinked_libs", unrelinked_libs_output) + enhance_ctx.debug_output("relinker_extra_outputs", relinker_extra_outputs) + + _register_gatorade_phase_evidence( + enhance_ctx, + original_shared_libs_by_platform.keys(), + native_library_merge_dir, + middle_gatorade_products, + unstripped_native_libraries_files, + relinked_libs_output, + defer_relink, + ) native_libs_for_primary_apk, exopackage_info = _get_exopackage_info(ctx, native_libs_always_in_primary_apk, native_libs, native_libs_metadata) return AndroidBinaryNativeLibsInfo( @@ -646,6 +1016,7 @@ def get_android_binary_native_library_info( non_root_module_native_lib_assets = [non_root_module_metadata_assets, non_root_module_lib_assets], generated_java_code = generated_java_code, unstripped_shared_libraries = unstripped_native_libraries_files, + validation_outputs = [jni_onload_check_report] if jni_onload_check_report else [], ) _NativeLibSubtargetArtifacts = record( @@ -653,6 +1024,10 @@ _NativeLibSubtargetArtifacts = record( unrelinked = Artifact | None, linker_command = Artifact | None, linker_argsfile = Artifact | None, + # Pre-inline module IR (.ll) emitted by the relink when the `preinline-ir` + # extra relinker output is enabled; a comment-only `.ll` otherwise. Surfaced + # as the `[preinline_ir]` sub-target so tooling (evt) can fetch it. + preinline_ir = Artifact | None, ) # Writes a directory of / shared libraries, and optionally a JSON @@ -662,6 +1037,23 @@ _NativeLibSubtargetArtifacts = record( # knows which libraries to replace); [unrelinked_libs] mirrors the per-library # [unrelinked] sub-target, exposing the unstripped linker output, and needs no # manifest (pass out_manifest = None). +def _write_relinker_extra_outputs_dir(ctx: AnalysisContext, libs_by_platform: dict[str, dict[str, SharedLibrary]], out_dir: Artifact): + # Only the configured types: the relinker records other things under + # `extra_outputs` as well, and they are not for building on their own. + output_types = getattr(ctx.attrs, "extra_relinker_outputs", []) + files = {} + for platform, libs in libs_by_platform.items(): + abi_directory = CPU_FILTER_TO_ABI_DIRECTORY[platform] + for soname, lib in libs.items(): + for output_type in output_types: + for info in lib.extra_outputs.get(output_type, []): + for artifact in info.default_outputs: + # Key by soname as well: two libraries in the same + # abi/type can emit outputs sharing a basename, which + # would otherwise collide and drop one silently. + files["{}/{}/{}/{}".format(output_type, abi_directory, soname, artifact.basename)] = artifact + ctx.actions.symlinked_dir(out_dir, files) + def _write_native_libs_dir_and_manifest(ctx, libs_by_platform, out_dir, out_manifest, stripped): lib_files = {} for platform in libs_by_platform: @@ -692,17 +1084,24 @@ def _post_native_lib_graph_finalization_steps( root_module_metadata_assets: Artifact, non_root_module_metadata_assets: Artifact, non_root_module_lib_assets: Artifact, + native_cmd_entries = None, + # True only on apps that neither merge nor relink: then the libs finalized here ARE the plain + # per-library links, recorded kind=link. When merging/relinking, those commands are captured at + # their own sites (kind=merge / kind=relink) and this must stay False to avoid double-recording. + record_final_link: bool = False, ) -> dict[str, dict[str, SharedLibrary]]: bolt_args = getattr(ctx.attrs, "native_library_bolt_args", None) pre_bolt_libs_by_platform = {} if bolt_args and len(bolt_args) != 0: - final_shared_libs_by_platform, pre_bolt_libs_by_platform = _bolt_libraries(ctx, final_shared_libs_by_platform, bolt_args) + final_shared_libs_by_platform, pre_bolt_libs_by_platform = _bolt_libraries( + ctx, final_shared_libs_by_platform, bolt_args, native_cmd_entries, record_final_link + ) unstripped_libs = {} linker_argsfiles_list = [] linker_commands_json = [] for platform, libs in final_shared_libs_by_platform.items() + pre_bolt_libs_by_platform.items(): - for lib in libs.values(): + for soname, lib in libs.items(): unstripped_libs[lib.lib.output] = platform if lib.lib.linker_argsfile: linker_argsfiles_list.append(lib.lib) @@ -712,6 +1111,14 @@ def _post_native_lib_graph_finalization_steps( "filename": lib.lib.output.short_path, }) + # Record the plain per-library link for [native_build_commands][link], but only on apps + # that neither merge nor relink (record_final_link). Merge and relink links are captured + # at their own sites, and on those paths the libs seen here are either already-recorded + # (merge/relink outputs) or command-less (late-gatorade dummies), so recording here would + # double-count or add null entries. + if record_final_link: + record_link_command(native_cmd_entries, "link", platform, soname, lib.lib) + ctx.actions.symlinked_dir(linker_argsfiles, {"{}".format(lib.output.basename): lib.linker_argsfile for lib in linker_argsfiles_list}) ctx.actions.write_json(linker_commands, linker_commands_json, with_inputs = False) ctx.actions.write(unstripped_native_libraries, unstripped_libs.keys()) @@ -789,6 +1196,8 @@ def _declare_library_subtargets( dynamic_outputs.append(linker_command_output) linker_argsfile_output = ctx.actions.declare_output(output_path + ".linker_argsfile", has_content_based_path = False) dynamic_outputs.append(linker_argsfile_output) + preinline_ir_output = ctx.actions.declare_output(output_path + ".preinline.ll", has_content_based_path = False) + dynamic_outputs.append(preinline_ir_output) output_path = output_path + ".unrelinked" unrelinked_lib_output = ctx.actions.declare_output(output_path, dir = True, has_content_based_path = False) @@ -798,6 +1207,7 @@ def _declare_library_subtargets( unrelinked = unrelinked_lib_output, linker_command = linker_command_output, linker_argsfile = linker_argsfile_output, + preinline_ir = preinline_ir_output, ) else: lib_outputs[soname] = _NativeLibSubtargetArtifacts( @@ -805,6 +1215,7 @@ def _declare_library_subtargets( unrelinked = None, linker_command = None, linker_argsfile = None, + preinline_ir = None, ) lib_outputs_by_platform[platform] = lib_outputs @@ -828,6 +1239,7 @@ def _link_library_subtargets( merged_lib_outputs = {} linker_commands_by_soname = {} linker_argsfiles_by_soname = {} + preinline_ir_by_soname = {} for soname, lib in final_shared_libs.items(): base_soname = soname @@ -846,6 +1258,11 @@ def _link_library_subtargets( } if lib.lib.linker_argsfile: linker_argsfiles_by_soname[soname] = lib.lib.linker_argsfile + preinline_ir = lib.extra_outputs.get("preinline-ir") if lib.extra_outputs else None + if preinline_ir: + preinline_ir_outputs = preinline_ir[0].default_outputs + if preinline_ir_outputs: + preinline_ir_by_soname[soname] = preinline_ir_outputs[0] for soname, lib_outputs in lib_outputs_by_platform[platform].items(): if soname in merged_lib_outputs: @@ -877,6 +1294,20 @@ def _link_library_subtargets( else: ctx.actions.write_json(outputs[lib_outputs.linker_argsfile], {}) + if lib_outputs.preinline_ir: + if soname in preinline_ir_by_soname: + ctx.actions.symlink_file(outputs[lib_outputs.preinline_ir], preinline_ir_by_soname[soname]) + else: + # `;` is an LLVM IR line comment, so a consumer that parses this + # unconditionally reads an empty module rather than choking on a + # placeholder that is not IR at all. + ctx.actions.write( + outputs[lib_outputs.preinline_ir], + "; no pre-inline IR for {}: the `preinline_ir` relinker output was not enabled for this library (constraint off, or soname absent from `gator.preinline_ir_sonames`)".format( + soname + ), + ) + def _create_library_subtargets( lib_outputs_by_platform: dict[str, dict[str, _NativeLibSubtargetArtifacts]], native_libs: Artifact, create_default_outputs: bool ): @@ -889,6 +1320,8 @@ def _create_library_subtargets( sub_targets["linker_command"] = [DefaultInfo(default_outputs = [output.linker_command])] if output.linker_argsfile: sub_targets["linker_argsfile"] = [DefaultInfo(default_outputs = [output.linker_argsfile])] + if output.preinline_ir: + sub_targets["preinline_ir"] = [DefaultInfo(default_outputs = [output.preinline_ir])] if output.unrelinked: sub_targets["unrelinked"] = [DefaultInfo(default_outputs = [output.unrelinked])] @@ -1502,6 +1935,69 @@ def _shared_lib_for_prebuilt_shared( label = target, ) +def _run_jni_onload_check( + ctx: AnalysisContext, + cxx_toolchain: CxxToolchainInfo, + check_inputs: list, + output: Artifact, +): + """Verify that no input to a merged library defines JNI_OnLoad. + + A constituent with allow_jni_merging = True has its JNI_OnLoad renamed by + jni_lib_merge.h, and the JNI_OnLoad symbol in the merged library is + manufactured by -Wl,--defsym. --defsym silently overrides a constituent's + own definition rather than colliding with it, so a constituent that has a + JNI_OnLoad but did not opt in has it discarded with no diagnostic and is + never registered on MergedSoMapping$Invoke_JNI_OnLoad -- an + UnsatisfiedLinkError at runtime and nothing at build time. + + This has to inspect the link *inputs*: by the time the merged library + exists, --defsym has erased the evidence. + """ + android_toolchain = ctx.attrs._android_toolchain[AndroidToolchainInfo] + + # write_json renders each artifact as its path and, with with_inputs, makes + # the objects inputs of the action so they are materialized for the scan. + manifest = ctx.actions.write_json( + "jni_onload_check_inputs.json", + [{"artifact": obj, "merged_lib": soname, "target": target} for soname, target, obj in check_inputs], + with_inputs = True, + has_content_based_path = False, + ) + ctx.actions.run( + cmd_args( + android_toolchain.jni_onload_check[RunInfo], + "--inputs", + manifest, + "--nm", + cxx_toolchain.binary_utilities_info.nm, + "--out", + output.as_output(), + ), + category = "jni_onload_check", + # Reads every object that goes into every merged library; running it + # remotely avoids shipping them back for a local scan. + prefer_remote = True, + ) + +def _link_info_object_artifacts(link_info: LinkInfo) -> list[Artifact]: + """The compiled inputs a LinkInfo contributes, for symbol inspection. + + Thin archives are represented by their member objects, which sidesteps the + fact that a thin archive's member paths are relative to the archive itself + and so cannot be read from an arbitrary working directory. + """ + artifacts = [] + for linkable in link_info.linkables: + if isinstance(linkable, ArchiveLinkable): + if linkable.archive.external_objects: + artifacts.extend(linkable.archive.external_objects) + else: + artifacts.append(linkable.archive.artifact) + elif isinstance(linkable, ObjectsLinkable): + artifacts.extend(linkable.objects or []) + return artifacts + def _get_merged_linkables_for_platform( ctx: AnalysisContext, cxx_toolchain: CxxToolchainInfo, @@ -1513,6 +2009,7 @@ def _get_merged_linkables_for_platform( merge_map: dict[str, [str, None]], merge_linker_args: dict[str, typing.Any], apk_module_graph: typing.Callable, + jni_onload_check_inputs: list | None = None, ) -> (dict[str, MergedSharedLibrary], MergedLinkablesDebugInfo): """ This takes the merge mapping and constructs the resulting merged shared libraries. @@ -1702,6 +2199,10 @@ def _get_merged_linkables_for_platform( solib_constituent_targets = [] group_deps = [] group_exported_deps = [] + # (target, object) pairs for the JNI_OnLoad check. Collected here + # because the constituents' link infos are not retained anywhere else, + # and associated with the soname below once it is known. + check_objects = [] for key in group_data.constituents: expect(target_to_link_group[key] == group) node = linkable_nodes[key] @@ -1719,6 +2220,11 @@ def _get_merged_linkables_for_platform( node = linkable_nodes[key] link_info = node.link_infos[archive_output_style].default + if jni_onload_check_inputs != None: + target_str = str(key.raw_target()) + for obj in _link_info_object_artifacts(link_info): + check_objects.append((target_str, obj)) + # the propagated link info should already be wrapped with exported flags. link_info = wrap_link_info( link_info, @@ -1740,6 +2246,12 @@ def _get_merged_linkables_for_platform( soname = linkable_nodes[group_data.constituents[0]].default_soname debug_info.with_default_soname.append((soname, group_data.constituents[0])) + # Only groups that are actually merged are checked. A group of one is a + # normal standalone library, where defining JNI_OnLoad is correct. + if jni_onload_check_inputs != None and is_actually_merged: + for target_str, obj in check_objects: + jni_onload_check_inputs.append((soname, target_str, obj)) + output_path = _platform_output_path(soname, platform) link_merge_info = LinkGroupMergeInfo( @@ -2040,7 +2552,11 @@ def _create_merged_link_args( return LinkArgs(infos = links), shlib_deps, link_traversal_cache def _bolt_libraries( - ctx: AnalysisContext, libraries_by_platform: dict[str, dict[str, SharedLibrary]], bolt_args: dict[str, ArgLike] + ctx: AnalysisContext, + libraries_by_platform: dict[str, dict[str, SharedLibrary]], + bolt_args: dict[str, ArgLike], + native_cmd_entries = None, + record_underlying_link: bool = False, ) -> ( dict[str, dict[str, SharedLibrary]], dict[str, dict[str, SharedLibrary]], @@ -2059,6 +2575,13 @@ def _bolt_libraries( bolted_libraries[soname] = original_shared_library continue + # On plain-link apps (record_underlying_link), record the bolted soname's underlying link + # BEFORE the swap: the bolted/pre-bolt objects below are command-less, so finalization + # skips them and this plain link would otherwise be lost. When merging/relinking, that + # underlying command is already captured at the merge/relink site, so skip it here. + if record_underlying_link: + record_link_command(native_cmd_entries, "link", platform, soname, original_shared_library.lib) + # Create pre-BOLT copy with .pre_bolt.so suffix pre_bolt_soname = soname.removesuffix(".so") + ".pre_bolt.so" pre_bolt_output_path = "pre-bolt-libs/{}/{}".format(platform, pre_bolt_soname) @@ -2089,6 +2612,8 @@ def _bolt_libraries( prebolt_lib = original_shared_library, output_path = output_path, bolt_args = bolt_args[soname], + native_cmd_entries = native_cmd_entries, + platform = platform, ) return bolted_libraries_by_platform, pre_bolt_libraries_by_platform @@ -2116,7 +2641,9 @@ def _bolt_libraries( # 5. extract the list of undefined symbols in the relinked libs (i.e. those symbols needed from dependencies and what had been # used in (1) above from higher nodes). -def relink_libraries(ctx: AnalysisContext, libraries_by_platform: dict[str, dict[str, SharedLibrary]]) -> dict[str, dict[str, SharedLibrary]]: +def relink_libraries( + ctx: AnalysisContext, libraries_by_platform: dict[str, dict[str, SharedLibrary]], native_cmd_entries = None +) -> dict[str, dict[str, SharedLibrary]]: relinker_extra_deps = getattr(ctx.attrs, "relinker_extra_deps", None) relinker_extra_args = getattr(ctx.attrs, "relinker_extra_args", {}) relinker_extra_args_all = getattr(ctx.attrs, "relinker_extra_args_all", []) @@ -2182,8 +2709,8 @@ def relink_libraries(ctx: AnalysisContext, libraries_by_platform: dict[str, dict ) extra_args = {} # @oss-enable - # @oss-disable[end= ]: extra_args = add_gatorade_relinker_args(ctx, cxx_toolchain, output_path) - relinker_output_args = get_extra_relinker_args(ctx, output_path) + # @oss-disable[end= ]: extra_args = add_gatorade_relinker_args(ctx, cxx_toolchain, output_path, soname, platform) + relinker_output_args = get_extra_relinker_args(ctx, output_path, soname) extra_args = merge_extra_linker_args([extra_args, relinker_output_args]) shared_lib = create_shared_lib( ctx, @@ -2202,12 +2729,13 @@ def relink_libraries(ctx: AnalysisContext, libraries_by_platform: dict[str, dict needed_symbols_files[soname] = unioned_needed_symbols_file relinked_libraries[soname] = shared_lib + record_link_command(native_cmd_entries, "relink", platform, soname, shared_lib.lib) return relinked_libraries_by_platform -def _relink_for_native_libs(ctx: AnalysisContext, libraries_by_platform: dict[str, dict[str, SharedLibrary]]) -> dict[str, dict[str, SharedLibrary]]: - return relink_libraries(ctx, libraries_by_platform) # @oss-enable - # @oss-disable[end= ]: return relink_for_native_libs(ctx, libraries_by_platform, relink_libraries, create_shared_lib) +def _relink_for_native_libs(ctx: AnalysisContext, libraries_by_platform: dict[str, dict[str, SharedLibrary]], native_cmd_entries = None) -> tuple: + return (relink_libraries(ctx, libraries_by_platform, native_cmd_entries), {}) # @oss-enable + # @oss-disable[end= ]: return relink_for_native_libs(ctx, libraries_by_platform, relink_libraries, create_shared_lib, native_cmd_entries) def extract_provided_symbols(ctx: AnalysisContext, toolchain: CxxToolchainInfo, lib: Artifact) -> Artifact: return extract_defined_syms(ctx, toolchain, lib, "relinker_extract_provided_symbols") @@ -2333,7 +2861,13 @@ def create_shared_lib( ) def _create_bolt_lib( - ctx: AnalysisContext, cxx_toolchain: CxxToolchainInfo, prebolt_lib: SharedLibrary, output_path: str, bolt_args: list[ArgLike] + ctx: AnalysisContext, + cxx_toolchain: CxxToolchainInfo, + prebolt_lib: SharedLibrary, + output_path: str, + bolt_args: list[ArgLike], + native_cmd_entries = None, + platform = None, ) -> SharedLibrary: soname = prebolt_lib.soname.ensure_str() bolt_output = ctx.actions.declare_output(output_path, has_content_based_path = False) @@ -2357,6 +2891,27 @@ def _create_bolt_lib( allow_cache_upload = True, ) + # Record the bolt command for [native_build_commands], referencing the plain output artifact + # (not `as_output()`, which cannot be re-serialized by write_json). + if native_cmd_entries != None: + native_cmd_entries.append( + native_build_command_entry( + kind = "bolt", + category = "bolt", + arch = platform, + soname = soname, + output = bolt_output.short_path, + argv = cmd_args( + cmd_args(cxx_toolchain.binary_utilities_info.bolt), + prebolt_lib.lib.output, + "-o", + bolt_output, + bolt_args, + ), + identifier = output_path, + ) + ) + linked_object = LinkedObject( output = bolt_output, unstripped_output = bolt_output, diff --git a/prelude/android/android_binary_resources_rules.bzl b/prelude/android/android_binary_resources_rules.bzl index 9715297..a5b28e7 100644 --- a/prelude/android/android_binary_resources_rules.bzl +++ b/prelude/android/android_binary_resources_rules.bzl @@ -26,12 +26,9 @@ load( "@prelude//java:java_providers.bzl", "JavaPackagingDep", # @unused Used as type ) -load("@prelude//utils:buckconfig.bzl", "read_bool") load("@prelude//utils:expect.bzl", "expect") load("@prelude//utils:utils.bzl", "flatten") -_optimized_resource_processing = read_bool("android", "optimized_resource_processing", default = False, root_cell = True) - _FilteredResourcesOutput = record( resource_infos = list[AndroidResourceInfo], voltron_res = list[Artifact], @@ -122,26 +119,21 @@ def get_android_binary_resources_info( exo_resources = ctx.actions.declare_output("exo_resources.apk", has_content_based_path = False) exo_resources_hash = ctx.actions.declare_output("exo_resources.apk.hash", has_content_based_path = False) ctx.actions.run( - cmd_args( - [ - android_toolchain.exo_resources_rewriter[RunInfo], - "--original-r-dot-txt", - aapt2_link_info.r_dot_txt, - "--new-r-dot-txt", - r_dot_txt.as_output(), - "--original-primary-apk-resources", - aapt2_link_info.primary_resources_apk, - "--new-primary-apk-resources", - primary_resources_apk.as_output(), - "--exo-resources", - exo_resources.as_output(), - "--exo-resources-hash", - exo_resources_hash.as_output(), - "--zipalign-tool", - android_toolchain.zipalign[RunInfo], - ] - + (["--optimized-processing"] if _optimized_resource_processing else []) - ), + cmd_args([ + android_toolchain.exo_resources_rewriter[RunInfo], + "--original-r-dot-txt", + aapt2_link_info.r_dot_txt, + "--new-r-dot-txt", + r_dot_txt.as_output(), + "--original-primary-apk-resources", + aapt2_link_info.primary_resources_apk, + "--new-primary-apk-resources", + primary_resources_apk.as_output(), + "--exo-resources", + exo_resources.as_output(), + "--exo-resources-hash", + exo_resources_hash.as_output(), + ]), category = "write_exo_resources", allow_cache_upload = True, ) @@ -170,7 +162,6 @@ def get_android_binary_resources_info( [r_dot_txt], override_symbols_paths, getattr(ctx.attrs, "duplicate_resource_whitelist", None), - getattr(ctx.attrs, "resource_union_package", None), referenced_resources_lists, generate_strings_and_ids_separately = generate_strings_and_ids_separately, remove_classes = ["{}.R".format(r_dot_java_package) for r_dot_java_package in r_dot_java_packages_to_exclude], @@ -359,6 +350,7 @@ def _maybe_filter_resources(ctx: AnalysisContext, resources: list[AndroidResourc r_dot_java_package = resource.r_dot_java_package, res = filtered_res, text_symbols = resource.text_symbols, + unused_resource_dep_validation_has_non_xml_resources = resource.unused_resource_dep_validation_has_non_xml_resources, ) filtered_resource_infos.append(filtered_resource) @@ -481,7 +473,7 @@ def get_manifest( manifest_entries.get("placeholders", {}), ) - if android_toolchain.set_application_id_to_specified_package and should_replace_application_id_placeholders: + if should_replace_application_id_placeholders: android_manifest_with_replaced_application_id = ctx.actions.declare_output("replaced/AndroidManifest.xml", has_content_based_path = False) replace_application_id_placeholders_cmd = cmd_args([ ctx.attrs._android_toolchain[AndroidToolchainInfo].replace_application_id_placeholders[RunInfo], @@ -489,9 +481,8 @@ def get_manifest( android_manifest, "--output", android_manifest_with_replaced_application_id.as_output(), + "--sanity-check-placeholders", ]) - if android_toolchain.should_run_sanity_check_for_placeholders: - replace_application_id_placeholders_cmd.add("--sanity-check-placeholders") ctx.actions.run(replace_application_id_placeholders_cmd, category = "replace_application_id_placeholders", allow_cache_upload = True) return android_manifest_with_replaced_application_id @@ -610,9 +601,6 @@ def _merge_assets( merge_assets_cmd.add("--binary-type", "aab" if is_bundle_build else "apk") - if _optimized_resource_processing: - merge_assets_cmd.add("--optimized-processing") - return merge_assets_cmd, merged_assets_output_hash if apk_module_graph_file: diff --git a/prelude/android/android_bundle.bzl b/prelude/android/android_bundle.bzl index 16a348d..4d89ec9 100644 --- a/prelude/android/android_bundle.bzl +++ b/prelude/android/android_bundle.bzl @@ -17,12 +17,13 @@ load( "AndroidBinaryResourcesInfo", "AndroidDerivedApkInfo", "DexFilesInfo", + "KeystoreInfo", ) load("@prelude//android:android_toolchain.bzl", "AndroidToolchainInfo") load("@prelude//android:bundletool_util.bzl", "derive_universal_apk") load("@prelude//android:util.bzl", "package_validators_decorator") -load("@prelude//java:java_providers.bzl", "KeystoreInfo") load("@prelude//java/utils:java_more_utils.bzl", "get_path_separator_for_exec_os") +load("@prelude//target_stats:target_stats.bzl", "target_stats_aggregate_providers_and_subtargets") load("@prelude//utils:argfile.bzl", "argfile") def android_bundle_impl(ctx: AnalysisContext) -> list[Provider]: @@ -48,6 +49,8 @@ def android_bundle_impl(ctx: AnalysisContext) -> list[Provider]: ) extra_providers = [] + if android_binary_info.preprocessed_java_classes_info: + extra_providers.append(android_binary_info.preprocessed_java_classes_info) sub_targets = {} sub_targets.update(android_binary_info.sub_targets) if ctx.attrs.use_derived_apk: @@ -75,27 +78,37 @@ def android_bundle_impl(ctx: AnalysisContext) -> list[Provider]: else: default_output = output_bundle + target_stats_providers, target_stats_subtargets = target_stats_aggregate_providers_and_subtargets( + ctx, + deps = android_binary_info.deps_by_platform[android_binary_info.primary_platform], + ) + sub_targets.update(target_stats_subtargets) + java_packaging_deps = android_binary_info.java_packaging_deps - return [ - DefaultInfo(default_output = default_output, other_outputs = android_binary_info.materialized_artifacts, sub_targets = sub_targets), - AndroidAabInfo( - aab = output_bundle, - manifest = android_binary_info.resources_info.manifest, - materialized_artifacts = android_binary_info.materialized_artifacts, - unstripped_shared_libraries = native_library_info.unstripped_shared_libraries, - ), - AndroidBinaryPrimaryPlatformInfo( - primary_platform = android_binary_info.primary_platform, - ), - TemplatePlaceholderInfo( - keyed_variables = { - "classpath": cmd_args([dep.jar for dep in java_packaging_deps if dep.jar], delimiter = get_path_separator_for_exec_os(ctx)), - "classpath_including_targets_with_no_output": cmd_args( - [dep.output_for_classpath_macro for dep in java_packaging_deps], delimiter = get_path_separator_for_exec_os(ctx) - ), - }, - ), - ] + extra_providers + return ( + [ + DefaultInfo(default_output = default_output, other_outputs = android_binary_info.materialized_artifacts, sub_targets = sub_targets), + AndroidAabInfo( + aab = output_bundle, + manifest = android_binary_info.resources_info.manifest, + materialized_artifacts = android_binary_info.materialized_artifacts, + unstripped_shared_libraries = native_library_info.unstripped_shared_libraries, + ), + AndroidBinaryPrimaryPlatformInfo( + primary_platform = android_binary_info.primary_platform, + ), + TemplatePlaceholderInfo( + keyed_variables = { + "classpath": cmd_args([dep.jar for dep in java_packaging_deps if dep.jar], delimiter = get_path_separator_for_exec_os(ctx)), + "classpath_including_targets_with_no_output": cmd_args( + [dep.output_for_classpath_macro for dep in java_packaging_deps], delimiter = get_path_separator_for_exec_os(ctx) + ), + }, + ), + ] + + extra_providers + + target_stats_providers + ) def build_bundle( output_filename: str, diff --git a/prelude/android/android_instrumentation_apk.bzl b/prelude/android/android_instrumentation_apk.bzl index eb0fc97..3e94879 100644 --- a/prelude/android/android_instrumentation_apk.bzl +++ b/prelude/android/android_instrumentation_apk.bzl @@ -19,6 +19,7 @@ load("@prelude//java:class_to_srcs.bzl", "merge_class_to_source_map_from_jar") load("@prelude//java:java_providers.bzl", "create_java_packaging_dep", "get_all_java_packaging_deps") load("@prelude//java:java_toolchain.bzl", "JavaToolchainInfo") load("@prelude//java/utils:java_utils.bzl", "get_class_to_source_map_info") +load("@prelude//target_stats:target_stats.bzl", "target_stats_aggregate_providers_and_subtargets") load("@prelude//utils:expect.bzl", "expect") load("@prelude//utils:utils.bzl", "flatten") @@ -62,10 +63,16 @@ This will lead to overbuilding and is not supported. Configuration {} not found is_self_instrumenting = ctx.attrs.is_self_instrumenting + all_java_packaging_deps = [packaging_dep for packaging_dep in get_all_java_packaging_deps(ctx, deps) if packaging_dep.dex] java_packaging_deps = [ packaging_dep - for packaging_dep in get_all_java_packaging_deps(ctx, deps) - if packaging_dep.dex and (is_self_instrumenting or packaging_dep.label.raw_target() not in apk_under_test_info.java_packaging_deps) + for packaging_dep in all_java_packaging_deps + if is_self_instrumenting or packaging_dep.label.raw_target() not in apk_under_test_info.java_packaging_deps + ] + desugar_classpath_jars = [ + packaging_dep.jar + for packaging_dep in all_java_packaging_deps + if not is_self_instrumenting and packaging_dep.label.raw_target() in apk_under_test_info.java_packaging_deps ] android_packageable_info = merge_android_packageable_info(ctx.label, ctx.actions, deps) @@ -94,6 +101,7 @@ This will lead to overbuilding and is not supported. Configuration {} not found enhance_ctx = create_enhancement_context(ctx) sub_targets = enhance_ctx.get_sub_targets() materialized_artifacts = [] + preprocessed_java_classes_info = None if not disable_pre_dex: pre_dexed_libs = [java_packaging_dep.dex for java_packaging_dep in java_packaging_deps] if ctx.attrs.use_split_dex: @@ -110,9 +118,9 @@ This will lead to overbuilding and is not supported. Configuration {} not found else: jars_to_owners = {packaging_dep.jar: packaging_dep.jar.owner.raw_target() for packaging_dep in java_packaging_deps} if ctx.attrs.preprocess_java_classes_bash: - jars_to_owners, materialized_artifacts_dir = get_preprocessed_java_classes(enhance_ctx, jars_to_owners) - if materialized_artifacts_dir: - materialized_artifacts.append(materialized_artifacts_dir) + jars_to_owners, preprocessed_java_classes_info = get_preprocessed_java_classes(enhance_ctx, jars_to_owners) + if preprocessed_java_classes_info: + materialized_artifacts.append(preprocessed_java_classes_info.materialized_artifacts_dir) if ctx.attrs.use_split_dex: dex_files_info = get_multi_dex( ctx, @@ -121,12 +129,14 @@ This will lead to overbuilding and is not supported. Configuration {} not found ctx.attrs.primary_dex_patterns, enable_bootstrap_dexes = ctx.attrs.enable_bootstrap_dexes, multidex_min_api = ctx.attrs.multidex_min_api, + classpath_jars = desugar_classpath_jars, ) else: dex_files_info = get_single_primary_dex( ctx, ctx.attrs._android_toolchain[AndroidToolchainInfo], jars_to_owners.keys(), + classpath_jars = desugar_classpath_jars, ) native_library_info = get_android_binary_native_library_info( enhance_ctx, @@ -168,9 +178,16 @@ This will lead to overbuilding and is not supported. Configuration {} not found ) sub_targets["transitive_class_to_src_map"] = [DefaultInfo(default_output = transitive_class_to_src_map)] - return [ - AndroidApkInfo(apk = output_apk, materialized_artifacts = materialized_artifacts, manifest = resources_info.manifest), - AndroidInstrumentationApkInfo(apk_under_test = ctx.attrs.apk[AndroidApkInfo].apk, is_self_instrumenting = is_self_instrumenting), - DefaultInfo(default_output = output_apk, other_outputs = materialized_artifacts, sub_targets = sub_targets | class_to_srcs_subtargets), - class_to_srcs, - ] + target_stats_providers, target_stats_subtargets = target_stats_aggregate_providers_and_subtargets(ctx, deps = deps) + sub_targets.update(target_stats_subtargets) + + return ( + [ + AndroidApkInfo(apk = output_apk, materialized_artifacts = materialized_artifacts, manifest = resources_info.manifest), + AndroidInstrumentationApkInfo(apk_under_test = ctx.attrs.apk[AndroidApkInfo].apk, is_self_instrumenting = is_self_instrumenting), + DefaultInfo(default_output = output_apk, other_outputs = materialized_artifacts, sub_targets = sub_targets | class_to_srcs_subtargets), + class_to_srcs, + ] + + ([preprocessed_java_classes_info] if preprocessed_java_classes_info else []) + + target_stats_providers + ) diff --git a/prelude/android/android_instrumentation_test.bzl b/prelude/android/android_instrumentation_test.bzl index 2450c7c..eae1e81 100644 --- a/prelude/android/android_instrumentation_test.bzl +++ b/prelude/android/android_instrumentation_test.bzl @@ -19,6 +19,8 @@ load( "merge_shared_libraries", "traverse_shared_library_info", ) +load("@prelude//target_stats:target_stats.bzl", "CycleMode", "target_stats_providers_and_subtargets") +load("@prelude//target_stats:target_stats_config.bzl", "TARGET_STATS_ENABLED") load("@prelude//test:inject_test_run_info.bzl", "inject_test_run_info") load("@prelude//tests:test_listing.bzl", "TestListingInfo") load("@prelude//utils:argfile.bzl", "at_argfile") @@ -204,13 +206,34 @@ def android_instrumentation_test_impl(ctx: AnalysisContext): test_info, run_info = inject_test_run_info(ctx, test_info) + target_stats_providers = [] + target_stats_subtargets = {} + if TARGET_STATS_ENABLED: + target_stats_tools = android_toolchain.target_stats_tools + if target_stats_tools != None: + target_stats_deps = [ctx.attrs.apk] + if ctx.attrs.instrumentation_test_listener != None: + target_stats_deps.append(ctx.attrs.instrumentation_test_listener) + target_stats_providers, target_stats_subtargets = target_stats_providers_and_subtargets( + ctx, + tools = target_stats_tools, + srcs = {src.short_path: src for src in ctx.attrs._test_srcs}, + deps = target_stats_deps, + cycle_mode = CycleMode("package"), + module_name = ctx.label.name, + ) + # We append additional args so that "buck2 run" will work with sane defaults run_info.args.add(cmd_args(["--auto-run-on-connected-device", "--output", ".", "--adb-executable-path", "adb"])) - return [ - test_info, - run_info, - DefaultInfo(), - ] + classmap_source_info + return ( + [ + test_info, + run_info, + DefaultInfo(sub_targets = target_stats_subtargets), + ] + + classmap_source_info + + target_stats_providers + ) def _compute_executor_overrides(ctx: AnalysisContext, instrumentation_test_can_run_locally: bool) -> dict[str, CommandExecutorConfig]: remote_execution_properties = { diff --git a/prelude/android/android_library.bzl b/prelude/android/android_library.bzl index e8f8b25..b1a8826 100644 --- a/prelude/android/android_library.bzl +++ b/prelude/android/android_library.bzl @@ -21,12 +21,9 @@ load("@prelude//graphql:graphql.bzl", "graphql_providers") load("@prelude//java:java_library.bzl", "build_java_library") load( "@prelude//java:java_providers.bzl", - "JavaClasspathEntry", # @unused Used as type - "JavaCompilingDepsTSet", "JavaLibraryInfo", "JavaProviders", # @unused Used as type "create_native_providers", - "single_library_compiling_deps", "to_list", ) load( @@ -36,6 +33,8 @@ load( "get_string_concat_inline_javac_args", ) load("@prelude//kotlin:kotlin_library.bzl", "build_kotlin_library") +load("@prelude//target_stats:target_stats.bzl", "CycleMode", "target_stats_providers_and_subtargets") +load("@prelude//target_stats:target_stats_config.bzl", "TARGET_STATS_ENABLED") load("@prelude//utils:expect.bzl", "expect") load("@prelude//utils:label_provider.bzl", "LabelInfo") @@ -49,6 +48,22 @@ def get_custom_jdk_info(ctx: AnalysisContext) -> CustomJdkInfo: system_image = ctx.attrs._android_toolchain[AndroidToolchainInfo].jdk_system_image, ) +def android_target_stats(ctx: AnalysisContext) -> (list[Provider], dict[str, list[Provider]]): + if not TARGET_STATS_ENABLED: + return [], {} + tools = ctx.attrs._android_toolchain[AndroidToolchainInfo].target_stats_tools + if tools == None: + return [], {} + return target_stats_providers_and_subtargets( + ctx, + tools = tools, + srcs = {src.short_path: src for src in ctx.attrs.srcs}, + # Target stats cover the compile graph, including non-packaged provided deps. + deps = (ctx.attrs.deps + ctx.attrs.exported_deps + ctx.attrs.runtime_deps + ctx.attrs.provided_deps + ctx.attrs.exported_provided_deps), + cycle_mode = CycleMode("package"), + module_name = ctx.label.name, + ) + def android_library_impl(ctx: AnalysisContext) -> list[Provider]: packaging_deps = ctx.attrs.deps + ctx.attrs.exported_deps + ctx.attrs.runtime_deps if ctx.attrs._build_only_native_code: @@ -74,9 +89,12 @@ def android_library_impl(ctx: AnalysisContext) -> list[Provider]: ), ] + target_stats_providers, target_stats_subtargets = android_target_stats(ctx) + java_providers, android_library_intellij_info = build_android_library( ctx = ctx, validation_deps_outputs = get_validation_deps_outputs(ctx), + extra_sub_targets = target_stats_subtargets, ) android_providers = [android_library_intellij_info] if android_library_intellij_info else [] @@ -95,6 +113,7 @@ def android_library_impl(ctx: AnalysisContext) -> list[Provider]: + [LabelInfo(labels = ctx.attrs.labels)] + graphql_providers(ctx) + capabilities_registration_providers(ctx) + + target_stats_providers ) def optional_jars(ctx: AnalysisContext) -> list[Artifact]: @@ -131,31 +150,21 @@ def optional_abi_jar_snapshots(ctx: AnalysisContext) -> list[Artifact]: def build_android_library( ctx: AnalysisContext, - r_dot_java: JavaClasspathEntry | None = None, + r_dot_java: JavaLibraryInfo | None = None, extra_sub_targets = {}, validation_deps_outputs: [list[Artifact], None] = None, - classpath_entries: JavaCompilingDepsTSet | None = None, ) -> (JavaProviders, [AndroidLibraryIntellijInfo, None]): custom_jdk_info = get_custom_jdk_info(ctx) - additional_classpath_entries_children = [classpath_entries] if classpath_entries else [] - - dummy_r_dot_java, android_library_intellij_info = _get_dummy_r_dot_java(ctx) + dummy_r_dot_java_info, android_library_intellij_info = _get_dummy_r_dot_java(ctx) extra_sub_targets = dict(extra_sub_targets) if r_dot_java: - additional_classpath_entries_children.append(single_library_compiling_deps(ctx.actions, r_dot_java)) - elif dummy_r_dot_java: - additional_classpath_entries_children.append(single_library_compiling_deps(ctx.actions, dummy_r_dot_java)) - extra_sub_targets["dummy_r_dot_java"] = [DefaultInfo(default_output = dummy_r_dot_java.full_library)] - - additional_classpath_entries = ( - ctx.actions.tset( - JavaCompilingDepsTSet, - children = additional_classpath_entries_children, - ) - if additional_classpath_entries_children - else None - ) + additional_classpath_entries = r_dot_java.compiling_deps + elif dummy_r_dot_java_info: + additional_classpath_entries = dummy_r_dot_java_info.compiling_deps + extra_sub_targets["dummy_r_dot_java"] = [DefaultInfo(default_output = dummy_r_dot_java_info.library_output.full_library)] + else: + additional_classpath_entries = None source_level, _ = get_java_version_attributes(ctx) extra_arguments = get_string_concat_inline_javac_args(source_level) @@ -186,7 +195,7 @@ def build_android_library( extra_arguments = extra_arguments, ), android_library_intellij_info -def _get_dummy_r_dot_java(ctx: AnalysisContext) -> (JavaClasspathEntry | None, [AndroidLibraryIntellijInfo, None]): +def _get_dummy_r_dot_java(ctx: AnalysisContext) -> (JavaLibraryInfo | None, [AndroidLibraryIntellijInfo, None]): android_resources = dedupe([ resource for resource in filter( @@ -202,12 +211,11 @@ def _get_dummy_r_dot_java(ctx: AnalysisContext) -> (JavaClasspathEntry | None, [ ctx, ctx.attrs._android_toolchain[AndroidToolchainInfo].merge_android_resources[RunInfo], android_resources, - ctx.attrs.resource_union_package, ) dummy_r_dot_java = dummy_r_dot_java_info.library_output return ( - dummy_r_dot_java, + dummy_r_dot_java_info, AndroidLibraryIntellijInfo( dummy_r_dot_java = dummy_r_dot_java.abi, android_resource_deps = android_resources, diff --git a/prelude/android/android_manifest.bzl b/prelude/android/android_manifest.bzl index 871a122..52423fd 100644 --- a/prelude/android/android_manifest.bzl +++ b/prelude/android/android_manifest.bzl @@ -50,7 +50,9 @@ def generate_android_manifest( if not manifests: manifests = [] elif isinstance(manifests, TransitiveSet): - manifests = manifests.project_as_args("artifacts", ordering = "topological") + # Manifests are usually resolved only at the binary level, so resolving + # projection nodes throughout the dependency graph is wasteful. + manifests = [entry.manifest for entry in manifests.traverse(ordering = "topological")] library_manifest_paths_file = argfile(actions = ctx.actions, name = "{}/library_manifest_paths_file".format(module_name), args = manifests) diff --git a/prelude/android/android_prebuilt_aar.bzl b/prelude/android/android_prebuilt_aar.bzl index 21b20a4..b06d0c6 100644 --- a/prelude/android/android_prebuilt_aar.bzl +++ b/prelude/android/android_prebuilt_aar.bzl @@ -81,13 +81,13 @@ def android_prebuilt_aar_impl(ctx: AnalysisContext) -> list[Provider]: res = res, res_priority = RESOURCE_PRIORITY_LOW, text_symbols = r_dot_txt, + unused_resource_dep_validation_has_non_xml_resources = True, ) dummy_r_dot_java_info = get_dummy_r_dot_java( ctx, android_toolchain.merge_android_resources[RunInfo], [resource_info], - None, ) android_resource_r_dot_info = AndroidResourceRDotInfo( diff --git a/prelude/android/android_providers.bzl b/prelude/android/android_providers.bzl index 4bdda3a..96be15c 100644 --- a/prelude/android/android_providers.bzl +++ b/prelude/android/android_providers.bzl @@ -58,6 +58,9 @@ AndroidBinaryNativeLibsInfo = record( native_libs_for_primary_apk = list[Artifact], generated_java_code = list[Artifact], unstripped_shared_libraries = [Artifact, None], + # Artifacts that must be produced for the binary to be valid. Threaded into + # the apk's validation_deps_outputs, so a failing check fails the build. + validation_outputs = list[Artifact], ) AndroidBinaryResourcesInfo = record( @@ -120,6 +123,13 @@ AndroidApkInfo = provider( }, ) +AndroidPreprocessedJavaClassesInfo = provider( + fields = { + "input_dir": provider_field(Artifact), + "materialized_artifacts_dir": provider_field(Artifact), + }, +) + # Exopackage secondary-dex dir (metadata.txt + secondary-N.dex.jar) so android_instrumentation_test # can push it to the device. Only set when the apk is built with exopackage secondary_dex mode. AndroidApkExopackageInfo = provider( @@ -182,6 +192,14 @@ AndroidInstrumentationApkInfo = provider( }, ) +KeystoreInfo = provider( + # @unsorted-dict-items + fields = { + "store": provider_field(Artifact), + "properties": provider_field(Artifact), + }, +) + ManifestInfo = record( target_label = TargetLabel, manifest = Artifact, @@ -239,6 +257,8 @@ AndroidResourceInfo = provider( "r_dot_java_package": provider_field(Artifact | None, default = None), # resources defined by this rule. May be empty "res": provider_field(Artifact | None, default = None), + # whether this resource contains non-XML resources + "unused_resource_dep_validation_has_non_xml_resources": provider_field(bool, default = False), # priority of the resources, may be 'low' or 'normal' "res_priority": provider_field(typing.Any, default = None), # str # symbols defined by the resources, if resources are present diff --git a/prelude/android/android_resource.bzl b/prelude/android/android_resource.bzl index a6ffa21..c2bbb11 100644 --- a/prelude/android/android_resource.bzl +++ b/prelude/android/android_resource.bzl @@ -43,6 +43,10 @@ def android_resource_impl(ctx: AnalysisContext) -> list[Provider]: res = _convert_to_artifact_dir(ctx, ctx.attrs.res, "res") asset = _convert_to_artifact_dir(ctx, ctx.attrs.assets, "assets") assets = [asset] if asset else [] + unused_resource_dep_validation_has_non_xml_resources = ctx.attrs.unused_resource_dep_validation_has_non_xml_resources + if unused_resource_dep_validation_has_non_xml_resources == None: + unused_resource_dep_validation_has_non_xml_resources = ctx.attrs.res != None + unused_resource_dep_validation_has_non_xml_resources = unused_resource_dep_validation_has_non_xml_resources or bool(assets) or ctx.attrs.manifest != None if res: aapt2_compile_output = aapt2_compile(ctx, res, ctx.attrs._android_toolchain[AndroidToolchainInfo]) @@ -65,6 +69,7 @@ def android_resource_impl(ctx: AnalysisContext) -> list[Provider]: res = res, res_priority = RESOURCE_PRIORITY_NORMAL, text_symbols = r_dot_txt_output, + unused_resource_dep_validation_has_non_xml_resources = unused_resource_dep_validation_has_non_xml_resources, ) else: resource_info = AndroidResourceInfo( @@ -79,6 +84,7 @@ def android_resource_impl(ctx: AnalysisContext) -> list[Provider]: res = None, res_priority = RESOURCE_PRIORITY_NORMAL, text_symbols = None, + unused_resource_dep_validation_has_non_xml_resources = unused_resource_dep_validation_has_non_xml_resources, ) providers.append(resource_info) providers.append(merge_android_packageable_info(ctx.label, ctx.actions, ctx.attrs.deps, manifest = ctx.attrs.manifest, resource_info = resource_info)) @@ -91,7 +97,6 @@ def android_resource_impl(ctx: AnalysisContext) -> list[Provider]: ctx, android_toolchain.merge_android_resources[RunInfo], [resource_info], - None, ) android_resource_r_dot_info = AndroidResourceRDotInfo( dummy_r_dot_java = dummy_r_dot_java_info.library_output.abi, diff --git a/prelude/android/android_toolchain.bzl b/prelude/android/android_toolchain.bzl index f26097e..afbb94b 100644 --- a/prelude/android/android_toolchain.bzl +++ b/prelude/android/android_toolchain.bzl @@ -12,13 +12,19 @@ AndroidPlatformInfo = provider( } ) +AidlToolchainInfo = provider( + fields = { + "aidl": provider_field(typing.Any, default = None), + "framework_aidl_file": provider_field(typing.Any, default = None), + } +) + AndroidToolchainInfo = provider( fields = { "aapt2": provider_field(typing.Any, default = None), "aapt2_filter_resources": provider_field(typing.Any, default = None), "aar_builder": provider_field(typing.Any, default = None), "adb": provider_field(typing.Any, default = None), - "aidl": provider_field(typing.Any, default = None), "android_bootclasspath": provider_field(typing.Any, default = None), "android_bootclasspath_snapshots": provider_field(typing.Any, default = []), "android_error_handler": provider_field(typing.Any, default = None), @@ -41,7 +47,6 @@ AndroidToolchainInfo = provider( "filter_dex_class_names": provider_field(typing.Any, default = None), "filter_prebuilt_native_library_dir": provider_field(typing.Any, default = None), "filter_resources": provider_field(typing.Any, default = None), - "framework_aidl_file": provider_field(typing.Any, default = None), # @oss-disable[end= ]: "gatorade_mergemap_tool": provider_field(typing.Any, default = None), "generate_build_config": provider_field(typing.Any, default = None), "generate_manifest": provider_field(typing.Any, default = None), @@ -51,6 +56,7 @@ AndroidToolchainInfo = provider( "instrumentation_test_runner_main_class": provider_field(typing.Any, default = None), "jar_splitter_command": provider_field(typing.Any, default = None), "jdk_system_image": provider_field(typing.Any, default = None), + "jni_onload_check": provider_field(typing.Any, default = None), "manifest_utils": provider_field(typing.Any, default = None), "merge_android_resource_sources": provider_field(typing.Any, default = None), "merge_android_resources": provider_field(typing.Any, default = None), @@ -69,9 +75,7 @@ AndroidToolchainInfo = provider( "replace_application_id_placeholders": provider_field(typing.Any, default = None), "secondary_dex_compression_command": provider_field(typing.Any, default = None), "secondary_dex_weight_limit": provider_field(typing.Any, default = None), - "set_application_id_to_specified_package": provider_field(typing.Any, default = None), - "should_run_sanity_check_for_placeholders": provider_field(typing.Any, default = None), - "sort_pre_dexed_files": provider_field(typing.Any, default = None), + "sort_pre_dexed_files": provider_field(typing.Any), "target_stats_tools": provider_field(typing.Any, default = None), "unpack_aar": provider_field(typing.Any, default = None), "zipalign": provider_field(typing.Any, default = None), diff --git a/prelude/android/apk_genrule.bzl b/prelude/android/apk_genrule.bzl index 1ae3108..dcd60da 100644 --- a/prelude/android/apk_genrule.bzl +++ b/prelude/android/apk_genrule.bzl @@ -8,27 +8,42 @@ load("@prelude//:genrule.bzl", "process_genrule") load("@prelude//android:android_apk.bzl", "get_install_info") -load("@prelude//android:android_providers.bzl", "AndroidAabInfo", "AndroidApkInfo", "AndroidApkUnderTestInfo", "AndroidDerivedApkInfo") +load( + "@prelude//android:android_providers.bzl", + "AndroidAabInfo", + "AndroidApkInfo", + "AndroidApkUnderTestInfo", + "AndroidDerivedApkInfo", + "AndroidPreprocessedJavaClassesInfo", + "KeystoreInfo", +) load("@prelude//android:android_toolchain.bzl", "AndroidToolchainInfo") load("@prelude//android:bundletool_util.bzl", "derive_universal_apk") +load("@prelude//android:native_build_commands.bzl", "GATORADE_PHASE_SUBTARGETS") load("@prelude//java:class_to_srcs.bzl", "JavaClassToSourceMapInfo") -load("@prelude//java:java_providers.bzl", "KeystoreInfo") load("@prelude//utils:expect.bzl", "expect") -# Native-library debug sub-targets that the wrapped android_apk/android_aab only -# exposes in some configurations (e.g. relinker or native merging enabled). -# Forward whichever happen to be present so they stay reachable through the -# apk_genrule wrapper. +# Native-library debug sub-targets forwarded through the apk_genrule wrapper only if present on the +# wrapped android_apk/android_aab. These exist only in certain configurations (e.g. relinked_libs +# and native_merge_debug require the relinker or native merging). Always-present sub-targets like +# native_libs, linker_commands and native_build_commands are forwarded in the required set below. _OPTIONAL_NATIVE_LIB_SUBTARGETS = [ + "gatorade_phase_evidence", "native_merge_debug", "relinked_libs", "relinked_libs_manifest", + "relinker_extra_outputs", "unrelinked_libs", ] def _forward_optional_native_lib_subtargets(input_subtargets: dict) -> dict: return {name: [input_subtargets[name][DefaultInfo]] for name in _OPTIONAL_NATIVE_LIB_SUBTARGETS if name in input_subtargets} +# The top-level Gatorade-phase sub-targets (`TARGET[early_gatorade]` etc.) are always present on the +# wrapped android_apk/android_aab, so forward them in the required set at every wrapper site. +def _forward_gatorade_phase_subtargets(input_subtargets: dict) -> dict: + return {name: [input_subtargets[name][DefaultInfo]] for name in GATORADE_PHASE_SUBTARGETS} + def apk_genrule_impl(ctx: AnalysisContext) -> list[Provider]: expect((ctx.attrs.apk == None) != (ctx.attrs.aab == None), "Exactly one of 'apk' and 'aab' must be specified") @@ -36,6 +51,7 @@ def apk_genrule_impl(ctx: AnalysisContext) -> list[Provider]: input_android_apk_subtargets = None input_android_apk_template_placeholder_info = None input_android_aab_subtargets = None + input_preprocessed_java_classes_info = None if ctx.attrs.apk != None: # TODO(T104150125) The underlying APK should not have exopackage enabled input_android_apk_info = ctx.attrs.apk[AndroidApkInfo] @@ -47,6 +63,8 @@ def apk_genrule_impl(ctx: AnalysisContext) -> list[Provider]: input_android_apk_under_test_info = ctx.attrs.apk[AndroidApkUnderTestInfo] input_android_apk_subtargets = ctx.attrs.apk[DefaultInfo].sub_targets input_android_apk_template_placeholder_info = ctx.attrs.apk[TemplatePlaceholderInfo].keyed_variables + if AndroidPreprocessedJavaClassesInfo in ctx.attrs.apk: + input_preprocessed_java_classes_info = ctx.attrs.apk[AndroidPreprocessedJavaClassesInfo] env_vars = { "APK": cmd_args(input_apk), @@ -61,6 +79,8 @@ def apk_genrule_impl(ctx: AnalysisContext) -> list[Provider]: input_materialized_artifacts = input_android_aab_info.materialized_artifacts input_android_aab_subtargets = ctx.attrs.aab[DefaultInfo].sub_targets input_unstripped_shared_libraries = input_android_aab_info.unstripped_shared_libraries + if AndroidPreprocessedJavaClassesInfo in ctx.attrs.aab: + input_preprocessed_java_classes_info = ctx.attrs.aab[AndroidPreprocessedJavaClassesInfo] env_vars = { "AAB": cmd_args(input_apk), @@ -127,11 +147,13 @@ def apk_genrule_impl(ctx: AnalysisContext) -> list[Provider]: ], "linker_argsfiles": [input_android_aab_subtargets["linker_argsfiles"][DefaultInfo]], "linker_commands": [input_android_aab_subtargets["linker_commands"][DefaultInfo]], + "native_build_commands": [input_android_aab_subtargets["native_build_commands"][DefaultInfo]], "native_libs": [input_android_aab_subtargets["native_libs"][DefaultInfo]], "unstripped_native_libraries": [input_android_aab_subtargets["unstripped_native_libraries"][DefaultInfo]], "unstripped_native_libraries_files": [input_android_aab_subtargets["unstripped_native_libraries_files"][DefaultInfo]], "unstripped_native_libraries_json": [input_android_aab_subtargets["unstripped_native_libraries_json"][DefaultInfo]], } + | _forward_gatorade_phase_subtargets(input_android_aab_subtargets) | _forward_optional_native_lib_subtargets(input_android_aab_subtargets), ), AndroidDerivedApkInfo( @@ -144,11 +166,13 @@ def apk_genrule_impl(ctx: AnalysisContext) -> list[Provider]: { "linker_argsfiles": [input_android_aab_subtargets["linker_argsfiles"][DefaultInfo]], "linker_commands": [input_android_aab_subtargets["linker_commands"][DefaultInfo]], + "native_build_commands": [input_android_aab_subtargets["native_build_commands"][DefaultInfo]], "native_libs": [input_android_aab_subtargets["native_libs"][DefaultInfo]], "unstripped_native_libraries": [input_android_aab_subtargets["unstripped_native_libraries"][DefaultInfo]], "unstripped_native_libraries_files": [input_android_aab_subtargets["unstripped_native_libraries_files"][DefaultInfo]], "unstripped_native_libraries_json": [input_android_aab_subtargets["unstripped_native_libraries_json"][DefaultInfo]], } + | _forward_gatorade_phase_subtargets(input_android_aab_subtargets) | _forward_optional_native_lib_subtargets(input_android_aab_subtargets) ) default_providers = [ @@ -168,11 +192,13 @@ def apk_genrule_impl(ctx: AnalysisContext) -> list[Provider]: "linker_argsfiles": [input_android_apk_subtargets["linker_argsfiles"][DefaultInfo]], "linker_commands": [input_android_apk_subtargets["linker_commands"][DefaultInfo]], "manifest": [input_android_apk_subtargets["manifest"][DefaultInfo]], + "native_build_commands": [input_android_apk_subtargets["native_build_commands"][DefaultInfo]], "native_libs": [input_android_apk_subtargets["native_libs"][DefaultInfo]], "unstripped_native_libraries": [input_android_apk_subtargets["unstripped_native_libraries"][DefaultInfo]], "unstripped_native_libraries_files": [input_android_apk_subtargets["unstripped_native_libraries_files"][DefaultInfo]], "unstripped_native_libraries_json": [input_android_apk_subtargets["unstripped_native_libraries_json"][DefaultInfo]], } + | _forward_gatorade_phase_subtargets(input_android_apk_subtargets) | _forward_optional_native_lib_subtargets(input_android_apk_subtargets) ) expect( @@ -220,4 +246,6 @@ def apk_genrule_impl(ctx: AnalysisContext) -> list[Provider]: aab_providers = filter(None, [output_aab_info]) apk_under_test_providers = filter(None, [input_android_apk_under_test_info]) - return default_providers + apk_providers + aab_providers + apk_under_test_providers + class_to_src_map + preprocessed_java_classes_providers = filter(None, [input_preprocessed_java_classes_info]) + + return default_providers + apk_providers + aab_providers + apk_under_test_providers + preprocessed_java_classes_providers + class_to_src_map diff --git a/prelude/android/bundletool_util.bzl b/prelude/android/bundletool_util.bzl index 2a21dde..78dbec5 100644 --- a/prelude/android/bundletool_util.bzl +++ b/prelude/android/bundletool_util.bzl @@ -6,8 +6,8 @@ # of this source tree. You may select, at your option, one of the # above-listed licenses. +load("@prelude//android:android_providers.bzl", "KeystoreInfo") # @unused used as type load("@prelude//android:android_toolchain.bzl", "AndroidToolchainInfo") -load("@prelude//java:java_providers.bzl", "KeystoreInfo") # @unused used as type def derive_universal_apk(ctx: AnalysisContext, android_toolchain: AndroidToolchainInfo, app_bundle: Artifact, keystore: [KeystoreInfo, None]) -> Artifact: output_apk = ctx.actions.declare_output("universal.apk", has_content_based_path = False) diff --git a/prelude/android/dex_rules.bzl b/prelude/android/dex_rules.bzl index d4a1c41..a84e405 100644 --- a/prelude/android/dex_rules.bzl +++ b/prelude/android/dex_rules.bzl @@ -34,7 +34,7 @@ load("@prelude//utils:utils.bzl", "flatten") # # The primary DEX is always stored in the root directory of the APK as `classes.dex`. # -# We have 5 different ways of storing our secondary DEX files, which are specified via the +# We have 4 different ways of storing our secondary DEX files, which are specified via the # `dex_compression` attribute: # 1) `raw` compression. This means that we create `classes2.dex`, `classes3.dex`, ..., # `classesN.dex` and store each of them in the root directory of the APK. @@ -45,9 +45,7 @@ load("@prelude//utils:utils.bzl", "flatten") # secondary dex subdirectory instead of to the root directory of the APK. # 3) `jar` compression. For each secondary DEX file, we put a `classes.dex` entry into a # JAR file, and store it as an asset at `assets/secondary-program-dex-jars/secondary-I.dex.jar` -# 4) `xz` compression. This is the same as `jar` compression, except that we run `xz` on the -# JAR file to produce `assets/secondary-program-dex-jars/secondary-I.dex.jar.xz`. -# 5) `xzs` compression. We do the same as `jar` compression, then concatenate all the jars +# 4) `xzs` compression. We do the same as `jar` compression, then concatenate all the jars # together and do `xz` compression on the result to produce a single # `assets/secondary-program-dex-jars/secondary.dex.jar.xzs`. # @@ -79,8 +77,8 @@ def _get_dex_compression(ctx: AnalysisContext) -> str: default_dex_compression = "jar" if is_exopackage_enabled_for_secondary_dexes else "raw" dex_compression = getattr(ctx.attrs, "dex_compression", None) or default_dex_compression expect( - dex_compression in ["raw", "raw_subdir", "jar", "xz", "xzs"], - "Only 'raw', 'raw_subdir', 'jar', 'xz' and 'xzs' dex compression are supported at this time!", + dex_compression in ["raw", "raw_subdir", "jar", "xzs"], + "Only 'raw', 'raw_subdir', 'jar' and 'xzs' dex compression are supported at this time!", ) return dex_compression @@ -94,7 +92,11 @@ def get_split_dex_merge_config(ctx: AnalysisContext, android_toolchain: AndroidT ) def get_single_primary_dex( - ctx: AnalysisContext, android_toolchain: AndroidToolchainInfo, java_library_jars: list[Artifact], is_optimized: bool = False + ctx: AnalysisContext, + android_toolchain: AndroidToolchainInfo, + java_library_jars: list[Artifact], + is_optimized: bool = False, + classpath_jars: list[Artifact] = [], ) -> DexFilesInfo: expect( not _is_exopackage_enabled_for_secondary_dex(ctx), @@ -107,6 +109,9 @@ def get_single_primary_dex( jar_to_dex_file = argfile(actions = ctx.actions, name = "jar_to_dex_file.txt", args = java_library_jars) d8_cmd.add(["--files-to-dex-list", jar_to_dex_file]) + if classpath_jars: + classpath_file = argfile(actions = ctx.actions, name = "classpath_jars.txt", args = classpath_jars) + d8_cmd.add(["--classpath-files", classpath_file]) d8_cmd.add(["--android-jar", android_toolchain.android_jar]) if not is_optimized: @@ -142,6 +147,7 @@ def get_multi_dex( enable_bootstrap_dexes = False, multidex_min_api: str | None = None, pre_dexed_inputs: bool = False, + classpath_jars: list[Artifact] = [], ) -> DexFilesInfo: expect( not _is_exopackage_enabled_for_secondary_dex(ctx), @@ -185,7 +191,8 @@ def get_multi_dex( secondary_dex_dir_srcs = {} all_inputs = flatten(module_to_inputs.values()) - all_inputs_list = argfile(actions = ctx.actions, name = "all_inputs_classpath.txt", args = all_inputs) + all_inputs_list = argfile(actions = ctx.actions, name = "all_inputs_classpath.txt", args = all_inputs + classpath_jars) + classpath_jars_list = argfile(actions = ctx.actions, name = "classpath_jars.txt", args = classpath_jars) if classpath_jars else None for module, module_inputs in module_to_inputs.items(): multi_dex_cmd = cmd_args(android_toolchain.multi_dex_command[RunInfo]) secondary_dex_compression_cmd = cmd_args(android_toolchain.secondary_dex_compression_command[RunInfo]) @@ -245,7 +252,11 @@ def get_multi_dex( ) secondary_dex_compression_cmd.add("--secondary-dex-output-dir", secondary_dex_dir_for_module.as_output()) inputs_to_dex = module_inputs - if not pre_dexed_inputs: + if not pre_dexed_inputs: + if is_root_module(module): + if classpath_jars_list: + multi_dex_cmd.add("--classpath-files", classpath_jars_list) + else: multi_dex_cmd.add("--classpath-files", all_inputs_list) multi_dex_cmd.add("--module", module) @@ -529,55 +540,41 @@ def merge_to_split_dex( ), ) - # Create lib metadata mapping: identifier -> owner target label + # Create lib metadata mapping: identifier -> owner target label. # This is used by the sort tool to determine which module each lib belongs to. - # Gated behind [android].use_sort_pre_dexed_files_tool buckconfig flag (default: false). - use_sort_tool = read_root_config("android", "use_sort_pre_dexed_files_tool", "false").lower() == "true" - sort_pre_dexed_files_tool = getattr(android_toolchain, "sort_pre_dexed_files", None) if use_sort_tool else None - dex_plan_file = None - - if sort_pre_dexed_files_tool: - lib_metadata = {} - for lib in pre_dexed_libs: - if lib.dex: - lib_metadata[lib.identifier] = str(lib.dex.owner.raw_target()) - lib_metadata_file = ctx.actions.write_json("pre_dexed_libs_metadata.json", lib_metadata, has_content_based_path = False) - - # Run the sort_pre_dexed_files tool to produce a dex plan. - # This replaces _sort_pre_dexed_files() which previously ran inside the lambda, - # taking ~3-4 seconds in the Starlark interpreter. The Python tool completes in ~50ms. - filter_dex_output_files = [input.weight_estimate_and_filtered_class_names_file for input in pre_dexed_libs_with_class_names_and_weight_estimates_files] - dex_plan_file = ctx.actions.declare_output("dex_plan.json", has_content_based_path = False) - sort_cmd = cmd_args([ - sort_pre_dexed_files_tool[RunInfo], - "--lib-metadata", - lib_metadata_file, - "--weight-limit", - str(split_dex_merge_config.secondary_dex_weight_limit_bytes), - "--output", - dex_plan_file.as_output(), - ]) - sort_cmd.add("--filter-dex-outputs") - sort_cmd.add(filter_dex_output_files) - if apk_module_graph_file: - sort_cmd.add("--module-graph") - sort_cmd.add(apk_module_graph_file) - if enable_bootstrap_dexes: - sort_cmd.add("--enable-bootstrap-dexes") - ctx.actions.run( - sort_cmd, - category = "sort_pre_dexed_files", - allow_cache_upload = True, - ) + lib_metadata = {} + for lib in pre_dexed_libs: + if lib.dex: + lib_metadata[lib.identifier] = str(lib.dex.owner.raw_target()) + lib_metadata_file = ctx.actions.write_json("pre_dexed_libs_metadata.json", lib_metadata, has_content_based_path = False) + + filter_dex_output_files = [input.weight_estimate_and_filtered_class_names_file for input in pre_dexed_libs_with_class_names_and_weight_estimates_files] + dex_plan_file = ctx.actions.declare_output("dex_plan.json", has_content_based_path = False) + sort_cmd = cmd_args([ + android_toolchain.sort_pre_dexed_files[RunInfo], + "--lib-metadata", + lib_metadata_file, + "--weight-limit", + str(split_dex_merge_config.secondary_dex_weight_limit_bytes), + "--output", + dex_plan_file.as_output(), + ]) + sort_cmd.add("--filter-dex-outputs") + sort_cmd.add(filter_dex_output_files) + if apk_module_graph_file: + sort_cmd.add("--module-graph") + sort_cmd.add(apk_module_graph_file) + if enable_bootstrap_dexes: + sort_cmd.add("--enable-bootstrap-dexes") + ctx.actions.run( + sort_cmd, + category = "sort_pre_dexed_files", + allow_cache_upload = True, + ) - if dex_plan_file: - input_artifacts = [dex_plan_file] - if apk_module_graph_file: - input_artifacts.append(apk_module_graph_file) - else: - input_artifacts = [input.weight_estimate_and_filtered_class_names_file for input in pre_dexed_libs_with_class_names_and_weight_estimates_files] + ( - [apk_module_graph_file] if apk_module_graph_file else [] - ) + input_artifacts = [dex_plan_file] + if apk_module_graph_file: + input_artifacts.append(apk_module_graph_file) primary_dex_artifact_list = ctx.actions.declare_output("pre_dexed_artifacts_for_primary_dex.txt", has_content_based_path = False) primary_dex_output = ctx.actions.declare_output("classes.dex", has_content_based_path = False) primary_dex_class_names_list = ctx.actions.declare_output("primary_dex_class_names_list.txt", has_content_based_path = False) @@ -597,70 +594,55 @@ def merge_to_split_dex( ] def merge_pre_dexed_libs(ctx: AnalysisContext, artifacts, outputs): - # We still need the module graph info for metadata.txt generation (module deps) - # and for the fallback path. + # We still need the module graph info for metadata.txt generation (module deps). apk_module_graph_info = ( get_apk_module_graph_info(ctx, apk_module_graph_file, artifacts) if apk_module_graph_file else get_root_module_only_apk_module_graph_info() ) module_to_canary_class_name_function = apk_module_graph_info.module_to_canary_class_name_function - if dex_plan_file: - # Fast path: read the pre-computed dex plan (produced by sort_pre_dexed_files tool). - plan = artifacts[dex_plan_file].read_json() - - # Build a lookup from identifier to DexLibraryInfo for resolving dex artifacts. - libs_by_id = {} - for batch in pre_dexed_libs_with_class_names_and_weight_estimates_files: - for lib in batch.libs: - libs_by_id[lib.identifier] = lib - - def resolve_plan_groups(groups): - resolved = [] - class_names_list = [] - for group in groups: - inputs_for_group = [] - for lib_id in group["lib_ids"]: - inputs_for_group.append( - DexInputWithSpecifiedClasses( - lib = libs_by_id[lib_id], - dex_class_names = [], # class names come from the plan - ) + plan = artifacts[dex_plan_file].read_json() + + # Build a lookup from identifier to DexLibraryInfo for resolving dex artifacts. + libs_by_id = {} + for batch in pre_dexed_libs_with_class_names_and_weight_estimates_files: + for lib in batch.libs: + libs_by_id[lib.identifier] = lib + + def resolve_plan_groups(groups): + resolved = [] + class_names_list = [] + for group in groups: + inputs_for_group = [] + for lib_id in group["lib_ids"]: + inputs_for_group.append( + DexInputWithSpecifiedClasses( + lib = libs_by_id[lib_id], + dex_class_names = [], # class names come from the plan ) - resolved.append(inputs_for_group) - class_names_list.append(group["class_names"]) - return resolved, class_names_list - - sorted_pre_dexed_inputs = [] - plan_class_names = {} - for module_plan in plan["modules"]: - module = module_plan["module"] - primary_groups, primary_class_names_list = resolve_plan_groups(module_plan["primary_groups"]) - secondary_groups, secondary_class_names_list = resolve_plan_groups(module_plan["secondary_groups"]) - - sorted_pre_dexed_inputs.append( - _SortedPreDexedInputs( - module = module, - primary_dex_inputs = primary_groups, - secondary_dex_inputs = secondary_groups, ) + resolved.append(inputs_for_group) + class_names_list.append(group["class_names"]) + return resolved, class_names_list + + sorted_pre_dexed_inputs = [] + plan_class_names = {} + for module_plan in plan["modules"]: + module = module_plan["module"] + primary_groups, primary_class_names_list = resolve_plan_groups(module_plan["primary_groups"]) + secondary_groups, secondary_class_names_list = resolve_plan_groups(module_plan["secondary_groups"]) + + sorted_pre_dexed_inputs.append( + _SortedPreDexedInputs( + module = module, + primary_dex_inputs = primary_groups, + secondary_dex_inputs = secondary_groups, ) - - for i, class_names in enumerate(primary_class_names_list): - plan_class_names[("primary", module, i)] = class_names - for i, class_names in enumerate(secondary_class_names_list): - plan_class_names[("secondary", module, i)] = class_names - else: - # Fallback path: sort in Starlark (old behavior) - sorted_pre_dexed_inputs = _sort_pre_dexed_files( - ctx, - artifacts, - pre_dexed_libs_with_class_names_and_weight_estimates_files, - split_dex_merge_config, - enable_bootstrap_dexes, - get_module_from_target = apk_module_graph_info.target_to_module_mapping_function, - module_to_canary_class_name_function = module_to_canary_class_name_function, ) - plan_class_names = None + + for i, class_names in enumerate(primary_class_names_list): + plan_class_names[("primary", module, i)] = class_names + for i, class_names in enumerate(secondary_class_names_list): + plan_class_names[("secondary", module, i)] = class_names root_module_secondary_dexes_for_symlinking = {} root_module_bootstrap_dexes_for_symlinking = {} @@ -681,10 +663,7 @@ def merge_to_split_dex( pre_dexed_artifacts = [primary_dex_input.lib.dex for primary_dex_input in primary_dex_inputs if primary_dex_input.lib.dex] if pre_dexed_artifacts: expect(is_root_module(module), "module {} should not have a primary dex!".format(module)) - if plan_class_names: - primary_class_names = plan_class_names[("primary", module, 0)] - else: - primary_class_names = flatten([primary_dex_input.dex_class_names for primary_dex_input in primary_dex_inputs]) + primary_class_names = plan_class_names[("primary", module, 0)] ctx.actions.write( outputs[primary_dex_class_names_list].as_output(), primary_class_names, @@ -703,10 +682,7 @@ def merge_to_split_dex( # If primary dex classes were spread to many based on weight, merge additional dex files here. for bootstrap_idx, bootstrap_dex_input_list in enumerate(additional_base_apk_dex_inputs): this_dex_number = base_apk_dex_files_count + 1 - if plan_class_names: - bootstrap_class_names = plan_class_names[("primary", module, bootstrap_idx + 1)] - else: - bootstrap_class_names = flatten([bootstrap_dex_input.dex_class_names for bootstrap_dex_input in bootstrap_dex_input_list]) + bootstrap_class_names = plan_class_names[("primary", module, bootstrap_idx + 1)] bootstrap_dex_class_list = ctx.actions.write( "class_list_for_bootstrap_dex_{}.txt".format(this_dex_number), bootstrap_class_names, @@ -773,31 +749,21 @@ def merge_to_split_dex( "pre_dexed_artifacts_for_secondary_dex_{}_for_module_{}.txt".format(this_dex_number, module), has_content_based_path = False ) - if plan_class_names: - # Fast path: class names from pre-computed plan, add canary class - canary_dex_input = _create_canary_class( - ctx, - i + 1, - module, - module_to_canary_class_name_function, - ctx.attrs._dex_toolchain[DexToolchainInfo], - ) - all_class_names = canary_dex_input.dex_class_names + plan_class_names[("secondary", module, i)] - secondary_dex_class_list = ctx.actions.write( - "class_list_for_secondary_dex_{}_for_module_{}.txt".format(this_dex_number, module), - all_class_names, - has_content_based_path = False, - ) - pre_dexed_artifacts = [canary_dex_input.lib.dex] if canary_dex_input.lib.dex else [] - pre_dexed_artifacts.extend([dex_input.lib.dex for dex_input in secondary_dex_inputs[i] if dex_input.lib.dex]) - else: - # Fallback path: class names from sorted inputs (includes canary from _sort_pre_dexed_files) - secondary_dex_class_list = ctx.actions.write( - "class_list_for_secondary_dex_{}_for_module_{}.txt".format(this_dex_number, module), - flatten([secondary_dex_input.dex_class_names for secondary_dex_input in secondary_dex_inputs[i]]), - has_content_based_path = False, - ) - pre_dexed_artifacts = [secondary_dex_input.lib.dex for secondary_dex_input in secondary_dex_inputs[i] if secondary_dex_input.lib.dex] + canary_dex_input = _create_canary_class( + ctx, + i + 1, + module, + module_to_canary_class_name_function, + ctx.attrs._dex_toolchain[DexToolchainInfo], + ) + all_class_names = canary_dex_input.dex_class_names + plan_class_names[("secondary", module, i)] + secondary_dex_class_list = ctx.actions.write( + "class_list_for_secondary_dex_{}_for_module_{}.txt".format(this_dex_number, module), + all_class_names, + has_content_based_path = False, + ) + pre_dexed_artifacts = [canary_dex_input.lib.dex] if canary_dex_input.lib.dex else [] + pre_dexed_artifacts.extend([dex_input.lib.dex for dex_input in secondary_dex_inputs[i] if dex_input.lib.dex]) _merge_dexes( ctx.actions, @@ -937,281 +903,6 @@ def _merge_dexes( error_handler = android_toolchain.android_error_handler, ) -def _sort_pre_dexed_files( - ctx: AnalysisContext, - artifacts, - pre_dexed_libs_with_class_names_and_weight_estimates_files: list[DexInputsWithClassNamesAndWeightEstimatesFile], - split_dex_merge_config: SplitDexMergeConfig, - enable_bootstrap_dexes: bool, - get_module_from_target: typing.Callable, - module_to_canary_class_name_function: typing.Callable, -) -> list[_SortedPreDexedInputs]: - sorted_pre_dexed_inputs_map = {} - - # DEX 64K limit enforcement. - # - # Each DEX file is limited to 65536 method_ids, field_ids, and type_ids. - # Per-library ref counts from the DEX header are summed as a conservative - # upper bound — the actual merged DEX has fewer refs because shared - # dependencies are deduplicated. - # - # The merge step uses _DEX_MERGE_OPTIONS = ["--no-desugar", "--no-optimize"], - # so D8 performs a pure mechanical merge with no synthetic generation. - # Merged refs are always <= sum of input refs, never more. - DEX_REF_LIMIT = 65536 - - # Tracking for when to spill over to another dex file based on weight estimate. - # Note that depending on given options, primary dex classes may be spread over N dex files - # (when minSdkVerion is high enough). - current_primary_dex_size_map = {} - current_primary_dex_inputs_map = {} - current_secondary_dex_size_map = {} - current_secondary_dex_inputs_map = {} - - # Tracking for ref-count-based splitting (DEX header counts). - current_primary_dex_method_refs_map = {} - current_primary_dex_field_refs_map = {} - current_primary_dex_type_refs_map = {} - current_secondary_dex_method_refs_map = {} - current_secondary_dex_field_refs_map = {} - current_secondary_dex_type_refs_map = {} - - def assign_pre_dexed_classes_to_secondary_dex( - dest: list[list[DexInputWithSpecifiedClasses]], - module: str, - lib: DexLibraryInfo, - weight_estimate: int, - dex_class_names: list[str], - current_dex_size_map: dict[str, int], # module to size - current_dex_inputs_map: dict[str, list[DexInputWithSpecifiedClasses]], # module to dex file that is being built up - emit_canaries: bool, - dex_weight_limit_bytes: int | None, - method_ref_count: int, - field_ref_count: int, - type_ref_count: int, - current_dex_method_refs_map: dict[str, int], - current_dex_field_refs_map: dict[str, int], - current_dex_type_refs_map: dict[str, int], - ): - if len(dex_class_names) == 0: - return - - current_dex_size = current_dex_size_map.get(module, 0) - should_start_new_dex = False - - # Check weight-based limit (existing behavior) - if dex_weight_limit_bytes != None and current_dex_size + weight_estimate > dex_weight_limit_bytes: - should_start_new_dex = True - - # Check ref-count-based limits (DEX 64K limits for methods, fields, and types). - # Only enforce when dex_weight_limit_bytes is set (i.e., splitting is enabled). - # For primary dex with bootstrap dexes disabled, dex_weight_limit_bytes is None - # and we must not split — extra primary groups become bootstrap dexes in - # assets/ (primary_dex_inputs[1:] below), but without the bootstrap - # classloader the runtime can't find them (ClassNotFoundException). The - # secondary dex metadata numbering (base_apk_dex_files_count) would also - # be wrong, producing mismatched filenames. - if dex_weight_limit_bytes != None: - current_methods = current_dex_method_refs_map.get(module, 0) - current_fields = current_dex_field_refs_map.get(module, 0) - current_types = current_dex_type_refs_map.get(module, 0) - if ( - current_methods + method_ref_count > DEX_REF_LIMIT - or current_fields + field_ref_count > DEX_REF_LIMIT - or current_types + type_ref_count > DEX_REF_LIMIT - ): - should_start_new_dex = True - - if should_start_new_dex: - current_dex_size = 0 - current_dex_inputs_map[module] = [] - current_dex_method_refs_map[module] = 0 - current_dex_field_refs_map[module] = 0 - current_dex_type_refs_map[module] = 0 - - current_dex_inputs = current_dex_inputs_map.setdefault(module, []) - if len(current_dex_inputs) == 0: - if emit_canaries: - canary_class_dex_input = _create_canary_class( - ctx, - len(dest) + 1, - module, - module_to_canary_class_name_function, - ctx.attrs._dex_toolchain[DexToolchainInfo], - ) - current_dex_inputs.append(canary_class_dex_input) - dest.append(current_dex_inputs) - - current_dex_size_map[module] = current_dex_size + weight_estimate - current_dex_method_refs_map[module] = current_dex_method_refs_map.get(module, 0) + method_ref_count - current_dex_field_refs_map[module] = current_dex_field_refs_map.get(module, 0) + field_ref_count - current_dex_type_refs_map[module] = current_dex_type_refs_map.get(module, 0) + type_ref_count - current_dex_inputs.append( - DexInputWithSpecifiedClasses(lib = lib, dex_class_names = dex_class_names), - ) - - def organize_pre_dexed_lib( - dest: list[list[DexInputWithSpecifiedClasses]], - module: str, - lib: DexLibraryInfo, - weight_estimate: int, - dex_class_names: list[str], - current_dex_size_map: dict[str, int], - current_dex_inputs_map: dict[str, list[DexInputWithSpecifiedClasses]], - emit_canaries: bool, - dex_weight_limit_bytes: int | None, - method_ref_count: int, - field_ref_count: int, - type_ref_count: int, - current_dex_method_refs_map: dict[str, int], - current_dex_field_refs_map: dict[str, int], - current_dex_type_refs_map: dict[str, int], - ): - if len(dex_class_names) == 0: - return - - should_start_new_dex = False - if dex_weight_limit_bytes != None and weight_estimate > dex_weight_limit_bytes: - should_start_new_dex = True - if dex_weight_limit_bytes != None and (method_ref_count > DEX_REF_LIMIT or field_ref_count > DEX_REF_LIMIT or type_ref_count > DEX_REF_LIMIT): - should_start_new_dex = True - - if should_start_new_dex: - # Given library is beyond the configured weight or ref limit; subdivide it into - # many dex files to lessen the likelihood of overflowing a dex. - num_classes = len(dex_class_names) - if dex_weight_limit_bytes != None and weight_estimate > dex_weight_limit_bytes: - chunks = weight_estimate / dex_weight_limit_bytes - else: - # Subdivide based on ref counts: use the more constrained dimension - max_refs = max(method_ref_count, field_ref_count, type_ref_count) - chunks = max_refs / DEX_REF_LIMIT - chunk_size = max(1, int(num_classes // chunks)) - for start_index in range(0, num_classes, chunk_size): - end_index = min(start_index + chunk_size, num_classes) - chunked_dex_class_names = dex_class_names[start_index:end_index] - - # Note: the original weight_estimate and ref counts will be reused for the - # chunk since individual class sizes are not exposed. - assign_pre_dexed_classes_to_secondary_dex( - dest, - module, - lib, - weight_estimate, - chunked_dex_class_names, - current_dex_size_map, - current_dex_inputs_map, - emit_canaries, - dex_weight_limit_bytes, - method_ref_count, - field_ref_count, - type_ref_count, - current_dex_method_refs_map, - current_dex_field_refs_map, - current_dex_type_refs_map, - ) - else: - # No need to further divide - assign_pre_dexed_classes_to_secondary_dex( - dest, - module, - lib, - weight_estimate, - dex_class_names, - current_dex_size_map, - current_dex_inputs_map, - emit_canaries, - dex_weight_limit_bytes, - method_ref_count, - field_ref_count, - type_ref_count, - current_dex_method_refs_map, - current_dex_field_refs_map, - current_dex_type_refs_map, - ) - - for pre_dexed_libs_with_class_names_and_weight_estimates in pre_dexed_libs_with_class_names_and_weight_estimates_files: - class_names_and_weight_estimates_json = artifacts[ - pre_dexed_libs_with_class_names_and_weight_estimates.weight_estimate_and_filtered_class_names_file - ].read_json() - for pre_dexed_lib in pre_dexed_libs_with_class_names_and_weight_estimates.libs: - module = get_module_from_target(str(pre_dexed_lib.dex.owner.raw_target())) - pre_dexed_lib_info = class_names_and_weight_estimates_json[pre_dexed_lib.identifier] - primary_dex_class_names = pre_dexed_lib_info["primary_dex_class_names"] - secondary_dex_class_names = pre_dexed_lib_info["secondary_dex_class_names"] - weight_estimate = int(pre_dexed_lib_info["weight_estimate"]) - - # Exact DEX ref counts from the header. Zero is valid for resource-only - # AARs (no code), which don't consume any 64K ref slots. - lib_method_refs = int(pre_dexed_lib_info["method_ref_count"]) - lib_field_refs = int(pre_dexed_lib_info["field_ref_count"]) - lib_type_refs = int(pre_dexed_lib_info["type_ref_count"]) - - module_pre_dexed_inputs = sorted_pre_dexed_inputs_map.setdefault( - module, - _SortedPreDexedInputs( - module = module, - primary_dex_inputs = [], - secondary_dex_inputs = [], - ), - ) - primary_dex_inputs = module_pre_dexed_inputs.primary_dex_inputs - secondary_dex_inputs = module_pre_dexed_inputs.secondary_dex_inputs - - if len(primary_dex_class_names) > 0 and not is_root_module(module): - # TODO(T148680617) We shouldn't allow classes that are specified to be in the - # primary dex to end up in a non-root module, but buck1 allows it and there are - # Voltron configs that rely on this, so we allow it too for migration purposes. - # fail("Non-root modules should not have anything that belongs in the primary dex, " + - # "but {} is assigned to module {} and has the following class names in the primary dex: {}\n".format( - # pre_dexed_lib.dex.owner, - # module, - # "\n".join(primary_dex_class_names), - # ), - # ) - secondary_dex_class_names.extend(primary_dex_class_names) - primary_dex_class_names = [] - - # Organize primary dex classes into logical dex file(s) - organize_pre_dexed_lib( - primary_dex_inputs, - module, - pre_dexed_lib, - weight_estimate, - primary_dex_class_names, - current_primary_dex_size_map, - current_primary_dex_inputs_map, - False, - split_dex_merge_config.secondary_dex_weight_limit_bytes if enable_bootstrap_dexes else None, - lib_method_refs, - lib_field_refs, - lib_type_refs, - current_primary_dex_method_refs_map, - current_primary_dex_field_refs_map, - current_primary_dex_type_refs_map, - ) - - # Organize secondary dex classes into logical dex file(s) - organize_pre_dexed_lib( - secondary_dex_inputs, - module, - pre_dexed_lib, - weight_estimate, - secondary_dex_class_names, - current_secondary_dex_size_map, - current_secondary_dex_inputs_map, - True, - split_dex_merge_config.secondary_dex_weight_limit_bytes, - lib_method_refs, - lib_field_refs, - lib_type_refs, - current_secondary_dex_method_refs_map, - current_secondary_dex_field_refs_map, - current_secondary_dex_type_refs_map, - ) - - return sorted_pre_dexed_inputs_map.values() - def _get_raw_secondary_dex_name(index: int, module: str, base_apk_dex_count: int) -> str: # Root module begins at 2 (primary classes.dex is 1) # Non-root module begins at 1 (classes.dex) diff --git a/prelude/android/gen_aidl.bzl b/prelude/android/gen_aidl.bzl index 8b2a6cf..2898eba 100644 --- a/prelude/android/gen_aidl.bzl +++ b/prelude/android/gen_aidl.bzl @@ -8,7 +8,7 @@ load("@prelude//java:java_toolchain.bzl", "JavaToolchainInfo") load("@prelude//os_lookup:defs.bzl", "Os", "OsLookup") -load(":android_toolchain.bzl", "AndroidToolchainInfo") +load(":android_toolchain.bzl", "AidlToolchainInfo") _AidlSourceInfo = provider( fields = { @@ -17,10 +17,10 @@ _AidlSourceInfo = provider( ) def gen_aidl_impl(ctx: AnalysisContext) -> list[Provider]: - android_toolchain = ctx.attrs._android_toolchain[AndroidToolchainInfo] + aidl_toolchain = ctx.attrs._aidl_toolchain[AidlToolchainInfo] aidl_cmd = cmd_args( - [android_toolchain.aidl] - + ["-p", android_toolchain.framework_aidl_file] + [aidl_toolchain.aidl] + + ["-p", aidl_toolchain.framework_aidl_file] + ["-I", ctx.attrs.import_path] + [a for path in ctx.attrs.import_paths for a in ["-I", path]], # We need the `aidl_srcs` files - otherwise the search on the `import_path` won't find anything. diff --git a/prelude/java/keystore.bzl b/prelude/android/keystore.bzl similarity index 93% rename from prelude/java/keystore.bzl rename to prelude/android/keystore.bzl index a06d70a..b083af0 100644 --- a/prelude/java/keystore.bzl +++ b/prelude/android/keystore.bzl @@ -6,7 +6,7 @@ # of this source tree. You may select, at your option, one of the # above-listed licenses. -load("@prelude//java:java_providers.bzl", "KeystoreInfo") +load(":android_providers.bzl", "KeystoreInfo") def keystore_impl(ctx: AnalysisContext) -> list[Provider]: sub_targets = {} diff --git a/prelude/android/native_build_commands.bzl b/prelude/android/native_build_commands.bzl new file mode 100644 index 0000000..295aa33 --- /dev/null +++ b/prelude/android/native_build_commands.bzl @@ -0,0 +1,124 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is dual-licensed under either the MIT license found in the +# LICENSE-MIT file in the root directory of this source tree or the Apache +# License, Version 2.0 found in the LICENSE-APACHE file in the root directory +# of this source tree. You may select, at your option, one of the +# above-listed licenses. + +# Shared schema for the [native_build_commands] sub-target, used by both +# android_binary_native_library_rules.bzl and meta_only/gatorade.bzl. This lives in a leaf module +# because gatorade.bzl is loaded by android_binary_native_library_rules.bzl and so cannot import the +# entry helper back from there. + +# Whether to capture per-source compile commands (the expensive [native_build_commands][compile] +# kind). Off by default; enable with `-c cxx.emit_native_build_commands=true`. Follows the +# FORCE_SINGLE_CPU config-backed-attr pattern. Single source of truth for both the cxx_library attr +# that carries compile commands onto the linkable graph (rules_impl.bzl) and the app-side compile +# emission (android_binary_native_library_rules.bzl). +EMIT_NATIVE_BUILD_COMMANDS = read_root_config("cxx", "emit_native_build_commands") in ("True", "true") + +# Command "kinds" collected by the [native_build_commands] sub-target. Each kind also gets its own +# filtered sub-target, e.g. `TARGET[native_build_commands][relink]`. Note `mergemap` (the one +# whole-graph merge-map PLAN computation) is distinct from `merge` (the per-soname merge link that +# combines constituent libraries), and `link` is reserved for a plain per-library link on apps that +# neither merge nor relink. +NATIVE_BUILD_COMMAND_KINDS = [ + "compile", + "mergemap", + "merge", + "link", + "relink", + "early_gatorade", + "middle_gatorade", + "late_gatorade", + "bolt", +] + +# The Gatorade phases exposed as top-level product sub-targets on Android app targets: building +# `TARGET[early_gatorade]` runs that phase and outputs the artifacts its gatorade invocation(s) +# produce, as a directory (like [native_libs]). Kept in this leaf module as the single source of +# truth for both the app rule that registers them and apk_genrule.bzl that forwards them through the +# redex/repack/resign wrapper. (Unrelated to NATIVE_BUILD_COMMAND_KINDS above, which is the separate +# [native_build_commands] JSON schema; these names coincide with those Gatorade kinds only by +# convention.) +GATORADE_PHASE_SUBTARGETS = [ + "early_gatorade", + "middle_gatorade", + "late_gatorade", +] + +# One entry in the [native_build_commands] JSON. `argv` is embedded as an ArgLike via +# write_json(with_inputs = False) so an entry renders its command without materializing the produced +# artifact, and `argsfile` lets a consumer expand the full flag list when the argv references an +# @argsfile. This shape is intentionally its own thing, not a compile_commands.json entry nor the +# [linker_commands] shape: it spans compile/link/merge/relink/gatorade/bolt uniformly, so consumers +# should key off `kind` rather than assume any single tool's schema. +# +# `argv` must not contain any `.as_output()` reference: write_json cannot re-serialize an output +# artifact, and doing so would also make this entry's writer a second producer of that artifact. +# Pass the plain output artifact instead. +# +# Field notes: +# - `argv`: for link/merge/relink/bolt/gatorade kinds this is the real executed command. For +# kind=compile it is the compile_commands.json-style invocation (compiler + @argsfile + source), +# NOT the exact object-producing action (it omits `-o`, LTO/bitcode flavor flags, dep-file and +# compiler-wrapper args); it mirrors what comp_db.bzl emits for a source. Use it to reproduce a +# representative compile, not to byte-match the build action. +# - `soname`: for per-library kinds, the FINAL shipped/merged soname the command contributes to. +# Best-effort for kind=compile: the compile->merged-soname map is built from each merged lib's +# PRIMARY constituents (matching shared_object_targets.txt), so an object that lands in a merged +# .so only as a non-primary / transitively-linked-static constituent is reported under its own +# `pre_merge_soname` rather than the merged soname. Split groups and late-gatorade code motion also +# make it approximate — cross-reference the app's merge.map / shared_object_targets.txt when an +# authoritative object->soname mapping is needed. None for whole-graph steps (e.g. compute_mergemap). +# - `pre_merge_soname`: for kind=compile, the per-cxx_library (pre-merge) soname the object was +# built for; None for other kinds. +# - `output`: the produced artifact's short_path for link/merge/relink/bolt/gatorade kinds; for +# kind=compile it is the compiled SOURCE path (the produced object is not available here — the +# full per-source attribution lives in `identifier`). +# - `argsfile`: the @argsfile the command references, if any. The sub-target MATERIALIZES it on disk +# for the link-family (merge/link/relink/bolt) and compile kinds, so a consumer that builds +# TARGET[native_build_commands] (or a per-kind filter) can read the flags. Gatorade argsfiles +# (early's input-containers list, late codegen and tmp-link) are recorded by PATH only, not +# materialized by the sub-target — build the app itself to obtain those files on disk. +def native_build_command_entry(kind, category, arch, soname, output, argv, argsfile = None, identifier = None, pre_merge_soname = None): + if kind not in NATIVE_BUILD_COMMAND_KINDS: + fail("native_build_command_entry: unknown kind {!r}; expected one of {}".format(kind, NATIVE_BUILD_COMMAND_KINDS)) + return { + "arch": arch, + "argsfile": argsfile, + "argv": argv, + "category": category, + "identifier": identifier, + "kind": kind, + "output": output, + "pre_merge_soname": pre_merge_soname, + "soname": soname, + } + +# Record a link-family entry (link/merge/relink/bolt) for a LinkedObject, deduplicating the identical +# guard+append that otherwise gets copy-pasted at every capture site. No-op when recording is off +# (native_cmd_entries == None) or the object carries no reified linker command (prebuilt copies, +# pre-bolt/late-gatorade dummy objects, and DistLTO all have none). `linked_object` is accepted +# untyped to keep this a leaf module (no prelude/linking import); it must expose the LinkedObject +# fields `output`, `linker_command`, and `linker_argsfile`. +# +# Intentionally NOT built on linking's make_link_command_debug_output: that helper also requires an +# argsfile and drops the command otherwise, which would silently lose links that inline all flags — +# unacceptable for a capture-everything sub-target. Here argsfile is optional (recorded as None). +def record_link_command(native_cmd_entries, kind, arch, soname, linked_object, category = "cxx_link", pre_merge_soname = None): + if native_cmd_entries == None or not linked_object.linker_command: + return + native_cmd_entries.append( + native_build_command_entry( + kind = kind, + category = category, + arch = arch, + soname = soname, + output = linked_object.output.short_path, + argv = linked_object.linker_command, + argsfile = linked_object.linker_argsfile, + pre_merge_soname = pre_merge_soname, + ), + ) diff --git a/prelude/android/preprocess_java_classes.bzl b/prelude/android/preprocess_java_classes.bzl index 61383e3..bdfcf96 100644 --- a/prelude/android/preprocess_java_classes.bzl +++ b/prelude/android/preprocess_java_classes.bzl @@ -6,13 +6,17 @@ # of this source tree. You may select, at your option, one of the # above-listed licenses. +load("@prelude//android:android_providers.bzl", "AndroidPreprocessedJavaClassesInfo") load("@prelude//android:android_toolchain.bzl", "AndroidToolchainInfo") load("@prelude//android:util.bzl", "EnhancementContext") load("@prelude//java:java_toolchain.bzl", "JavaToolchainInfo") load("@prelude//java/utils:java_more_utils.bzl", "get_path_separator_for_exec_os") load("@prelude//utils:expect.bzl", "expect") -def get_preprocessed_java_classes(enhance_ctx: EnhancementContext, input_jars: dict[Artifact, TargetLabel]) -> (dict[Artifact, TargetLabel], Artifact | None): +def get_preprocessed_java_classes( + enhance_ctx: EnhancementContext, + input_jars: dict[Artifact, TargetLabel], +) -> (dict[Artifact, TargetLabel], AndroidPreprocessedJavaClassesInfo | None): if not input_jars: return {}, None @@ -74,4 +78,7 @@ def get_preprocessed_java_classes(enhance_ctx: EnhancementContext, input_jars: d enhance_ctx.debug_output("preprocess_java_classes_input_jars_map", input_jars_map) enhance_ctx.debug_output("preprocess_java_classes_materialized_artifacts_dir", materialized_artifacts_dir) - return output_jars_to_owners, materialized_artifacts_dir + return output_jars_to_owners, AndroidPreprocessedJavaClassesInfo( + input_dir = input_dir, + materialized_artifacts_dir = materialized_artifacts_dir, + ) diff --git a/prelude/android/r_dot_java.bzl b/prelude/android/r_dot_java.bzl index 517e3db..a9b8a63 100644 --- a/prelude/android/r_dot_java.bzl +++ b/prelude/android/r_dot_java.bzl @@ -11,9 +11,6 @@ load("@prelude//android:android_toolchain.bzl", "AndroidToolchainInfo") load("@prelude//java:java_library.bzl", "compile_to_jar") load("@prelude//java:java_providers.bzl", "JavaLibraryInfo", "single_library_compiling_deps") load("@prelude//utils:argfile.bzl", "argfile") -load("@prelude//utils:buckconfig.bzl", "read_bool") - -_optimized_resource_processing = read_bool("android", "optimized_resource_processing", default = False, root_cell = True) RDotJavaSourceCode = record( r_dot_java_source_code_dir = Artifact, @@ -24,12 +21,8 @@ RDotJavaSourceCode = record( ids_source_code_zipped = Artifact | None, ) -def get_dummy_r_dot_java( - ctx: AnalysisContext, merge_android_resources_tool: RunInfo, android_resources: list[AndroidResourceInfo], union_package: [str, None] -) -> JavaLibraryInfo: - r_dot_java_source_code = _generate_r_dot_java_source_code( - ctx, merge_android_resources_tool, android_resources, "dummy_r_dot_java", union_package = union_package - ) +def get_dummy_r_dot_java(ctx: AnalysisContext, merge_android_resources_tool: RunInfo, android_resources: list[AndroidResourceInfo]) -> JavaLibraryInfo: + r_dot_java_source_code = _generate_r_dot_java_source_code(ctx, merge_android_resources_tool, android_resources, "dummy_r_dot_java") return _compile_r_dot_java( ctx, r_dot_java_source_code.r_dot_java_source_code_zipped, @@ -44,7 +37,6 @@ def generate_r_dot_javas( uber_r_dot_txt_files: list[Artifact], override_symbols_paths: list[Artifact], duplicate_resources_allowlist: Artifact | None, - union_package: [str, None], referenced_resources_lists: list[Artifact], generate_strings_and_ids_separately: [bool, None] = True, remove_classes: list[str] = [], @@ -72,7 +64,6 @@ def generate_r_dot_javas( uber_r_dot_txt_files = uber_r_dot_txt_files, override_symbols_paths = override_symbols_paths, duplicate_resources_allowlist = duplicate_resources_allowlist, - union_package = union_package, referenced_resources_lists = referenced_resources_lists, ) @@ -113,7 +104,6 @@ def _generate_r_dot_java_source_code( uber_r_dot_txt_files: list[Artifact] = [], override_symbols_paths: list[Artifact] = [], duplicate_resources_allowlist: Artifact | None = None, - union_package: [str, None] = None, referenced_resources_lists: list[Artifact] = [], ) -> RDotJavaSourceCode: merge_resources_cmd = cmd_args(merge_android_resources_tool) @@ -174,18 +164,12 @@ def _generate_r_dot_java_source_code( if duplicate_resources_allowlist != None: merge_resources_cmd.add(["--duplicate-resource-allowlist-path", duplicate_resources_allowlist]) - if union_package != None: - merge_resources_cmd.add(["--union-package", union_package]) - if referenced_resources_lists: referenced_resources_file = argfile( actions = ctx.actions, name = "referenced_resources_lists", args = referenced_resources_lists, has_content_based_path = True ) merge_resources_cmd.add(["--referenced-resources-lists", referenced_resources_file]) - if _optimized_resource_processing: - merge_resources_cmd.add("--optimized-processing") - ctx.actions.run(merge_resources_cmd, category = "r_dot_java_merge_resources", identifier = identifier) return RDotJavaSourceCode( diff --git a/prelude/android/relinker_linker_outputs.bzl b/prelude/android/relinker_linker_outputs.bzl index 8de1de8..704fb34 100644 --- a/prelude/android/relinker_linker_outputs.bzl +++ b/prelude/android/relinker_linker_outputs.bzl @@ -38,13 +38,51 @@ def _llvm_stats_flags_factory() -> typing.Callable: return factory +def _preinline_ir_outputs_factory(output_path: str) -> typing.Callable: + """Factory for the relinker pre-inline IR (.ll) output.""" + + def factory(ctx: AnalysisContext) -> ExtraLinkerOutputs: + ir_file = ctx.actions.declare_output(output_path + ".preinline.ll", has_content_based_path = False) + return ExtraLinkerOutputs( + artifacts = {"preinline-ir": ir_file}, + providers = {"preinline-ir": [DefaultInfo(default_output = ir_file)]}, + ) + + return factory + +def _preinline_ir_flags_factory() -> typing.Callable: + """Factory for the relinker pre-inline IR linker flags. + + Threads the gator `-full-lto-postlink-preinline-ir-dump=` mllvm option + into the LTO backend. `.as_output()` is what registers the linker-written + file as a tracked, materializable buck output (a bare `-Wl,-mllvm` string + would be discarded by RE).""" + + def factory(_ctx: AnalysisContext, outputs: dict[str, Artifact]) -> list[ArgLike]: + return [ + cmd_args( + outputs["preinline-ir"].as_output(), + format = "-Wl,-mllvm,-full-lto-postlink-preinline-ir-dump={}", + ), + ] + + return factory + def _get_extra_relinker_output_types(ctx: AnalysisContext) -> list[str]: """Get list of enabled extra relinker output types from attribute.""" return getattr(ctx.attrs, "extra_relinker_outputs", []) -def get_extra_relinker_args(ctx: AnalysisContext, output_path: str) -> dict[str, typing.Any]: - """Get extra linker outputs factory dict for all enabled relinker outputs.""" +def get_extra_relinker_args(ctx: AnalysisContext, output_path: str, soname: str = "") -> dict[str, typing.Any]: + """Get extra linker outputs factory dict for all enabled relinker outputs. + + `soname` scopes the (multi-GB) pre-inline IR dump: when `preinline_ir_sonames` + is set, only libraries in that allowlist emit the dump output+flag, so a whole + -app build does not dump every library. An empty allowlist keeps legacy + behavior (every relinked library dumps).""" output_types = _get_extra_relinker_output_types(ctx) + allowlist = getattr(ctx.attrs, "preinline_ir_sonames", []) + if allowlist and "preinline-ir" in output_types and soname not in allowlist: + output_types = [t for t in output_types if t != "preinline-ir"] if not output_types: return {} @@ -56,6 +94,10 @@ def get_extra_relinker_args(ctx: AnalysisContext, output_path: str) -> dict[str, result = _llvm_stats_outputs_factory(output_path)(ctx) artifacts |= result.artifacts providers |= result.providers + elif output_type == "preinline-ir": + result = _preinline_ir_outputs_factory(output_path)(ctx) + artifacts |= result.artifacts + providers |= result.providers return ExtraLinkerOutputs(artifacts = artifacts, providers = providers) def combined_flags_factory(ctx: AnalysisContext, outputs: dict[str, Artifact]) -> list[ArgLike]: @@ -63,6 +105,8 @@ def get_extra_relinker_args(ctx: AnalysisContext, output_path: str) -> dict[str, for output_type in output_types: if output_type == "llvm-stats": flags += _llvm_stats_flags_factory()(ctx, outputs) + elif output_type == "preinline-ir": + flags += _preinline_ir_flags_factory()(ctx, outputs) return flags return { diff --git a/prelude/android/robolectric_test.bzl b/prelude/android/robolectric_test.bzl index 5045cc1..bb27957 100644 --- a/prelude/android/robolectric_test.bzl +++ b/prelude/android/robolectric_test.bzl @@ -7,7 +7,7 @@ # above-listed licenses. load("@prelude//android:android_binary_resources_rules.bzl", "get_android_binary_resources_info") -load("@prelude//android:android_library.bzl", "build_android_library", "optional_jars") +load("@prelude//android:android_library.bzl", "android_target_stats", "build_android_library", "optional_jars") load("@prelude//android:android_providers.bzl", "merge_android_packageable_info") load("@prelude//android:android_toolchain.bzl", "AndroidToolchainInfo") load("@prelude//java:java_providers.bzl", "JavaLibraryInfo") @@ -16,6 +16,11 @@ load("@prelude//java:java_toolchain.bzl", "JavaToolchainInfo") load("@prelude//test:inject_test_run_info.bzl", "inject_test_run_info") load("@prelude//utils:expect.bzl", "expect") +def _target_build_file_path(target): + cell = "" if target.cell in ["", "fbsource"] else target.cell + "/" + package = target.package + "/" if target.package else "" + return cell + package + "BUCK" + def robolectric_test_impl(ctx: AnalysisContext) -> list[Provider]: if ctx.attrs._build_only_native_code: return [DefaultInfo()] @@ -52,13 +57,44 @@ def robolectric_test_impl(ctx: AnalysisContext) -> list[Provider]: manifest_entries = ctx.attrs.manifest_entries, ) + resource_source_map = None + resource_source_map_jar = ctx.attrs.env.get("ROBOLECTRIC_RESOURCE_SOURCE_MAP_JAR") + if resource_source_map_jar: + resource_infos = [resource for resource in resources_info.unfiltered_resource_infos if resource.res != None] + resource_dirs = [cmd_args([resource.res, _target_build_file_path(resource.raw_target)], delimiter = "\t") for resource in resource_infos] + resource_dirs_file = ctx.actions.write("resource_source_map_resource_dirs", resource_dirs, has_content_based_path = False) + asset_infos = [ + (asset, _target_build_file_path(resource.raw_target)) for resource in resources_info.unfiltered_resource_infos for asset in resource.assets + ] + asset_dirs = [cmd_args([asset, owner_build_file], delimiter = "\t") for asset, owner_build_file in asset_infos] + asset_dirs_file = ctx.actions.write("resource_source_map_asset_dirs", asset_dirs, has_content_based_path = False) + resource_source_map = ctx.actions.declare_output("resource_source_map.tsv", has_content_based_path = False) + resource_source_map_cmd = cmd_args([ + ctx.attrs._java_toolchain[JavaToolchainInfo].java[RunInfo], + "-jar", + resource_source_map_jar, + "--resource-dirs", + resource_dirs_file, + "--asset-dirs", + asset_dirs_file, + "--output", + resource_source_map.as_output(), + ]) + resource_source_map_cmd.add(cmd_args(hidden = [resource.res for resource in resource_infos] + [asset for asset, _ in asset_infos])) + ctx.actions.run(resource_source_map_cmd, category = "robolectric_resource_source_map", allow_cache_upload = True) + test_config_properties_file = ctx.actions.write( "test_config.properties", [ # Replace \ with \\ for Windows compatibility cmd_args(["android_resource_apk", resources_info.primary_resources_apk], delimiter = "=", replace_regex = ("\\\\\\b", "\\\\")), cmd_args(["android_merged_manifest", resources_info.manifest], delimiter = "=", replace_regex = ("\\\\\\b", "\\\\")), - ], + ] + + ( + [cmd_args(["robolectric_resource_source_map", resource_source_map], delimiter = "=", replace_regex = ("\\\\\\b", "\\\\"))] + if resource_source_map + else [] + ), has_content_based_path = False, ) @@ -76,21 +112,23 @@ def robolectric_test_impl(ctx: AnalysisContext) -> list[Provider]: ".", ]) ctx.actions.run(jar_cmd, category = "test_config_properties_jar_cmd") - extra_cmds.append(cmd_args(hidden = [resources_info.primary_resources_apk, resources_info.manifest])) + extra_cmds.append(cmd_args(hidden = [resources_info.primary_resources_apk, resources_info.manifest] + ([resource_source_map] if resource_source_map else []))) - r_dot_javas = [r_dot_java.library_info.library_output for r_dot_java in resources_info.r_dot_java_infos if r_dot_java.library_info.library_output] + r_dot_javas = [r_dot_java.library_info for r_dot_java in resources_info.r_dot_java_infos if r_dot_java.library_info.library_output] expect(len(r_dot_javas) <= 1, "android_library only works with single R.java") - extra_sub_targets = {} + target_stats_providers, extra_sub_targets = android_target_stats(ctx) + if resource_source_map: + extra_sub_targets["resource_source_map"] = [DefaultInfo(default_output = resource_source_map)] if r_dot_javas: r_dot_java = r_dot_javas[0] - extra_sub_targets["r_dot_java"] = [DefaultInfo(default_output = r_dot_java.full_library)] + extra_sub_targets["r_dot_java"] = [DefaultInfo(default_output = r_dot_java.library_output.full_library)] else: r_dot_java = None java_providers, _ = build_android_library(ctx, r_dot_java = r_dot_java, extra_sub_targets = extra_sub_targets) extra_classpath_entries = [test_config_properties_jar] + ctx.attrs._android_toolchain[AndroidToolchainInfo].android_bootclasspath + optional_jars(ctx) - extra_classpath_entries.extend([r_dot_java.full_library for r_dot_java in r_dot_javas]) + extra_classpath_entries.extend([r_dot_java.library_output.full_library for r_dot_java in r_dot_javas]) external_runner_test_info = build_junit_test( ctx, java_providers.java_library_info, @@ -115,5 +153,6 @@ def robolectric_test_impl(ctx: AnalysisContext) -> list[Provider]: output_for_classpath_macro = java_providers.java_library_info.output_for_classpath_macro, ) providers.append(java_library_without_compiling_deps) + providers.extend(target_stats_providers) return providers diff --git a/prelude/apple/apple_binary.bzl b/prelude/apple/apple_binary.bzl index f4e2064..eef92b4 100644 --- a/prelude/apple/apple_binary.bzl +++ b/prelude/apple/apple_binary.bzl @@ -14,6 +14,7 @@ load( "@prelude//apple:modularization_dependency_graph.bzl", "create_modularization_dep_graph_subtargets_and_provider", ) +load("@prelude//apple:xcassets_asset_symbols.bzl", "meta_xcassets_asset_symbol_usage_providers_and_subtargets") # @oss-disable[end= ]: load( # @oss-disable[end= ]: "@prelude//apple/meta_only:linker_outputs.bzl", # @oss-disable[end= ]: "extra_distributed_thin_lto_opt_outputs_merger", @@ -54,12 +55,15 @@ load( "@prelude//cxx:cxx_types.bzl", "CxxRuleAdditionalParams", "CxxRuleConstructorParams", + "CxxRuleSubTargetParams", + "xcode_data_enabled", ) load("@prelude//cxx:cxx_utility.bzl", "cxx_attrs_get_allow_cache_upload") load( "@prelude//cxx:headers.bzl", "HeaderMode", "cxx_attr_headers", + "cxx_attr_headers_list", "cxx_get_regular_cxx_headers_layout", "prepare_headers", ) @@ -81,6 +85,8 @@ load( "@prelude//linking:linkable_graph.bzl", "LinkableGraph", ) +load("@prelude//target_stats:target_stats.bzl", "CycleMode", "target_stats_providers_and_subtargets") +load("@prelude//target_stats:target_stats_config.bzl", "TARGET_STATS_ENABLED") load("@prelude//utils:arglike.bzl", "ArgLike") load("@prelude//utils:utils.bzl", "filter_and_map_idx", "map_val") load(":apple_bundle_types.bzl", "AppleBundleLinkerMapInfo", "AppleMinDeploymentVersionInfo") @@ -92,7 +98,7 @@ load(":apple_error_handler.bzl", "apple_build_error_handler", "cxx_error_deseria load(":apple_frameworks.bzl", "get_framework_search_path_flags") load(":apple_rpaths.bzl", "get_rpath_flags_for_apple_binary") load(":apple_target_sdk_version.bzl", "get_min_deployment_version_for_node") -load(":apple_utility.bzl", "get_apple_cxx_headers_layout", "get_apple_stripped_attr_value_with_default_fallback") +load(":apple_utility.bzl", "get_apple_cxx_headers_layout", "get_apple_stripped_attr_value_with_default_fallback", "target_stats_header_name") load(":debug.bzl", "AppleDebuggableInfo") load(":resource_groups.bzl", "create_resource_graph") load(":xcode.bzl", "apple_populate_xcode_attributes") @@ -162,6 +168,7 @@ def apple_binary_impl(ctx: AnalysisContext) -> [list[Provider], Promise]: stripped = get_apple_stripped_attr_value_with_default_fallback(ctx) constructor_params = CxxRuleConstructorParams( rule_type = "apple_binary", + generate_sub_targets = CxxRuleSubTargetParams(xcode_data = xcode_data_enabled()), headers_layout = get_apple_cxx_headers_layout(ctx), extra_link_flags = extra_link_flags, extra_hidden = validation_deps_outputs, @@ -254,6 +261,9 @@ def apple_binary_impl(ctx: AnalysisContext) -> [list[Provider], Promise]: all_deps = non_exported_deps + exported_deps + meta_xcassets_usage_providers, meta_xcassets_usage_subtargets = meta_xcassets_asset_symbol_usage_providers_and_subtargets(ctx, cxx_srcs, swift_srcs) + cxx_output.sub_targets.update(meta_xcassets_usage_subtargets) + index_store_subtargets, index_store_info = create_index_store_subtargets_and_provider(ctx, index_stores, swift_index_stores, all_deps) cxx_output.sub_targets.update(index_store_subtargets) @@ -269,6 +279,29 @@ def apple_binary_impl(ctx: AnalysisContext) -> [list[Provider], Promise]: ) cxx_output.sub_targets.update(mod_dep_graph_subtargets) + target_stats_providers = [] + if TARGET_STATS_ENABLED: + target_stats_tools = get_cxx_toolchain_info(ctx).target_stats_tools + if target_stats_tools != None: + # Headers as well as srcs: file_cycles derives its ObjC import + # edges from the header entries, so without them a binary can + # never report a cycle. + target_stats_srcs = {src.file.short_path: src.file for src in cxx_srcs + swift_srcs} + target_stats_srcs.update({ + target_stats_header_name(header): header.artifact + for header in cxx_attr_headers_list(ctx, ctx.attrs.headers, get_apple_cxx_headers_layout(ctx)) + }) + target_stats_providers, target_stats_subtargets = target_stats_providers_and_subtargets( + ctx, + tools = target_stats_tools, + srcs = target_stats_srcs, + deps = all_deps, + cycle_mode = CycleMode("file"), + module_name = module_name, + swift_dot = swift_compile.modularization_dependency_graph if swift_compile else None, + ) + cxx_output.sub_targets.update(target_stats_subtargets) + validation_providers = [ValidationInfo(validations = cxx_output.validation_specs)] if cxx_output.validation_specs else [] all_diagnostics = [] @@ -287,24 +320,30 @@ def apple_binary_impl(ctx: AnalysisContext) -> [list[Provider], Promise]: ] providers = ( - [ - DefaultInfo(default_output = cxx_output.binary, sub_targets = cxx_output.sub_targets), - RunInfo(args = cmd_args(cxx_output.binary, hidden = cxx_output.runtime_files)), - AppleEntitlementsInfo(entitlements_file = ctx.attrs.entitlements_file), - AppleDebuggableInfo(dsyms = [dsym_artifact], binaries = [unstripped_binary], debug_info_tset = cxx_output.external_debug_info), - cxx_output.xcode_data, - cxx_output.compilation_db, - merge_bundle_linker_maps_info(bundle_infos), - UnstrippedLinkOutputInfo(artifact = unstripped_binary), - index_store_info, - mod_dep_graph_info, - ] + filter( + None, + [ + DefaultInfo(default_output = cxx_output.binary, sub_targets = cxx_output.sub_targets), + RunInfo(args = cmd_args(cxx_output.binary, hidden = cxx_output.runtime_files)), + AppleEntitlementsInfo(entitlements_file = ctx.attrs.entitlements_file), + AppleDebuggableInfo(dsyms = [dsym_artifact], binaries = [unstripped_binary], debug_info_tset = cxx_output.external_debug_info), + cxx_output.xcode_data, + cxx_output.compilation_db, + merge_bundle_linker_maps_info(bundle_infos), + UnstrippedLinkOutputInfo(artifact = unstripped_binary), + index_store_info, + mod_dep_graph_info, + swift_dependency_info, + ], + ) + [resource_graph] + min_version_providers + link_command_providers + sanitizer_runtime_providers + validation_providers + diagnostics_providers + + target_stats_providers + + meta_xcassets_usage_providers ) if cxx_output.xplugins_debug_artifacts_info: diff --git a/prelude/apple/apple_bundle.bzl b/prelude/apple/apple_bundle.bzl index 7568736..91c7560 100644 --- a/prelude/apple/apple_bundle.bzl +++ b/prelude/apple/apple_bundle.bzl @@ -26,6 +26,7 @@ load( load("@prelude//apple/user:apple_selected_debug_path_file.bzl", "SELECTED_DEBUG_PATH_FILE_NAME") load("@prelude//apple/user:apple_selective_debugging.bzl", "AppleSelectiveDebuggingInfo") load("@prelude//apple/validation:required_reasons.bzl", "get_required_reasons_validator_output") +load("@prelude//apple/validation:swiftmodule_change_analysis.bzl", "get_swiftmodule_change_analysis_output") load( "@prelude//cxx:cxx_transitive_diagnostics.bzl", "cxx_transitive_diagnostics_combine", @@ -52,13 +53,19 @@ load( "UnstrippedLinkOutputInfo", "make_link_command_debug_output_json_info", ) +load("@prelude//target_stats:target_stats.bzl", "target_stats_aggregate_providers_and_subtargets") load("@prelude//utils:arglike.bzl", "ArgLike") load("@prelude//utils:lazy.bzl", "lazy") load( "@prelude//utils:utils.bzl", "flatten", ) -load("@prelude//xplugins:debug_artifacts.bzl", "xplugins_get_debug_artifacts_info", "xplugins_get_debug_artifacts_subtargets") +load( + "@prelude//xplugins:debug_artifacts.bzl", + "xplugins_get_debug_artifacts_info", + "xplugins_get_debug_artifacts_subtargets", + "xplugins_get_function_mapping_manifest_info", +) load(":apple_bundle_destination.bzl", "AppleBundleDestination") load( ":apple_bundle_part.bzl", @@ -423,7 +430,13 @@ def apple_bundle_impl(ctx: AnalysisContext) -> list[Provider]: sub_targets["linker-maps"] = [DefaultInfo(default_output = linker_maps_directory)] xplugins_debug_info = xplugins_get_debug_artifacts_info(ctx, deps_with_binary) - sub_targets["xplugins"] = xplugins_get_debug_artifacts_subtargets(ctx.actions, xplugins_debug_info) + app_binary = get_default_binary_dep(ctx.attrs.binary) if ctx.attrs.extension == "app" and xplugins_debug_info else None + xplugins_function_mapping_manifest_info = xplugins_get_function_mapping_manifest_info( + ctx.actions, + app_binary.label if app_binary else None, + xplugins_debug_info, + ) + sub_targets["xplugins"] = xplugins_get_debug_artifacts_subtargets(xplugins_function_mapping_manifest_info) link_cmd_debug_file, link_cmd_debug_info = _link_command_debug_data(ctx.actions, deps_with_binary) sub_targets["linker.command"] = [DefaultInfo(default_outputs = filter(None, [link_cmd_debug_file]))] @@ -431,7 +444,11 @@ def apple_bundle_impl(ctx: AnalysisContext) -> list[Provider]: # dsyms dsym_input_binary_arg = _get_dsym_input_binary_arg(ctx, binary_outputs, primary_binary_path_arg) binary_dsym_artifacts = _get_bundle_binary_dsym_artifacts(ctx, binary_outputs, dsym_input_binary_arg) - dep_dsym_artifacts = flatten([info.dsyms for info in deps_debuggable_infos]) + # The same dSYM can reach a bundle through multiple dependency paths. + dep_dsym_artifacts = set() + for info in deps_debuggable_infos: + dep_dsym_artifacts.update(info.dsyms) + dep_dsym_artifacts = list(dep_dsym_artifacts) dsym_artifacts = binary_dsym_artifacts + dep_dsym_artifacts if dsym_artifacts: @@ -512,6 +529,9 @@ def apple_bundle_impl(ctx: AnalysisContext) -> list[Provider]: mod_dep_graph_subtargets, mod_dep_graph_info = _modularization_dep_graph_data(ctx, deps_with_binary) sub_targets.update(mod_dep_graph_subtargets) + target_stats_providers, target_stats_subtargets = target_stats_aggregate_providers_and_subtargets(ctx, deps = deps_with_binary) + sub_targets.update(target_stats_subtargets) + bundle_and_dsym_info_json = { "bundle": bundle, "dsym": dsym_json_info.json_object, @@ -556,6 +576,8 @@ def apple_bundle_impl(ctx: AnalysisContext) -> list[Provider]: ) sub_targets["check"] = [DefaultInfo(default_output = None, other_outputs = transitive_diagnostic_artifacts)] + sub_targets["swiftmodule-change-analysis"] = [DefaultInfo(default_output = get_swiftmodule_change_analysis_output(ctx, deps_with_binary))] + providers = ( [ DefaultInfo(default_output = bundle, sub_targets = sub_targets), @@ -592,11 +614,13 @@ def apple_bundle_impl(ctx: AnalysisContext) -> list[Provider]: mod_dep_graph_info, info_plist_info, ] + + target_stats_providers + bundle_result.providers + validation_providers ) if xplugins_debug_info: providers.append(xplugins_debug_info) + providers.append(xplugins_function_mapping_manifest_info) return providers diff --git a/prelude/apple/apple_bundle_destination.bzl b/prelude/apple/apple_bundle_destination.bzl index bacfcbc..4e4e567 100644 --- a/prelude/apple/apple_bundle_destination.bzl +++ b/prelude/apple/apple_bundle_destination.bzl @@ -126,34 +126,34 @@ def bundle_relative_path_for_destination(destination: AppleBundleDestination, sd else: bundle_destinations = _get_apple_bundle_destinations_for_sdk_name(sdk_name) - if destination.value == "resources": + if destination == AppleBundleDestination("resources"): return bundle_destinations.resources - elif destination.value == "frameworks": + elif destination == AppleBundleDestination("frameworks"): return bundle_destinations.frameworks - elif destination.value == "executables": + elif destination == AppleBundleDestination("executables"): return bundle_destinations.executables - elif destination.value == "extensionkit_extensions": + elif destination == AppleBundleDestination("extensionkit_extensions"): return bundle_destinations.extensionkit_extensions - elif destination.value == "plugins": + elif destination == AppleBundleDestination("plugins"): return bundle_destinations.plugins - elif destination.value == "xpcservices": + elif destination == AppleBundleDestination("xpcservices"): return bundle_destinations.xpcservices - elif destination.value == "metadata": + elif destination == AppleBundleDestination("metadata"): return bundle_destinations.metadata - elif destination.value == "watchapp": + elif destination == AppleBundleDestination("watchapp"): return bundle_destinations.watchapp - elif destination.value == "headers": + elif destination == AppleBundleDestination("headers"): return bundle_destinations.headers - elif destination.value == "modules": + elif destination == AppleBundleDestination("modules"): return bundle_destinations.modules - elif destination.value == "quicklook": + elif destination == AppleBundleDestination("quicklook"): return bundle_destinations.quicklook - elif destination.value == "bundleroot": + elif destination == AppleBundleDestination("bundleroot"): return bundle_destinations.bundleroot - elif destination.value == "loginitems": + elif destination == AppleBundleDestination("loginitems"): return bundle_destinations.loginitems - elif destination.value == "launchagents": + elif destination == AppleBundleDestination("launchagents"): return bundle_destinations.launchagents - elif destination.value == "appclips": + elif destination == AppleBundleDestination("appclips"): return bundle_destinations.appclips fail("Unsupported Apple bundle destination {}".format(destination)) diff --git a/prelude/apple/apple_bundle_part.bzl b/prelude/apple/apple_bundle_part.bzl index ec6eaec..0058f78 100644 --- a/prelude/apple/apple_bundle_part.bzl +++ b/prelude/apple/apple_bundle_part.bzl @@ -28,6 +28,15 @@ load(":apple_sdk_metadata.bzl", "get_apple_sdk_metadata_for_sdk_name") load(":apple_swift_stdlib.bzl", "should_copy_swift_stdlib") load(":apple_toolchain_types.bzl", "AppleToolchainInfo", "AppleToolsInfo") +# Must match the null case returned by `serialize_signing_context_data`: +# https://www.internalfb.com/code/fbsource/fbcode/buck2/prelude/apple/tools/bundling/signing_context_data.py?lines=39%2C58 +_EMPTY_SIGNING_CONTEXT_DATA = { + "provisioning_profile_data_base64": None, + "selected_identity": None, + "signing_context": None, + "version": 1, +} + # Defines where and what should be copied into AppleBundlePart = record( # A file or directory which content should be copied @@ -79,6 +88,16 @@ def bundle_output(ctx: AnalysisContext) -> Artifact: output = ctx.actions.declare_output(bundle_dir_name, has_content_based_path = False) return output +def _bundling_log_args(ctx: AnalysisContext, output_name: str): + if not ctx.attrs._bundling_log_file_enabled: + return ([], None) + + output = ctx.actions.declare_output(output_name, has_content_based_path = False) + args = ["--log-file", output.as_output()] + if ctx.attrs._bundling_log_file_level: + args.extend(["--log-level-file", ctx.attrs._bundling_log_file_level]) + return (args, output) + def assemble_bundle( ctx: AnalysisContext, bundle: Artifact, @@ -101,10 +120,8 @@ def assemble_bundle( tools = ctx.attrs._apple_tools[AppleToolsInfo] tool = tools.assemble_bundle - # Defines common codesign args that can be passed to all codesign-like tools - codesign_args = [] - - # Defines codesign args for bundling only + codesign_selection_args = [] + codesign_execution_args = [] codesign_bundle_extra_args = [] codesign_tool = ctx.attrs._apple_toolchain[AppleToolchainInfo].codesign @@ -129,7 +146,7 @@ def assemble_bundle( else: fail("Code signing configuration `{}` not supported".format(code_signing_configuration)) - codesign_required = codesign_type.value in ["distribution", "adhoc"] + codesign_required = codesign_type != CodeSignType("skip") swift_support_required = swift_stdlib_args and (not ctx.attrs.skip_copying_swift_stdlib) and should_copy_swift_stdlib(bundle.extension) sdk_name = get_apple_sdk_name(ctx) @@ -161,7 +178,7 @@ def assemble_bundle( swift_args = [] if codesign_required: - codesign_args += [ + codesign_execution_args += [ "--codesign", ] @@ -180,30 +197,35 @@ def assemble_bundle( {profile.short_path: profile for profile in source.profiles}, has_content_based_path = False, ) - codesign_args.extend(["--profiles-dir", profiles_dir]) + codesign_selection_args.extend(["--profiles-dir", profiles_dir]) elif source.directory: - codesign_args.extend(["--profiles-dir", source.directory]) + codesign_selection_args.extend(["--profiles-dir", source.directory]) identities_command = ctx.attrs._apple_toolchain[AppleToolchainInfo].codesign_identities_command if ctx.attrs._codesign_identities_command_override: identities_command = ctx.attrs._codesign_identities_command_override[RunInfo] identities_command_args = ["--codesign-identities-command", cmd_args(identities_command)] if identities_command else [] - codesign_args.extend(identities_command_args) + codesign_selection_args.extend(identities_command_args) - if codesign_type.value == "adhoc": - codesign_args.append("--ad-hoc") + if codesign_type == CodeSignType("adhoc"): + codesign_selection_args.append("--ad-hoc") if ctx.attrs.codesign_identity: - codesign_args.extend(["--ad-hoc-codesign-identity", ctx.attrs.codesign_identity]) + codesign_selection_args.extend(["--ad-hoc-codesign-identity", ctx.attrs.codesign_identity]) if profile_selection_required: - codesign_args.append("--embed-provisioning-profile-when-signing-ad-hoc") + codesign_selection_args.append("--embed-provisioning-profile-when-signing-ad-hoc") + + entitlements_args = get_entitlements_codesign_args(ctx, codesign_type) + codesign_execution_args += entitlements_args - codesign_args += get_entitlements_codesign_args(ctx, codesign_type) if getattr(ctx.attrs, "entitlements_suffixed_key_map", None): - codesign_args += ["--entitlements-suffixed-key-map", json.encode(ctx.attrs.entitlements_suffixed_key_map)] + arg = ["--entitlements-suffixed-key-map", json.encode(ctx.attrs.entitlements_suffixed_key_map)] + codesign_execution_args += arg if getattr(ctx.attrs, "entitlements_removed_keys", None): - codesign_args += ["--entitlements-removed-keys", json.encode(ctx.attrs.entitlements_removed_keys)] + arg = ["--entitlements-removed-keys", json.encode(ctx.attrs.entitlements_removed_keys)] + codesign_execution_args += arg if getattr(ctx.attrs, "entitlements_removed_values_map", None): - codesign_args += ["--entitlements-removed-values-map", json.encode(ctx.attrs.entitlements_removed_values_map)] + arg = ["--entitlements-removed-values-map", json.encode(ctx.attrs.entitlements_removed_values_map)] + codesign_execution_args += arg codesign_bundle_extra_args += _get_extra_codesign_args(ctx) info_plist_args = ( @@ -216,28 +238,84 @@ def assemble_bundle( if info_plist_part else [] ) - codesign_args.extend(info_plist_args) + codesign_execution_args.extend(info_plist_args) if ctx.attrs.provisioning_profile_filter: - codesign_args.extend([ + arg = [ "--provisioning-profile-filter", ctx.attrs.provisioning_profile_filter, - ]) + ] + codesign_selection_args.extend(arg) strict_provisioning_profile_search = value_or(ctx.attrs.strict_provisioning_profile_search, ctx.attrs._strict_provisioning_profile_search_default) if strict_provisioning_profile_search: - codesign_args.append("--strict-provisioning-profile-search") + codesign_selection_args.append("--strict-provisioning-profile-search") if ctx.attrs._fast_provisioning_profile_parsing_enabled: - codesign_args.append("--fast-provisioning-profile-parsing") + codesign_selection_args.append("--fast-provisioning-profile-parsing") if ctx.attrs._no_check_certificates: - codesign_args.append("--no-check-certificates") - elif codesign_type.value == "skip": + codesign_selection_args.append("--no-check-certificates") + + if ctx.attrs.entitlements_verification_check_enabled: + codesign_selection_args.append("--verify-entitlements") + + elif codesign_type == CodeSignType("skip"): pass else: fail("Code sign type `{}` not supported".format(codesign_type)) + force_local_bundling = codesign_required + env = {} + cache_buster = ctx.attrs._bundling_cache_buster + if cache_buster: + env["BUCK2_BUNDLING_CACHE_BUSTER"] = cache_buster + + if codesign_required: + signing_context_path = ctx.actions.declare_output("signing_context.json", has_content_based_path = False) + resolve_command = [ + tools.resolve_signing_context, + "--output", + signing_context_path.as_output(), + ] + resolution_log_args, resolution_log_output = _bundling_log_args(ctx, "signing_context_resolution_log.txt") + resolve_command.extend(resolution_log_args) + ctx.actions.run( + cmd_args( + resolve_command + platform_args + codesign_selection_args + codesign_execution_args, + ), + local_only = force_local_bundling, + prefer_local = not force_local_bundling, + category = "apple_resolve_signing_context", + env = env, + error_handler = apple_build_error_handler, + ) + signing_context_path_arg = ["--signing-context-path", signing_context_path] + + signing_info_output = ctx.actions.declare_output("signing-info.json", has_content_based_path = False) + ctx.actions.run( + cmd_args( + [ + tools.signing_info, + "--signing-context-path", + signing_context_path, + "--output", + signing_info_output.as_output(), + ], + ), + category = "apple_signing_info", + error_handler = apple_build_error_handler, + ) + else: + resolution_log_output = None + # Avoid adding a resolver action to every unsigned bundle target. + signing_context_path = ctx.actions.write_json( + "signing_context.json", + _EMPTY_SIGNING_CONTEXT_DATA, + ) + signing_context_path_arg = ["--signing-context-path", signing_context_path] + signing_info_output = ctx.actions.write_json("signing-info.json", {}) + # - Always request codesign manifest, even if signing not required. # Removes the need for conditional subtargets and fields in JSON output. # - Manifest file name reflects whether signing is required or not. @@ -249,8 +327,6 @@ def assemble_bundle( codesign_manifest.as_output(), ] - verify_entitlements_args = ["--verify-entitlements"] if ctx.attrs.entitlements_verification_check_enabled else [] - extra_context_args = [ "--resources-destination", bundle_relative_path_for_destination(AppleBundleDestination("resources"), sdk_name, ctx.attrs.extension, ctx.attrs.versioned_macos_bundle), @@ -264,12 +340,12 @@ def assemble_bundle( "--spec", spec_file, ] - + codesign_args + + codesign_execution_args + codesign_bundle_extra_args - + verify_entitlements_args + platform_args + swift_args - + extra_context_args, + + extra_context_args + + signing_context_path_arg, hidden = [part.source for part in all_parts] + [part.codesign_entitlements for part in all_parts if part.codesign_entitlements] + @@ -304,12 +380,11 @@ def assemble_bundle( command.add("--profile-output", profile_output) subtargets = {} - bundling_log_output = None - if ctx.attrs._bundling_log_file_enabled: - bundling_log_output = ctx.actions.declare_output("bundling_log.txt", has_content_based_path = False) - command.add("--log-file", bundling_log_output.as_output()) - if ctx.attrs._bundling_log_file_level: - command.add("--log-level-file", ctx.attrs._bundling_log_file_level) + if resolution_log_output: + subtargets["signing-context-resolution-log"] = [DefaultInfo(default_output = resolution_log_output)] + bundling_log_args, bundling_log_output = _bundling_log_args(ctx, "bundling_log.txt") + command.add(bundling_log_args) + if bundling_log_output: subtargets["bundling-log"] = [DefaultInfo(default_output = bundling_log_output)] command.add("--check-conflicts") @@ -325,8 +400,15 @@ def assemble_bundle( if bundle_telemetry_logger: command.add("--bundle-telemetry-logger", bundle_telemetry_logger) - signing_info_output = ctx.actions.declare_output("signing-info.json", has_content_based_path = False) - command.add("--signing-info-output", signing_info_output.as_output()) + ctx.actions.run( + command, + local_only = force_local_bundling, + prefer_local = not force_local_bundling, + category = category, + env = env, + error_handler = apple_build_error_handler, + **run_incremental_args, + ) command_json = ctx.actions.declare_output("bundling_command.json", has_content_based_path = False) command_json_cmd_args = ctx.actions.write_json(command_json, command, with_inputs = True, pretty = True) @@ -355,22 +437,6 @@ def assemble_bundle( providers = [AppleBundleManifestInfo(manifest = bundle_manifest)] - env = {} - cache_buster = ctx.attrs._bundling_cache_buster - if cache_buster: - env["BUCK2_BUNDLING_CACHE_BUSTER"] = cache_buster - - force_local_bundling = codesign_type.value != "skip" - ctx.actions.run( - command, - local_only = force_local_bundling, - prefer_local = not force_local_bundling, - category = category, - env = env, - error_handler = apple_build_error_handler, - **run_incremental_args, - ) - codesign_manifest_tree = _make_codesign_manifest_tree(ctx, codesign_manifest, codesign_manifest_parts) codesign_manifest_tree_json = _get_codesign_manifest_tree_as_json(codesign_manifest_tree) codesign_manifest_tree_json_file = ctx.actions.declare_output("codesign_manifest_tree.json", has_content_based_path = False) @@ -408,16 +474,14 @@ def assemble_bundle( ctx.actions.run( cmd_args( [ - tools.signing_context, + tools.provisioning_manifest, "--output", signing_context_output.as_output(), ] - + platform_args - + codesign_args, + + signing_context_path_arg, ), - local_only = force_local_bundling, - prefer_local = not force_local_bundling, category = "apple_provisioning_manifest", + error_handler = apple_build_error_handler, ) signing_context_tree = _make_signing_context_tree(ctx, signing_context_output, signing_context_parts) @@ -564,7 +628,7 @@ def _detect_codesign_type(ctx: AnalysisContext, skip_adhoc_signing: bool) -> Cod return CodeSignType("adhoc" if is_ad_hoc_sufficient else "distribution") codesign_type = compute_codesign_type() - if skip_adhoc_signing and codesign_type.value == "adhoc": + if skip_adhoc_signing and codesign_type == CodeSignType("adhoc"): codesign_type = CodeSignType("skip") return codesign_type @@ -574,10 +638,10 @@ def _get_extra_codesign_args(ctx: AnalysisContext) -> list[str]: return ["--codesign-args={}".format(flag) for flag in codesign_args] def _should_embed_provisioning_profile(ctx: AnalysisContext, codesign_type: CodeSignType) -> bool: - if codesign_type.value == "distribution": + if codesign_type == CodeSignType("distribution"): return True - if codesign_type.value == "adhoc": + if codesign_type == CodeSignType("adhoc"): # The config-based override value takes priority over target value if ctx.attrs._embed_provisioning_profile_when_adhoc_code_signing != None: return ctx.attrs._embed_provisioning_profile_when_adhoc_code_signing diff --git a/prelude/apple/apple_entitlements.bzl b/prelude/apple/apple_entitlements.bzl index 0e1d1f0..0ddf076 100644 --- a/prelude/apple/apple_entitlements.bzl +++ b/prelude/apple/apple_entitlements.bzl @@ -19,10 +19,10 @@ def get_entitlements_codesign_args(ctx: AnalysisContext, codesign_type: CodeSign return entitlements_args def should_include_entitlements(ctx: AnalysisContext, codesign_type: CodeSignType) -> bool: - if codesign_type.value == "distribution": + if codesign_type == CodeSignType("distribution"): return True - if codesign_type.value == "adhoc": + if codesign_type == CodeSignType("adhoc"): # The config-based override value takes priority over target value if ctx.attrs._use_entitlements_when_adhoc_code_signing != None: return ctx.attrs._use_entitlements_when_adhoc_code_signing diff --git a/prelude/apple/apple_finalize_codesign.bzl b/prelude/apple/apple_finalize_codesign.bzl index 00454e0..b80a1d3 100644 --- a/prelude/apple/apple_finalize_codesign.bzl +++ b/prelude/apple/apple_finalize_codesign.bzl @@ -6,6 +6,7 @@ # of this source tree. You may select, at your option, one of the # above-listed licenses. +load("@prelude//xplugins:types.bzl", "XPluginsFunctionMappingManifestInfo") load( ":apple_bundle_types.bzl", "AppleBundleInfo", @@ -19,7 +20,8 @@ load( def _apple_finalize_bundle_impl(ctx): original_bundle = ctx.attrs.bundle - bundle_artifact = original_bundle[DefaultInfo].default_outputs[0] + original_bundle_info = original_bundle[AppleBundleInfo] + bundle_artifact = original_bundle_info.bundle finalized_bundle = ctx.actions.declare_output(bundle_artifact.basename, has_content_based_path = False) cmd = cmd_args([ @@ -37,7 +39,6 @@ def _apple_finalize_bundle_impl(ctx): identifier = bundle_artifact.basename, ) - original_bundle_info = original_bundle[AppleBundleInfo] finalized_bundle_info = AppleBundleInfo( bundle = finalized_bundle, bundle_type = original_bundle_info.bundle_type, @@ -48,6 +49,9 @@ def _apple_finalize_bundle_impl(ctx): ) forwarded_providers = [original_bundle[AppleDebuggableInfo], original_bundle[AppleInfoPlistInfo], original_bundle[AppleBundleLinkerMapInfo]] + xplugins_function_mapping_manifest_info = original_bundle.get(XPluginsFunctionMappingManifestInfo) + if xplugins_function_mapping_manifest_info: + forwarded_providers.append(xplugins_function_mapping_manifest_info) return [ DefaultInfo(default_output = finalized_bundle), @@ -56,7 +60,7 @@ def _apple_finalize_bundle_impl(ctx): apple_finalize_bundle = rule( attrs = { - "bundle": attrs.dep(), + "bundle": attrs.dep(providers = [AppleBundleInfo]), "finalizer": attrs.exec_dep(providers = [RunInfo]), "sign_key": attrs.string(default = "fbios-debug"), }, diff --git a/prelude/apple/apple_framework_versions.bzl b/prelude/apple/apple_framework_versions.bzl index 0fde1de..96407aa 100644 --- a/prelude/apple/apple_framework_versions.bzl +++ b/prelude/apple/apple_framework_versions.bzl @@ -647,6 +647,7 @@ FRAMEWORK_INTRODUCED_VERSIONS = { "macosx": (10, 9, 0), "watchos": (2, 0, 0), }, + "MarketplaceKit": {"iphoneos": (17, 4, 0)}, "Matter": { "appletvos": (16, 0, 0), "iphoneos": (16, 0, 0), @@ -958,6 +959,13 @@ FRAMEWORK_INTRODUCED_VERSIONS = { "macosx": (10, 9, 0), "watchos": (3, 0, 0), }, + "StateReporting": { + "appletvos": (27, 0, 0), + "iphoneos": (27, 0, 0), + "macosx": (27, 0, 0), + "visionos": (27, 0, 0), + "watchos": (27, 0, 0), + }, "StickerKit": { "iphoneos": (18, 0, 0), "macosx": (15, 0, 0), diff --git a/prelude/apple/apple_library.bzl b/prelude/apple/apple_library.bzl index 5eaa55e..dd2374c 100644 --- a/prelude/apple/apple_library.bzl +++ b/prelude/apple/apple_library.bzl @@ -15,6 +15,7 @@ load("@prelude//:paths.bzl", "paths") load("@prelude//:validation_deps.bzl", "get_validation_deps_outputs") load("@prelude//apple:apple_dsym.bzl", "DSYM_SUBTARGET", "get_apple_dsym") load("@prelude//apple:apple_stripping.bzl", "apple_strip_args") +load("@prelude//apple:xcassets_asset_symbols.bzl", "meta_xcassets_asset_symbol_usage_providers_and_subtargets") # @oss-disable[end= ]: load("@prelude//apple/meta_only:apple_library_meta_validation.bzl", "apple_library_validate_for_meta_restrictions") # @oss-disable[end= ]: load( # @oss-disable[end= ]: "@prelude//apple/meta_only:linker_outputs.bzl", @@ -87,6 +88,7 @@ load( "CxxRuleConstructorParams", "CxxRuleProviderParams", "CxxRuleSubTargetParams", + "xcode_data_enabled", ) load("@prelude//cxx:cxx_utility.bzl", "cxx_attrs_get_allow_cache_upload") load("@prelude//cxx:headers.bzl", "cxx_attr_exported_headers", "cxx_attr_headers_list") @@ -104,6 +106,7 @@ load( "@prelude//cxx:preprocessor.bzl", "CPreprocessor", "CPreprocessorInfo", # @unused Used as a type + "EMPTY_CPREPROCESSOR", ) load("@prelude//cxx:target_sdk_version.bzl", "get_unversioned_target_triple") load("@prelude//graphql:graphql.bzl", "graphql_providers") @@ -129,7 +132,7 @@ load(":apple_library_types.bzl", "AppleLibraryInfo") load(":apple_modular_utility.bzl", "MODULE_CACHE_PATH") load(":apple_rpaths.bzl", "get_rpath_flags_for_library") load(":apple_target_sdk_version.bzl", "get_min_deployment_version_for_node") -load(":apple_utility.bzl", "get_apple_cxx_headers_layout", "get_apple_stripped_attr_value_with_default_fallback", "get_module_name") +load(":apple_utility.bzl", "get_apple_cxx_headers_layout", "get_apple_stripped_attr_value_with_default_fallback", "get_module_name", "target_stats_header_name") load( ":debug.bzl", "AppleDebuggableInfo", @@ -204,6 +207,7 @@ def apple_library_impl(ctx: AnalysisContext) -> [Promise, list[Provider]]: ctx, AppleLibraryAdditionalParams( rule_type = "apple_library", + generate_sub_targets = CxxRuleSubTargetParams(xcode_data = xcode_data_enabled()), generate_providers = CxxRuleProviderParams( java_packaging_info = False, java_global_code_info = False, @@ -330,18 +334,19 @@ def _make_mockingbird_library_info_provider(ctx: AnalysisContext) -> list[Mockin if len(swift_sources) == 0: return [] + mockingbird_module_name = get_module_name(ctx) deps_mockingbird_infos = filter(None, [dep.get(MockingbirdLibraryInfo) for dep in cxx_attr_deps(ctx)]) exported_deps_mockingbird_infos = filter(None, [dep.get(MockingbirdLibraryInfo) for dep in cxx_attr_exported_deps(ctx)]) children = [] - dep_names = [] - exported_dep_names = [] + dep_module_names = [] + exported_dep_module_names = [] for info in deps_mockingbird_infos: - dep_names.append(info.name) + dep_module_names.append(info.name) children.append(info.tset) for info in exported_deps_mockingbird_infos: - exported_dep_names.append(info.name) + exported_dep_module_names.append(info.name) children.append(info.tset) mockingbird_srcs_folder = ctx.actions.declare_output("mockingbird_srcs_" + ctx.attrs.name, dir = True, has_content_based_path = False) @@ -352,10 +357,10 @@ def _make_mockingbird_library_info_provider(ctx: AnalysisContext) -> list[Mockin ) mockingbird_record = MockingbirdLibraryRecord( - name = ctx.attrs.name, + name = mockingbird_module_name, srcs = [src.file for src in swift_sources], - dep_names = dep_names, - exported_dep_names = exported_dep_names, + dep_names = dep_module_names, + exported_dep_names = exported_dep_module_names, type = MockingbirdTargetType("library"), src_dir = mockingbird_srcs_folder, ) @@ -364,7 +369,7 @@ def _make_mockingbird_library_info_provider(ctx: AnalysisContext) -> list[Mockin return [ MockingbirdLibraryInfo( - name = ctx.attrs.name, + name = mockingbird_module_name, tset = mockingbird_tset, ) ] @@ -420,7 +425,7 @@ def apple_library_rule_constructor_params_and_swift_providers( objc_swift_interface = swift_compile_result.objc_swift_interface swift_object_files = swift_compile.object_files if swift_compile else [] - swift_pre = CPreprocessor() + swift_pre = EMPTY_CPREPROCESSOR if swift_compile: # If we have Swift we export the extended modulemap that includes # the ObjC exported headers and the -Swift.h header. @@ -495,6 +500,10 @@ def apple_library_rule_constructor_params_and_swift_providers( extra_apple_providers = _make_apple_library_info_provider(ctx, swift_objc_header) + _make_apple_library_for_distribution_info_provider( ctx, swift_library_for_distribution_output ) + meta_xcassets_usage_providers, meta_xcassets_usage_subtargets = ( + ([], {}) if is_test_target else meta_xcassets_asset_symbol_usage_providers_and_subtargets(ctx, cxx_srcs, swift_srcs) + ) + extra_apple_providers += meta_xcassets_usage_providers # Always provide a valid JSON object, so that tooling can depend on its existance modulemap_info_json = {"modulemap": exported_pre.modulemap_artifact} if (exported_pre and exported_pre.modulemap_artifact) else {} @@ -503,6 +512,7 @@ def apple_library_rule_constructor_params_and_swift_providers( modulemap_info_providers = [DefaultInfo(default_output = modulemap_info_json_file, other_outputs = [modulemap_info_json_cmd_args])] subtargets = { + "ast": [DefaultInfo(default_output = swift_compile_result.ast)], "modulemap-info": modulemap_info_providers, "objc-swift-interface": [objc_swift_interface], "swift-compilation-database": [DefaultInfo(default_output = None)], @@ -510,6 +520,7 @@ def apple_library_rule_constructor_params_and_swift_providers( "swiftinterface": [DefaultInfo(default_output = swift_compile_result.swiftinterface)], "swiftmodule": [DefaultInfo(default_output = None)], } + subtargets.update(meta_xcassets_usage_subtargets) if swift_compile and swift_compile.compiled_underlying_pcm_artifact: subtargets["underlying-pcm"] = [DefaultInfo(default_output = swift_compile.compiled_underlying_pcm_artifact)] @@ -542,6 +553,7 @@ def apple_library_rule_constructor_params_and_swift_providers( ] subtargets["swiftmodule"] = [DefaultInfo(default_output = swift_compile.swiftmodule)] + subtargets["swift-objc-header"] = [DefaultInfo(default_output = swift_compile.exported_swift_header)] # Always provide the subtarget, so that clients don't need to handle conditional existence subtargets["swift.check"] = [DefaultInfo(default_output = swift_compile.typecheck_file if swift_compile else None)] @@ -572,6 +584,12 @@ def apple_library_rule_constructor_params_and_swift_providers( extra_preprocessors = [swift_pre, modular_pre], extra_exported_preprocessors = filter(None, [exported_pre]), srcs = cxx_srcs, + target_stats_cycle_mode = "file", + target_stats_module_name = get_module_name(ctx), + target_stats_extra_srcs = { + target_stats_header_name(header): header.artifact for header in cxx_attr_headers_list(ctx, ctx.attrs.headers, header_layout) + exported_hdrs + }, + target_stats_swift_dot = swift_compile.modularization_dependency_graph if swift_compile else None, additional = CxxRuleAdditionalParams( srcs = swift_srcs, argsfiles = swift_compile.argsfiles if swift_compile else CompileArgsfiles(), diff --git a/prelude/apple/apple_package.bzl b/prelude/apple/apple_package.bzl index b82b2bb..bd76dc7 100644 --- a/prelude/apple/apple_package.bzl +++ b/prelude/apple/apple_package.bzl @@ -7,11 +7,13 @@ # above-listed licenses. load("@prelude//apple:apple_toolchain_types.bzl", "AppleToolsInfo") +load("@prelude//xplugins:types.bzl", "XPluginsFunctionMappingManifestInfo") load( ":apple_bundle_types.bzl", "AppleBundleInfo", "AppleBundleLinkerMapInfo", "AppleInfoPlistInfo", + "ApplePackageExtension", ) load(":apple_package_config.bzl", "IpaCompressionLevel") load(":apple_package_types.bzl", "ApplePackageInfo") @@ -57,7 +59,7 @@ def apple_package_impl(ctx: AnalysisContext) -> list[Provider]: ctx.actions.run(process_ipa_cmd, category = category) - return [ + providers = [ DefaultInfo( default_output = package, sub_targets = sub_targets, @@ -65,7 +67,7 @@ def apple_package_impl(ctx: AnalysisContext) -> list[Provider]: ApplePackageInfo( name = package_name, bundle_info = ctx.attrs.bundle[AppleBundleInfo], - extension = ctx.attrs.ext, + extension = ApplePackageExtension(ctx.attrs.ext), package = package, dsyms = ctx.attrs.bundle[AppleDebuggableInfo].dsyms, info_plist = ctx.attrs.bundle[AppleInfoPlistInfo].info_plist, @@ -73,6 +75,10 @@ def apple_package_impl(ctx: AnalysisContext) -> list[Provider]: unstripped_binaries = ctx.attrs.bundle[AppleDebuggableInfo].binaries, ), ] + xplugins_function_mapping_manifest_info = ctx.attrs.bundle.get(XPluginsFunctionMappingManifestInfo) + if xplugins_function_mapping_manifest_info: + providers.append(xplugins_function_mapping_manifest_info) + return providers def _get_ipa_contents(ctx: AnalysisContext) -> Artifact: ipa_package_dep = ctx.attrs._ipa_package @@ -96,13 +102,13 @@ def _get_default_package_cmd(ctx: AnalysisContext, unprocessed_ipa_contents: Art return process_ipa_cmd def _compression_level_arg(compression_level: IpaCompressionLevel) -> str: - if compression_level.value == "none": + if compression_level == IpaCompressionLevel("none"): return "0" - elif compression_level.value == "default": + elif compression_level == IpaCompressionLevel("default"): return "6" - elif compression_level.value == "min": + elif compression_level == IpaCompressionLevel("min"): return "1" - elif compression_level.value == "max": + elif compression_level == IpaCompressionLevel("max"): return "9" else: fail("Unknown .ipa compression level: " + str(compression_level)) diff --git a/prelude/apple/apple_package_types.bzl b/prelude/apple/apple_package_types.bzl index 2f9d0b4..ead36a1 100644 --- a/prelude/apple/apple_package_types.bzl +++ b/prelude/apple/apple_package_types.bzl @@ -6,13 +6,13 @@ # of this source tree. You may select, at your option, one of the # above-listed licenses. -load(":apple_bundle_types.bzl", "AppleBundleInfo") +load(":apple_bundle_types.bzl", "AppleBundleInfo", "ApplePackageExtension") ApplePackageInfo = provider( fields = { "bundle_info": provider_field(AppleBundleInfo), "dsyms": provider_field(list[Artifact]), - "extension": provider_field(str), + "extension": provider_field(ApplePackageExtension), "info_plist": provider_field(Artifact), "linker_maps": provider_field(list[Artifact]), "name": provider_field(str), diff --git a/prelude/apple/apple_resource_types.bzl b/prelude/apple/apple_resource_types.bzl index f11922b..482b380 100644 --- a/prelude/apple/apple_resource_types.bzl +++ b/prelude/apple/apple_resource_types.bzl @@ -10,6 +10,7 @@ load("@prelude//:artifacts.bzl", "ArtifactOutputs") load(":apple_asset_catalog_types.bzl", "AppleAssetCatalogSpec") load(":apple_core_data_types.bzl", "AppleCoreDataSpec") load(":scene_kit_assets_types.bzl", "SceneKitAssetsSpec") +load(":xcassets_asset_symbols.bzl", "MetaXcassetsAssetSymbolSpec") # Represents the values for the `destination` field of `apple_resource` # @@ -52,9 +53,17 @@ CxxResourceSpec = record( resources = field(dict[str, ArtifactOutputs], {}), ) +# A selected asset catalog with the resource-graph target that provided it. +SelectedAppleAssetCatalogSpec = record( + asset_catalog_spec = field(AppleAssetCatalogSpec), + xcassets_symbol_spec = field([MetaXcassetsAssetSymbolSpec, None], None), + target = field(Label), +) + AppleResourceSelectionOutput = record( resource_specs = field(list[AppleResourceSpec]), asset_catalog_specs = field(list[AppleAssetCatalogSpec]), + selected_asset_catalog_specs = field(list[SelectedAppleAssetCatalogSpec], []), core_data_specs = field(list[AppleCoreDataSpec]), scene_kit_assets_spec = field(list[SceneKitAssetsSpec]), cxx_resource_specs = field(list[CxxResourceSpec]), diff --git a/prelude/apple/apple_rules_decls.bzl b/prelude/apple/apple_rules_decls.bzl index e77c4a5..1177106 100644 --- a/prelude/apple/apple_rules_decls.bzl +++ b/prelude/apple/apple_rules_decls.bzl @@ -707,6 +707,7 @@ apple_library = prelude_rule( "supports_shlib_interfaces": attrs.bool(default = True), "swift_compilation_mode": attrs.enum(SwiftCompilationMode.values(), default = "wmo"), "swift_compiler_flags": attrs.list(attrs.arg(), default = []), + "swift_dump_ast_subtarget_enabled": attrs.bool(default = read_bool("apple", "swift_dump_ast_subtarget_enabled", default = False, root_cell = True)), "swift_interface_compilation_enabled": attrs.bool(default = True), "swift_macro_deps": attrs.list(attrs.plugin_dep(kind = SwiftMacroPlugin), default = []), "swift_version": attrs.enum(SwiftVersion, default = SwiftVersion[0]), @@ -1559,12 +1560,15 @@ apple_tools = prelude_rule( "ipa_package_maker": attrs.exec_dep(providers = [RunInfo]), "make_modulemap": attrs.exec_dep(providers = [RunInfo]), "make_vfsoverlay": attrs.exec_dep(providers = [RunInfo]), + "provisioning_manifest": attrs.exec_dep(providers = [RunInfo]), + "resolve_signing_context": attrs.exec_dep(providers = [RunInfo]), "selective_debugging_scrubber": attrs.exec_dep(providers = [RunInfo]), - "signing_context": attrs.exec_dep(providers = [RunInfo]), "signing_context_tree_postprocessor": attrs.exec_dep(providers = [RunInfo]), + "signing_info": attrs.exec_dep(providers = [RunInfo]), "split_arch_combine_dsym_bundles_tool": attrs.exec_dep(providers = [RunInfo]), "spm_packager": attrs.exec_dep(providers = [RunInfo]), "static_archive_linker": attrs.exec_dep(providers = [RunInfo]), + "swiftmodule_change_analysis": attrs.option(attrs.exec_dep(providers = [RunInfo]), default = None), "xcframework_maker": attrs.exec_dep(providers = [RunInfo]), }, ) diff --git a/prelude/apple/apple_test.bzl b/prelude/apple/apple_test.bzl index 5da4b2f..f561961 100644 --- a/prelude/apple/apple_test.bzl +++ b/prelude/apple/apple_test.bzl @@ -28,12 +28,13 @@ load( "@prelude//cxx:cxx_sources.bzl", "CxxSrcWithFlags", # @unused Used as a type ) -load("@prelude//cxx:cxx_types.bzl", "CxxRuleProviderParams", "CxxRuleSubTargetParams") +load("@prelude//cxx:cxx_types.bzl", "CxxRuleProviderParams", "CxxRuleSubTargetParams", "xcode_data_enabled") load( "@prelude//cxx:linker.bzl", "SharedLibraryFlagOverrides", ) load("@prelude//ide_integrations/xcode:data.bzl", "XcodeDataInfoKeys") +load("@prelude//target_stats:target_stats_types.bzl", "TargetStatsInfo") load( "@prelude//utils:dicts.bzl", "flatten_x", @@ -109,6 +110,7 @@ def apple_test_impl(ctx: AnalysisContext) -> [list[Provider], Promise]: compilation_database = True, headers = False, link_group_map = False, + xcode_data = xcode_data_enabled(), ), generate_providers = CxxRuleProviderParams( compilation_database = True, @@ -155,6 +157,8 @@ def apple_test_impl(ctx: AnalysisContext) -> [list[Provider], Promise]: xctest_swift_support_needed = p.support_needed elif isinstance(p, AppleDebuggableInfo): debug_info = project_artifacts(ctx.actions, p.debug_info_tset) + elif isinstance(p, TargetStatsInfo): + cxx_providers.append(p) elif isinstance(p, ValidationInfo): cxx_providers.append(p) expect(xctest_swift_support_needed != None, "Expected `XCTestSwiftSupportInfo` provider to be present") @@ -227,13 +231,16 @@ def apple_test_impl(ctx: AnalysisContext) -> [list[Provider], Promise]: sub_targets[_XCTOOLCHAIN_SUB_TARGET] = ctx.attrs._apple_xctoolchain.providers return ( - [ - DefaultInfo(default_output = xctest_bundle, sub_targets = sub_targets), - _get_test_info(ctx, xctest_bundle, test_host_app_bundle, ui_test_target_app_bundle = ui_test_target_app_bundle), - cxx_library_output.index_store_info, - cxx_library_output.xcode_data_info, - cxx_library_output.cxx_compilationdb_info, - ] + filter( + None, + [ + DefaultInfo(default_output = xctest_bundle, sub_targets = sub_targets), + _get_test_info(ctx, xctest_bundle, test_host_app_bundle, ui_test_target_app_bundle = ui_test_target_app_bundle), + cxx_library_output.index_store_info, + cxx_library_output.xcode_data_info, + cxx_library_output.cxx_compilationdb_info, + ], + ) + bundle_result.providers + cxx_providers ) diff --git a/prelude/apple/apple_toolchain_types.bzl b/prelude/apple/apple_toolchain_types.bzl index 3b555ed..c0fa42c 100644 --- a/prelude/apple/apple_toolchain_types.bzl +++ b/prelude/apple/apple_toolchain_types.bzl @@ -55,7 +55,9 @@ AppleToolsInfo = provider( # @unsorted-dict-items fields = { "assemble_bundle": provider_field(RunInfo), - "signing_context": provider_field(RunInfo), + "provisioning_manifest": provider_field(RunInfo), + "resolve_signing_context": provider_field(RunInfo), + "signing_info": provider_field(RunInfo), "split_arch_combine_dsym_bundles_tool": provider_field(RunInfo), "dry_codesign_tool": provider_field(RunInfo), "adhoc_codesign_tool": provider_field(RunInfo), @@ -72,5 +74,6 @@ AppleToolsInfo = provider( "static_archive_linker": provider_field(RunInfo), "spm_packager": provider_field(RunInfo), "bundle_telemetry_logger": provider_field(RunInfo | None, default = None), + "swiftmodule_change_analysis": provider_field(RunInfo | None, default = None), }, ) diff --git a/prelude/apple/apple_tools.bzl b/prelude/apple/apple_tools.bzl index 8264dbe..f9cb17a 100644 --- a/prelude/apple/apple_tools.bzl +++ b/prelude/apple/apple_tools.bzl @@ -13,7 +13,9 @@ def apple_tools_impl(ctx: AnalysisContext) -> list[Provider]: DefaultInfo(), AppleToolsInfo( assemble_bundle = ctx.attrs.assemble_bundle[RunInfo], - signing_context = ctx.attrs.signing_context[RunInfo], + provisioning_manifest = ctx.attrs.provisioning_manifest[RunInfo], + resolve_signing_context = ctx.attrs.resolve_signing_context[RunInfo], + signing_info = ctx.attrs.signing_info[RunInfo], split_arch_combine_dsym_bundles_tool = ctx.attrs.split_arch_combine_dsym_bundles_tool[RunInfo], dry_codesign_tool = ctx.attrs.dry_codesign_tool[RunInfo], adhoc_codesign_tool = ctx.attrs.adhoc_codesign_tool[RunInfo], @@ -30,5 +32,6 @@ def apple_tools_impl(ctx: AnalysisContext) -> list[Provider]: static_archive_linker = ctx.attrs.static_archive_linker[RunInfo], spm_packager = ctx.attrs.spm_packager[RunInfo], bundle_telemetry_logger = ctx.attrs.bundle_telemetry_logger[RunInfo] if ctx.attrs.bundle_telemetry_logger else None, + swiftmodule_change_analysis = ctx.attrs.swiftmodule_change_analysis[RunInfo] if ctx.attrs.swiftmodule_change_analysis else None, ), ] diff --git a/prelude/apple/apple_utility.bzl b/prelude/apple/apple_utility.bzl index b736039..438b070 100644 --- a/prelude/apple/apple_utility.bzl +++ b/prelude/apple/apple_utility.bzl @@ -8,9 +8,19 @@ load("@prelude//apple:apple_toolchain_types.bzl", "AppleToolchainInfo") load("@prelude//apple/swift:swift_toolchain_types.bzl", "SwiftToolchainInfo") -load("@prelude//cxx:headers.bzl", "CxxHeadersLayout", "CxxHeadersNaming") +load("@prelude//cxx:headers.bzl", "CHeader", "CxxHeadersLayout", "CxxHeadersNaming") load("@prelude//utils:utils.bzl", "value_or") +def target_stats_header_name(header: CHeader) -> str: + """The path a target refers to a header by. + + A header given as a dict is named by its key, which is the only thing + separating two files that share a basename. A plain list keeps its + package-relative path: Apple's naming would flatten it to the basename, + which loses the sub-directory for no benefit here. + """ + return header.name if header.named else header.artifact.short_path + def get_apple_cxx_headers_layout(ctx: AnalysisContext) -> CxxHeadersLayout: namespace = value_or(ctx.attrs.header_path_prefix, ctx.attrs.name) return CxxHeadersLayout(namespace = namespace, naming = CxxHeadersNaming("apple")) diff --git a/prelude/apple/mockingbird/mockingbird_types.bzl b/prelude/apple/mockingbird/mockingbird_types.bzl index 54a4d6c..43f4902 100644 --- a/prelude/apple/mockingbird/mockingbird_types.bzl +++ b/prelude/apple/mockingbird/mockingbird_types.bzl @@ -12,20 +12,20 @@ MockingbirdTargetType = enum("library", "test") MockingbirdLibraryInfo = provider( fields = { - # The name of the target. + # The Swift module name. "name": provider_field(str), - # Contains a tset with this target's MockingbirdLibraryRecord as the value - # and all of its dependency's MockingbirdLibraryRecord in the children. + # Contains a tset with this module's MockingbirdLibraryRecord as the value + # and all of its dependencies' MockingbirdLibraryRecord in the children. "tset": provider_field(MockingbirdLibraryInfoTSet), }, ) MockingbirdLibraryRecord = record( - # The names of this target's dependencies. + # The Swift module names of this module's dependencies. dep_names = field(list[str]), - # The names of this target's exported dependencies. + # The Swift module names of this module's exported dependencies. exported_dep_names = field(list[str]), - # The name of the target. + # The Swift module name. name = str, # Swift sources in this target. srcs = field(list[Artifact]), diff --git a/prelude/apple/prebuilt_apple_framework.bzl b/prelude/apple/prebuilt_apple_framework.bzl index 32e152f..d49d149 100644 --- a/prelude/apple/prebuilt_apple_framework.bzl +++ b/prelude/apple/prebuilt_apple_framework.bzl @@ -19,6 +19,8 @@ load( "@prelude//apple/swift:swift_compilation.bzl", "create_swift_dependency_info", "get_external_debug_info_tsets", + "get_external_swift_ast_dump_tsets", + "get_external_swiftmodule_change_analysis_tsets", "get_swift_anonymous_targets_for_prebuilt_framework", ) load( @@ -276,12 +278,31 @@ def _compile_swiftinterface( tags = [ArtifactInfoTag("swift_debug_info")], ) + swiftmodule_change_analysis_tset = make_artifact_tset( + actions = ctx.actions, + # Just the swiftmodule, not the underlying PCM (unlike debug_info_tset + # above): PCM-only changes aren't swiftmodule changes, and including it + # would re-run this analysis action whenever the PCM changes for no + # reason. + artifacts = [swift_compiled_module.output_artifact], + children = get_external_swiftmodule_change_analysis_tsets(False, ctx.attrs.deps), + label = ctx.label, + ) + + swift_ast_dump_tset = make_artifact_tset( + actions = ctx.actions, + children = get_external_swift_ast_dump_tsets(False, ctx.attrs.deps), + label = ctx.label, + ) + swift_dependency_info = create_swift_dependency_info( ctx, ctx.attrs.deps, deps_providers, swift_compiled_module, debug_info_tset, + swiftmodule_change_analysis_tset, + swift_ast_dump_tset, False, ) diff --git a/prelude/apple/prebuilt_apple_xcframework_macro_impl.bzl b/prelude/apple/prebuilt_apple_xcframework_macro_impl.bzl index 2b7e8fc..9b048be 100644 --- a/prelude/apple/prebuilt_apple_xcframework_macro_impl.bzl +++ b/prelude/apple/prebuilt_apple_xcframework_macro_impl.bzl @@ -53,7 +53,7 @@ def _generate_framework_and_dsym_select_maps(prebuilt_xcframework_args, platform # - We have specified a default_arch and it matches # - We haven't specified a default_arch in which case # we take the first match - if (default_arch != None and arch == default_arch.value) or framework_select_map.get("DEFAULT") == None: + if (default_arch != None and AppleArch(arch) == default_arch) or framework_select_map.get("DEFAULT") == None: framework_select_map["DEFAULT"] = ":" + assemble_name dsym_select_map["DEFAULT"] = dsym_artifacts diff --git a/prelude/apple/resource_groups.bzl b/prelude/apple/resource_groups.bzl index 7ad669b..6e1d2c4 100644 --- a/prelude/apple/resource_groups.bzl +++ b/prelude/apple/resource_groups.bzl @@ -13,8 +13,9 @@ load( ) load(":apple_asset_catalog_types.bzl", "AppleAssetCatalogSpec") load(":apple_core_data_types.bzl", "AppleCoreDataSpec") -load(":apple_resource_types.bzl", "AppleResourceSelectionOutput", "AppleResourceSpec", "CxxResourceSpec") +load(":apple_resource_types.bzl", "AppleResourceSelectionOutput", "AppleResourceSpec", "CxxResourceSpec", "SelectedAppleAssetCatalogSpec") load(":scene_kit_assets_types.bzl", "SceneKitAssetsSpec") +load(":xcassets_asset_symbols.bzl", "MetaXcassetsAssetSymbolSpec") ResourceGroupInfo = provider( # @unsorted-dict-items @@ -52,6 +53,9 @@ ResourceGraphNode = record( scene_kit_assets_spec = field([SceneKitAssetsSpec, None], None), # Actual resource data, present when node corresponds to `cxx_library` target containing resources. cxx_resource_spec = field([CxxResourceSpec, None], None), + # Asset symbol metadata from an asset catalog opted into trimming. Carries the + # producer and catalog identity through bundle-level resource selection. + xcassets_symbol_spec = field([MetaXcassetsAssetSymbolSpec, None], None), ) ResourceGraphTSet = transitive_set() @@ -75,6 +79,7 @@ def create_resource_graph( core_data_spec: [AppleCoreDataSpec, None] = None, scene_kit_assets_spec: [SceneKitAssetsSpec, None] = None, cxx_resource_spec: [CxxResourceSpec, None] = None, + xcassets_symbol_spec: [MetaXcassetsAssetSymbolSpec, None] = None, should_propagate: bool = True, ) -> ResourceGraphInfo: # Collect deps and exported_deps with resources that should propagate. @@ -101,6 +106,7 @@ def create_resource_graph( core_data_spec = core_data_spec, scene_kit_assets_spec = scene_kit_assets_spec, cxx_resource_spec = cxx_resource_spec, + xcassets_symbol_spec = xcassets_symbol_spec, ) children = [child_node.nodes for child_node in dep_graphs + exported_dep_graphs] return ResourceGraphInfo( @@ -168,6 +174,7 @@ def get_filtered_resources( resource_specs = [] asset_catalog_specs = [] + selected_asset_catalog_specs = [] core_data_specs = [] scene_kit_assets_specs = [] cxx_resource_specs = [] @@ -192,6 +199,13 @@ def get_filtered_resources( asset_catalog_spec = node.asset_catalog_spec if asset_catalog_spec: asset_catalog_specs.append(asset_catalog_spec) + selected_asset_catalog_specs.append( + SelectedAppleAssetCatalogSpec( + asset_catalog_spec = asset_catalog_spec, + xcassets_symbol_spec = node.xcassets_symbol_spec, + target = target, + ), + ) core_data_spec = node.core_data_spec if core_data_spec: core_data_specs.append(core_data_spec) @@ -205,6 +219,7 @@ def get_filtered_resources( return AppleResourceSelectionOutput( resource_specs = resource_specs, asset_catalog_specs = asset_catalog_specs, + selected_asset_catalog_specs = selected_asset_catalog_specs, core_data_specs = core_data_specs, scene_kit_assets_spec = scene_kit_assets_specs, cxx_resource_specs = cxx_resource_specs, diff --git a/prelude/apple/swift/apple_sdk_clang_module.bzl b/prelude/apple/swift/apple_sdk_clang_module.bzl index 2e70d1a..9e0f526 100644 --- a/prelude/apple/swift/apple_sdk_clang_module.bzl +++ b/prelude/apple/swift/apple_sdk_clang_module.bzl @@ -23,6 +23,7 @@ def apple_sdk_clang_module_impl(ctx: AnalysisContext) -> list[Provider]: input_relative_path = ctx.attrs.modulemap_relative_path, deps = ctx.attrs.deps, cxx_deps = ctx.attrs.cxx_deps, + overlays_transitive_deps = ctx.attrs.overlays_transitive_deps, ) return [ @@ -42,5 +43,7 @@ apple_sdk_clang_module = rule( "module_name": attrs.string(), "modulemap_relative_path": attrs.string(), "overlays": attrs.dict(key = attrs.string(), value = attrs.list(attrs.string(), default = []), sorted = False, default = {}), + # Transitive closure filtered to overlay owners and underlyings. + "overlays_transitive_deps": attrs.list(attrs.string(), default = []), }, ) diff --git a/prelude/apple/swift/apple_sdk_modules_utility.bzl b/prelude/apple/swift/apple_sdk_modules_utility.bzl index 9d2924b..574da48 100644 --- a/prelude/apple/swift/apple_sdk_modules_utility.bzl +++ b/prelude/apple/swift/apple_sdk_modules_utility.bzl @@ -6,10 +6,10 @@ # of this source tree. You may select, at your option, one of the # above-listed licenses. -load("@prelude//utils:set.bzl", "set") load( ":swift_toolchain_types.bzl", "SdkSwiftOverlayInfo", + "SdkUncompiledModuleInfo", "SwiftCompiledModuleTset", "SwiftToolchainInfo", # @unused Used as a type "WrappedSdkCompiledModuleInfo", @@ -40,36 +40,54 @@ def get_uncompiled_sdk_deps(sdk_modules: list[str], required_modules: list[str], if not is_sdk_modules_provided(toolchain): fail("SDK deps are not set for swift_toolchain") - all_sdk_modules = sdk_modules + required_modules - all_sdk_modules = set(all_sdk_modules) + swift_map = toolchain.uncompiled_swift_sdk_modules_deps + clang_map = toolchain.uncompiled_clang_sdk_modules_deps - sdk_deps = [] - sdk_overlays = [] - - def process_sdk_module_dep(dep_name, uncompiled_sdk_modules_map): - if dep_name not in uncompiled_sdk_modules_map: - return + direct_names = set(sdk_modules + required_modules) - sdk_dep = uncompiled_sdk_modules_map[dep_name] - sdk_deps.append(sdk_dep) - - if SdkSwiftOverlayInfo not in sdk_dep: - return + sdk_deps = [] + for sdk_module_dep_name in list(direct_names): + if sdk_module_dep_name in swift_map: + sdk_deps.append(swift_map[sdk_module_dep_name]) + if sdk_module_dep_name in clang_map: + sdk_deps.append(clang_map[sdk_module_dep_name]) - overlay_info = sdk_dep[SdkSwiftOverlayInfo] - for underlying_module, overlay_modules in overlay_info.overlays.items(): - # Only add a cross import SDK overlay if both modules associated with the overlay are required - if all_sdk_modules.contains(underlying_module): - # Cross import overlays themselves are always Swift modules, but the underlying module - # can be a Swift module or a Clang module - sdk_overlays.extend([ - toolchain.uncompiled_swift_sdk_modules_deps[overlay_name] - for overlay_name in overlay_modules - if overlay_name in toolchain.uncompiled_swift_sdk_modules_deps - ]) + # ModuleA declares a cross-import overlay on ModuleADependency, which the + # compiler auto-loads whenever both are visible, even if ModuleADependency + # only arrives transitively. Match overlays against the precomputed + # overlay-filtered transitive closure, not just the direct sdk_modules. + reachable = set(direct_names) + for sdk_module_dep_name in list(direct_names): + for uncompiled_sdk_modules_map in (swift_map, clang_map): + if sdk_module_dep_name not in uncompiled_sdk_modules_map: + continue + sdk_dep = uncompiled_sdk_modules_map[sdk_module_dep_name] + if SdkUncompiledModuleInfo not in sdk_dep: + continue + transitive_names = sdk_dep[SdkUncompiledModuleInfo].overlays_transitive_deps or [] + for transitive_name in transitive_names: + reachable.add(transitive_name) - for sdk_module_dep_name in all_sdk_modules.list(): - process_sdk_module_dep(sdk_module_dep_name, toolchain.uncompiled_swift_sdk_modules_deps) - process_sdk_module_dep(sdk_module_dep_name, toolchain.uncompiled_clang_sdk_modules_deps) + sdk_overlays = [] + # Seed with the direct deps: an overlay listed explicitly in sdk_modules + # is already in sdk_deps above and must not be added a second time. + seen_overlays = set(direct_names) + for reachable_name in list(reachable): + for uncompiled_sdk_modules_map in (swift_map, clang_map): + if reachable_name not in uncompiled_sdk_modules_map: + continue + sdk_dep = uncompiled_sdk_modules_map[reachable_name] + if SdkSwiftOverlayInfo not in sdk_dep: + continue + overlay_info = sdk_dep[SdkSwiftOverlayInfo] + for underlying_module, overlay_modules in overlay_info.overlays.items(): + # Only add a cross import SDK overlay if both modules associated with the overlay are required + if underlying_module in reachable: + # Cross import overlays themselves are always Swift modules, but the underlying module + # can be a Swift module or a Clang module + for overlay_name in overlay_modules: + if overlay_name in swift_map and overlay_name not in seen_overlays: + seen_overlays.add(overlay_name) + sdk_overlays.append(swift_map[overlay_name]) return sdk_deps + sdk_overlays diff --git a/prelude/apple/swift/apple_sdk_swift_module.bzl b/prelude/apple/swift/apple_sdk_swift_module.bzl index 8434d3e..eee5a93 100644 --- a/prelude/apple/swift/apple_sdk_swift_module.bzl +++ b/prelude/apple/swift/apple_sdk_swift_module.bzl @@ -36,6 +36,7 @@ def apple_sdk_swift_module_impl(ctx: AnalysisContext) -> list[Provider]: is_framework = ctx.attrs.is_framework, is_swiftmodule = True, module_name = ctx.attrs.module_name, + overlays_transitive_deps = ctx.attrs.overlays_transitive_deps, partial_cmd = cmd, target = target, ) @@ -56,6 +57,8 @@ apple_sdk_swift_module = rule( # which has a special suffix to distinguish Swift and Clang modules with the same name "module_name": attrs.string(), "overlays": attrs.dict(key = attrs.string(), value = attrs.list(attrs.string(), default = []), sorted = False, default = {}), + # Transitive closure filtered to overlay owners and underlyings. + "overlays_transitive_deps": attrs.list(attrs.string(), default = []), # A prefixed path ($SDKROOT/$PLATFORM_DIR) to swiftinterface textual file. "swiftinterface_relative_path": attrs.option(attrs.string(), default = None), # if `swiftinterface` is None represents a Root node. "target": attrs.string(), diff --git a/prelude/apple/swift/swift_compilation.bzl b/prelude/apple/swift/swift_compilation.bzl index 61d5b66..84eeb13 100644 --- a/prelude/apple/swift/swift_compilation.bzl +++ b/prelude/apple/swift/swift_compilation.bzl @@ -32,6 +32,7 @@ load( ) load("@prelude//cxx:argsfiles.bzl", "CompileArgsfile", "CompileArgsfiles") load("@prelude//cxx:cxx_context.bzl", "get_cxx_platform_info", "get_cxx_toolchain_info") +load("@prelude//cxx:cxx_library_utility.bzl", "cxx_attr_deps", "cxx_attr_exported_deps") load( "@prelude//cxx:cxx_sources.bzl", "CxxSrcWithFlags", # @unused Used as a type @@ -174,6 +175,7 @@ SwiftCompileResult = record( swift_compilation = field(SwiftCompilationOutput | None), objc_swift_interface = field(DefaultInfo), swiftinterface = field(Artifact | None), + ast = field(Artifact | None), ) SwiftDebugInfo = record( @@ -445,7 +447,7 @@ def compile_swift( objc_swift_interface_info = _create_objc_swift_interface(ctx, shared_flags, module_name) if not srcs: - return SwiftCompileResult(swift_compilation = None, objc_swift_interface = objc_swift_interface_info, swiftinterface = None) + return SwiftCompileResult(swift_compilation = None, objc_swift_interface = objc_swift_interface_info, swiftinterface = None, ast = None) # Content-based path hashing and Swift incremental compilation are incompatible. if uses_content_based_paths and should_build_swift_incrementally(ctx): @@ -479,6 +481,17 @@ def compile_swift( output_swiftinterface, ) + output_ast = None + if getattr(ctx.attrs, "swift_dump_ast_subtarget_enabled", False): + output_ast = ctx.actions.declare_output(module_name + "-ast-dump", dir = True, has_content_based_path = uses_content_based_paths) + _compile_dump_ast( + ctx, + toolchain, + shared_flags, + srcs, + output_ast, + ) + # When compiling with WMO or incremental with split actions enabled, we compile # the swiftmodule separately. In pure incremental mode, we generate the swiftmodule # as part of the compile action to make use of incrementality. @@ -576,7 +589,13 @@ def compile_swift( swiftmodule = output_swiftmodule, typecheck_file = typecheck_file, compiled_underlying_pcm_artifact = exported_compiled_underlying_pcm.output_artifact if exported_compiled_underlying_pcm else None, - dependency_info = get_swift_dependency_info(ctx, output_swiftmodule, deps_providers, is_macro), + dependency_info = get_swift_dependency_info( + ctx, + output_swiftmodule, + deps_providers, + is_macro, + swift_ast_dump_artifacts = filter(None, [output_ast]), + ), pre = pre, exported_pre = exported_pp_info, exported_swift_header = exported_swift_header.artifact, @@ -595,6 +614,7 @@ def compile_swift( ), objc_swift_interface = objc_swift_interface_info, swiftinterface = output_swiftinterface, + ast = output_ast, ) def _compile_swiftinterface( @@ -626,6 +646,49 @@ def _compile_swiftinterface( supports_serialized_errors = False, ) +def _compile_dump_ast(ctx: AnalysisContext, toolchain: SwiftToolchainInfo, shared_flags: cmd_args, srcs: list[CxxSrcWithFlags], output_ast_dir: Artifact): + ast_output = output_ast_dir.as_output() + output_file_map = {} + for src in srcs: + # safe_name includes path in case basenames clash + safe_name = src.file.short_path.replace("/", "_") + output_file_map[src.file] = { + "ast-dump": cmd_args(ast_output, format = "{}/" + safe_name + ".ast", delimiter = ""), + } + + argfile_cmd = cmd_args(shared_flags) + if ctx.attrs.swift_module_skip_function_bodies: + argfile_cmd.add([ + "-Xfrontend", + "-experimental-skip-non-inlinable-function-bodies-without-types", + ]) + + additional_flags = cmd_args(["-dump-ast"]) + additional_flags.add(cmd_args(hidden = ast_output)) + + # `-dump-ast` disables `-wmo`. Without an explicit `-j`, those jobs run + # serially over each file in larger modules, becoming potential bottlenecks. + additional_flags.add([ + "-enable-batch-mode", + "-driver-batch-size-limit", + str(INCREMENTAL_SWIFT_COMPILE_BATCH_SIZE), + "-j", + str(INCREMENTAL_SWIFT_COMPILE_MAX_NUM_THREADS), + ]) + + _compile_with_argsfile( + ctx = ctx, + category = "dump_ast", + shared_flags = argfile_cmd, + srcs = srcs, + additional_flags = additional_flags, + toolchain = toolchain, + num_threads = INCREMENTAL_SWIFT_COMPILE_MAX_NUM_THREADS, + output_file_map = output_file_map, + supports_serialized_errors = False, + incremental_build_allowed = False, + ) + # We use separate actions for swiftmodule and object file output. This # improves build parallelism at the cost of duplicated work, but by disabling # type checking in function bodies the swiftmodule compilation can be done much @@ -819,7 +882,7 @@ def _compile_object( else: num_threads = 1 swiftdeps = [] - output_object = ctx.actions.declare_output(module_name + ".o", has_content_based_path = uses_content_based_paths) + output_object = ctx.actions.declare_output(module_name + SWIFT_EXTENSION + ".o", has_content_based_path = uses_content_based_paths) objects = [output_object] object_format = toolchain.object_format.value embed_bitcode = False @@ -1379,6 +1442,12 @@ def _get_swift_paths_tsets(is_macro: bool, deps: list[Dependency]) -> list[Swift def get_external_debug_info_tsets(is_macro: bool, deps: list[Dependency]) -> list[ArtifactTSet]: return [d.debug_info_tset for d in _get_swift_dependency_info(is_macro, deps)] +def get_external_swiftmodule_change_analysis_tsets(is_macro: bool, deps: list[Dependency]) -> list[ArtifactTSet]: + return [d.swiftmodule_change_analysis_tset for d in _get_swift_dependency_info(is_macro, deps)] + +def get_external_swift_ast_dump_tsets(is_macro: bool, deps: list[Dependency]) -> list[ArtifactTSet]: + return [d.swift_ast_dump_tset for d in _get_swift_dependency_info(is_macro, deps)] + def get_swift_pcm_uncompile_info( ctx: AnalysisContext, propagated_exported_preprocessor_info: [CPreprocessorInfo, None], exported_pre: [CPreprocessor, None] ) -> [SwiftPCMUncompiledInfo, None]: @@ -1417,7 +1486,14 @@ def get_swift_pcm_uncompile_info( return None def create_swift_dependency_info( - ctx: AnalysisContext, deps, deps_providers: list, compiled_info: [SwiftCompiledModuleInfo, None], debug_info_tset: ArtifactTSet, is_macro: bool + ctx: AnalysisContext, + deps, + deps_providers: list, + compiled_info: [SwiftCompiledModuleInfo, None], + debug_info_tset: ArtifactTSet, + swiftmodule_change_analysis_tset: ArtifactTSet, + swift_ast_dump_tset: ArtifactTSet, + is_macro: bool, ): # We pass through the SDK swiftmodules here to match Buck 1 behaviour. This is # pretty loose, but it matches Buck 1 behavior so cannot be improved until @@ -1437,9 +1513,17 @@ def create_swift_dependency_info( has_exported_headers = len(getattr(ctx.attrs, "exported_headers", [])) > 0, is_modular = ctx.attrs.modular, is_macro = is_macro, + swift_ast_dump_tset = swift_ast_dump_tset, + swiftmodule_change_analysis_tset = swiftmodule_change_analysis_tset, ) -def get_swift_dependency_info(ctx: AnalysisContext, output_module: Artifact | None, deps_providers: list, is_macro: bool) -> SwiftDependencyInfo: +def get_swift_dependency_info( + ctx: AnalysisContext, + output_module: Artifact | None, + deps_providers: list, + is_macro: bool, + swift_ast_dump_artifacts: list[Artifact] = [], +) -> SwiftDependencyInfo: exported_deps = _exported_deps(ctx) if output_module: @@ -1453,20 +1537,46 @@ def get_swift_dependency_info(ctx: AnalysisContext, output_module: Artifact | No else: compiled_info = None + all_deps = ctx.attrs.deps + getattr(ctx.attrs, "exported_deps", []) + debug_info_tset = make_artifact_tset( actions = ctx.actions, artifacts = filter(None, [output_module]), - children = get_external_debug_info_tsets(is_macro, ctx.attrs.deps + getattr(ctx.attrs, "exported_deps", [])), + children = get_external_debug_info_tsets(is_macro, all_deps), label = ctx.label, tags = [ArtifactInfoTag("swift_debug_info")], ) + # Deliberately broader than `all_deps` above (also covers toolchain + # `default_deps` and `deps_query`, via `cxx_attr_deps`/`cxx_attr_exported_deps`): + # this tset is new, so widening it to match the rest of the codebase's + # convention has no existing behavior to preserve. `debug_info_tset` and + # `swift_ast_dump_tset` predate this and keep the narrower `all_deps` to + # avoid changing their already-shipped output. + swiftmodule_change_analysis_deps = cxx_attr_deps(ctx) + cxx_attr_exported_deps(ctx) + + swiftmodule_change_analysis_tset = make_artifact_tset( + actions = ctx.actions, + artifacts = filter(None, [output_module]), + children = get_external_swiftmodule_change_analysis_tsets(is_macro, swiftmodule_change_analysis_deps), + label = ctx.label, + ) + + swift_ast_dump_tset = make_artifact_tset( + actions = ctx.actions, + artifacts = swift_ast_dump_artifacts, + children = get_external_swift_ast_dump_tsets(is_macro, all_deps), + label = ctx.label, + ) + return create_swift_dependency_info( ctx, exported_deps, deps_providers, compiled_info, debug_info_tset, + swiftmodule_change_analysis_tset, + swift_ast_dump_tset, is_macro, ) diff --git a/prelude/apple/swift/swift_toolchain_types.bzl b/prelude/apple/swift/swift_toolchain_types.bzl index 0453303..f8a4f28 100644 --- a/prelude/apple/swift/swift_toolchain_types.bzl +++ b/prelude/apple/swift/swift_toolchain_types.bzl @@ -79,6 +79,8 @@ SdkUncompiledModuleInfo = provider( "is_swiftmodule": provider_field(typing.Any, default = None), # A real name of a module, without distinguishing suffixes. "module_name": provider_field(typing.Any, default = None), + # Transitive closure filtered to overlay owners and underlyings. + "overlays_transitive_deps": provider_field(typing.Any, default = None), # Partial arguments, required to compile a particular SDK module. "partial_cmd": provider_field(typing.Any, default = None), # A string of the compiler target triple to use for clang module deps, eg arm64-apple-ios16.4 diff --git a/prelude/apple/swift/swift_types.bzl b/prelude/apple/swift/swift_types.bzl index bfff3fb..70cee42 100644 --- a/prelude/apple/swift/swift_types.bzl +++ b/prelude/apple/swift/swift_types.bzl @@ -41,6 +41,12 @@ SwiftDependencyInfo = provider( "is_macro": provider_field(bool), # If this target provides a clang module, used to validate non-modular dep exports "is_modular": provider_field(bool), + # Tset of this target's own `-dump-ast` output directory (see the `[ast]` + # subtarget) merged with the same tset from all deps. + "swift_ast_dump_tset": provider_field(ArtifactTSet), + # Tset of this target's own `.swiftmodule` output merged with the + # same tsets from all deps. + "swiftmodule_change_analysis_tset": provider_field(ArtifactTSet), } ) diff --git a/prelude/apple/tools/BUCK b/prelude/apple/tools/BUCK index e09f249..4e248e9 100644 --- a/prelude/apple/tools/BUCK +++ b/prelude/apple/tools/BUCK @@ -1,6 +1,6 @@ # @noautodeps load("@prelude//utils:source_listing.bzl", "source_listing") -load(":defs.bzl", "apple_oso_scrubber_target", "bundle_telemetry_logger_target") +load(":defs.bzl", "apple_oso_scrubber_target", "bundle_telemetry_logger_target", "swiftmodule_change_analysis_target") oncall("build_infra") @@ -20,12 +20,15 @@ apple_tools( ipa_package_maker = ":ipa_package_maker", make_modulemap = ":make_modulemap", make_vfsoverlay = ":make_vfsoverlay", + provisioning_manifest = "prelude//apple/tools/bundling:provisioning_manifest", + resolve_signing_context = "prelude//apple/tools/bundling:resolve_signing_context", selective_debugging_scrubber = apple_oso_scrubber_target(), - signing_context = "prelude//apple/tools/bundling:signing_context", signing_context_tree_postprocessor = "prelude//apple/tools/signing_context_tree_postprocessor:signing_context_tree_postprocessor", + signing_info = "prelude//apple/tools/bundling:signing_info", split_arch_combine_dsym_bundles_tool = ":split_arch_combine_dsym_bundles_tool", spm_packager = ":spm_packager", static_archive_linker = ":static_archive_linker", + swiftmodule_change_analysis = swiftmodule_change_analysis_target(), visibility = ["PUBLIC"], xcframework_maker = ":xcframework_maker", ) diff --git a/prelude/apple/tools/bundling/BUCK b/prelude/apple/tools/bundling/BUCK index d896560..aa8b958 100644 --- a/prelude/apple/tools/bundling/BUCK +++ b/prelude/apple/tools/bundling/BUCK @@ -17,14 +17,42 @@ python_binary( ) python_binary( - name = "signing_context", - main = "signing_context_main.py", + name = "provisioning_manifest", + main = "provisioning_manifest_main.py", visibility = ["PUBLIC"], deps = [ ":lib", ], ) +python_binary( + name = "resolve_signing_context", + main = "resolve_signing_context_main.py", + visibility = ["PUBLIC"], + deps = [ + ":lib", + ], +) + +python_binary( + name = "signing_info", + main = "signing_info_main.py", + visibility = ["PUBLIC"], + deps = [ + ":lib", + ], +) + +python_library( + name = "scripts-typing", + srcs = [ + "resolve_signing_context_main.py", + "signing_info_main.py", + ], + typing = True, + deps = [":lib"], +) + python_library( name = "lib", srcs = glob( diff --git a/prelude/apple/tools/bundling/incremental_utils.py b/prelude/apple/tools/bundling/incremental_utils.py index b5df5a4..3b16b20 100644 --- a/prelude/apple/tools/bundling/incremental_utils.py +++ b/prelude/apple/tools/bundling/incremental_utils.py @@ -121,6 +121,13 @@ def _codesigned_on_copy_paths_from_previous_build_which_are_present_in_current_b def _get_new_digest(action_metadata: Dict[Path, str], path: Path) -> str: + # A spec source is normally an action input recorded under exactly this + # path, and this runs once per file in the bundle, so try the lookup before + # resolving: `Path.resolve()` walks and stats every component of the path. + digest = action_metadata.get(path) + if digest is not None: + return digest + # While a resource file can be in a symlinked folder, like the `ghi/def` example below, # ``` # project_root diff --git a/prelude/apple/tools/bundling/incremental_utils_test.py b/prelude/apple/tools/bundling/incremental_utils_test.py index c9fbc3d..47b226c 100644 --- a/prelude/apple/tools/bundling/incremental_utils_test.py +++ b/prelude/apple/tools/bundling/incremental_utils_test.py @@ -11,6 +11,7 @@ import unittest from pathlib import Path from typing import Generator +from unittest.mock import patch from apple.tools.code_signing.codesign_bundle import CodesignConfiguration @@ -599,6 +600,29 @@ def test_calculate_incremental_state(self): ], ) + def test_calculate_incremental_state_prefers_action_metadata_to_resolving( + self, + ) -> None: + with tempfile.TemporaryDirectory() as project_root, chdir(project_root): + Path("foo").write_text("hello") + action_metadata = {Path("foo"): "hash(foo)"} + spec = [BundleSpecItem(src="foo", dst="foo")] + with patch.object( + Path, "resolve", side_effect=AssertionError("path was resolved") + ): + state = calculate_incremental_state(spec, action_metadata) + self.assertEqual( + state, + [ + IncrementalStateItem( + source=Path("foo"), + destination_relative_to_bundle=Path("foo"), + digest="hash(foo)", + resolved_symlink=None, + ), + ], + ) + def test_calculate_incremental_state_with_ds_store(self) -> None: with tempfile.TemporaryDirectory() as project_root, chdir(project_root): # project_root diff --git a/prelude/apple/tools/bundling/logging_utils.py b/prelude/apple/tools/bundling/logging_utils.py new file mode 100644 index 0000000..aa184b3 --- /dev/null +++ b/prelude/apple/tools/bundling/logging_utils.py @@ -0,0 +1,62 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is dual-licensed under either the MIT license found in the +# LICENSE-MIT file in the root directory of this source tree or the Apache +# License, Version 2.0 found in the LICENSE-APACHE file in the root directory +# of this source tree. You may select, at your option, one of the +# above-listed licenses. + +from __future__ import annotations + +import logging +import sys +from pathlib import Path + + +def configure_logging( + stderr_level: int, + file_level: int, + log_path: Path | None, +) -> None: + stderr_handler = logging.StreamHandler() + stderr_handler.setLevel(stderr_level) + log_format = ( + "%(asctime)s - %(name)s - %(levelname)s - %(message)s (%(filename)s:%(lineno)d)" + ) + stderr_handler.setFormatter( + _ColoredLogFormatter(log_format) + if sys.stderr.isatty() + else logging.Formatter(log_format) + ) + + handlers: list[logging.Handler] = [stderr_handler] + if log_path is not None: + # Declared log outputs must exist even when no messages are emitted, and + # incremental runs must not retain messages from an earlier invocation. + log_path.write_text("") + file_handler = logging.FileHandler(log_path, encoding="utf-8") + file_handler.setFormatter(logging.Formatter(log_format)) + file_handler.setLevel(file_level) + handlers.append(file_handler) + + logging.basicConfig(level=logging.DEBUG, handlers=handlers) + + +class _ColoredLogFormatter(logging.Formatter): + _colors: dict[int, str] = { + logging.DEBUG: "\x1b[m", + logging.INFO: "\x1b[37m", + logging.WARNING: "\x1b[33m", + logging.ERROR: "\x1b[31m", + logging.CRITICAL: "\x1b[1;31m", + } + _reset_color = "\x1b[0m" + + def __init__(self, text_format: str) -> None: + self.text_format = text_format + + def format(self, record: logging.LogRecord) -> str: + formatter = logging.Formatter( + self._colors.get(record.levelno, "") + self.text_format + self._reset_color + ) + return formatter.format(record) diff --git a/prelude/apple/tools/bundling/main.py b/prelude/apple/tools/bundling/main.py index 94d455a..7ec808f 100644 --- a/prelude/apple/tools/bundling/main.py +++ b/prelude/apple/tools/bundling/main.py @@ -6,7 +6,7 @@ # of this source tree. You may select, at your option, one of the # above-listed licenses. -# pyre-strict +from __future__ import annotations import argparse import cProfile @@ -15,22 +15,20 @@ import os import pstats import subprocess -import sys import tempfile from pathlib import Path -from typing import Dict, List, Optional +from typing import List, Optional from apple.tools.code_signing.codesign_bundle import ( - AdhocSigningContext, codesign_bundle, CodesignConfiguration, CodesignedPath, - selection_profile_context_from_signing_context, - SigningContextWithProfileSelection, write_empty_codesign_manifest, ) -from apple.tools.code_signing.provisioning_profile_metadata import ( - ProvisioningProfileMetadata, +from apple.tools.code_signing.signing_context_types import ( + AdhocSigningContext, + selection_profile_context_from_signing_context, + SigningContextWithProfileSelection, ) from apple.tools.re_compatibility_utils.writable import make_dir_recursively_writable @@ -44,9 +42,14 @@ parse_incremental_state, ) from .incremental_utils import codesigned_on_copy_item +from .logging_utils import configure_logging from .signing_context import ( - add_args_for_signing_context, - signing_context_and_selected_identity_from_args, + add_args_for_bundling_execution, + add_args_for_signing_context_path, +) +from .signing_context_data import ( + load_signing_context_data_from_file, + SigningContextData, ) from .swift_support import run_swift_stdlib_tool, SwiftSupportArguments @@ -221,14 +224,8 @@ def _args_parser() -> argparse.ArgumentParser: help="Path to bundle telemetry logger tool. If provided, will be invoked after bundle assembly completes.", ) - parser.add_argument( - "--signing-info-output", - type=Path, - required=False, - help="Path to the output JSON file for simplified signing identity metadata.", - ) - - add_args_for_signing_context(parser) + add_args_for_bundling_execution(parser) + add_args_for_signing_context_path(parser) return parser @@ -304,7 +301,21 @@ def _get_codesigned_paths_from_spec( return codesigned_paths -def _main(spec_temp_dir: tempfile.TemporaryDirectory) -> None: +def _load_signing_context( + args: argparse.Namespace, + temp_dir: Path, +) -> tuple[AdhocSigningContext | SigningContextWithProfileSelection | None, str | None]: + if not args.codesign: + return None, None + + signing_context_data = load_signing_context_data_from_file( + args.signing_context_path + ) + _materialize_selected_profile(signing_context_data, temp_dir) + return signing_context_data.signing_context, signing_context_data.selected_identity + + +def _main(spec_temp_dir: str) -> None: args_parser = _args_parser() args = args_parser.parse_args() @@ -323,14 +334,7 @@ def _main(spec_temp_dir: tempfile.TemporaryDirectory) -> None: ): args.codesign_args.remove("--digest-algorithm=sha1") - if args.log_file: - with open(args.log_file, "w") as _: - # We need to open the log file for two reasons: - # - Ensure it exists after action runs, as it's an output and thus required - # - It gets erased, so that we get new logs when doing incremental bundling - pass - - _setup_logging( + configure_logging( stderr_level=getattr(logging, args.log_level_stderr.upper()), file_level=getattr(logging, args.log_level_file.upper()), log_path=args.log_file, @@ -341,20 +345,11 @@ def _main(spec_temp_dir: tempfile.TemporaryDirectory) -> None: if profiling_enabled: pr.enable() - signing_context, selected_identity_argument = ( - signing_context_and_selected_identity_from_args(args) + signing_context, selected_identity_argument = _load_signing_context( + args, + Path(spec_temp_dir), ) - if args.signing_info_output: - selection_profile_context = selection_profile_context_from_signing_context( - signing_context - ) - signing_info = _build_signing_info_json( - args, selected_identity_argument, selection_profile_context - ) - with open(args.signing_info_output, "w") as signing_info_file: - json.dump(signing_info, signing_info_file, indent=4) - with args.spec.open(mode="rb") as spec_file: spec = json.load(spec_file, object_hook=lambda d: BundleSpecItem(**d)) spec = _deduplicate_spec(spec) @@ -498,41 +493,22 @@ def _main(spec_temp_dir: tempfile.TemporaryDirectory) -> None: telemetry_tmp_dir.cleanup() -def _build_signing_info_json( - args: argparse.Namespace, - selected_identity: Optional[str], - selection_profile_context: Optional[SigningContextWithProfileSelection], -) -> dict: - if not args.codesign: - return {} - - signing_info: dict = { - "codesign_type": "adhoc" if args.ad_hoc else "distribution", - } - - if selected_identity: - signing_info["codesign_identity"] = selected_identity - - if selection_profile_context: - selected_profile_info = selection_profile_context.selected_profile_info - profile_metadata: ProvisioningProfileMetadata = selected_profile_info.profile - if profile_metadata.provisioned_devices is not None: - provisioned_devices = "list" - elif profile_metadata.provisions_all_devices: - provisioned_devices = "all" - else: - provisioned_devices = "none" - signing_info["provisioning_profile"] = { - "uuid": profile_metadata.uuid, - "file_name": profile_metadata.file_path.name, - "provisioned_devices": provisioned_devices, - } - signing_info["signing_certificate"] = { - "fingerprint": selected_profile_info.identity.fingerprint, - "subject_common_name": selected_profile_info.identity.subject_common_name, - } +def _materialize_selected_profile( + signing_context_data: SigningContextData, + output_dir: Path, +) -> None: + profile_context = selection_profile_context_from_signing_context( + signing_context_data.signing_context + ) + if profile_context is None: + return + if signing_context_data.provisioning_profile_data is None: + raise ValueError("Signing context is missing profile data") - return signing_info + profile = profile_context.selected_profile_info.profile + materialized_path = output_dir / profile.file_path.name + materialized_path.write_bytes(signing_context_data.provisioning_profile_data) + profile.file_path = materialized_path def _get_selected_profile_path( @@ -767,52 +743,6 @@ def _deduplicate_spec(spec: List[BundleSpecItem]) -> List[BundleSpecItem]: return deduplicated_spec -def _setup_logging( - stderr_level: int, file_level: int, log_path: Optional[Path] -) -> None: - stderr_handler = logging.StreamHandler() - stderr_handler.setLevel(stderr_level) - log_format = ( - "%(asctime)s - %(name)s - %(levelname)s - %(message)s (%(filename)s:%(lineno)d)" - ) - stderr_handler.setFormatter( - ColoredLogFormatter(log_format) - if sys.stderr.isatty() - else logging.Formatter(log_format) - ) - - handlers: List[logging.Handler] = [stderr_handler] - - if log_path: - file_handler = logging.FileHandler(log_path, encoding="utf-8") - file_handler.setFormatter(logging.Formatter(log_format)) - file_handler.setLevel(file_level) - handlers.append(file_handler) - - logging.basicConfig(level=logging.DEBUG, handlers=handlers) - - -class ColoredLogFormatter(logging.Formatter): - _colors: Dict[int, str] = { - logging.DEBUG: "\x1b[m", - logging.INFO: "\x1b[37m", - logging.WARNING: "\x1b[33m", - logging.ERROR: "\x1b[31m", - logging.CRITICAL: "\x1b[1;31m", - } - _reset_color = "\x1b[0m" - - def __init__(self, text_format: str) -> None: - self.text_format = text_format - - def format(self, record: logging.LogRecord) -> str: - colored_format = ( - self._colors[record.levelno] + self.text_format + self._reset_color - ) - formatter = logging.Formatter(colored_format) - return formatter.format(record) - - if __name__ == "__main__": with tempfile.TemporaryDirectory() as spec_temp_dir: _main(spec_temp_dir) diff --git a/prelude/apple/tools/bundling/signing_context_main.py b/prelude/apple/tools/bundling/provisioning_manifest_main.py similarity index 59% rename from prelude/apple/tools/bundling/signing_context_main.py rename to prelude/apple/tools/bundling/provisioning_manifest_main.py index 5852529..0770fdf 100644 --- a/prelude/apple/tools/bundling/signing_context_main.py +++ b/prelude/apple/tools/bundling/provisioning_manifest_main.py @@ -6,26 +6,23 @@ # of this source tree. You may select, at your option, one of the # above-listed licenses. -# pyre-strict +from __future__ import annotations import argparse import base64 import json from pathlib import Path -from apple.tools.code_signing.codesign_bundle import ( +from apple.tools.code_signing.signing_context_types import ( selection_profile_context_from_signing_context, ) -from .signing_context import ( - add_args_for_signing_context, - signing_context_and_selected_identity_from_args, -) +from .signing_context_data import load_signing_context_data_from_file def _main() -> None: parser = argparse.ArgumentParser( - description="Tool which outputs the signing context for an apple_bundle().", + description="Generate a provisioning manifest from a signing context.", ) parser.add_argument( "--output", @@ -33,15 +30,19 @@ def _main() -> None: type=Path, help="Path to the output JSON file.", ) - add_args_for_signing_context(parser) + parser.add_argument( + "--signing-context-path", + type=Path, + required=True, + help="Path to the precomputed signing context JSON.", + ) args = parser.parse_args() - signing_context, selected_identity = ( - signing_context_and_selected_identity_from_args(args) + signing_context_data = load_signing_context_data_from_file( + args.signing_context_path ) - selection_profile_context = selection_profile_context_from_signing_context( - signing_context + signing_context_data.signing_context ) with open(args.output, "w") as output_file: @@ -49,21 +50,19 @@ def _main() -> None: "version": 1, } - if selected_identity: - # Adhoc and Developer ID builds will only have `codesign_identity` - # (in which case, it would the human readable identity). - # - # For provisioned builds (i.e., with a prov profile), it would be the - # signing cert fingerprint (i.e., SHA1 hash of cert in DER format) - signing_context_json_obj["codesign_identity"] = selected_identity + if signing_context_data.selected_identity: + # Ad hoc and Developer ID signing use the human-readable identity; + # profile-based signing uses the certificate fingerprint. + signing_context_json_obj["codesign_identity"] = ( + signing_context_data.selected_identity + ) if selection_profile_context: selected_profile_info = selection_profile_context.selected_profile_info profile_metadata = selected_profile_info.profile - with open(profile_metadata.file_path, "rb") as prov_profile_file: - prov_profile_as_base64_utf8 = base64.standard_b64encode( - prov_profile_file.read() - ).decode() + profile_data = signing_context_data.provisioning_profile_data + if profile_data is None: + raise ValueError("Signing context is missing profile data") signing_context_json_obj["provisioning_profile"] = { "uuid": profile_metadata.uuid, @@ -72,7 +71,7 @@ def _main() -> None: "subject_common_name": selected_profile_info.identity.subject_common_name, }, "file_name": profile_metadata.file_path.name, - "file_data_base64": prov_profile_as_base64_utf8, + "file_data_base64": base64.b64encode(profile_data).decode(), } json.dump( diff --git a/prelude/apple/tools/bundling/resolve_signing_context_main.py b/prelude/apple/tools/bundling/resolve_signing_context_main.py new file mode 100644 index 0000000..0d9a54a --- /dev/null +++ b/prelude/apple/tools/bundling/resolve_signing_context_main.py @@ -0,0 +1,75 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is dual-licensed under either the MIT license found in the +# LICENSE-MIT file in the root directory of this source tree or the Apache +# License, Version 2.0 found in the LICENSE-APACHE file in the root directory +# of this source tree. You may select, at your option, one of the +# above-listed licenses. + +import argparse +import logging +from pathlib import Path + +from .logging_utils import configure_logging +from .signing_context import ( + add_args_for_signing_context, + signing_context_and_selected_identity_from_args, +) +from .signing_context_data import write_signing_context_data_to_file + + +def _args_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Tool which resolves signing context and caches it for downstream bundling and signing_context tools.", + ) + parser.add_argument( + "--output", + required=True, + type=Path, + help="Path to the output JSON file containing the signing context.", + ) + parser.add_argument( + "--log-level-stderr", + choices=["debug", "info", "warning", "error", "critical"], + type=str, + required=False, + default="warning", + help="Logging level for messages written to stderr.", + ) + parser.add_argument( + "--log-level-file", + choices=["debug", "info", "warning", "error", "critical"], + type=str, + required=False, + default="info", + help="Logging level for messages written to a log file.", + ) + + add_args_for_signing_context(parser) + + return parser + + +def _main() -> None: + parser = _args_parser() + args = parser.parse_args() + + configure_logging( + stderr_level=getattr(logging, args.log_level_stderr.upper()), + file_level=getattr(logging, args.log_level_file.upper()), + log_path=args.log_file, + ) + + signing_context, selected_identity = ( + signing_context_and_selected_identity_from_args(args) + ) + + write_signing_context_data_to_file( + path=args.output, + signing_context=signing_context, + selected_identity=selected_identity, + ) + + +if __name__ == "__main__": + _main() diff --git a/prelude/apple/tools/bundling/signing_context.py b/prelude/apple/tools/bundling/signing_context.py index adb3032..10d7f8f 100644 --- a/prelude/apple/tools/bundling/signing_context.py +++ b/prelude/apple/tools/bundling/signing_context.py @@ -6,79 +6,46 @@ # of this source tree. You may select, at your option, one of the # above-listed licenses. +from __future__ import annotations + import argparse import json import shlex from pathlib import Path -from typing import Optional, Union from apple.tools.code_signing.apple_platform import ApplePlatform from apple.tools.code_signing.codesign_bundle import ( - AdhocSigningContext, signing_context_with_profile_selection, - SigningContextWithProfileSelection, ) from apple.tools.code_signing.list_codesign_identities import ( AdHocListCodesignIdentities, ListCodesignIdentities, ) +from apple.tools.code_signing.signing_context_types import ( + AdhocSigningContext, + SigningContextWithProfileSelection, +) -def add_args_for_signing_context(parser: argparse.ArgumentParser): - parser.add_argument( - "--codesign", - action="store_true", - help="Should the final bundle be codesigned.", - ) - parser.add_argument( - "--ad-hoc", - action="store_true", - help="Perform ad-hoc signing if set.", - ) - parser.add_argument( - "--ad-hoc-codesign-identity", - metavar="", - type=str, - required=False, - help="Codesign identity to use when ad-hoc signing is performed. Should be present when selection of provisioining profile is requested for ad-hoc signing.", - ) - parser.add_argument( - "--codesign-identities-command", - metavar='<"/signing/identities --available">', - type=str, - required=False, - help="Command listing available code signing identities. If it's not provided `security` utility is assumed to be available and is used.", - ) +def add_args_for_signing_context_path(parser: argparse.ArgumentParser) -> None: parser.add_argument( - "--profiles-dir", - metavar="", + "--signing-context-path", + metavar="", type=Path, - required=False, - action="append", - help="Required if non-ad-hoc code signing is requested. Path to directory with provisioning profile files. Can be specified multiple times.", - ) - parser.add_argument( - "--embed-provisioning-profile-when-signing-ad-hoc", - action="store_true", - help="Perform selection of provisioining profile and embed it into final bundle when ad-hoc signing if set.", - ) - parser.add_argument( - "--fast-provisioning-profile-parsing", - action="store_true", - help="Uses experimental faster provisioning profile parsing.", + required=True, + help="Path to the precomputed signing context JSON.", ) + + +def add_args_for_bundling_execution(parser: argparse.ArgumentParser) -> None: + _add_common_execution_args(parser) + + +def _add_common_execution_args(parser: argparse.ArgumentParser) -> None: parser.add_argument( - "--strict-provisioning-profile-search", + "--codesign", action="store_true", - required=False, - help="Fail code signing if more than one matching profile found.", - ) - parser.add_argument( - "--provisioning-profile-filter", - metavar="", - type=str, - required=False, - help="Regex to disambiguate multiple matching profiles, evaluated against provisioning profile filename.", + help="Should the final bundle be codesigned.", ) parser.add_argument( "--entitlements", @@ -135,6 +102,61 @@ def add_args_for_signing_context(parser: argparse.ArgumentParser): required=False, help="Path to a log file. If present logging will be directed to this file in addition to stderr.", ) + + +def add_args_for_signing_context_selection( + parser: argparse.ArgumentParser, +) -> None: + parser.add_argument( + "--ad-hoc", + action="store_true", + help="Perform ad-hoc signing if set.", + ) + parser.add_argument( + "--ad-hoc-codesign-identity", + metavar="", + type=str, + required=False, + help="Codesign identity to use when ad-hoc signing is performed. Should be present when selection of provisioining profile is requested for ad-hoc signing.", + ) + parser.add_argument( + "--codesign-identities-command", + metavar='<"/signing/identities --available">', + type=str, + required=False, + help="Command listing available code signing identities. If it's not provided `security` utility is assumed to be available and is used.", + ) + parser.add_argument( + "--profiles-dir", + metavar="", + type=Path, + required=False, + action="append", + help="Required if non-ad-hoc code signing is requested. Path to directory with provisioning profile files. Can be specified multiple times.", + ) + parser.add_argument( + "--embed-provisioning-profile-when-signing-ad-hoc", + action="store_true", + help="Perform selection of provisioining profile and embed it into final bundle when ad-hoc signing if set.", + ) + parser.add_argument( + "--fast-provisioning-profile-parsing", + action="store_true", + help="Uses experimental faster provisioning profile parsing.", + ) + parser.add_argument( + "--strict-provisioning-profile-search", + action="store_true", + required=False, + help="Fail code signing if more than one matching profile found.", + ) + parser.add_argument( + "--provisioning-profile-filter", + metavar="", + type=str, + required=False, + help="Regex to disambiguate multiple matching profiles, evaluated against provisioning profile filename.", + ) parser.add_argument( "--verify-entitlements", action="store_true", @@ -148,12 +170,14 @@ def add_args_for_signing_context(parser: argparse.ArgumentParser): ) +def add_args_for_signing_context(parser: argparse.ArgumentParser) -> None: + _add_common_execution_args(parser) + add_args_for_signing_context_selection(parser) + + def signing_context_and_selected_identity_from_args( args: argparse.Namespace, -) -> ( - Optional[Union[AdhocSigningContext, SigningContextWithProfileSelection]], - Optional[str], -): +) -> tuple[AdhocSigningContext | SigningContextWithProfileSelection | None, str | None]: if args.codesign: if not args.info_plist_source: raise RuntimeError( diff --git a/prelude/apple/tools/bundling/signing_context_data.py b/prelude/apple/tools/bundling/signing_context_data.py new file mode 100644 index 0000000..21720ad --- /dev/null +++ b/prelude/apple/tools/bundling/signing_context_data.py @@ -0,0 +1,157 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is dual-licensed under either the MIT license found in the +# LICENSE-MIT file in the root directory of this source tree or the Apache +# License, Version 2.0 found in the LICENSE-APACHE file in the root directory +# of this source tree. You may select, at your option, one of the +# above-listed licenses. + +from __future__ import annotations + +import base64 +import binascii +import json +from dataclasses import dataclass +from pathlib import Path + +from apple.tools.code_signing.serialization import ( + expect_dict, + expect_keys, + expect_optional_str, + expect_str, +) +from apple.tools.code_signing.signing_context_types import ( + AdhocSigningContext, + selection_profile_context_from_signing_context, + SigningContextWithProfileSelection, +) + +_VERSION = 1 + +SigningContext = AdhocSigningContext | SigningContextWithProfileSelection + + +@dataclass(frozen=True) +class SigningContextData: + signing_context: SigningContext | None + selected_identity: str | None + provisioning_profile_data: bytes | None + + +def serialize_signing_context_data( + signing_context: SigningContext | None, + selected_identity: str | None, +) -> dict[str, object]: + profile_context = selection_profile_context_from_signing_context(signing_context) + profile_data = ( + profile_context.selected_profile_info.profile.file_path.read_bytes() + if profile_context is not None + else None + ) + return { + "version": _VERSION, + "selected_identity": selected_identity, + "signing_context": _serialize_signing_context(signing_context), + "provisioning_profile_data_base64": ( + base64.b64encode(profile_data).decode() + if profile_data is not None + else None + ), + } + + +def _serialize_signing_context( + signing_context: SigningContext | None, +) -> dict[str, object] | None: + if signing_context is None: + return None + if isinstance(signing_context, SigningContextWithProfileSelection): + return {"kind": "distribution", "distribution": signing_context.to_dict()} + if isinstance(signing_context, AdhocSigningContext): + return {"kind": "adhoc", "adhoc": signing_context.to_dict()} + raise ValueError(f"Unexpected signing context type: {type(signing_context)}") + + +def deserialize_signing_context_data(value: object) -> SigningContextData: + data = expect_dict(value, "SigningContextData") + expect_keys( + data, + "SigningContextData", + frozenset( + { + "version", + "selected_identity", + "signing_context", + "provisioning_profile_data_base64", + } + ), + ) + version = data["version"] + if type(version) is not int or version != _VERSION: + raise ValueError(f"Expected signing context version {_VERSION}, got {version}") + + signing_context = _deserialize_signing_context(data["signing_context"]) + profile_data = _decode_profile_data(data["provisioning_profile_data_base64"]) + profile_context = selection_profile_context_from_signing_context(signing_context) + if (profile_context is None) != (profile_data is None): + raise ValueError( + "provisioning_profile_data_base64 must be present exactly when a " + "provisioning profile is selected" + ) + + return SigningContextData( + signing_context=signing_context, + selected_identity=expect_optional_str( + data["selected_identity"], "SigningContextData.selected_identity" + ), + provisioning_profile_data=profile_data, + ) + + +def _deserialize_signing_context(value: object) -> SigningContext | None: + if value is None: + return None + data = expect_dict(value, "SigningContextData.signing_context") + kind = expect_str(data.get("kind"), "SigningContextData.signing_context.kind") + if kind == "distribution": + expect_keys( + data, + "SigningContextData.signing_context", + frozenset({"kind", "distribution"}), + ) + return SigningContextWithProfileSelection.from_dict(data["distribution"]) + if kind == "adhoc": + expect_keys( + data, + "SigningContextData.signing_context", + frozenset({"kind", "adhoc"}), + ) + return AdhocSigningContext.from_dict(data["adhoc"]) + raise ValueError(f"Unknown signing context kind: {kind}") + + +def _decode_profile_data(value: object) -> bytes | None: + encoded = expect_optional_str( + value, "SigningContextData.provisioning_profile_data_base64" + ) + if encoded is None: + return None + try: + return base64.b64decode(encoded, validate=True) + except binascii.Error as error: + raise ValueError("Invalid provisioning profile base64 data") from error + + +def write_signing_context_data_to_file( + path: Path, + signing_context: SigningContext | None, + selected_identity: str | None, +) -> None: + data = serialize_signing_context_data(signing_context, selected_identity) + with path.open("w") as output: + json.dump(data, output, indent=4) + + +def load_signing_context_data_from_file(path: Path) -> SigningContextData: + with path.open() as source: + return deserialize_signing_context_data(json.load(source)) diff --git a/prelude/apple/tools/bundling/signing_context_data_test.py b/prelude/apple/tools/bundling/signing_context_data_test.py new file mode 100644 index 0000000..1b8da1d --- /dev/null +++ b/prelude/apple/tools/bundling/signing_context_data_test.py @@ -0,0 +1,219 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is dual-licensed under either the MIT license found in the +# LICENSE-MIT file in the root directory of this source tree or the Apache +# License, Version 2.0 found in the LICENSE-APACHE file in the root directory +# of this source tree. You may select, at your option, one of the +# above-listed licenses. + +import datetime +import json +import unittest +from pathlib import Path +from tempfile import TemporaryDirectory + +from apple.tools.code_signing.identity import CodeSigningIdentity +from apple.tools.code_signing.info_plist_metadata import InfoPlistMetadata +from apple.tools.code_signing.provisioning_profile_metadata import ( + ProvisioningProfileMetadata, +) +from apple.tools.code_signing.provisioning_profile_selection import ( + SelectedProvisioningProfileInfo, +) +from apple.tools.code_signing.signing_context_types import ( + AdhocSigningContext, + SigningContextWithProfileSelection, +) + +from .signing_context_data import ( + deserialize_signing_context_data, + serialize_signing_context_data, +) + + +class TestSigningContextData(unittest.TestCase): + def setUp(self) -> None: + self.temp_dir = TemporaryDirectory() + self.profile_path = Path(self.temp_dir.name) / "profile.mobileprovision" + self.profile_path.write_bytes(b"profile data") + + def tearDown(self) -> None: + self.temp_dir.cleanup() + + def test_null_context(self): + data = serialize_signing_context_data(None, None) + context_data = deserialize_signing_context_data(data) + self.assertIsNone(context_data.signing_context) + self.assertIsNone(context_data.selected_identity) + self.assertIsNone(context_data.provisioning_profile_data) + + def test_adhoc_without_profile(self): + ctx = AdhocSigningContext(codesign_identity="-", profile_selection_context=None) + data = serialize_signing_context_data(ctx, "-") + json_str = json.dumps(data) + loaded = json.loads(json_str) + context_data = deserialize_signing_context_data(loaded) + ctx2 = context_data.signing_context + self.assertEqual(context_data.selected_identity, "-") + self.assertIsNone(context_data.provisioning_profile_data) + assert isinstance(ctx2, AdhocSigningContext) + self.assertEqual(ctx2.codesign_identity, "-") + self.assertIsNone(ctx2.profile_selection_context) + + def test_distribution_context(self): + expiration = datetime.datetime(2025, 1, 1, 12, 0, 0) + creation = datetime.datetime(2024, 1, 1, 12, 0, 0) + profile = ProvisioningProfileMetadata( + file_path=self.profile_path, + uuid="UUID-1234", + expiration_date=expiration, + platforms=frozenset(["iOS"]), + developer_certificate_fingerprints=frozenset( + ["ABCDEF1234567890ABCDEF1234567890ABCDEF12"] + ), + entitlements={"application-identifier": "TEAM.com.test.app"}, + creation_date=creation, + provisions_all_devices=False, + provisioned_devices=["device1"], + ) + identity = CodeSigningIdentity( + fingerprint="ABCDEF1234567890ABCDEF1234567890ABCDEF12", + subject_common_name="Apple Development: Test", + ) + selected_info = SelectedProvisioningProfileInfo( + profile=profile, identity=identity + ) + info_plist_metadata = InfoPlistMetadata( + bundle_id="com.test.app", bundle_type="APPL", is_watchos_app=False + ) + ctx = SigningContextWithProfileSelection( + info_plist_source=Path("/tmp/Info.plist"), + info_plist_destination=Path("Info.plist"), + info_plist_metadata=info_plist_metadata, + selected_profile_info=selected_info, + ) + data = serialize_signing_context_data(ctx, identity.fingerprint) + json_str = json.dumps(data) + loaded = json.loads(json_str) + context_data = deserialize_signing_context_data(loaded) + ctx2 = context_data.signing_context + assert isinstance(ctx2, SigningContextWithProfileSelection) + self.assertEqual(context_data.selected_identity, identity.fingerprint) + self.assertEqual(context_data.provisioning_profile_data, b"profile data") + self.assertEqual(ctx2.info_plist_source, Path("/tmp/Info.plist")) + self.assertEqual(ctx2.info_plist_destination, Path("Info.plist")) + self.assertEqual(ctx2.info_plist_metadata.bundle_id, "com.test.app") + self.assertEqual(ctx2.selected_profile_info.profile.uuid, "UUID-1234") + self.assertEqual( + ctx2.selected_profile_info.profile.file_path, + self.profile_path, + ) + self.assertEqual( + ctx2.selected_profile_info.identity.fingerprint, identity.fingerprint + ) + self.assertEqual( + ctx2.selected_profile_info.profile.provisions_all_devices, False + ) + self.assertEqual( + ctx2.selected_profile_info.profile.provisioned_devices, ["device1"] + ) + + def test_adhoc_with_profile(self): + expiration = datetime.datetime(2025, 1, 1, 12, 0, 0) + profile = ProvisioningProfileMetadata( + file_path=self.profile_path, + uuid="UUID-5678", + expiration_date=expiration, + platforms=frozenset(["iOS"]), + developer_certificate_fingerprints=frozenset( + ["AAAABBBBCCCCDDDDEEEEFFFF0000111122223333"] + ), + entitlements={}, + creation_date=None, + provisions_all_devices=True, + provisioned_devices=None, + ) + identity = CodeSigningIdentity( + fingerprint="AAAABBBBCCCCDDDDEEEEFFFF0000111122223333", + subject_common_name="Apple Development: Adhoc", + ) + selected_info = SelectedProvisioningProfileInfo( + profile=profile, identity=identity + ) + info_plist_metadata = InfoPlistMetadata( + bundle_id="com.test.app", bundle_type="APPL", is_watchos_app=False + ) + distribution_ctx = SigningContextWithProfileSelection( + info_plist_source=Path("/tmp/Info.plist"), + info_plist_destination=Path("Info.plist"), + info_plist_metadata=info_plist_metadata, + selected_profile_info=selected_info, + ) + adhoc_ctx = AdhocSigningContext( + codesign_identity="Apple Development: Adhoc", + profile_selection_context=distribution_ctx, + ) + data = serialize_signing_context_data(adhoc_ctx, "Apple Development: Adhoc") + json_str = json.dumps(data) + loaded = json.loads(json_str) + context_data = deserialize_signing_context_data(loaded) + ctx2 = context_data.signing_context + assert isinstance(ctx2, AdhocSigningContext) + self.assertEqual(ctx2.codesign_identity, "Apple Development: Adhoc") + assert ctx2.profile_selection_context is not None + self.assertEqual( + ctx2.profile_selection_context.selected_profile_info.profile.uuid, + "UUID-5678", + ) + self.assertEqual(context_data.selected_identity, "Apple Development: Adhoc") + self.assertEqual(context_data.provisioning_profile_data, b"profile data") + + def test_version_mismatch_raises(self): + bad_data = { + "version": 999, + "selected_identity": None, + "signing_context": None, + "provisioning_profile_data_base64": None, + } + with self.assertRaises(ValueError) as cm: + deserialize_signing_context_data(bad_data) + + msg = str(cm.exception) + self.assertIn("Expected signing context version", msg) + self.assertIn("999", msg) + + bad_data_old = { + "version": 2, + "selected_identity": "-", + "signing_context": { + "kind": "adhoc", + "adhoc": {"codesign_identity": "-", "profile_selection_context": None}, + }, + "provisioning_profile_data_base64": None, + } + with self.assertRaises(ValueError): + deserialize_signing_context_data(bad_data_old) + + def test_profile_context_requires_profile_data(self): + profile = ProvisioningProfileMetadata( + file_path=self.profile_path, + uuid="UUID", + expiration_date=datetime.datetime(2025, 1, 1), + platforms=frozenset(["iOS"]), + developer_certificate_fingerprints=frozenset(["ABC"]), + entitlements={}, + ) + context = SigningContextWithProfileSelection( + info_plist_source=Path("/tmp/Info.plist"), + info_plist_destination=Path("Info.plist"), + info_plist_metadata=InfoPlistMetadata("com.test", "APPL", False), + selected_profile_info=SelectedProvisioningProfileInfo( + profile=profile, + identity=CodeSigningIdentity("ABC", "Developer"), + ), + ) + data = serialize_signing_context_data(context, "ABC") + data["provisioning_profile_data_base64"] = None + + with self.assertRaisesRegex(ValueError, "must be present exactly"): + deserialize_signing_context_data(data) diff --git a/prelude/apple/tools/bundling/signing_info.py b/prelude/apple/tools/bundling/signing_info.py new file mode 100644 index 0000000..a8df645 --- /dev/null +++ b/prelude/apple/tools/bundling/signing_info.py @@ -0,0 +1,77 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is dual-licensed under either the MIT license found in the +# LICENSE-MIT file in the root directory of this source tree or the Apache +# License, Version 2.0 found in the LICENSE-APACHE file in the root directory +# of this source tree. You may select, at your option, one of the +# above-listed licenses. + + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from apple.tools.code_signing.signing_context_types import ( + AdhocSigningContext, + selection_profile_context_from_signing_context, + SigningContextWithProfileSelection, +) + +from .signing_context_data import load_signing_context_data_from_file + + +def build_signing_info( + signing_context: AdhocSigningContext | SigningContextWithProfileSelection | None, + selected_identity: str | None, +) -> dict[str, Any]: + if signing_context is None: + return {} + + if isinstance(signing_context, AdhocSigningContext): + codesign_type = "adhoc" + elif isinstance(signing_context, SigningContextWithProfileSelection): + codesign_type = "distribution" + else: + raise RuntimeError(f"Unexpected signing context type: {type(signing_context)}") + + signing_info: dict[str, Any] = { + "codesign_type": codesign_type, + } + + if selected_identity: + signing_info["codesign_identity"] = selected_identity + + selection_profile_context = selection_profile_context_from_signing_context( + signing_context + ) + + if selection_profile_context: + selected_profile_info = selection_profile_context.selected_profile_info + profile_metadata = selected_profile_info.profile + if profile_metadata.provisioned_devices is not None: + provisioned_devices = "list" + elif profile_metadata.provisions_all_devices: + provisioned_devices = "all" + else: + provisioned_devices = "none" + + signing_info["provisioning_profile"] = { + "uuid": profile_metadata.uuid, + "file_name": profile_metadata.file_path.name, + "provisioned_devices": provisioned_devices, + } + signing_info["signing_certificate"] = { + "fingerprint": selected_profile_info.identity.fingerprint, + "subject_common_name": selected_profile_info.identity.subject_common_name, + } + + return signing_info + + +def build_signing_info_from_file(path: Path) -> dict[str, Any]: + signing_context_data = load_signing_context_data_from_file(path) + return build_signing_info( + signing_context_data.signing_context, + signing_context_data.selected_identity, + ) diff --git a/prelude/apple/tools/bundling/signing_info_main.py b/prelude/apple/tools/bundling/signing_info_main.py new file mode 100644 index 0000000..695cfde --- /dev/null +++ b/prelude/apple/tools/bundling/signing_info_main.py @@ -0,0 +1,44 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is dual-licensed under either the MIT license found in the +# LICENSE-MIT file in the root directory of this source tree or the Apache +# License, Version 2.0 found in the LICENSE-APACHE file in the root directory +# of this source tree. You may select, at your option, one of the +# above-listed licenses. + +import argparse +import json +from pathlib import Path + +from .signing_info import build_signing_info_from_file + + +def _main() -> None: + parser = argparse.ArgumentParser( + description="Tool which outputs signing-info JSON from a precomputed signing context. " + "This is cheap (no profile directory scan, no security tool) and allows the " + "[signing-info] subtarget to be built without running bundling.", + ) + parser.add_argument( + "--signing-context-path", + type=Path, + required=True, + help="Path to precomputed signing context JSON.", + ) + parser.add_argument( + "--output", + type=Path, + required=True, + help="Path to the output signing-info JSON file.", + ) + + args = parser.parse_args() + + signing_info = build_signing_info_from_file(args.signing_context_path) + + with open(args.output, "w") as f: + json.dump(signing_info, f, indent=4) + + +if __name__ == "__main__": + _main() diff --git a/prelude/apple/tools/bundling/signing_info_test.py b/prelude/apple/tools/bundling/signing_info_test.py new file mode 100644 index 0000000..bf0c145 --- /dev/null +++ b/prelude/apple/tools/bundling/signing_info_test.py @@ -0,0 +1,130 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is dual-licensed under either the MIT license found in the +# LICENSE-MIT file in the root directory of this source tree or the Apache +# License, Version 2.0 found in the LICENSE-APACHE file in the root directory +# of this source tree. You may select, at your option, one of the +# above-listed licenses. + +import datetime +import unittest +from pathlib import Path + +from apple.tools.code_signing.identity import CodeSigningIdentity +from apple.tools.code_signing.info_plist_metadata import InfoPlistMetadata +from apple.tools.code_signing.provisioning_profile_metadata import ( + ProvisioningProfileMetadata, +) +from apple.tools.code_signing.provisioning_profile_selection import ( + SelectedProvisioningProfileInfo, +) +from apple.tools.code_signing.signing_context_types import ( + AdhocSigningContext, + SigningContextWithProfileSelection, +) + +from .signing_info import build_signing_info + + +class TestSigningInfo(unittest.TestCase): + def test_no_signing_context(self): + info = build_signing_info(None, None) + self.assertEqual(info, {}) + + def test_adhoc_without_profile(self): + ctx = AdhocSigningContext(codesign_identity="-", profile_selection_context=None) + info = build_signing_info(ctx, "-") + self.assertEqual(info["codesign_type"], "adhoc") + self.assertEqual(info["codesign_identity"], "-") + self.assertNotIn("provisioning_profile", info) + + def test_distribution_with_devices_list(self): + expiration = datetime.datetime(2025, 1, 1, 12, 0, 0) + profile = ProvisioningProfileMetadata( + file_path=Path("/tmp/my.mobileprovision"), + uuid="UUID-123", + expiration_date=expiration, + platforms=frozenset(["iOS"]), + developer_certificate_fingerprints=frozenset(["ABC"]), + entitlements={}, + creation_date=None, + provisions_all_devices=False, + provisioned_devices=["dev1", "dev2"], + ) + identity = CodeSigningIdentity( + fingerprint="ABC", subject_common_name="Apple Dev" + ) + selected = SelectedProvisioningProfileInfo(profile=profile, identity=identity) + metadata = InfoPlistMetadata( + bundle_id="com.test", bundle_type="APPL", is_watchos_app=False + ) + ctx = SigningContextWithProfileSelection( + info_plist_source=Path("/tmp/Info.plist"), + info_plist_destination=Path("Info.plist"), + info_plist_metadata=metadata, + selected_profile_info=selected, + ) + info = build_signing_info(ctx, "ABC") + self.assertEqual(info["codesign_type"], "distribution") + self.assertEqual(info["codesign_identity"], "ABC") + self.assertEqual(info["provisioning_profile"]["uuid"], "UUID-123") + self.assertEqual( + info["provisioning_profile"]["file_name"], "my.mobileprovision" + ) + self.assertEqual(info["provisioning_profile"]["provisioned_devices"], "list") + self.assertEqual(info["signing_certificate"]["fingerprint"], "ABC") + + def test_provisioned_devices_all_and_none(self): + expiration = datetime.datetime(2025, 1, 1, 12, 0, 0) + + profile_all = ProvisioningProfileMetadata( + file_path=Path("/tmp/all.mobileprovision"), + uuid="UUID-ALL", + expiration_date=expiration, + platforms=frozenset(["iOS"]), + developer_certificate_fingerprints=frozenset(["ABC"]), + entitlements={}, + creation_date=None, + provisions_all_devices=True, + provisioned_devices=None, + ) + identity = CodeSigningIdentity(fingerprint="ABC", subject_common_name="Dev") + selected_all = SelectedProvisioningProfileInfo( + profile=profile_all, identity=identity + ) + metadata = InfoPlistMetadata( + bundle_id="com.test", bundle_type="APPL", is_watchos_app=False + ) + ctx_all = SigningContextWithProfileSelection( + info_plist_source=Path("/tmp/Info.plist"), + info_plist_destination=Path("Info.plist"), + info_plist_metadata=metadata, + selected_profile_info=selected_all, + ) + info_all = build_signing_info(ctx_all, "ABC") + self.assertEqual(info_all["provisioning_profile"]["provisioned_devices"], "all") + + profile_none = ProvisioningProfileMetadata( + file_path=Path("/tmp/none.mobileprovision"), + uuid="UUID-NONE", + expiration_date=expiration, + platforms=frozenset(["iOS"]), + developer_certificate_fingerprints=frozenset(["ABC"]), + entitlements={}, + creation_date=None, + provisions_all_devices=False, + provisioned_devices=None, + ) + selected_none = SelectedProvisioningProfileInfo( + profile=profile_none, identity=identity + ) + ctx_none = SigningContextWithProfileSelection( + info_plist_source=Path("/tmp/Info.plist"), + info_plist_destination=Path("Info.plist"), + info_plist_metadata=metadata, + selected_profile_info=selected_none, + ) + info_none = build_signing_info(ctx_none, "ABC") + self.assertEqual( + info_none["provisioning_profile"]["provisioned_devices"], "none" + ) diff --git a/prelude/apple/tools/code_signing/BUCK b/prelude/apple/tools/code_signing/BUCK index 621d130..2018505 100644 --- a/prelude/apple/tools/code_signing/BUCK +++ b/prelude/apple/tools/code_signing/BUCK @@ -33,6 +33,13 @@ python_library( visibility = ["PUBLIC"], ) +python_library( + name = "serialization", + srcs = ["serialization.py"], + typing = True, + visibility = ["PUBLIC"], +) + python_library( name = "lib", srcs = glob( @@ -43,6 +50,7 @@ python_library( "*_test.py", "main.py", "apple_platform.py", + "serialization.py", ], ), resources = [":dummy_binary_for_signing_configured"], @@ -54,6 +62,7 @@ python_library( # @oss-disable[end= ]: "prelude//apple/tools/meta_only/codesign_rust:check_adhoc_signature_lib", # @oss-disable[end= ]: "prelude//apple/tools/meta_only/entitlements_mismatch:check_entitlements_lib", ":apple_platform_lib", + ":serialization", ], ) diff --git a/prelude/apple/tools/code_signing/codesign_bundle.py b/prelude/apple/tools/code_signing/codesign_bundle.py index e84b53e..c31e11b 100644 --- a/prelude/apple/tools/code_signing/codesign_bundle.py +++ b/prelude/apple/tools/code_signing/codesign_bundle.py @@ -65,6 +65,11 @@ DefaultReadProvisioningProfileCommandFactory, IReadProvisioningProfileCommandFactory, ) +from .signing_context_types import ( + AdhocSigningContext, + selection_profile_context_from_signing_context, + SigningContextWithProfileSelection, +) _default_read_provisioning_profile_command_factory = ( DefaultReadProvisioningProfileCommandFactory() @@ -253,36 +258,6 @@ def _select_best_provisioning_profile( ) -@dataclass -class SigningContextWithProfileSelection: - info_plist_source: Path - info_plist_destination: Path - info_plist_metadata: InfoPlistMetadata - selected_profile_info: SelectedProvisioningProfileInfo - - -@dataclass -class AdhocSigningContext: - codesign_identity: str - profile_selection_context: Optional[SigningContextWithProfileSelection] - - def __init__( - self, - codesign_identity: Optional[str] = None, - profile_selection_context: Optional[SigningContextWithProfileSelection] = None, - ) -> None: - self.codesign_identity = codesign_identity or "-" - self.profile_selection_context = profile_selection_context - - def identity(self) -> CodeSigningIdentity: - if self.profile_selection_context: - return self.profile_selection_context.selected_profile_info.identity - return CodeSigningIdentity( - fingerprint=self.codesign_identity, - subject_common_name="", - ) - - def signing_context_with_profile_selection( info_plist_source: Path, info_plist_destination: Path, @@ -339,23 +314,6 @@ def write_empty_codesign_manifest(codesign_manifest_path: Path, bundle_path: Pat json.dump(codesign_manifest, codesign_manifest_file, indent=4) -def selection_profile_context_from_signing_context( - signing_context: Optional[ - Union[AdhocSigningContext, SigningContextWithProfileSelection] - ], -) -> Optional[SigningContextWithProfileSelection]: - if signing_context: - if isinstance(signing_context, SigningContextWithProfileSelection): - selection_profile_context = signing_context - elif isinstance(signing_context, AdhocSigningContext): - selection_profile_context = signing_context.profile_selection_context - else: - raise RuntimeError( - f"Unexpected type of signing context `{type(signing_context)}`" - ) - return selection_profile_context - - def _postprocess_entitlements_if_needed_for_adhoc_signed_bundle( bundle_path: CodesignedPath, tmp_dir: str, @@ -944,7 +902,7 @@ def obtain_keychain_permissions( path=CodesignedPath( path=Path(dummy_binary_copied), entitlements=None, - flags=[], + flags=["--timestamp=none"], extra_file_paths=None, ), identity_fingerprint=identity_fingerprint, diff --git a/prelude/apple/tools/code_signing/identity.py b/prelude/apple/tools/code_signing/identity.py index 9e4f45e..52bf152 100644 --- a/prelude/apple/tools/code_signing/identity.py +++ b/prelude/apple/tools/code_signing/identity.py @@ -15,6 +15,8 @@ from enum import Enum from typing import List +from .serialization import expect_dict, expect_keys, expect_str + @dataclass class CodeSigningIdentity: @@ -41,3 +43,27 @@ def parse_security_stdout(cls, text: str) -> List[CodeSigningIdentity]: ) for match in re.finditer(cls._pattern, text) ] + + def to_dict(self) -> dict[str, str]: + return { + "fingerprint": self.fingerprint, + "subject_common_name": self.subject_common_name, + } + + @staticmethod + def from_dict(value: object) -> CodeSigningIdentity: + data = expect_dict(value, "CodeSigningIdentity") + expect_keys( + data, + "CodeSigningIdentity", + frozenset({"fingerprint", "subject_common_name"}), + ) + return CodeSigningIdentity( + fingerprint=expect_str( + data["fingerprint"], "CodeSigningIdentity.fingerprint" + ), + subject_common_name=expect_str( + data["subject_common_name"], + "CodeSigningIdentity.subject_common_name", + ), + ) diff --git a/prelude/apple/tools/code_signing/info_plist_metadata.py b/prelude/apple/tools/code_signing/info_plist_metadata.py index e573278..c3c035f 100644 --- a/prelude/apple/tools/code_signing/info_plist_metadata.py +++ b/prelude/apple/tools/code_signing/info_plist_metadata.py @@ -15,6 +15,14 @@ from apple.tools.plistlib_utils import detect_format_and_load +from .serialization import ( + expect_bool, + expect_dict, + expect_keys, + expect_optional_str, + expect_str, +) + @dataclass class InfoPlistMetadata: @@ -30,3 +38,28 @@ def from_file(info_plist_file: IO[bytes]) -> InfoPlistMetadata: root.get("CFBundlePackageType"), root.get("WKApplication", False), ) + + def to_dict(self) -> dict[str, object]: + return { + "bundle_id": self.bundle_id, + "bundle_type": self.bundle_type, + "is_watchos_app": self.is_watchos_app, + } + + @staticmethod + def from_dict(value: object) -> InfoPlistMetadata: + data = expect_dict(value, "InfoPlistMetadata") + expect_keys( + data, + "InfoPlistMetadata", + frozenset({"bundle_id", "bundle_type", "is_watchos_app"}), + ) + return InfoPlistMetadata( + bundle_id=expect_str(data["bundle_id"], "InfoPlistMetadata.bundle_id"), + bundle_type=expect_optional_str( + data["bundle_type"], "InfoPlistMetadata.bundle_type" + ), + is_watchos_app=expect_bool( + data["is_watchos_app"], "InfoPlistMetadata.is_watchos_app" + ), + ) diff --git a/prelude/apple/tools/code_signing/provisioning_profile_metadata.py b/prelude/apple/tools/code_signing/provisioning_profile_metadata.py index 60e4dec..501179a 100644 --- a/prelude/apple/tools/code_signing/provisioning_profile_metadata.py +++ b/prelude/apple/tools/code_signing/provisioning_profile_metadata.py @@ -10,15 +10,99 @@ from __future__ import annotations +import base64 +import binascii import hashlib from dataclasses import dataclass from datetime import datetime from pathlib import Path -from typing import Any, Optional +from typing import Any, cast, Optional from apple.tools.plistlib_utils import detect_format_and_loads from .app_id import AppId +from .serialization import ( + expect_dict, + expect_keys, + expect_optional_bool, + expect_optional_str_list, + expect_str, + expect_str_list, +) + +_ENTITLEMENTS_BYTES_MARKER = "__apple_code_signing_bytes_b64__" +_ENTITLEMENTS_DATETIME_MARKER = "__apple_code_signing_datetime_iso__" +_ENTITLEMENTS_ESCAPED_DICT_MARKER = "__apple_code_signing_escaped_dict__" +_ENTITLEMENTS_RESERVED_MARKERS = ( + _ENTITLEMENTS_BYTES_MARKER, + _ENTITLEMENTS_DATETIME_MARKER, + _ENTITLEMENTS_ESCAPED_DICT_MARKER, +) + + +def _serialize_datetime(dt: Optional[datetime]) -> Optional[str]: + if dt is None: + return None + return dt.isoformat() + + +def _deserialize_datetime(value: object, path: str) -> Optional[datetime]: + if value is None: + return None + return datetime.fromisoformat(expect_str(value, path)) + + +def _make_entitlements_json_safe(obj: Any) -> Any: + """Encode plist-only scalar types as tagged JSON objects.""" + if isinstance(obj, bytes): + return {_ENTITLEMENTS_BYTES_MARKER: base64.standard_b64encode(obj).decode()} + if isinstance(obj, datetime): + return {_ENTITLEMENTS_DATETIME_MARKER: obj.isoformat()} + if isinstance(obj, dict): + encoded = {k: _make_entitlements_json_safe(v) for k, v in obj.items()} + # A marker-shaped dictionary is valid plist data and must not be + # mistaken for one of the tagged scalar representations when decoded. + if len(obj) == 1 and next(iter(obj)) in _ENTITLEMENTS_RESERVED_MARKERS: + return {_ENTITLEMENTS_ESCAPED_DICT_MARKER: encoded} + return encoded + if isinstance(obj, (list, tuple)): + return [_make_entitlements_json_safe(x) for x in obj] + return obj + + +def _restore_entitlements_from_json_safe(obj: object) -> Any: + if isinstance(obj, dict): + data = expect_dict(obj, "entitlements") + if set(data) == {_ENTITLEMENTS_BYTES_MARKER}: + raw = obj[_ENTITLEMENTS_BYTES_MARKER] + encoded = expect_str(raw, f"entitlements.{_ENTITLEMENTS_BYTES_MARKER}") + try: + return base64.b64decode(encoded, validate=True) + except binascii.Error as error: + raise ValueError("Invalid base64 entitlement value") from error + if set(data) == {_ENTITLEMENTS_DATETIME_MARKER}: + raw = obj[_ENTITLEMENTS_DATETIME_MARKER] + return datetime.fromisoformat( + expect_str(raw, f"entitlements.{_ENTITLEMENTS_DATETIME_MARKER}") + ) + if set(data) == {_ENTITLEMENTS_ESCAPED_DICT_MARKER}: + escaped = expect_dict( + data[_ENTITLEMENTS_ESCAPED_DICT_MARKER], + f"entitlements.{_ENTITLEMENTS_ESCAPED_DICT_MARKER}", + ) + return { + key: _restore_entitlements_from_json_safe(value) + for key, value in escaped.items() + } + return { + key: _restore_entitlements_from_json_safe(value) + for key, value in data.items() + } + if isinstance(obj, list): + return [_restore_entitlements_from_json_safe(value) for value in obj] + if obj is None or isinstance(obj, (bool, int, float, str)): + return obj + raise ValueError(f"Unsupported entitlement JSON value: {type(obj).__name__}") @dataclass @@ -102,3 +186,82 @@ def __hash__(self) -> int: self.developer_certificate_fingerprints, ) ) + + def to_dict(self) -> dict[str, object]: + return { + "file_path": str(self.file_path), + "uuid": self.uuid, + "expiration_date": _serialize_datetime(self.expiration_date), + "creation_date": _serialize_datetime(self.creation_date), + "platforms": sorted(self.platforms), + "developer_certificate_fingerprints": sorted( + self.developer_certificate_fingerprints + ), + "entitlements": _make_entitlements_json_safe(self.entitlements), + "provisions_all_devices": self.provisions_all_devices, + "provisioned_devices": self.provisioned_devices, + } + + @staticmethod + def from_dict(value: object) -> ProvisioningProfileMetadata: + data = expect_dict(value, "ProvisioningProfileMetadata") + expect_keys( + data, + "ProvisioningProfileMetadata", + frozenset( + { + "file_path", + "uuid", + "expiration_date", + "creation_date", + "platforms", + "developer_certificate_fingerprints", + "entitlements", + "provisions_all_devices", + "provisioned_devices", + } + ), + ) + expiration = _deserialize_datetime( + data["expiration_date"], "ProvisioningProfileMetadata.expiration_date" + ) + if expiration is None: + raise ValueError( + "ProvisioningProfileMetadata.expiration_date must be a string" + ) + creation = _deserialize_datetime( + data["creation_date"], "ProvisioningProfileMetadata.creation_date" + ) + entitlements = _restore_entitlements_from_json_safe(data["entitlements"]) + if not isinstance(entitlements, dict): + raise ValueError( + "ProvisioningProfileMetadata.entitlements must be an object" + ) + return ProvisioningProfileMetadata( + file_path=Path( + expect_str(data["file_path"], "ProvisioningProfileMetadata.file_path") + ), + uuid=expect_str(data["uuid"], "ProvisioningProfileMetadata.uuid"), + expiration_date=expiration, + platforms=frozenset( + expect_str_list( + data["platforms"], "ProvisioningProfileMetadata.platforms" + ) + ), + developer_certificate_fingerprints=frozenset( + expect_str_list( + data["developer_certificate_fingerprints"], + "ProvisioningProfileMetadata.developer_certificate_fingerprints", + ) + ), + entitlements=cast(dict[str, Any], entitlements), + creation_date=creation, + provisions_all_devices=expect_optional_bool( + data["provisions_all_devices"], + "ProvisioningProfileMetadata.provisions_all_devices", + ), + provisioned_devices=expect_optional_str_list( + data["provisioned_devices"], + "ProvisioningProfileMetadata.provisioned_devices", + ), + ) diff --git a/prelude/apple/tools/code_signing/provisioning_profile_selection.py b/prelude/apple/tools/code_signing/provisioning_profile_selection.py index 183eefa..eaed9ab 100644 --- a/prelude/apple/tools/code_signing/provisioning_profile_selection.py +++ b/prelude/apple/tools/code_signing/provisioning_profile_selection.py @@ -8,6 +8,8 @@ # pyre-strict +from __future__ import annotations + import datetime import logging import re @@ -29,6 +31,7 @@ UnsupportedPlatform, ) from .provisioning_profile_metadata import ProvisioningProfileMetadata +from .serialization import expect_dict, expect_keys _LOGGER: logging.Logger = logging.getLogger(__name__) @@ -173,6 +176,25 @@ class SelectedProvisioningProfileInfo: profile: ProvisioningProfileMetadata identity: CodeSigningIdentity + def to_dict(self) -> dict[str, Any]: + return { + "identity": self.identity.to_dict(), + "profile": self.profile.to_dict(), + } + + @staticmethod + def from_dict(value: object) -> SelectedProvisioningProfileInfo: + data = expect_dict(value, "SelectedProvisioningProfileInfo") + expect_keys( + data, + "SelectedProvisioningProfileInfo", + frozenset({"identity", "profile"}), + ) + return SelectedProvisioningProfileInfo( + profile=ProvisioningProfileMetadata.from_dict(data["profile"]), + identity=CodeSigningIdentity.from_dict(data["identity"]), + ) + def _filter_matching_selected_provisioning_profile_infos( selected_profile_infos: list[SelectedProvisioningProfileInfo], diff --git a/prelude/apple/tools/code_signing/serialization.py b/prelude/apple/tools/code_signing/serialization.py new file mode 100644 index 0000000..6546df2 --- /dev/null +++ b/prelude/apple/tools/code_signing/serialization.py @@ -0,0 +1,67 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is dual-licensed under either the MIT license found in the +# LICENSE-MIT file in the root directory of this source tree or the Apache +# License, Version 2.0 found in the LICENSE-APACHE file in the root directory +# of this source tree. You may select, at your option, one of the +# above-listed licenses. + +from typing import cast + + +def expect_dict(value: object, path: str) -> dict[str, object]: + if not isinstance(value, dict) or not all( + isinstance(key, str) for key in value.keys() + ): + raise ValueError(f"{path} must be an object") + return cast(dict[str, object], value) + + +def expect_keys( + data: dict[str, object], + path: str, + expected: frozenset[str], +) -> None: + actual = frozenset(data) + if actual != expected: + missing = sorted(expected - actual) + unexpected = sorted(actual - expected) + raise ValueError( + f"{path} has invalid keys: missing={missing}, unexpected={unexpected}" + ) + + +def expect_str(value: object, path: str) -> str: + if not isinstance(value, str): + raise ValueError(f"{path} must be a string") + return value + + +def expect_optional_str(value: object, path: str) -> str | None: + if value is None: + return None + return expect_str(value, path) + + +def expect_bool(value: object, path: str) -> bool: + if not isinstance(value, bool): + raise ValueError(f"{path} must be a boolean") + return value + + +def expect_optional_bool(value: object, path: str) -> bool | None: + if value is None: + return None + return expect_bool(value, path) + + +def expect_str_list(value: object, path: str) -> list[str]: + if not isinstance(value, list) or not all(isinstance(item, str) for item in value): + raise ValueError(f"{path} must be a list of strings") + return cast(list[str], value) + + +def expect_optional_str_list(value: object, path: str) -> list[str] | None: + if value is None: + return None + return expect_str_list(value, path) diff --git a/prelude/apple/tools/code_signing/signing_context_types.py b/prelude/apple/tools/code_signing/signing_context_types.py new file mode 100644 index 0000000..91e99ff --- /dev/null +++ b/prelude/apple/tools/code_signing/signing_context_types.py @@ -0,0 +1,143 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is dual-licensed under either the MIT license found in the +# LICENSE-MIT file in the root directory of this source tree or the Apache +# License, Version 2.0 found in the LICENSE-APACHE file in the root directory +# of this source tree. You may select, at your option, one of the +# above-listed licenses. + +# pyre-strict + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import Optional, Union + +from .identity import CodeSigningIdentity +from .info_plist_metadata import InfoPlistMetadata +from .provisioning_profile_selection import SelectedProvisioningProfileInfo +from .serialization import expect_dict, expect_keys, expect_str + + +@dataclass +class SigningContextWithProfileSelection: + info_plist_source: Path + info_plist_destination: Path + info_plist_metadata: InfoPlistMetadata + selected_profile_info: SelectedProvisioningProfileInfo + + def to_dict(self) -> dict[str, object]: + return { + "info_plist_source": str(self.info_plist_source), + "info_plist_destination": str(self.info_plist_destination), + "info_plist_metadata": self.info_plist_metadata.to_dict(), + "selected_profile_info": self.selected_profile_info.to_dict(), + } + + @staticmethod + def from_dict(value: object) -> SigningContextWithProfileSelection: + data = expect_dict(value, "SigningContextWithProfileSelection") + expect_keys( + data, + "SigningContextWithProfileSelection", + frozenset( + { + "info_plist_source", + "info_plist_destination", + "info_plist_metadata", + "selected_profile_info", + } + ), + ) + return SigningContextWithProfileSelection( + info_plist_source=Path( + expect_str( + data["info_plist_source"], + "SigningContextWithProfileSelection.info_plist_source", + ) + ), + info_plist_destination=Path( + expect_str( + data["info_plist_destination"], + "SigningContextWithProfileSelection.info_plist_destination", + ) + ), + info_plist_metadata=InfoPlistMetadata.from_dict( + data["info_plist_metadata"] + ), + selected_profile_info=SelectedProvisioningProfileInfo.from_dict( + data["selected_profile_info"] + ), + ) + + +@dataclass +class AdhocSigningContext: + codesign_identity: str + profile_selection_context: Optional[SigningContextWithProfileSelection] + + def __init__( + self, + codesign_identity: Optional[str] = None, + profile_selection_context: Optional[SigningContextWithProfileSelection] = None, + ) -> None: + self.codesign_identity = codesign_identity or "-" + self.profile_selection_context = profile_selection_context + + def identity(self) -> CodeSigningIdentity: + if self.profile_selection_context: + return self.profile_selection_context.selected_profile_info.identity + return CodeSigningIdentity( + fingerprint=self.codesign_identity, + subject_common_name="", + ) + + def to_dict(self) -> dict[str, object]: + return { + "codesign_identity": self.codesign_identity, + "profile_selection_context": ( + self.profile_selection_context.to_dict() + if self.profile_selection_context + else None + ), + } + + @staticmethod + def from_dict(value: object) -> AdhocSigningContext: + data = expect_dict(value, "AdhocSigningContext") + expect_keys( + data, + "AdhocSigningContext", + frozenset({"codesign_identity", "profile_selection_context"}), + ) + psc_data = data["profile_selection_context"] + psc = ( + SigningContextWithProfileSelection.from_dict(psc_data) + if psc_data is not None + else None + ) + return AdhocSigningContext( + codesign_identity=expect_str( + data["codesign_identity"], "AdhocSigningContext.codesign_identity" + ), + profile_selection_context=psc, + ) + + +def selection_profile_context_from_signing_context( + signing_context: Optional[ + Union[AdhocSigningContext, SigningContextWithProfileSelection] + ], +) -> Optional[SigningContextWithProfileSelection]: + if signing_context: + if isinstance(signing_context, SigningContextWithProfileSelection): + selection_profile_context = signing_context + elif isinstance(signing_context, AdhocSigningContext): + selection_profile_context = signing_context.profile_selection_context + else: + raise RuntimeError( + f"Unexpected type of signing context `{type(signing_context)}`" + ) + return selection_profile_context + return None diff --git a/prelude/apple/tools/code_signing/signing_context_types_test.py b/prelude/apple/tools/code_signing/signing_context_types_test.py new file mode 100644 index 0000000..c7e5b54 --- /dev/null +++ b/prelude/apple/tools/code_signing/signing_context_types_test.py @@ -0,0 +1,316 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is dual-licensed under either the MIT license found in the +# LICENSE-MIT file in the root directory of this source tree or the Apache +# License, Version 2.0 found in the LICENSE-APACHE file in the root directory +# of this source tree. You may select, at your option, one of the +# above-listed licenses. + +import datetime +import json +import unittest +from pathlib import Path + +from .identity import CodeSigningIdentity +from .info_plist_metadata import InfoPlistMetadata +from .provisioning_profile_metadata import ProvisioningProfileMetadata +from .provisioning_profile_selection import SelectedProvisioningProfileInfo +from .signing_context_types import ( + AdhocSigningContext, + SigningContextWithProfileSelection, +) + + +class TestIdentitySerialization(unittest.TestCase): + def test_round_trip(self): + identity = CodeSigningIdentity( + fingerprint="ABCDEF1234567890ABCDEF1234567890ABCDEF12", + subject_common_name="Apple Development: Test", + ) + data = identity.to_dict() + restored = CodeSigningIdentity.from_dict(data) + self.assertEqual(identity, restored) + json_str = json.dumps(data) + loaded = json.loads(json_str) + restored2 = CodeSigningIdentity.from_dict(loaded) + self.assertEqual(identity, restored2) + + def test_rejects_unknown_fields(self): + with self.assertRaisesRegex(ValueError, "unexpected=.*team_id"): + CodeSigningIdentity.from_dict( + { + "fingerprint": "ABC", + "subject_common_name": "Developer", + "team_id": "TEAM", + } + ) + + def test_rejects_invalid_field_type(self): + with self.assertRaisesRegex(ValueError, "fingerprint must be a string"): + CodeSigningIdentity.from_dict( + {"fingerprint": 123, "subject_common_name": "Developer"} + ) + + +class TestInfoPlistMetadataSerialization(unittest.TestCase): + def test_round_trip(self): + metadata = InfoPlistMetadata( + bundle_id="com.test.app", bundle_type="APPL", is_watchos_app=False + ) + data = metadata.to_dict() + restored = InfoPlistMetadata.from_dict(data) + self.assertEqual(metadata, restored) + + def test_optional_fields(self): + data = {"bundle_id": "com.test", "bundle_type": None, "is_watchos_app": False} + restored = InfoPlistMetadata.from_dict(data) + self.assertEqual(restored.bundle_id, "com.test") + self.assertIsNone(restored.bundle_type) + + def test_requires_watchos_field(self): + with self.assertRaisesRegex(ValueError, "missing=.*is_watchos_app"): + InfoPlistMetadata.from_dict({"bundle_id": "com.test", "bundle_type": None}) + + +class TestProvisioningProfileMetadataSerialization(unittest.TestCase): + def test_round_trip_basic(self): + expiration = datetime.datetime(2025, 1, 1, 12, 0, 0) + creation = datetime.datetime(2024, 1, 1, 12, 0, 0) + profile = ProvisioningProfileMetadata( + file_path=Path("/tmp/profile.mobileprovision"), + uuid="UUID-1234", + expiration_date=expiration, + platforms=frozenset(["iOS"]), + developer_certificate_fingerprints=frozenset(["ABCDEF1234567890"]), + entitlements={"application-identifier": "TEAM.com.test.app"}, + creation_date=creation, + provisions_all_devices=False, + provisioned_devices=["device1"], + ) + data = profile.to_dict() + restored = ProvisioningProfileMetadata.from_dict(data) + self.assertEqual(restored.uuid, profile.uuid) + self.assertEqual(restored.expiration_date, profile.expiration_date) + self.assertEqual(restored.creation_date, profile.creation_date) + self.assertEqual(restored.platforms, profile.platforms) + self.assertEqual( + restored.developer_certificate_fingerprints, + profile.developer_certificate_fingerprints, + ) + self.assertEqual(restored.entitlements, profile.entitlements) + + def test_bytes_and_datetime_entitlements(self): + expiration = datetime.datetime(2025, 1, 1, 12, 0, 0) + profile = ProvisioningProfileMetadata( + file_path=Path("/tmp/profile.mobileprovision"), + uuid="UUID-BYTES", + expiration_date=expiration, + platforms=frozenset(["iOS"]), + developer_certificate_fingerprints=frozenset(["ABC"]), + entitlements={ + "data-key": b"\x00\x01\x02binary", + "date-key": datetime.datetime(2024, 6, 1, 10, 0, 0), + "nested": {"inner-bytes": b"inner"}, + }, + creation_date=None, + provisions_all_devices=False, + provisioned_devices=None, + ) + data = profile.to_dict() + json_str = json.dumps(data) + loaded = json.loads(json_str) + restored = ProvisioningProfileMetadata.from_dict(loaded) + self.assertEqual(restored.entitlements["data-key"], b"\x00\x01\x02binary") + self.assertEqual( + restored.entitlements["date-key"], + datetime.datetime(2024, 6, 1, 10, 0, 0), + ) + self.assertEqual(restored.entitlements["nested"]["inner-bytes"], b"inner") + + def test_rejects_missing_fields(self): + profile = self._profile() + data = profile.to_dict() + del data["platforms"] + with self.assertRaisesRegex(ValueError, "missing=.*platforms"): + ProvisioningProfileMetadata.from_dict(data) + + def test_rejects_malformed_entitlement_marker(self): + profile = self._profile() + data = profile.to_dict() + data["entitlements"] = {"data-key": {"__apple_code_signing_bytes_b64__": 123}} + with self.assertRaisesRegex(ValueError, "must be a string"): + ProvisioningProfileMetadata.from_dict(data) + + def test_marker_shaped_entitlement_dicts_round_trip(self): + profile = self._profile() + profile.entitlements = { + "bytes-marker": {"__apple_code_signing_bytes_b64__": "YWJj"}, + "datetime-marker": { + "__apple_code_signing_datetime_iso__": "2025-01-01T12:00:00" + }, + "escape-marker": { + "__apple_code_signing_escaped_dict__": {"nested": "value"} + }, + } + + serialized = json.loads(json.dumps(profile.to_dict())) + restored = ProvisioningProfileMetadata.from_dict(serialized) + + self.assertEqual(profile.entitlements, restored.entitlements) + + def _profile(self) -> ProvisioningProfileMetadata: + return ProvisioningProfileMetadata( + file_path=Path("/tmp/profile.mobileprovision"), + uuid="UUID-STRICT", + expiration_date=datetime.datetime(2025, 1, 1, 12, 0, 0), + platforms=frozenset(["iOS"]), + developer_certificate_fingerprints=frozenset(["ABC"]), + entitlements={}, + creation_date=None, + provisions_all_devices=False, + provisioned_devices=None, + ) + + +class TestSelectedProvisioningProfileInfoSerialization(unittest.TestCase): + def test_round_trip(self): + expiration = datetime.datetime(2025, 1, 1, 12, 0, 0) + profile = ProvisioningProfileMetadata( + file_path=Path("/tmp/profile.mobileprovision"), + uuid="UUID-1234", + expiration_date=expiration, + platforms=frozenset(["iOS"]), + developer_certificate_fingerprints=frozenset(["ABC"]), + entitlements={}, + creation_date=None, + provisions_all_devices=False, + provisioned_devices=None, + ) + identity = CodeSigningIdentity( + fingerprint="ABC", subject_common_name="Apple Development: Test" + ) + data_no_file = { + "identity": identity.to_dict(), + "profile": profile.to_dict(), + } + json_str = json.dumps(data_no_file) + loaded = json.loads(json_str) + restored = SelectedProvisioningProfileInfo.from_dict(loaded) + self.assertEqual(restored.profile.uuid, "UUID-1234") + self.assertEqual(restored.identity.fingerprint, "ABC") + + +class TestSigningContextWithProfileSelectionSerialization(unittest.TestCase): + def test_round_trip(self): + expiration = datetime.datetime(2025, 1, 1, 12, 0, 0) + profile = ProvisioningProfileMetadata( + file_path=Path("/tmp/profile.mobileprovision"), + uuid="UUID-1234", + expiration_date=expiration, + platforms=frozenset(["iOS"]), + developer_certificate_fingerprints=frozenset(["ABC"]), + entitlements={}, + creation_date=None, + provisions_all_devices=False, + provisioned_devices=None, + ) + identity = CodeSigningIdentity(fingerprint="ABC", subject_common_name="Dev") + selected = SelectedProvisioningProfileInfo(profile=profile, identity=identity) + metadata = InfoPlistMetadata( + bundle_id="com.test.app", bundle_type="APPL", is_watchos_app=False + ) + ctx = SigningContextWithProfileSelection( + info_plist_source=Path("/tmp/Info.plist"), + info_plist_destination=Path("Info.plist"), + info_plist_metadata=metadata, + selected_profile_info=selected, + ) + dict_without_file = { + "info_plist_source": str(ctx.info_plist_source), + "info_plist_destination": str(ctx.info_plist_destination), + "info_plist_metadata": metadata.to_dict(), + "selected_profile_info": { + "identity": identity.to_dict(), + "profile": profile.to_dict(), + }, + } + json_str = json.dumps(dict_without_file) + loaded = json.loads(json_str) + restored = SigningContextWithProfileSelection.from_dict(loaded) + self.assertEqual(restored.info_plist_source, Path("/tmp/Info.plist")) + self.assertEqual(restored.info_plist_metadata.bundle_id, "com.test.app") + self.assertEqual(restored.selected_profile_info.profile.uuid, "UUID-1234") + + +class TestAdhocSigningContextSerialization(unittest.TestCase): + def test_adhoc_without_profile(self): + ctx = AdhocSigningContext(codesign_identity="-", profile_selection_context=None) + data = ctx.to_dict() + self.assertIsNone(data["profile_selection_context"]) + json_str = json.dumps(data) + loaded = json.loads(json_str) + restored = AdhocSigningContext.from_dict(loaded) + self.assertEqual(restored.codesign_identity, "-") + self.assertIsNone(restored.profile_selection_context) + + def test_rejects_null_codesign_identity(self): + with self.assertRaisesRegex(ValueError, "codesign_identity must be a string"): + AdhocSigningContext.from_dict( + { + "codesign_identity": None, + "profile_selection_context": None, + } + ) + + def test_adhoc_with_profile_explicit_none_check(self): + # Ensures from_dict uses `is not None` check, not truthiness, so empty dict case would be preserved if ever occurs + expiration = datetime.datetime(2025, 1, 1, 12, 0, 0) + profile = ProvisioningProfileMetadata( + file_path=Path("/tmp/profile.mobileprovision"), + uuid="UUID-5678", + expiration_date=expiration, + platforms=frozenset(["iOS"]), + developer_certificate_fingerprints=frozenset(["AAA"]), + entitlements={}, + creation_date=None, + provisions_all_devices=True, + provisioned_devices=None, + ) + identity = CodeSigningIdentity(fingerprint="AAA", subject_common_name="Adhoc") + selected = SelectedProvisioningProfileInfo(profile=profile, identity=identity) + metadata = InfoPlistMetadata( + bundle_id="com.test.app", bundle_type="APPL", is_watchos_app=False + ) + distribution = SigningContextWithProfileSelection( + info_plist_source=Path("/tmp/Info.plist"), + info_plist_destination=Path("Info.plist"), + info_plist_metadata=metadata, + selected_profile_info=selected, + ) + adhoc = AdhocSigningContext( + codesign_identity="Apple Development: Adhoc", + profile_selection_context=distribution, + ) + data = adhoc.to_dict() + json_str = json.dumps( + { + "codesign_identity": data["codesign_identity"], + "profile_selection_context": { + "info_plist_source": str(distribution.info_plist_source), + "info_plist_destination": str(distribution.info_plist_destination), + "info_plist_metadata": metadata.to_dict(), + "selected_profile_info": { + "identity": identity.to_dict(), + "profile": profile.to_dict(), + }, + }, + } + ) + loaded = json.loads(json_str) + restored = AdhocSigningContext.from_dict(loaded) + self.assertEqual(restored.codesign_identity, "Apple Development: Adhoc") + assert restored.profile_selection_context is not None + self.assertEqual( + restored.profile_selection_context.selected_profile_info.profile.uuid, + "UUID-5678", + ) diff --git a/prelude/apple/tools/defs.bzl b/prelude/apple/tools/defs.bzl index 2afa022..dfa21be 100644 --- a/prelude/apple/tools/defs.bzl +++ b/prelude/apple/tools/defs.bzl @@ -45,3 +45,7 @@ def apple_oso_scrubber_target(): def bundle_telemetry_logger_target(): return read_root_config("apple", "bundle_telemetry_logger", None) + +def swiftmodule_change_analysis_target(): + # Overridable via buckconfig either way. + return read_root_config("apple", "swiftmodule_change_analysis", None) diff --git a/prelude/apple/tools/selective_debugging/scrubber.py b/prelude/apple/tools/selective_debugging/scrubber.py index 234cc78..5457736 100644 --- a/prelude/apple/tools/selective_debugging/scrubber.py +++ b/prelude/apple/tools/selective_debugging/scrubber.py @@ -64,11 +64,11 @@ def _get_target_output_path_from_debug_file_path( # Without content based path: # buck-out/isolation_dir/gen/project_cell/{hash}/.../__name__/libFoo.a # buck-out/isolation_dir/gen/project_cell/{hash}/.../__name__/__objects__/bar.o - # buck-out/isolation_dir/gen/project_cell/{hash}/.../__name__/swift_object_file.o + # buck-out/isolation_dir/gen/project_cell/{hash}/.../__name__/swift_object_file.swift.o # With content based path: # buck-out/isolation_dir/gen/project_cell/.../__name__/{hash}/libFoo.a # buck-out/isolation_dir/gen/project_cell/.../__name__/__objects__/{hash}/bar.o - # buck-out/isolation_dir/gen/project_cell/.../__name__/{hash}/swift_object_file.o + # buck-out/isolation_dir/gen/project_cell/.../__name__/{hash}/swift_object_file.swift.o parts = debug_target_path.split("/") # We are doing the traverse in reverse order because this way we'll find the first diff --git a/prelude/apple/user/watch_transition.bzl b/prelude/apple/user/watch_transition.bzl index 00e1dc9..a987877 100644 --- a/prelude/apple/user/watch_transition.bzl +++ b/prelude/apple/user/watch_transition.bzl @@ -21,9 +21,12 @@ def _watch_transition_impl(platform: PlatformInfo, refs: struct) -> PlatformInfo # - If the old OS constraint was iOS or watchOS, set the new constraint to be always watchOS. # - If the old SDK constraint was iOS, replace with the equivalent watchOS constraint. # - Return a new platform with the updated constraints. - updated_constraints = transition_utils.filtered_platform_constraints( - platform, [refs.os[ConstraintSettingInfo].label, refs.sdk[ConstraintSettingInfo].label] - ) + constraint_settings_to_remove = [ + refs.os[ConstraintSettingInfo].label, + refs.sdk[ConstraintSettingInfo].label, + ] + # @oss-disable[end= ]: constraint_settings_to_remove.append(refs.memory_profiler[ConstraintValueInfo].setting.label) + updated_constraints = transition_utils.filtered_platform_constraints(platform, constraint_settings_to_remove) # Update OS constraint old_os = transition_utils.get_constraint_value(platform, refs.os[ConstraintSettingInfo]) @@ -71,6 +74,7 @@ watch_transition = transition( "ios": "config//os/constraints:iphoneos", "ios_device_sdk": "config//os/sdk/apple/constraints:iphoneos", "ios_simulator_sdk": "config//os/sdk/apple/constraints:iphonesimulator", + # @oss-disable[end= ]: "memory_profiler": "config//build_mode/apple/constraints:enable-memory-profiler-constraint-value", "os": "config//os/constraints:os", "sdk": "config//os/sdk/apple/constraints:_", "watchos": "config//os/constraints:watchos", diff --git a/prelude/apple/validation/swiftmodule_change_analysis.bzl b/prelude/apple/validation/swiftmodule_change_analysis.bzl new file mode 100644 index 0000000..f21171f --- /dev/null +++ b/prelude/apple/validation/swiftmodule_change_analysis.bzl @@ -0,0 +1,125 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is dual-licensed under either the MIT license found in the +# LICENSE-MIT file in the root directory of this source tree or the Apache +# License, Version 2.0 found in the LICENSE-APACHE file in the root directory +# of this source tree. You may select, at your option, one of the +# above-listed licenses. + +load( + "@prelude//:artifact_tset.bzl", + "make_artifact_tset", + "stringify_artifact_label", +) +load("@prelude//apple:apple_toolchain_types.bzl", "AppleToolsInfo") +load("@prelude//apple/swift:swift_types.bzl", "SwiftDependencyInfo") + +# Backs the apple_bundle `[swiftmodule-change-analysis]` subtarget +def get_swiftmodule_change_analysis_output(ctx: AnalysisContext, deps: list[Dependency]) -> Artifact | None: + apple_tools = ctx.attrs._apple_tools[AppleToolsInfo] + + if apple_tools.swiftmodule_change_analysis == None: + return None + + swiftmodule_change_analysis_tsets = [dep[SwiftDependencyInfo].swiftmodule_change_analysis_tset for dep in deps if SwiftDependencyInfo in dep] + if not swiftmodule_change_analysis_tsets: + return None + + tset = make_artifact_tset( + actions = ctx.actions, + label = ctx.label, + children = swiftmodule_change_analysis_tsets, + ) + if tset._tset == None: + return None + + label_to_artifacts = {} + for infos in tset._tset.traverse(): + for info in infos: + label_to_artifacts.setdefault(stringify_artifact_label(info.label), []).extend(info.artifacts) + + if not label_to_artifacts: + return None + + manifest = ctx.actions.write_json( + "swiftmodule_change_analysis/manifest.json", + label_to_artifacts, + # with_inputs = True here means any change to a dependency's + # swiftmodule will re-trigger this action. + with_inputs = True, + has_content_based_path = False, + ) + + # Only populated when `apple.swift_dump_ast_subtarget_enabled` is set, so + # this is `None` for the vast majority of builds. Ignored by the default + # bootstrap tool; consumed by richer out-of-tree implementations (see + # `apple.swiftmodule_change_analysis` buckconfig) that classify changes + # via AST diffing instead of a raw `.swiftinterface` text diff. + ast_dump_tsets = [dep[SwiftDependencyInfo].swift_ast_dump_tset for dep in deps if SwiftDependencyInfo in dep] + ast_tset = make_artifact_tset( + actions = ctx.actions, + label = ctx.label, + children = ast_dump_tsets, + ) + ast_manifest = None + if ast_tset._tset != None: + label_to_ast_artifacts = {} + for infos in ast_tset._tset.traverse(): + for info in infos: + if info.artifacts: + label_to_ast_artifacts.setdefault(stringify_artifact_label(info.label), []).extend(info.artifacts) + + if label_to_ast_artifacts: + ast_manifest = ctx.actions.write_json( + "swiftmodule_change_analysis/ast_manifest.json", + label_to_ast_artifacts, + with_inputs = True, + has_content_based_path = False, + ) + + # Not content-addressed, and `no_outputs_cleanup` below keeps this file + # around across runs, so the checker can compare the current state + # against what it saw the last time this action ran. + state = ctx.actions.declare_output("swiftmodule_change_analysis/state.json", has_content_based_path = False) + output = ctx.actions.declare_output("swiftmodule_change_analysis/differences.json", has_content_based_path = False) + + swiftmodule_change_analysis_tool = apple_tools.swiftmodule_change_analysis + + command = cmd_args([ + swiftmodule_change_analysis_tool, + "--target", + stringify_artifact_label(ctx.label), + "--manifest", + manifest, + "--state", + state.as_output(), + "--output", + output.as_output(), + ]) + + if ast_manifest: + command.add("--ast-manifest", ast_manifest) + + # The manifest only bakes in each artifact's *path*; without also listing + # them as hidden inputs here, Buck2 wouldn't know this action actually + # depends on their content, and `metadata_path` below wouldn't include + # their digests. + command.add(cmd_args(hidden = [artifact for artifacts in label_to_artifacts.values() for artifact in artifacts])) + + ctx.actions.run( + command, + category = "swiftmodule_change_analysis", + identifier = ctx.attrs.name, + # prefer_local as only the local action will do anything useful + prefer_local = True, + allow_cache_upload = False, + no_outputs_cleanup = True, + # Buck2 already knows a content digest for every one of this + # action's inputs (it needs one for its own caching); this hands + # them to the tool as a `path -> digest` JSON file so it can compare + # digests instead of reading and hashing `.swiftmodule` files itself. + metadata_env_var = "SWIFTMODULE_CHANGE_ANALYSIS_METADATA", + metadata_path = "swiftmodule_change_analysis/action_metadata.json", + ) + + return output diff --git a/prelude/apple/versions.bzl b/prelude/apple/versions.bzl index d4fb7e7..d352c44 100644 --- a/prelude/apple/versions.bzl +++ b/prelude/apple/versions.bzl @@ -41,6 +41,7 @@ TARGET_SDK_VERSIONS = [ "17.2", "17.4", "18.0", + "18.4", "26.0", "26.1", ] diff --git a/prelude/apple/xcassets_asset_symbols.bzl b/prelude/apple/xcassets_asset_symbols.bzl new file mode 100644 index 0000000..ee35381 --- /dev/null +++ b/prelude/apple/xcassets_asset_symbols.bzl @@ -0,0 +1,25 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is dual-licensed under either the MIT license found in the +# LICENSE-MIT file in the root directory of this source tree or the Apache +# License, Version 2.0 found in the LICENSE-APACHE file in the root directory +# of this source tree. You may select, at your option, one of the +# above-listed licenses. + +# @oss-disable[end= ]: # Asset symbol hooks implemented in meta_only; open source gets no-op stubs. + +# @oss-disable[end= ]: load("@prelude//apple/meta_only:meta_xcassets_asset_symbol_spec.bzl", _MetaXcassetsAssetSymbolSpec = "MetaXcassetsAssetSymbolSpec") +# @oss-disable[end= ]: load( + # @oss-disable[end= ]: "@prelude//apple/meta_only:meta_xcassets_asset_symbol_usage.bzl", + # @oss-disable[end= ]: _usage_providers_and_subtargets = "meta_xcassets_asset_symbol_usage_providers_and_subtargets", +# @oss-disable[end= ]: ) +load("@prelude//cxx:cxx_sources.bzl", "CxxSrcWithFlags") + +# @oss-disable[end= ]: MetaXcassetsAssetSymbolSpec = _MetaXcassetsAssetSymbolSpec +MetaXcassetsAssetSymbolSpec = record() # @oss-enable + +def meta_xcassets_asset_symbol_usage_providers_and_subtargets( + ctx: AnalysisContext, cxx_srcs: list[CxxSrcWithFlags], swift_srcs: list[CxxSrcWithFlags] +) -> (list[Provider], dict[str, list[Provider]]): + # @oss-disable[end= ]: return _usage_providers_and_subtargets(ctx, cxx_srcs, swift_srcs) + return [], {} # @oss-enable diff --git a/prelude/cfg/modifier/cfg_constructor.bzl b/prelude/cfg/modifier/cfg_constructor.bzl index 0b094cc..c2d5435 100644 --- a/prelude/cfg/modifier/cfg_constructor.bzl +++ b/prelude/cfg/modifier/cfg_constructor.bzl @@ -39,6 +39,12 @@ def _get_buckconfig_backed_modifiers(extra_data: struct, configuring_exec_dep: b return None return getattr(extra_data, "buckconfig_backed_modifiers", None) +def _has_buckconfig_backed_modifiers(refs: dict[str, ProviderCollection], target: str | None) -> bool: + if not target: + return False + info = refs[target][BuckconfigBackedModifierInfo] + return bool(info.pre_platform_modifiers or info.post_platform_modifiers or info.pre_cli_modifiers) + def cfg_constructor_pre_constraint_analysis( *, legacy_platform: PlatformInfo | None, @@ -124,7 +130,8 @@ def cfg_constructor_post_constraint_analysis(*, refs: dict[str, ProviderCollecti Returns a PlatformInfo """ - if not (params.package_modifiers or params.target_modifiers or params.cli_modifiers): + buckconfig_backed_modifiers = _get_buckconfig_backed_modifiers(params.extra_data, params.configuring_exec_dep) + if not (params.package_modifiers or params.target_modifiers or params.cli_modifiers or _has_buckconfig_backed_modifiers(refs, buckconfig_backed_modifiers)): # If there is no modifier and legacy platform is specified, # then return the legacy platform as is without changing the label or # configuration. @@ -138,8 +145,6 @@ def cfg_constructor_post_constraint_analysis(*, refs: dict[str, ProviderCollecti ) constraint_setting_to_modifier_infos = {} - cli_modifier_validation = getattr(params.extra_data, "cli_modifier_validation", None) - buckconfig_backed_modifiers = _get_buckconfig_backed_modifiers(params.extra_data, params.configuring_exec_dep) if buckconfig_backed_modifiers: apply_buckconfig_backed_modifiers( @@ -181,19 +186,11 @@ def cfg_constructor_post_constraint_analysis(*, refs: dict[str, ProviderCollecti for modifier in params.cli_modifiers: if modifier: - constraint_setting_label, _ = get_and_insert_modifier_info( + get_and_insert_modifier_info( constraint_setting_to_modifier_infos = constraint_setting_to_modifier_infos, refs = refs, modifier = modifier, location = ModifierCliLocation(), ) - # Exclude CLI modifier allowlist validation when evaluating the exec configuration, - # because modifiers from CLI are not applied to exec dependencies. - # Instead, we treat the original platform constraints as "CLI modifiers" so they take precedence. - if params.configuring_exec_dep: - continue - if cli_modifier_validation: - cli_modifier_validation(constraint_setting_label, modifier) - return resolve_configuration(constraint_setting_to_modifier_infos) diff --git a/prelude/cfg/modifier/common.bzl b/prelude/cfg/modifier/common.bzl index cd87569..3e85347 100644 --- a/prelude/cfg/modifier/common.bzl +++ b/prelude/cfg/modifier/common.bzl @@ -103,12 +103,16 @@ def get_modifier_info(refs: dict[str, ProviderCollection], modifier: Modifier, l default = None elif key != "_type": cfg_info = refs[key][ConfigurationInfo] - if cfg_info.values: + root_values = cfg_info.root_values + if cfg_info.values or root_values: + buckconfig_values = cfg_info.values | root_values soft_error( "starlark_config_setting_non_empty_buckconfig_values_in_conditional_modifier", - "config_setting `{}` defines buckconfig values {} which are NOT supported in conditional modifiers.\n".format(key, cfg_info.values) + "config_setting `{}` defines buckconfig values {} which are NOT supported in conditional modifiers.\n".format(key, buckconfig_values) + "These buckconfig values are being IGNORED.\n\n" - + "Action required: Remove the `values` parameter from this config_setting {} and use only `constraint_values` instead.\n".format(key) + + "Action required: Remove the `values` and `root_values` parameters from this config_setting {} and use only `constraint_values` instead.\n".format( + key + ) + "Note: This may become a hard error in the future to prevent silent misconfiguration.", quiet = True, stack = False, diff --git a/prelude/cfg/modifier/name.bzl b/prelude/cfg/modifier/name.bzl index 6bd8c37..83fd975 100644 --- a/prelude/cfg/modifier/name.bzl +++ b/prelude/cfg/modifier/name.bzl @@ -21,7 +21,7 @@ NAMED_CONSTRAINT_SETTINGS = { "ovr_config//cpu/constraints:cpu": None, "ovr_config//runtime/constraints:runtime": None, "ovr_config//runtime/constraints:runtime_version": None, - "ovr_config//os/sdk/apple/constraints:_": None, + "ovr_config//os/sdk/apple/constraints:_": (lambda label: str(label.sub_target[0])), "ovr_config//os/sdk/android/ndk/constraints:version": None, "ovr_config//os/version/android/constraints:api-level": (lambda label: "api" + str(label.name).split("-")[-1]), "ovr_config//toolchain/clang/constraints:clang-toolchain-version": (lambda label: "clang" + str(label.name)), diff --git a/prelude/configurations/rules.bzl b/prelude/configurations/rules.bzl index 311557c..d3be067 100644 --- a/prelude/configurations/rules.bzl +++ b/prelude/configurations/rules.bzl @@ -18,7 +18,13 @@ _ExecutionModifierInfo = provider( def config_setting_impl(ctx): subinfos = [util.constraint_values_to_configuration(ctx.attrs.constraint_values)] - subinfos.append(ConfigurationInfo(constraints = {}, values = ctx.attrs.values)) + subinfos.append( + util.configuration_info( + constraints = {}, + values = ctx.attrs.values, + root_values = ctx.attrs.root_values, + ) + ) cfg_info = util.configuration_info_union(subinfos) providers = [DefaultInfo(), cfg_info] if len(ctx.attrs.constraint_values) == 1: diff --git a/prelude/configurations/util.bzl b/prelude/configurations/util.bzl index e8cdf25..54a21ea 100644 --- a/prelude/configurations/util.bzl +++ b/prelude/configurations/util.bzl @@ -6,9 +6,16 @@ # of this source tree. You may select, at your option, one of the # above-listed licenses. +def _configuration_info(constraints, values, root_values = None): + return ConfigurationInfo( + constraints = constraints, + values = values, + root_values = root_values or {}, + ) + def _configuration_info_union(infos): if len(infos) == 0: - return ConfigurationInfo( + return _configuration_info( constraints = {}, values = {}, ) @@ -16,15 +23,21 @@ def _configuration_info_union(infos): return infos[0] constraints = {k: v for info in infos for (k, v) in info.constraints.items()} values = {k: v for info in infos for (k, v) in info.values.items()} - return ConfigurationInfo( + root_values = {k: v for info in infos for (k, v) in info.root_values.items()} + return _configuration_info( constraints = constraints, values = values, + root_values = root_values, ) def _constraint_values_to_configuration(values): - return ConfigurationInfo(constraints = {info[ConstraintValueInfo].setting.label: info[ConstraintValueInfo] for info in values}, values = {}) + return _configuration_info( + constraints = {info[ConstraintValueInfo].setting.label: info[ConstraintValueInfo] for info in values}, + values = {}, + ) util = struct( + configuration_info = _configuration_info, configuration_info_union = _configuration_info_union, constraint_values_to_configuration = _constraint_values_to_configuration, ) diff --git a/prelude/cxx/anon_link.bzl b/prelude/cxx/anon_link.bzl index 0c4b76a..cf327e5 100644 --- a/prelude/cxx/anon_link.bzl +++ b/prelude/cxx/anon_link.bzl @@ -10,7 +10,6 @@ load( "@prelude//:artifact_tset.bzl", "ArtifactInfo", "make_artifact_tset", - "stringify_artifact_label", ) load( "@prelude//cxx:cxx_toolchain_types.bzl", @@ -36,74 +35,115 @@ load( "link_options", ) -def _serialize_linkable(linkable): +# The link recipe is encoded as one JSON string of pure scalars/counts plus two +# flat streams (flags and artifacts) that both sides walk in the same traversal +# order. Encoding each linkable as nested attr tuples instead costs the daemon +# ~1KiB of coerced-attr nodes per linkable, which dominates buck2 memory on +# large link-group graphs (measured ~1GiB retained on a 14-target fbcode dev +# cohort); the artifacts must remain real `attrs.source()` values for input +# tracking, but everything else can live in the string. +# +# The anon target identity therefore hashes (recipe, flags, artifacts), which +# discriminates exactly like the nested encoding, except that external debug +# info labels are no longer part of the identity: deserialization always +# discarded them, so they only served to split otherwise-identical identities. + +_LINK_ARGS_FLAGS_TAG = 0 +_LINK_ARGS_INFOS_TAG = 1 + +_LINKABLE_ARCHIVE_TAG = 0 +_LINKABLE_OBJECTS_TAG = 1 +_LINKABLE_SHARED_TAG = 2 + +def _encode_linkable(linkable, artifacts: list[Artifact]): if isinstance(linkable, ArchiveLinkable): - return ( - "archive", - ( - ( - linkable.archive.artifact, - linkable.archive.external_objects, - linkable.archive.archive_contents_type.value, - ), - linkable.link_whole, - linkable.linker_type.value, - linkable.supports_lto, - ), - ) + artifacts.append(linkable.archive.artifact) + artifacts.extend(linkable.archive.external_objects) + return [ + _LINKABLE_ARCHIVE_TAG, + len(linkable.archive.external_objects), + linkable.archive.archive_contents_type.value, + linkable.link_whole, + linkable.linker_type.value, + linkable.supports_lto, + ] if isinstance(linkable, ObjectsLinkable): - return ( - "objects", - ( - linkable.objects, - linkable.link_whole, - linkable.linker_type.value, - ), - ) + artifacts.extend(linkable.objects) + return [ + _LINKABLE_OBJECTS_TAG, + len(linkable.objects), + linkable.link_whole, + linkable.linker_type.value, + ] if isinstance(linkable, SharedLibLinkable): - return ( - "shared", - ( - linkable.lib, - linkable.link_without_soname, - ), - ) + artifacts.append(linkable.lib) + return [ + _LINKABLE_SHARED_TAG, + linkable.link_without_soname, + ] fail('cannot serialize linkable "{}"'.format(str(linkable))) -def _serialize_link_info(info: LinkInfo): - external_debug_info = [] +def _encode_link_info(info: LinkInfo, flags: list, artifacts: list[Artifact]): + external_debug_artifacts = [] if info.external_debug_info._tset != None: - for infos in info.external_debug_info._tset.traverse(): - external_debug_info.extend(infos) - external_debug_info = dedupe(external_debug_info) - return ( - info.name, - info.pre_flags, - info.post_flags, - [_serialize_linkable(linkable) for linkable in info.linkables], - # TODO(agallagher): It appears anon-targets don't allow passing in `label`. - [(stringify_artifact_label(info.label), info.artifacts) for info in external_debug_info], - [m.version for m in info.metadata], - ) - -def _serialize_link_args(link: LinkArgs): - if link.flags != None: - return ("flags", link.flags) + external_debug_infos = [] + for tset_infos in info.external_debug_info._tset.traverse(): + external_debug_infos.extend(tset_infos) + for artifact_info in dedupe(external_debug_infos): + external_debug_artifacts.extend(artifact_info.artifacts) - if link.infos != None: - return ("infos", [_serialize_link_info(info) for info in link.infos]) + flags.extend(info.pre_flags) + flags.extend(info.post_flags) + linkables = [_encode_linkable(linkable, artifacts) for linkable in info.linkables] + artifacts.extend(external_debug_artifacts) - fail("cannot serialize link args") + return [ + len(info.pre_flags), + len(info.post_flags), + linkables, + len(external_debug_artifacts), + [m.version for m in info.metadata], + ] -def _serialize_links(links: list[LinkArgs]): - return [_serialize_link_args(link) for link in links] +def _anon_link_flags(values: list) -> list: + result = [] + for value in values: + if isinstance(value, TransitiveSetArgsProjection) and value.projection_name == "linker_flags": + # Anonymous attrs accept resolved macros, but not transitive-set projections. + for shared_flags in value.transitive_set.traverse(ordering = "postorder"): + result.extend(shared_flags.linker_flags) + else: + result.append(value) + return result def serialize_anon_attrs(output: str, result_type: CxxLinkResultType, opts: LinkOptions) -> dict[str, typing.Any]: + # Anonymous links cannot run distributed ThinLTO (its dynamic outputs are + # unsupported in anon targets; every anonymous caller disables it), and the + # encoding relies on that: `LinkInfo.name` is not serialized, and dist + # ThinLTO planning is the one consumer that reads it. + if opts.enable_distributed_thinlto: + fail("anonymous links do not support distributed ThinLTO") + + recipe = [] + flags = [] + artifacts = [] + for link in opts.links: + if link.flags != None: + link_flags = _anon_link_flags(link.flags) + recipe.append([_LINK_ARGS_FLAGS_TAG, len(link_flags)]) + flags.extend(link_flags) + elif link.infos != None: + recipe.append([_LINK_ARGS_INFOS_TAG, [_encode_link_info(info, flags, artifacts) for info in link.infos]]) + else: + fail("cannot serialize link args") + return dict( - links = _serialize_links(opts.links), + links_recipe = json.encode(recipe), + links_flags = flags, + links_artifacts = artifacts, output = output, import_library = opts.import_library, link_execution_preference = opts.link_execution_preference.value, @@ -115,14 +155,17 @@ def serialize_anon_attrs(output: str, result_type: CxxLinkResultType, opts: Link allow_cache_upload = opts.allow_cache_upload, ) -def _deserialize_linkable(linkable: (str, typing.Any)) -> typing.Any: - typ, payload = linkable +def _decode_linkable(spec, artifacts, artifact_cursor: list[int]) -> typing.Any: + kind = spec[0] - if typ == "archive": - (artifact, external_objects, archive_contents_type), link_whole, linker_type, supports_lto = payload + if kind == _LINKABLE_ARCHIVE_TAG: + _, num_external_objects, archive_contents_type, link_whole, linker_type, supports_lto = spec + archive_artifact = artifacts[artifact_cursor[0]] + external_objects = artifacts[artifact_cursor[0] + 1 : artifact_cursor[0] + 1 + num_external_objects] + artifact_cursor[0] += 1 + num_external_objects return ArchiveLinkable( archive = Archive( - artifact = artifact, + artifact = archive_artifact, external_objects = external_objects, archive_contents_type = ArchiveContentsType(archive_contents_type), ), @@ -131,51 +174,81 @@ def _deserialize_linkable(linkable: (str, typing.Any)) -> typing.Any: supports_lto = supports_lto, ) - if typ == "objects": - objects, link_whole, linker_type = payload + if kind == _LINKABLE_OBJECTS_TAG: + _, num_objects, link_whole, linker_type = spec + objects = artifacts[artifact_cursor[0] : artifact_cursor[0] + num_objects] + artifact_cursor[0] += num_objects return ObjectsLinkable( objects = objects, link_whole = link_whole, linker_type = LinkerType(linker_type), ) - if typ == "shared": - lib, link_without_soname = payload + if kind == _LINKABLE_SHARED_TAG: + _, link_without_soname = spec + lib = artifacts[artifact_cursor[0]] + artifact_cursor[0] += 1 return SharedLibLinkable( lib = lib, link_without_soname = link_without_soname, ) - fail("Invalid linkable type: {}".format(typ)) + fail("Invalid linkable kind: {}".format(kind)) + +def _decode_link_info(actions: AnalysisActions, label: Label, spec, flags, flag_cursor: list[int], artifacts, artifact_cursor: list[int]) -> LinkInfo: + num_pre_flags, num_post_flags, linkable_specs, num_external_debug_artifacts, metadata = spec + + pre_flags = flags[flag_cursor[0] : flag_cursor[0] + num_pre_flags] + post_flags = flags[flag_cursor[0] + num_pre_flags : flag_cursor[0] + num_pre_flags + num_post_flags] + flag_cursor[0] += num_pre_flags + num_post_flags + + linkables = [_decode_linkable(linkable_spec, artifacts, artifact_cursor) for linkable_spec in linkable_specs] + + external_debug_artifacts = artifacts[artifact_cursor[0] : artifact_cursor[0] + num_external_debug_artifacts] + artifact_cursor[0] += num_external_debug_artifacts -def _deserialize_link_info(actions: AnalysisActions, label: Label, info) -> LinkInfo: - name, pre_flags, post_flags, linkables, external_debug_info, metadata = info return LinkInfo( - name = name, + name = None, pre_flags = pre_flags, post_flags = post_flags, - linkables = [_deserialize_linkable(linkable) for linkable in linkables], + linkables = linkables, external_debug_info = make_artifact_tset( actions = actions, - infos = [ArtifactInfo(label = label, artifacts = artifacts, tags = []) for _label, artifacts in external_debug_info], + infos = [ArtifactInfo(label = label, artifacts = external_debug_artifacts, tags = [])] if external_debug_artifacts else [], ), metadata = [DepMetadata(version = v) for v in metadata], ) -def _deserialize_link_args(actions: AnalysisActions, label: Label, link: (str, typing.Any)) -> LinkArgs: - typ, payload = link - - if typ == "flags": - return LinkArgs(flags = payload) - - if typ == "infos": - return LinkArgs(infos = [_deserialize_link_info(actions, label, info) for info in payload]) +def deserialize_anon_attrs(actions: AnalysisActions, label: Label, attrs: struct) -> (str, CxxLinkResultType, LinkOptions): + flag_cursor = [0] + artifact_cursor = [0] + links = [] + for entry in json.decode(attrs.links_recipe): + if entry[0] == _LINK_ARGS_FLAGS_TAG: + num_flags = entry[1] + links.append(LinkArgs(flags = attrs.links_flags[flag_cursor[0] : flag_cursor[0] + num_flags])) + flag_cursor[0] += num_flags + elif entry[0] == _LINK_ARGS_INFOS_TAG: + links.append( + LinkArgs( + infos = [_decode_link_info(actions, label, spec, attrs.links_flags, flag_cursor, attrs.links_artifacts, artifact_cursor) for spec in entry[1]] + ) + ) + else: + fail("Invalid link args kind: {}".format(entry[0])) - fail("invalid link args type: {}".format(typ)) + if flag_cursor[0] != len(attrs.links_flags) or artifact_cursor[0] != len(attrs.links_artifacts): + fail( + "anon link recipe did not consume its flag/artifact streams exactly (flags {}/{}, artifacts {}/{})".format( + flag_cursor[0], + len(attrs.links_flags), + artifact_cursor[0], + len(attrs.links_artifacts), + ) + ) -def deserialize_anon_attrs(actions: AnalysisActions, label: Label, attrs: struct) -> (str, CxxLinkResultType, LinkOptions): opts = link_options( - links = [_deserialize_link_args(actions, label, link) for link in attrs.links], + links = links, import_library = attrs.import_library, link_execution_preference = LinkExecutionPreference(attrs.link_execution_preference), dwp_execution_preference = LinkExecutionPreference(attrs.dwp_execution_preference) if attrs.dwp_execution_preference != None else None, @@ -189,8 +262,8 @@ def deserialize_anon_attrs(actions: AnalysisActions, label: Label, attrs: struct return (attrs.output, result_type, opts) -# The attributes -- and their serialzied type -- that can be passed to an -# anonymous link. +# The attributes -- and their serialized types -- that can be passed to an +# anonymous link. See the module docblock for the stream encoding contract. ANON_ATTRS = { "allow_cache_upload": attrs.bool(), "category_suffix": attrs.string(), @@ -199,67 +272,9 @@ ANON_ATTRS = { "identifier": attrs.option(attrs.string(), default = None), "import_library": attrs.option(attrs.source(), default = None), "link_execution_preference": attrs.enum(LinkExecutionPreference.values()), - "links": attrs.list( - # List[LinkArgs] - attrs.tuple( - attrs.enum(["flags", "infos"]), # "flags" or "infos" - attrs.one_of( - attrs.list(attrs.one_of(attrs.string(), attrs.arg())), # flags - attrs.list( - # infos: [LinkInfo] - attrs.tuple( - # LinkInfo - attrs.option(attrs.string(), default = None), # name - attrs.list(attrs.one_of(attrs.string(), attrs.arg())), # pre_flags - attrs.list(attrs.one_of(attrs.string(), attrs.arg())), # post_flags - attrs.list( - # linkables - attrs.tuple( - attrs.enum(["archive", "objects", "shared"]), - attrs.one_of( - attrs.tuple( - # ObjectsLinkable - attrs.list(attrs.source()), # objects - attrs.bool(), # link_whole - attrs.enum(LinkerType.values()), # linker_type - ), - attrs.tuple( - # ArchiveLinkable - attrs.tuple( - # Archive - attrs.source(), # archive - attrs.list(attrs.source()), # external_objects - attrs.enum(ArchiveContentsType.values()), # archive_contents_type - ), - attrs.bool(), # link_whole - attrs.enum(LinkerType.values()), # linker_type - attrs.bool(), # supports_lto - ), - attrs.tuple( - # SharedLibLinkable - attrs.source(), # lib - attrs.bool(), # link_without_soname - ), - ), - ), - ), - attrs.list( - # external_debug_info - attrs.tuple( - # TODO(agallagher): It appears anon-targets don't - # allow passing in `label`. - attrs.string(), # label - attrs.list(attrs.source()), # artifacts - ), - ), - # metadata - attrs.list(attrs.string()), - ), - ), - ), - ), - default = [], - ), + "links_artifacts": attrs.list(attrs.source(), default = []), + "links_flags": attrs.list(attrs.arg(), default = []), + "links_recipe": attrs.string(default = "[]"), "output": attrs.string(), "result_type": attrs.enum(CxxLinkResultType.values()), "separate_debug_info": attrs.bool(default = False), diff --git a/prelude/cxx/archive.bzl b/prelude/cxx/archive.bzl index e784516..642f93c 100644 --- a/prelude/cxx/archive.bzl +++ b/prelude/cxx/archive.bzl @@ -48,6 +48,8 @@ def archive_flags(archiver_type: str, linker_type: LinkerType, use_archiver_flag return [flags] +_ARCHIVE_ENV = {"ZERO_AR_DATE": "1"} + # Create a static library from a list of object files. def _archive( ctx: AnalysisContext, name: str, args: cmd_args, thin: bool, prefer_local: bool, allow_cache_upload: bool, force_disable_content_based_path: bool = False @@ -96,7 +98,7 @@ def _archive( # other archivers do not support such a flag. Some implementations, notably # Xcode's, instead support zeroing the timestamp by way of an environment # variable. - env = {"ZERO_AR_DATE": "1"} + env = _ARCHIVE_ENV category = "archive" if thin: diff --git a/prelude/cxx/argsfiles.bzl b/prelude/cxx/argsfiles.bzl index 8795700..356c501 100644 --- a/prelude/cxx/argsfiles.bzl +++ b/prelude/cxx/argsfiles.bzl @@ -14,10 +14,17 @@ CompileArgsfile = record( file = field(Artifact), # This argsfile as a command form that would use the argsfile (includes dependent inputs). cmd_form = field(cmd_args), + # Command form for the NVCC dry run: the argsfile, the argsfiles it + # references and the header maps the flags name, but not the header trees + # or other artifacts behind the flags. Only built for CUDA sources. + cmd_form_cuda_dryrun = field(cmd_args | None, None), # Args as written to the argsfile (with shell quoting applied). args = field(cmd_args), # Args aggregated for the argsfile excluding file prefix args (excludes shell quoting). args_without_file_prefix_args = field(cmd_args), + # The generated GCC specs file carrying the file-prefix-map flags, when the + # compiler takes them that way. None for compilers given the flags directly. + file_prefix_specs = field(Artifact | None, None), ) CompileArgsfiles = record( diff --git a/prelude/cxx/bitcode.bzl b/prelude/cxx/bitcode.bzl index 21a3be7..f685346 100644 --- a/prelude/cxx/bitcode.bzl +++ b/prelude/cxx/bitcode.bzl @@ -65,7 +65,7 @@ def make_bitcode_bundle(ctx: AnalysisContext, name: str, objects: list[Artifact] linker_info = get_cxx_toolchain_info(ctx).linker_info - args = cmd_args(format = '"{}"') + args = cmd_args(quote = "shell") if ignore_native: args.add("--ignore-non-bitcode") diff --git a/prelude/cxx/comp_db.bzl b/prelude/cxx/comp_db.bzl index 0bb8bb3..1a6aa03 100644 --- a/prelude/cxx/comp_db.bzl +++ b/prelude/cxx/comp_db.bzl @@ -14,6 +14,7 @@ load( "CxxSrcCompileCommand", # @unused Used as a type ) load(":cxx_context.bzl", "get_cxx_toolchain_info") +load(":cxx_library_utility.bzl", "EMPTY_DEFAULT_INFO") # Provider that exposes the compilation database information CxxCompilationDbInfo = provider( @@ -82,7 +83,7 @@ def _create_comp_database_impl( actions.run(cmd, category = "cxx_compilation_database_merge", identifier = identifier) - return [DefaultInfo()] + return [EMPTY_DEFAULT_INFO] _dynamic_compilation_database_rule = dynamic_actions( impl = _create_comp_database_impl, diff --git a/prelude/cxx/compile.bzl b/prelude/cxx/compile.bzl index cefce08..7027199 100644 --- a/prelude/cxx/compile.bzl +++ b/prelude/cxx/compile.bzl @@ -21,12 +21,15 @@ load( "DepFileType", "HeaderExtension", "HeadersDepFiles", + "IndexStoreFactory", "UseHeaderUnitsMode", ) load( "@prelude//cxx:cuda.bzl", "CudaCompileInfo", "CudaCompileStyle", # @unused Used as a type + "CudaDistributedCompileSpec", # @unused Used as a type + "create_cuda_distributed_compiles", "cuda_compile", "declare_cuda_dist_compile_output", ) @@ -37,6 +40,7 @@ load( "XCODE_ARG_SUBSTITUTIONS", ) load("@prelude//linking:lto.bzl", "LtoMode") +load("@prelude//utils:argfile.bzl", "argsfile_with_artifacts", _mk_argsfile = "mk_argsfile") load("@prelude//utils:expect.bzl", "expect") load( "@prelude//utils:utils.bzl", @@ -54,8 +58,15 @@ load( "get_output_flags", ) load(":cxx_context.bzl", "get_cxx_toolchain_info") +load( + ":cxx_flags.bzl", + "CxxFlagsInfo", + "cxx_flags_projection_name", + "cxx_flags_tset", +) +load(":cxx_library_utility.bzl", "EMPTY_DEFAULT_INFO") load(":cxx_sources.bzl", "CxxSrcWithFlags") -load(":cxx_toolchain_types.bzl", "CxxObjectFormat", "DepTrackingMode") +load(":cxx_toolchain_types.bzl", "CxxObjectFormat", "DepTrackingMode", "compiler_info_with_argsfiles") load(":cxx_types.bzl", "CxxRuleConstructorParams") load(":debug.bzl", "SplitDebugMode") load( @@ -74,15 +85,28 @@ load( "get_flags_for_compiler_type", ) +# Output folder shared by every per-source object declaration; the shared CUDA +# plan must declare its outputs under the same folder as the objects it feeds. +_OBJECTS_FOLDER = "__objects__" + +_CUDA_DRYRUN_SOURCE_PLACEHOLDER = "__buck2_cuda_dryrun_placeholder__.cu" + # Record containing compile info that will be passed to the dynamic action CxxCompileInfo = record( - compile_cmd = field(CxxSrcCompileCommand), + src = field(Artifact), + args = field(list[typing.Any]), short_path = field(str), filename_base = field(str), index_store_base = field(str | None), # filename_base for index store compilation identifier = field(str), folder_name = field(str), - flavor_flags = field(dict[str, list[str]]), +) + +CxxSharedCompileInfo = record( + cxx_compile_cmd = field(CxxCompileCommand), + index_store_factory = field(IndexStoreFactory | None), + error_handler = field([typing.Callable, None]), + expect_eligible_for_dedupe = field(bool), ) # Input for a single CXX compilation - declared artifacts and compile info @@ -91,6 +115,11 @@ CxxCompileInput = record( info = field(CxxCompileInfo), ) +CxxSharedCompileCommandArgs = record( + before_src = field(cmd_args), + after_src = field(cmd_args), +) + def _project_clang_traces_as_args(traces: list[Artifact]): return cmd_args(traces) @@ -217,6 +246,7 @@ def create_compile_cmds( inherited_preprocessor_infos: list[CPreprocessorInfo], add_coverage_instrumentation_compiler_flags: bool, compile_pch: CxxPrecompiledHeader | None = None, + filename_prefix: str = "", ) -> CxxCompileCommandOutput: """ Forms the CxxSrcCompileCommand to use for each source file based on it's extension @@ -279,10 +309,20 @@ def create_compile_cmds( # of the same extension they will have some of the same flags. Save on # allocations by caching and reusing these objects. for ext in src_extensions: - cmd = _generate_base_compile_command(actions, target_label, toolchain, impl_params, pre, headers_tag, ext) + cmd = _generate_base_compile_command( + actions, + target_label, + toolchain, + impl_params, + pre, + headers_tag, + ext, + filename_prefix = filename_prefix, + ) cxx_compile_cmd_by_ext[ext] = cmd argsfile_by_ext[ext.value] = cmd.argsfile - xcode_argsfile_by_ext[ext.value] = cmd.xcode_argsfile + if cmd.xcode_argsfile != None: + xcode_argsfile_by_ext[ext.value] = cmd.xcode_argsfile # Add -fprofile-list for selective coverage instrumentation. # The profile list file is generated by a clang_profile_list rule and tells @@ -360,6 +400,7 @@ def _prepare_cxx_compilation( use_header_units: UseHeaderUnitsMode, separate_debug_info: bool, cuda_compile_style: CudaCompileStyle | None, + shared_cuda_dist_output: CudaDistributedCompileOutput | None, compile_pch: CxxPrecompiledHeader | None, ) -> CxxCompileInput: """ @@ -385,7 +426,7 @@ def _prepare_cxx_compilation( identifier = "{} ({})".format(identifier, flavor.value) content_based = src_compile_cmd.uses_content_based_paths - folder_name = "__objects__" + folder_name = _OBJECTS_FOLDER compiler_type = src_compile_cmd.cxx_compile_cmd.compiler_type # Declare main object output @@ -490,13 +531,12 @@ def _prepare_cxx_compilation( # globally on a toolchain level. object_has_external_debug_info = toolchain.split_debug_mode == SplitDebugMode("single") - flavor_flags = build_flavor_flags(toolchain.compiler_flavor_flags, src_compile_cmd.cxx_compile_cmd.compiler_type) clang_trace = _declare_clang_trace_output(toolchain, compiler_type, actions, filename_base, content_based) # Only declare CUDA distributed compile outputs for CUDA source files cuda_dist_output = None if src_compile_cmd.src.extension == ".cu" and cuda_compile_style == CudaCompileStyle("dist"): - cuda_dist_output = declare_cuda_dist_compile_output( + cuda_dist_output = shared_cuda_dist_output or declare_cuda_dist_compile_output( actions, CudaCompileInfo( filename = filename_base, @@ -547,13 +587,13 @@ def _prepare_cxx_compilation( ) info = CxxCompileInfo( - compile_cmd = src_compile_cmd, + src = src_compile_cmd.src, + args = src_compile_cmd.args, short_path = short_path, filename_base = filename_base, index_store_base = index_store_base, identifier = identifier, folder_name = folder_name, - flavor_flags = flavor_flags, ) return CxxCompileInput( @@ -565,14 +605,13 @@ def _compile_single_cxx( actions: AnalysisActions, label: Label, toolchain: CxxToolchainInfo, - bitcode_args: list, flavors: set[CxxCompileFlavor], compile_pch: CxxPrecompiledHeader | None, precompiled_header: Dependency | None, cuda_compile_style: CudaCompileStyle | None, - use_header_units: UseHeaderUnitsMode, # CxxCompileInfo fields info: CxxCompileInfo, + shared_info: CxxSharedCompileInfo, # Output artifacts object: OutputArtifact, external_debug_info: OutputArtifact | None, @@ -585,10 +624,15 @@ def _compile_single_cxx( clang_tidy_diagnostics: OutputArtifact | None, preproc: OutputArtifact, index_store: OutputArtifact | None, - dist_cuda: (OutputArtifact, OutputArtifact, OutputArtifact) | None, + dist_cuda: CudaDistributedCompileOutput | None, + prepare_cuda_dist: bool, + cuda_shared_plan_identifier: str | None, + shared_compile_args: CxxSharedCompileCommandArgs, + shared_auxiliary_args: CxxSharedCompileCommandArgs, + shared_dryrun_args: CxxSharedCompileCommandArgs | None, pch_object: OutputArtifact | None, json_error: OutputArtifact | None, -) -> None: +) -> CudaDistributedCompileSpec | None: """ EXECUTION PHASE: Creates compilation actions for a single source file. @@ -601,22 +645,24 @@ def _compile_single_cxx( """ # Extract info fields - src_compile_cmd = info.compile_cmd + src = info.src + src_args = info.args filename_base = info.filename_base index_store_base = info.index_store_base identifier = info.identifier folder_name = info.folder_name short_path = info.short_path - flavor_flags = info.flavor_flags + cxx_compile_cmd = shared_info.cxx_compile_cmd # Get compiler type - compiler_type = src_compile_cmd.cxx_compile_cmd.compiler_type - content_based = src_compile_cmd.uses_content_based_paths + compiler_type = cxx_compile_cmd.compiler_type + content_based = cxx_compile_cmd.allow_content_based_paths - if src_compile_cmd.src.extension == ".cu": + cuda_dist_spec = None + if src.extension == ".cu": output_args = None elif compile_pch: - if src_compile_cmd.cxx_compile_cmd.compiler_type == "windows": + if compiler_type == "windows": output_args = [ cmd_args(object, format = "/Fp{}"), cmd_args(pch_object, format = "/Fo{}"), @@ -634,23 +680,45 @@ def _compile_single_cxx( else: output_args = get_output_flags(compiler_type, object) + base_compile_cmd_override = None + if src.extension == ".cu" and cuda_compile_style == CudaCompileStyle("dist"): + compiler_for_dryrun = getattr(toolchain.cuda_compiler_info, "compiler_for_dryrun", None) + if compiler_for_dryrun != None: + base_compile_cmd_override = _get_compile_base( + toolchain, + toolchain.cuda_compiler_info, + use_wrapper = False, + compiler_override = compiler_for_dryrun, + ) + cmd = _get_base_compile_cmd( - bitcode_args = bitcode_args, - src_compile_cmd = src_compile_cmd, - flavors = flavors, - flavor_flags = flavor_flags, - use_header_units = use_header_units, + src = src, + src_args = src_args, + cxx_compile_cmd = cxx_compile_cmd, output_args = output_args, + base_compile_cmd_override = base_compile_cmd_override, + shared_args = shared_compile_args, ) + cuda_prepare_cmd = None + if prepare_cuda_dist: + expect(shared_dryrun_args != None, "distributed CUDA prepare requires dry-run compile args") + cuda_prepare_cmd = _get_base_compile_cmd( + src = src, + src_args = src_args, + cxx_compile_cmd = cxx_compile_cmd, + base_compile_cmd_override = base_compile_cmd_override, + source_override = _CUDA_DRYRUN_SOURCE_PLACEHOLDER, + shared_args = shared_dryrun_args, + ) if index_store: compile_index_store_cmd = _get_base_compile_cmd( - bitcode_args = bitcode_args, - src_compile_cmd = src_compile_cmd, - flavors = flavors, - flavor_flags = toolchain.compiler_flavor_flags, + src = src, + src_args = src_args, + cxx_compile_cmd = cxx_compile_cmd, + shared_args = shared_auxiliary_args, ) - src_compile_cmd.index_store_factory.compile( + shared_info.index_store_factory.compile( actions, label, index_store, @@ -668,16 +736,16 @@ def _compile_single_cxx( pch_flavor = flavor.value if flavor.value else pch_flavor target = pch_subtargets[pch_flavor].get(CPrecompiledHeaderInfo) - cmd.add(_get_use_pch_args(src_compile_cmd, target, precompiled_header[CPrecompiledHeaderInfo])) + cmd.add(_get_use_pch_args(src, cxx_compile_cmd, target, precompiled_header[CPrecompiledHeaderInfo])) action_dep_files = {} - headers_dep_files = src_compile_cmd.cxx_compile_cmd.headers_dep_files + headers_dep_files = cxx_compile_cmd.headers_dep_files - # Distributed NVCC compilation doesn't support dep files because we'll - # dryrun cmd and the dep files won't be materialized. - # TODO (T219249723): investigate if dep files are needed for dist nvcc. - if src_compile_cmd.src.extension == ".cu": + # CUDA compilation attaches dep files itself (see cuda_mono_compile and + # cuda_distributed_compile), because the flags have to land on the real + # compile sub-command rather than on the nvcc driver command we dryrun here. + if src.extension == ".cu": headers_dep_files = None if headers_dep_files: @@ -685,7 +753,7 @@ def _compile_single_cxx( actions, cmd, headers_dep_files, - src_compile_cmd.src, + src, filename_base, action_dep_files, ) @@ -727,7 +795,7 @@ def _compile_single_cxx( cmd, ) - if src_compile_cmd.src.extension == ".cu": + if src.extension == ".cu": expect(cuda_compile_style != None, "CUDA compile style should be configured for targets with .cu sources") cuda_compile_info = CudaCompileInfo( filename = filename_base, @@ -736,67 +804,62 @@ def _compile_single_cxx( uses_content_based_paths = content_based, ) - cuda_dist_output = None - if dist_cuda: - cuda_dist_output = CudaDistributedCompileOutput( - nvcc_dag = dist_cuda[0].as_input(), - nvcc_env = dist_cuda[1].as_input(), - hostcc_argsfile = dist_cuda[2].as_input(), - ) - cuda_compile( + cuda_dist_spec = cuda_compile( actions, - toolchain, cmd, object, - src_compile_cmd, + src, + cxx_compile_cmd, cuda_compile_info, action_dep_files, allow_dep_file_cache_upload = False, - error_handler = src_compile_cmd.error_handler, + error_handler = shared_info.error_handler, cuda_compile_style = cuda_compile_style, - cuda_dist_output = cuda_dist_output, + cuda_dist_output = dist_cuda, + cuda_prepare_cmd = cuda_prepare_cmd, + shared_plan_identifier = cuda_shared_plan_identifier, ) else: is_producing_compiled_pch = bool(compile_pch) is_consuming_compiled_pch = bool(precompiled_header and precompiled_header[CPrecompiledHeaderInfo].compiled) actions.run( cmd, - category = src_compile_cmd.cxx_compile_cmd.category, + category = cxx_compile_cmd.category, identifier = identifier, dep_files = action_dep_files, - allow_cache_upload = src_compile_cmd.cxx_compile_cmd.allow_cache_upload, + allow_cache_upload = cxx_compile_cmd.allow_cache_upload, allow_dep_file_cache_upload = False, - error_handler = src_compile_cmd.error_handler, + error_handler = shared_info.error_handler, outputs_for_error_handler = outputs_for_error_handler, local_only = is_producing_compiled_pch or is_consuming_compiled_pch, - expect_eligible_for_dedupe = src_compile_cmd.expect_eligible_for_dedupe, + expect_eligible_for_dedupe = shared_info.expect_eligible_for_dedupe, ) # Generate asm for compiler which accept `-S` (TODO: support others) if assembly: assembly_cmd = _get_base_compile_cmd( - bitcode_args = bitcode_args, - src_compile_cmd = src_compile_cmd, - flavors = flavors, - flavor_flags = toolchain.compiler_flavor_flags, + src = src, + src_args = src_args, + cxx_compile_cmd = cxx_compile_cmd, output_args = ["-S"] + get_output_flags(compiler_type, assembly), + shared_args = shared_auxiliary_args, ) actions.run( assembly_cmd, - category = src_compile_cmd.cxx_compile_cmd.category, + category = cxx_compile_cmd.category, identifier = identifier + " (assembly)", - allow_cache_upload = src_compile_cmd.cxx_compile_cmd.allow_cache_upload, + allow_cache_upload = cxx_compile_cmd.allow_cache_upload, allow_dep_file_cache_upload = False, - error_handler = src_compile_cmd.error_handler, + error_handler = shared_info.error_handler, ) if diagnostics: syntax_only_cmd = _get_base_compile_cmd( - bitcode_args = bitcode_args, - src_compile_cmd = src_compile_cmd, - flavors = flavors, - flavor_flags = toolchain.compiler_flavor_flags, + src = src, + src_args = src_args, + cxx_compile_cmd = cxx_compile_cmd, output_args = ["-fsyntax-only"], + shared_args = shared_auxiliary_args, ) diagnostics_dep_files = {} if headers_dep_files: @@ -804,7 +867,7 @@ def _compile_single_cxx( actions = actions, cmd = syntax_only_cmd, headers_dep_files = headers_dep_files, - src = src_compile_cmd.src, + src = src, filename_base = "{}.check".format(filename_base), action_dep_files = diagnostics_dep_files, ) @@ -817,9 +880,9 @@ def _compile_single_cxx( category = "check", identifier = short_path, dep_files = diagnostics_dep_files, - allow_cache_upload = src_compile_cmd.cxx_compile_cmd.allow_cache_upload, + allow_cache_upload = cxx_compile_cmd.allow_cache_upload, allow_dep_file_cache_upload = False, - error_handler = src_compile_cmd.error_handler, + error_handler = shared_info.error_handler, ) if clang_tidy_diagnostics: @@ -829,79 +892,117 @@ def _compile_single_cxx( toolchain.internal_tools.clang_tidy_wrapper, cmd_args(clang_tidy_diagnostics, format = "--output={}"), cmd_args(clang_tidy_tool, format = "--clang-tidy={}"), - cmd_args(src_compile_cmd.src, format = "--source={}"), + cmd_args(src, format = "--source={}"), "--", - src_compile_cmd.cxx_compile_cmd.base_compile_cmd, - src_compile_cmd.cxx_compile_cmd.argsfile.cmd_form, - src_compile_cmd.args, + cxx_compile_cmd.base_compile_cmd, + cxx_compile_cmd.argsfile.cmd_form, + src_args, ], category = "clang_tidy", identifier = short_path, - allow_cache_upload = src_compile_cmd.cxx_compile_cmd.allow_cache_upload, + allow_cache_upload = cxx_compile_cmd.allow_cache_upload, allow_dep_file_cache_upload = False, - error_handler = src_compile_cmd.error_handler, + error_handler = shared_info.error_handler, ) # Generate pre-processed sources preproc_cmd = _get_base_compile_cmd( - bitcode_args = bitcode_args, - src_compile_cmd = src_compile_cmd, - flavors = flavors, - flavor_flags = toolchain.compiler_flavor_flags, + src = src, + src_args = src_args, + cxx_compile_cmd = cxx_compile_cmd, output_args = [COMMON_PREPROCESSOR_OUTPUT_ARGS, get_output_flags(compiler_type, preproc)], + shared_args = shared_auxiliary_args, ) actions.run( preproc_cmd, - category = src_compile_cmd.cxx_compile_cmd.category, + category = cxx_compile_cmd.category, identifier = identifier + " (preprocessor)", - allow_cache_upload = src_compile_cmd.cxx_compile_cmd.allow_cache_upload, + allow_cache_upload = cxx_compile_cmd.allow_cache_upload, allow_dep_file_cache_upload = False, - error_handler = src_compile_cmd.error_handler, + error_handler = shared_info.error_handler, ) + return cuda_dist_spec + def _get_base_compile_cmd( - bitcode_args: cmd_args | list, - src_compile_cmd: CxxSrcCompileCommand, - flavors: set[CxxCompileFlavor], - flavor_flags: dict[str, list[str]], + src: Artifact, + src_args: list[typing.Any], + cxx_compile_cmd: CxxCompileCommand, + shared_args: CxxSharedCompileCommandArgs, output_args: list | None = None, - use_header_units: UseHeaderUnitsMode = UseHeaderUnitsMode("none"), + base_compile_cmd_override = None, + source_override: str | None = None, ) -> cmd_args: """ - Construct a shared compile command for a single CXX source based on - `src_compile_command` and other compilation options. + Construct a compile command for a single CXX source from its + `cxx_compile_cmd`, the extension-shared `shared_args`, and other + compilation options. """ - cmd = cmd_args(src_compile_cmd.cxx_compile_cmd.base_compile_cmd) + cmd = cmd_args(base_compile_cmd_override if base_compile_cmd_override != None else cxx_compile_cmd.base_compile_cmd) if output_args: cmd.add(output_args) - if use_header_units == UseHeaderUnitsMode("pcm") and src_compile_cmd.cxx_compile_cmd.header_units_argsfile: - cmd.add(src_compile_cmd.cxx_compile_cmd.header_units_argsfile.cmd_form) - elif use_header_units == UseHeaderUnitsMode("stub") and src_compile_cmd.cxx_compile_cmd.header_unit_stubs_argsfile: - cmd.add(cmd_args(hidden = src_compile_cmd.cxx_compile_cmd.header_unit_stubs_argsfile.file)) + cmd.add(shared_args.before_src) + + if source_override == None: + cmd.add(src_args) + else: + expect( + src_args and src_args[-1] == src, + "source override requires the source artifact to be the final compile argument", + ) + cmd.add(src_args[:-1]) + cmd.add(source_override) + + cmd.add(shared_args.after_src) - cmd.add(src_compile_cmd.cxx_compile_cmd.argsfile.cmd_form) + return cmd + +def _get_shared_compile_command_args( + bitcode_args: cmd_args | list, + cxx_compile_cmd: CxxCompileCommand, + flavors: set[CxxCompileFlavor], + flavor_flags: dict[str, list[str]], + use_header_units: UseHeaderUnitsMode = UseHeaderUnitsMode("none"), + cuda_dryrun: bool = False, +) -> CxxSharedCompileCommandArgs: + before_src = cmd_args() + + if use_header_units == UseHeaderUnitsMode("pcm") and cxx_compile_cmd.header_units_argsfile: + before_src.add(cxx_compile_cmd.header_units_argsfile.cmd_form) + elif use_header_units == UseHeaderUnitsMode("stub") and cxx_compile_cmd.header_unit_stubs_argsfile: + before_src.add(cmd_args(hidden = cxx_compile_cmd.header_unit_stubs_argsfile.file)) + + if cuda_dryrun: + dryrun_form = cxx_compile_cmd.argsfile.cmd_form_cuda_dryrun + expect(dryrun_form != None, "compile argsfile has no CUDA dry-run form") + before_src.add(dryrun_form) + else: + before_src.add(cxx_compile_cmd.argsfile.cmd_form) for flavor in flavors: flags = flavor_flags.get(flavor.value) if flags: - cmd.add(flags) - - cmd.add(src_compile_cmd.args) + before_src.add(flags) - cmd.add(bitcode_args) - - return cmd + return CxxSharedCompileCommandArgs( + before_src = before_src, + after_src = cmd_args(bitcode_args), + ) def toolchain_supports_flavor(toolchain: CxxToolchainInfo, flavor: CxxCompileFlavor) -> bool: return flavor.value in toolchain.supported_compile_flavors and toolchain.compiler_flavor_flags.get(flavor.value) != None -def build_flavor_flags(flavor_flags: dict[str, list[str]], compiler_type: str) -> dict[str, list[str]]: +# Fallback when the toolchain configures no flavor flags at all, for clang and +# gcc. Ideally this would be a default value in the toolchain definition itself, +# but it is compiler-dependent so we can't have it there. Module-level so every +# per-source compile record shares one frozen value instead of allocating a +# fresh dict+list per TU. +_DEFAULT_PIC_FLAVOR_FLAGS = {CxxCompileFlavor("pic").value: ["-fPIC"]} + +def build_flavor_flags(flavor_flags: dict[str, typing.Any], compiler_type: str) -> dict[str, typing.Any]: if not flavor_flags and compiler_type in ["clang", "gcc"]: - # If there are no configured flavor flags for toolchain at all - # we fallback to default fPIC for clang and gcc. Ideally this should be default - # value in toolchain definition itself, but it is compiler-dependent so we can't have it there. - return {CxxCompileFlavor("pic").value: ["-fPIC"]} + return _DEFAULT_PIC_FLAVOR_FLAGS return flavor_flags @@ -916,20 +1017,22 @@ def _cxx_dynamic_compile( compile_pch: CxxPrecompiledHeader | None, cuda_compile_style: CudaCompileStyle | None, infos: list[CxxCompileInfo], + shared_infos: dict[str, CxxSharedCompileInfo], object: list[OutputArtifact], - external_debug_info: list[OutputArtifact | None], - clang_remarks: list[OutputArtifact | None], - clang_llvm_statistics: list[OutputArtifact | None], - clang_trace: list[OutputArtifact | None], - gcno_file: list[OutputArtifact | None], + external_debug_info: dict[int, OutputArtifact], + clang_remarks: dict[int, OutputArtifact], + clang_llvm_statistics: dict[int, OutputArtifact], + clang_trace: dict[int, OutputArtifact], + gcno_file: dict[int, OutputArtifact], assembly: list[OutputArtifact | None], diagnostics: list[OutputArtifact | None], - clang_tidy_diagnostics: list[OutputArtifact | None], + clang_tidy_diagnostics: dict[int, OutputArtifact], preproc: list[OutputArtifact], - index_store: list[OutputArtifact | None], - dist_cuda: list[None | (OutputArtifact, OutputArtifact, OutputArtifact)], - pch_object: list[OutputArtifact | None], - json_error: list[OutputArtifact | None], + index_store: dict[int, OutputArtifact], + dist_cuda: dict[int, (OutputArtifact, OutputArtifact, OutputArtifact)], + shared_dist_cuda: (OutputArtifact, OutputArtifact, OutputArtifact) | None, + pch_object: dict[int, OutputArtifact], + json_error: dict[int, OutputArtifact], ) -> list[Provider]: """ DYNAMIC ACTION CALLBACK: The bridge between declaration and execution phases. @@ -961,35 +1064,125 @@ def _cxx_dynamic_compile( one per source file. """ flavors_set = set(flavors) + shared_cuda_specs = [] + shared_cuda_dist_output = None + shared_cuda_plan_src_idx = None + if shared_dist_cuda != None: + shared_cuda_dist_output = CudaDistributedCompileOutput( + nvcc_dag = shared_dist_cuda[0].as_input(), + nvcc_env = shared_dist_cuda[1].as_input(), + hostcc_argsfile = shared_dist_cuda[2].as_input(), + ) + cuda_shared_plan_identifier = _cuda_plan_identifier(flavors_set) if shared_cuda_dist_output != None else None + shared_compile_args = {} + shared_auxiliary_args = {} + shared_dryrun_args = {} + for extension, shared_info in shared_infos.items(): + flavor_flags = build_flavor_flags(toolchain.compiler_flavor_flags, shared_info.cxx_compile_cmd.compiler_type) + shared_compile_args[extension] = _get_shared_compile_command_args( + bitcode_args = bitcode_args, + cxx_compile_cmd = shared_info.cxx_compile_cmd, + flavors = flavors_set, + flavor_flags = flavor_flags, + use_header_units = use_header_units, + ) + shared_auxiliary_args[extension] = _get_shared_compile_command_args( + bitcode_args = bitcode_args, + cxx_compile_cmd = shared_info.cxx_compile_cmd, + flavors = flavors_set, + flavor_flags = toolchain.compiler_flavor_flags, + ) + if extension == ".cu" and cuda_compile_style == CudaCompileStyle("dist"): + shared_dryrun_args[extension] = _get_shared_compile_command_args( + bitcode_args = bitcode_args, + cxx_compile_cmd = shared_info.cxx_compile_cmd, + flavors = flavors_set, + flavor_flags = flavor_flags, + use_header_units = use_header_units, + cuda_dryrun = True, + ) for i in range(len(infos)): - _compile_single_cxx( + extension = infos[i].src.extension + is_cuda = extension == ".cu" + + # Exactly one prepare action must bind the shared plan outputs. Any + # shared CUDA source is a valid representative, since sharing requires + # identical plan-affecting args across sources; use the first one. + if is_cuda and shared_cuda_dist_output != None and shared_cuda_plan_src_idx == None: + shared_cuda_plan_src_idx = i + source_dist_cuda = shared_cuda_dist_output if is_cuda else None + source_dist_cuda_outputs = dist_cuda.get(i) + if source_dist_cuda == None and source_dist_cuda_outputs != None: + source_dist_cuda = CudaDistributedCompileOutput( + nvcc_dag = source_dist_cuda_outputs[0].as_input(), + nvcc_env = source_dist_cuda_outputs[1].as_input(), + hostcc_argsfile = source_dist_cuda_outputs[2].as_input(), + ) + prepare_cuda_dist = source_dist_cuda != None and (shared_cuda_dist_output == None or i == shared_cuda_plan_src_idx) + cuda_dist_spec = _compile_single_cxx( actions = actions, label = label, toolchain = toolchain, - bitcode_args = bitcode_args, flavors = flavors_set, compile_pch = compile_pch, precompiled_header = precompiled_header, cuda_compile_style = cuda_compile_style, - use_header_units = use_header_units, info = infos[i], + shared_info = shared_infos[extension], object = object[i], - external_debug_info = external_debug_info[i], - clang_remarks = clang_remarks[i], - clang_llvm_statistics = clang_llvm_statistics[i], - clang_trace = clang_trace[i], - gcno_file = gcno_file[i], + external_debug_info = external_debug_info.get(i), + clang_remarks = clang_remarks.get(i), + clang_llvm_statistics = clang_llvm_statistics.get(i), + clang_trace = clang_trace.get(i), + gcno_file = gcno_file.get(i), assembly = assembly[i], diagnostics = diagnostics[i], - clang_tidy_diagnostics = clang_tidy_diagnostics[i], + clang_tidy_diagnostics = clang_tidy_diagnostics.get(i), preproc = preproc[i], - index_store = index_store[i], - dist_cuda = dist_cuda[i], - pch_object = pch_object[i], - json_error = json_error[i], + index_store = index_store.get(i), + dist_cuda = source_dist_cuda, + prepare_cuda_dist = prepare_cuda_dist, + cuda_shared_plan_identifier = cuda_shared_plan_identifier, + shared_compile_args = shared_compile_args[extension], + shared_auxiliary_args = shared_auxiliary_args[extension], + shared_dryrun_args = shared_dryrun_args.get(extension), + pch_object = pch_object.get(i), + json_error = json_error.get(i), + ) + if cuda_dist_spec != None: + if shared_cuda_dist_output != None: + shared_cuda_specs.append(cuda_dist_spec) + else: + cxx_compile_cmd = shared_infos[extension].cxx_compile_cmd + create_cuda_distributed_compiles( + actions, + toolchain, + source_dist_cuda, + [cuda_dist_spec], + cxx_compile_cmd.allow_cache_upload, + # The argsfile closure carries the file-prefix specs file + # that replayed host compiles reference by path. + cmd_args(cxx_compile_cmd.base_compile_cmd, hidden = cxx_compile_cmd.argsfile.cmd_form), + cxx_compile_cmd.headers_dep_files, + ) + + if shared_cuda_specs: + cxx_compile_cmd = shared_infos[".cu"].cxx_compile_cmd + expect( + shared_cuda_plan_src_idx != None, + "shared CUDA plan outputs were declared but no .cu source is present to bind them", + ) + create_cuda_distributed_compiles( + actions, + toolchain, + shared_cuda_dist_output, + shared_cuda_specs, + cxx_compile_cmd.allow_cache_upload, + cmd_args(cxx_compile_cmd.base_compile_cmd, hidden = cxx_compile_cmd.argsfile.cmd_form), + cxx_compile_cmd.headers_dep_files, ) - return [DefaultInfo()] + return [EMPTY_DEFAULT_INFO] # https://buck2.build/docs/api/build/AnalysisActions/#analysisactionsdynamic_output_new # Dynamic actions factory for batch CXX compilation @@ -997,31 +1190,73 @@ _dynamic_compile_rule = dynamic_actions( impl = _cxx_dynamic_compile, attrs = { "assembly": dynattrs.list(dynattrs.option(dynattrs.output())), - "bitcode_args": dynattrs.list(dynattrs.value(str)), - "clang_llvm_statistics": dynattrs.list(dynattrs.option(dynattrs.output())), - "clang_remarks": dynattrs.list(dynattrs.option(dynattrs.output())), - "clang_tidy_diagnostics": dynattrs.list(dynattrs.option(dynattrs.output())), - "clang_trace": dynattrs.list(dynattrs.option(dynattrs.output())), + "bitcode_args": dynattrs.value(list[str]), + "clang_llvm_statistics": dynattrs.dict(int, dynattrs.output()), + "clang_remarks": dynattrs.dict(int, dynattrs.output()), + "clang_tidy_diagnostics": dynattrs.dict(int, dynattrs.output()), + "clang_trace": dynattrs.dict(int, dynattrs.output()), "compile_pch": dynattrs.option(dynattrs.value(CxxPrecompiledHeader)), "cuda_compile_style": dynattrs.option(dynattrs.value(CudaCompileStyle)), "diagnostics": dynattrs.list(dynattrs.option(dynattrs.output())), - "dist_cuda": dynattrs.list(dynattrs.option(dynattrs.tuple(dynattrs.output(), dynattrs.output(), dynattrs.output()))), - "external_debug_info": dynattrs.list(dynattrs.option(dynattrs.output())), - "flavors": dynattrs.list(dynattrs.value(CxxCompileFlavor)), - "gcno_file": dynattrs.list(dynattrs.option(dynattrs.output())), - "index_store": dynattrs.list(dynattrs.option(dynattrs.output())), - "infos": dynattrs.list(dynattrs.value(CxxCompileInfo)), - "json_error": dynattrs.list(dynattrs.option(dynattrs.output())), + "dist_cuda": dynattrs.dict(int, dynattrs.tuple(dynattrs.output(), dynattrs.output(), dynattrs.output())), + "external_debug_info": dynattrs.dict(int, dynattrs.output()), + "flavors": dynattrs.value(list[CxxCompileFlavor]), + "gcno_file": dynattrs.dict(int, dynattrs.output()), + "index_store": dynattrs.dict(int, dynattrs.output()), + "infos": dynattrs.value(list[CxxCompileInfo]), + "json_error": dynattrs.dict(int, dynattrs.output()), "label": dynattrs.value(Label), "object": dynattrs.list(dynattrs.output()), - "pch_object": dynattrs.list(dynattrs.option(dynattrs.output())), + "pch_object": dynattrs.dict(int, dynattrs.output()), "precompiled_header": dynattrs.option(dynattrs.value(Dependency)), "preproc": dynattrs.list(dynattrs.output()), + "shared_dist_cuda": dynattrs.option(dynattrs.tuple(dynattrs.output(), dynattrs.output(), dynattrs.output())), + "shared_infos": dynattrs.dict(str, dynattrs.value(CxxSharedCompileInfo)), "toolchain": dynattrs.value(CxxToolchainInfo), "use_header_units": dynattrs.value(UseHeaderUnitsMode), }, ) +def _cuda_plan_flavor(flavors: set[CxxCompileFlavor]) -> str: + return ".".join(sorted([flavor.value for flavor in flavors])) or "default" + +def _cuda_plan_identifier(flavors: set[CxxCompileFlavor]) -> str: + return "cuda_compile_plan ({})".format(_cuda_plan_flavor(flavors)) + +def maybe_get_shared_cuda_dist_output( + actions: AnalysisActions, + src_compile_cmds: list[CxxSrcCompileCommand], + cuda_compile_style: CudaCompileStyle | None, + flavors: set[CxxCompileFlavor], +) -> CudaDistributedCompileOutput | None: + cuda_src_compile_cmds = [cmd for cmd in src_compile_cmds if cmd.src.extension == ".cu"] + if cuda_compile_style != CudaCompileStyle("dist") or not cuda_src_compile_cmds: + return None + + for cmd in cuda_src_compile_cmds: + # Same-extension sources already share one `cxx_compile_cmd`, so per-source + # args, duplicate-basename disambiguation (`index`), and path style are the + # only sharing hazards. + # create_compile_cmds appends source flags before "-c" and the source artifact. + # Only commands with no source-specific flags can share an NVCC plan. + if len(cmd.args) != 2 or cmd.args[0] != "-c" or cmd.args[1] != cmd.src or cmd.index != None: + return None + if cmd.uses_content_based_paths != cuda_src_compile_cmds[0].uses_content_based_paths: + return None + + # Plan outputs are named per flavor set: callers invoke compile_cxx at most + # once per flavor set within a target, and a collision fails analysis loudly. + cuda_plan_flavor = _cuda_plan_flavor(flavors) + return declare_cuda_dist_compile_output( + actions, + CudaCompileInfo( + filename = "cuda_compile_plan.{}".format(cuda_plan_flavor), + identifier = _cuda_plan_identifier(flavors), + output_prefix = _OBJECTS_FOLDER, + uses_content_based_paths = cuda_src_compile_cmds[0].uses_content_based_paths, + ), + ) + def compile_cxx( actions: AnalysisActions, target_label: Label, @@ -1062,24 +1297,51 @@ def compile_cxx( objects = [] - # Lists for dynamic action inputs + # Dynamic action inputs infos = [] object_outputs = [] - external_debug_info_outputs = [] - clang_remarks_outputs = [] - clang_llvm_statistics_outputs = [] - clang_trace_outputs = [] - gcno_file_outputs = [] + external_debug_info_outputs = {} + clang_remarks_outputs = {} + clang_llvm_statistics_outputs = {} + clang_trace_outputs = {} + gcno_file_outputs = {} assembly_outputs = [] diagnostics_outputs = [] - clang_tidy_diagnostics_outputs = [] + clang_tidy_diagnostics_outputs = {} preproc_outputs = [] - index_store_outputs = [] - dist_cuda_outputs = [] - pch_object_outputs = [] - json_error_outputs = [] + index_store_outputs = {} + dist_cuda_outputs = {} + pch_object_outputs = {} + json_error_outputs = {} + shared_infos = {} for src_compile_cmd in src_compile_cmds: + extension = src_compile_cmd.src.extension + shared_info = shared_infos.get(extension) + if shared_info == None: + shared_infos[extension] = CxxSharedCompileInfo( + cxx_compile_cmd = src_compile_cmd.cxx_compile_cmd, + index_store_factory = src_compile_cmd.index_store_factory, + error_handler = src_compile_cmd.error_handler, + expect_eligible_for_dedupe = src_compile_cmd.expect_eligible_for_dedupe, + ) + else: + expect( + src_compile_cmd.cxx_compile_cmd == shared_info.cxx_compile_cmd + and src_compile_cmd.index_store_factory == shared_info.index_store_factory + and src_compile_cmd.error_handler == shared_info.error_handler + and src_compile_cmd.expect_eligible_for_dedupe == shared_info.expect_eligible_for_dedupe, + "sources with the same extension must share compile command inputs", + ) + + shared_cuda_dist_output = maybe_get_shared_cuda_dist_output( + actions, + src_compile_cmds, + cuda_compile_style, + flavors, + ) + + for source_index, src_compile_cmd in enumerate(src_compile_cmds): cxx_compile_input = _prepare_cxx_compilation( actions = actions, toolchain = toolchain, @@ -1090,6 +1352,7 @@ def compile_cxx( use_header_units = use_header_units, separate_debug_info = separate_debug_info, cuda_compile_style = cuda_compile_style, + shared_cuda_dist_output = shared_cuda_dist_output, compile_pch = compile_pch, ) @@ -1099,19 +1362,33 @@ def compile_cxx( # Collect outputs - call .as_output() on each artifact declared = cxx_compile_input.declared_artifacts object_outputs.append(declared.object.as_output()) - external_debug_info_outputs.append(map_val(as_output, declared.external_debug_info)) - clang_remarks_outputs.append(map_val(as_output, declared.clang_remarks)) - clang_llvm_statistics_outputs.append(map_val(as_output, declared.clang_llvm_statistics)) - clang_trace_outputs.append(map_val(as_output, declared.clang_trace)) - gcno_file_outputs.append(map_val(as_output, declared.gcno_file)) + if declared.external_debug_info: + external_debug_info_outputs[source_index] = declared.external_debug_info.as_output() + if declared.clang_remarks: + clang_remarks_outputs[source_index] = declared.clang_remarks.as_output() + if declared.clang_llvm_statistics: + clang_llvm_statistics_outputs[source_index] = declared.clang_llvm_statistics.as_output() + if declared.clang_trace: + clang_trace_outputs[source_index] = declared.clang_trace.as_output() + if declared.gcno_file: + gcno_file_outputs[source_index] = declared.gcno_file.as_output() assembly_outputs.append(map_val(as_output, declared.assembly)) diagnostics_outputs.append(map_val(as_output, declared.diagnostics)) - clang_tidy_diagnostics_outputs.append(map_val(as_output, declared.clang_tidy_diagnostics)) + if declared.clang_tidy_diagnostics: + clang_tidy_diagnostics_outputs[source_index] = declared.clang_tidy_diagnostics.as_output() preproc_outputs.append(declared.preproc.as_output()) - index_store_outputs.append(map_val(as_output, declared.index_store)) - dist_cuda_outputs.append(map_val(lambda d: (d.nvcc_dag.as_output(), d.nvcc_env.as_output(), d.hostcc_argsfile.as_output()), declared.dist_cuda)) - pch_object_outputs.append(map_val(as_output, declared.pch_object)) - json_error_outputs.append(map_val(as_output, declared.json_error)) + if declared.index_store: + index_store_outputs[source_index] = declared.index_store.as_output() + if shared_cuda_dist_output == None and declared.dist_cuda: + dist_cuda_outputs[source_index] = ( + declared.dist_cuda.nvcc_dag.as_output(), + declared.dist_cuda.nvcc_env.as_output(), + declared.dist_cuda.hostcc_argsfile.as_output(), + ) + if declared.pch_object: + pch_object_outputs[source_index] = declared.pch_object.as_output() + if declared.json_error: + json_error_outputs[source_index] = declared.json_error.as_output() objects.append(declared) @@ -1138,6 +1415,8 @@ def compile_cxx( pch_object = pch_object_outputs, precompiled_header = precompiled_header, preproc = preproc_outputs, + shared_dist_cuda = map_val(lambda d: (d.nvcc_dag.as_output(), d.nvcc_env.as_output(), d.hostcc_argsfile.as_output()), shared_cuda_dist_output), + shared_infos = shared_infos, toolchain = toolchain, use_header_units = use_header_units, ) @@ -1654,11 +1933,18 @@ def _get_category(ext: CxxExtension) -> str: # This should be unreachable as long as we handle all enum values fail("Unknown extension: " + ext.value) -def _get_compile_base(toolchain: CxxToolchainInfo, compiler_info: typing.Any, use_wrapper) -> cmd_args: +def _get_compile_base(toolchain: CxxToolchainInfo, compiler_info: typing.Any, use_wrapper, compiler_override = None) -> cmd_args: """ Given a compiler info returned by _get_compiler_info, form the base compile args. + + `compiler_override` substitutes the compiler executable while keeping the + toolchain-level wrapping (e.g. cwd remapping) that all compiles of this + compiler type must run under. """ - compiler = compiler_info.compiler_with_wrapper if compiler_info.compiler_with_wrapper and use_wrapper else compiler_info.compiler + if compiler_override != None: + compiler = compiler_override + else: + compiler = compiler_info.compiler_with_wrapper if compiler_info.compiler_with_wrapper and use_wrapper else compiler_info.compiler if toolchain.remap_cwd and compiler_info.compiler_type in ["clang", "clang_windows", "clang_cl", "gcc"]: return cmd_args(toolchain.internal_tools.remap_cwd, compiler) @@ -1699,6 +1985,60 @@ def _add_compiler_info_flags(compiler_info: typing.Any) -> list: return cmd +def mk_toolchain_precompile_argsfile(actions: AnalysisActions, name: str, argsfile: cmd_args, filter_argsfile: RunInfo) -> cmd_args: + """ + Copy a toolchain argsfile with flags that are invalid for C++20 module + precompilation filtered out. One filter action per toolchain, run only if + a precompile demands it. `argsfile` is an `argsfile_with_artifacts()` + value; the filtered copy carries everything the original carried. + """ + filtered = actions.declare_output(name + "_toolchain_args_precompile", has_content_based_path = True) + actions.run( + [filter_argsfile, _PRECOMPILE_OPTION_IGNORE_REGEX_STR, argsfile, filtered.as_output()], + category = "filter_modules_precompile_argsfile", + identifier = name, + ) + return argsfile_with_artifacts(filtered, argsfile) + +def compiler_info_with_toolchain_argsfiles( + actions: AnalysisActions, name: str, ctor: typing.Callable, compiler_info: typing.Any, precompile_filter: RunInfo | None = None +) -> typing.Any: + """ + Given a compiler_info that can have a new version made with ctor, this + will generate new `argsfile`/`argsfile_xcode` containing the compiler_info's + preprocessor and compiler flags. Compile commands use these per-toolchain + files, so targets using the toolchain don't each write their own + `{ext}.toolchain_cxx_args` copy of the same flags. With `precompile_filter`, + also generate `argsfile_precompile`. + """ + if compiler_info == None: + return None + is_nasm = compiler_info.compiler_type == "nasm" + flags = _add_compiler_info_flags(compiler_info) + content_based = bool(compiler_info.supports_content_based_paths) + argsfile = _mk_argsfile( + actions = actions, + name = name + "_toolchain_args", + args = create_cmd_args(is_nasm, False, flags), + has_content_based_path = content_based, + ) + argsfile_xcode = _mk_argsfile( + actions = actions, + name = name + "_toolchain_args_xcode", + args = create_cmd_args(is_nasm, True, flags), + has_content_based_path = content_based, + ) + argsfile_precompile = None + if precompile_filter != None: + argsfile_precompile = mk_toolchain_precompile_argsfile(actions, name, argsfile, precompile_filter) + return compiler_info_with_argsfiles( + compiler_info, + ctor, + argsfile, + argsfile_xcode, + argsfile_precompile, + ) + def _add_compiler_type_flags(target_label: Label, compiler_type: str, ext: CxxExtension) -> list: cmd = [] cmd.append(get_flags_for_reproducible_build(target_label, compiler_type)) @@ -1767,6 +2107,49 @@ def _filter_precompile_args(args: list[typing.Any]) -> list[typing.Any]: [_filter_precompile_args(arg) if type(arg) == type([]) else arg if not should_ignore(arg) else None for arg in args], ) +def _cxx_flags_args(flags, ext: str, is_precompile: bool): + if not is_precompile: + return flags.project_as_args( + cxx_flags_projection_name(ext), + ordering = "postorder", + ) + + values = flags.traverse(ordering = "postorder") + args = [ + [ + value.preprocessor_flags, + cxx_by_language_ext(value.lang_preprocessor_flags, ext), + cxx_by_language_ext(value.lang_compiler_flags, ext), + value.compiler_flags, + ] + for value in values + ] + return _filter_precompile_args(args) + +def _cxx_flags_argsfile_anon_impl(ctx: AnalysisContext): + flags = cxx_flags_tset(ctx.actions, ctx.attrs.flags) + if flags == None: + fail("cxx flags argsfile requires at least one `cxx_flags` target") + args = _cxx_flags_args(flags, ctx.attrs.src_extension, ctx.attrs.is_precompile) + content = create_cmd_args(ctx.attrs.is_nasm, ctx.attrs.is_xcode_argsfile, args) + argsfile, _ = ctx.actions.write("cxx_flags_args", content, allow_args = True, has_content_based_path = True) + return [DefaultInfo(default_outputs = [argsfile])] + +_cxx_flags_argsfile_anon_rule = anon_rule( + impl = _cxx_flags_argsfile_anon_impl, + attrs = { + "flags": attrs.list(attrs.dep(providers = [CxxFlagsInfo])), + "is_nasm": attrs.bool(), + "is_precompile": attrs.bool(), + "is_xcode_argsfile": attrs.bool(), + "src_extension": attrs.string(), + }, + artifact_promise_mappings = { + "argsfile": lambda x: x[DefaultInfo].default_outputs[0], + }, + doc = "Creates a shared argsfile from one or more nested `cxx_flags` targets.", +) + def _filter_precompile_argsfile_anon_impl(ctx: AnalysisContext): argsfile = ctx.actions.declare_output("filtered_args", has_content_based_path = True) ctx.actions.run( @@ -1833,8 +2216,8 @@ def _mk_argsfiles( ) return argsfile - def write_for_dep_file_filtering(filename: str, content, absolute: bool = False) -> (Artifact, Artifact): - # We write two versions of the file: + def write_for_dep_file_filtering(filename: str, content, absolute: bool = False) -> (Artifact, Artifact | None): + # Dep-file filtering needs two versions of the file: # - The first is a tagged file with the optionally content-based paths, which # will be passed to the compiler as usual, but will NOT be marked used, so # whether the paths in it change or not, the action will NOT rerun. @@ -1850,6 +2233,11 @@ def _mk_argsfiles( absolute = absolute, has_content_based_path = uses_content_based_paths, ) + # Precompile actions do not filter inputs with dep files, so the real + # argsfile already tracks flag changes. + if is_precompile: + return (output, None) + redacted_output, _ = actions.write( filename + "_redacted", content, @@ -1860,7 +2248,7 @@ def _mk_argsfiles( ) return (output, redacted_output) - def mk_argsfile_for_dep_file_filtering(filename: str, args, absolute: bool = False) -> (Artifact, Artifact): + def mk_argsfile_for_dep_file_filtering(filename: str, args, absolute: bool = False) -> (Artifact, Artifact | None): return write_for_dep_file_filtering( filename, create_cmd_args(is_nasm, is_xcode_argsfile, args), @@ -1870,8 +2258,34 @@ def _mk_argsfiles( def make_toolchain_argsfile(): compiler_info_flags = _add_compiler_info_flags(compiler_info) - # Use the argsfile from the compiler info if it exists. - if compiler_info.argsfile and not is_xcode_argsfile: + if is_precompile: + if impl_params._cxx_toolchain == None: + # TODO(nml): Compared to get_cxx_toolchain_info(), we don't support + # AppleToolchain for C++20 modules. Update this if that changes. + fail("C++20 modules are not supported for AppleToolchain") + + if compiler_info.argsfile_precompile and not is_xcode_argsfile: + compiler_info_argsfile = compiler_info.argsfile_precompile + else: + # The anon filter takes an `attrs.source()` argsfile, so write + # a per-target copy of the flags for it. + unfiltered_argsfile = mk_argsfile(filename_prefix + "toolchain_cxx_args", compiler_info_flags) + filtered_info_argsfile = actions.anon_target( + _filter_precompile_argsfile_anon_rule, + { + "allow_cache_upload": impl_params.allow_cache_upload, + "src": unfiltered_argsfile, + "_cxx_toolchain": impl_params._cxx_toolchain, + }, + ).artifact("argsfile") + + # The promise artifact doesn't carry the artifacts the flags + # reference forward automatically for us; re-pair them. + compiler_info_argsfile = argsfile_with_artifacts( + actions.assert_has_content_based_path(filtered_info_argsfile), + compiler_info_flags, + ) + elif compiler_info.argsfile and not is_xcode_argsfile: compiler_info_argsfile = compiler_info.argsfile elif compiler_info.argsfile_xcode and is_xcode_argsfile: compiler_info_argsfile = compiler_info.argsfile_xcode @@ -1880,22 +2294,6 @@ def _mk_argsfiles( compiler_info_filename = filename_prefix + "toolchain_cxx_args" compiler_info_argsfile = mk_argsfile(compiler_info_filename, compiler_info_flags) - if is_precompile: - if impl_params._cxx_toolchain == None: - # TODO(nml): Compared to get_cxx_toolchain_info(), we don't support - # AppleToolchain for C++20 modules. Update this if that changes. - fail("C++20 modules are not supported for AppleToolchain") - - filtered_info_argsfile = actions.anon_target( - _filter_precompile_argsfile_anon_rule, - { - "allow_cache_upload": impl_params.allow_cache_upload, - "src": compiler_info_argsfile, - "_cxx_toolchain": impl_params._cxx_toolchain, - }, - ).artifact("argsfile") - compiler_info_argsfile = actions.assert_has_content_based_path(filtered_info_argsfile) - argsfiles.append(compiler_info_argsfile) args_list.append(compiler_info_flags) @@ -1924,6 +2322,39 @@ def _mk_argsfiles( make_compiler_type_argsfile() + def make_cxx_flags_argsfile(): + flag_deps = impl_params.cxx_flags + if not flag_deps: + return + + flags = cxx_flags_tset(actions, flag_deps) + projection = flags.project_as_args( + cxx_flags_projection_name(ext.value), + ordering = "postorder", + ) + if impl_params.anon_targets_allowed: + argsfile = actions.anon_target( + _cxx_flags_argsfile_anon_rule, + { + "flags": flag_deps, + "is_nasm": is_nasm, + "is_precompile": is_precompile, + "is_xcode_argsfile": is_xcode_argsfile, + "src_extension": ext.value, + }, + ).artifact("argsfile") + argsfile = actions.assert_has_content_based_path(argsfile) + else: + argsfile = mk_argsfile( + filename_prefix + "cxx_flags_args", + _cxx_flags_args(flags, ext.value, is_precompile), + ) + + argsfiles.append(argsfile_with_artifacts(argsfile, projection)) + args_list.append(projection) + + make_cxx_flags_argsfile() + def make_deps_argsfile(): deps_args = [] if is_precompile: @@ -1942,7 +2373,7 @@ def _mk_argsfiles( argsfiles.append( cmd_args( headers_tag.tag_artifacts(deps_argsfile_for_compiler), - hidden = deps_argsfile_for_buck_action_rerun, + hidden = [deps_argsfile_for_buck_action_rerun] if deps_argsfile_for_buck_action_rerun != None else [], ) ) args_list.extend(deps_args) @@ -1983,6 +2414,9 @@ def _mk_argsfiles( # Create a copy of the args so that we can continue to modify it later. args_without_file_prefix_args = cmd_args(args_list) + # Holder, because the closure below cannot bind an enclosing name. + file_prefix_specs = [] + def make_file_prefix_argsfile(): if is_precompile: # The precompile_args field overrides these. @@ -1993,21 +2427,23 @@ def _mk_argsfiles( # driver (as_compiler_info) and do accept these flags. return + if not preprocessor.set.reduce("has_file_prefix_args"): + return + # Put file_prefix_args in argsfile, make sure they do not appear when evaluating $(cxxppflags) # to avoid "argument too long" errors file_prefix_args = headers_tag.tag_artifacts(preprocessor.set.project_as_args("file_prefix_args")) - if compiler_info.compiler_type == "clang": - if ext.value == ".cu": - # NVCC passes argsfile flags to device sub-tools (cicc, ptxas) which don't - # understand -ffile-prefix-map. - file_prefix_args = cmd_args(file_prefix_args, replace_regex = ("-ffile-prefix-map=", "-fdebug-prefix-map=")) - else: - # -fcoverage-prefix-map is only supported by clang. Include coverage prefix args - # only for clang to avoid breaking non-clang compilers (e.g. GCC for CUDA) that - # receive flags from dependencies via the preprocessor set. - coverage_prefix_args = headers_tag.tag_artifacts(preprocessor.set.project_as_args("coverage_prefix_args")) - file_prefix_args = cmd_args(file_prefix_args, coverage_prefix_args) + if ext.value == ".cu": + # NVCC passes argsfile flags to device sub-tools (cicc, ptxas) which don't + # understand -ffile-prefix-map. + file_prefix_args = cmd_args(file_prefix_args, replace_regex = ("-ffile-prefix-map=", "-fdebug-prefix-map=")) + elif compiler_info.compiler_type == "clang": + # -fcoverage-prefix-map is only supported by clang. Include coverage prefix args + # only for clang to avoid breaking non-clang compilers that receive flags from + # dependencies via the preprocessor set. + coverage_prefix_args = headers_tag.tag_artifacts(preprocessor.set.project_as_args("coverage_prefix_args")) + file_prefix_args = cmd_args(file_prefix_args, coverage_prefix_args) file_prefix_args_filename = filename_prefix + "file_prefix_cxx_args" @@ -2035,6 +2471,7 @@ def _mk_argsfiles( format = "-specs={}", hidden = specs_file_for_buck_action_rerun, ) + file_prefix_specs.append(specs_file) else: prefix_ref = file_prefix_args @@ -2071,18 +2508,31 @@ def _mk_argsfiles( args = create_cmd_args(is_nasm, is_xcode_argsfile, args_list) input_args = [args, file_args] + if argsfile_for_buck_action_rerun != None: + input_args.append(argsfile_for_buck_action_rerun) + argsfile_format = "-@{}" if is_nasm else "@{}" cmd_form = cmd_args( headers_tag.tag_artifacts(argsfile), - format = "-@{}" if is_nasm else "@{}", - hidden = input_args + [argsfile_for_buck_action_rerun], + format = argsfile_format, + hidden = input_args, ) + # The NVCC dry run expands the flags and reads header maps, nothing else behind them. + cmd_form_cuda_dryrun = None + if ext.value == ".cu": + cmd_form_cuda_dryrun = cmd_args( + argsfile, + format = argsfile_format, + hidden = [file_args, preprocessor.set.project_as_args("header_maps")], + ) return CompileArgsfile( file = argsfile, cmd_form = cmd_form, + cmd_form_cuda_dryrun = cmd_form_cuda_dryrun, args = args, args_without_file_prefix_args = args_without_file_prefix_args, + file_prefix_specs = file_prefix_specs[0] if file_prefix_specs else None, ) def _mk_header_units_argsfile( @@ -2202,7 +2652,15 @@ def _generate_base_compile_command( ) argsfile = gen_argsfiles(is_xcode_argsfile = False) - xcode_argsfile = gen_argsfiles(is_xcode_argsfile = True) + + # The Xcode argsfile is consumed only by the `xcode-argsfiles` sub-target and + # `XcodeDataInfo`, both of which are Xcode/IDE features gated on `xcode_data`. + # Skip generating it (per source extension, per target) otherwise, saving + # memory on non-IDE builds. + if impl_params.generate_sub_targets.xcode_data: + xcode_argsfile = gen_argsfiles(is_xcode_argsfile = True) + else: + xcode_argsfile = None header_units_argsfile = _mk_header_units_argsfile( actions, @@ -2238,21 +2696,26 @@ def _generate_base_compile_command( allow_content_based_paths = allow_content_based_paths, ) -def _get_use_pch_args(src_compile_cmd: CxxSrcCompileCommand, compile_with_pch: CPrecompiledHeaderInfo, precompiled_header: CPrecompiledHeaderInfo) -> cmd_args: - if precompiled_header.clanguage != src_compile_cmd.src.extension: +def _get_use_pch_args( + src: Artifact, + cxx_compile_cmd: CxxCompileCommand, + compile_with_pch: CPrecompiledHeaderInfo, + precompiled_header: CPrecompiledHeaderInfo, +) -> cmd_args: + if precompiled_header.clanguage != src.extension: return cmd_args() pch_args = cmd_args() - if src_compile_cmd.cxx_compile_cmd.compiler_type in ["windows"]: + if cxx_compile_cmd.compiler_type in ["windows"]: pch_args.add(cmd_args(compile_with_pch.basename, format = "/Yu{}")) pch_args.add(cmd_args(compile_with_pch.basename, format = "/FI{}")) pch_args.add(cmd_args(compile_with_pch.header, format = "/Fp{}")) - elif src_compile_cmd.cxx_compile_cmd.compiler_type in ["clang"]: + elif cxx_compile_cmd.compiler_type in ["clang"]: pch_args.add("-Xclang", "-include-pch", "-Xclang", compile_with_pch.header) else: fail( "Warning: Unsupported compiler type for precompiled header usage: {}".format( - src_compile_cmd.cxx_compile_cmd.compiler_type, + cxx_compile_cmd.compiler_type, ) ) diff --git a/prelude/cxx/compile_types.bzl b/prelude/cxx/compile_types.bzl index ebdaadf..25dfbf4 100644 --- a/prelude/cxx/compile_types.bzl +++ b/prelude/cxx/compile_types.bzl @@ -86,8 +86,8 @@ CxxCompileCommand = record( base_compile_cmd = field(cmd_args), # The argsfile of arguments from the rule and it's dependencies. argsfile = field(CompileArgsfile), - # The argsfile to use for Xcode integration. - xcode_argsfile = field(CompileArgsfile), + # The argsfile to use for Xcode integration. Requires xcode_data subtarget. + xcode_argsfile = field(CompileArgsfile | None), # The argsfile containing header units args. header_units_argsfile = field(CompileArgsfile | None), # The argsfile containing header unit stubs args. diff --git a/prelude/cxx/cuda.bzl b/prelude/cxx/cuda.bzl index 162d68d..6154091 100644 --- a/prelude/cxx/cuda.bzl +++ b/prelude/cxx/cuda.bzl @@ -6,7 +6,7 @@ # of this source tree. You may select, at your option, one of the # above-listed licenses. -load("@prelude//cxx:compile_types.bzl", "CudaDistributedCompileOutput", "CxxSrcCompileCommand") +load("@prelude//cxx:compile_types.bzl", "CudaDistributedCompileOutput", "CxxCompileCommand", "HeadersDepFiles") load("@prelude//cxx:compiler.bzl", "get_output_flags") load("@prelude//cxx:cxx_toolchain_types.bzl", "CxxToolchainInfo") load("@prelude//cxx:headers.bzl", "add_headers_dep_files") @@ -34,6 +34,21 @@ CudaCompileStyle = enum( "dist", ) +CudaDistributedCompileSpec = record( + cuda_compile_info = field(CudaCompileInfo), + original_cmd = field(cmd_args), + argsfile_args = field(cmd_args), + file_prefix_specs = field(Artifact | None), + output_declared_artifact = field(OutputArtifact), + src = field(Artifact), +) + +PreparedCudaCommand = record( + cmd_node = field(dict[str, typing.Any]), + exe = field(str), + parts = field(list[typing.Any]), +) + def declare_cuda_dist_compile_output(actions: AnalysisActions, cuda_compile_info: CudaCompileInfo) -> CudaDistributedCompileOutput: """ Declare output artifacts for CUDA distributed compilation upfront. @@ -70,7 +85,8 @@ def cuda_mono_compile( actions: AnalysisActions, cmd: cmd_args, object: OutputArtifact, - src_compile_cmd: CxxSrcCompileCommand, + src: Artifact, + cxx_compile_cmd: CxxCompileCommand, cuda_compile_info: CudaCompileInfo, action_dep_files: dict[str, ArtifactTag], allow_dep_file_cache_upload: bool, @@ -82,86 +98,119 @@ def cuda_mono_compile( """ # Bind the object output for monolithic NVCC compilation. - cmd.add(get_output_flags(src_compile_cmd.cxx_compile_cmd.compiler_type, object)) - headers_dep_files = src_compile_cmd.cxx_compile_cmd.headers_dep_files + cmd.add(get_output_flags(cxx_compile_cmd.compiler_type, object)) + headers_dep_files = cxx_compile_cmd.headers_dep_files if headers_dep_files: cmd = add_headers_dep_files( actions, cmd, headers_dep_files, - src_compile_cmd.src, + src, cuda_compile_info.filename, action_dep_files, ) actions.run( cmd, - category = src_compile_cmd.cxx_compile_cmd.category, + category = cxx_compile_cmd.category, identifier = cuda_compile_info.identifier, dep_files = action_dep_files, - allow_cache_upload = src_compile_cmd.cxx_compile_cmd.allow_cache_upload, + allow_cache_upload = cxx_compile_cmd.allow_cache_upload, allow_dep_file_cache_upload = allow_dep_file_cache_upload, error_handler = error_handler, ) def cuda_distributed_compile( actions: AnalysisActions, - toolchain: CxxToolchainInfo, cmd: cmd_args, object: OutputArtifact, cuda_dist_output: CudaDistributedCompileOutput, - src_compile_cmd: CxxSrcCompileCommand, + src: Artifact, + cxx_compile_cmd: CxxCompileCommand, cuda_compile_info: CudaCompileInfo, -) -> None: + cuda_prepare_cmd: cmd_args | None, + shared_plan_identifier: str | None, +) -> CudaDistributedCompileSpec: """ - Compile a CUDA file using distributed compilation. - NVCC provides the compilation plan, but compilation is split into - one Buck action per sub-command. + Set up distributed compilation for a CUDA file: optionally register the + nvcc -dryrun prepare action that produces the target's shared compilation + plan, and return the spec from which create_cuda_distributed_compiles + later creates one Buck action per NVCC sub-command. + + Compilation modes that require a whole-program device link step are NOT + supported here: relocatable device code (-rdc=true / --device-c), device + LTO (-dlto), and PTX-only codegen all make nvcc -dryrun emit an nvlink + sub-command that dist NVCC cannot distribute. Such targets must set + cuda_compile_style = "mono". If one of these modes reaches this path, the + dryrun parser (tools/build/cuda/nvcc_dryrun.py) fails the + cuda_compile_prepare action with an actionable error that names the source + .cu and points to the mono escape hatch. """ hostcc_argsfile = cuda_dist_output.hostcc_argsfile - # We'll first run nvcc with -dryrun. So do not bind the object file yet. - cmd.add(["-o", object.short_path]) - original_cmd = cmd.copy() - cmd.add([ - "-_NVCC_DRYRUN_", - "-_NVCC_HOSTCC_ARGSFILE_", - as_output(hostcc_argsfile), - "-_NVCC_DRYRUN_ENV_OUT_", - as_output(cuda_dist_output.nvcc_env), - "-_NVCC_DRYRUN_DAG_OUT_", - as_output(cuda_dist_output.nvcc_dag), - ]) - - # Run nvcc with -dryrun to create the inputs needed for dist nvcc. - actions.run(cmd, category = "cuda_compile_prepare", identifier = cuda_compile_info.identifier) + # The object is produced by the replayed plan sub-actions, not by this + # command, so embed its path as a string rather than binding the output. + cmd_with_output = cmd_args(cmd, ["-o", object.short_path]) + original_cmd = cmd_args( + cmd_with_output, + hidden = [ + cxx_compile_cmd.base_compile_cmd, + cxx_compile_cmd.argsfile.cmd_form, + ], + ) - actions.dynamic_output_new( - _nvcc_dynamic_compile_rule( - toolchain = toolchain, - cuda_compile_info = cuda_compile_info, - src_compile_cmd = src_compile_cmd, - original_cmd = original_cmd, - hostcc_argsfile = hostcc_argsfile, - plan_artifact = cuda_dist_output.nvcc_dag, - env_artifact = cuda_dist_output.nvcc_env, - output_declared_artifact = object, + # The dry run never opens this file. It matches the flag, echoed as one + # `-specs=` token, so the plan can carry a placeholder in place of a + # path that moves with the header closure. + file_prefix_specs = cxx_compile_cmd.argsfile.file_prefix_specs + + if cuda_prepare_cmd != None: + prepare_cmd = cmd_args( + cuda_prepare_cmd, + ["-o", object.short_path], + [ + "-_NVCC_DRYRUN_", + "-_NVCC_HOSTCC_ARGSFILE_", + as_output(hostcc_argsfile), + "-_NVCC_DRYRUN_ENV_OUT_", + as_output(cuda_dist_output.nvcc_env), + "-_NVCC_DRYRUN_DAG_OUT_", + as_output(cuda_dist_output.nvcc_dag), + ], ) + if file_prefix_specs != None: + prepare_cmd.add( + "-_NVCC_FILE_PREFIX_SPECS_", + cmd_args(file_prefix_specs, format = "-specs={}"), + ) + + # Run nvcc with -dryrun to create the inputs needed for dist nvcc. + actions.run(prepare_cmd, category = "cuda_compile_prepare", identifier = shared_plan_identifier or cuda_compile_info.identifier) + + return CudaDistributedCompileSpec( + cuda_compile_info = cuda_compile_info, + original_cmd = original_cmd, + argsfile_args = cxx_compile_cmd.argsfile.args, + file_prefix_specs = file_prefix_specs, + output_declared_artifact = object, + src = src, ) # Keep the old cuda_compile function for backward compatibility def cuda_compile( actions: AnalysisActions, - toolchain: CxxToolchainInfo, cmd: cmd_args, object: OutputArtifact, - src_compile_cmd: CxxSrcCompileCommand, + src: Artifact, + cxx_compile_cmd: CxxCompileCommand, cuda_compile_info: CudaCompileInfo, action_dep_files: dict[str, ArtifactTag], allow_dep_file_cache_upload: bool, error_handler: [typing.Callable, None], cuda_compile_style: CudaCompileStyle | None, cuda_dist_output: CudaDistributedCompileOutput | None = None, -) -> None: + cuda_prepare_cmd: cmd_args | None = None, + shared_plan_identifier: str | None = None, +) -> CudaDistributedCompileSpec | None: """ Compile a CUDA file using either monolithic or distributed compilation. This is a convenience function that dispatches to the appropriate implementation. @@ -171,7 +220,8 @@ def cuda_compile( actions, cmd, object, - src_compile_cmd, + src, + cxx_compile_cmd, cuda_compile_info, action_dep_files, allow_dep_file_cache_upload, @@ -181,23 +231,68 @@ def cuda_compile( elif cuda_compile_style == CudaCompileStyle("dist"): if cuda_dist_output == None: fail("cuda_dist_output is required for distributed CUDA compilation") - cuda_distributed_compile( + return cuda_distributed_compile( actions, - toolchain, cmd, object, cuda_dist_output, - src_compile_cmd, + src, + cxx_compile_cmd, cuda_compile_info, + cuda_prepare_cmd, + shared_plan_identifier, ) - return None else: fail("Unsupported CUDA compile style: {}".format(cuda_compile_style)) +def create_cuda_distributed_compiles( + actions: AnalysisActions, + toolchain: CxxToolchainInfo, + cuda_dist_output: CudaDistributedCompileOutput, + specs: list[CudaDistributedCompileSpec], + allow_cache_upload: bool, + cuda_cxx_compile_inputs: cmd_args, + headers_dep_files: HeadersDepFiles | None, +) -> None: + if not specs: + return + + # The argsfile is shared by the target's sources, so one fingerprint covers + # them all. It carries the same flags with content-based paths rendered as a + # placeholder: it moves when a flag really changes, not when a path does. + hostcc_argsfile_fingerprint = None + if headers_dep_files != None: + hostcc_argsfile_fingerprint, _ = actions.write( + "__redacted__/{}.hostcc_argsfile_fingerprint".format(specs[0].cuda_compile_info.filename), + cmd_args(specs[0].original_cmd, specs[0].argsfile_args, quote = "shell"), + allow_args = True, + has_content_based_path = specs[0].cuda_compile_info.uses_content_based_paths, + use_dep_files_placeholder_for_content_based_paths = True, + ) + + actions.dynamic_output_new( + _nvcc_dynamic_compile_rule( + toolchain = toolchain, + cuda_compile_infos = [spec.cuda_compile_info for spec in specs], + srcs = [spec.src for spec in specs], + allow_cache_upload = allow_cache_upload, + cuda_cxx_compile_inputs = cuda_cxx_compile_inputs, + headers_dep_files = headers_dep_files, + original_cmds = [spec.original_cmd for spec in specs], + hostcc_argsfile = cuda_dist_output.hostcc_argsfile, + hostcc_argsfile_fingerprint = hostcc_argsfile_fingerprint, + file_prefix_specs = specs[0].file_prefix_specs, + plan_artifact = cuda_dist_output.nvcc_dag, + env_artifact = cuda_dist_output.nvcc_env, + output_declared_artifacts = [spec.output_declared_artifact for spec in specs], + ) + ) + def _create_file_to_artifact_map( actions: AnalysisActions, plan_json: list[dict[str, typing.Any]], - src_compile_cmd: CxxSrcCompileCommand, + src: Artifact, + cuda_compile_info: CudaCompileInfo, output_declared_artifact: OutputArtifact, uses_content_based_paths: bool, ) -> dict[str, Artifact | OutputArtifact]: @@ -209,10 +304,11 @@ def _create_file_to_artifact_map( for input in node_inputs: if input not in file2artifact: if input.endswith(".cu"): - file2artifact[input] = src_compile_cmd.src + file2artifact[input] = src else: input_artifact = actions.declare_output( - input, + "__cuda_intermediates__", + "{}/{}".format(cuda_compile_info.filename, input), has_content_based_path = uses_content_based_paths, ) file2artifact[input] = input_artifact @@ -222,7 +318,8 @@ def _create_file_to_artifact_map( file2artifact[output] = output_declared_artifact else: output_artifact = actions.declare_output( - output, + "__cuda_intermediates__", + "{}/{}".format(cuda_compile_info.filename, output), has_content_based_path = uses_content_based_paths, ) file2artifact[output] = output_artifact @@ -236,7 +333,12 @@ def _create_nvcc_subcmd_env(env_artifact: ArtifactValue) -> dict[str, str]: subcmd_env[key] = value return subcmd_env -def _include_symlinked_stubs_dir(actions: AnalysisActions, file2artifact: dict[str, typing.Any], subcmd: cmd_args) -> None: +def _include_symlinked_stubs_dir( + actions: AnalysisActions, + cuda_compile_info: CudaCompileInfo, + file2artifact: dict[str, typing.Any], + subcmd: cmd_args, +) -> None: """ .cudafe1.stub.c and .fatbin.c files are hardcoded into the cudafe1.cpp file and its includes like below: @@ -252,6 +354,7 @@ def _include_symlinked_stubs_dir(actions: AnalysisActions, file2artifact: dict[s """ stubs_dir = actions.declare_output( "__stubs__", + cuda_compile_info.filename, dir = True, has_content_based_path = True, ) @@ -259,8 +362,7 @@ def _include_symlinked_stubs_dir(actions: AnalysisActions, file2artifact: dict[s for file, artifact in file2artifact.items(): if file.endswith(".cudafe1.stub.c") or file.endswith(".fatbin.c"): # Remove the parent paths because the includes are the filenames only. - # We can do this because each dynamic compile deals with only one CUDA - # source file. + # Each stubs directory is scoped to one CUDA source file. stubs[artifact.basename] = artifact symlinked_dir = actions.symlinked_dir( stubs_dir, @@ -269,97 +371,233 @@ def _include_symlinked_stubs_dir(actions: AnalysisActions, file2artifact: dict[s ) subcmd.add(cmd_args(symlinked_dir, format = "-I{}")) +def _prepare_cuda_command(cmd_node: dict[str, typing.Any], hostcc_wp_form: cmd_args, specs_form: cmd_args | None) -> PreparedCudaCommand: + parts = [] + common = cmd_args() + for token in cmd_node["cmd"][1:]: + placeholder = None + if "{input}" in token: + placeholder = "input" + elif "{output}" in token: + placeholder = "output" + elif "{source_path}" in token: + placeholder = "source_path" + + if placeholder != None: + parts.append(("common", common)) + left, right = token.split("{" + placeholder + "}", 1) + + # Replay substitutes exactly one placeholder per token; a second + # one would be passed through literally. + for kind in ("input", "output", "source_path"): + if "{" + kind + "}" in left + right: + fail("plan token carries multiple placeholders: {}".format(token)) + parts.append((placeholder, (left, right))) + common = cmd_args() + elif token.startswith("-Wp,@"): + common.add(hostcc_wp_form) + elif token == "{file_prefix_specs}": + if specs_form == None: + fail("CUDA plan references file-prefix specs without a specs artifact") + common.add(specs_form) + else: + common.add(token) + parts.append(("common", common)) + return PreparedCudaCommand(cmd_node = cmd_node, exe = cmd_node["cmd"][0], parts = parts) + +def _cuda_replay_input_kind(category: str) -> str: + # Device phase categories appear both arch-suffixed (`cuda_opt_sm90a`) and + # bare: the plan parser emits the bare form whenever an intermediate file + # name carries no compute_{arch} component, e.g. for arch suffixes its + # normalization does not recognize. + if category == "cuda_host_preproc" or category == "cuda_device_preproc" or category.startswith("cuda_device_preproc_"): + return "preprocess" + if category == "cuda_cxx_compile": + return "host_compile" + if category in ("cudafepp", "cuda_fatbin", "cuda_opt", "cuda_cubin_lower") or category.startswith("cuda_opt_") or category.startswith("cuda_cubin_lower_"): + return "cuda_tool" + fail("Unsupported distributed CUDA phase category `{}`; update its input policy".format(category)) + def _nvcc_dynamic_compile( actions: AnalysisActions, toolchain: CxxToolchainInfo, - cuda_compile_info: CudaCompileInfo, - src_compile_cmd: CxxSrcCompileCommand, - original_cmd: cmd_args, + cuda_compile_infos: list[CudaCompileInfo], + srcs: list[Artifact], + allow_cache_upload: bool, + cuda_cxx_compile_inputs: cmd_args, + headers_dep_files: HeadersDepFiles | None, + original_cmds: list[cmd_args], hostcc_argsfile: Artifact, + hostcc_argsfile_fingerprint: Artifact | None, + file_prefix_specs: Artifact | None, plan_artifact: ArtifactValue, env_artifact: ArtifactValue, - output_declared_artifact: OutputArtifact, + output_declared_artifacts: list[OutputArtifact], ) -> list[Provider]: + num_sources = len(cuda_compile_infos) + if len(srcs) != num_sources or len(original_cmds) != num_sources or len(output_declared_artifacts) != num_sources: + fail( + "per-source dist CUDA lists must be the same length, got {}/{}/{}/{}".format( + len(cuda_compile_infos), + len(srcs), + len(original_cmds), + len(output_declared_artifacts), + ) + ) + plan = plan_artifact.read_json() - content_based = cuda_compile_info.uses_content_based_paths - file2artifact = _create_file_to_artifact_map( - actions, - plan, - src_compile_cmd, - output_declared_artifact, - content_based, - ) subcmd_env = _create_nvcc_subcmd_env(env_artifact) - for cmd_node in plan: - subcmd = cmd_args() - exe = cmd_node["cmd"].pop(0) - if "g++" in exe or "clang++" in exe: - # Add the original command as a hidden dependency, so that - # we have access to the host compiler and header files. - subcmd.add(cmd_args(hidden = original_cmd)) - elif "ptxas" in exe: - # Ptxas occasionally produces an empty output. The root cause - # is unknown as we're unable to reproduce it locally. Check the - # output is not empty - subcmd.add(toolchain.internal_tools.check_nonempty_output) - subcmd.add(exe) - - if content_based and cmd_node["category"] == "cuda_cxx_compile": - _include_symlinked_stubs_dir(actions, file2artifact, subcmd) - - for token in cmd_node["cmd"]: - # Replace the {input} and {output} placeholders with the actual - # artifacts. node["inputs"] and node["outputs"] are used as a - # queue here where the files will always be correctly replaced - # in a FIFO order. - if "{input}" in token: - input = cmd_node["inputs"].pop(0) - left, right = token.split("{input}", 1) - subcmd.add(cmd_args([left, file2artifact[input], right], delimiter = "")) - elif "{output}" in token: - output = cmd_node["outputs"].pop(0) - left, right = token.split("{output}", 1) - artifact = file2artifact[output] - if isinstance(artifact, Artifact): - bindable = artifact.as_output() - else: - bindable = artifact - subcmd.add(cmd_args([left, bindable, right], delimiter = "")) - elif token.startswith("-Wp,@"): - subcmd.add(cmd_args(hostcc_argsfile, format = "-Wp,@{}")) - else: - subcmd.add(token) - - # Some nodes have hidden dependencies (deps that don't appear in - # the cmd). Add them to the hidden field of cmd_args. - if cmd_node["hidden"]: - subcmd.add(cmd_args(hidden = [file2artifact[f] for f in cmd_node["hidden"]])) - - # Add the cuda toolchain deps so that we can find the Nvidia tools - # and CUDA header files. - subcmd.add(cmd_args(hidden = [toolchain.cuda_compiler_info.compiler])) - actions.run( - subcmd, - category = cmd_node["category"], - env = subcmd_env, - identifier = cuda_compile_info.identifier, - allow_cache_upload = src_compile_cmd.cxx_compile_cmd.allow_cache_upload, - prefer_remote = True if "preproc" in cmd_node["category"] else False, + # The plan is shared across the target's sources, so these references are + # built once from the representative source. Tagging lets dep-file filtering + # drop their paths from each sub-action key; the untagged fingerprint is what + # still reruns the sub-actions when a flag changes. + if headers_dep_files != None and hostcc_argsfile_fingerprint != None: + hostcc_wp_form = cmd_args( + headers_dep_files.tag.tag_artifacts(hostcc_argsfile), + format = "-Wp,@{}", + hidden = [hostcc_argsfile_fingerprint], ) + else: + hostcc_wp_form = cmd_args(hostcc_argsfile, format = "-Wp,@{}") + + specs_form = None + if file_prefix_specs != None: + if headers_dep_files != None: + specs_form = cmd_args( + headers_dep_files.tag.tag_artifacts(file_prefix_specs), + format = "-specs={}", + ) + else: + specs_form = cmd_args(file_prefix_specs, format = "-specs={}") + + prepared_commands = [_prepare_cuda_command(cmd_node, hostcc_wp_form, specs_form) for cmd_node in plan] + cuda_toolchain = getattr(toolchain.cuda_compiler_info, "compiler_for_dryrun", None) or toolchain.cuda_compiler_info.compiler + cuda_tool_inputs = cmd_args(hidden = [cuda_toolchain]) + + for cuda_compile_info, src, original_cmd, output_declared_artifact in zip( + cuda_compile_infos, + srcs, + original_cmds, + output_declared_artifacts, + ): + content_based = cuda_compile_info.uses_content_based_paths + file2artifact = _create_file_to_artifact_map( + actions, + plan, + src, + cuda_compile_info, + output_declared_artifact, + content_based, + ) + + category_counts = {} + for prepared in prepared_commands: + cmd_node = prepared.cmd_node + subcmd = cmd_args() + exe = prepared.exe + input_kind = _cuda_replay_input_kind(cmd_node["category"]) + if input_kind == "preprocess": + subcmd.add(cmd_args(hidden = original_cmd)) + elif input_kind == "host_compile": + subcmd.add(cmd_args(hidden = cuda_cxx_compile_inputs)) + else: + subcmd.add(cuda_tool_inputs) + + if "ptxas" in exe: + # Ptxas occasionally produces an empty output. The root cause + # is unknown as we're unable to reproduce it locally. Check the + # output is not empty + subcmd.add(toolchain.internal_tools.check_nonempty_output) + subcmd.add(exe) + + # The stubs directory must lead the include path so its generated + # stub files shadow any same-named file in later include dirs. + if content_based and cmd_node["category"] == "cuda_cxx_compile": + _include_symlinked_stubs_dir(actions, cuda_compile_info, file2artifact, subcmd) + + input_index = 0 + output_index = 0 + for kind, value in prepared.parts: + if kind == "common": + subcmd.add(value) + elif kind == "input": + input = cmd_node["inputs"][input_index] + input_index += 1 + subcmd.add(cmd_args([value[0], file2artifact[input], value[1]], delimiter = "")) + elif kind == "output": + output = cmd_node["outputs"][output_index] + output_index += 1 + artifact = file2artifact[output] + bindable = artifact.as_output() if isinstance(artifact, Artifact) else artifact + subcmd.add(cmd_args([value[0], bindable, value[1]], delimiter = "")) + elif kind == "source_path": + # The contents of the `source_path` files are not actually + # used as inputs to the sub-action, they are used primarily + # to name the actual source path for inclusion in things like + # DWARF information. However, artifacts with content-based-paths + # can't know their paths until their content is known, so they + # cannot be included as ignore_artifacts=True + # TODO(jtbraun): when has_content_based_path is available, + # use that as the condition here + if src.is_source: + subcmd.add(cmd_args([value[0], src, value[1]], delimiter = "", ignore_artifacts = True)) + else: + subcmd.add(cmd_args([value[0], src, value[1]], delimiter = "")) + else: + fail("unhandled placeholder kind: {}".format(kind)) + + # Some nodes have hidden dependencies (deps that don't appear in + # the cmd). Add them to the hidden field of cmd_args. + if cmd_node["hidden"]: + subcmd.add(cmd_args(hidden = [file2artifact[f] for f in cmd_node["hidden"]])) + + # Host-compiler sub-actions pin a declared header closure; dep files + # let the compiler report what it actually read so buck can prune the + # rest from the action key. + action_dep_files = {} + if input_kind != "cuda_tool" and headers_dep_files != None: + # Categories can repeat within a plan, so disambiguate with a + # per-category ordinal. + ordinal = category_counts.get(cmd_node["category"], 0) + category_counts[cmd_node["category"]] = ordinal + 1 + subcmd = add_headers_dep_files( + actions, + subcmd, + headers_dep_files, + src, + "{}/{}/{}".format(cuda_compile_info.filename, cmd_node["category"], ordinal), + action_dep_files, + ) + + actions.run( + subcmd, + category = cmd_node["category"], + env = subcmd_env, + identifier = cuda_compile_info.identifier, + dep_files = action_dep_files, + allow_cache_upload = allow_cache_upload, + allow_dep_file_cache_upload = False, + prefer_remote = True if "preproc" in cmd_node["category"] else False, + ) return [DefaultInfo()] _nvcc_dynamic_compile_rule = dynamic_actions( impl = _nvcc_dynamic_compile, attrs = { - "cuda_compile_info": dynattrs.value(CudaCompileInfo), + "allow_cache_upload": dynattrs.value(bool), + "cuda_compile_infos": dynattrs.list(dynattrs.value(CudaCompileInfo)), + "cuda_cxx_compile_inputs": dynattrs.value(cmd_args), "env_artifact": dynattrs.artifact_value(), + "file_prefix_specs": dynattrs.option(dynattrs.value(Artifact)), + "headers_dep_files": dynattrs.option(dynattrs.value(HeadersDepFiles)), "hostcc_argsfile": dynattrs.value(Artifact), - "original_cmd": dynattrs.value(cmd_args), - "output_declared_artifact": dynattrs.output(), + "hostcc_argsfile_fingerprint": dynattrs.option(dynattrs.value(Artifact)), + "original_cmds": dynattrs.list(dynattrs.value(cmd_args)), + "output_declared_artifacts": dynattrs.list(dynattrs.output()), "plan_artifact": dynattrs.artifact_value(), - "src_compile_cmd": dynattrs.value(CxxSrcCompileCommand), + "srcs": dynattrs.list(dynattrs.value(Artifact)), "toolchain": dynattrs.value(CxxToolchainInfo), }, ) diff --git a/prelude/cxx/cxx.bzl b/prelude/cxx/cxx.bzl index c20853f..237a145 100644 --- a/prelude/cxx/cxx.bzl +++ b/prelude/cxx/cxx.bzl @@ -9,8 +9,8 @@ load("@prelude//:paths.bzl", "paths") load( "@prelude//:resources.bzl", - "ResourceInfo", "gather_resources", + "make_resource_info", ) load( "@prelude//android:android_providers.bzl", @@ -25,6 +25,7 @@ load( "@prelude//cxx:cuda.bzl", "CudaCompileStyle", ) +load("@prelude//cxx:cxx_flags.bzl", "cxx_attr_flags") load("@prelude//cxx:cxx_sources.bzl", "get_srcs_with_flags") load( "@prelude//cxx:cxx_toolchain_types.bzl", @@ -66,7 +67,7 @@ load( ) load( "@prelude//linking:linkable_graph.bzl", - "DlopenableLibraryInfo", + "DLOPENABLE_LIBRARY_INFO_MARKER", "LinkableGraph", "create_linkable_graph", "create_linkable_graph_node", @@ -89,6 +90,8 @@ load("@prelude//linking:strip.bzl", "strip_debug_info") load("@prelude//linking:types.bzl", "Linkage") load("@prelude//os_lookup:defs.bzl", "Os", "OsLookup") load("@prelude//python:manifest.bzl", "create_manifest_for_entries") +load("@prelude//target_stats:target_stats.bzl", "CycleMode", "target_stats_providers_and_subtargets") +load("@prelude//target_stats:target_stats_config.bzl", "TARGET_STATS_ENABLED") load("@prelude//test:inject_test_run_info.bzl", "inject_test_run_info") load( "@prelude//tests:re_utils.bzl", @@ -135,6 +138,7 @@ load( "CxxRuleProviderParams", "CxxRuleSubTargetParams", "LinkPreference", + "xcode_data_enabled", ) load(":gcno.bzl", "GcnoFilesInfo") load( @@ -146,6 +150,7 @@ load( load( ":headers.bzl", "CPrecompiledHeaderInfo", + "cxx_attr_headers_list", "cxx_get_regular_cxx_headers_layout", ) load( @@ -241,7 +246,7 @@ def cxx_library_generate(ctx: AnalysisContext, rule_type: str) -> list[Provider] if ctx.attrs._is_building_android_binary: sub_target_params, provider_params = _get_params_for_android_binary_cxx_library() else: - sub_target_params = CxxRuleSubTargetParams() + sub_target_params = CxxRuleSubTargetParams(xcode_data = xcode_data_enabled()) provider_params = CxxRuleProviderParams( third_party_build = True, ) @@ -254,6 +259,7 @@ def cxx_library_generate(ctx: AnalysisContext, rule_type: str) -> list[Provider] generate_sub_targets = sub_target_params, generate_providers = provider_params, compiler_flags = ctx.attrs.compiler_flags, + cxx_flags = cxx_attr_flags(ctx), lang_compiler_flags = ctx.attrs.lang_compiler_flags, preprocessor_flags = ctx.attrs.preprocessor_flags, lang_preprocessor_flags = ctx.attrs.lang_preprocessor_flags, @@ -324,6 +330,7 @@ def cxx_binary_impl(ctx: AnalysisContext) -> list[Provider]: ) params = CxxRuleConstructorParams( rule_type = "cxx_binary", + generate_sub_targets = CxxRuleSubTargetParams(xcode_data = xcode_data_enabled()), executable_name = ctx.attrs.executable_name, headers_layout = cxx_get_regular_cxx_headers_layout(ctx), srcs = get_srcs_with_flags(ctx), @@ -333,6 +340,7 @@ def cxx_binary_impl(ctx: AnalysisContext) -> list[Provider]: exe_allow_cache_upload = cxx_attrs_get_allow_cache_upload(ctx.attrs), extra_link_roots = linkables(ctx.attrs.link_group_deps), compiler_flags = ctx.attrs.compiler_flags, + cxx_flags = cxx_attr_flags(ctx), lang_compiler_flags = ctx.attrs.lang_compiler_flags, preprocessor_flags = ctx.attrs.preprocessor_flags, lang_preprocessor_flags = ctx.attrs.lang_preprocessor_flags, @@ -413,17 +421,23 @@ def cxx_binary_impl(ctx: AnalysisContext) -> list[Provider]: # by another rule, such as by $(exe ...) or exec_dep. other_outputs = output.runtime_files + (output.external_debug_info_artifacts if get_cxx_toolchain_info(ctx).materialize_external_debug_info else []) - return [ - DefaultInfo( - default_output = output.binary, - other_outputs = other_outputs, - sub_targets = output.sub_targets, - ), - RunInfo(args = cmd_args(output.binary, hidden = output.runtime_files)), - output.compilation_db, - output.xcode_data, - output.dist_info, - ] + extra_providers + return ( + filter( + None, + [ + DefaultInfo( + default_output = output.binary, + other_outputs = other_outputs, + sub_targets = output.sub_targets, + ), + RunInfo(args = cmd_args(output.binary, hidden = output.runtime_files)), + output.compilation_db, + output.xcode_data, + output.dist_info, + ], + ) + + extra_providers + ) def _prebuilt_item(_ctx: AnalysisContext, item: [typing.Any, None]) -> [typing.Any, None]: """ @@ -799,7 +813,7 @@ def _create_prebuilt_library_providers( # Mark libraries that support `dlopen`. if ctx.attrs.supports_python_dlopen: - providers.append(DlopenableLibraryInfo()) + providers.append(DLOPENABLE_LIBRARY_INFO_MARKER) linkable_graph = create_linkable_graph( ctx, @@ -996,12 +1010,12 @@ def prebuilt_cxx_library_impl(ctx: AnalysisContext) -> list[Provider]: ) providers.append( - ResourceInfo( - resources = gather_resources( + make_resource_info( + gather_resources( label = ctx.label, deps = first_order_deps + exported_first_order_deps, - ) - ) + ), + ), ) return providers @@ -1029,15 +1043,19 @@ def cxx_test_impl(ctx: AnalysisContext) -> list[Provider]: ) # TODO(T110378115): have the runinfo contain the correct test running args + headers_layout = cxx_get_regular_cxx_headers_layout(ctx) + srcs = get_srcs_with_flags(ctx) params = CxxRuleConstructorParams( rule_type = "cxx_test", - headers_layout = cxx_get_regular_cxx_headers_layout(ctx), - srcs = get_srcs_with_flags(ctx), + generate_sub_targets = CxxRuleSubTargetParams(xcode_data = xcode_data_enabled()), + headers_layout = headers_layout, + srcs = srcs, link_group_info = link_group_info, auto_link_group_specs = get_auto_link_group_specs(ctx, link_group_info), prefer_stripped_objects = ctx.attrs.prefer_stripped_objects, extra_link_roots = linkables(ctx.attrs.link_group_deps), compiler_flags = ctx.attrs.compiler_flags, + cxx_flags = cxx_attr_flags(ctx), lang_compiler_flags = ctx.attrs.lang_compiler_flags, preprocessor_flags = ctx.attrs.preprocessor_flags, lang_preprocessor_flags = ctx.attrs.lang_preprocessor_flags, @@ -1061,6 +1079,28 @@ def cxx_test_impl(ctx: AnalysisContext) -> list[Provider]: ) output = cxx_executable(ctx, params, is_cxx_test = True) + target_stats_providers = [] + if TARGET_STATS_ENABLED: + target_stats_tools = get_cxx_toolchain_info(ctx).target_stats_tools + if target_stats_tools != None: + target_stats_srcs = {src.file.short_path: src.file for src in srcs} + target_stats_srcs.update({ + # Named headers use the dict key; list headers retain their + # package-relative path. This matches the collector and avoids + # collisions between headers with the same basename. + (header.name if header.named else header.artifact.short_path): header.artifact + for header in cxx_attr_headers_list(ctx, ctx.attrs.headers, headers_layout) + }) + target_stats_providers, target_stats_subtargets = target_stats_providers_and_subtargets( + ctx, + tools = target_stats_tools, + srcs = target_stats_srcs, + deps = cxx_attr_deps(ctx), + cycle_mode = CycleMode("file"), + module_name = ctx.label.name, + ) + output.sub_targets.update(target_stats_subtargets) + command = [cmd_args(output.binary, hidden = output.runtime_files)] + ctx.attrs.args # Setup RE executors based on the `remote_execution` param. @@ -1073,9 +1113,10 @@ def cxx_test_impl(ctx: AnalysisContext) -> list[Provider]: sub_targets = output.sub_targets, ), output.compilation_db, - output.xcode_data, output.dist_info, ] + if output.xcode_data: + providers.append(output.xcode_data) providers.extend( inject_test_run_info( ctx, @@ -1102,6 +1143,8 @@ def cxx_test_impl(ctx: AnalysisContext) -> list[Provider]: if get_cxx_toolchain_info(ctx).gcno_files and output.gcno_files: providers.append(GcnoFilesInfo(gcno_files = output.gcno_files)) + providers.extend(target_stats_providers) + return providers def _get_params_for_android_binary_cxx_library() -> (CxxRuleSubTargetParams, CxxRuleProviderParams): diff --git a/prelude/cxx/cxx_bolt.bzl b/prelude/cxx/cxx_bolt.bzl index 2bf64f1..234d5de 100644 --- a/prelude/cxx/cxx_bolt.bzl +++ b/prelude/cxx/cxx_bolt.bzl @@ -16,6 +16,8 @@ load( ) load(":cxx_context.bzl", "get_cxx_toolchain_info") +PRE_BOLT_SUFFIX = "-wrapper" + CxxBoltOutput = record( output = field(Artifact), dwo_output = field(Artifact | None), @@ -33,22 +35,21 @@ def bolt( generate_dwp: bool, allow_cache_upload: bool = False, ) -> CxxBoltOutput: - output_name = prebolt_output.short_path.removesuffix("-wrapper") + output_name = prebolt_output.short_path.removesuffix(PRE_BOLT_SUFFIX) postbolt_output = ctx.actions.declare_output(output_name, has_content_based_path = False) dwo_output = None cxx_toolchain_info = get_cxx_toolchain_info(ctx) - bolt_exe = cxx_toolchain_info.binary_utilities_info.bolt - bolt_msdk = cxx_toolchain_info.binary_utilities_info.bolt_msdk + bolt_tool = cxx_toolchain_info.binary_utilities_info.bolt - if not (bolt_exe or bolt_msdk) or not cxx_use_bolt(ctx): - fail("Cannot use bolt if bolt_msdk is not available or bolt profile is not available") + if not bolt_tool or not cxx_use_bolt(ctx): + fail("Cannot use bolt if bolt tool is not available or bolt profile is not available") materialized_external_debug_info = project_artifacts(ctx.actions, external_debug_info) # bolt command format: # {llvm_bolt} {input_bin} -o $OUT -data={fdata} {args} args = cmd_args( - bolt_exe if bolt_exe else cmd_args(bolt_msdk, format = "{}/bin/llvm-bolt"), + bolt_tool, prebolt_output, "-o", postbolt_output.as_output(), diff --git a/prelude/cxx/cxx_executable.bzl b/prelude/cxx/cxx_executable.bzl index 3f6c886..6ece157 100644 --- a/prelude/cxx/cxx_executable.bzl +++ b/prelude/cxx/cxx_executable.bzl @@ -29,6 +29,7 @@ load( ) load( "@prelude//cxx:cxx_bolt.bzl", + "PRE_BOLT_SUFFIX", "cxx_use_bolt", ) load( @@ -62,6 +63,14 @@ load( "XcodeDataInfo", "generate_xcode_data", ) +load("@prelude//linking:add_elf_sections.bzl", "PRE_ADD_ELF_SECTIONS_SUFFIX", "get_elf_sections") +load( + "@prelude//linking:generated_build_info.bzl", + "compile_generated_build_info", + "generate_build_info", + "generate_build_info_shared_library", + "generated_build_info_is_shared_library", +) load( "@prelude//linking:link_groups.bzl", "gather_link_group_libs", @@ -81,6 +90,7 @@ load( "make_link_command_debug_output_json_info", "process_link_strategy_for_pic_behavior", "to_link_strategy", + "unpack_link_args", ) load( "@prelude//linking:linkable_graph.bzl", @@ -119,7 +129,6 @@ load( "@prelude//xplugins:types.bzl", "XPluginsDebugArtifactsInfo", ) -load("@prelude//xplugins:utils.bzl", "get_xplugins_usage_info", "get_xplugins_usage_subtargets") load( ":argsfiles.bzl", "ARGSFILES_SUBTARGET", @@ -244,7 +253,7 @@ CxxExecutableOutput = record( # All link group links that were generated in the executable. auto_link_groups = field(dict[str, LinkedObject], {}), compilation_db = CxxCompilationDbInfo, - xcode_data = XcodeDataInfo, + xcode_data = [XcodeDataInfo, None], linker_map_data = [CxxLinkerMapData, None], gc_sections_data = [CxxGcSectionsData, None], link_command_debug_output = field([LinkCommandDebugOutput, None], None), @@ -254,6 +263,7 @@ CxxExecutableOutput = record( validation_specs = field(list[ValidationSpec], []), gcno_files = field(list[Artifact], []), xplugins_debug_artifacts_info = field(XPluginsDebugArtifactsInfo | None, None), + build_info_manifest_entries = field(Artifact | None, None), ) def cxx_executable(ctx: AnalysisContext, impl_params: CxxRuleConstructorParams, is_cxx_test: bool = False) -> CxxExecutableOutput: @@ -326,8 +336,10 @@ def cxx_executable(ctx: AnalysisContext, impl_params: CxxRuleConstructorParams, if get_cxx_toolchain_info(ctx).gcno_files: gcno_files += flatten([dep[GcnoFilesInfo].gcno_files for dep in cxx_deps if GcnoFilesInfo in dep]) - sub_targets[ARGSFILES_SUBTARGET] = [get_argsfiles_output(ctx, compile_cmd_output.argsfiles.relative, ARGSFILES_SUBTARGET)] - sub_targets[XCODE_ARGSFILES_SUB_TARGET] = [get_argsfiles_output(ctx, compile_cmd_output.argsfiles.xcode, XCODE_ARGSFILES_SUB_TARGET)] + if impl_params.generate_sub_targets.argsfiles: + sub_targets[ARGSFILES_SUBTARGET] = [get_argsfiles_output(ctx, compile_cmd_output.argsfiles.relative, ARGSFILES_SUBTARGET)] + if impl_params.generate_sub_targets.xcode_data: + sub_targets[XCODE_ARGSFILES_SUB_TARGET] = [get_argsfiles_output(ctx, compile_cmd_output.argsfiles.xcode, XCODE_ARGSFILES_SUB_TARGET)] sub_targets[OBJECTS_SUBTARGET] = [DefaultInfo(sub_targets = cxx_objects_sub_targets(cxx_outs))] if impl_params.generate_sub_targets and impl_params.generate_sub_targets.clang_traces: @@ -440,7 +452,7 @@ def cxx_executable(ctx: AnalysisContext, impl_params: CxxRuleConstructorParams, # Gather link inputs. own_link_flags = ( get_cxx_toolchain_info(ctx).linker_info.binary_linker_flags - + cxx_attr_linker_flags(ctx) + + cxx_attr_linker_flags(ctx, impl_params.cxx_flags) + impl_params.extra_link_flags + impl_params.extra_exported_link_flags ) @@ -701,6 +713,43 @@ def cxx_executable(ctx: AnalysisContext, impl_params: CxxRuleConstructorParams, toolchain_info = get_cxx_toolchain_info(ctx) linker_info = toolchain_info.linker_info + generated_build_info_external_debug_info = [] + binary_linker_flags = ctx.attrs.binary_linker_flags + generated_build_info_args = [] + if getattr(ctx.attrs, "_generated_build_info_enabled", False): + generated_build_info_args = [flag for flag in binary_linker_flags if _is_build_info_linker_flag(flag)] + binary_linker_flags = [flag for flag in binary_linker_flags if not _is_build_info_linker_flag(flag)] + generated_build_info_invalidation_inputs = ( + [out.object for out in cxx_outs] + + [unpack_link_args(dep_links)] + + [shared_lib.lib.output for shared_lib in shared_libs] + + impl_params.generated_build_info_invalidation_inputs + ) + generated_build_info_shared_library = None + if generated_build_info_is_shared_library(ctx): + binary_linker_flags += ["--build-info=none"] + generated_build_info_shared_library = generate_build_info_shared_library( + ctx, + dep_links, + generator_args = generated_build_info_args, + invalidation_inputs = generated_build_info_invalidation_inputs, + ) + if generated_build_info_shared_library: + shared_libs.append(generated_build_info_shared_library.library) + sub_targets["generated_build_info"] = [ + DefaultInfo(default_output = generated_build_info_shared_library.library.lib.output), + ] + else: + generated_build_info = generate_build_info( + ctx, + generator_args = generated_build_info_args, + invalidation_inputs = generated_build_info_invalidation_inputs, + ) + if generated_build_info: + generated_build_info_compile_output = compile_generated_build_info(ctx, generated_build_info) + generated_build_info_external_debug_info = generated_build_info_compile_output.external_debug_info + binary_linker_flags += generated_build_info_compile_output.objects + generated_build_info.linker_flags + links = [ LinkArgs( infos = [ @@ -720,14 +769,17 @@ def cxx_executable(ctx: AnalysisContext, impl_params: CxxRuleConstructorParams, artifacts = ( [out.object for out in cxx_outs if out.object_has_external_debug_info] + [out.external_debug_info for out in cxx_outs if out.external_debug_info != None] + + generated_build_info_external_debug_info + (impl_params.extra_link_input if impl_params.extra_link_input_has_external_debug_info else []) ), ), ), ] ), - dep_links, - ] + impl_params.extra_link_args + ] + if generated_build_info_shared_library: + links.append(generated_build_info_shared_library.link_args) + links += [dep_links] + impl_params.extra_link_args # If there are hidden dependencies to this target then add them as # hidden link args. @@ -752,6 +804,7 @@ def cxx_executable(ctx: AnalysisContext, impl_params: CxxRuleConstructorParams, shared_libs if impl_params.exe_shared_libs_link_tree else [], impl_params.executable_name, linker_info.binary_extension, + binary_linker_flags, link_options( links = links, link_weight = linker_info.link_weight, @@ -769,6 +822,7 @@ def cxx_executable(ctx: AnalysisContext, impl_params: CxxRuleConstructorParams, incremental_link = incremental_link, has_hip_device_debug = has_hip_device_debug, ), + build_info_json = (generated_build_info_shared_library.json if generated_build_info_shared_library else None), ) binary = link_result.exe runtime_files = link_result.runtime_files @@ -777,16 +831,18 @@ def cxx_executable(ctx: AnalysisContext, impl_params: CxxRuleConstructorParams, gc_sections_data = link_result.gc_sections_data # Define the xcode data sub target - xcode_data_default_info, xcode_data_info = generate_xcode_data( - ctx, - rule_type = impl_params.rule_type, - output = binary.output, - populate_rule_specific_attributes_func = impl_params.cxx_populate_xcode_attributes_func, - srcs = impl_params.srcs + impl_params.additional.srcs, - argsfiles = compile_cmd_output.argsfiles.xcode, - product_name = get_cxx_executable_product_name(ctx, has_hip_device_debug), - ) - sub_targets[XCODE_DATA_SUB_TARGET] = xcode_data_default_info + xcode_data_info = None + if impl_params.generate_sub_targets.xcode_data: + xcode_data_default_info, xcode_data_info = generate_xcode_data( + ctx, + rule_type = impl_params.rule_type, + output = binary.output, + populate_rule_specific_attributes_func = impl_params.cxx_populate_xcode_attributes_func, + srcs = impl_params.srcs + impl_params.additional.srcs, + argsfiles = compile_cmd_output.argsfiles.xcode, + product_name = get_cxx_executable_product_name(ctx, has_hip_device_debug), + ) + sub_targets[XCODE_DATA_SUB_TARGET] = xcode_data_default_info # Info about dynamic-linked libraries for fbpkg integration: # - the symlink dir that's part of RPATH @@ -1028,11 +1084,6 @@ def cxx_executable(ctx: AnalysisContext, impl_params: CxxRuleConstructorParams, sub_targets.update(link_result.extra_outputs) - # Propagate xplugins providers - xplugins_usage_info = get_xplugins_usage_info(ctx.actions, cxx_deps) - if xplugins_usage_info: - sub_targets.update(get_xplugins_usage_subtargets(ctx, xplugins_usage_info, link_group_info)) - xplugins_debug_artifacts_info = xplugins_get_debug_artifacts_info(ctx, cxx_deps) return CxxExecutableOutput( @@ -1063,6 +1114,7 @@ def cxx_executable(ctx: AnalysisContext, impl_params: CxxRuleConstructorParams, validation_specs = get_attrs_validation_specs(ctx), gcno_files = dedupe(gcno_files), xplugins_debug_artifacts_info = xplugins_debug_artifacts_info, + build_info_manifest_entries = (generated_build_info_shared_library.manifest_entries if generated_build_info_shared_library else None), ) _CxxLinkExecutableResult = record( @@ -1127,7 +1179,13 @@ def _get_shared_library_symlink_deps( return shlib_deps def _link_into_executable( - ctx: AnalysisContext, shared_libs: list[SharedLibrary], executable_name: [str, None], binary_extension: str, opts: LinkOptions + ctx: AnalysisContext, + shared_libs: list[SharedLibrary], + executable_name: [str, None], + binary_extension: str, + binary_linker_flags: list[typing.Any], + opts: LinkOptions, + build_info_json: Artifact | None = None, ) -> _CxxLinkExecutableResult: if executable_name and binary_extension and executable_name.endswith(binary_extension): # don't append .exe if it already is .exe @@ -1149,20 +1207,21 @@ def _link_into_executable( ctx = ctx, output = output, result_type = CxxLinkResultType("executable"), - # ctx.attrs.binary_linker_flags should come after default link flags so it can be used to override default settings + # Binary linker flags should come after default link flags so they can override default settings. opts = merge_link_options( opts, binary_links = [ LinkArgs( infos = [ LinkInfo( - pre_flags = ctx.attrs.binary_linker_flags, + pre_flags = binary_linker_flags, ), ] ), ], links = [LinkArgs(flags = executable_args.extra_link_args)] + opts.links, ), + build_info_json = build_info_json, ) return _CxxLinkExecutableResult( @@ -1177,14 +1236,27 @@ def _link_into_executable( extra_outputs = link_result.extra_outputs if link_result.extra_outputs else {}, ) -def get_cxx_executable_product_name(ctx: AnalysisContext, has_hip_device_debug: bool = False) -> str: - name = ctx.label.name +def get_cxx_post_link_suffix(ctx: AnalysisContext, has_hip_device_debug: bool = False) -> str: + """ + The suffix the linker's output must carry so that each post-link stage of + `cxx_link_into` can strip the part it owns, leaving the canonical name. + + Appended in reverse of the order the stages run, so the outermost suffix + belongs to the stage that runs first. + """ + suffix = "" if cxx_stamp_build_info(ctx): - # build_info_stamping is executed after BOLT, make sure the prestamp flag is the innermost prefix - name += PRE_STAMPED_SUFFIX + suffix += PRE_STAMPED_SUFFIX + if get_elf_sections(ctx): + suffix += PRE_ADD_ELF_SECTIONS_SUFFIX if has_hip_device_debug and hip_debug_extract_available(get_cxx_toolchain_info(ctx)): - # Pre-suffix so hip_debug_extract can strip back to canonical name. - name += PRE_EXTRACT_SUFFIX + suffix += PRE_EXTRACT_SUFFIX if cxx_use_bolt(ctx): - name += "-wrapper" - return name + suffix += PRE_BOLT_SUFFIX + return suffix + +def get_cxx_executable_product_name(ctx: AnalysisContext, has_hip_device_debug: bool = False) -> str: + return ctx.label.name + get_cxx_post_link_suffix(ctx, has_hip_device_debug) + +def _is_build_info_linker_flag(flag) -> bool: + return flag.startswith("--build-info") diff --git a/prelude/cxx/cxx_flags.bzl b/prelude/cxx/cxx_flags.bzl new file mode 100644 index 0000000..87a46b6 --- /dev/null +++ b/prelude/cxx/cxx_flags.bzl @@ -0,0 +1,113 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is dual-licensed under either the MIT license found in the +# LICENSE-MIT file in the root directory of this source tree or the Apache +# License, Version 2.0 found in the LICENSE-APACHE file in the root directory +# of this source tree. You may select, at your option, one of the +# above-listed licenses. + +load(":attr_selection.bzl", "cxx_by_language_ext") + +CxxFlags = record( + compiler_flags = field(list[typing.Any], []), + lang_compiler_flags = field(dict[typing.Any, typing.Any], {}), + lang_preprocessor_flags = field(dict[typing.Any, typing.Any], {}), + linker_flags = field(list[typing.Any], []), + preprocessor_flags = field(list[typing.Any], []), +) + +def _compiler_flags(flags: CxxFlags): + return cmd_args(flags.compiler_flags) + +def _linker_flags(flags: CxxFlags): + return cmd_args(flags.linker_flags) + +def _preprocessor_flags(flags: CxxFlags): + return cmd_args(flags.preprocessor_flags) + +def _flags_for_ext(ext: str): + def project(flags: CxxFlags): + return cmd_args( + flags.preprocessor_flags, + cxx_by_language_ext(flags.lang_preprocessor_flags, ext), + cxx_by_language_ext(flags.lang_compiler_flags, ext), + flags.compiler_flags, + ) + + return project + +CxxFlagsTSet = transitive_set( + args_projections = { + "asm": _flags_for_ext(".asm"), + "asmpp": _flags_for_ext(".S"), + "c": _flags_for_ext(".c"), + "compiler_flags": _compiler_flags, + "cuda": _flags_for_ext(".cu"), + "cxx": _flags_for_ext(".cpp"), + "hip": _flags_for_ext(".hip"), + "linker_flags": _linker_flags, + "objc": _flags_for_ext(".m"), + "objcxx": _flags_for_ext(".mm"), + "preprocessor_flags": _preprocessor_flags, + }, +) + +CxxFlagsInfo = provider( + fields = { + "flags": provider_field(typing.Any), # CxxFlagsTSet + }, +) + +def cxx_flags_impl(ctx: AnalysisContext) -> list[Provider]: + return [ + DefaultInfo(), + CxxFlagsInfo( + flags = ctx.actions.tset( + CxxFlagsTSet, + value = CxxFlags( + compiler_flags = ctx.attrs.compiler_flags, + lang_compiler_flags = ctx.attrs.lang_compiler_flags, + lang_preprocessor_flags = ctx.attrs.lang_preprocessor_flags, + linker_flags = ctx.attrs.linker_flags, + preprocessor_flags = ctx.attrs.preprocessor_flags, + ), + children = [dep[CxxFlagsInfo].flags for dep in ctx.attrs.deps], + ), + ), + ] + +def cxx_attr_flags(ctx: AnalysisContext) -> list[Dependency]: + return getattr(ctx.attrs, "flags", []) + +def cxx_flags_tset(actions: AnalysisActions, flag_deps: list[Dependency]): + if not flag_deps: + return None + return actions.tset( + CxxFlagsTSet, + children = [dep[CxxFlagsInfo].flags for dep in flag_deps], + ) + +def cxx_flags_projection_name(ext: str) -> str: + if ext == ".c": + return "c" + elif ext in (".cpp", ".cc", ".cl", ".cxx", ".c++", ".bc"): + return "cxx" + elif ext == ".m": + return "objc" + elif ext == ".mm": + return "objcxx" + elif ext in (".s", ".sx", ".S"): + return "asmpp" + elif ext == ".cu": + return "cuda" + elif ext == ".hip": + return "hip" + elif ext in (".asm", ".asmpp"): + return "asm" + fail("Unexpected file extension: " + ext) + +def cxx_flags_linker_flags(actions: AnalysisActions, flag_deps: list[Dependency]) -> list[typing.Any]: + flags = cxx_flags_tset(actions, flag_deps) + if flags == None: + return [] + return [flags.project_as_args("linker_flags", ordering = "postorder")] diff --git a/prelude/cxx/cxx_library.bzl b/prelude/cxx/cxx_library.bzl index 865c134..2e91712 100644 --- a/prelude/cxx/cxx_library.bzl +++ b/prelude/cxx/cxx_library.bzl @@ -15,8 +15,8 @@ load("@prelude//:attrs_validators.bzl", "get_attrs_validation_specs") load("@prelude//:paths.bzl", "paths") load( "@prelude//:resources.bzl", - "ResourceInfo", "gather_resources", + "make_resource_info", ) load( "@prelude//android:android_providers.bzl", @@ -64,8 +64,8 @@ load( load("@prelude//linking:execution_preference.bzl", "LinkExecutionPreference", "get_link_execution_preference") load( "@prelude//linking:link_groups.bzl", + "EMPTY_LINK_GROUP_LIB_INFO", "LinkGroupLib", # @unused Used as a type - "LinkGroupLibInfo", "gather_link_group_libs", "merge_link_group_lib_info", ) @@ -101,7 +101,7 @@ load( ) load( "@prelude//linking:linkable_graph.bzl", - "DlopenableLibraryInfo", + "DLOPENABLE_LIBRARY_INFO_MARKER", "LinkableRootInfo", "ReducedLinkableGraph", # @unused used as a type "create_linkable_graph", @@ -110,9 +110,18 @@ load( "linkable_deps", "reduce_linkable_graph", ) -load("@prelude//linking:shared_libraries.bzl", "NamedLinkedObject", "SharedLibraryInfo", "create_flavored_shared_libraries", "merge_shared_libraries") +load( + "@prelude//linking:shared_libraries.bzl", + "EMPTY_SHARED_LIBRARY_INFO", + "NamedLinkedObject", + "SharedLibraryInfo", + "create_flavored_shared_libraries", + "merge_shared_libraries", +) load("@prelude//linking:strip.bzl", "strip_debug_info") load("@prelude//linking:types.bzl", "Linkage") +load("@prelude//target_stats:target_stats.bzl", "CycleMode", "target_stats_providers_and_subtargets") +load("@prelude//target_stats:target_stats_config.bzl", "TARGET_STATS_ENABLED") load( "@prelude//third-party:build.bzl", "create_third_party_build_info", @@ -128,7 +137,7 @@ load( "value_or", ) load("@prelude//xplugins:debug_artifacts.bzl", "xplugins_get_debug_artifacts_info") -load("@prelude//xplugins:utils.bzl", "get_xplugins_usage_info", "get_xplugins_usage_subtargets") +load("@prelude//xplugins:utils.bzl", "get_xplugins_usage_info") load(":archive.bzl", "make_archive") load( ":argsfiles.bzl", @@ -169,6 +178,7 @@ load( ) load( ":cxx_library_utility.bzl", + "EMPTY_DEFAULT_INFO", "OBJECTS_SUBTARGET", "cxx_attr_dep_metadata", "cxx_attr_deps", @@ -414,7 +424,6 @@ def cxx_library_parameterized(ctx: AnalysisContext, impl_params: CxxRuleConstruc """ Defines the outputs for a cxx library, return the default output and any subtargets and providers based upon the requested params. """ - if not cxx_platform_supported(ctx): sub_targets = {} @@ -425,7 +434,7 @@ def cxx_library_parameterized(ctx: AnalysisContext, impl_params: CxxRuleConstruc return _CxxLibraryParameterizedOutput( providers = [ DefaultInfo(default_output = None, sub_targets = sub_targets), - SharedLibraryInfo(set = None), + EMPTY_SHARED_LIBRARY_INFO, ], sub_targets = sub_targets, ) @@ -586,6 +595,7 @@ def cxx_library_parameterized(ctx: AnalysisContext, impl_params: CxxRuleConstruc if impl_params.generate_sub_targets.argsfiles: sub_targets[ARGSFILES_SUBTARGET] = [get_argsfiles_output(ctx, compiled_srcs.compile_cmds.argsfiles.relative, ARGSFILES_SUBTARGET)] + if impl_params.generate_sub_targets.xcode_data: sub_targets[XCODE_ARGSFILES_SUB_TARGET] = [get_argsfiles_output(ctx, compiled_srcs.compile_cmds.argsfiles.xcode, XCODE_ARGSFILES_SUB_TARGET)] if impl_params.generate_sub_targets.clang_remarks: @@ -991,7 +1001,7 @@ def cxx_library_parameterized(ctx: AnalysisContext, impl_params: CxxRuleConstruc own_exported_preprocessors.extend(header_unit_preprocessors) else: sub_targets["header-unit"] = [ - DefaultInfo(), + EMPTY_DEFAULT_INFO, cxx_merge_cpreprocessors( ctx.actions, own_exported_preprocessors, @@ -1071,8 +1081,8 @@ def cxx_library_parameterized(ctx: AnalysisContext, impl_params: CxxRuleConstruc frameworks_linkable = frameworks_linkable, swiftmodule_linkable = swiftmodule_linkable, ), - LinkGroupLibInfo(libs = {}), - SharedLibraryInfo(set = None), + EMPTY_LINK_GROUP_LIB_INFO, + EMPTY_SHARED_LIBRARY_INFO, ] + additional_providers if getattr(ctx.attrs, "supports_header_symlink_subtarget", False): @@ -1130,7 +1140,7 @@ def cxx_library_parameterized(ctx: AnalysisContext, impl_params: CxxRuleConstruc sub_targets.update(mod_dep_graph_subtargets) providers.append(mod_dep_graph_info) - linker_flags = cxx_attr_linker_flags_all(ctx) + linker_flags = cxx_attr_linker_flags_all(ctx, impl_params.cxx_flags) # Omnibus root provider. linkable_root = None @@ -1187,7 +1197,7 @@ def cxx_library_parameterized(ctx: AnalysisContext, impl_params: CxxRuleConstruc # Mark libraries that support `dlopen`. if getattr(ctx.attrs, "supports_python_dlopen", False): - providers.append(DlopenableLibraryInfo()) + providers.append(DLOPENABLE_LIBRARY_INFO_MARKER) # Augment and provide the linkable graph. if impl_params.generate_providers.linkable_graph: @@ -1205,6 +1215,10 @@ def cxx_library_parameterized(ctx: AnalysisContext, impl_params: CxxRuleConstruc # If we don't have link input for this link style, we pass in `None` so # that omnibus knows to avoid it. include_in_android_mergemap = getattr(ctx.attrs, "include_in_android_merge_map_output", True) and default_output != None, + # Attach this library's per-source compile commands so app rules can aggregate a + # native compile-command database. Gated (default off) to keep this off the graph + # for normal builds; see _emit_native_build_commands. + compile_cmds = compiled_srcs.compile_cmds.src_compile_cmds if getattr(ctx.attrs, "_emit_native_build_commands", False) else [], link_infos = library_outputs.link_infos, shared_libs = shared_libs, linker_flags = linker_flags, @@ -1221,13 +1235,13 @@ def cxx_library_parameterized(ctx: AnalysisContext, impl_params: CxxRuleConstruc if impl_params.generate_providers.resources: resources = cxx_attr_resources(ctx) providers.append( - ResourceInfo( - resources = gather_resources( + make_resource_info( + gather_resources( label = ctx.label, resources = resources, deps = deps_all_non_exported_first, - ) - ) + ), + ), ) if impl_params.generate_providers.cxx_resources_as_apple_resources: apple_resource_graph = create_resource_graph( @@ -1345,8 +1359,8 @@ def cxx_library_parameterized(ctx: AnalysisContext, impl_params: CxxRuleConstruc linkage_providers = [ linkage_merged_link_info, propagated_preprocessor, - LinkGroupLibInfo(libs = {}), - SharedLibraryInfo(set = None), + EMPTY_LINK_GROUP_LIB_INFO, + EMPTY_SHARED_LIBRARY_INFO, ] # Only create LinkableGraph if we have all required output styles for this linkage @@ -1363,6 +1377,7 @@ def cxx_library_parameterized(ctx: AnalysisContext, impl_params: CxxRuleConstruc deps = non_exported_deps, exported_deps = exported_deps, include_in_android_mergemap = getattr(ctx.attrs, "include_in_android_merge_map_output", True) and default_output != None, + compile_cmds = compiled_srcs.compile_cmds.src_compile_cmds if getattr(ctx.attrs, "_emit_native_build_commands", False) else [], link_infos = library_outputs.link_infos, shared_libs = shared_libs, linker_flags = linker_flags, @@ -1382,11 +1397,9 @@ def cxx_library_parameterized(ctx: AnalysisContext, impl_params: CxxRuleConstruc ) sub_targets["prefer-{}".format(linkage.value)] = linkage_providers - # Propagate xplugins providers xplugins_usage_info = get_xplugins_usage_info(ctx.actions, deps_all_non_exported_first) if xplugins_usage_info: providers.append(xplugins_usage_info) - sub_targets.update(get_xplugins_usage_subtargets(ctx, xplugins_usage_info, link_group_info)) xplugins_debug_info = xplugins_get_debug_artifacts_info( ctx, @@ -1395,6 +1408,25 @@ def cxx_library_parameterized(ctx: AnalysisContext, impl_params: CxxRuleConstruc if xplugins_debug_info: providers.append(xplugins_debug_info) + if TARGET_STATS_ENABLED: + target_stats_tools = get_cxx_toolchain_info(ctx).target_stats_tools + if target_stats_tools != None: + # Keyed by each file's path within the target, which is what a + # consumer can line up against the target's own source list. + target_stats_srcs = {src.file.short_path: src.file for src in impl_params.srcs + impl_params.additional.srcs} + target_stats_srcs.update(impl_params.target_stats_extra_srcs) + target_stats_providers, target_stats_subtargets = target_stats_providers_and_subtargets( + ctx, + tools = target_stats_tools, + srcs = target_stats_srcs, + deps = non_exported_deps + exported_deps, + cycle_mode = CycleMode(impl_params.target_stats_cycle_mode), + module_name = impl_params.target_stats_module_name or ctx.label.name, + swift_dot = impl_params.target_stats_swift_dot, + ) + providers.extend(target_stats_providers) + sub_targets.update(target_stats_subtargets) + if impl_params.generate_providers.default: if False: # TODO(nga): `default_output.unstripped` is never `None`. @@ -1549,6 +1581,7 @@ def cxx_compile_srcs( add_coverage_instrumentation_compiler_flags: bool, compile_pch: CxxPrecompiledHeader | None = None, own_exported_preprocessors: list[CPreprocessor] = [], + filename_prefix: str = "", ) -> _CxxCompiledSourcesOutput: """ Compile objects we'll need for archives and shared libraries. @@ -1564,6 +1597,7 @@ def cxx_compile_srcs( inherited_preprocessor_infos = inherited_non_exported_preprocessor_infos + inherited_exported_preprocessor_infos, add_coverage_instrumentation_compiler_flags = add_coverage_instrumentation_compiler_flags, compile_pch = compile_pch, + filename_prefix = filename_prefix, ) # Define header unit. @@ -1714,7 +1748,7 @@ def _form_library_outputs( gcno_files = [] linker_info = get_cxx_toolchain_info(ctx).linker_info - linker_flags = cxx_attr_linker_flags_all(ctx) + linker_flags = cxx_attr_linker_flags_all(ctx, impl_params.cxx_flags) # Add in exported linker flags. def ldflags(inner: LinkInfo) -> LinkInfo: @@ -2351,7 +2385,7 @@ def _shared_library( # does, but the intent of exported link flags are to wrap the link output # that we propagate up the tree, rather than being used locally when # generating a link product. - linker_flags = cxx_attr_linker_flags_all(ctx) + linker_flags = cxx_attr_linker_flags_all(ctx, impl_params.cxx_flags) link_info = LinkInfo( dist_thin_lto_codegen_flags = getattr(ctx.attrs, "dist_thin_lto_codegen_flags", []), pre_flags = (linker_flags.flags + linker_flags.exported_flags + getattr(ctx.attrs, "local_linker_flags", [])), diff --git a/prelude/cxx/cxx_library_utility.bzl b/prelude/cxx/cxx_library_utility.bzl index 0254736..500dcc0 100644 --- a/prelude/cxx/cxx_library_utility.bzl +++ b/prelude/cxx/cxx_library_utility.bzl @@ -26,6 +26,7 @@ load( "from_named_set", ) load(":cxx_context.bzl", "get_cxx_platform_info", "get_cxx_toolchain_info") +load(":cxx_flags.bzl", "cxx_flags_linker_flags") load( ":cxx_toolchain_types.bzl", "LinkerType", @@ -34,6 +35,8 @@ load( OBJECTS_SUBTARGET = "objects" +EMPTY_DEFAULT_INFO = DefaultInfo() + # The dependencies, and the default deps (if selected) def cxx_attr_deps(ctx: AnalysisContext) -> list[Dependency]: deps = ctx.attrs.deps @@ -65,8 +68,8 @@ def cxx_attr_exported_deps(ctx: AnalysisContext) -> list[Dependency]: return exported_deps -def cxx_attr_linker_flags_all(ctx: AnalysisContext) -> LinkerFlags: - flags = cxx_attr_linker_flags(ctx) +def cxx_attr_linker_flags_all(ctx: AnalysisContext, cxx_flags: list[Dependency] = []) -> LinkerFlags: + flags = cxx_attr_linker_flags(ctx, cxx_flags) local_linker_script_flags_attr = getattr(ctx.attrs, "local_linker_script_flags", None) if local_linker_script_flags_attr: @@ -92,8 +95,8 @@ def cxx_inherited_link_info(first_order_deps: list[Dependency]) -> list[MergedLi return filter_and_map_idx(MergedLinkInfo, first_order_deps) # Linker flags -def cxx_attr_linker_flags(ctx: AnalysisContext) -> list[typing.Any]: - linker_flags = list(ctx.attrs.linker_flags) +def cxx_attr_linker_flags(ctx: AnalysisContext, cxx_flags: list[Dependency] = []) -> list[typing.Any]: + linker_flags = cxx_flags_linker_flags(ctx.actions, cxx_flags) + ctx.attrs.linker_flags return linker_flags def cxx_attr_link_style(ctx: AnalysisContext) -> LinkStyle: diff --git a/prelude/cxx/cxx_toolchain.bzl b/prelude/cxx/cxx_toolchain.bzl index d3d75b2..67f8bb5 100644 --- a/prelude/cxx/cxx_toolchain.bzl +++ b/prelude/cxx/cxx_toolchain.bzl @@ -7,6 +7,7 @@ # above-listed licenses. load("@prelude//:is_full_meta_repo.bzl", "is_full_meta_repo") +load("@prelude//cxx:compile.bzl", "compiler_info_with_toolchain_argsfiles") load( "@prelude//cxx:cxx_toolchain_types.bzl", "AsCompilerInfo", @@ -129,6 +130,7 @@ def cxx_toolchain_impl(ctx): cuda_info = ( CudaCompilerInfo( compiler = ctx.attrs.cuda_compiler[RunInfo], + compiler_for_dryrun = ctx.attrs.cuda_compiler_for_dryrun[RunInfo] if ctx.attrs.cuda_compiler_for_dryrun else None, compiler_type = ctx.attrs.cuda_compiler_type or ctx.attrs.compiler_type, compiler_flags = cmd_args(ctx.attrs.cuda_compiler_flags), preprocessor_flags = cmd_args(ctx.attrs.cuda_preprocessor_flags), @@ -169,6 +171,24 @@ def cxx_toolchain_impl(ctx): else None ) + c_info = compiler_info_with_toolchain_argsfiles(ctx.actions, "c", CCompilerInfo, c_info) + objc_info = compiler_info_with_toolchain_argsfiles(ctx.actions, "objc", ObjcCompilerInfo, objc_info) + cxx_info = compiler_info_with_toolchain_argsfiles( + ctx.actions, + "cxx", + CxxCompilerInfo, + cxx_info, + # Only the cxx info can be precompiled for C++20 modules. + precompile_filter = ctx.attrs.internal_tools[CxxInternalTools].filter_argsfile, + ) + objcxx_info = compiler_info_with_toolchain_argsfiles(ctx.actions, "objcxx", ObjcxxCompilerInfo, objcxx_info) + asm_info = compiler_info_with_toolchain_argsfiles(ctx.actions, "asm", AsmCompilerInfo, asm_info) + as_info = compiler_info_with_toolchain_argsfiles(ctx.actions, "as", AsCompilerInfo, as_info) + cuda_info = compiler_info_with_toolchain_argsfiles(ctx.actions, "cuda", CudaCompilerInfo, cuda_info) + hip_info = compiler_info_with_toolchain_argsfiles(ctx.actions, "hip", HipCompilerInfo, hip_info) + cvtres_info = compiler_info_with_toolchain_argsfiles(ctx.actions, "cvtres", CvtresCompilerInfo, cvtres_info) + rc_info = compiler_info_with_toolchain_argsfiles(ctx.actions, "rc", RcCompilerInfo, rc_info) + linker_type = LinkerType(ctx.attrs.linker_type) linker_info = LinkerInfo( archiver = ctx.attrs.archiver[RunInfo], @@ -228,13 +248,13 @@ def cxx_toolchain_impl(ctx): utilities_info = BinaryUtilitiesInfo( bolt = ctx.attrs.bolt[RunInfo] if ctx.attrs.bolt else None, custom_tools = {name: dep[RunInfo] for name, dep in ctx.attrs.custom_tools.items()}, + elf_stamp = ctx.attrs.elf_stamp[RunInfo] if ctx.attrs.elf_stamp else None, nm = ctx.attrs.nm[RunInfo], objcopy = ctx.attrs.objcopy_for_shared_library_interface[RunInfo], objdump = ctx.attrs.objdump[RunInfo] if ctx.attrs.objdump else None, ranlib = ctx.attrs.ranlib[RunInfo] if ctx.attrs.ranlib else None, strip = ctx.attrs.strip[RunInfo], dwp = ctx.attrs.dwp[RunInfo] if ctx.attrs.dwp else None, - bolt_msdk = None, ) strip_flags_info = StripFlagsInfo( @@ -314,12 +334,14 @@ def cxx_toolchain_extra_attributes(is_toolchain_rule): "compiler_flavor_flags": attrs.dict(key = attrs.string(), value = attrs.list(attrs.string()), default = {}), "cpp_dep_tracking_mode": attrs.enum(DepTrackingMode.values(), default = "makefile"), "cuda_compiler": attrs.option(dep_type(providers = [RunInfo]), default = None), + "cuda_compiler_for_dryrun": attrs.option(dep_type(providers = [RunInfo]), default = None), "cuda_dep_tracking_mode": attrs.enum(DepTrackingMode.values(), default = "makefile"), "custom_tools": attrs.dict(key = attrs.string(), value = dep_type(providers = [RunInfo]), default = {}), "cvtres_compiler": attrs.option(dep_type(providers = [RunInfo]), default = None), "cxx_compiler": dep_type(providers = [RunInfo]), "default_deps": attrs.list(dep_type(), default = []), "dwp": attrs.option(dep_type(providers = [RunInfo]), default = None), + "elf_stamp": attrs.option(dep_type(providers = [RunInfo]), default = None), "gcno_files": attrs.bool(default = False), "generate_gc_sections": attrs.bool(default = False), "generate_linker_maps": attrs.bool(default = False), diff --git a/prelude/cxx/cxx_toolchain_types.bzl b/prelude/cxx/cxx_toolchain_types.bzl index 9b2e549..9b3ef3e 100644 --- a/prelude/cxx/cxx_toolchain_types.bzl +++ b/prelude/cxx/cxx_toolchain_types.bzl @@ -10,6 +10,8 @@ load("@prelude//cxx:cxx_apple_linker_flags.bzl", "apple_extra_darwin_linker_flag load("@prelude//cxx:debug.bzl", "SplitDebugMode") load("@prelude//target_stats:target_stats_tools.bzl", "TargetStatsToolsInfo") +CXX_COMPILER_TYPES = ["clang", "clang_cl", "clang_windows", "gcc", "windows", "windows_ml64"] + LinkerType = enum("gnu", "darwin", "windows", "wasm") IncrementalLinkingMode = enum("disabled", "supported", "enabled") @@ -99,9 +101,9 @@ LinkerInfo = provider( BinaryUtilitiesInfo = provider( fields = { "bolt": provider_field(typing.Any, default = None), - "bolt_msdk": provider_field(typing.Any, default = None), "custom_tools": provider_field(dict[str, RunInfo], default = {}), "dwp": provider_field(typing.Any, default = None), + "elf_stamp": provider_field([RunInfo, None], default = None), "nm": provider_field(typing.Any, default = None), "objcopy": provider_field(typing.Any, default = None), "objdump": provider_field(typing.Any, default = None), @@ -147,11 +149,14 @@ _compiler_fields = [ "compiler", "compiler_type", "compiler_flags", - # An optional @argsfile `Artifact` that contains the preprocessor flags and the compiler flags. - "argsfile", # `Artifact | None` - # An optional @argsfile `Artifact` that contains the preprocessor flags and the compiler flags + # An optional argsfile that contains the preprocessor flags and the compiler flags. + "argsfile", # `cmd_args | None` + # An optional argsfile that contains the preprocessor flags and the compiler flags, # formatted for xcode. - "argsfile_xcode", # `Artifact | None` + "argsfile_xcode", # `cmd_args | None` + # An optional copy of `argsfile` filtered for C++20 module precompilation, + # as an `argsfile_with_artifacts()` value. + "argsfile_precompile", # `cmd_args | None` "preprocessor", "preprocessor_type", "preprocessor_flags", @@ -165,7 +170,7 @@ _compiler_fields = [ AsCompilerInfo = provider(fields = _compiler_fields) AsmCompilerInfo = provider(fields = _compiler_fields) CCompilerInfo = provider(fields = _compiler_fields) -CudaCompilerInfo = provider(fields = _compiler_fields) +CudaCompilerInfo = provider(fields = _compiler_fields + ["compiler_for_dryrun"]) CvtresCompilerInfo = provider(fields = _compiler_fields) CxxCompilerInfo = provider(fields = _compiler_fields) HipCompilerInfo = provider(fields = _compiler_fields) @@ -173,6 +178,20 @@ ObjcCompilerInfo = provider(fields = _compiler_fields) ObjcxxCompilerInfo = provider(fields = _compiler_fields) RcCompilerInfo = provider(fields = _compiler_fields) +def compiler_info_with_argsfiles( + compiler_info: typing.Any, ctor: typing.Callable, argsfile: cmd_args, argsfile_xcode: cmd_args, argsfile_precompile: cmd_args | None = None +) -> typing.Any: + fields = {k: getattr(compiler_info, k) for k in _compiler_fields} + fields["argsfile"] = argsfile + fields["argsfile_xcode"] = argsfile_xcode + fields["argsfile_precompile"] = argsfile_precompile + + # CudaCompilerInfo carries fields beyond the shared set. + compiler_for_dryrun = getattr(compiler_info, "compiler_for_dryrun", None) + if compiler_for_dryrun != None: + fields["compiler_for_dryrun"] = compiler_for_dryrun + return ctor(**fields) + DistLtoToolsInfo = provider( fields = dict( planner = dict[LinkerType, RunInfo], diff --git a/prelude/cxx/cxx_types.bzl b/prelude/cxx/cxx_types.bzl index cbfd0ef..53c364e 100644 --- a/prelude/cxx/cxx_types.bzl +++ b/prelude/cxx/cxx_types.bzl @@ -84,12 +84,16 @@ CxxRuleSubTargetParams = record( headers = field(bool, True), link_group_map = field(bool, True), link_style_outputs = field(bool, True), - xcode_data = field(bool, True), + xcode_data = field(bool, False), objects = field(bool, True), bitcode_bundle = field(bool, True), header_unit = field(bool, True), ) +def xcode_data_enabled() -> bool: + # Project generators must manually enable xcode data because it measurably increases memory usage. + return read_root_config("cxx", "enable_xcode_data", "false").lower() == "true" + # Parameters to control which providers to define when processing Cxx rules. # By default, generates all providers. CxxRuleProviderParams = record( @@ -163,6 +167,8 @@ CxxRuleConstructorParams = record( # These flags are _not_ propagated up the dep tree. extra_link_flags = field(list[typing.Any], []), extra_binary_link_flags = field(list[typing.Any], []), + # Inputs that invalidate generated build info without becoming inputs to the executable link. + generated_build_info_invalidation_inputs = field(list[typing.Any], []), # Additional artifacts to be linked together with the cxx compilation output. extra_link_input = field(list[Artifact], []), # If True the extra_link_input should be considered as external debug info. @@ -247,6 +253,7 @@ CxxRuleConstructorParams = record( extra_shared_library_interfaces = field([list[Artifact], None], None), # Compiler flags compiler_flags = field(list[typing.Any], []), + cxx_flags = field(list[Dependency], []), lang_compiler_flags = field(dict[typing.Any, typing.Any], {}), # Preprocessor flags preprocessor_flags = field(list[typing.Any], []), @@ -299,4 +306,10 @@ CxxRuleConstructorParams = record( supports_stripping = field(bool, True), # Whether to set expect_eligible_for_dedupe on compile actions. expect_eligible_for_dedupe = field(bool, False), + target_stats_cycle_mode = field(str, "none"), + target_stats_extra_srcs = field(dict[str, Artifact], {}), + target_stats_swift_dot = field(Artifact | None, None), + # The module the cycle tools resolve imports against. Not how the target is + # reported -- that stays the target's label. + target_stats_module_name = field(str | None, None), ) diff --git a/prelude/cxx/dist_lto/darwin/dist_lto.bzl b/prelude/cxx/dist_lto/darwin/dist_lto.bzl index ec2df73..c1f0ce8 100644 --- a/prelude/cxx/dist_lto/darwin/dist_lto.bzl +++ b/prelude/cxx/dist_lto/darwin/dist_lto.bzl @@ -148,14 +148,15 @@ def complete_distributed_link_with_expanded_archive_link_data( def name_for_obj(link_name: str, object_artifact: Artifact) -> str: """Creates a unique name/path we can use for a particular object file input""" prefix = "{}/{}".format(link_name, object_artifact.short_path) - - # it's possible (though unlikely) that we can get duplicate name/short_path, so just uniquify them - if prefix in recorded_outputs: - recorded_outputs[prefix] += 1 - extra = recorded_outputs[prefix] + case_insensitive_prefix = prefix.lower() + # it's possible (though unlikely) that we can get duplicate name/short_path, so just uniquify them. + # Make sure we case normalize before to avoid collisions on case in-sensitive file systems. + if case_insensitive_prefix in recorded_outputs: + recorded_outputs[case_insensitive_prefix] += 1 + extra = recorded_outputs[case_insensitive_prefix] prefix = "{}-{}".format(prefix, extra) else: - recorded_outputs[prefix] = 1 + recorded_outputs[case_insensitive_prefix] = 1 return prefix names = {} @@ -789,11 +790,13 @@ def cxx_darwin_dist_link( if isinstance(linkable, ArchiveLinkable) and not linkable.archive.external_objects: link_name = link.name or "unknown" archive_name_candidate = "{}-{}".format(link_name, linkable.archive.artifact.short_path) - if archive_name_candidate in recorded_artifact_names: - recorded_artifact_names[archive_name_candidate] += 1 - archive_name = "{}-{}".format(archive_name_candidate, recorded_artifact_names[archive_name_candidate]) + # On case in-sensitive file systems, we need to make sure that, we normalize case before checking for duplicates + case_insensitive_archive_name_candidate = archive_name_candidate.lower() + if case_insensitive_archive_name_candidate in recorded_artifact_names: + recorded_artifact_names[case_insensitive_archive_name_candidate] += 1 + archive_name = "{}-{}".format(archive_name_candidate, recorded_artifact_names[case_insensitive_archive_name_candidate]) else: - recorded_artifact_names[archive_name_candidate] = 1 + recorded_artifact_names[case_insensitive_archive_name_candidate] = 1 archive_name = archive_name_candidate archive_manifest = ctx.actions.declare_output( diff --git a/prelude/cxx/dist_lto/dist_lto.bzl b/prelude/cxx/dist_lto/dist_lto.bzl index d606421..c1b4838 100644 --- a/prelude/cxx/dist_lto/dist_lto.bzl +++ b/prelude/cxx/dist_lto/dist_lto.bzl @@ -14,6 +14,7 @@ load( load("@prelude//:paths.bzl", "paths") load( "@prelude//cxx:cxx_bolt.bzl", + "PRE_BOLT_SUFFIX", "bolt", "cxx_use_bolt", ) @@ -132,6 +133,7 @@ def cxx_gnu_dist_link( # This action will only happen if split_dwarf is enabled via the toolchain. dwp_tool_available: bool = True, executable_link: bool = True, + build_info_json: Artifact | None = None, ) -> LinkedObject: """ Perform a distributed thin-lto link into the supplied output @@ -163,6 +165,9 @@ def cxx_gnu_dist_link( normalized_identifier = identifier.replace("[", "_").replace("]", "_").replace(" ", "_") if identifier != None else None enable_late_build_info_stamping = executable_link and cxx_stamp_build_info(ctx) + + # Unlike the plain link, a `None` (no preference) narrows to `False` here, + # because this also feeds `bolt` and `strip_object`, which take a plain bool. enable_cache_upload = opts.allow_cache_upload or enable_late_build_info_stamping link_action_execution_properties = get_action_execution_attributes(opts.link_execution_preference) @@ -823,7 +828,7 @@ def cxx_gnu_dist_link( split_debug_output = None if dwp_tool_available: - dwp_output = ctx.actions.declare_output(output.short_path.removesuffix("-wrapper") + ".dwp", has_content_based_path = False) + dwp_output = ctx.actions.declare_output(output.short_path.removesuffix(PRE_BOLT_SUFFIX) + ".dwp", has_content_based_path = False) def dynamic_run_dwp_action(ctx: AnalysisContext, artifacts, outputs): plan = artifacts[link_plan_out].read_json() @@ -881,7 +886,16 @@ def cxx_gnu_dist_link( strip_args = opts.strip_args_factory(ctx) if opts.strip_args_factory else cmd_args() final_output = strip_object(ctx, cxx_toolchain, final_output, strip_args, category_suffix, allow_cache_upload = enable_cache_upload) - final_output = stamp_build_info(ctx, final_output, links = opts.links) if executable_link else final_output + final_output = ( + stamp_build_info( + ctx, + final_output, + links = opts.links, + build_info_json = build_info_json, + ) + if executable_link + else final_output + ) return LinkedObject( output = final_output, diff --git a/prelude/cxx/dist_lto/tools/dist_lto_planner_gnu.py b/prelude/cxx/dist_lto/tools/dist_lto_planner_gnu.py index 1d8338c..25d5cfb 100644 --- a/prelude/cxx/dist_lto/tools/dist_lto_planner_gnu.py +++ b/prelude/cxx/dist_lto/tools/dist_lto_planner_gnu.py @@ -405,6 +405,7 @@ def index_path(path): "crtendS.o", "glibc/lib/crtn.o", ] + GENERATED_BUILD_INFO_OBJECT_BASENAME_PREFIX = "__buck2_generated_build_info.c" with open(index_path("index")) as indexfile: for line in indexfile: line = line.strip() @@ -461,7 +462,11 @@ def index_path(path): for line in full_index_input: line = line.strip() - if any(filter(line.endswith, KNOWN_REMOVABLE_DEPS_SUFFIX)): + if any( + filter(line.endswith, KNOWN_REMOVABLE_DEPS_SUFFIX) + ) or os.path.basename(line).startswith( + GENERATED_BUILD_INFO_OBJECT_BASENAME_PREFIX + ): continue # LLVM IR files that are indexed in Phase 2 Thin Link are reported by `ld.lld` to exist under the directory `args.index`, # but they actually exist at the path with the prefix `args.index` removed. diff --git a/prelude/cxx/dwp.bzl b/prelude/cxx/dwp.bzl index d0679cf..9ee436c 100644 --- a/prelude/cxx/dwp.bzl +++ b/prelude/cxx/dwp.bzl @@ -63,6 +63,9 @@ def run_dwp_action( prefer_remote = action_execution_properties.prefer_remote, local_only = action_execution_properties.local_only, force_full_hybrid_if_capable = action_execution_properties.full_hybrid, + # dwp merges the debug info of the corpus the link consumed and its + # memory scales the same way, so share the link's local-slot weight. + weight = toolchain.linker_info.link_weight, ) def dwp( diff --git a/prelude/cxx/groups.bzl b/prelude/cxx/groups.bzl index 79a1a06..14d1bc3 100644 --- a/prelude/cxx/groups.bzl +++ b/prelude/cxx/groups.bzl @@ -40,6 +40,7 @@ _VALID_ATTRS = [ "discard_group", "linker_flags", "linker_script", + "version_script", "exported_linker_flags", "link_execution_preference", "no_as_needed", @@ -114,6 +115,7 @@ def parse_groups_definitions( discard_group = attrs.get("discard_group", False), linker_flags = attrs.get("linker_flags", []), linker_script = attrs.get("linker_script", None), + version_script = attrs.get("version_script", None), exported_linker_flags = attrs.get("exported_linker_flags", []), link_execution_preference = attrs.get("link_execution_preference", None), no_as_needed = attrs.get("no_as_needed", False), @@ -171,10 +173,7 @@ def _parse_filter(entry: str) -> GroupFilterInfo: regex_expr = regex("^{}$".format(label_regex), fancy = False) def matches_regex(_r, _t, labels): - for label in labels: - if regex_expr.match(label): - return True - return False + return regex_expr.any_match(labels) return GroupFilterInfo( matches = matches_regex, diff --git a/prelude/cxx/groups_types.bzl b/prelude/cxx/groups_types.bzl index d8bc345..7c9d4d3 100644 --- a/prelude/cxx/groups_types.bzl +++ b/prelude/cxx/groups_types.bzl @@ -69,6 +69,13 @@ GroupAttrs = record( linker_flags = field(list[typing.Any], []), # Passes a linker script to the link group shared library's link command. linker_script = field([Artifact, None], None), + # Passes a version script to the link group shared library's link command. + # Tagging a group's symbols with a version keeps them from satisfying an + # identically-named *versioned* reference in another copy of the same library + # loaded into the process, which is otherwise allowed: glibc lets an + # unversioned definition satisfy a versioned reference, so whichever copy + # reaches the global scope first captures the other. + version_script = field([Artifact, None], None), # Adds additional linker flags to apply to dependents that link against the # link group's shared object. exported_linker_flags = field(list[typing.Any], []), diff --git a/prelude/cxx/headers.bzl b/prelude/cxx/headers.bzl index 1ed1fc0..58cf748 100644 --- a/prelude/cxx/headers.bzl +++ b/prelude/cxx/headers.bzl @@ -73,6 +73,8 @@ HeaderStyle = enum( Headers = record( include_path = field(cmd_args), + # The header map on the include path, when the header mode uses one. + header_map = field(Artifact | None, None), # NOTE(agallagher): Used for module hack replacement. symlink_tree = field(Artifact | None, None), # -fdebug-prefix-map args that map symlinked headers to source path @@ -268,6 +270,7 @@ def prepare_headers( hmap = _mk_hmap(actions, cxx_toolchain_info, output_name, headers, allow_cache_upload, uses_content_based_paths) return Headers( include_path = cmd_args(hmap, hidden = srcs.values()), + header_map = hmap, ) symlink_dir = actions.symlinked_dir( output_name, @@ -298,6 +301,7 @@ def prepare_headers( coverage_prefix_args = cmd_args(cmd_args(symlink_dir, format = "-fcoverage-prefix-map={}=" + replacement)) return Headers( include_path = cmd_args(hmap, hidden = symlink_dir), + header_map = hmap, symlink_tree = symlink_dir, file_prefix_args = file_prefix_args, coverage_prefix_args = coverage_prefix_args, diff --git a/prelude/cxx/link.bzl b/prelude/cxx/link.bzl index b9d3246..41643d2 100644 --- a/prelude/cxx/link.bzl +++ b/prelude/cxx/link.bzl @@ -32,6 +32,7 @@ load( "@prelude//cxx/dist_lto/darwin:dist_lto.bzl", "cxx_darwin_dist_link", ) +load("@prelude//linking:add_elf_sections.bzl", "add_elf_sections_to_executable") load("@prelude//linking:execution_preference.bzl", "LinkExecutionPreference", "LinkExecutionPreferenceInfo", "get_action_execution_attributes") load( "@prelude//linking:link_info.bzl", @@ -59,6 +60,7 @@ load( ) load(":bitcode.bzl", "make_bitcode_bundle") load(":cxx_context.bzl", "get_cxx_toolchain_info") +load(":cxx_library_utility.bzl", "EMPTY_DEFAULT_INFO") load( ":cxx_link_utility.bzl", "LinkArgsOutput", @@ -177,6 +179,8 @@ def cxx_link_into( output: Artifact, result_type: CxxLinkResultType, opts: LinkOptions, + output_has_content_based_path: bool = False, + build_info_json: Artifact | None = None, ) -> CxxLinkResult: cxx_toolchain_info = opts.cxx_toolchain or get_cxx_toolchain_info(ctx) linker_info = cxx_toolchain_info.linker_info @@ -246,6 +250,7 @@ def cxx_link_into( gc_sections_output, dwp_tool_available, is_result_executable, + build_info_json, ) extra_outputs = {} else: @@ -457,6 +462,14 @@ def cxx_link_into( enable_late_build_info_stamping = is_result_executable and cxx_stamp_build_info(ctx) + if is_incremental_link: + allow_cache_upload = False + elif enable_late_build_info_stamping: + allow_cache_upload = True + else: + # Preserves `None`: no preference, as opposed to a decision not to upload. + allow_cache_upload = opts.allow_cache_upload + ctx.actions.run( command, prefer_local = action_execution_properties.prefer_local and not is_incremental_link, @@ -466,10 +479,9 @@ def cxx_link_into( category = category, identifier = opts.identifier, force_full_hybrid_if_capable = action_execution_properties.full_hybrid, - allow_cache_upload = (opts.allow_cache_upload or enable_late_build_info_stamping) and not is_incremental_link, + allow_cache_upload = allow_cache_upload, error_handler = opts.error_handler, no_outputs_cleanup = is_incremental_link, - eager_materialization_enabled = True, ) external_debug_info = link_external_debug_info( @@ -484,6 +496,7 @@ def cxx_link_into( strip_args = opts.strip_args_factory(ctx) if opts.strip_args_factory else cmd_args() output = strip_object(ctx, cxx_toolchain_info, output, strip_args, opts.category_suffix, allow_cache_upload = enable_late_build_info_stamping) + prebolt_output = output use_bolt = is_result_executable and cxx_use_bolt(ctx) if use_bolt: bolt_output = bolt(ctx, output, external_debug_info, opts.identifier, dwp_tool_available, allow_cache_upload = enable_late_build_info_stamping) @@ -527,13 +540,19 @@ def cxx_link_into( output = renamed if is_result_executable: - output = stamp_build_info(ctx, output, links = opts.links) + output = add_elf_sections_to_executable(ctx, output, has_content_based_path = output_has_content_based_path) + output = stamp_build_info( + ctx, + output, + links = opts.links, + build_info_json = build_info_json, + ) linked_object = LinkedObject( output = output, link_args = opts.links + opts.binary_links, bitcode_bundle = bitcode_artifact.artifact if bitcode_artifact else None, - prebolt_output = output, + prebolt_output = prebolt_output, unstripped_output = unstripped_output, dwp = dwp_artifact, external_debug_info = external_debug_info, @@ -585,7 +604,7 @@ def _anon_link_impl(ctx): split_debug_output_placeholder = ctx.actions.write("placeholder_split_debug_output", "", has_content_based_path = False) return [ - DefaultInfo(), + EMPTY_DEFAULT_INFO, _AnonLinkInfo(result = link_result), _AnonLinkInfoPlaceholder(dwp = dwp_placeholder, split_debug_output = split_debug_output_placeholder), ] diff --git a/prelude/cxx/link_groups.bzl b/prelude/cxx/link_groups.bzl index 70807d0..3f33e22 100644 --- a/prelude/cxx/link_groups.bzl +++ b/prelude/cxx/link_groups.bzl @@ -993,6 +993,10 @@ def _stub_library( identifier = name, category_suffix = "stub_library", link_execution_preference = stub_execution_preference, + # Anonymous stubs are canonical (owner-independent) and tiny, so + # sharing them through the action cache is cheap and lets other + # owners/builds skip the link entirely. + allow_cache_upload = anonymous, ), anonymous = anonymous, ) @@ -1190,6 +1194,11 @@ def create_link_groups( + link_group_spec.group.attrs.linker_flags + link_group_spec.group.attrs.exported_linker_flags + ([cmd_args(link_group_spec.group.attrs.linker_script, format = "-Wl,--script={}")] if link_group_spec.group.attrs.linker_script else []) + + ( + [cmd_args(link_group_spec.group.attrs.version_script, format = "-Wl,--version-script={}")] + if link_group_spec.group.attrs.version_script + else [] + ) ), params = create_link_group_params, ) diff --git a/prelude/cxx/link_types.bzl b/prelude/cxx/link_types.bzl index 8c1ed27..906209c 100644 --- a/prelude/cxx/link_types.bzl +++ b/prelude/cxx/link_types.bzl @@ -52,7 +52,8 @@ LinkOptions = record( # A function/lambda which will generate the strip args using the ctx. strip_args_factory = [typing.Callable, None], import_library = Artifact | None, - allow_cache_upload = bool, + # `None` expresses no preference, leaving the choice to buck2 + allow_cache_upload = [bool, None], cxx_toolchain = [CxxToolchainInfo, None], # Force callers to use link_options() or merge_link_options() to create. __private_use_link_options_function_to_construct = None, @@ -80,7 +81,7 @@ def link_options( strip: bool = False, strip_args_factory = None, import_library: Artifact | None = None, - allow_cache_upload: bool = False, + allow_cache_upload: [bool, None] = False, cxx_toolchain: [CxxToolchainInfo, None] = None, error_handler: [typing.Callable, None] = None, extra_linker_outputs_factory: typing.Callable | None = None, @@ -138,7 +139,7 @@ def merge_link_options( strip: [bool, _NotProvided] = _NOT_PROVIDED, strip_args_factory = _NOT_PROVIDED, import_library: [Artifact, None, _NotProvided] = _NOT_PROVIDED, - allow_cache_upload: [bool, _NotProvided] = _NOT_PROVIDED, + allow_cache_upload: [bool, None, _NotProvided] = _NOT_PROVIDED, cxx_toolchain: [CxxToolchainInfo, _NotProvided] = _NOT_PROVIDED, incremental_link: [bool, _NotProvided] = _NOT_PROVIDED, ) -> LinkOptions: diff --git a/prelude/cxx/preprocessor.bzl b/prelude/cxx/preprocessor.bzl index 8dd3655..b4c21f6 100644 --- a/prelude/cxx/preprocessor.bzl +++ b/prelude/cxx/preprocessor.bzl @@ -17,6 +17,7 @@ load( ) load(":attr_selection.bzl", "cxx_by_language_ext") load(":cxx_context.bzl", "get_cxx_toolchain_info") +load(":cxx_toolchain_types.bzl", "CXX_COMPILER_TYPES") load( ":headers.bzl", "CHeader", # @unused Used as a type @@ -50,6 +51,9 @@ CPreprocessorArgs = record( coverage_prefix_args = field(list[typing.Any], []), # Arguments used for module precompilation, replacing args precompile_args = field(list[typing.Any], []), + # Header maps named by `args`, without the header trees behind them. Tools + # that resolve a header map by reading it need these and nothing else. + header_maps = field(list[Artifact], []), ) HeaderUnit = record( @@ -88,6 +92,20 @@ CPreprocessor = record( raw_system_include_dirs = field(list[str], []), ) +_EMPTY_CPREPROCESSOR_ARGS = CPreprocessorArgs() +EMPTY_CPREPROCESSOR = CPreprocessor() + +_EMPTY_SYSTEM_INCLUDE_DIRS_BY_COMPILER_TYPE = { + compiler_type: SystemIncludeDirs(compiler_type = compiler_type, include_dirs = []) for compiler_type in CXX_COMPILER_TYPES +} + +def _system_include_dirs(compiler_type: str, include_dirs: list[CellPath]) -> SystemIncludeDirs: + if not include_dirs: + shared = _EMPTY_SYSTEM_INCLUDE_DIRS_BY_COMPILER_TYPE.get(compiler_type) + if shared != None: + return shared + return SystemIncludeDirs(compiler_type = compiler_type, include_dirs = include_dirs) + # Methods for transitive_sets must be declared prior to their use. def _cpreprocessor_args(pres: list[CPreprocessor]): @@ -108,6 +126,9 @@ def _cpreprocessor_precompile_args(pres: list[CPreprocessor]): args.add(pre.args.precompile_args) return args +def _cpreprocessor_header_maps(pres: list[CPreprocessor]): + return cmd_args(hidden = [pre.args.header_maps for pre in pres]) + def _cpreprocessor_header_units_args_impl(pres: list[CPreprocessor], stub: bool): args = cmd_args() for pre in pres: @@ -146,6 +167,13 @@ def _cpreprocessor_coverage_prefix_args(pres: list[CPreprocessor]): args.add(pre.args.coverage_prefix_args) return args +def _cpreprocessor_has_file_prefix_args(children: list[bool], pres: [list[CPreprocessor], None]): + if pres: + for pre in pres: + if pre.args.file_prefix_args or pre.args.coverage_prefix_args: + return True + return any(children) + def _cpreprocessor_include_dirs(pres: list[CPreprocessor]): args = cmd_args() for pre in pres: @@ -172,6 +200,7 @@ CPreprocessorTSet = transitive_set( "args": _cpreprocessor_args, "coverage_prefix_args": _cpreprocessor_coverage_prefix_args, "file_prefix_args": _cpreprocessor_file_prefix_args, + "header_maps": _cpreprocessor_header_maps, "header_unit_stubs_args": _cpreprocessor_header_unit_stubs_args, "header_units_args": _cpreprocessor_header_units_args, "include_dirs": _cpreprocessor_include_dirs, @@ -179,6 +208,7 @@ CPreprocessorTSet = transitive_set( "precompile_args": _cpreprocessor_precompile_args, }, reductions = { + "has_file_prefix_args": _cpreprocessor_has_file_prefix_args, "has_header_units_args": _cpreprocessor_has_header_units_args, "uses_modules": _cpreprocessor_uses_modules, }, @@ -222,17 +252,24 @@ def cxx_merge_cpreprocessors(actions: AnalysisActions, own: list[CPreprocessor], set = actions.tset(CPreprocessorTSet, **kwargs), ) +# Module globals are recursively frozen before callers can observe them. +_INCLUDE_FLAG_ARGS = { + "-I": cmd_args("-I"), + "-isystem": cmd_args("-isystem"), +} + def _format_include_arg(flag: str, path: cmd_args, compiler_type: str) -> list[cmd_args]: if compiler_type == "windows": return [cmd_args(path, format = flag + "{}")] else: - return [cmd_args(flag), path] + shared = _INCLUDE_FLAG_ARGS.get(flag) + return [shared if shared != None else cmd_args(flag), path] def format_system_include_arg(path: cmd_args, compiler_type: str) -> list[cmd_args]: if compiler_type == "windows": return [cmd_args(path, format = "/external:I{}")] else: - return [cmd_args("-isystem"), path] + return [_INCLUDE_FLAG_ARGS["-isystem"], path] def cxx_exported_preprocessor_info(ctx: AnalysisContext, headers_layout: CxxHeadersLayout, extra_preprocessors: list[CPreprocessor] = []) -> CPreprocessor: """ @@ -293,13 +330,11 @@ def cxx_exported_preprocessor_info(ctx: AnalysisContext, headers_layout: CxxHead header_units.extend(pre.header_units) return CPreprocessor( - args = CPreprocessorArgs( - args = args.args, file_prefix_args = args.file_prefix_args, coverage_prefix_args = args.coverage_prefix_args, precompile_args = args.precompile_args - ), + args = args, headers = exported_headers, raw_headers = raw_headers, include_dirs = include_dirs, - system_include_dirs = SystemIncludeDirs(compiler_type = compiler_type, include_dirs = system_include_dirs), + system_include_dirs = _system_include_dirs(compiler_type, system_include_dirs), modular_args = modular_args, header_units = header_units, raw_include_dirs = ctx.attrs.public_include_directories, @@ -355,9 +390,12 @@ def get_exported_preprocessor_args( # Propagate the exported header tree. file_prefix_args = [] coverage_prefix_args = [] + header_maps = [] if header_root != None: args.extend(_header_style_args(style, header_root.include_path, compiler_type)) precompile_args.extend(_header_style_args(style, precompile_root.include_path, compiler_type)) + if header_root.header_map != None: + header_maps.append(header_root.header_map) if header_root.file_prefix_args != None: file_prefix_args.append(header_root.file_prefix_args) if header_root.coverage_prefix_args != None: @@ -376,8 +414,13 @@ def get_exported_preprocessor_args( for pre in extra_preprocessors: args.extend(pre.args.args) precompile_args.extend(pre.args.precompile_args) + header_maps.extend(pre.args.header_maps) - return CPreprocessorArgs(args = args, file_prefix_args = file_prefix_args, coverage_prefix_args = coverage_prefix_args, precompile_args = precompile_args) + if not args and not file_prefix_args and not coverage_prefix_args and not precompile_args: + return _EMPTY_CPREPROCESSOR_ARGS + return CPreprocessorArgs( + args = args, file_prefix_args = file_prefix_args, coverage_prefix_args = coverage_prefix_args, precompile_args = precompile_args, header_maps = header_maps + ) def cxx_private_preprocessor_info( ctx: AnalysisContext, @@ -457,10 +500,13 @@ def _cxx_private_preprocessor_info( args = _get_private_preprocessor_args(ctx, header_map, compiler_type, all_raw_headers) + # The remaining CPreprocessor fields use their defaults below, so these + # checks cover every field in the record. + if not headers and not all_raw_headers and not include_dirs and not uses_modules and args == _EMPTY_CPREPROCESSOR_ARGS: + return EMPTY_CPREPROCESSOR + return CPreprocessor( - args = CPreprocessorArgs( - args = args.args, file_prefix_args = args.file_prefix_args, coverage_prefix_args = args.coverage_prefix_args, precompile_args = args.precompile_args - ), + args = args, headers = headers, raw_headers = all_raw_headers, include_dirs = include_dirs, @@ -475,6 +521,7 @@ def _get_private_preprocessor_args( cxx_toolchain_info = get_cxx_toolchain_info(ctx) file_prefix_args = [] coverage_prefix_args = [] + header_maps = [] header_mode = map_val(HeaderMode, getattr(ctx.attrs, "header_mode", None)) allow_cache_upload = cxx_attrs_get_allow_cache_upload(ctx.attrs) header_root = prepare_headers( @@ -489,6 +536,8 @@ def _get_private_preprocessor_args( ) if header_root != None: args.extend(_format_include_arg("-I", header_root.include_path, compiler_type)) + if header_root.header_map != None: + header_maps.append(header_root.header_map) if header_root.file_prefix_args != None: file_prefix_args.append(header_root.file_prefix_args) if header_root.coverage_prefix_args != None: @@ -501,7 +550,9 @@ def _get_private_preprocessor_args( # appears to do the job (and not e.g. expand to `""`). args.append(cmd_args(hidden = all_raw_headers)) - return CPreprocessorArgs(args = args, file_prefix_args = file_prefix_args, coverage_prefix_args = coverage_prefix_args) + if not args and not file_prefix_args and not coverage_prefix_args: + return _EMPTY_CPREPROCESSOR_ARGS + return CPreprocessorArgs(args = args, file_prefix_args = file_prefix_args, coverage_prefix_args = coverage_prefix_args, header_maps = header_maps) def _by_language_cxx(x: dict[typing.Any, typing.Any]) -> list[typing.Any]: return cxx_by_language_ext(x, ".cpp") diff --git a/prelude/cxx/shared_library_interface.bzl b/prelude/cxx/shared_library_interface.bzl index 518f7bf..2429a46 100644 --- a/prelude/cxx/shared_library_interface.bzl +++ b/prelude/cxx/shared_library_interface.bzl @@ -8,6 +8,7 @@ load("@prelude//:paths.bzl", "paths") load(":cxx_context.bzl", "get_cxx_toolchain_info") +load(":cxx_library_utility.bzl", "EMPTY_DEFAULT_INFO") load(":cxx_toolchain_types.bzl", "CxxToolchainInfo") def _shared_library_interface(ctx: AnalysisContext, output: str, identifier: str, shared_lib: [Artifact, Promise]) -> Artifact: @@ -46,7 +47,7 @@ def _anon_shared_library_interface_impl(ctx): shared_lib = ctx.attrs.shared_lib, identifier = ctx.attrs.identifier, ) - return [DefaultInfo(), _InterfaceInfo(artifact = output)] + return [EMPTY_DEFAULT_INFO, _InterfaceInfo(artifact = output)] # Anonymous wrapper for `extract_symbol_names`. _anon_shared_library_interface = anon_rule( diff --git a/prelude/cxx/symbols.bzl b/prelude/cxx/symbols.bzl index a882a4c..14df90c 100644 --- a/prelude/cxx/symbols.bzl +++ b/prelude/cxx/symbols.bzl @@ -7,6 +7,7 @@ # above-listed licenses. load("@prelude//:paths.bzl", "paths") +load("@prelude//cxx:cxx_library_utility.bzl", "EMPTY_DEFAULT_INFO") load( "@prelude//cxx:cxx_toolchain_types.bzl", "CxxToolchainInfo", @@ -174,7 +175,7 @@ def _anon_extract_symbol_names_impl(ctx): undefined_weak = ctx.attrs.undefined_weak, allow_cache_upload = ctx.attrs.allow_cache_upload, ) - return [DefaultInfo(), _SymbolsInfo(artifact = output)] + return [EMPTY_DEFAULT_INFO, _SymbolsInfo(artifact = output)] # Anonymous wrapper for `extract_symbol_names`. _anon_extract_symbol_names_impl_rule = anon_rule( diff --git a/prelude/cxx/user/cxx_toolchain_override.bzl b/prelude/cxx/user/cxx_toolchain_override.bzl index 55aaa56..61b0801 100644 --- a/prelude/cxx/user/cxx_toolchain_override.bzl +++ b/prelude/cxx/user/cxx_toolchain_override.bzl @@ -6,6 +6,7 @@ # of this source tree. You may select, at your option, one of the # above-listed licenses. +load("@prelude//cxx:compile.bzl", "compiler_info_with_toolchain_argsfiles") load( "@prelude//cxx:cxx_toolchain_types.bzl", "AsCompilerInfo", @@ -112,6 +113,23 @@ def _cxx_toolchain_override(ctx): allow_cache_upload = _pick_raw(ctx.attrs.cxx_compiler_allow_cache_upload, base_cxx_info.allow_cache_upload), supports_content_based_paths = base_cxx_info.supports_content_based_paths, ) + + # The base toolchain's argsfiles don't reflect the overridden flags, so + # write fresh ones for every compiler info this rule reconstructs. + as_info = compiler_info_with_toolchain_argsfiles(ctx.actions, "as", AsCompilerInfo, as_info) + asm_info = compiler_info_with_toolchain_argsfiles(ctx.actions, "asm", AsmCompilerInfo, asm_info) + c_info = compiler_info_with_toolchain_argsfiles(ctx.actions, "c", CCompilerInfo, c_info) + objc_info = compiler_info_with_toolchain_argsfiles(ctx.actions, "objc", ObjcCompilerInfo, objc_info) + cxx_info = compiler_info_with_toolchain_argsfiles( + ctx.actions, + "cxx", + CxxCompilerInfo, + cxx_info, + # Only the cxx info can be precompiled for C++20 modules. + precompile_filter = ctx.attrs.internal_tools[CxxInternalTools].filter_argsfile, + ) + objcxx_info = compiler_info_with_toolchain_argsfiles(ctx.actions, "objcxx", ObjcxxCompilerInfo, objcxx_info) + base_linker_info = base_toolchain.linker_info linker_type = LinkerType(ctx.attrs.linker_type) if ctx.attrs.linker_type != None else base_linker_info.type pdb_expected = is_pdb_generated(linker_type, ctx.attrs.linker_flags) if ctx.attrs.linker_flags != None else base_linker_info.is_pdb_generated @@ -199,13 +217,13 @@ def _cxx_toolchain_override(ctx): base_binary_utilities_info = base_toolchain.binary_utilities_info binary_utilities_info = BinaryUtilitiesInfo( bolt = _pick_bin(ctx.attrs.bolt, base_binary_utilities_info.bolt), + elf_stamp = _pick_bin(ctx.attrs.elf_stamp, base_binary_utilities_info.elf_stamp), nm = _pick_bin(ctx.attrs.nm, base_binary_utilities_info.nm), objcopy = _pick_bin(ctx.attrs.objcopy, base_binary_utilities_info.objcopy), objdump = _pick_bin(ctx.attrs.objdump, base_binary_utilities_info.objdump), ranlib = _pick_bin(ctx.attrs.ranlib, base_binary_utilities_info.ranlib), strip = _pick_bin(ctx.attrs.strip, base_binary_utilities_info.strip), dwp = _pick_bin(ctx.attrs.dwp, base_binary_utilities_info.dwp), - bolt_msdk = base_binary_utilities_info.bolt_msdk, ) base_strip_flags_info = base_toolchain.strip_flags_info @@ -236,6 +254,7 @@ def _cxx_toolchain_override(ctx): libclang = value_or(ctx.attrs.libclang, base_toolchain.libclang), llvm_link = ctx.attrs.llvm_link[RunInfo] if ctx.attrs.llvm_link != None else base_toolchain.llvm_link, # the rest are used without overrides + target_stats_tools = base_toolchain.target_stats_tools, cuda_compiler_info = base_toolchain.cuda_compiler_info, hip_compiler_info = base_toolchain.hip_compiler_info, hip_debug_extract = base_toolchain.hip_debug_extract, @@ -287,6 +306,7 @@ cxx_toolchain_override_registration_spec = RuleRegistrationSpec( "cxx_compiler_flags": attrs.option(attrs.list(attrs.arg()), default = None), "cxx_preprocessor_flags": attrs.option(attrs.list(attrs.arg()), default = None), "dwp": attrs.option(attrs.exec_dep(providers = [RunInfo]), default = None), + "elf_stamp": attrs.option(attrs.exec_dep(providers = [RunInfo]), default = None), "force_full_hybrid_if_capable": attrs.option(attrs.bool(), default = None), "generate_gc_sections": attrs.option(attrs.bool(), default = None), "generate_linker_maps": attrs.option(attrs.bool(), default = None), diff --git a/prelude/cxx/windows_resource.bzl b/prelude/cxx/windows_resource.bzl index 552e11d..f6b6854 100644 --- a/prelude/cxx/windows_resource.bzl +++ b/prelude/cxx/windows_resource.bzl @@ -10,10 +10,10 @@ load("@prelude//cxx:cxx_context.bzl", "get_cxx_toolchain_info") load("@prelude//cxx:cxx_library_utility.bzl", "cxx_attr_deps") load("@prelude//cxx:headers.bzl", "cxx_get_regular_cxx_headers_layout") load("@prelude//cxx:preprocessor.bzl", "cxx_inherited_preprocessor_infos", "cxx_merge_cpreprocessors", "cxx_private_preprocessor_info") -load("@prelude//linking:link_groups.bzl", "LinkGroupLibInfo") +load("@prelude//linking:link_groups.bzl", "EMPTY_LINK_GROUP_LIB_INFO") load("@prelude//linking:link_info.bzl", "LibOutputStyle", "LinkInfo", "LinkInfos", "ObjectsLinkable", "create_merged_link_info") load("@prelude//linking:linkable_graph.bzl", "create_linkable_graph") -load("@prelude//linking:shared_libraries.bzl", "SharedLibraryInfo") +load("@prelude//linking:shared_libraries.bzl", "EMPTY_SHARED_LIBRARY_INFO") def windows_resource_impl(ctx: AnalysisContext) -> list[Provider]: non_exported_deps = cxx_attr_deps(ctx) @@ -92,8 +92,8 @@ def windows_resource_impl(ctx: AnalysisContext) -> list[Provider]: providers = [ DefaultInfo(default_output = None), - SharedLibraryInfo(set = None), - LinkGroupLibInfo(libs = {}), + EMPTY_SHARED_LIBRARY_INFO, + EMPTY_LINK_GROUP_LIB_INFO, create_linkable_graph(ctx), create_merged_link_info( ctx, diff --git a/prelude/decls/android_rules.bzl b/prelude/decls/android_rules.bzl index 0649e6a..ee4995f 100644 --- a/prelude/decls/android_rules.bzl +++ b/prelude/decls/android_rules.bzl @@ -24,7 +24,7 @@ load("@prelude//decls:test_common.bzl", "test_common") load("@prelude//transitions:constraint_overrides.bzl", "constraint_overrides") load("@prelude//utils:clear_platform.bzl", "clear_platform_transition") load(":android_common.bzl", "android_common") -load(":common.bzl", "AnnotationProcessingTool", "SourceAbiVerificationMode", "TestType", "buck", "prelude_rule") +load(":common.bzl", "TestType", "buck", "prelude_rule") load(":core_rules.bzl", "TargetCpuType") load(":genrule_common.bzl", "genrule_common") load(":java_rules.bzl", "dex_min_sdk_version") @@ -32,9 +32,7 @@ load(":jvm_common.bzl", "jvm_common") load(":re_test_common.bzl", "re_test_common") load(":toolchains_common.bzl", "toolchains_common") -CompressionAlgorithm = ["xz", "zstd"] - -DexStore = ["raw", "raw_subdir", "jar", "xz", "xzs"] +DexStore = ["raw", "raw_subdir", "jar", "xzs"] DuplicateResourceBehaviour = ["allow_by_default", "ban_by_default"] @@ -132,7 +130,6 @@ ANDROID_BINARY_BUNDLE_COMMON_ATTRS = ( doc = "A map of module names to lists of targets, where the targets should seed the corresponding module. The seed targets and their exclusive dependencies are packaged into the APK in separate files to allow them to be loaded independently.", ), "application_module_dependencies": attrs.option(attrs.dict(key = attrs.string(), value = attrs.list(attrs.string()), sorted = False), default = None, doc = ""), - "asset_compression_algorithm": attrs.option(attrs.enum(CompressionAlgorithm), default = None, doc = ""), "banned_duplicate_resource_types": attrs.list( attrs.enum(RType), default = [], @@ -146,7 +143,6 @@ ANDROID_BINARY_BUNDLE_COMMON_ATTRS = ( ), "build_string_source_map": attrs.bool(default = False, doc = ""), "compiled_resource_apks": attrs.list(attrs.source(), default = [], doc = ""), - "compress_asset_libraries": attrs.bool(default = False, doc = ""), "cpu_filters": attrs.list( attrs.enum(TargetCpuType), default = [], @@ -187,8 +183,9 @@ android_library(), android_resource(), cxx_library(), groovy_library(), java_lib # @oss-disable[end= ]: "gatorade_phases": attrs.set(attrs.enum(GatoradePhase), default = []), "ignore_aapt_proguard_config": attrs.bool(default = False, doc = "If true, the proguard config automatically generated by aapt will be ignored."), "include_build_info_file": attrs.bool( - default = read_root_config("android", "include_build_info_file", "false").lower() == "true", - doc = "Bake `assets/BuildInfo.json` with the buck2 `BUCK_BUILD_ID` into the APK so an installed build can be resolved back to its buck2 build and source commit. On by default for non-release builds (`[android] include_build_info_file` in `fbsource-common.bcfg`); pass `-c android.include_build_info_file=false` to disable. The `apk_build` action invokes a toolchain-provided generator to write the file, so the baked id refreshes when the APK content changes and a no-op rebuild reuses a prior (equally valid) id.", + # @oss-disable[end= ]: default = True, + default = False, # @oss-enable + doc = "Bake `assets/BuildInfo.json` with the buck2 `BUCK_BUILD_ID` into the APK so an installed build can be resolved back to its buck2 build and source commit. On by default for non-release builds in fbsource; set this attribute to `False` to disable. The `apk_build` action invokes a toolchain-provided generator to write the file, so the baked id refreshes when the APK content changes and a no-op rebuild reuses a prior (equally valid) id.", ), "includes_vector_drawables": attrs.bool( default = False, @@ -274,8 +271,8 @@ for how to resolve duplicate classes using this attribute.""", "packaged_locales": attrs.list(attrs.string(), default = [], doc = ""), "packaging_options": attrs.dict(key = attrs.string(), value = attrs.list(attrs.string()), default = {}, doc = ""), "post_filter_resources_cmd": attrs.option(attrs.arg(), default = None, doc = ""), + "preinline_ir_sonames": attrs.list(attrs.string(), default = [], doc = "Sonames the pre-inline IR dump is restricted to (empty = all)."), "preprocess_java_classes_bash": attrs.option(attrs.arg(), default = None, doc = ""), - "preprocess_java_classes_cmd": attrs.option(attrs.arg(), default = None, doc = ""), "preprocess_java_classes_deps": attrs.list(attrs.dep(), default = [], doc = ""), "primary_dex_patterns": attrs.list(attrs.string(), default = [], doc = ""), "proguard_config": attrs.option( @@ -301,7 +298,6 @@ for how to resolve duplicate classes using this attribute.""", "resource_compression": attrs.enum(ResourceCompressionMode, default = "disabled", doc = ""), "resource_filter": attrs.list(attrs.string(), default = [], doc = ""), "resource_stable_ids": attrs.option(attrs.source(), default = None, doc = ""), - "resource_union_package": attrs.option(attrs.string(), default = None, doc = ""), "secondary_dex_weight_limit": attrs.option(attrs.int(), default = None, doc = ""), "shared_libraries_to_exclude": attrs.list(attrs.label(), default = [], doc = ""), "skip_crunch_pngs": attrs.option(attrs.bool(), default = None, doc = "If True, PNGs in the APK are not crushed by aapt."), @@ -433,16 +429,14 @@ android_aar = prelude_rule( and its dependencies. """, ), - "annotation_processing_tool": attrs.option(attrs.enum(AnnotationProcessingTool), default = None), "build_config_values_file": attrs.option(attrs.source(), default = None), - "compress_asset_libraries": attrs.default_only(attrs.bool(default = False)), "cpu_filters": attrs.list(attrs.enum(TargetCpuType), default = ALL_CPU_FILTERS), "enable_relinker": attrs.bool(default = False), "excluded_java_deps": attrs.list(attrs.dep(), default = []), "extra_arguments": attrs.list(attrs.string(), default = []), "extra_kotlinc_arguments": attrs.list(attrs.arg(anon_target_compatible = True), default = []), "extra_relinker_outputs": attrs.list(attrs.string(), default = []), - "friend_paths": attrs.list(attrs.dep(), default = []), + "preinline_ir_sonames": attrs.list(attrs.string(), default = []), # @oss-disable[end= ]: "gatorade_extra_args": attrs.list(attrs.arg(), default = [], doc = "Extra Gatorade cross-library step arguments"), # @oss-disable[end= ]: "gatorade_phases": attrs.set(attrs.enum(GatoradePhase), default = []), "hardcode_permissions_for_deterministic_output": attrs.option( @@ -495,17 +489,14 @@ android_aar = prelude_rule( ), "relinker_whitelist": attrs.list(attrs.regex(), default = []), "required_for_source_only_abi": attrs.bool(default = False), - "resource_union_package": attrs.option(attrs.string(), default = None), "resources": attrs.list(attrs.source(), default = []), "resources_root": attrs.option(attrs.string(), default = None), "runtime_deps": attrs.list(attrs.dep(), default = []), "source": attrs.option(attrs.string(), default = None), - "source_abi_verification_mode": attrs.option(attrs.enum(SourceAbiVerificationMode), default = None), "source_only_abi_deps": attrs.list(attrs.dep(), default = []), "srcs": attrs.list(attrs.source(), default = []), "strip_libraries": attrs.default_only(attrs.bool(default = not DISABLE_STRIPPING)), "target": attrs.option(attrs.string(), default = None), - "use_jvm_abi_gen": attrs.option(attrs.bool(), default = None), "_android_toolchain": toolchains_common.android(), "_cxx_toolchain": attrs.split_transition_dep(cfg = cpu_split_transition, default = "toolchains//:android-hack"), "_is_building_android_binary": attrs.default_only(attrs.bool(default = True)), @@ -844,7 +835,6 @@ android_instrumentation_apk = prelude_rule( "min_sdk_version": attrs.option(attrs.int(), default = None), "multidex_min_api": attrs.option(attrs.string(), default = None), "preprocess_java_classes_bash": attrs.option(attrs.arg(), default = None), - "preprocess_java_classes_cmd": attrs.option(attrs.arg(), default = None), "preprocess_java_classes_deps": attrs.list(attrs.dep(), default = []), "primary_dex_patterns": attrs.list(attrs.string(), default = []), "use_split_dex": attrs.option(attrs.bool(), default = None), @@ -1018,17 +1008,6 @@ android_library = prelude_rule( } | android_common.manifest_arg() | { - "annotation_processing_tool": attrs.option( - attrs.enum(AnnotationProcessingTool), - default = None, - doc = """ - Specifies the tool to use for annotation processing. Possible values: "kapt" or "javac". - "kapt" allows running Java annotation processors against Kotlin sources while backporting - it for Java sources too. - "javac" works only against Java sources, Kotlin sources won't have access to generated - classes at compile time. - """, - ), "deps": attrs.list( attrs.dep(), default = [], @@ -1092,7 +1071,7 @@ android_library = prelude_rule( | { "android_optional_jars": attrs.option(attrs.list(attrs.dep()), default = None), "capabilities_registrations": attrs.option(attrs.list(attrs.any()), default = None), - "friend_paths": attrs.list(attrs.dep(), default = []), + "friend_paths": jvm_common.friend_paths_attr(), "jar_postprocessor": attrs.option(attrs.exec_dep(), default = None), "java_version": attrs.option(attrs.string(), default = None), "language": attrs.option(attrs.enum(JvmLanguage), default = None), @@ -1107,11 +1086,8 @@ android_library = prelude_rule( sources, not from any of its dependencies. """, ), - "resource_union_package": attrs.option(attrs.string(), default = None), "resources_root": attrs.option(attrs.string(), default = None), "runtime_deps": attrs.list(attrs.dep(), default = []), - "source_abi_verification_mode": attrs.option(attrs.enum(SourceAbiVerificationMode), default = None), - "use_jvm_abi_gen": attrs.option(attrs.bool(), default = None), VALIDATION_DEPS_ATTR_NAME: attrs.set(attrs.dep(), sorted = True, default = []), "_android_toolchain": toolchains_common.android(), "_build_only_native_code": attrs.default_only(attrs.bool(default = is_build_only_native_code())), @@ -1336,6 +1312,7 @@ android_resource = prelude_rule( Android resources, although this option is deprecated and might be removed in the future. """, ), + "unused_resource_dep_validation_has_non_xml_resources": attrs.option(attrs.bool(), default = None), "package": attrs.option( attrs.string(), default = None, @@ -1428,7 +1405,6 @@ android_resource = prelude_rule( Android resources, although this option is deprecated and might be removed in the future. """, ), - "resource_union": attrs.bool(default = False), "_android_toolchain": toolchains_common.android(), "_build_only_native_code": attrs.default_only(attrs.bool(default = is_build_only_native_code())), "_is_building_android_binary": attrs.default_only(attrs.bool(default = False)), @@ -1618,7 +1594,7 @@ gen_aidl = prelude_rule( A list of rules that must be built before this rule. """, ), - "_android_toolchain": toolchains_common.android(), + "_aidl_toolchain": toolchains_common.android_aidl(), "_exec_os_type": buck.exec_os_type_arg(), "_java_toolchain": toolchains_common.java_for_android(), } @@ -1754,7 +1730,6 @@ robolectric_test = prelude_rule( buck.inject_test_env_arg() | { "android_optional_jars": attrs.option(attrs.list(attrs.dep()), default = None), - "annotation_processing_tool": attrs.option(attrs.enum(AnnotationProcessingTool), default = None), "compiled_resource_apks": attrs.list(attrs.source(), default = []), "cxx_library_allowlist": attrs.list( attrs.dep(), @@ -1777,7 +1752,7 @@ robolectric_test = prelude_rule( List of additional arguments to pass into the Kotlin compiler. """, ), - "friend_paths": attrs.list(attrs.dep(), default = []), + "friend_paths": jvm_common.friend_paths_attr(), "jar_postprocessor": attrs.option(attrs.exec_dep(), default = None), "java": attrs.option(attrs.dep(), default = None), "java_agents": attrs.list(attrs.source(), default = []), @@ -1795,7 +1770,6 @@ robolectric_test = prelude_rule( "remove_classes": attrs.list(attrs.regex(), default = []), "required_for_source_only_abi": attrs.bool(default = False), "resource_stable_ids": attrs.option(attrs.source(), default = None), - "resource_union_package": attrs.option(attrs.string(), default = None), "resources": attrs.list(attrs.source(), default = []), "resources_root": attrs.option(attrs.string(), default = None), "robolectric_runtime_dependencies": attrs.list(attrs.source(), default = []), @@ -1810,7 +1784,6 @@ robolectric_test = prelude_rule( "run_test_separately": attrs.bool(default = False), "runtime_deps": attrs.list(attrs.dep(), default = []), "source": attrs.option(attrs.string(), default = None), - "source_abi_verification_mode": attrs.option(attrs.enum(SourceAbiVerificationMode), default = None), "source_only_abi_deps": attrs.list(attrs.dep(), default = []), "srcs": attrs.list(attrs.source(), default = []), "supports_test_execution_caching": attrs.bool(default = False), @@ -1821,8 +1794,6 @@ robolectric_test = prelude_rule( "test_type": attrs.option(attrs.enum(TestType), default = None), "unbundled_resources_root": attrs.option(attrs.source(allow_directory = True), default = None), "use_cxx_libraries": attrs.option(attrs.bool(), default = None), - "use_dependency_order_classpath": attrs.option(attrs.bool(), default = None), - "use_jvm_abi_gen": attrs.option(attrs.bool(), default = None), "vm_args": attrs.list(attrs.arg(), default = []), "_android_toolchain": toolchains_common.android(), "_build_only_native_code": attrs.default_only(attrs.bool(default = is_build_only_native_code())), diff --git a/prelude/decls/common.bzl b/prelude/decls/common.bzl index 3459e32..704a8c8 100644 --- a/prelude/decls/common.bzl +++ b/prelude/decls/common.bzl @@ -32,8 +32,6 @@ prelude_rule = record( AbiGenerationMode = ["class", "source", "source_only", "none"] -AnnotationProcessingTool = ["kapt", "javac"] - CxxRuntimeType = ["dynamic", "static"] CxxSourceType = [ @@ -70,8 +68,6 @@ OnDuplicateEntry = ["fail", "overwrite", "append"] RawHeadersAsHeadersMode = ["enabled", "disabled"] -SourceAbiVerificationMode = ["off", "log", "fail"] - TestType = ["junit", "junit5", "testng"] UnusedDependenciesAction = ["unknown", "fail", "warn", "ignore", "unrecognized"] diff --git a/prelude/decls/core_rules.bzl b/prelude/decls/core_rules.bzl index 8b5f27c..5e55b4e 100644 --- a/prelude/decls/core_rules.bzl +++ b/prelude/decls/core_rules.bzl @@ -267,7 +267,26 @@ config_setting = prelude_rule( # @unsorted-dict-items { "constraint_values": attrs.list(attrs.configuration_label(), default = []), - "values": attrs.dict(key = attrs.string(), value = attrs.string(), sorted = False, default = {}), + "values": attrs.dict( + key = attrs.string(), + value = attrs.string(), + sorted = False, + default = {}, + doc = """ + Buckconfig key -> expected value, evaluated from the target's cell (like `read_config`). + A given `section.key` may appear in either `values` or `root_values`, but not both. + """, + ), + "root_values": attrs.dict( + key = attrs.string(), + value = attrs.string(), + sorted = False, + default = {}, + doc = """ + Buckconfig key -> expected value, evaluated from the root cell (like `read_root_config`). + A given `section.key` may appear in either `values` or `root_values`, but not both. + """, + ), } ), ) diff --git a/prelude/decls/cxx_common.bzl b/prelude/decls/cxx_common.bzl index 51284a7..9d789ea 100644 --- a/prelude/decls/cxx_common.bzl +++ b/prelude/decls/cxx_common.bzl @@ -11,6 +11,7 @@ # the generated docs, and so those should be verified to be accurate and # well-formatted (and then delete this TODO) +load("@prelude//cxx:cxx_flags.bzl", "CxxFlagsInfo") load(":common.bzl", "CxxSourceType", "DefaultDepsMode", "IncludeType", "RawHeadersAsHeadersMode", "RuntimeDependencyHandling") def _srcs_arg(): @@ -498,6 +499,20 @@ def _expect_eligible_for_dedupe_arg(): "expect_eligible_for_dedupe": attrs.bool(default = False), } +def _flags_arg(): + return { + "flags": attrs.list( + attrs.dep(providers = [CxxFlagsInfo]), + default = [], + doc = """ + `cxx_flags` targets whose flags apply before this target's own. Their transitive + definitions are combined into a shared argsfile, with nested flags first and + direct entries in list order. Repeated and diamond dependencies contribute once. + They cannot contribute `exported_*` or post-linker flag categories. + """, + ), + } + cxx_common = struct( srcs_arg = _srcs_arg, deps_arg = _deps_arg, @@ -534,5 +549,6 @@ cxx_common = struct( use_fbcc_rust_wrapper_arg = _use_fbcc_rust_wrapper_arg, use_content_based_paths_arg = _use_content_based_paths_arg, expect_eligible_for_dedupe_arg = _expect_eligible_for_dedupe_arg, + flags_arg = _flags_arg, supports_stripping = _supports_stripping, ) diff --git a/prelude/decls/cxx_rules.bzl b/prelude/decls/cxx_rules.bzl index 8a39beb..e823bd2 100644 --- a/prelude/decls/cxx_rules.bzl +++ b/prelude/decls/cxx_rules.bzl @@ -14,6 +14,8 @@ load("@prelude//:attrs_validators.bzl", "validation_common") load("@prelude//apple:apple_common.bzl", "apple_common") load("@prelude//cxx:cuda.bzl", "CudaCompileStyle") +load("@prelude//cxx:cxx_flags.bzl", "CxxFlagsInfo") +load("@prelude//cxx:cxx_toolchain_types.bzl", "CXX_COMPILER_TYPES") load("@prelude//cxx:cxx_types.bzl", "LinkPreference") load("@prelude//cxx:headers.bzl", "CPrecompiledHeaderInfo") load("@prelude//cxx:link_groups_types.bzl", "LINK_GROUP_MAP_ATTR") @@ -38,39 +40,43 @@ BUILD_INFO_ATTR = attrs.dict( ) def _cxx_binary_and_test_attrs(): - ret = { - "anonymous_link_groups": attrs.bool(default = False), - "auto_link_groups": attrs.bool(default = False), - # Linker flags that only apply to the executable link, used for link - # strategies (e.g. link groups) which may link shared libraries from - # top-level binary context. - "binary_linker_flags": attrs.list(attrs.arg(anon_target_compatible = True), default = []), - "bolt_flags": attrs.list(attrs.arg(), default = []), - "bolt_profile": attrs.option(attrs.source(), default = None), - # These flags will only be used to instrument a target - # when coverage for that target is enabled by a header - # selected for coverage either in the target or in one - # of the target's dependencies. - "coverage_instrumentation_compiler_flags": attrs.list(attrs.string(), default = []), - # Optional clang_profile_list target for selective coverage instrumentation via -fprofile-list. - "coverage_profile_list": attrs.option(attrs.dep(), default = None), - "cuda_compile_style": attrs.enum(CudaCompileStyle.values(), default = "mono"), - "enable_distributed_thinlto": attrs.bool(default = False), - "exported_needs_coverage_instrumentation": attrs.bool(default = False), - "extra_dwp_flags": attrs.list(attrs.string(), default = []), - "link_execution_preference": link_execution_preference_attr(), - "link_group_map": LINK_GROUP_MAP_ATTR, - "link_group_min_binary_node_count": attrs.option(attrs.int(), default = None), - "link_ordering": attrs.option(attrs.enum(LinkOrdering.values()), default = None), - "link_preference": attrs.enum(LinkPreference.values(), default = "default"), - "link_whole": attrs.default_only(attrs.bool(default = False)), - "precompiled_header": attrs.option(attrs.dep(providers = [CPrecompiledHeaderInfo]), default = None), - "resources": attrs.named_set(attrs.one_of(attrs.dep(), attrs.source(allow_directory = True)), sorted = True, default = []), - "separate_debug_info": attrs.bool(default = False), - "_build_info": BUILD_INFO_ATTR, - "_cxx_hacks": attrs.dep(default = "prelude//cxx/tools:cxx_hacks"), - "_cxx_toolchain": toolchains_common.cxx(), - } | validation_common.attrs_validators_arg() + ret = ( + { + "anonymous_link_groups": attrs.bool(default = False), + "auto_link_groups": attrs.bool(default = False), + # Linker flags that only apply to the executable link, used for link + # strategies (e.g. link groups) which may link shared libraries from + # top-level binary context. + "binary_linker_flags": attrs.list(attrs.arg(anon_target_compatible = True), default = []), + "bolt_flags": attrs.list(attrs.arg(), default = []), + "bolt_profile": attrs.option(attrs.source(), default = None), + # These flags will only be used to instrument a target + # when coverage for that target is enabled by a header + # selected for coverage either in the target or in one + # of the target's dependencies. + "coverage_instrumentation_compiler_flags": attrs.list(attrs.string(), default = []), + # Optional clang_profile_list target for selective coverage instrumentation via -fprofile-list. + "coverage_profile_list": attrs.option(attrs.dep(), default = None), + "cuda_compile_style": attrs.enum(CudaCompileStyle.values(), default = "mono"), + "enable_distributed_thinlto": attrs.bool(default = False), + "exported_needs_coverage_instrumentation": attrs.bool(default = False), + "extra_dwp_flags": attrs.list(attrs.string(), default = []), + "link_execution_preference": link_execution_preference_attr(), + "link_group_map": LINK_GROUP_MAP_ATTR, + "link_group_min_binary_node_count": attrs.option(attrs.int(), default = None), + "link_ordering": attrs.option(attrs.enum(LinkOrdering.values()), default = None), + "link_preference": attrs.enum(LinkPreference.values(), default = "default"), + "link_whole": attrs.default_only(attrs.bool(default = False)), + "precompiled_header": attrs.option(attrs.dep(providers = [CPrecompiledHeaderInfo]), default = None), + "resources": attrs.named_set(attrs.one_of(attrs.dep(), attrs.source(allow_directory = True)), sorted = True, default = []), + "separate_debug_info": attrs.bool(default = False), + "_build_info": BUILD_INFO_ATTR, + "_cxx_hacks": attrs.dep(default = "prelude//cxx/tools:cxx_hacks"), + "_cxx_toolchain": toolchains_common.cxx(), + } + | cxx_common.flags_arg() + | validation_common.attrs_validators_arg() + ) ret.update(constraint_overrides.attributes) return ret @@ -78,8 +84,6 @@ ArchiverProviderType = ["bsd", "gnu", "llvm", "windows", "windows_clang"] CxxTestType = ["gtest", "boost"] -CxxToolProviderType = ["clang", "clang_cl", "clang_windows", "gcc", "windows", "windows_ml64"] - LinkerProviderType = ["darwin", "gnu", "windows", "unknown", "wasm"] PicType = ["pic", "pdc"] @@ -394,6 +398,7 @@ cxx_genrule = prelude_rule( library_attrs = ( # @unsorted-dict-items cxx_common.srcs_arg() + | cxx_common.flags_arg() | cxx_common.headers_arg() | cxx_common.exported_headers_arg() | cxx_common.exported_header_style_arg() @@ -758,6 +763,47 @@ cxx_precompiled_header = prelude_rule( ), ) +cxx_flags = prelude_rule( + name = "cxx_flags", + docs = """ + A `cxx_flags()` target holds flags that C++ targets pull in through their + `flags` attribute. Its `deps` can reference other `cxx_flags` + targets; nested flags apply before the including target's flags, and all + shared flags apply before the consuming C++ target's own flags. + + Consumers share a compiler argsfile for the transitive flag set. + + Write-to-file macros (`$(@...)`) are unsupported in all flag attributes. + """, + examples = """ + ``` + cxx_flags( + name = "common_warnings", + compiler_flags = ["-Wall", "-Wextra"], + ) + + cxx_library( + name = "lib", + srcs = ["lib.cpp"], + flags = [":common_warnings"], + ) + ``` + """, + further = None, + attrs = ( + { + "compiler_flags": attrs.list(attrs.arg(), default = []), + "deps": attrs.list(attrs.dep(providers = [CxxFlagsInfo]), default = []), + "lang_compiler_flags": attrs.dict(key = attrs.enum(CxxSourceType), value = attrs.list(attrs.arg()), sorted = False, default = {}), + "lang_preprocessor_flags": attrs.dict(key = attrs.enum(CxxSourceType), value = attrs.list(attrs.arg()), sorted = False, default = {}), + "linker_flags": attrs.list(attrs.arg(anon_target_compatible = True), default = []), + "preprocessor_flags": attrs.list(attrs.arg(), default = []), + } + | buck.labels_arg() + | buck.contacts_arg() + ), +) + windows_resource = prelude_rule( name = "windows_resource", docs = """ @@ -983,16 +1029,16 @@ cxx_toolchain = prelude_rule( "archiver_type": attrs.enum(ArchiverProviderType), "asm_compiler": attrs.option(attrs.source(), default = None), "asm_compiler_flags": attrs.list(attrs.arg(), default = []), - "asm_compiler_type": attrs.option(attrs.enum(CxxToolProviderType), default = None), + "asm_compiler_type": attrs.option(attrs.enum(CXX_COMPILER_TYPES), default = None), "asm_preprocessor": attrs.option(attrs.source(), default = None), "asm_preprocessor_flags": attrs.list(attrs.arg(), default = []), - "asm_preprocessor_type": attrs.option(attrs.enum(CxxToolProviderType), default = None), + "asm_preprocessor_type": attrs.option(attrs.enum(CXX_COMPILER_TYPES), default = None), "assembler": attrs.source(), "assembler_flags": attrs.list(attrs.arg(), default = []), "assembler_preprocessor": attrs.option(attrs.source(), default = None), "assembler_preprocessor_flags": attrs.list(attrs.arg(), default = []), - "assembler_preprocessor_type": attrs.option(attrs.enum(CxxToolProviderType), default = None), - "assembler_type": attrs.option(attrs.enum(CxxToolProviderType), default = None), + "assembler_preprocessor_type": attrs.option(attrs.enum(CXX_COMPILER_TYPES), default = None), + "assembler_type": attrs.option(attrs.enum(CXX_COMPILER_TYPES), default = None), "binary_extension": attrs.option(attrs.string(), default = None), "binary_linker_flags": attrs.list( attrs.arg(anon_target_compatible = True), @@ -1008,22 +1054,22 @@ cxx_toolchain = prelude_rule( "bolt": attrs.source(), "c_compiler": attrs.source(), "c_compiler_flags": attrs.list(attrs.arg(), default = []), - "c_compiler_type": attrs.option(attrs.enum(CxxToolProviderType), default = None), + "c_compiler_type": attrs.option(attrs.enum(CXX_COMPILER_TYPES), default = None), "c_preprocessor_flags": attrs.list(attrs.arg(), default = []), "cache_links": attrs.bool(default = False), - "compiler_type": attrs.option(attrs.enum(CxxToolProviderType), default = None), + "compiler_type": attrs.option(attrs.enum(CXX_COMPILER_TYPES), default = None), "cuda_compiler": attrs.option(attrs.source(), default = None), "cuda_compiler_flags": attrs.list(attrs.arg(), default = []), - "cuda_compiler_type": attrs.option(attrs.enum(CxxToolProviderType), default = None), + "cuda_compiler_type": attrs.option(attrs.enum(CXX_COMPILER_TYPES), default = None), "cuda_preprocessor_flags": attrs.list(attrs.arg(), default = []), "custom_tools": attrs.dict(key = attrs.string(), value = attrs.source(), default = {}), "cvtres_compiler": attrs.option(attrs.source(), default = None), "cvtres_compiler_flags": attrs.list(attrs.arg(), default = []), - "cvtres_compiler_type": attrs.option(attrs.enum(CxxToolProviderType), default = None), + "cvtres_compiler_type": attrs.option(attrs.enum(CXX_COMPILER_TYPES), default = None), "cvtres_preprocessor_flags": attrs.list(attrs.arg(), default = []), "cxx_compiler": attrs.source(), "cxx_compiler_flags": attrs.list(attrs.arg(), default = []), - "cxx_compiler_type": attrs.option(attrs.enum(CxxToolProviderType), default = None), + "cxx_compiler_type": attrs.option(attrs.enum(CXX_COMPILER_TYPES), default = None), "cxx_preprocessor_flags": attrs.list(attrs.arg(), default = []), "debug_path_prefix_map_sanitizer_format": attrs.option(attrs.string(), default = None), "dist_thin_lto_codegen_flags": attrs.list(attrs.arg(), default = []), @@ -1038,7 +1084,7 @@ cxx_toolchain = prelude_rule( "headers_as_raw_headers_mode": attrs.option(attrs.enum(HeadersAsRawHeadersMode), default = None), "hip_compiler": attrs.option(attrs.source(), default = None), "hip_compiler_flags": attrs.list(attrs.arg(), default = []), - "hip_compiler_type": attrs.option(attrs.enum(CxxToolProviderType), default = None), + "hip_compiler_type": attrs.option(attrs.enum(CXX_COMPILER_TYPES), default = None), "hip_preprocessor_flags": attrs.list(attrs.arg(), default = []), "link_metadata_flag": attrs.option(attrs.string(), default = None), "link_style": attrs.enum( @@ -1064,7 +1110,7 @@ cxx_toolchain = prelude_rule( "ranlib_flags": attrs.list(attrs.arg(), default = []), "rc_compiler": attrs.option(attrs.source(), default = None), "rc_compiler_flags": attrs.list(attrs.arg(), default = []), - "rc_compiler_type": attrs.option(attrs.enum(CxxToolProviderType), default = None), + "rc_compiler_type": attrs.option(attrs.enum(CXX_COMPILER_TYPES), default = None), "rc_preprocessor_flags": attrs.list(attrs.arg(), default = []), "requires_archives": attrs.bool(default = False), "shared_dep_runtime_ld_flags": attrs.list(attrs.arg(), default = []), @@ -1413,6 +1459,7 @@ cxx_rules = struct( cxx_binary = cxx_binary, cxx_genrule = cxx_genrule, cxx_library = cxx_library, + cxx_flags = cxx_flags, cxx_precompiled_header = cxx_precompiled_header, windows_resource = windows_resource, cxx_test = cxx_test, diff --git a/prelude/decls/erlang_rules.bzl b/prelude/decls/erlang_rules.bzl index 3bbf1c8..5829d4e 100644 --- a/prelude/decls/erlang_rules.bzl +++ b/prelude/decls/erlang_rules.bzl @@ -10,11 +10,7 @@ load("@prelude//erlang:erlang_application.bzl", "StartTypeValues") load("@prelude//erlang:erlang_info.bzl", "ErlangAppIncludeInfo", "ErlangAppInfo", "ErlangAppOrTestInfo") load(":common.bzl", "buck", "prelude_rule") load(":re_test_common.bzl", "re_test_common") - -def re_test_args(): - # remove reference to fbcode targets - args = re_test_common.test_args() - return {"remote_execution": args["remote_execution"]} +load(":test_common.bzl", "test_common") common_attributes = ( buck.labels_arg() @@ -363,13 +359,33 @@ rules_attributes = { default = {}, doc = """ The mapping listed here maps from boot script name (e.g. "start.boot") to a binary (e.g. "$(location ...)") generating the boot script. - The binary receives three input arguments: + The binary receives four input arguments: 1. path of to a json file containing the applications, i.e. `["app1", ("app2", "load"), ...]` 2. the location of the lib dir containing the `_` folders 3. the output location of the boot script 4. the output location of the script file + followed by the values of `extra_bootscript_builder_args`, if any. """, ), + "default_bootscript_name": attrs.string( + default = "start", + doc = """ + The boot script a runnable release boots, named without its `.boot` extension and looked up in + `releases//`. The default is `start`, the release's default boot script, produced by + `generate_default_bootscript`. When the release also contains + `releases//.vm.args`, the launcher passes it as `-args_file`. + """, + ), + "extra_bootscript_builder_args": attrs.list( + attrs.arg(), + default = [], + doc = """ + Additional arguments that are appended, in order, to every `bootscript_builders` invocation + after the four positional arguments. This lets a custom boot script builder be parameterised + by the release that uses it. Macros such as `$(location //some:target)` are expanded, and the + targets they name are made available to the invocation. + """, + ), "generate_default_bootscript": attrs.bool( default = True, doc = """ @@ -383,6 +399,16 @@ rules_attributes = { Please note, that at the moment the erts folder is just `erts/`. """, ), + "is_executable": attrs.bool( + default = False, + doc = """ + This field controls whether the release is runnable. If set, a launcher is generated at + `bin/` that boots the release with the bundled emulator, which is why it + requires `include_erts = True`. The boot script it uses is `default_bootscript_name`. + + When the release contains `releases//sys.config`, the launcher passes it as `-config`. + """, + ), "overlays": attrs.dict( key = attrs.string(), value = attrs.list(attrs.dep()), @@ -513,7 +539,8 @@ rules_attributes = { ), } | common_shell_attributes - | re_test_args(), + | test_common.attributes() + | re_test_common.test_args(), } attributes = {name: dict(rules_attributes[name], **common_attributes) for name in rules_attributes} diff --git a/prelude/decls/java_rules.bzl b/prelude/decls/java_rules.bzl index 75ebdb2..fa7de12 100644 --- a/prelude/decls/java_rules.bzl +++ b/prelude/decls/java_rules.bzl @@ -19,7 +19,7 @@ load("@prelude//android:configuration.bzl", "is_building_android_binary_attr") load("@prelude//android:min_sdk_version.bzl", "get_min_sdk_version_constraint_value_name", "get_min_sdk_version_range") load("@prelude//decls:test_common.bzl", "test_common") load("@prelude//transitions:constraint_overrides.bzl", "constraint_overrides") -load(":common.bzl", "SourceAbiVerificationMode", "TestType", "buck", "prelude_rule") +load(":common.bzl", "TestType", "buck", "prelude_rule") load(":jvm_common.bzl", "jvm_common") load(":re_test_common.bzl", "re_test_common") load(":toolchains_common.bzl", "toolchains_common") @@ -390,7 +390,6 @@ java_library = prelude_rule( "proguard_config": attrs.option(attrs.source(), default = None), "resources_root": attrs.option(attrs.string(), default = None), "runtime_deps": attrs.list(attrs.dep(), default = []), - "source_abi_verification_mode": attrs.option(attrs.enum(SourceAbiVerificationMode), default = None), VALIDATION_DEPS_ATTR_NAME: attrs.set(attrs.dep(), sorted = True, default = []), "_build_only_native_code": attrs.default_only(attrs.bool(default = is_build_only_native_code())), "_dex_min_sdk_version": attrs.option(attrs.int(), default = dex_min_sdk_version()), @@ -562,14 +561,11 @@ java_test = prelude_rule( "resources_root": attrs.option(attrs.string(), default = None), "runner": attrs.option(attrs.dep(), default = None), "runtime_deps": attrs.list(attrs.dep(), default = []), - "source_abi_verification_mode": attrs.option(attrs.enum(SourceAbiVerificationMode), default = None), "source_only_abi_deps": attrs.list(attrs.dep(), default = []), - "specs": attrs.option(attrs.arg(json = True), default = None), "supports_test_execution_caching": attrs.bool(default = False), "test_case_timeout_ms": attrs.option(attrs.int(), default = None), "test_class_names_file": attrs.option(attrs.source(), default = None), "unbundled_resources_root": attrs.option(attrs.source(allow_directory = True), default = None), - "use_dependency_order_classpath": attrs.option(attrs.bool(), default = None), "_build_only_native_code": attrs.default_only(attrs.bool(default = is_build_only_native_code())), "_exec_os_type": buck.exec_os_type_arg(), "_is_building_android_binary": attrs.default_only(attrs.bool(default = False)), @@ -589,50 +585,6 @@ java_test = prelude_rule( cfg = constraint_overrides.transition, ) -java_test_runner = prelude_rule( - name = "java_test_runner", - docs = """ - A `java_test_runner()` rule defines a custom test runner JAR - (with a `main_class`) that can be referenced by `java_test()` - targets via their `runner` attribute, replacing the default - JUnit/TestNG runner. - """, - examples = None, - further = None, - attrs = ( - # @unsorted-dict-items - { - "deps": attrs.list(attrs.dep(), default = []), - "exported_deps": attrs.list(attrs.dep(), default = []), - "exported_provided_deps": attrs.list(attrs.dep(), default = []), - "extra_arguments": attrs.list(attrs.string(), default = []), - "java_version": attrs.option(attrs.string(), default = None), - "main_class": attrs.string(default = ""), - "manifest_file": attrs.option(attrs.source(), default = None), - "maven_coords": attrs.option(attrs.string(), default = None), - "proguard_config": attrs.option(attrs.source(), default = None), - "provided_deps": attrs.list(attrs.dep(), default = []), - "remove_classes": attrs.list(attrs.regex(), default = []), - "required_for_source_only_abi": attrs.bool(default = False), - "resources": attrs.list(attrs.source(), default = []), - "resources_root": attrs.option(attrs.string(), default = None), - "runtime_deps": attrs.list(attrs.dep(), default = []), - "source": attrs.option(attrs.string(), default = None), - "source_abi_verification_mode": attrs.option(attrs.enum(SourceAbiVerificationMode), default = None), - "source_only_abi_deps": attrs.list(attrs.dep(), default = []), - "srcs": attrs.list(attrs.source(), default = []), - "target": attrs.option(attrs.string(), default = None), - } - | buck.licenses_arg() - | buck.labels_arg() - | buck.contacts_arg() - | jvm_common.abi_generation_mode() - | jvm_common.annotation_processors() - | jvm_common.plugins() - | jvm_common.javac() - ), -) - prebuilt_jar = prelude_rule( name = "prebuilt_jar", docs = """ @@ -724,6 +676,5 @@ java_rules = struct( java_library = java_library, java_plugin = java_plugin, java_test = java_test, - java_test_runner = java_test_runner, prebuilt_jar = prebuilt_jar, ) diff --git a/prelude/decls/jvm_common.bzl b/prelude/decls/jvm_common.bzl index cfa0bc0..a10d500 100644 --- a/prelude/decls/jvm_common.bzl +++ b/prelude/decls/jvm_common.bzl @@ -200,7 +200,7 @@ def _k2(): def _incremental(): return { "incremental": attrs.bool( - default = True, + default = False, doc = """ Enables Kotlin incremental compilation. """, @@ -352,8 +352,22 @@ def _javac(): ), } +def _friend_paths_attr(): + return attrs.list( + attrs.one_of(attrs.dep(), attrs.string()), + default = [], + doc = """ + Modules whose `internal` declarations this rule may access, passed to the Kotlin compiler as + friend-paths. + + An entry is either a target (`//foo/bar:baz`) or a build target pattern covering more than one + target (`//foo/bar/...` recursively, or `//foo/bar:` for a single package). +""", + ) + jvm_common = struct( test_env = _test_env, + friend_paths_attr = _friend_paths_attr, resources_arg = _resources_arg, remove_classes_arg = _remove_classes_arg, provided_deps = _provided_deps, diff --git a/prelude/decls/kotlin_rules.bzl b/prelude/decls/kotlin_rules.bzl index dd237a8..1b9b7e5 100644 --- a/prelude/decls/kotlin_rules.bzl +++ b/prelude/decls/kotlin_rules.bzl @@ -16,7 +16,7 @@ load("@prelude//:validation_deps.bzl", "VALIDATION_DEPS_ATTR_NAME") load("@prelude//android:build_only_native_code.bzl", "is_build_only_native_code") load("@prelude//android:configuration.bzl", "is_building_android_binary_attr") load("@prelude//decls:test_common.bzl", "test_common") -load(":common.bzl", "AnnotationProcessingTool", "SourceAbiVerificationMode", "TestType", "buck", "prelude_rule") +load(":common.bzl", "TestType", "buck", "prelude_rule") load(":java_rules.bzl", "dex_min_sdk_version") load(":jvm_common.bzl", "jvm_common") load(":re_test_common.bzl", "re_test_common") @@ -103,17 +103,6 @@ kotlin_library = prelude_rule( } | jvm_common.resources_arg() | { - "annotation_processing_tool": attrs.option( - attrs.enum(AnnotationProcessingTool), - default = None, - doc = """ - Specifies the tool to use for annotation processing. Possible values: "kapt" or "javac". - "kapt" allows running Java annotation processors against Kotlin sources while backporting - it for Java sources too. - "javac" works only against Java sources, Kotlin sources won't have access to generated - classes at compile time. - """, - ), "deps": attrs.list( attrs.dep(), default = [], @@ -129,14 +118,7 @@ kotlin_library = prelude_rule( List of additional arguments to pass into the Kotlin compiler. """, ), - "friend_paths": attrs.list( - attrs.dep(), - default = [], - doc = """ - List of source paths to pass into the Kotlin compiler as friend-paths, that is, modules - you can have access to internal methods. - """, - ), + "friend_paths": jvm_common.friend_paths_attr(), } | jvm_common.annotation_processors() | jvm_common.remove_classes_arg() @@ -167,10 +149,8 @@ kotlin_library = prelude_rule( "resources_root": attrs.option(attrs.string(), default = None), "runtime_deps": attrs.list(attrs.dep(), default = []), "source": attrs.option(attrs.string(), default = None), - "source_abi_verification_mode": attrs.option(attrs.enum(SourceAbiVerificationMode), default = None), "source_only_abi_deps": attrs.list(attrs.dep(), default = []), "target": attrs.option(attrs.string(), default = None), - "use_jvm_abi_gen": attrs.option(attrs.bool(), default = None), VALIDATION_DEPS_ATTR_NAME: attrs.set(attrs.dep(), sorted = True, default = []), "_build_only_native_code": attrs.default_only(attrs.bool(default = is_build_only_native_code())), "_dex_min_sdk_version": attrs.option(attrs.int(), default = dex_min_sdk_version()), @@ -268,7 +248,6 @@ kotlin_test = prelude_rule( | jvm_common.test_env() | jvm_common.abi_generation_mode() | { - "annotation_processing_tool": attrs.option(attrs.enum(AnnotationProcessingTool), default = None), "cxx_library_allowlist": attrs.list( attrs.dep(), default = [], @@ -283,7 +262,7 @@ kotlin_test = prelude_rule( "exported_provided_deps": attrs.list(attrs.dep(), default = []), "extra_arguments": attrs.list(attrs.string(), default = []), "extra_kotlinc_arguments": attrs.list(attrs.arg(anon_target_compatible = True), default = []), - "friend_paths": attrs.list(attrs.dep(), default = []), + "friend_paths": jvm_common.friend_paths_attr(), "java": attrs.option(attrs.dep(), default = None), "java_agents": attrs.list(attrs.source(), default = []), "java_version": attrs.option(attrs.string(), default = None), @@ -298,7 +277,6 @@ kotlin_test = prelude_rule( "resources_root": attrs.option(attrs.string(), default = None), "runtime_deps": attrs.list(attrs.dep(), default = []), "source": attrs.option(attrs.string(), default = None), - "source_abi_verification_mode": attrs.option(attrs.enum(SourceAbiVerificationMode), default = None), "source_only_abi_deps": attrs.list(attrs.dep(), default = []), "supports_test_execution_caching": attrs.bool(default = False), "target": attrs.option(attrs.string(), default = None), @@ -307,8 +285,6 @@ kotlin_test = prelude_rule( "test_info_type": attrs.option(attrs.string(), default = None), "unbundled_resources_root": attrs.option(attrs.source(allow_directory = True), default = None), "use_cxx_libraries": attrs.option(attrs.bool(), default = None), - "use_dependency_order_classpath": attrs.option(attrs.bool(), default = None), - "use_jvm_abi_gen": attrs.option(attrs.bool(), default = None), "_build_only_native_code": attrs.default_only(attrs.bool(default = is_build_only_native_code())), "_exec_os_type": buck.exec_os_type_arg(), "_is_building_android_binary": attrs.default_only(attrs.bool(default = False)), diff --git a/prelude/decls/python_rules.bzl b/prelude/decls/python_rules.bzl index 42eed9f..2df8d35 100644 --- a/prelude/decls/python_rules.bzl +++ b/prelude/decls/python_rules.bzl @@ -126,6 +126,11 @@ def _python_executable_attrs(): "_cxx_hacks": attrs.default_only(attrs.dep(default = "prelude//cxx/tools:cxx_hacks")), "_cxx_toolchain": toolchains_common.cxx(), "_exec_os_type": buck.exec_os_type_arg(), + "_gen_build_info": attrs.option(attrs.exec_dep(providers = [RunInfo]), default = None), + "_generated_build_info_enabled": attrs.bool(default = False), + "_generated_build_info_mode": attrs.enum(["full", "stable"], default = "stable"), + "_generated_build_info_spec": BUILD_INFO_ATTR, + "_late_build_info_stamping": attrs.any(default = None), "_python_internal_tools": python_common.internal_tools_arg(), "_python_toolchain": toolchains_common.python(), "_target_os_type": buck.target_os_type_arg(), diff --git a/prelude/decls/rust_rules.bzl b/prelude/decls/rust_rules.bzl index 0c89912..f4957da 100644 --- a/prelude/decls/rust_rules.bzl +++ b/prelude/decls/rust_rules.bzl @@ -60,9 +60,19 @@ _RUST_EXECUTABLE_ATTRIBUTES = { # Unlike cxx which supports pre-defined link groups, we only support # auto_link_groups in rust "auto_link_groups": attrs.bool(default = True), - # BOLT post-link optimization only applies when deferred_link_enabled is True. + # BOLT post-link optimization only applies when cxx performs the link + # (advanced_unstable_linking toolchains; see rust_link_binary). "bolt_flags": attrs.list(attrs.arg(), default = []), "bolt_profile": attrs.option(attrs.source(), default = None), + # Contents to embed into the linked executable as named ELF sections, added + # after BOLT and before build info stamping. Only applies when cxx performs + # the link (advanced_unstable_linking toolchains; see rust_link_binary). + "elf_sections": attrs.dict( + key = attrs.string(), + value = attrs.source(), + sorted = True, + default = {}, + ), "enable_distributed_thinlto": attrs.bool(default = False), "extra_dwp_flags": attrs.list(attrs.string(), default = []), # Opt the final executable output into content-based (immutable) buck-out diff --git a/prelude/decls/toolchains_common.bzl b/prelude/decls/toolchains_common.bzl index f7cdf11..8ee554b 100644 --- a/prelude/decls/toolchains_common.bzl +++ b/prelude/decls/toolchains_common.bzl @@ -6,7 +6,7 @@ # of this source tree. You may select, at your option, one of the # above-listed licenses. -load("@prelude//android:android_toolchain.bzl", "AndroidPlatformInfo", "AndroidToolchainInfo") +load("@prelude//android:android_toolchain.bzl", "AidlToolchainInfo", "AndroidPlatformInfo", "AndroidToolchainInfo") load("@prelude//csharp:toolchain.bzl", "CSharpToolchainInfo") load("@prelude//go:toolchain.bzl", "GoToolchainInfo") load("@prelude//go_bootstrap:go_bootstrap.bzl", "GoBootstrapToolchainInfo") @@ -39,6 +39,9 @@ def _toolchain(lang: str, providers: list[typing.Any], *, default: typing.Any = def _android_toolchain(): return _toolchain("android", [AndroidToolchainInfo, AndroidPlatformInfo, TestListingInfo]) +def _android_aidl_toolchain(): + return _toolchain("android_aidl", [AidlToolchainInfo]) + def _csharp_toolchain(): return _toolchain("csharp", [CSharpToolchainInfo]) @@ -117,6 +120,7 @@ def _test_toolchain(): toolchains_common = struct( android = _android_toolchain, + android_aidl = _android_aidl_toolchain, csharp = _csharp_toolchain, cxx = _cxx_toolchain, cython = _cython_toolchain, diff --git a/prelude/erlang/common_test/common/include/buck_ct_records.hrl b/prelude/erlang/common_test/common/include/buck_ct_records.hrl index 172118b..a355550 100644 --- a/prelude/erlang/common_test/common/include/buck_ct_records.hrl +++ b/prelude/erlang/common_test/common/include/buck_ct_records.hrl @@ -47,7 +47,9 @@ trampolines :: [binary()], timeout :: timeout(), ct_stdout_fingerprint :: ct_stdout:fingerprint(), - ct_stdout_streaming :: output_to_stdout | no_output_to_stdout + ct_stdout_streaming :: output_to_stdout | no_output_to_stdout, + %% Receives {run_succeed | run_failed, Result, ProgressMarkersOffsets} when the CT run ends + result_recipient :: pid() }). -record(run_specs, { diff --git a/prelude/erlang/common_test/common/src/artifact_annotations.erl b/prelude/erlang/common_test/common/src/artifact_annotations.erl index ba97d39..3eec71a 100644 --- a/prelude/erlang/common_test/common/src/artifact_annotations.erl +++ b/prelude/erlang/common_test/common/src/artifact_annotations.erl @@ -7,7 +7,6 @@ %% @format -module(artifact_annotations). --compile(warn_missing_spec_all). -moduledoc """ This file acts as a manual erlang sync for the thrift type struct TestResultArtifactAnnotations defined in https://fburl.com/code/r2t4vclb @@ -17,6 +16,14 @@ We mostly expect next iterations of thrift data structure to include more testArtifactTypes. Those should be manually added to the test_artifact_type() here. """. +-compile(warn_missing_spec_all). + +%% Public API +-export([serialize/1, create_artifact_annotation/2, default_annotation/1, test_metrics_artifact_annotation/2]). +-export_type([ + annotation_function/0, + test_result_artifact_annotations/0 +]). -import(common_util, [unicode_characters_to_binary/1]). @@ -37,13 +44,6 @@ test_artifact_type() here. -type annotation_function() :: fun((file:filename()) -> test_result_artifact_annotations()). -%% Public API --export([serialize/1, create_artifact_annotation/2, default_annotation/1, test_metrics_artifact_annotation/2]). --export_type([ - annotation_function/0, - test_result_artifact_annotations/0 -]). - -spec serialize(test_result_artifact_annotations()) -> iodata(). serialize(ArtifactAnnotation) -> json:encode(ArtifactAnnotation). @@ -70,7 +70,7 @@ default_annotation(FileName) -> test_metrics_artifact_annotation(FileName, Tests) -> FirstTest = hd(Tests), FirstTestName = unicode_characters_to_binary( - cth_tpx_test_tree:qualified_name( + common_util:qualified_name( lists:reverse(FirstTest#ct_test.groups), FirstTest#ct_test.test_name ) diff --git a/prelude/erlang/common_test/common/src/buck_ct_parser.erl b/prelude/erlang/common_test/common/src/buck_ct_parser.erl index 8885b55..7711b09 100644 --- a/prelude/erlang/common_test/common/src/buck_ct_parser.erl +++ b/prelude/erlang/common_test/common/src/buck_ct_parser.erl @@ -7,11 +7,11 @@ %% @format -module(buck_ct_parser). --compile(warn_missing_spec_all). -moduledoc """ Utilities method to parse string args given to the test binary via user input. """. +-compile(warn_missing_spec_all). -import(common_util, [unicode_characters_to_list/1]). diff --git a/prelude/erlang/common_test/common/src/buck_ct_provider.erl b/prelude/erlang/common_test/common/src/buck_ct_provider.erl index 06523c5..2d8e333 100644 --- a/prelude/erlang/common_test/common/src/buck_ct_provider.erl +++ b/prelude/erlang/common_test/common/src/buck_ct_provider.erl @@ -12,18 +12,6 @@ behavior module defining callbacks for buck2 test providers """. -compile(warn_missing_spec_all). --include("buck_ct_records.hrl"). - --type state() :: term(). - --type return_type() :: {ok, state()} | {error, term()}. - --type buck_ct_provider() :: {atom(), state()}. - --type init_argument_type() :: any(). - --include_lib("kernel/include/logger.hrl"). - -export([ do_init/2, do_pre_listing/1, @@ -36,6 +24,17 @@ behavior module defining callbacks for buck2 test providers init/2, pre_listing/1, post_listing/1, pre_running/1, post_running/1, terminate/1 ]). +-include("buck_ct_records.hrl"). +-include_lib("kernel/include/logger.hrl"). + +-type state() :: term(). + +-type return_type() :: {ok, state()} | {error, term()}. + +-type buck_ct_provider() :: {atom(), state()}. + +-type init_argument_type() :: any(). + % ------------------- Behaviors Callbacks ------------------------- %%% Initialize the state of the provider diff --git a/prelude/erlang/common_test/common/src/common_util.erl b/prelude/erlang/common_test/common/src/common_util.erl index afa1740..187cc86 100644 --- a/prelude/erlang/common_test/common/src/common_util.erl +++ b/prelude/erlang/common_test/common/src/common_util.erl @@ -7,17 +7,22 @@ %% % @format -module(common_util). +-compile(warn_missing_spec_all). -export([ unicode_characters_to_list/1, unicode_characters_to_binary/1, + qualified_name/2, + parse_test_name/2, + filename_all_to_filename/1, get_env/1, set_env/2 ]). --compile(warn_missing_spec_all). + +-include_lib("common/include/buck_ct_records.hrl"). -spec unicode_characters_to_list(unicode:chardata()) -> string(). unicode_characters_to_list(CharData) -> @@ -25,6 +30,38 @@ unicode_characters_to_list(CharData) -> R when not is_tuple(R) -> R end. +-doc """ +Gets the name for a testcase in a given group-path +The groups order expected here is [leaf_group, ...., root_group] +""". +-spec qualified_name(Groups, TestCase) -> string() when + Groups :: [atom()], + TestCase :: string() | atom(). +qualified_name(Groups, TestCase) -> + StringGroups = [atom_to_list(Group) || Group <- Groups], + JoinedGroups = string:join(lists:reverse(StringGroups), ":"), + Raw = io_lib:format("~ts.~ts", [JoinedGroups, TestCase]), + unicode_characters_to_list(Raw). + +-doc """ +Parse the test name, and decompose it into the test, group and suite atoms +""". +-spec parse_test_name(string(), atom()) -> #ct_test{}. +parse_test_name(Test, Suite) -> + [Groups0, TestName] = string:split(Test, ".", all), + Groups1 = + case Groups0 of + [] -> []; + _ -> string:split(Groups0, ":", all) + end, + Groups = [list_to_atom(GroupStr) || GroupStr <:- Groups1], + #ct_test{ + suite = Suite, + groups = Groups, + test_name = list_to_atom(TestName), + canonical_name = Test + }. + -spec unicode_characters_to_binary(unicode:chardata()) -> binary(). unicode_characters_to_binary(Chars) -> case unicode:characters_to_binary(Chars) of diff --git a/prelude/erlang/common_test/common/src/ct_error_printer.erl b/prelude/erlang/common_test/common/src/ct_error_printer.erl index d7a0907..8b3b0d8 100644 --- a/prelude/erlang/common_test/common/src/ct_error_printer.erl +++ b/prelude/erlang/common_test/common/src/ct_error_printer.erl @@ -7,11 +7,11 @@ %% @format -module(ct_error_printer). --compile([warn_missing_spec_all]). -moduledoc """ Helper module to format errors arising from common_test executions, in particular as generated by the assert.hrl module. """. +-compile([warn_missing_spec_all]). %% Public API -export([format_error/2, format_error/3, format_reason/1]). diff --git a/prelude/erlang/common_test/common/src/ct_stdout.erl b/prelude/erlang/common_test/common/src/ct_stdout.erl index 8258252..69ea93f 100644 --- a/prelude/erlang/common_test/common/src/ct_stdout.erl +++ b/prelude/erlang/common_test/common/src/ct_stdout.erl @@ -18,6 +18,8 @@ to delimit the output of each test-case. -export([emit_progress/4]). -export([process_raw_stdout_log/3, collect_method_stdout/4]). -export([init_process_stdout_state/3, process_stdout_line/2]). +-export_type([progress/0, callback/0, progress_line/0, offset/0, fingerprint/0, collected_stdout/0]). +-export_type([process_stdout_state/0]). -import(common_util, [unicode_characters_to_binary/1]). @@ -27,9 +29,6 @@ to delimit the output of each test-case. %% Types %% --------------------------------------------------------------------------- --export_type([progress/0, callback/0, progress_line/0, offset/0, fingerprint/0, collected_stdout/0]). --export_type([process_stdout_state/0]). - -type progress() :: started | finished. -type callback() :: init_per | end_per. -type progress_line() :: binary(). diff --git a/prelude/erlang/common_test/test_binary/src/list_test.erl b/prelude/erlang/common_test/common/src/list_test.erl similarity index 100% rename from prelude/erlang/common_test/test_binary/src/list_test.erl rename to prelude/erlang/common_test/common/src/list_test.erl index 9be1655..fd4e07d 100644 --- a/prelude/erlang/common_test/test_binary/src/list_test.erl +++ b/prelude/erlang/common_test/common/src/list_test.erl @@ -9,10 +9,10 @@ -module(list_test). -compile(warn_missing_spec_all). --include_lib("common/include/tpx_records.hrl"). - -export([list_tests/2]). +-include_lib("common/include/tpx_records.hrl"). + -import(common_util, [unicode_characters_to_binary/1]). -type ct_groupname() :: ct_suite:ct_groupname(). diff --git a/prelude/erlang/common_test/common/src/test_artifact_directory.erl b/prelude/erlang/common_test/common/src/test_artifact_directory.erl index c0b0508..76b539d 100644 --- a/prelude/erlang/common_test/common/src/test_artifact_directory.erl +++ b/prelude/erlang/common_test/common/src/test_artifact_directory.erl @@ -13,6 +13,11 @@ Used by TPX to upload diagnostic reports. """. -compile(warn_missing_spec_all). +%% Public API +-export([prepare/3, link_to_artifact_dir/3, find_log_private/1]). + +-export_type([dir_path/0]). + -include_lib("common/include/buck_ct_records.hrl"). -include_lib("kernel/include/logger.hrl"). @@ -20,11 +25,6 @@ Used by TPX to upload diagnostic reports. -define(raw_file_access, prim_file). -%% Public API --export([prepare/3, link_to_artifact_dir/3, find_log_private/1]). - --export_type([dir_path/0]). - -type dir_path() :: file:filename() | undefined. % Gets the artifactory directory path. @@ -120,6 +120,17 @@ prepare(ExecutionDir, Tests, ArtifactAnnotationFunction) -> end ). +-doc """ +Use hardlinks inside RE which doesn't follow symlinks like local execution uploads. +""". +-spec make_artifact_link(file:filename_all(), file:filename_all()) -> + ok | {error, badarg | file:posix()}. +make_artifact_link(File, ArtifactPath) -> + case os:getenv("INSIDE_RE_WORKER") of + "1" -> file:make_link(File, ArtifactPath); + _ -> file:make_symlink(filename:absname(File), ArtifactPath) + end. + -spec link_to_artifact_dir(File, Root, ArtifactAnnotationMFA) -> ok when File :: file:filename_all(), Root :: file:filename_all(), @@ -139,7 +150,7 @@ link_to_artifact_dir(File, Root, ArtifactAnnotationMFA) -> unicode_characters_to_list(string:replace(RelativePath, "/", ".", all)), case filelib:is_file(File, ?raw_file_access) of true -> - file:make_symlink(filename:absname(File), join_paths(ArtifactDir, FullFileName)), + make_artifact_link(File, join_paths(ArtifactDir, FullFileName)), Annotation = artifact_annotations:create_artifact_annotation(FullFileName, ArtifactAnnotationMFA), dump_annotation(Annotation, FullFileName); _ -> diff --git a/prelude/erlang/common_test/common/test/ct_stdout_SUITE.erl b/prelude/erlang/common_test/common/test/ct_stdout_SUITE.erl index 1d5db05..739197d 100644 --- a/prelude/erlang/common_test/common/test/ct_stdout_SUITE.erl +++ b/prelude/erlang/common_test/common/test/ct_stdout_SUITE.erl @@ -9,10 +9,6 @@ -module(ct_stdout_SUITE). -typing([eqwalizer]). --include_lib("stdlib/include/assert.hrl"). - --define(UNICODE_REPLACEMENT_CHAR_UTF8, <<239, 191, 189>>). - -export([all/0]). -export([ @@ -21,6 +17,10 @@ test_collect_method_stdout_sanitizes_invalid_utf8_without_truncation/1 ]). +-include_lib("stdlib/include/assert.hrl"). + +-define(UNICODE_REPLACEMENT_CHAR_UTF8, <<239, 191, 189>>). + all() -> [ test_collect_method_stdout_sanitizes_invalid_utf8_in_truncated_prefix, diff --git a/prelude/erlang/common_test/cth_hooks/src/cth_skip_cases.erl b/prelude/erlang/common_test/cth_hooks/src/cth_skip_cases.erl new file mode 100644 index 0000000..95d7ae8 --- /dev/null +++ b/prelude/erlang/common_test/cth_hooks/src/cth_skip_cases.erl @@ -0,0 +1,136 @@ +%% Copyright (c) Meta Platforms, Inc. and affiliates. +%% +%% This source code is licensed under both the MIT license found in the +%% LICENSE-MIT file in the root directory of this source tree and the Apache +%% License, Version 2.0 found in the LICENSE-APACHE file in the root directory +%% of this source tree. + +%% @format +-module(cth_skip_cases). +-compile(warn_missing_spec_all). + +%% Callbacks +-export([init/2]). + +-export([pre_init_per_suite/3]). +-export([pre_init_per_group/4]). +-export([post_init_per_group/5]). +-export([post_end_per_group/5]). + +-export([pre_init_per_testcase/4]). + +-type ct_suite() :: module(). +-type ct_groupname() :: ct_suite:ct_groupname(). +-type ct_testname() :: ct_suite:ct_testname(). +-type ct_config() :: ct_suite:ct_config(). +-type ct_config_or_skip_or_fail() :: + ct_config() | {skip, term()} | {fail, term()}. +-type ct_config_or_skip_or_fail_or_term() :: + ct_config_or_skip_or_fail() | ok | term(). + +-type qualified_name() :: string(). +-type reason() :: unicode:chardata(). +-type skip_spec() :: {qualified_name(), reason()}. + +-type state() :: #{ + skips := [skip_spec()], + groups := cth_tpx_test_tree:group_path() +}. + +-doc """ +Skips suite, group, or testcase callbacks whose qualified name matches one of +the configured regular expressions, each with its own reason. + +`pre_init_per_suite` matches against `""`, `pre_init_per_group` matches the +qualified group name with an empty testcase, and `pre_init_per_testcase` +matches the fully-qualified testcase name that +`common_util:qualified_name/2` reports. The first matching pattern wins. +""". +-spec init(Id, Opts) -> {ok, state()} when + Id :: term(), + Opts :: [skip_spec()]. +init(_Id, Opts) -> + {ok, #{skips => Opts, groups => []}}. + +-spec pre_init_per_suite(_Suite, Config, State) -> {ct_config_or_skip_or_fail(), state()} when + _Suite :: ct_suite(), + Config :: ct_config(), + State :: state(). +pre_init_per_suite(_Suite, Config, State = #{skips := Skips}) -> + case find_skip("", Skips) of + {ok, Reason} -> + {{skip, Reason}, State}; + error -> + {Config, State} + end. + +-spec pre_init_per_group(_Suite, Group, Config, State) -> {ct_config_or_skip_or_fail(), state()} when + _Suite :: ct_suite(), + Group :: ct_groupname(), + Config :: ct_config(), + State :: state(). +pre_init_per_group(_Suite, Group, Config, State = #{skips := Skips, groups := Groups}) -> + State1 = State#{groups := [Group | Groups]}, + case find_skip(common_util:qualified_name([Group | Groups], ""), Skips) of + {ok, Reason} -> + {{skip, Reason}, State1}; + error -> + {Config, State1} + end. + +-spec post_init_per_group( + ct_suite(), ct_groupname(), ct_config(), ct_config_or_skip_or_fail_or_term(), state() +) -> + {ct_config_or_skip_or_fail_or_term(), state()}. +post_init_per_group(_Suite, Group, _Config, Return, State) when not is_list(Return) -> + {Return, pop_group(Group, State)}; +post_init_per_group(_Suite, _Group, _Config, Return, State) -> + {Return, State}. + +-spec post_end_per_group( + ct_suite(), ct_groupname(), ct_config(), ct_config_or_skip_or_fail_or_term(), state() +) -> + {ct_config_or_skip_or_fail_or_term(), state()}. +post_end_per_group(_Suite, Group, _Config, Return, State) -> + {Return, pop_group(Group, State)}. + +-spec pre_init_per_testcase + (_Suite, _TestCase, SkipOrFail, State) -> {ct_config_or_skip_or_fail(), state()} when + _Suite :: ct_suite(), + _TestCase :: ct_testname(), + SkipOrFail :: ct_config_or_skip_or_fail(), + State :: state(); + (_Suite, TestCase, Config, State) -> {ct_config_or_skip_or_fail(), state()} when + _Suite :: ct_suite(), + TestCase :: ct_testname(), + Config :: ct_config(), + State :: state(). +pre_init_per_testcase(_Suite, _TestCase, {Tag, _Reason} = SkipOrFail, State) when + Tag =:= skip; Tag =:= fail +-> + {SkipOrFail, State}; +pre_init_per_testcase(_Suite, TestCase, Config, State = #{skips := Skips, groups := Groups}) -> + case find_skip(common_util:qualified_name(Groups, TestCase), Skips) of + {ok, Reason} -> + {{skip, Reason}, State}; + error -> + {Config, State} + end. + +-spec find_skip(Name, Skips) -> {ok, Reason} | error when + Name :: qualified_name(), + Skips :: [skip_spec()], + Reason :: reason(). +find_skip(_Name, []) -> + error; +find_skip(Name, [{Pattern, Reason} | Rest]) -> + case re:run(Name, Pattern, [{capture, none}]) of + match -> {ok, Reason}; + nomatch -> find_skip(Name, Rest) + end. + +-spec pop_group(ct_groupname(), state()) -> state(). +pop_group(Group, State = #{groups := [Group | Groups]}) -> + State#{groups := Groups}; +pop_group(_Group, State) -> + State. diff --git a/prelude/erlang/common_test/cth_hooks/src/cth_tpx.erl b/prelude/erlang/common_test/cth_hooks/src/cth_tpx.erl index 1fa0eb7..4cff7d1 100644 --- a/prelude/erlang/common_test/cth_hooks/src/cth_tpx.erl +++ b/prelude/erlang/common_test/cth_hooks/src/cth_tpx.erl @@ -38,6 +38,10 @@ %% please dialyzer -export([ok_group/1, fail_group/1]). +-export_type([ + shared_state/0 +]). + -import(common_util, [unicode_characters_to_list/1]). %% For tests purposes @@ -52,10 +56,6 @@ %% Types %% ----------------------------------------------------------------------------- --export_type([ - shared_state/0 -]). - -type tree_node() :: cth_tpx_test_tree:tree_node(). -type group_path() :: cth_tpx_test_tree:group_path(). -type outcome() :: cth_tpx_test_tree:outcome(). @@ -293,10 +293,8 @@ post_init_per_suite(Suite, _Config, {skip, {failed, _} = Reason} = Error, HookSt end); post_init_per_suite(Suite, _Config, {skip, Reason} = Error, HookState) -> on_shared_state(HookState, ?FUNCTION_NAME, Error, fun(State) -> - % In this case the init_per_suite returns with a {skip, Reason} - % It then passed fine. Desc = fmt_init_or_end(Suite, init_per_suite, Reason, ~"SKIPPED"), - {Error, add_result(?INIT_PER_SUITE, passed, Desc, State)} + {Error, add_result(?INIT_PER_SUITE, skipped, Desc, State)} end); post_init_per_suite(Suite, _Config, {fail, Reason} = Error, HookState) -> on_shared_state(HookState, ?FUNCTION_NAME, Error, fun(State) -> @@ -610,7 +608,7 @@ method_name(Method, Groups) -> MethodName0 -> atom_to_list(MethodName0) end, - cth_tpx_test_tree:qualified_name(Groups, MethodName). + common_util:qualified_name(Groups, MethodName). -spec pre_end_per_testcase(ct_suite(), ct_testname(), ct_config(), hook_state()) -> {ct_config(), hook_state()}. pre_end_per_testcase(_Suite, TC, Config, HookState) -> diff --git a/prelude/erlang/common_test/cth_hooks/src/cth_tpx_role.erl b/prelude/erlang/common_test/cth_hooks/src/cth_tpx_role.erl index 9465af8..2f4bc75 100644 --- a/prelude/erlang/common_test/cth_hooks/src/cth_tpx_role.erl +++ b/prelude/erlang/common_test/cth_hooks/src/cth_tpx_role.erl @@ -63,6 +63,11 @@ priorities: """. -compile(warn_missing_spec_all). +-export([ + role_priority/1, + is_responsible/2 +]). + -export_type([ role/0, responsibility/0 @@ -70,11 +75,6 @@ priorities: -type role() :: top | bot. --export([ - role_priority/1, - is_responsible/2 -]). - -doc """ Default hook priority for the role - In CT, the hook with the lowest numerical value has "highest priority" and is diff --git a/prelude/erlang/common_test/cth_hooks/src/cth_tpx_test_tree.erl b/prelude/erlang/common_test/cth_hooks/src/cth_tpx_test_tree.erl index 044f152..85d26c6 100644 --- a/prelude/erlang/common_test/cth_hooks/src/cth_tpx_test_tree.erl +++ b/prelude/erlang/common_test/cth_hooks/src/cth_tpx_test_tree.erl @@ -9,7 +9,14 @@ -module(cth_tpx_test_tree). -compile([warn_missing_spec_all]). --include("method_ids.hrl"). +%% Public API +-export([ + new_node/1, + new_leaf/1, + + register_result/4, + collect_results/3 +]). -export_type([ tree/0, @@ -27,16 +34,7 @@ method_id/0 ]). -%% Public API --export([ - qualified_name/2, - - new_node/1, - new_leaf/1, - - register_result/4, - collect_results/3 -]). +-include("method_ids.hrl"). -import(common_util, [unicode_characters_to_list/1]). @@ -93,19 +91,6 @@ ends := [collected_method_result()] }. --doc """ -Gets the name for a testcase in a given group-path -The groups order expected here is [leaf_group, ...., root_group] -""". --spec qualified_name(Groups, TestCase) -> string() when - Groups :: group_path(), - TestCase :: name(). -qualified_name(Groups, TestCase) -> - StringGroups = [atom_to_list(Group) || Group <- Groups], - JoinedGroups = string:join(lists:reverse(StringGroups), ":"), - Raw = io_lib:format("~ts.~ts", [JoinedGroups, TestCase]), - unicode_characters_to_list(Raw). - %% Tree creation and update -doc """ @@ -202,7 +187,7 @@ Provides a result for a given specific requested_result. TestCase :: atom(), CollectedStdOut :: ct_stdout:collected_stdout(). collect_result(TreeResult, Groups, TestCase, CollectedStdOut) -> - QualifiedName = qualified_name(lists:reverse(Groups), TestCase), + QualifiedName = common_util:qualified_name(lists:reverse(Groups), TestCase), LeafResult = collect_result(TreeResult, [], [], Groups, TestCase, QualifiedName, CollectedStdOut), #{ends := EndsResults, main := MainResult} = LeafResult, MainResultWithEndFailure = report_end_failure(EndsResults, MainResult), @@ -380,12 +365,14 @@ get_missing_result(Inits, QualifiedName, CollectedStdOut) -> handle_skipped_result(Inits, MainResult, CollectedStdOut). -doc """ -Generates an user informative message in the case of the missing result by attempting to find the right init to blame. - -Notice that an Erlang test-result can be `skipped` if it is either skipped by the user or was skipped because of an init failure. -As `skipped` is an error state in tpx, if it was skipped by the user, the test is reported as omitted, which is not an error state. -In the case where it is skipped because of init failure, it is reported as failed with appropriate user message reporting -to the init to be blamed. +Attributes a test case that left no result of its own, or that reports itself as +`skipped`, to the init that is responsible for it. + +`Inits` runs innermost first, and the first one that did not pass is the one blamed: the +case takes that init's outcome, with a message naming the init and quoting its details. +`failed`, `timeout`, `infra_failure`, and `skipped` therefore carry through as +themselves; an `omitted` init is blamed as a failure. When every init passed, +`MainResult` stands. """. -spec handle_skipped_result(Inits, MainResult, CollectedStdOut) -> collected_method_result() when Inits :: [collected_method_result()], @@ -435,7 +422,17 @@ handle_skipped_result([Init | Inits], MainResult = #{name := Name}, CollectedStd passed -> handle_skipped_result(Inits, MainResult, CollectedStdOut); skipped -> - handle_skipped_result(Inits, MainResult, CollectedStdOut); + #{ + name => Name, + outcome => skipped, + details => + io_lib:format( + ~"Skipped because init ~ts was skipped, with reason:\n ~ts", + [maps:get(name, Init), maps:get(details, Init)] + ), + + std_out => InitStdOut + }; omitted -> #{ name => Name, diff --git a/prelude/erlang/common_test/test_binary/src/json_interfacer.erl b/prelude/erlang/common_test/test_binary/src/json_interfacer.erl index eb457a1..d60c67b 100644 --- a/prelude/erlang/common_test/test_binary/src/json_interfacer.erl +++ b/prelude/erlang/common_test/test_binary/src/json_interfacer.erl @@ -13,10 +13,12 @@ defined in tpx [here](https://www.internalfb.com/code/fbsource/[bb9e81daacad]/fb """. -compile(warn_missing_spec_all). --export([write_json_output/2, format_json/1, status_name/1]). +-export([write_json_output/4, format_json/3, status_name/1]). -import(common_util, [unicode_characters_to_binary/1]). +-define(RESULT_EXEC_VERSION, 1). + -define(PASSED, <<"PASSED">>). -define(FAILED, <<"FAILED">>). -define(SKIPPED, <<"SKIPPED">>). @@ -72,15 +74,37 @@ summary(infra_failure) -> ?INFRA_FAILURE. ends := [formatted_result()] }. --spec write_json_output(file:filename_all(), [collected_result()]) -> {ok, file:filename_all()}. -write_json_output(OutputDir, TpxResults) -> +-type formatted_output() :: + #{ + version := pos_integer(), + status := integer(), + summary := binary(), + details := binary(), + test_results := [formatted_case_result()] + }. + +-spec write_json_output(file:filename_all(), status(), unicode:chardata(), [collected_result()]) -> + {ok, file:filename_all()}. +write_json_output(OutputDir, SuiteStatus, SuiteDetails, TpxResults) -> OuptputFile = filename:join(OutputDir, "result_exec.json"), - file:write_file(OuptputFile, format_json(TpxResults), [raw, binary]), + file:write_file( + OuptputFile, format_json(SuiteStatus, SuiteDetails, TpxResults), [raw, binary] + ), {ok, OuptputFile}. --spec format_json([collected_result()]) -> iodata(). -format_json(TpxResults) -> - json:encode([format_case(CaseResult) || CaseResult <- TpxResults]). +-spec format_json(status(), unicode:chardata(), [collected_result()]) -> iodata(). +format_json(SuiteStatus, SuiteDetails, TpxResults) -> + json:encode(format_output(SuiteStatus, SuiteDetails, TpxResults)). + +-spec format_output(status(), unicode:chardata(), [collected_result()]) -> formatted_output(). +format_output(SuiteStatus, SuiteDetails, TpxResults) -> + #{ + version => ?RESULT_EXEC_VERSION, + status => status(SuiteStatus), + summary => summary(SuiteStatus), + details => unicode_characters_to_binary(SuiteDetails), + test_results => [format_case(CaseResult) || CaseResult <- TpxResults] + }. -spec format_case(collected_result()) -> formatted_case_result(). format_case( diff --git a/prelude/erlang/common_test/test_binary/src/listing_interfacer.erl b/prelude/erlang/common_test/test_binary/src/listing_interfacer.erl index b9ee49c..72c2dd2 100644 --- a/prelude/erlang/common_test/test_binary/src/listing_interfacer.erl +++ b/prelude/erlang/common_test/test_binary/src/listing_interfacer.erl @@ -10,9 +10,10 @@ -compile(warn_missing_spec_all). --include_lib("common/include/tpx_records.hrl"). -export([produce_json_file/2]). +-include_lib("common/include/tpx_records.hrl"). + -import(common_util, [unicode_characters_to_binary/1]). -spec produce_json_file(OutputDir, TestCase) -> ok when diff --git a/prelude/erlang/common_test/test_binary/src/test_binary.erl b/prelude/erlang/common_test/test_binary/src/test_binary.erl index 34bd2a5..e2aa134 100644 --- a/prelude/erlang/common_test/test_binary/src/test_binary.erl +++ b/prelude/erlang/common_test/test_binary/src/test_binary.erl @@ -299,14 +299,14 @@ stdout_streaming(_) -> output_to_stdout. -spec print_results(file:filename()) -> boolean(). print_results(ResultsFile) -> {ok, Data} = file:read_file(ResultsFile, [raw]), - Results = json:decode(Data), - {Summary, AnyFailure} = lists:foldl(fun print_individual_results/2, {#{}, false}, Results), + #{<<"status">> := SuiteStatus, <<"test_results">> := Results} = json:decode(Data), + {Summary, AnyTestFailure} = lists:foldl(fun print_individual_results/2, {#{}, false}, Results), io:format("~n~10s: ~b~n~n", ["TOTAL", lists:sum(maps:values(Summary))]), [ io:format("~10ts: ~b~n", [json_interfacer:status_name(Result), Amount]) || Result := Amount <- Summary ], - AnyFailure. + AnyTestFailure orelse json_interfacer:status_name(SuiteStatus) =/= passed. -spec print_individual_results(map(), Acc) -> Acc when Acc :: {#{non_neg_integer() => non_neg_integer()}, boolean()}. print_individual_results(Result, {Summary, AnyFailure}) -> diff --git a/prelude/erlang/common_test/test_binary/src/test_info.erl b/prelude/erlang/common_test/test_binary/src/test_info.erl index 0d8287f..f4d876c 100644 --- a/prelude/erlang/common_test/test_binary/src/test_info.erl +++ b/prelude/erlang/common_test/test_binary/src/test_info.erl @@ -10,10 +10,11 @@ -compile(warn_missing_spec_all). -export([load_from_file/1, write_to_file/2, try_make_path_relative/1]). +-export_type([test_info/0]). + -include_lib("common/include/buck_ct_records.hrl"). -type test_info() :: #test_info{}. --export_type([test_info/0]). -import(common_util, [unicode_characters_to_list/1, unicode_characters_to_binary/1]). diff --git a/prelude/erlang/common_test/test_binary/src/test_runner.erl b/prelude/erlang/common_test/test_binary/src/test_runner.erl index 5238cc2..68d8cc4 100644 --- a/prelude/erlang/common_test/test_binary/src/test_runner.erl +++ b/prelude/erlang/common_test/test_binary/src/test_runner.erl @@ -9,14 +9,12 @@ -module(test_runner). -compile(warn_missing_spec_all). +-export([run_tests/6]). + -include_lib("common/include/tpx_records.hrl"). -include_lib("common/include/buck_ct_records.hrl"). -include_lib("kernel/include/logger.hrl"). --export([run_tests/6, mark_success/2, mark_failure/2]). - --export([parse_test_name/2]). - -import(common_util, [unicode_characters_to_list/1, unicode_characters_to_binary/1]). -define(DEFAULT_OUTPUT_FORMAT, json). @@ -36,7 +34,7 @@ run_tests(Tests, #test_info{} = TestInfo, OutputDir, Listing, Timeout, StdoutStr SuiteBin when is_binary(SuiteBin) -> binary_to_atom(SuiteBin); SuiteStr when is_list(SuiteStr) -> list_to_atom(SuiteStr) end, - StructuredTests = [parse_test_name(Test, Suite) || Test <- Tests], + StructuredTests = [common_util:parse_test_name(Test, Suite) || Test <- Tests], case StructuredTests of [] -> throw(no_tests_to_run); @@ -64,7 +62,8 @@ run_tests(Tests, #test_info{} = TestInfo, OutputDir, Listing, Timeout, StdoutStr trampolines = TestInfo#test_info.trampolines, timeout = Timeout, ct_stdout_fingerprint = ct_stdout:make_fingerprint(), - ct_stdout_streaming = StdoutStreaming + ct_stdout_streaming = StdoutStreaming, + result_recipient = self() } ) end. @@ -110,7 +109,6 @@ execute_test_suite(TestEnv) -> TestEnv :: #test_env{}, Timeout :: timeout(). run_test(TestEnv, Timeout) -> - register(?MODULE, self()), application:set_env(test_exec, test_env, TestEnv, [{persistent, true}]), case application:ensure_all_started(test_exec, temporary) of {ok, _Apps} -> @@ -135,9 +133,9 @@ run_test(TestEnv, Timeout) -> after Timeout -> ensure_test_exec_stopped(), ErrorMsg = - "\n***************************************************************\n" - "* the suite timed out, all tests will be reported as failure. *\n" - "***************************************************************\n", + "\n************************\n" + "* the suite timed out. *\n" + "************************\n", test_run_timeout(TestEnv, ErrorMsg) end; {error, Reason} -> @@ -214,14 +212,14 @@ provide_output_file( Results = case Status of failed -> - collect_results_broken_run(Tests, Suite, ~"internal crash", ResultExec, LogFilesForCrashes); + collect_results_broken_run( + Tests, Suite, ~"internal crash", ResultExec, LogFilesForCrashes + ); timeout -> - % Suite timeout: this is typically a user error, so we don't want to display the - % executor logs. - StdOutLogFile = #{ct_executor_stdout => StdOutFile}, - collect_results_broken_run(Tests, Suite, ~"", ResultExec, StdOutLogFile); + % Once the suite watchdog fires, group/suite teardown may not + % have completed, so no testcase has an independent verdict. + []; passed -> - % Here we either passed or timeout. case file:read_file(ResultsFile, [raw]) of {ok, JsonFile} -> TreeResults = decode_erlang_term(JsonFile), @@ -233,7 +231,9 @@ provide_output_file( ResultsFile ] ), - collect_results_broken_run(Tests, Suite, ErrorMsg, ResultExec, LogFilesForCrashes); + collect_results_broken_run( + Tests, Suite, ErrorMsg, ResultExec, LogFilesForCrashes + ); _ -> {ok, CollectedStdOut} = ct_stdout:collect_method_stdout( StdOutFile, @@ -247,11 +247,14 @@ provide_output_file( ErrorMsg = io_lib:format(~"ct failed to produced results file ~tp", [ ResultsFile ]), - collect_results_broken_run(Tests, Suite, ErrorMsg, ResultExec, LogFilesForCrashes) + collect_results_broken_run( + Tests, Suite, ErrorMsg, ResultExec, LogFilesForCrashes + ) end end, - {ok, _ResultOuptuFile} = json_interfacer:write_json_output(OutputDir, Results), + {ok, _ResultOuptuFile} = + json_interfacer:write_json_output(OutputDir, Status, ResultExec, Results), test_artifact_directory:link_to_artifact_dir( StdOutFile, OutputDir, ArtifactAnnotationFunction ), @@ -327,10 +330,10 @@ collect_results_broken_run(Tests, _Suite, ErrorMsg, ResultExec, RelevantLogFiles main => #{ name => lists:flatten( io_lib:format("~ts.[main_testcase]", [ - % We need to reverse the list of groups as the method cth_tpx_test_tree:qualified_name expects them + % We need to reverse the list of groups as the method common_util:qualified_name expects them % in the reverse order (as it is designed to be called when exploring the tree of results % where we push at each time the group we are in, leading to them being in reverse order). - cth_tpx_test_tree:qualified_name( + common_util:qualified_name( lists:reverse(Test#ct_test.groups), Test#ct_test.test_name ) @@ -359,32 +362,48 @@ provided by ct displaying results of all the tests ran. Tests :: [#ct_test{}], CollectedStdOut :: ct_stdout:collected_stdout(). collect_results_fine_run(TreeResults, Tests, CollectedStdOut) -> - cth_tpx_test_tree:collect_results(TreeResults, maps:from_list(get_requested_tests(Tests)), CollectedStdOut). + cth_tpx_test_tree:collect_results( + TreeResults, get_requested_tests_by_group_path(Tests), CollectedStdOut + ). -doc """ -Returns a list of the tests by classifying from the (sequence) of groups they belong. -The list is [{[sequence of groups] => [list of tests belonging to this sequence]}]. -We make sure to respect the group / test insertion order. That is, if the sequence is -g1.t1, g2.t2, g1.t2, g1.t3, g2.t2, we produce: -[g1.[t1,t2,t3], g2.[t1,t2]] +Splits the requested tests into consecutive runs of tests sharing a (sequence) of groups. +The list is [{[sequence of groups], [list of tests belonging to this sequence]}], in the order the +tests were requested, so a sequence returned to later gets a further entry. That is, if the sequence +is g1.t1, g2.t1, g1.t2, g1.t3, g2.t2, we produce: +[{g1,[t1]}, {g2,[t1]}, {g1,[t2,t3]}, {g2,[t2]}] +The tests are expected in listing order, the depth first traversal of all/0 with groups/0 expanded +that reorder_tests/2 puts them in. The runs only follow the order the suite declares under that +precondition. """. --spec get_requested_tests([#ct_test{}]) -> [{[atom()], [atom()]}]. +-spec get_requested_tests(Tests) -> [{GroupPath, TestCases}] when + Tests :: [#ct_test{}], + GroupPath :: cth_tpx_test_tree:group_path(), + TestCases :: [ct_suite:ct_testname()]. get_requested_tests(Tests) -> - {TestMap, RevOrderedKeys} = lists:foldl( - fun(Test, {Map, Keys}) -> - Groups = Test#ct_test.groups, - TestName = Test#ct_test.test_name, - case Map of - #{Groups := Existing} -> - {Map#{Groups => [TestName | Existing]}, Keys}; - _ -> - {Map#{Groups => [TestName]}, [Groups | Keys]} + lists:foldr( + fun(#ct_test{groups = Groups, test_name = TestName}, Runs) -> + case Runs of + [{Groups, TestNames} | Rest] -> [{Groups, [TestName | TestNames]} | Rest]; + _ -> [{Groups, [TestName]} | Runs] end end, - {#{}, []}, + [], Tests - ), - [{Key, lists:reverse(maps:get(Key, TestMap))} || Key <- lists:reverse(RevOrderedKeys)]. + ). + +-spec get_requested_tests_by_group_path(Tests) -> #{GroupPath => TestCases} when + Tests :: [#ct_test{}], + GroupPath :: cth_tpx_test_tree:group_path(), + TestCases :: [ct_suite:ct_testname()]. +get_requested_tests_by_group_path(Tests) -> + lists:foldr( + fun(#ct_test{groups = Groups, test_name = TestName}, Acc) -> + maps:update_with(Groups, fun(TestNames) -> [TestName | TestNames] end, [TestName], Acc) + end, + #{}, + Tests + ). -doc """ Built the test_spec selecting the requested tests and @@ -454,25 +473,6 @@ add_spec_if_absent({Key, Value}, CtOpts) -> _ -> CtOpts end. --doc """ -Parse the test name, and decompose it into the test, group and suite atoms -""". --spec parse_test_name(string(), atom()) -> #ct_test{}. -parse_test_name(Test, Suite) -> - [Groups0, TestName] = string:split(Test, ".", all), - Groups1 = - case Groups0 of - [] -> []; - _ -> string:split(Groups0, ":", all) - end, - Groups = [list_to_atom(GroupStr) || GroupStr <:- Groups1], - #ct_test{ - suite = Suite, - groups = Groups, - test_name = list_to_atom(TestName), - canonical_name = Test - }. - -spec reorder_tests(list(#ct_test{}), #test_spec_test_case{}) -> list(#ct_test{}). reorder_tests(Tests, #test_spec_test_case{testcases = TestCases}) -> % This is the ordered lists of test from the suite as @@ -503,26 +503,6 @@ set_up_log_dir(OutputDir) -> ok = filelib:ensure_path(LogDir), LogDir. --doc """ -Informs the test runner of a successful test run. -""". --spec mark_success(Result, ProgressMarkersOffsets) -> ok when - Result :: unicode:chardata(), - ProgressMarkersOffsets :: #{ct_stdout:progress_line() => ct_stdout:offset()}. -mark_success(Result, ProgressMarkersOffsets) -> - ?MODULE ! {run_succeed, Result, ProgressMarkersOffsets}, - ok. - --doc """ -Informs the test runner of a fataled test run. -""". --spec mark_failure(Result, ProgressMarkersOffsets) -> ok when - Result :: unicode:chardata(), - ProgressMarkersOffsets :: #{ct_stdout:progress_line() => ct_stdout:offset()}. -mark_failure(Error, ProgressMarkersOffsets) -> - ?MODULE ! {run_failed, Error, ProgressMarkersOffsets}, - ok. - -doc """ CtOpts must be tuple as defined here: https://www.erlang.org/doc/apps/common_test/run_test_chapter.html#test-specification-syntax diff --git a/prelude/erlang/common_test/test_cli_lib/src/test.erl b/prelude/erlang/common_test/test_cli_lib/src/test.erl index 8f28132..914c193 100644 --- a/prelude/erlang/common_test/test_cli_lib/src/test.erl +++ b/prelude/erlang/common_test/test_cli_lib/src/test.erl @@ -15,8 +15,6 @@ User-Facing library for quick-iteration testing of Common Test -typing([eqwalizer]). -compile(warn_missing_spec_all). --include_lib("common/include/buck_ct_records.hrl"). - %% Public API -export([ start/0, @@ -40,6 +38,8 @@ User-Facing library for quick-iteration testing of Common Test list_impl/1 ]). +-include_lib("common/include/buck_ct_records.hrl"). + -define(raw_file_access, prim_file). -type test_id() :: string() | non_neg_integer() | atom(). diff --git a/prelude/erlang/common_test/test_cli_lib/test/test_cli_e2e_SUITE.erl b/prelude/erlang/common_test/test_cli_lib/test/test_cli_e2e_SUITE.erl index 8a00a93..c3f8706 100644 --- a/prelude/erlang/common_test/test_cli_lib/test/test_cli_e2e_SUITE.erl +++ b/prelude/erlang/common_test/test_cli_lib/test/test_cli_e2e_SUITE.erl @@ -7,11 +7,6 @@ -module(test_cli_e2e_SUITE). -typing([eqwalizer]). -% elp:ignore WA003 (better_assertions) - Open Source --include_lib("stdlib/include/assert.hrl"). --include_lib("common_test/include/ct.hrl"). --include_lib("common/include/buck_ct_records.hrl"). - -export([suite/0, all/0, init_per_suite/1, end_per_suite/1]). -export([ @@ -19,6 +14,11 @@ test_try_make_path_relative/1 ]). +% elp:ignore WA003 (better_assertions) - Open Source +-include_lib("stdlib/include/assert.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("common/include/buck_ct_records.hrl"). + suite() -> [{appatic, #{enable_autoclean => true}}]. @@ -26,7 +26,7 @@ all() -> [test_list, test_try_make_path_relative]. init_per_suite(Config) -> - PrivDir = ?config(priv_dir, Config), + PrivDir = proplists:get_value(priv_dir, Config), TestInfoFile = filename:join(PrivDir, <<"test_info">>), {ok, [ErlCmd]} = init:get_argument(progname), diff --git a/prelude/erlang/common_test/test_exec/src/ct_daemon_core.erl b/prelude/erlang/common_test/test_exec/src/ct_daemon_core.erl index 0667004..c4cd1b4 100644 --- a/prelude/erlang/common_test/test_exec/src/ct_daemon_core.erl +++ b/prelude/erlang/common_test/test_exec/src/ct_daemon_core.erl @@ -12,10 +12,6 @@ Stateless Core functionality for ct_daemon """. -compile(warn_missing_spec_all). --include_lib("common/include/tpx_records.hrl"). --include_lib("common/include/buck_ct_records.hrl"). --include_lib("kernel/include/logger.hrl"). - %% Public API -export([ test_suites/0, @@ -29,6 +25,12 @@ Stateless Core functionality for ct_daemon path_timetrap/1 ]). +-export_type([reason/0, run_result/0, setup/0]). + +-include_lib("common/include/tpx_records.hrl"). +-include_lib("common/include/buck_ct_records.hrl"). +-include_lib("kernel/include/logger.hrl"). + -type reason() :: term(). -type run_result() :: term(). @@ -58,8 +60,6 @@ Stateless Core functionality for ct_daemon | {skip, Where :: atom(), Reason :: term()} | {fail, Where :: atom(), Reason :: term()}. --export_type([reason/0, run_result/0, setup/0]). - -define(DEFAULT_TIMETRAP, 30 * 60 * 1000). -type ct_test_result() :: diff --git a/prelude/erlang/common_test/test_exec/src/ct_daemon_hooks.erl b/prelude/erlang/common_test/test_exec/src/ct_daemon_hooks.erl index a3c295b..173604f 100644 --- a/prelude/erlang/common_test/test_exec/src/ct_daemon_hooks.erl +++ b/prelude/erlang/common_test/test_exec/src/ct_daemon_hooks.erl @@ -7,7 +7,6 @@ %% @format -module(ct_daemon_hooks). --compile(warn_missing_spec_all). -moduledoc """ Implementation of hooks functionality. We mimic the behaviour of common test hooks so that they can run in test shell @@ -26,8 +25,6 @@ common test hooks so that they can run in test shell get_hooks/0 ]). --export_type([part/0]). - %% gen_server callbacks -export([ init/1, @@ -35,6 +32,8 @@ common test hooks so that they can run in test shell handle_cast/2 ]). +-export_type([part/0]). + -type id() :: term(). -type opts() :: term(). -type hook_state() :: term(). diff --git a/prelude/erlang/common_test/test_exec/src/ct_daemon_node.erl b/prelude/erlang/common_test/test_exec/src/ct_daemon_node.erl index 0175cbb..6bd7688 100644 --- a/prelude/erlang/common_test/test_exec/src/ct_daemon_node.erl +++ b/prelude/erlang/common_test/test_exec/src/ct_daemon_node.erl @@ -9,13 +9,15 @@ -module(ct_daemon_node). -compile(warn_missing_spec_all). --include_lib("kernel/include/logger.hrl"). - %% Public API -export([start/1, start/2, stop/0, alive/0, get_node/0]). -export([node_main/1, get_domain_type/0]). +-export_type([config/0]). + +-include_lib("kernel/include/logger.hrl"). + -import(common_util, [unicode_characters_to_binary/1, filename_all_to_filename/1]). -define(LOG_BASE, "/tmp/ct_daemon"). @@ -32,8 +34,6 @@ | {ct_hooks, [atom() | {atom(), [term()]}]} | {output_dir, file:filename_all()}. --export_type([config/0]). - -doc """ start node for running tests in isolated way and keep state """. diff --git a/prelude/erlang/common_test/test_exec/src/ct_daemon_runner.erl b/prelude/erlang/common_test/test_exec/src/ct_daemon_runner.erl index 4e608d7..216774f 100644 --- a/prelude/erlang/common_test/test_exec/src/ct_daemon_runner.erl +++ b/prelude/erlang/common_test/test_exec/src/ct_daemon_runner.erl @@ -12,8 +12,6 @@ gen_server holding state between test runs """. -compile(warn_missing_spec_all). --include_lib("kernel/include/logger.hrl"). - -behavior(gen_server). %% gen_server API @@ -22,6 +20,10 @@ gen_server holding state between test runs %% Public API -export([start_monitor/2, name/1]). +-export_type([discover_error/0]). + +-include_lib("kernel/include/logger.hrl"). + -type state() :: #{ enumerated_tests => #{test_id() => string()}, output_dir => file:filename_all(), @@ -39,8 +41,6 @@ gen_server holding state between test runs | invalid_regex | {invalid_regex, {string(), non_neg_integer()}}. --export_type([discover_error/0]). - -spec start_monitor(Node, OutputDir) -> gen_server:start_mon_ret() when Node :: node(), OutputDir :: file:filename_all(). @@ -219,7 +219,7 @@ flatten_enumerated_tests(Tests) -> -spec run_test(string(), state()) -> {ct_daemon_core:run_result(), state()}. run_test(Test, State = #{output_dir := OutputDir, setup := InSetupState}) -> #{suite := Suite, name := Name} = ct_daemon_core:from_qualified(Test), - Spec = test_runner:parse_test_name(Name, Suite), + Spec = common_util:parse_test_name(Name, Suite), ?LOG_INFO("discovered test ~tp with spec ~tp", [Name, Spec]), diff --git a/prelude/erlang/common_test/test_exec/src/ct_executor.erl b/prelude/erlang/common_test/test_exec/src/ct_executor.erl index 8faa606..1ca2f26 100644 --- a/prelude/erlang/common_test/test_exec/src/ct_executor.erl +++ b/prelude/erlang/common_test/test_exec/src/ct_executor.erl @@ -7,17 +7,20 @@ %% @format -module(ct_executor). --compile(warn_missing_spec_all). -moduledoc """ Simple trampoline for ct_run. Notably allows us to call post/pre method on the node if needed, e.g for coverage. """. +-compile(warn_missing_spec_all). + +-export([run/1]). + +% For testing +-export([preload_app_file_atoms/1, split_args/1]). -include_lib("kernel/include/logger.hrl"). -include_lib("common/include/buck_ct_records.hrl"). --export([run/1]). - %% `ct_run_arg()` represents an option accepted by ct:run_test/1, such as %% `multiply_timetraps` or `ct_hooks`. %% For all the options, see https://www.erlang.org/doc/man/ct#run_test-1 @@ -68,9 +71,6 @@ Notably allows us to call post/pre method on the node if needed, e.g for coverag | {providers, [{Name :: atom(), Args :: term()}]} | {common_app_env, #{binary() => binary()}}. -% For testing --export([preload_app_file_atoms/1, split_args/1]). - -define(raw_file_access, prim_file). -spec run([string()]) -> no_return(). diff --git a/prelude/erlang/common_test/test_exec/src/ct_executor_watchdog.erl b/prelude/erlang/common_test/test_exec/src/ct_executor_watchdog.erl index 652ba9b..25789de 100644 --- a/prelude/erlang/common_test/test_exec/src/ct_executor_watchdog.erl +++ b/prelude/erlang/common_test/test_exec/src/ct_executor_watchdog.erl @@ -18,7 +18,7 @@ this shutdown handled by it. """. -compile(warn_missing_spec_all). --include_lib("kernel/include/logger.hrl"). +-behaviour(gen_server). -export([start_link_client/1, start_server/0]). @@ -29,7 +29,7 @@ this shutdown handled by it. handle_cast/2 ]). --behaviour(gen_server). +-include_lib("kernel/include/logger.hrl"). -record(state, { socket :: gen_tcp:socket() | undefined diff --git a/prelude/erlang/common_test/test_exec/src/ct_runner.erl b/prelude/erlang/common_test/test_exec/src/ct_runner.erl index d8c164c..9a8f2b4 100644 --- a/prelude/erlang/common_test/test_exec/src/ct_runner.erl +++ b/prelude/erlang/common_test/test_exec/src/ct_runner.erl @@ -16,9 +16,6 @@ communicates the result to the test runner. -behavior(gen_server). -export([start_link/1]). --include_lib("common/include/buck_ct_records.hrl"). --include_lib("kernel/include/logger.hrl"). --define(raw_file_access, prim_file). -export([ init/1, @@ -37,6 +34,13 @@ communicates the result to the test runner. project_root/0 ]). +-export_type([port_settings/0]). + +-include_lib("common/include/buck_ct_records.hrl"). +-include_lib("kernel/include/logger.hrl"). + +-define(raw_file_access, prim_file). + -import(common_util, [unicode_characters_to_binary/1, filename_all_to_filename/1]). -type opt() :: @@ -62,8 +66,6 @@ communicates the result to the test runner. -type port_settings() :: [opt()]. --export_type([port_settings/0]). - -type initial_state() :: #{ test_env := #test_env{} }. @@ -114,7 +116,7 @@ handle_continue({run, PortEpmd}, #{test_env := TestEnv} = State0) -> erl_error:format_exception(Class, Reason, Stack) ]), ?LOG_ERROR(ErrorMsg), - test_runner:mark_failure(ErrorMsg, #{}), + report_result(TestEnv, run_failed, ErrorMsg, #{}), {stop, ct_runner_failed, State0} end. @@ -133,14 +135,14 @@ handle_continue({run, PortEpmd}, #{test_env := TestEnv} = State0) -> Port :: erlang:port(), Reason :: term(). -handle_info({Port, {exit_status, ExitStatus}}, #{port := Port} = State) -> +handle_info({Port, {exit_status, ExitStatus}}, #{port := Port, test_env := TestEnv} = State) -> CtStdoutState = maps:get(ct_stdout_state, State), {eof, ProgressMarkersOffsets} = ct_stdout:process_stdout_line(eof, CtStdoutState), case ExitStatus of 0 -> ResultMsg = "ct_runner finished successfully with exit status 0", ?LOG_DEBUG(ResultMsg), - test_runner:mark_success(ResultMsg, ProgressMarkersOffsets); + report_result(TestEnv, run_succeed, ResultMsg, ProgressMarkersOffsets); _ -> ErrorMsg = case ExitStatus of @@ -156,7 +158,7 @@ handle_info({Port, {exit_status, ExitStatus}}, #{port := Port} = State) -> ]) end, ?LOG_ERROR(ErrorMsg), - test_runner:mark_failure(ErrorMsg, ProgressMarkersOffsets) + report_result(TestEnv, run_failed, ErrorMsg, ProgressMarkersOffsets) end, {stop, {ct_run_finished, ExitStatus}, State}; handle_info({Port, {data, Data}}, State0 = #{port := Port}) -> @@ -185,6 +187,15 @@ terminate(_Reason, #{port := Port}) -> terminate(_Reason, _State) -> ok. +-spec report_result(TestEnv, Outcome, Result, ProgressMarkersOffsets) -> ok when + TestEnv :: #test_env{}, + Outcome :: run_succeed | run_failed, + Result :: unicode:chardata(), + ProgressMarkersOffsets :: #{ct_stdout:progress_line() => ct_stdout:offset()}. +report_result(#test_env{result_recipient = Recipient}, Outcome, Result, ProgressMarkersOffsets) -> + Recipient ! {Outcome, Result, ProgressMarkersOffsets}, + ok. + -doc """ Executes the test in a new node by launching ct_run. """. diff --git a/prelude/erlang/common_test/test_exec/src/epmd_manager.erl b/prelude/erlang/common_test/test_exec/src/epmd_manager.erl index 666e27d..bf7c50a 100644 --- a/prelude/erlang/common_test/test_exec/src/epmd_manager.erl +++ b/prelude/erlang/common_test/test_exec/src/epmd_manager.erl @@ -13,7 +13,7 @@ each suite execution. """. -compile(warn_missing_spec_all). --include_lib("common/include/buck_ct_records.hrl"). +-behaviour(gen_server). %% UI methods -export([start_link/1, get_epmd_out_path/1, get_port/0]). @@ -21,7 +21,7 @@ each suite execution. %% gen_server callbacks -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2]). --behaviour(gen_server). +-include_lib("common/include/buck_ct_records.hrl"). -define(raw_file_access, prim_file). diff --git a/prelude/erlang/common_test/test_exec/src/test_exec.erl b/prelude/erlang/common_test/test_exec/src/test_exec.erl index 73d1e0b..0b433f2 100644 --- a/prelude/erlang/common_test/test_exec/src/test_exec.erl +++ b/prelude/erlang/common_test/test_exec/src/test_exec.erl @@ -16,8 +16,9 @@ with the epmd daemon. -behavior(application). --include_lib("kernel/include/logger.hrl"). -export([start/2, stop/1, kill_process/1]). + +-include_lib("kernel/include/logger.hrl"). -include_lib("common/include/buck_ct_records.hrl"). -spec start(Type, Args) -> {'ok', pid()} | {'error', supervisor:startlink_err()} when diff --git a/prelude/erlang/common_test/test_exec/test/ct_daemon_capture_SUITE.erl b/prelude/erlang/common_test/test_exec/test/ct_daemon_capture_SUITE.erl index 62ea2ee..c88dae4 100644 --- a/prelude/erlang/common_test/test_exec/test/ct_daemon_capture_SUITE.erl +++ b/prelude/erlang/common_test/test_exec/test/ct_daemon_capture_SUITE.erl @@ -7,9 +7,6 @@ -module(ct_daemon_capture_SUITE). -typing([eqwalizer]). -% elp:ignore WA003 (better_assertions) - Open Source --include_lib("stdlib/include/assert.hrl"). - -export([all/0, init_per_testcase/2, end_per_testcase/2]). -export([ test_start_stop/1, @@ -21,6 +18,9 @@ test_unknown_request/1 ]). +% elp:ignore WA003 (better_assertions) - Open Source +-include_lib("stdlib/include/assert.hrl"). + all() -> [ test_start_stop, diff --git a/prelude/erlang/common_test/test_exec/test/ct_executor_SUITE.erl b/prelude/erlang/common_test/test_exec/test/ct_executor_SUITE.erl index 323dddd..54a38ce 100644 --- a/prelude/erlang/common_test/test_exec/test/ct_executor_SUITE.erl +++ b/prelude/erlang/common_test/test_exec/test/ct_executor_SUITE.erl @@ -7,9 +7,6 @@ -module(ct_executor_SUITE). -typing([eqwalizer]). -% elp:ignore WA003 (better_assertions) - Open Source --include_lib("stdlib/include/assert.hrl"). - -export([all/0]). -export([ @@ -17,6 +14,9 @@ test_split_args/1 ]). +% elp:ignore WA003 (better_assertions) - Open Source +-include_lib("stdlib/include/assert.hrl"). + all() -> [ test_split_args, diff --git a/prelude/erlang/elp.bxl b/prelude/erlang/elp.bxl index de4d3b6..001dbff 100644 --- a/prelude/erlang/elp.bxl +++ b/prelude/erlang/elp.bxl @@ -178,6 +178,10 @@ def _elp_config(ctx): if deps: deps_list = [_resolve_target_label(tgt.raw_target(), alias_map) for tgt in deps.value()] + test_binary_lib = attrs.get("_test_binary_lib") + if test_binary_lib: + deps_list.append(_resolve_target_label(test_binary_lib.value().raw_target(), alias_map)) + apps = attrs.get("applications") if apps: apps_list = [_resolve_target_label(tgt.raw_target(), alias_map) for tgt in apps.value()] @@ -186,6 +190,15 @@ def _elp_config(ctx): if included_apps: included_apps_list = [_resolve_target_label(tgt.raw_target(), alias_map) for tgt in included_apps.value()] + # These end up verbatim in the generated `.app` file, so unlike the + # attributes above they hold application names, not target labels. + extra_properties_map = {} + extra_properties = attrs.get("extra_properties") + if extra_properties: + # Absent on target types that do not declare the attribute at all, + # and `None` on those that declare it and leave it unset. + extra_properties_map = extra_properties.value() or {} + is_generated = "generated" in attrs.get("labels").value() gen_srcs = [] srcs_attr = attrs.get("srcs") @@ -223,6 +236,7 @@ def _elp_config(ctx): apps = apps_list, included_apps = included_apps_list, extra_includes = extra_includes_list, + extra_properties = extra_properties_map, origin = typ, ) diff --git a/prelude/erlang/erlang_application.bzl b/prelude/erlang/erlang_application.bzl index 2bcd79c..e3c407c 100644 --- a/prelude/erlang/erlang_application.bzl +++ b/prelude/erlang/erlang_application.bzl @@ -369,14 +369,14 @@ def _link_src_dir(ctx: AnalysisContext, *, extra_srcs: list[Artifact]) -> Artifa def _build_start_dependencies(ctx: AnalysisContext) -> list[StartDependencySet]: return build_apps_start_dependencies( - ctx, + ctx.actions, [(app, StartType("permanent")) for app in ctx.attrs.applications], ) + build_apps_start_dependencies( - ctx, + ctx.actions, [(app, StartType("load")) for app in ctx.attrs.included_applications], ) -def build_apps_start_dependencies(ctx: AnalysisContext, apps: list[(Dependency, StartType)]) -> list[StartDependencySet]: +def build_apps_start_dependencies(actions: AnalysisActions, apps: list[(Dependency, StartType)]) -> list[StartDependencySet]: start_dependencies = [] for app, start_type in apps[::-1]: app_spec = _build_start_spec(app[ErlangAppInfo], start_type) @@ -386,7 +386,7 @@ def build_apps_start_dependencies(ctx: AnalysisContext, apps: list[(Dependency, else: children = app[ErlangAppInfo].start_dependencies - app_set = ctx.actions.tset( + app_set = actions.tset( StartDependencySet, value = app_spec, children = children, diff --git a/prelude/erlang/erlang_error_handler.bzl b/prelude/erlang/erlang_error_handler.bzl index c3f240b..9e79726 100644 --- a/prelude/erlang/erlang_error_handler.bzl +++ b/prelude/erlang/erlang_error_handler.bzl @@ -57,20 +57,13 @@ def erlang_extract_otp_app_error_handler(ctx: ActionErrorCtx) -> list[ActionSubE apps_list = "\n".join([" - {}".format(app) for app in available_apps]) available_apps_section = "\n\n Available OTP applications ({count}):\n{apps_list}".format(count = len(available_apps), apps_list = apps_list) - remediation = """To fix: - - 1. Verify '{app_name}' exists in your OTP installation: + remediation = """To fix, verify '{app_name}' exists in your OTP installation: $ ls -Al "{root_dir}/lib/{app_name}-*" - 2. Regenerate your toolchain's otp_versions.bzl file: - - $ python3 buck2/prelude/erlang/toolchain/generate_otp_versions.py my_otp_versions.bzl - Common causes: - The application is not part of your OTP version (e.g., jinterface was removed in OTP 27) - - You're using a toolchain configured for a different OTP version - - Your toolchain's 'applications' list needs updating{available_apps_section}""".format( + - You're using a toolchain configured for a different OTP version{available_apps_section}""".format( app_name = app_name, root_dir = error_json["root_dir"], available_apps_section = available_apps_section, @@ -120,5 +113,4 @@ def erlang_extract_otp_app_error_handler(ctx: ActionErrorCtx) -> list[ActionSubE erlang_action_error_handler = ErlangErrorHandlers( erlc = erlang_erlc_action_error_handler, - extract_otp_app = erlang_extract_otp_app_error_handler, ) diff --git a/prelude/erlang/erlang_erts.bzl b/prelude/erlang/erlang_erts.bzl new file mode 100644 index 0000000..d935420 --- /dev/null +++ b/prelude/erlang/erlang_erts.bzl @@ -0,0 +1,140 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is dual-licensed under either the MIT license found in the +# LICENSE-MIT file in the root directory of this source tree or the Apache +# License, Version 2.0 found in the LICENSE-APACHE file in the root directory +# of this source tree. You may select, at your option, one of the +# above-listed licenses. + +load("@prelude//:paths.bzl", "paths") +load( + ":erlang_error_handler.bzl", + "erlang_extract_otp_app_error_handler", +) +load( + ":erlang_info.bzl", + "ErlangOTPBinariesInfo", + "ErtsToolchainInfo", + "Tool", +) +load( + ":erlang_otp_application.bzl", + "otp_applications", +) +load( + ":erlang_paths.bzl", + "strip_extension", +) +load( + ":erlang_toolchain.bzl", + "default_toolchain_script_args_post", + "default_toolchain_script_args_pre", +) + +# The extraction runs `erl`, so its exec platform has to match the target platform or the +# artifacts come out for the wrong one. There is no DEFAULT: a target platform that is not +# listed here fails to configure rather than silently producing a foreign-architecture release. +_DEFAULT_EXEC_COMPATIBLE_WITH = select({ + "config//os/constraints:os[unspecified]": [], + "config//os:linux": select({ + "config//cpu:arm64": [ + "config//cpu/constraints:arm64", + "config//os/constraints:linux", + ], + "config//cpu:x86_64": [ + "config//cpu/constraints:x86_64", + "config//os/constraints:linux", + ], + }), + "config//os:macos": select({ + "config//cpu:arm64": [ + "config//cpu/constraints:arm64", + "config//os/constraints:macos", + ], + "config//cpu:x86_64": [ + "config//cpu/constraints:x86_64", + "config//os/constraints:macos", + ], + }), + "config//os:windows": select({ + "config//cpu:arm64": [ + "config//cpu/constraints:arm64", + "config//os/constraints:windows", + ], + "config//cpu:x86_64": [ + "config//cpu/constraints:x86_64", + "config//os/constraints:windows", + ], + }), +}) + +def _extractor(ctx: AnalysisContext, binaries: ErlangOTPBinariesInfo, env: dict[str, str]) -> Tool: + src = ctx.attrs._extract_from_otp + name = strip_extension(src.basename) + beam = ctx.actions.declare_output(name, name + ".beam", has_content_based_path = False) + erlc = cmd_args(binaries.erlc, hidden = binaries.erl) + ctx.actions.run( + cmd_args(erlc, "+deterministic", "-o", cmd_args(beam.as_output(), parent = 1), src), + category = "erlc", + identifier = src.short_path, + env = env, + ) + return cmd_args( + binaries.erl, + cmd_args(beam, parent = 1, prepend = "-pa"), + default_toolchain_script_args_pre, + cmd_args(name, ":main(init:get_plain_arguments())", delimiter = ""), + default_toolchain_script_args_post, + ) + +def _erlang_erts_impl(ctx: AnalysisContext) -> list[Provider]: + binaries = ctx.attrs.otp_binaries[ErlangOTPBinariesInfo] + env = ctx.attrs.env + extractor = _extractor(ctx, binaries, env) + + def extract(name: str, args: list[str], category: str, dir: bool = True) -> Artifact: + out = ctx.actions.declare_output(name, dir = dir, has_content_based_path = False) + ctx.actions.run( + cmd_args(extractor, args, out.as_output()), + category = category, + identifier = ctx.attrs.name, + env = env, + ) + return out + + # one action per application, so a release only ever extracts what its own closure names + applications = {} + for application in otp_applications: + out = ctx.actions.declare_output(paths.join("lib", application), dir = True, has_content_based_path = False) + ctx.actions.run( + cmd_args(extractor, paths.join("lib", application + "-*"), out.as_output()), + category = "extract_otp_app", + identifier = application, + env = env, + error_handler = erlang_extract_otp_app_error_handler, + ) + applications[application] = out + + return [ + DefaultInfo(), + ErtsToolchainInfo( + applications = applications, + erts = extract("erts", ["extract_into", "erts-*"], "extract_erts"), + headers = extract("erts_headers", ["erts-*/include"], "extract_erts_headers"), + otp_no_dot_erlang_boot = extract("otp_no_dot_erlang_boot", ["bin/no_dot_erlang.boot"], "extract_otp_boot", False), + otp_start_boot = extract("otp_start_boot", ["bin/start.boot"], "extract_otp_start_boot", False), + versions = extract("versions.json", ["versions"], "introspect_otp_versions", False), + ), + ] + +_erlang_erts_rule = rule( + impl = _erlang_erts_impl, + attrs = { + "env": attrs.dict(key = attrs.string(), value = attrs.string(), default = {}), + "otp_binaries": attrs.toolchain_dep(), + "_extract_from_otp": attrs.default_only(attrs.source(default = "prelude//erlang/erts:extract_from_otp.erl")), + }, +) + +def erlang_erts(exec_compatible_with = _DEFAULT_EXEC_COMPATIBLE_WITH, **kwargs): + _erlang_erts_rule(exec_compatible_with = exec_compatible_with, **kwargs) diff --git a/prelude/erlang/erlang_headers.bzl b/prelude/erlang/erlang_headers.bzl index 4abc88a..d03ecc9 100644 --- a/prelude/erlang/erlang_headers.bzl +++ b/prelude/erlang/erlang_headers.bzl @@ -9,10 +9,12 @@ load(":erlang_toolchain.bzl", "get_toolchain") def _erlang_headers(ctx: AnalysisContext) -> list[Provider]: - headers = get_toolchain(ctx).erts_toolchain_info.headers + erts_toolchain_info = get_toolchain(ctx).erts_toolchain_info + if erts_toolchain_info == None: + fail("%s exposes the ERTS headers, but its toolchain has no `erts_toolchain_info` to take them from" % (str(ctx.label),)) return [ DefaultInfo( - default_outputs = [headers], + default_outputs = [erts_toolchain_info.headers], ), ] diff --git a/prelude/erlang/erlang_info.bzl b/prelude/erlang/erlang_info.bzl index 96b8073..17c3a19 100644 --- a/prelude/erlang/erlang_info.bzl +++ b/prelude/erlang/erlang_info.bzl @@ -137,27 +137,22 @@ Tools = record( escript = field(Tool), ) -ErtsToolchainApplicationInfo = provider( - fields = { - "name": provider_field(str), - "version": provider_field(str), - }, -) - ErtsToolchainInfo = provider( fields = { - "applications": provider_field(list[ErtsToolchainApplicationInfo]), - "erts_version": provider_field(str), + # the directory each OTP application ships as, by application name + "applications": provider_field(dict[str, Artifact]), + # directory holding `erts-` + "erts": provider_field(Artifact), "headers": provider_field(Artifact), "otp_no_dot_erlang_boot": provider_field(Artifact), "otp_start_boot": provider_field(Artifact), - "output": provider_field(Artifact), + # json, `{"erts_version": ..., "applications": {: }}` + "versions": provider_field(Artifact), }, ) ErlangErrorHandlers = record( erlc = field(typing.Callable[[ActionErrorCtx], list[ActionSubError]]), - extract_otp_app = field(typing.Callable[[ActionErrorCtx], list[ActionSubError]]), ) # toolchain provider @@ -194,14 +189,12 @@ ErlangToolchainInfo = provider( "boot_script_builder": provider_field(Tool), # build release_variables "release_variables_builder": provider_field(Tool), - # copying erts - "extract_from_otp": provider_field(Tool), # beams we need for various reasons "utility_modules": provider_field(Artifact), # env to be set for toolchain invocations "env": provider_field(dict[str, str]), # Erlang Runtime System (ERTS) toolchain metadata - "erts_toolchain_info": provider_field(ErtsToolchainInfo), + "erts_toolchain_info": provider_field(ErtsToolchainInfo | None, default = None), # error handler "error_handler": provider_field(ErlangErrorHandlers), }, diff --git a/prelude/erlang/erlang_otp_application.bzl b/prelude/erlang/erlang_otp_application.bzl index b72e36c..5a5c428 100644 --- a/prelude/erlang/erlang_otp_application.bzl +++ b/prelude/erlang/erlang_otp_application.bzl @@ -6,11 +6,6 @@ # of this source tree. You may select, at your option, one of the # above-listed licenses. -load("@prelude//:paths.bzl", "paths") -load( - ":erlang_build.bzl", - "erlang_build", -) load(":erlang_info.bzl", "ErlangAppInfo", "ErlangAppOrTestInfo") load( ":erlang_toolchain.bzl", @@ -82,42 +77,12 @@ def _erlang_otp_application_impl(ctx: AnalysisContext) -> list[Provider]: toolchain = get_toolchain(ctx) - # Look up version from toolchain applications - version = None - for app_info in toolchain.erts_toolchain_info.applications: - if app_info.name == ctx.attrs.name: - version = app_info.version - break - - # Support dynamic mode (for include_erts=False) and explicit mode (for include_erts=True) - if version == None: - # Application not in toolchain's explicit list - # Use wildcard to attempt discovery at build time - # This allows the analysis phase to succeed even if the app doesn't exist - # The actual error will happen during extraction if the app is truly missing - wildcard = paths.join("lib", ctx.attrs.name + "-*") - app_dir = ctx.actions.declare_output(ctx.attrs.name, dir = True, has_content_based_path = False) - version = "dynamic" - else: - # Explicit version found - use versioned directory - wildcard = paths.join("lib", ctx.attrs.name + "-" + version) - app_dir = ctx.actions.declare_output(ctx.attrs.name + "-" + version, dir = True, has_content_based_path = False) - - erlang_build.utils.run_with_env( - ctx, - toolchain, - cmd_args(toolchain.extract_from_otp, wildcard, app_dir.as_output()), - identifier = ctx.attrs.name, - category = "extract_otp_app", - error_handler = toolchain.error_handler.extract_otp_app, - ) - return [ DefaultInfo(), ErlangAppOrTestInfo(), ErlangAppInfo( name = ctx.attrs.name, - version = version, + version = "dynamic", beams = [], includes = [], dependencies = {}, @@ -125,7 +90,7 @@ def _erlang_otp_application_impl(ctx: AnalysisContext) -> list[Provider]: start_dependencies = None, include_dir = None, virtual = True, - app_folder = app_dir, + app_folder = toolchain.erts_toolchain_info.applications.get(ctx.attrs.name) if toolchain.erts_toolchain_info else None, ), ] diff --git a/prelude/erlang/erlang_release.bzl b/prelude/erlang/erlang_release.bzl index 3eb746c..8bfe69a 100644 --- a/prelude/erlang/erlang_release.bzl +++ b/prelude/erlang/erlang_release.bzl @@ -7,6 +7,7 @@ # above-listed licenses. load("@prelude//:paths.bzl", "paths") +load("@prelude//utils:arglike.bzl", "ArgLike") # @unused Used as type load( ":erlang_application.bzl", "StartDependencySet", @@ -29,95 +30,283 @@ load( # Erlang Releases according to https://www.erlang.org/doc/design_principles/release_structure.html +Release = record( + dir = Artifact, + # `bin/` on its own, so a release that installs another's launcher does not take its tree + launcher = field(Artifact | None, None), + # what the release is assembled from that analysis already knows, by path from the release root + entries = dict[str, Artifact], +) + +LauncherLines = record( + name = str, + head = list[str], + tail = list[str], +) + +ReleaseConfig = record( + # the target the release is built for, only used to point failures at it + label = Label, + name = str, + version = str, + # the applications the release starts, in order, with the type each is started with, and + # together with their transitive dependencies what `lib/` is drawn from + applications = list[(Dependency, StartType)], + toolchain = Toolchain, + # the environment every toolchain invocation runs with, `None` for the toolchain's own + os_env = field(dict[str, str] | None, None), + include_erts = bool, + is_executable = bool, + generate_default_bootscript = bool, + default_bootscript_name = str, + bootscript_builders = dict[str, cmd_args], + extra_bootscript_builder_args = list[ArgLike], + # the config files the launcher hands the emulator, in the order they are applied, so a later + # one overrides an earlier one. Each is a path from the release root without the `.config` + # extension, the form `-config` names a file with, and has to be part of the release. + config_paths = list[str], + # artifacts to install, mapping the directory they go into, from the release root, to their contents + overlays = dict[str, list[Artifact]], + # artifacts to install, by their path from the release root, that the release does not build itself + extra_entries = field(dict[str, Artifact], {}), +) + def erlang_release_impl(ctx: AnalysisContext) -> list[Provider]: - apps = flatten_dependencies(_dependencies(ctx)) + config = _release_config(ctx) - all_outputs = _build_release(ctx, apps) - release_dir = _symlink_primary_toolchain_output(ctx, all_outputs) - return [DefaultInfo(default_output = release_dir), ErlangReleaseInfo(name = _relname(ctx))] + release = build_release(ctx.actions, config) + providers = [DefaultInfo(default_output = release.dir), ErlangReleaseInfo(name = config.name)] -def _build_release(ctx: AnalysisContext, apps: ErlAppDependencies) -> dict[str, Artifact]: + if config.is_executable: + # the launcher reaches the rest of the release through relative symlinks, so running it + # needs the whole tree materialised, not just `bin/` + launcher = release.dir.project(_launcher_path(config)).with_associated_artifacts([release.dir]) + providers.append(RunInfo(cmd_args(launcher))) + + return providers + +def _release_config(ctx: AnalysisContext) -> ReleaseConfig: + applications = _applications(ctx) toolchain = get_toolchain(ctx) + overlays = { + target: [artifact for dep in deps for artifact in dep[DefaultInfo].default_outputs + dep[DefaultInfo].other_outputs] + for target, deps in ctx.attrs.overlays.items() + } + + # an `erlang_release` is configured with the one `sys.config` the OTP layout gives it + sys_config = paths.join("releases", ctx.attrs.version, "sys") + config_paths = [sys_config] if sys_config + ".config" in _overlay_paths(overlays) else [] + + return ReleaseConfig( + label = ctx.label, + name = ctx.attrs.release_name if ctx.attrs.release_name else ctx.attrs.name, + version = ctx.attrs.version, + applications = applications, + toolchain = toolchain, + os_env = getattr(ctx.attrs, "os_env", None), + include_erts = ctx.attrs.include_erts, + is_executable = ctx.attrs.is_executable, + generate_default_bootscript = ctx.attrs.generate_default_bootscript, + default_bootscript_name = ctx.attrs.default_bootscript_name, + bootscript_builders = {script_name: builder[RunInfo].args for script_name, builder in ctx.attrs.bootscript_builders.items()}, + extra_bootscript_builder_args = ctx.attrs.extra_bootscript_builder_args, + config_paths = config_paths, + overlays = overlays, + ) - # Validate include_erts configuration - _validate_include_erts(ctx, toolchain) +def _applications(ctx: AnalysisContext) -> list[(Dependency, StartType)]: + """Extract the applications, with their start type, from the `applications` field, order preserving""" + applications = [] + for dep in ctx.attrs.applications: + if type(dep) == "tuple": + applications.append((dep[0], StartType(dep[1]))) + else: + applications.append((dep, StartType("permanent"))) + return applications - # OTP base structure - lib_dir = build_lib_dir(ctx, apps) +def build_release(actions: AnalysisActions, config: ReleaseConfig) -> Release: + """Build an OTP release, returning the release root. - # erts - maybe_erts = _build_erts(ctx, toolchain) + The versioned parts of the layout, `erts-` and `lib/-`, are laid + out by a dynamic action: the versions come from the toolchain's OTP rather than from analysis. + """ + _validate_include_erts(config) + _validate_is_executable(config) - maybe_boot_scripts = _build_boot_scripts(ctx, toolchain, lib_dir["lib"]) + all_apps = flatten_dependencies([app for app, _ in config.applications]) + own_apps = _own_applications(all_apps) + otp_apps = _otp_applications(config, all_apps) if config.include_erts else {} + erts_toolchain_info = config.toolchain.erts_toolchain_info - # start_erl.data for releases with bundled ERTS - maybe_start_erl_data = _build_start_erl_data(ctx, toolchain) + if erts_toolchain_info == None: + lib_dir = actions.symlinked_dir(paths.join(erlang_build.utils.BUILD_DIR, "lib"), own_apps, has_content_based_path = False) + else: + lib_dir = actions.declare_output(paths.join(erlang_build.utils.BUILD_DIR, "lib"), dir = True, has_content_based_path = False) + actions.dynamic_output_new( + _assemble_lib_dir( + otp_apps = otp_apps, + out = lib_dir.as_output(), + own_apps = own_apps, + versions = erts_toolchain_info.versions, + ) + ) - # release specific variables in bin/release_variables - release_variables = _build_release_variables(ctx, toolchain) + entries = {"lib": lib_dir} + entries.update(_build_boot_scripts(actions, config, lib_dir)) + entries.update(_build_overlays(config.overlays)) + entries.update(_build_release_variables(actions, config)) - # Overlays - overlays = _build_overlays(ctx) + for entry, artifact in config.extra_entries.items(): + if entry in entries: + fail("%s is given `%s` to install, which its own release builds" % (str(config.label), entry)) + entries[entry] = artifact - # link output - all_outputs = {} - for outputs in [ - lib_dir, - maybe_boot_scripts, - maybe_start_erl_data, - overlays, - release_variables, - maybe_erts, - ]: - all_outputs.update(outputs) + launcher = _build_launcher(config, entries) + if launcher != None and _launcher_path(config) in entries: + fail("the launcher of %s is installed at %s, which the release already contains" % (str(config.label), _launcher_path(config))) + + launcher_file = None + if launcher != None: + launcher_file = actions.declare_output(paths.join(erlang_build.utils.BUILD_DIR, "launcher", launcher.name), has_content_based_path = False) + + if erts_toolchain_info == None: + release_dir = actions.symlinked_dir(config.name, entries, has_content_based_path = False) + else: + release_dir = actions.declare_output(config.name, dir = True, has_content_based_path = False) + actions.dynamic_output_new( + _assemble_release( + entries = entries, + include_erts = config.include_erts, + launcher = launcher, + launcher_out = launcher_file.as_output() if launcher_file != None else None, + otp_erts = erts_toolchain_info.erts, + out = release_dir.as_output(), + version = config.version, + versions = erts_toolchain_info.versions, + ), + ) + return Release(dir = release_dir, entries = entries, launcher = launcher_file) + +def _assemble_lib_dir_impl( + actions: AnalysisActions, otp_apps: dict[str, Artifact], own_apps: dict[str, Artifact], versions: ArtifactValue, out: OutputArtifact +) -> list[Provider]: + srcs = dict(own_apps) + if otp_apps: + app_versions = versions.read_json()["applications"] + for app, app_folder in otp_apps.items(): + if app not in app_versions: + fail("the toolchain's OTP does not contain the application `%s`" % (app,)) + srcs["{}-{}".format(app, app_versions[app])] = app_folder + actions.symlinked_dir(out, srcs) + return [] + +_assemble_lib_dir = dynamic_actions( + impl = _assemble_lib_dir_impl, + attrs = { + "otp_apps": dynattrs.value(dict[str, Artifact]), + "out": dynattrs.output(), + "own_apps": dynattrs.value(dict[str, Artifact]), + "versions": dynattrs.artifact_value(), + }, +) - return all_outputs +def _assemble_release_impl( + actions: AnalysisActions, + entries: dict[str, Artifact], + include_erts: bool, + launcher: LauncherLines | None, + launcher_out: OutputArtifact | None, + otp_erts: Artifact, + version: str, + versions: ArtifactValue, + out: OutputArtifact, +) -> list[Provider]: + otp = versions.read_json() + erts_dir = "erts-{}".format(otp["erts_version"]) + + srcs = dict(entries) + if include_erts: + srcs[erts_dir] = otp_erts.project(erts_dir) + + start_erl_data = actions.declare_output("start_erl.data", has_content_based_path = False) + actions.write(start_erl_data, "{} {}\n".format(otp["erts_version"], version)) + srcs[paths.join("releases", "start_erl.data")] = start_erl_data + + if launcher != None: + lines = launcher.head + ['BINDIR="$ROOTDIR/{}/bin"'.format(erts_dir)] + launcher.tail + srcs[paths.join("bin", launcher.name)] = actions.write(launcher_out, lines, is_executable = True) + + actions.symlinked_dir(out, srcs) + return [] + +_assemble_release = dynamic_actions( + impl = _assemble_release_impl, + attrs = { + "entries": dynattrs.value(dict[str, Artifact]), + "include_erts": dynattrs.value(bool), + "launcher": dynattrs.value(LauncherLines | None), + "launcher_out": dynattrs.option(dynattrs.output()), + "otp_erts": dynattrs.value(Artifact), + "out": dynattrs.output(), + "version": dynattrs.value(str), + "versions": dynattrs.artifact_value(), + }, +) def build_lib_dir(ctx: AnalysisContext, all_apps: ErlAppDependencies) -> dict[str, Artifact]: """Build lib dir according to OTP specifications. .. seealso:: `OTP Design Principles Release Structure `_ """ - include_erts = False - if "include_erts" in dir(ctx.attrs): - include_erts = ctx.attrs.include_erts - - link_spec = { - (dep[ErlangAppInfo].name + "-" + dep[ErlangAppInfo].version): dep[ErlangAppInfo].app_folder - for dep in all_apps.values() - if ErlangAppInfo in dep and (include_erts or not dep[ErlangAppInfo].virtual) - } - lib_dir = ctx.actions.symlinked_dir( paths.join(erlang_build.utils.BUILD_DIR, "lib"), - link_spec, + _own_applications(all_apps), has_content_based_path = False, ) return {"lib": lib_dir} -def _build_boot_scripts(ctx: AnalysisContext, toolchain: Toolchain, lib_dir: Artifact) -> dict[str, Artifact]: +def _otp_applications(config: ReleaseConfig, all_apps: ErlAppDependencies) -> dict[str, Artifact]: + applications = { + "erts": config.toolchain.erts_toolchain_info.applications["erts"], + } + for dep in all_apps.values(): + if ErlangAppInfo not in dep or not dep[ErlangAppInfo].virtual: + continue + app_info = dep[ErlangAppInfo] + if app_info.app_folder == None: + fail("%s needs the OTP application `%s`, which the toolchain's OTP does not ship" % (str(config.label), app_info.name)) + applications[app_info.name] = app_info.app_folder + return applications + +def _own_applications(all_apps: ErlAppDependencies) -> dict[str, Artifact]: + return { + (dep[ErlangAppInfo].name + "-" + dep[ErlangAppInfo].version): dep[ErlangAppInfo].app_folder + for dep in all_apps.values() + if ErlangAppInfo in dep and not dep[ErlangAppInfo].virtual + } + +def _build_boot_scripts(actions: AnalysisActions, config: ReleaseConfig, lib_dir: Artifact) -> dict[str, Artifact]: link_spec = {} - if ctx.attrs.generate_default_bootscript: - maybe_default_boot_script = _build_default_boot_scripts(ctx, toolchain, lib_dir) + if config.generate_default_bootscript: + maybe_default_boot_script = _build_default_boot_scripts(actions, config, lib_dir) link_spec.update(maybe_default_boot_script) # write applications spec to file - data = [_app_info_to_data(app_info) for app_info in ctx.attrs.applications] - spec_file = ctx.actions.write_json( + data = [(app[ErlangAppInfo].name, start_type.value) for app, start_type in config.applications] + spec_file = actions.write_json( paths.join(erlang_build.utils.BUILD_DIR, "bootscripts", "applications_json"), data, has_content_based_path = False, ) - for script_name, builder in ctx.attrs.bootscript_builders.items(): - builder_args = builder[RunInfo].args - custom_boot_script_spec = _build_custom_boot_scripts(ctx, toolchain, spec_file, script_name, builder_args, lib_dir) + for script_name, builder in config.bootscript_builders.items(): + custom_boot_script_spec = _build_custom_boot_scripts(actions, config, spec_file, script_name, builder, lib_dir) link_spec.update(custom_boot_script_spec) return link_spec -def _build_default_boot_scripts(ctx: AnalysisContext, toolchain: Toolchain, lib_dir: Artifact) -> dict[str, Artifact]: +def _build_default_boot_scripts(actions: AnalysisActions, config: ReleaseConfig, lib_dir: Artifact) -> dict[str, Artifact]: """Build Name.rel, start.script, and start.boot in the release folder. Boot scripts are always generated regardless of include_erts setting. @@ -125,20 +314,16 @@ def _build_default_boot_scripts(ctx: AnalysisContext, toolchain: Toolchain, lib_ When include_erts=True, explicit versions from the toolchain are used and additional no_dot_erlang boot scripts are generated for the self-contained release. """ - release_name = _relname(ctx) + release_name = config.name - start_type_mapping = _dependencies_with_start_types(ctx) - root_apps = _dependencies(ctx) - root_apps_names = [app[ErlangAppInfo].name for app in root_apps] + root_apps_names = [app[ErlangAppInfo].name for app, _ in config.applications] + start_dependencies = build_apps_start_dependencies(actions, config.applications) - root_apps_with_start_type = [(app, start_type_mapping[_app_name(app)]) for app in root_apps] - start_dependencies = build_apps_start_dependencies(ctx, root_apps_with_start_type) - - root_set = ctx.actions.tset( + root_set = actions.tset( StartDependencySet, value = StartSpec( name = "__ignored__", - version = ctx.attrs.version, + version = config.version, start_type = StartType("permanent"), resolved = False, ), @@ -174,17 +359,17 @@ def _build_default_boot_scripts(ctx: AnalysisContext, toolchain: Toolchain, lib_ "apps": release_applications[::-1], "lib_dir": lib_dir, "name": release_name, - "version": ctx.attrs.version, + "version": config.version, } - spec_file = ctx.actions.write_json(paths.join(erlang_build.utils.BUILD_DIR, "boot_script_spec.json"), data, with_inputs = True, has_content_based_path = False) + spec_file = actions.write_json(paths.join(erlang_build.utils.BUILD_DIR, "boot_script_spec.json"), data, with_inputs = True, has_content_based_path = False) - scripts_dir = ctx.actions.declare_output(erlang_build.utils.BUILD_DIR, "scripts", dir = True, has_content_based_path = False) + scripts_dir = actions.declare_output(erlang_build.utils.BUILD_DIR, "scripts", dir = True, has_content_based_path = False) - erlang_build.utils.run_with_env( - ctx, - toolchain, - cmd_args(toolchain.boot_script_builder, spec_file, scripts_dir.as_output()), + _run_with_env( + actions, + config, + cmd_args(config.toolchain.boot_script_builder, spec_file, scripts_dir.as_output()), category = "build_boot_script", identifier = release_name, ) @@ -197,13 +382,13 @@ def _build_default_boot_scripts(ctx: AnalysisContext, toolchain: Toolchain, lib_ ] # Only include no_dot_erlang boot scripts for self-contained releases with bundled ERTS - if ctx.attrs.include_erts: + if config.include_erts: boot_files.extend([ "no_dot_erlang.script", "no_dot_erlang.boot", ]) - result = {paths.join("releases", ctx.attrs.version, file): scripts_dir.project(file) for file in boot_files} + result = {paths.join("releases", config.version, file): scripts_dir.project(file) for file in boot_files} # Place OTP's boot files in bin/ so erl can find them at ROOTDIR/bin/. # When erl runs from bundled ERTS (erts-VSN/bin/erl), it resolves ROOTDIR @@ -213,214 +398,173 @@ def _build_default_boot_scripts(ctx: AnalysisContext, toolchain: Toolchain, lib_ # - `erl` bare gives a clean shell (uses bin/start.boot) # - `erl -boot no_dot_erlang` works for ectl and other tools # mini_start explicitly uses releases/VERSION/start.boot for service startup. - if ctx.attrs.include_erts: - result[paths.join("bin", "start.boot")] = toolchain.erts_toolchain_info.otp_start_boot - result[paths.join("bin", "no_dot_erlang.boot")] = toolchain.erts_toolchain_info.otp_no_dot_erlang_boot + if config.include_erts: + result[paths.join("bin", "start.boot")] = config.toolchain.erts_toolchain_info.otp_start_boot + result[paths.join("bin", "no_dot_erlang.boot")] = config.toolchain.erts_toolchain_info.otp_no_dot_erlang_boot return result def _build_custom_boot_scripts( - ctx: AnalysisContext, toolchain: Toolchain, spec_file: Artifact, script_name: str, builder: cmd_args, lib_dir: Artifact + actions: AnalysisActions, config: ReleaseConfig, spec_file: Artifact, script_name: str, builder: cmd_args, lib_dir: Artifact ) -> dict[str, Artifact]: - boot_script = ctx.actions.declare_output(paths.join(erlang_build.utils.BUILD_DIR, "bootscripts", script_name), has_content_based_path = False) + boot_script = actions.declare_output(paths.join(erlang_build.utils.BUILD_DIR, "bootscripts", script_name), has_content_based_path = False) raw_script_name = paths.replace_extension(script_name, ".script") - raw_script = ctx.actions.declare_output(paths.join(erlang_build.utils.BUILD_DIR, "bootscripts", raw_script_name), has_content_based_path = False) - - erlang_build.utils.run_with_env( - ctx, - toolchain, - cmd_args(builder, spec_file, lib_dir, boot_script.as_output(), raw_script.as_output()), + raw_script = actions.declare_output(paths.join(erlang_build.utils.BUILD_DIR, "bootscripts", raw_script_name), has_content_based_path = False) + + _run_with_env( + actions, + config, + cmd_args( + builder, + spec_file, + lib_dir, + boot_script.as_output(), + raw_script.as_output(), + config.extra_bootscript_builder_args, + ), category = "build_custom_boot_script", identifier = script_name, ) return { - paths.join("releases", ctx.attrs.version, script_name): boot_script, - paths.join("releases", ctx.attrs.version, raw_script_name): raw_script, + paths.join("releases", config.version, script_name): boot_script, + paths.join("releases", config.version, raw_script_name): raw_script, } -def _app_info_to_data(app_info: Dependency | (Dependency, str)) -> (str, str): - if type(app_info) == "tuple": - app_info, start_type = app_info - else: - start_type = "permanent" - - erlang_app = app_info[ErlangAppInfo] - return (erlang_app.name, start_type) - -def _build_overlays(ctx: AnalysisContext) -> dict[str, Artifact]: +def _build_overlays(overlays: dict[str, list[Artifact]]) -> dict[str, Artifact]: installed = {} - for target, deps in ctx.attrs.overlays.items(): - for dep in deps: - for artifact in dep[DefaultInfo].default_outputs + dep[DefaultInfo].other_outputs: - link_path = paths.normalize(paths.join(target, artifact.basename)) - if link_path in installed: - fail("multiple overlays defined for the same location: %s" % (link_path,)) - installed[link_path] = artifact + for target, artifacts in overlays.items(): + for artifact in artifacts: + link_path = _overlay_path(target, artifact) + if link_path in installed: + fail("multiple overlays defined for the same location: %s" % (link_path,)) + installed[link_path] = artifact return installed -def _build_release_variables(ctx: AnalysisContext, toolchain: Toolchain) -> dict[str, Artifact]: - release_name = _relname(ctx) +def _overlay_paths(overlays: dict[str, list[Artifact]]) -> list[str]: + return [_overlay_path(target, artifact) for target, artifacts in overlays.items() for artifact in artifacts] + +def _overlay_path(target: str, artifact: Artifact) -> str: + return paths.normalize(paths.join(target, artifact.basename)) + +def _build_release_variables(actions: AnalysisActions, config: ReleaseConfig) -> dict[str, Artifact]: + release_name = config.name short_path = "bin/release_variables" - release_variables = ctx.actions.declare_output( + release_variables = actions.declare_output( erlang_build.utils.BUILD_DIR, "release_variables", has_content_based_path = False, ) - spec_file = ctx.actions.write_json( + spec_file = actions.write_json( paths.join(erlang_build.utils.BUILD_DIR, "relvars.json"), { "REL_NAME": release_name, - "REL_VSN": ctx.attrs.version, + "REL_VSN": config.version, }, has_content_based_path = False, ) - erlang_build.utils.run_with_env( - ctx, - toolchain, - cmd_args(toolchain.release_variables_builder, spec_file, release_variables.as_output()), + _run_with_env( + actions, + config, + cmd_args(config.toolchain.release_variables_builder, spec_file, release_variables.as_output()), category = "build_release_variables", identifier = release_name, ) return {short_path: release_variables} -def _build_erts(ctx: AnalysisContext, toolchain: Toolchain) -> dict[str, Artifact]: - if not ctx.attrs.include_erts: - return {} +def _build_launcher(config: ReleaseConfig, release_files: dict[str, Artifact]) -> LauncherLines | None: + """Generate bin/, a launcher booting the release with the bundled emulator. - release_name = _relname(ctx) - - erts_dir = ctx.actions.symlink_file( - paths.join( - erlang_build.utils.BUILD_DIR, - release_name, - "erts-{}".format(toolchain.erts_toolchain_info.erts_version), - ), - toolchain.erts_toolchain_info.output, - has_content_based_path = False, - ) - - return {"erts-{}".format(toolchain.erts_toolchain_info.erts_version): erts_dir} - -def _build_start_erl_data(ctx: AnalysisContext, toolchain: Toolchain) -> dict[str, Artifact]: - """Generate start_erl.data file for releases with bundled ERTS. - - This file contains the ERTS version and release version, - used by the release boot scripts to determine which ERTS and - release to start. - - Format: - Example: 15.1 1.0.0 + Everything the emulator is told is resolved here rather than at runtime: the erts version, so + the launcher addresses `erts-` directly, and the boot script, `vm.args` and the config + files, so the launcher does not depend on what it was invoked as. Every path is relative to + ROOTDIR, so the release stays relocatable. The tool name is still taken from the launcher's own + basename, so one release can serve several tools that differ only in the arguments they get. """ - if not ctx.attrs.include_erts: - return {} - - content = "{} {}\n".format( - toolchain.erts_toolchain_info.erts_version, - ctx.attrs.version, - ) - - start_erl_data = ctx.actions.write( - paths.join(erlang_build.utils.BUILD_DIR, "start_erl.data"), - content, - has_content_based_path = False, - ) - - return {"releases/start_erl.data": start_erl_data} - -def _symlink_primary_toolchain_output(ctx: AnalysisContext, artifacts: dict[str, Artifact]) -> Artifact: - return ctx.actions.symlinked_dir( - _relname(ctx), - artifacts, - has_content_based_path = False, - ) - -def _relname(ctx: AnalysisContext) -> str: - return ctx.attrs.release_name if ctx.attrs.release_name else ctx.attrs.name - -def _dependencies(ctx: AnalysisContext) -> list[Dependency]: - """Extract dependencies from `applications` field, order preserving""" - deps = [] - for dep in ctx.attrs.applications: - if type(dep) == "tuple": - deps.append(dep[0]) - else: - deps.append(dep) - return deps - -def _dependencies_with_start_types(ctx: AnalysisContext) -> dict[str, StartType]: - """Extract mapping from dependency to start type from `applications` field, this is not order preserving""" - deps = {} - for dep in ctx.attrs.applications: - if type(dep) == "tuple": - deps[_app_name(dep[0])] = StartType(dep[1]) - else: - deps[_app_name(dep)] = StartType("permanent") - return deps - -def _app_name(app: Dependency) -> str: - """Helper to unwrap the name for an erlang application dependency""" - return app[ErlangAppInfo].name - -def _validate_include_erts(ctx: AnalysisContext, toolchain: Toolchain) -> None: - """Validate that include_erts is properly configured with required version information""" - if not ctx.attrs.include_erts: - return - - # Check if applications list is empty (dynamic mode) - if not toolchain.erts_toolchain_info.applications: - fail( - """ -ERROR: include_erts=True requires explicit OTP application versions in your erlang_toolchain. - -Currently, your erlang_toolchain does not have the 'applications' attribute configured, -which is required for creating self-contained releases with bundled ERTS. - -To fix this: - -1. Generate OTP version information from your Erlang installation: - - $ python3 buck2/prelude/erlang/toolchain/generate_otp_versions.py my_otp_versions.bzl + if not config.is_executable: + return None + + boot_script = paths.join("releases", config.version, config.default_bootscript_name) + vm_args = boot_script + ".vm.args" + + for config_path in config.config_paths: + if config_path + ".config" not in release_files: + fail("%s is configured with `%s.config`, which none of its overlays installs" % (str(config.label), config_path)) + + if boot_script + ".boot" not in release_files: + fail("%s boots with `%s.boot`, which the release does not contain" % (str(config.label), boot_script)) + + head = [ + "#!/usr/bin/env bash", + "set -euo pipefail", + # macOS has no `readlink -f`, so the symlink chain is followed one hop at a time + 'SELF="${BASH_SOURCE[0]}"', + "HOPS=0", + "while :; do", + ' ROOTDIR="$(cd -P "$(dirname "$SELF")/.." && pwd)"', + ' if [ -e "$ROOTDIR/{}.boot" ]; then'.format(boot_script), + " break", + " fi", + ' if [ ! -L "$SELF" ]; then', + ' echo "$0: cannot find the release root above $SELF" >&2', + " exit 1", + " fi", + " HOPS=$((HOPS + 1))", + " if [ $HOPS -gt 40 ]; then", + ' echo "$0: too many symlink hops resolving release root" >&2', + " exit 1", + " fi", + ' SELFDIR="$(cd -P "$(dirname "$SELF")" && pwd)"', + ' SELF="$(readlink "$SELF")"', + ' case "$SELF" in', + " /*) ;;", + ' *) SELF="$SELFDIR/$SELF" ;;', + " esac", + "done", + ] + tail = [ + 'TOOL="$(basename "$0")"', + "export ROOTDIR BINDIR", + 'exec "$BINDIR/erlexec" \\', + ' -boot "$ROOTDIR/{}" \\'.format(boot_script), + ] + if vm_args in release_files: + tail.append(' -args_file "$ROOTDIR/{}" \\'.format(vm_args)) + for config_path in config.config_paths: + tail.append(' -config "$ROOTDIR/{}" \\'.format(config_path)) + tail += [ + ' -extra "$TOOL" ${1+"$@"}', + "", + ] -2. Commit the generated file and load it in your BUCK file: + return LauncherLines(name = config.name, head = head, tail = tail) - load(":my_otp_versions.bzl", "get_otp_applications", "get_erts_version") +def _launcher_path(config: ReleaseConfig) -> str: + return paths.join("bin", config.name) -3. Configure your erlang_toolchain with the application versions: +def _run_with_env(actions: AnalysisActions, config: ReleaseConfig, args: cmd_args, **kwargs): + """run interface that injects the environment the release's toolchain invocations run with""" + env = config.os_env if config.os_env != None else config.toolchain.env - erlang_toolchain( - name = "my-toolchain", - applications = get_otp_applications(), - erts_version = get_erts_version(), - otp_binaries = "...", - # ... other configuration - ) + if "env" in kwargs: + kwargs["env"].update(env) + else: + kwargs["env"] = env -Alternatively, if you don't need a self-contained release with bundled ERTS, -set include_erts=False (or remove it, as False is the default). + actions.run(args, **kwargs) -Documentation: https://buck2.build/docs/prelude/erlang/ -Target: {target} -""".format(target = str(ctx.label)) - ) +def _validate_is_executable(config: ReleaseConfig) -> None: + """Validate that a runnable release ships the emulator its launcher runs""" + if config.is_executable and not config.include_erts: + fail("is_executable = True requires include_erts = True, the launcher runs the emulator from the release's own erts folder: %s" % (str(config.label),)) - # Check if erts_version is still dynamic - if toolchain.erts_toolchain_info.erts_version == "dynamic": +def _validate_include_erts(config: ReleaseConfig) -> None: + """Validate that a release bundling the emulator has a toolchain to take it from""" + if config.include_erts and config.toolchain.erts_toolchain_info == None: fail( - """ -ERROR: include_erts=True requires an explicit erts_version in your erlang_toolchain. - -Current erts_version is 'dynamic' which only works when include_erts=False. - -Please ensure you've configured your erlang_toolchain with: - - applications = get_otp_applications() # from generated .bzl file - - erts_version = get_erts_version() # from generated .bzl file - -See the error message above for how to generate the version configuration. - -Target: {target} -""".format(target = str(ctx.label)) + "include_erts = True requires the toolchain `%s` to set erts_toolchain_info, there is no ERTS nor OTP applications to take from it otherwise: %s" + % (config.toolchain.name, str(config.label)) ) diff --git a/prelude/erlang/erlang_tests.bzl b/prelude/erlang/erlang_tests.bzl index 61cb3f8..55630f5 100644 --- a/prelude/erlang/erlang_tests.bzl +++ b/prelude/erlang/erlang_tests.bzl @@ -7,6 +7,7 @@ # above-listed licenses. load("@prelude//:paths.bzl", "paths") +load("@prelude//tests:re_utils.bzl", "get_re_executors_from_props") load( ":erlang_build.bzl", "erlang_build", @@ -188,7 +189,8 @@ def _build_erlang_test(ctx: AnalysisContext, dep_info: ErlangDependencyInfo, bin additional_args = additional_shell_args, ) - re_executor = get_re_executor_from_props(ctx) + # Setup RE executors based on the `remote_execution` param. + re_executors = get_re_executors_from_props(ctx) external_runner_info = ExternalRunnerTestInfo( type = "erlang_test", command = [cmd], @@ -197,7 +199,8 @@ def _build_erlang_test(ctx: AnalysisContext, dep_info: ErlangDependencyInfo, bin contacts = ctx.attrs.contacts, run_from_project_root = True, use_project_relative_paths = True, - default_executor = re_executor, + default_executor = re_executors.default_executor, + executor_overrides = re_executors.executor_overrides, ) test_info = ErlangTestInfo( name = suite_name, @@ -323,30 +326,3 @@ def is_target(suite: str) -> bool: if suite.find("//") != -1: return True return False - -def get_re_executor_from_props(ctx: AnalysisContext) -> [CommandExecutorConfig, None]: - """ - Convert the `remote_execution` properties param into a `CommandExecutorConfig` - to use with test providers. - """ - - re_props = ctx.attrs.remote_execution - if re_props == None: - return None - - re_props_copy = dict(re_props) - capabilities = re_props_copy.pop("capabilities") - use_case = re_props_copy.pop("use_case") - remote_cache_enabled = re_props_copy.pop("remote_cache_enabled", None) - if re_props_copy: - unexpected_props = ", ".join(re_props_copy.keys()) - fail("found unexpected re props: " + unexpected_props) - - return CommandExecutorConfig( - local_enabled = False, - remote_enabled = True, - remote_execution_properties = capabilities, - remote_execution_use_case = use_case or "tpx-default", - remote_cache_enabled = remote_cache_enabled, - remote_execution_action_key = None, - ) diff --git a/prelude/erlang/erlang_toolchain.bzl b/prelude/erlang/erlang_toolchain.bzl index a8febb5..85632c4 100644 --- a/prelude/erlang/erlang_toolchain.bzl +++ b/prelude/erlang/erlang_toolchain.bzl @@ -19,7 +19,6 @@ load( "ErlangOTPBinariesInfo", "ErlangParseTransformInfo", "ErlangToolchainInfo", - "ErtsToolchainApplicationInfo", "ErtsToolchainInfo", "Tool", "Tools", @@ -45,7 +44,6 @@ ToolchainUtillInfo = provider( "escript_trampoline": provider_field(Artifact), "escript_builder": provider_field(Artifact), "release_variables_builder": provider_field(Artifact), - "extract_from_otp": provider_field(Artifact), "utility_modules": provider_field(list[Artifact]), }, ) @@ -110,16 +108,8 @@ def _erlang_toolchain_impl(ctx: AnalysisContext) -> list[Provider]: dependency_merger = _gen_toolchain_script(ctx, env, utils.dependency_merger, otp_binaries, utility_modules) escript_builder = _gen_toolchain_script(ctx, env, utils.escript_builder, otp_binaries, utility_modules) release_variables_builder = _gen_toolchain_script(ctx, env, utils.release_variables_builder, otp_binaries, utility_modules) - extract_from_otp = _gen_toolchain_script(ctx, env, utils.extract_from_otp, otp_binaries, utility_modules) - if ctx.attrs.erts_toolchain_info != None: - if ctx.attrs.erts_version != "unknown" or ctx.attrs.applications: - fail( - "erlang_toolchain: erts_toolchain_info is mutually exclusive with erts_version and applications; the supplied dep already carries that metadata" - ) - erts_toolchain_info = ctx.attrs.erts_toolchain_info[ErtsToolchainInfo] - else: - erts_toolchain_info = _extract_erts_toolchain_info(ctx, env, extract_from_otp) + erts_toolchain_info = ctx.attrs.erts_toolchain_info[ErtsToolchainInfo] if ctx.attrs.erts_toolchain_info else None return [ DefaultInfo(), @@ -138,7 +128,6 @@ def _erlang_toolchain_impl(ctx: AnalysisContext) -> list[Provider]: escript_builder = escript_builder, otp_binaries = otp_binaries, release_variables_builder = release_variables_builder, - extract_from_otp = extract_from_otp, core_parse_transforms = core_parse_transforms, parse_transforms = parse_transforms, parse_transforms_filters = ctx.attrs.parse_transforms_filters, @@ -148,55 +137,6 @@ def _erlang_toolchain_impl(ctx: AnalysisContext) -> list[Provider]: ), ] -def _extract_erts_toolchain_info(ctx: AnalysisContext, env: dict[str, str], extract_from_otp: Tool) -> ErtsToolchainInfo: - erts_toolchain_application_info_list = [ - ErtsToolchainApplicationInfo( - name = application["name"], - version = application["version"], - ) - for application in ctx.attrs.applications - ] - - otp_start_boot = ctx.actions.declare_output("otp_start_boot", has_content_based_path = False) - ctx.actions.run( - cmd_args(extract_from_otp, "bin/start.boot", otp_start_boot.as_output()), - identifier = ctx.attrs.name + "_start_boot", - category = "extract_otp_boot", - env = env, - ) - otp_no_dot_erlang_boot = ctx.actions.declare_output("otp_no_dot_erlang_boot", has_content_based_path = False) - ctx.actions.run( - cmd_args(extract_from_otp, "bin/no_dot_erlang.boot", otp_no_dot_erlang_boot.as_output()), - identifier = ctx.attrs.name + "_no_dot_erlang_boot", - category = "extract_otp_boot", - env = env, - ) - - erts_dir = "erts-{}".format("*" if ctx.attrs.erts_version == "dynamic" else ctx.attrs.erts_version) - erts_headers = ctx.actions.declare_output("erts_headers", dir = True, has_content_based_path = False) - ctx.actions.run( - cmd_args(extract_from_otp, paths.join(erts_dir, "include"), erts_headers.as_output()), - identifier = ctx.attrs.name + "_headers", - category = "extract_erts_headers", - env = env, - ) - - erts_toolchain_info = ErtsToolchainInfo( - applications = erts_toolchain_application_info_list, - erts_version = ctx.attrs.erts_version, - headers = erts_headers, - otp_start_boot = otp_start_boot, - otp_no_dot_erlang_boot = otp_no_dot_erlang_boot, - output = ctx.actions.declare_output("erts-{}".format(ctx.attrs.erts_version), dir = True, has_content_based_path = False), - ) - ctx.actions.run( - cmd_args(extract_from_otp, erts_dir, erts_toolchain_info.output.as_output()), - identifier = ctx.attrs.name, - category = "extract_erts", - env = env, - ) - return erts_toolchain_info - def _gen_parse_transforms(ctx: AnalysisContext, erlc: Tool, env: dict[str, str], parse_transforms: list[Dependency]) -> dict[str, cmd_args]: transforms = {} for dep in parse_transforms: @@ -268,7 +208,6 @@ def _gen_toolchain_script(ctx: AnalysisContext, env: dict[str, str], script: Art erlang_toolchain = rule( impl = _erlang_toolchain_impl, attrs = { - "applications": attrs.list(attrs.dict(key = attrs.string(), value = attrs.string()), default = []), "core_parse_transforms": attrs.list(attrs.dep(), default = ["@prelude//erlang/toolchain:transform_project_root"]), "emu_flags": attrs.one_of( attrs.list(attrs.string()), @@ -284,8 +223,6 @@ erlang_toolchain = rule( default = [], ), "erts_toolchain_info": attrs.option(attrs.dep(providers = [ErtsToolchainInfo]), default = None), - # ERTS version and OTP application metadata - "erts_version": attrs.string(default = "unknown"), "otp_binaries": attrs.toolchain_dep(), "parse_transforms": attrs.list(attrs.dep()), "parse_transforms_filters": attrs.dict(key = attrs.string(), value = attrs.list(attrs.string())), @@ -364,7 +301,6 @@ def _toolchain_utils(ctx: AnalysisContext) -> list[Provider]: escript_trampoline = ctx.attrs.escript_trampoline, escript_builder = ctx.attrs.escript_builder, release_variables_builder = ctx.attrs.release_variables_builder, - extract_from_otp = ctx.attrs.extract_from_otp, utility_modules = ctx.attrs.utility_modules, ), ] @@ -381,7 +317,6 @@ toolchain_utilities = rule( "dependency_merger": attrs.source(), "escript_builder": attrs.source(), "escript_trampoline": attrs.source(), - "extract_from_otp": attrs.source(), "release_variables_builder": attrs.source(), "utility_modules": attrs.list(attrs.source()), }, diff --git a/prelude/erlang/erts/BUCK b/prelude/erlang/erts/BUCK new file mode 100644 index 0000000..335a459 --- /dev/null +++ b/prelude/erlang/erts/BUCK @@ -0,0 +1,11 @@ +load("@prelude//utils:source_listing.bzl", "source_listing") + +oncall("build_infra") + +source_listing() + +export_file( + name = "extract_from_otp.erl", + src = "extract_from_otp.erl", + visibility = ["PUBLIC"], +) diff --git a/prelude/erlang/toolchain/extract_from_otp.erl b/prelude/erlang/erts/extract_from_otp.erl similarity index 75% rename from prelude/erlang/toolchain/extract_from_otp.erl rename to prelude/erlang/erts/extract_from_otp.erl index 38b31ce..898d7dc 100644 --- a/prelude/erlang/toolchain/extract_from_otp.erl +++ b/prelude/erlang/erts/extract_from_otp.erl @@ -10,10 +10,12 @@ -compile(warn_missing_spec_all). -author("loscher@meta.com"). -moduledoc """ -Copy ERTS for releases to the given location +Take parts of the OTP a release is built from, and introspect the versions it ships usage: - extract_from_otp.erl wildcard target + extract_from_otp.erl versions Target + extract_from_otp.erl extract_into Wildcard Target + extract_from_otp.erl Wildcard Target """. %% escript API @@ -35,6 +37,10 @@ usage: %%%============================================================================= -spec main([string()]) -> ok. +main(["versions", Target]) -> + ok = file:write_file(Target, json:encode(versions())); +main(["extract_into", Wildcard, Target]) -> + ok = extract_into(Wildcard, Target); main([Wildcard, Target]) -> ok = extract(Wildcard, Target); main(_) -> @@ -96,6 +102,45 @@ extract(Wildcard, Target) -> erlang:halt(1) end. +-spec extract_into(Wildcard, Target) -> ok when Wildcard :: string(), Target :: string(). +extract_into(Wildcard, Target) -> + FullWildcard = filename:join(code:root_dir(), Wildcard), + case filelib:wildcard(FullWildcard, ".", prim_file) of + [] -> + ?extract_from_otp_error(?no_matches_for_wildcard, #{ + message => ?FMT("No matches found for wildcard: ~ts", [FullWildcard]), + wildcard => bin(FullWildcard), + root_dir => bin(code:root_dir()) + }), + erlang:halt(1); + Paths -> + ok = filelib:ensure_path(Target), + lists:foreach( + fun(Path) -> + ok = copy_dir(Path, filename:join(Target, filename:basename(Path))) + end, + Paths + ) + end. + +-spec versions() -> #{binary() => binary() | #{binary() => binary()}}. +versions() -> + LibDir = filename:join(code:root_dir(), "lib"), + Applications = lists:foldl( + fun(Entry, Acc) -> + case string:split(Entry, "-", trailing) of + [Name, Version] -> Acc#{bin(Name) => bin(Version)}; + _ -> Acc + end + end, + #{}, + filelib:wildcard("*-*", LibDir, prim_file) + ), + #{ + <<"erts_version">> => bin(erlang:system_info(version)), + <<"applications">> => Applications + }. + -spec extract_app_name_from_wildcard(string()) -> {ok, string()} | error. extract_app_name_from_wildcard(Wildcard) -> % Expected format: "lib/appname-*" @@ -161,4 +206,11 @@ parse_installed_application_versions(Dirs) -> -spec usage() -> ok. usage() -> - io:format(standard_error, "needs exactly one argument: extract_from_otp.escript wildcard target~n", []). + io:format( + standard_error, + "usage:~n" + " extract_from_otp.erl versions Target~n" + " extract_from_otp.erl extract_into Wildcard Target~n" + " extract_from_otp.erl Wildcard Target~n", + [] + ). diff --git a/prelude/erlang/toolchain/BUCK b/prelude/erlang/toolchain/BUCK index e1ae45f..facd4f4 100644 --- a/prelude/erlang/toolchain/BUCK +++ b/prelude/erlang/toolchain/BUCK @@ -34,7 +34,6 @@ toolchain_utilities( dependency_merger = "dependency_merger.erl", escript_builder = "escript_builder.erl", escript_trampoline = "escript_trampoline.sh", - extract_from_otp = "extract_from_otp.erl", release_variables_builder = "release_variables_builder.erl", utility_modules = ["dependency_utils.erl"], visibility = ["PUBLIC"], diff --git a/prelude/erlang/toolchain/README.md b/prelude/erlang/toolchain/README.md deleted file mode 100644 index 56c7521..0000000 --- a/prelude/erlang/toolchain/README.md +++ /dev/null @@ -1,125 +0,0 @@ -# Erlang Toolchain Version Generation - -This directory contains utilities for generating OTP application version information required for creating self-contained Erlang releases with bundled ERTS (`include_erts=True`). - -## Quick Start - -### 1. Generate Version Information - -Run the script to extract version information from your Erlang installation: - -```bash -python3 buck2/prelude/erlang/toolchain/generate_otp_versions.py my_otp_versions.bzl -``` - -This creates a `.bzl` file containing version information for all OTP applications. - -### 2. Use in Your BUCK File - -Load the generated file and configure your toolchain: - -```python -load(":my_otp_versions.bzl", "get_otp_applications", "get_erts_version") - -erlang_toolchain( - name = "erlang-27", - applications = get_otp_applications(), - erts_version = get_erts_version(), - otp_binaries = "//path/to:otp-binaries", - # ... other configuration -) -``` - -### 3. Enable Bundled ERTS - -Now you can create self-contained releases: - -```python -erlang_release( - name = "myapp", - applications = [":myapp"], - include_erts = True, # Bundles ERTS in the release -) -``` - -## Why is This Needed? - -Self-contained releases with bundled ERTS require versioned directory structures (e.g., `kernel-10.1` instead of `kernel-*`) to create proper OTP releases. This ensures: - -- **Reproducibility**: Exact versions are locked in the build -- **Portability**: Release doesn't depend on system-installed Erlang -- **Isolation**: Each release bundles its own runtime - -For releases without bundled ERTS (`include_erts=False`, the default), version information is optional and the system will use wildcard patterns. - -## Files - -- **`generate_otp_versions.py`**: Python script that extracts version info from OTP -- **`otp_versions.bzl`**: Generated OTP version information for the current Erlang installation - -## Updating Versions - -When you upgrade your Erlang/OTP installation, regenerate the version file: - -```bash -$ python3 buck2/prelude/erlang/toolchain/generate_otp_versions.py my_otp_versions.bzl -$ git add my_otp_versions.bzl -$ git commit -m "Update OTP versions for Erlang/OTP 28" -``` - -## Troubleshooting - -### Error: "include_erts=True requires explicit OTP application versions" - -This means you're trying to create a bundled release without configuring version information. Follow the Quick Start steps above. - -### Error: "Could not find version for OTP application: foo" - -The application `foo` is not in your generated version file. Either: -1. Regenerate the file with your current Erlang installation -2. The application is not part of OTP (third-party app) - depend on it directly instead - -### Different Erlang Versions for Different Targets - -You can generate multiple version files for different Erlang installations: - -```bash -# For OTP 27 -$ /path/to/otp27/bin/python3 generate_otp_versions.py otp27_versions.bzl - -# For OTP 26 -$ /path/to/otp26/bin/python3 generate_otp_versions.py otp26_versions.bzl -``` - -Then create separate toolchains for each: - -```python -load(":otp27_versions.bzl", otp27_apps = "get_otp_applications", otp27_erts = "get_erts_version") -load(":otp26_versions.bzl", otp26_apps = "get_otp_applications", otp26_erts = "get_erts_version") - -erlang_toolchain( - name = "erlang-27", - applications = otp27_apps(), - erts_version = otp27_erts(), - ... -) - -erlang_toolchain( - name = "erlang-26", - applications = otp26_apps(), - erts_version = otp26_erts(), - ... -) -``` - -## Technical Details - -The generated `.bzl` file contains: -- List of all OTP applications with exact versions (e.g., `{"name": "kernel", "version": "10.1"}`) -- ERTS version (e.g., `"15.1"`) -- OTP release number (e.g., `"27"`) - -This information is used by: -- `erlang_otp_application.bzl`: To create versioned app directories -- `erlang_release.bzl`: To create proper ERTS directory structure -- `boot_script_builder.erl`: To generate boot scripts with correct versions diff --git a/prelude/erlang/toolchain/boot_script_builder.erl b/prelude/erlang/toolchain/boot_script_builder.erl index e3f9dbc..a245091 100644 --- a/prelude/erlang/toolchain/boot_script_builder.erl +++ b/prelude/erlang/toolchain/boot_script_builder.erl @@ -65,7 +65,7 @@ build_no_dot_erlang_boot( PathOption, RelName, RelVersion, - _OTPAppMapping + OTPAppMapping ) -> {OTPApps, _Others} = lists:partition( fun(#{<<"resolved">> := Resolved}) -> not Resolved end, @@ -80,7 +80,8 @@ build_no_dot_erlang_boot( OTPApps ) of - {value, #{<<"version">> := KernelVersion}} -> + {value, _} -> + #{<<"kernel">> := #{version := KernelVersion}} = OTPAppMapping, {kernel, binary_to_list(KernelVersion)}; false -> error({abort, "kernel version not found in OTPApps"}) @@ -94,7 +95,8 @@ build_no_dot_erlang_boot( OTPApps ) of - {value, #{<<"version">> := StdLibVersion}} -> + {value, _} -> + #{<<"stdlib">> := #{version := StdLibVersion}} = OTPAppMapping, {stdlib, binary_to_list(StdLibVersion)}; false -> error({abort, "stdlib version not found in OTPApps"}) diff --git a/prelude/erlang/toolchain/generate_otp_versions.py b/prelude/erlang/toolchain/generate_otp_versions.py deleted file mode 100755 index ebc1c11..0000000 --- a/prelude/erlang/toolchain/generate_otp_versions.py +++ /dev/null @@ -1,245 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This source code is dual-licensed under either the MIT license found in the -# LICENSE-MIT file in the root directory of this source tree or the Apache -# License, Version 2.0 found in the LICENSE-APACHE file in the root directory -# of this source tree. You may select, at your option, one of the -# above-listed licenses. - -""" -Generate OTP application version information for use with erlang_toolchain - -This script extracts version information for all OTP applications from the -current Erlang runtime by reading the installed_application_versions file, -which is the canonical source maintained by OTP itself. - -Usage: - python3 generate_otp_versions.py [output_file.bzl] - -Example: - $ python3 generate_otp_versions.py my_otp_versions.bzl - Generated: my_otp_versions.bzl - ERTS version: 15.1 - OTP release: 27 - Applications: 34 - -Then in your BUCK file: - load(":my_otp_versions.bzl", "get_otp_applications", "get_erts_version") - - erlang_toolchain( - name = "erlang-27", - applications = get_otp_applications(), - erts_version = get_erts_version(), - ... - ) -""" - -import os -import subprocess -import sys -from datetime import datetime -from typing import Dict, List - - -def get_erl_command() -> str: - """Find the erl command in PATH.""" - erl = subprocess.run(["which", "erl"], capture_output=True, text=True, check=False) - if erl.returncode != 0: - raise RuntimeError( - "Could not find 'erl' command. Please ensure Erlang is installed and in PATH." - ) - return erl.stdout.strip() - - -def run_erl_eval(expression: str) -> str: - """Execute an Erlang expression and return the output.""" - cmd = [ - "erl", - "-eval", - f'io:format("~s", [{expression}])', - "-s", - "init", - "stop", - "-noshell", - ] - result = subprocess.run(cmd, capture_output=True, text=True, check=True) - return result.stdout.strip() - - -def get_erts_version() -> str: - """Get the ERTS version from the current Erlang installation.""" - return run_erl_eval("erlang:system_info(version)") - - -def get_otp_release() -> str: - """Get the OTP release number.""" - return run_erl_eval("erlang:system_info(otp_release)") - - -def get_releases_dir() -> str: - """Get the OTP releases directory path for the current OTP release.""" - return run_erl_eval( - 'filename:join([code:root_dir(), "releases", erlang:system_info(otp_release)])' - ) - - -def get_otp_applications() -> List[Dict[str, str]]: - """ - Get all OTP applications with their versions by reading the - installed_application_versions file. - - This file is maintained by OTP and contains the canonical list of - all installed applications with their exact versions. - - Returns: - List of dicts with 'name' and 'version' keys, sorted by name - """ - releases_dir = get_releases_dir() - installed_apps_file = os.path.join(releases_dir, "installed_application_versions") - - if not os.path.exists(installed_apps_file): - raise RuntimeError( - f"Could not find installed_application_versions file at: {installed_apps_file}\n" - f"This file should exist in a standard Erlang/OTP installation." - ) - - applications = [] - - with open(installed_apps_file, "r") as f: - for line in f: - line = line.strip() - if not line: - continue - - # Format is: appname-version - # Split on first hyphen to separate name from version - parts = line.split("-", 1) - if len(parts) == 2: - name, version = parts - applications.append({"name": name, "version": version}) - - # Sort by name for consistent output - applications.sort(key=lambda x: x["name"]) - - return applications - - -def generate_bzl_content( - erts_version: str, otp_release: str, applications: List[Dict[str, str]] -) -> str: - """Generate the Starlark (.bzl) file content.""" - - timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") - - # Header - lines = [ - "# Copyright (c) Meta Platforms, Inc. and affiliates.", - "#", - "# This source code is dual-licensed under either the MIT license found in the", - "# LICENSE-MIT file in the root directory of this source tree or the Apache", - "# License, Version 2.0 found in the LICENSE-APACHE file in the root directory", - "# of this source tree. You may select, at your option, one of the", - "# above-listed licenses.", - "", - "# Auto-generated OTP application version information", - "# DO NOT EDIT THIS FILE MANUALLY", - "#", - f"# Generated on: {timestamp}", - f"# Erlang/OTP: {otp_release}", - f"# ERTS version: {erts_version}", - f"# Applications: {len(applications)}", - "#", - "# To regenerate this file, run:", - "# python3 buck2/prelude/erlang/toolchain/generate_otp_versions.py ", - "", - ] - - # Applications list - lines.append("_OTP_APPLICATIONS = [") - for app in applications: - lines.append(f' {{"name": "{app["name"]}", "version": "{app["version"]}"}},') - lines.append("]") - lines.append("") - - # Constants - lines.append(f'_ERTS_VERSION = "{erts_version}"') - lines.append(f'_OTP_RELEASE = "{otp_release}"') - lines.append("") - - # Helper functions - lines.extend( - [ - "def get_otp_applications():", - ' """Returns list of OTP applications with versions for erlang_toolchain.', - "", - " Returns:", - " List of dicts with 'name' and 'version' keys", - "", - " Example:", - " erlang_toolchain(", - ' name = "my-toolchain",', - " applications = get_otp_applications(),", - " erts_version = get_erts_version(),", - " ...", - " )", - ' """', - " return _OTP_APPLICATIONS", - "", - "def get_erts_version():", - ' """Returns the ERTS version string.', - "", - " Returns:", - ' String containing the ERTS version (e.g., "15.1")', - ' """', - " return _ERTS_VERSION", - "", - "def get_otp_release():", - ' """Returns the OTP release number.', - "", - " Returns:", - ' String containing the OTP release (e.g., "27")', - ' """', - " return _OTP_RELEASE", - ] - ) - - return "\n".join(lines) + "\n" - - -def main(): - """Main entry point.""" - if len(sys.argv) > 2: - print(f"Usage: {sys.argv[0]} [output_file.bzl]", file=sys.stderr) - sys.exit(1) - - output_file = sys.argv[1] if len(sys.argv) == 2 else "otp_versions.bzl" - - try: - print("Extracting OTP version information...") - - # Get version information - erts_version = get_erts_version() - otp_release = get_otp_release() - applications = get_otp_applications() - - # Generate .bzl file content - bzl_content = generate_bzl_content(erts_version, otp_release, applications) - - # Write to file - with open(output_file, "w") as f: - f.write(bzl_content) - - # Print summary - print(f"Generated: {output_file}") - print(f"ERTS version: {erts_version}") - print(f"OTP release: {otp_release}") - print(f"Applications: {len(applications)}") - - except Exception as e: - print(f"Error: {e}", file=sys.stderr) - sys.exit(1) - - -if __name__ == "__main__": - main() diff --git a/prelude/erlang/toolchain/otp_versions.bzl b/prelude/erlang/toolchain/otp_versions.bzl deleted file mode 100644 index ec29e06..0000000 --- a/prelude/erlang/toolchain/otp_versions.bzl +++ /dev/null @@ -1,90 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# -# This source code is dual-licensed under either the MIT license found in the -# LICENSE-MIT file in the root directory of this source tree or the Apache -# License, Version 2.0 found in the LICENSE-APACHE file in the root directory -# of this source tree. You may select, at your option, one of the -# above-listed licenses. - -# Auto-generated OTP application version information -# DO NOT EDIT THIS FILE MANUALLY -# -# Generated on: 2025-12-20 15:19:52 -# Erlang/OTP: 27 -# ERTS version: 15.1 -# Applications: 34 -# -# To regenerate this file, run: -# python3 buck2/prelude/erlang/toolchain/generate_otp_versions.py - -_OTP_APPLICATIONS = [ - {"name": "asn1", "version": "5.3.1"}, - {"name": "common_test", "version": "1.27.1"}, - {"name": "compiler", "version": "8.5.2"}, - {"name": "crypto", "version": "5.5.1"}, - {"name": "debugger", "version": "5.4"}, - {"name": "dialyzer", "version": "5.2.1"}, - {"name": "diameter", "version": "2.4.1"}, - {"name": "edoc", "version": "1.3.2"}, - {"name": "eldap", "version": "1.2.13"}, - {"name": "erl_interface", "version": "5.5.2"}, - {"name": "erts", "version": "15.1"}, - {"name": "et", "version": "1.7.1"}, - {"name": "eunit", "version": "2.9.1"}, - {"name": "ftp", "version": "1.2.3"}, - {"name": "inets", "version": "9.3"}, - {"name": "kernel", "version": "10.1"}, - {"name": "megaco", "version": "4.6"}, - {"name": "mnesia", "version": "4.23.2"}, - {"name": "observer", "version": "2.16"}, - {"name": "os_mon", "version": "2.10"}, - {"name": "parsetools", "version": "2.6"}, - {"name": "public_key", "version": "1.16.2"}, - {"name": "reltool", "version": "1.0.1"}, - {"name": "runtime_tools", "version": "2.1.1"}, - {"name": "sasl", "version": "4.2.2"}, - {"name": "snmp", "version": "5.17"}, - {"name": "ssh", "version": "5.2.2"}, - {"name": "ssl", "version": "11.2.2"}, - {"name": "stdlib", "version": "6.1"}, - {"name": "syntax_tools", "version": "3.2.1"}, - {"name": "tftp", "version": "1.2.1"}, - {"name": "tools", "version": "4.1"}, - {"name": "wx", "version": "2.4.3"}, - {"name": "xmerl", "version": "2.1"}, -] - -_ERTS_VERSION = "15.1" -_OTP_RELEASE = "27" - -def get_otp_applications(): - """Returns list of OTP applications with versions for erlang_toolchain. - - Returns: - List of dicts with 'name' and 'version' keys - - Example: - erlang_toolchain( - name = "my-toolchain", - applications = get_otp_applications(), - erts_version = get_erts_version(), - ... - ) - """ - return _OTP_APPLICATIONS - -def get_erts_version(): - """Returns the ERTS version string. - - Returns: - String containing the ERTS version (e.g., "15.1") - """ - return _ERTS_VERSION - -def get_otp_release(): - """Returns the OTP release number. - - Returns: - String containing the OTP release (e.g., "27") - """ - return _OTP_RELEASE diff --git a/prelude/genrule.bzl b/prelude/genrule.bzl index 34144cd..a425adb 100644 --- a/prelude/genrule.bzl +++ b/prelude/genrule.bzl @@ -9,8 +9,8 @@ # Implementation of the `genrule` build rule. load("@prelude//:cache_mode.bzl", "CacheModeInfo") -load("@prelude//:genrule_local_labels.bzl", "genrule_labels_require_local") -load("@prelude//:genrule_prefer_local_labels.bzl", "genrule_labels_prefer_local") +load("@prelude//:genrule_local_labels.bzl", "resolved_genrule_labels_require_local") +load("@prelude//:genrule_prefer_local_labels.bzl", "resolved_genrule_labels_prefer_local") load("@prelude//:genrule_toolchain.bzl", "GenruleToolchainInfo") load("@prelude//:is_full_meta_repo.bzl", "is_full_meta_repo") load("@prelude//android:build_only_native_code.bzl", "is_build_only_native_code") @@ -71,10 +71,10 @@ def _requires_build_root(ctx: AnalysisContext) -> bool: return False def _requires_local(ctx: AnalysisContext) -> bool: - return genrule_labels_require_local(ctx.attrs.labels) + return resolved_genrule_labels_require_local(ctx.attrs.labels) def _prefers_local(ctx: AnalysisContext) -> bool: - return genrule_labels_prefer_local(ctx.attrs.labels) + return resolved_genrule_labels_prefer_local(ctx.attrs.labels) def _ignore_artifacts(ctx: AnalysisContext) -> bool: return "buck2_ignore_artifacts" in ctx.attrs.labels @@ -166,6 +166,7 @@ _HEADER_EXTENSIONS = [ ".hxx", ".cuh", ".inc", + ".tcc", ] def _is_header(path: str) -> bool: @@ -308,15 +309,18 @@ def process_genrule( delimiter = " " # Setup environment variables. - srcs = cmd_args(delimiter = delimiter) - for symlink in symlinks: - srcs.add(cmd_args(srcs_artifact, format = path_sep.join([".", "{}", symlink.replace("/", path_sep)]))) + no_srcs_environment = _requires_no_srcs_environment(ctx) env_vars = { "GEN_DIR": "GEN_DIR_DEPRECATED", "OUT": out_env.as_output(), "SRCDIR": cmd_args(srcs_artifact, format = path_sep.join([".", "{}"])), - "SRCS": srcs, - } | {k: cmd_args(v) for k, v in getattr(ctx.attrs, "env", {}).items()} + } + if not no_srcs_environment: + srcs = cmd_args(delimiter = delimiter) + for symlink in symlinks: + srcs.add(cmd_args(srcs_artifact, format = path_sep.join([".", "{}", symlink.replace("/", path_sep)]))) + env_vars["SRCS"] = srcs + env_vars |= {k: cmd_args(v) for k, v in getattr(ctx.attrs, "env", {}).items()} # RE will cache successful actions that don't produce the desired outptuts, # so if that happens and _then_ we add a local-only label, we'll get a @@ -336,9 +340,6 @@ def process_genrule( if cacheable and cache_bust: env_vars["__BUCK2_ALLOW_CACHE_UPLOADS_CACHE_BUSTER"] = "" - if _requires_no_srcs_environment(ctx): - env_vars.pop("SRCS") - for key, value in extra_env_vars.items(): env_vars[key] = value diff --git a/prelude/genrule_local_labels.bzl b/prelude/genrule_local_labels.bzl index af5754f..312ee16 100644 --- a/prelude/genrule_local_labels.bzl +++ b/prelude/genrule_local_labels.bzl @@ -188,3 +188,9 @@ def genrule_labels_require_local(labels): return False return selects.apply(labels, check_labels) + +def resolved_genrule_labels_require_local(labels: list[str]) -> bool: + for label in labels: + if label in _GENRULE_LOCAL_LABELS: + return True + return False diff --git a/prelude/genrule_prefer_local_labels.bzl b/prelude/genrule_prefer_local_labels.bzl index 1c692c6..72dcbcc 100644 --- a/prelude/genrule_prefer_local_labels.bzl +++ b/prelude/genrule_prefer_local_labels.bzl @@ -16,7 +16,7 @@ _GENRULE_PREFER_LOCAL_LABELS = set([ "large_copy", ]) -def genrule_labels_prefer_local(labels): +def resolved_genrule_labels_prefer_local(labels: list[str]) -> bool: for label in labels: if label in _GENRULE_PREFER_LOCAL_LABELS: return True diff --git a/prelude/go/go_test.bzl b/prelude/go/go_test.bzl index 1aff297..a7841d2 100644 --- a/prelude/go/go_test.bzl +++ b/prelude/go/go_test.bzl @@ -39,9 +39,6 @@ def _gen_test_main( """ Generate a `main.go` which calls tests from the given sources. """ - cover_pkgs_argsfile = ctx.actions.declare_output("cover_pkgs_argsfile", has_content_based_path = True) - ctx.actions.write(cover_pkgs_argsfile, [["--cover-pkgs", pkg] for pkg in cover_packages]) - output = ctx.actions.declare_output("main.go", has_content_based_path = True) cmd = [] cmd.append(ctx.attrs._testmaingen[RunInfo]) @@ -51,7 +48,10 @@ def _gen_test_main( cmd.append(cmd_args(pkg_import_path, format = "--import-path={}")) if coverage_mode != None: cmd.extend(["--cover-mode", coverage_mode.value]) - cmd.append(cmd_args(cover_pkgs_argsfile, format = "@{}")) + if cover_packages: + cover_pkgs_argsfile = ctx.actions.declare_output("cover_pkgs_argsfile", has_content_based_path = True) + ctx.actions.write(cover_pkgs_argsfile, [["--cover-pkgs", pkg] for pkg in cover_packages]) + cmd.append(cmd_args(cover_pkgs_argsfile, format = "@{}")) cmd.append(cmd_args(test_go_files_argsfile, format = "@{}")) ctx.actions.run(cmd_args(cmd), category = "go_test_main_gen", allow_cache_upload = ctx.attrs._go_toolchain[GoToolchainInfo].allow_cache_upload) return output diff --git a/prelude/go/link.bzl b/prelude/go/link.bzl index 2bf15dc..25615b0 100644 --- a/prelude/go/link.bzl +++ b/prelude/go/link.bzl @@ -281,7 +281,10 @@ def link( ) ) - # stamp only executable targets + # Stamp only executable targets. An internal link never runs the ld + # wrapper, so there is no fb_build_info placeholder and the stamper adds + # the section; an external link runs the wrapper, which reserves the + # placeholder when the rule stamps. if build_mode in [GoBuildMode("exe"), GoBuildMode("pie")]: output = stamp_build_info(ctx, output, has_content_based_path = True) diff --git a/prelude/go/package_builder.bzl b/prelude/go/package_builder.bzl index d6f0a03..e9dc960 100644 --- a/prelude/go/package_builder.bzl +++ b/prelude/go/package_builder.bzl @@ -355,9 +355,14 @@ def build_package( if params.standard and params.pkg_import_path in _incomplete_pkgs_allow_list: complete_flag = False + go_files_to_compile = covered_go_files + transformed_cgo_files + + # Use argsfile to avoid command length limit on Windows. The source list + # is identical for the shared and non-shared variants, so they share it. + srcs_argsfile = actions.write("srcs.go_package_argsfile", go_files_to_compile, has_content_based_path = True) if go_files_to_compile else None + def build_variant(shared: bool) -> (Artifact, Artifact): build_variant_id = "shared" if shared else "non-shared" # use tomake artifacts and actions unique - go_files_to_compile = covered_go_files + transformed_cgo_files required_imports = go_list.imports | implicit_imports( pkg_name = go_list.pkg_name, @@ -382,6 +387,7 @@ def build_package( pkg_import_path = params.pkg_import_path, main = params.main, go_srcs = go_files_to_compile, + srcs_argsfile = srcs_argsfile, importcfg = importcfg, compiler_flags = params.compiler_flags, shared = shared, @@ -436,6 +442,7 @@ def _compile( pkg_import_path: str, main: bool, go_srcs: list[Artifact], + srcs_argsfile: Artifact | None, importcfg: Artifact, compiler_flags: list[str], shared: bool, @@ -460,9 +467,6 @@ def _compile( asmhdr = actions.declare_output("__asmhdr__{}/go_asm.h".format(build_variant_id), has_content_based_path = True) if gen_asmhdr else None - # Use argsfile to avoid command length limit on Windows - srcs_argsfile = actions.write(build_variant_id + "_srcs.go_package_argsfile", go_srcs, has_content_based_path = True) - compile_cmd = cmd_args( [ go_toolchain.go_wrapper, diff --git a/prelude/go/tools/gobuckify/lib/buck.go b/prelude/go/tools/gobuckify/lib/buck.go index 9aa112a..90f0846 100644 --- a/prelude/go/tools/gobuckify/lib/buck.go +++ b/prelude/go/tools/gobuckify/lib/buck.go @@ -12,6 +12,7 @@ package gobuckifylib import ( "fmt" + "maps" "slices" "strings" ) @@ -22,10 +23,11 @@ type OSDeps struct { ArchDeps map[string]*ArchDeps } -// ArchDeps is a map of Buck-Arch to set of go packages +// ArchDeps is a map of Buck-Arch to CGO-specific dependency sets type ArchDeps struct { - Arch string - Deps *StringSet + Arch string + CommonDeps *StringSet + CgoDeps map[string]*StringSet } // BuckTarget is a buck-friendly representation of a go package @@ -36,39 +38,60 @@ type BuckTarget struct { EmbedFiles StringSet CommonDeps []string PlatformDeps map[string]*OSDeps - TargetCompatibleWith map[string][]string // os => []arch + TargetCompatibleWith map[string]map[string]*StringSet // os => arch => cgo constraints } // Normalise prepares BuckTarget to be written to a file: // - Moves common dependencies to CommonDeps out of PlatformDeps // - Removes empty PlatformDeps -// - Removes TargetCompatibleWith if all platforms are compatible -func (b *BuckTarget) Normalise(totalPlatformNumber int) { +// - Removes TargetCompatibleWith if all configurations are compatible +func (b *BuckTarget) Normalise(totalConfigurationNumber int) { freqmap := make(map[string]int) for _, osDeps := range b.PlatformDeps { for _, archDeps := range osDeps.ArchDeps { - for dep := range *archDeps.Deps { - freqmap[dep]++ + for _, deps := range archDeps.CgoDeps { + for dep := range *deps { + freqmap[dep]++ + } } } } - // If a dependency is used in all platforms, it can be moved to CommonDeps + // If a dependency is used in all configurations, it can be moved to CommonDeps for dep, freq := range freqmap { - if freq == totalPlatformNumber { + if freq == totalConfigurationNumber { b.CommonDeps = append(b.CommonDeps, dep) for _, osDeps := range b.PlatformDeps { for _, archDeps := range osDeps.ArchDeps { - archDeps.Deps.Remove(dep) + for _, deps := range archDeps.CgoDeps { + deps.Remove(dep) + } } } } } + // Identical dependencies with and without cgo do not need a cgo select. + for _, osDeps := range b.PlatformDeps { + for _, archDeps := range osDeps.ArchDeps { + disabled, hasDisabled := archDeps.CgoDeps["prelude//go/constraints:cgo_enabled[false]"] + enabled, hasEnabled := archDeps.CgoDeps["prelude//go/constraints:cgo_enabled[true]"] + if hasDisabled && hasEnabled && disabled.Len() > 0 && maps.Equal(*disabled, *enabled) { + archDeps.CommonDeps = disabled + archDeps.CgoDeps = nil + } + } + } + // Remove empty PlatformDeps for os, osDeps := range b.PlatformDeps { for arch, archDeps := range osDeps.ArchDeps { - if archDeps.Deps.Len() == 0 { + for cgo, deps := range archDeps.CgoDeps { + if deps.Len() == 0 { + delete(archDeps.CgoDeps, cgo) + } + } + if archDeps.CommonDeps == nil && len(archDeps.CgoDeps) == 0 { delete(osDeps.ArchDeps, arch) } } @@ -79,16 +102,14 @@ func (b *BuckTarget) Normalise(totalPlatformNumber int) { slices.Sort(b.CommonDeps) compatibleWithNumber := 0 - for _, archList := range b.TargetCompatibleWith { - compatibleWithNumber += len(archList) - } - - if compatibleWithNumber == totalPlatformNumber { - b.TargetCompatibleWith = nil // all platforms are compatible + for _, archMap := range b.TargetCompatibleWith { + for _, cgoConstraints := range archMap { + compatibleWithNumber += cgoConstraints.Len() + } } - for _, archList := range b.TargetCompatibleWith { - slices.Sort(archList) // sort arch list to render it deterministically + if compatibleWithNumber == totalConfigurationNumber { + b.TargetCompatibleWith = nil // all configurations are compatible } } @@ -96,8 +117,8 @@ func (b *BuckTarget) Normalise(totalPlatformNumber int) { type BuckTargets map[string]*BuckTarget // AddPackage adds a package to the buck targets map -func (b *BuckTargets) AddPackage(pkg *Package, buckOS, buckArch string) { - // If package with the same import path and os/arch already added, its data be replaced +func (b *BuckTargets) AddPackage(pkg *Package, buckOS, buckArch string, cgoEnabled bool) { + // If a package with the same import path and configuration is added again, replace its data. var target *BuckTarget var ok bool if target, ok = (*b)[pkg.ImportPath]; !ok { @@ -107,13 +128,20 @@ func (b *BuckTargets) AddPackage(pkg *Package, buckOS, buckArch string) { PlatformDeps: make(map[string]*OSDeps), EmbedFiles: *NewSet(), IsBinary: pkg.Name == "main", - TargetCompatibleWith: map[string][]string{}, + TargetCompatibleWith: make(map[string]map[string]*StringSet), } (*b)[pkg.ImportPath] = target } + cgoConstraint := fmt.Sprintf("prelude//go/constraints:cgo_enabled[%t]", cgoEnabled) target.EmbedFiles.AddList(pkg.EmbedFiles) - target.TargetCompatibleWith[buckOS] = append(target.TargetCompatibleWith[buckOS], buckArch) + if target.TargetCompatibleWith[buckOS] == nil { + target.TargetCompatibleWith[buckOS] = make(map[string]*StringSet) + } + if target.TargetCompatibleWith[buckOS][buckArch] == nil { + target.TargetCompatibleWith[buckOS][buckArch] = NewSet() + } + target.TargetCompatibleWith[buckOS][buckArch].Add(cgoConstraint) if target.PlatformDeps[buckOS] == nil { target.PlatformDeps[buckOS] = &OSDeps{ @@ -122,16 +150,19 @@ func (b *BuckTargets) AddPackage(pkg *Package, buckOS, buckArch string) { } } - target.PlatformDeps[buckOS].ArchDeps[buckArch] = &ArchDeps{ - Arch: buckArch, - Deps: NewSet(), + if target.PlatformDeps[buckOS].ArchDeps[buckArch] == nil { + target.PlatformDeps[buckOS].ArchDeps[buckArch] = &ArchDeps{ + Arch: buckArch, + CgoDeps: make(map[string]*StringSet), + } } + target.PlatformDeps[buckOS].ArchDeps[buckArch].CgoDeps[cgoConstraint] = NewSet() for _, dep := range pkg.Imports { if !strings.ContainsRune(dep, '.') { continue // skip stdlib deps } - target.PlatformDeps[buckOS].ArchDeps[buckArch].Deps.Add(dep) + target.PlatformDeps[buckOS].ArchDeps[buckArch].CgoDeps[cgoConstraint].Add(dep) } } diff --git a/prelude/go/tools/gobuckify/lib/buck_test.go b/prelude/go/tools/gobuckify/lib/buck_test.go index 41633e1..fd82f32 100644 --- a/prelude/go/tools/gobuckify/lib/buck_test.go +++ b/prelude/go/tools/gobuckify/lib/buck_test.go @@ -12,7 +12,6 @@ package gobuckifylib import ( "reflect" - "slices" "testing" ) @@ -54,136 +53,72 @@ func TestTargetLabelFromImportPath(t *testing.T) { } func TestBuckTargetNormalise(t *testing.T) { - tests := []struct { - name string - target *BuckTarget - totalPlatformNumber int - wantCommonDeps []string - wantPlatformDepsLen int - wantTargetCompatibleWith map[string][]string + targets := make(BuckTargets) + for _, platform := range []struct { + os string + imports []string }{ - { - name: "move common deps and clean-up target_compatible_with", - target: &BuckTarget{ - Name: "test", - ImportPath: "github.com/example/test", - CommonDeps: []string{}, - PlatformDeps: map[string]*OSDeps{ - "linux": { - OS: "linux", - ArchDeps: map[string]*ArchDeps{ - "x86_64": { - Arch: "x86_64", - Deps: func() *StringSet { - s := NewSet() - s.Add("common/dep1") - s.Add("common/dep2") - s.Add("linux/dep") - return s - }(), - }, - }, - }, - "darwin": { - OS: "darwin", - ArchDeps: map[string]*ArchDeps{ - "x86_64": { - Arch: "x86_64", - Deps: func() *StringSet { - s := NewSet() - s.Add("common/dep1") - s.Add("common/dep2") - s.Add("darwin/dep") - return s - }(), - }, - }, - }, - }, - TargetCompatibleWith: map[string][]string{ - "linux": {"x86_64"}, - "darwin": {"x86_64"}, - }, - }, - totalPlatformNumber: 2, - wantCommonDeps: []string{"common/dep1", "common/dep2"}, - wantPlatformDepsLen: 2, // Both platforms still have platform-specific deps - wantTargetCompatibleWith: nil, - }, - { - name: "remove empty platform deps", - target: &BuckTarget{ - Name: "test", + {"linux", []string{"example.com/common/dep1", "example.com/common/dep2", "example.com/linux/dep"}}, + {"darwin", []string{"example.com/common/dep1", "example.com/common/dep2", "example.com/darwin/dep"}}, + } { + for _, cgoEnabled := range []bool{false, true} { + targets.AddPackage(&Package{ ImportPath: "github.com/example/test", - CommonDeps: []string{}, - PlatformDeps: map[string]*OSDeps{ - "linux": { - OS: "linux", - ArchDeps: map[string]*ArchDeps{ - "x86_64": { - Arch: "x86_64", - Deps: NewSet(), - }, - }, - }, - "darwin": { - OS: "darwin", - ArchDeps: map[string]*ArchDeps{ - "x86_64": { - Arch: "x86_64", - Deps: NewSet(), - }, - }, - }, - }, - TargetCompatibleWith: map[string][]string{ - "linux": {"x86_64"}, - "darwin": {"x86_64"}, - }, - }, - totalPlatformNumber: 4, - wantCommonDeps: []string{}, - wantPlatformDepsLen: 0, // All platform deps should be removed as they're now empty - wantTargetCompatibleWith: map[string][]string{ - "linux": {"x86_64"}, - "darwin": {"x86_64"}, - }, - }, + Imports: platform.imports, + }, platform.os, "x86_64", cgoEnabled) + } } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - tt.target.Normalise(tt.totalPlatformNumber) - - // Check CommonDeps - if !reflect.DeepEqual(tt.target.CommonDeps, tt.wantCommonDeps) { - t.Errorf("CommonDeps = %v, want %v", tt.target.CommonDeps, tt.wantCommonDeps) - } + target := targets["github.com/example/test"] + target.Normalise(4) - // Check PlatformDeps length - if len(tt.target.PlatformDeps) != tt.wantPlatformDepsLen { - t.Errorf("PlatformDeps length = %d, want %d", len(tt.target.PlatformDeps), tt.wantPlatformDepsLen) - } + if !reflect.DeepEqual(target.CommonDeps, []string{"example.com/common/dep1", "example.com/common/dep2"}) { + t.Errorf("CommonDeps = %v", target.CommonDeps) + } + if len(target.PlatformDeps) != 2 { + t.Errorf("PlatformDeps length = %d, want 2", len(target.PlatformDeps)) + } + for os, want := range map[string]string{ + "linux": "example.com/linux/dep", + "darwin": "example.com/darwin/dep", + } { + archDeps := target.PlatformDeps[os].ArchDeps["x86_64"] + if archDeps.CommonDeps == nil { + t.Errorf("%s common architecture deps = nil", os) + continue + } + if got := archDeps.CommonDeps.SortedList(); !reflect.DeepEqual(got, []string{want}) { + t.Errorf("%s common architecture deps = %v, want %v", os, got, []string{want}) + } + if len(archDeps.CgoDeps) != 0 { + t.Errorf("%s CGO deps = %v, want none", os, archDeps.CgoDeps) + } + } + if target.TargetCompatibleWith != nil { + t.Errorf("TargetCompatibleWith = %v, want nil", target.TargetCompatibleWith) + } - // Check TargetCompatibleWith - if !reflect.DeepEqual(tt.target.TargetCompatibleWith, tt.wantTargetCompatibleWith) { - t.Errorf("TargetCompatibleWith = %v, want %v", tt.target.TargetCompatibleWith, tt.wantTargetCompatibleWith) - } - }) + for _, cgoEnabled := range []bool{false, true} { + targets.AddPackage(&Package{ + ImportPath: "github.com/example/empty", + }, "linux", "x86_64", cgoEnabled) + } + emptyTarget := targets["github.com/example/empty"] + emptyTarget.Normalise(2) + if len(emptyTarget.PlatformDeps) != 0 { + t.Errorf("PlatformDeps = %v, want none", emptyTarget.PlatformDeps) } } func TestBuckTargetsAddPackage(t *testing.T) { tests := []struct { - name string - pkg *Package - buckOS string - buckArch string - wantName string - wantImportPath string - wantIsBinary bool - wantEmbedFiles []string - wantDepsCount int + name string + pkg *Package + buckOS string + buckArch string + wantName string + wantIsBinary bool + wantDepsCount int }{ { name: "add library package", @@ -193,13 +128,11 @@ func TestBuckTargetsAddPackage(t *testing.T) { Imports: []string{"github.com/example/dep1", "github.com/example/dep2", "fmt"}, EmbedFiles: []string{"embed1.txt", "embed2.txt"}, }, - buckOS: "linux", - buckArch: "x86_64", - wantName: "testpkg", - wantImportPath: "github.com/example/testpkg", - wantIsBinary: false, - wantEmbedFiles: []string{"embed1.txt", "embed2.txt"}, - wantDepsCount: 2, // Only non-stdlib deps + buckOS: "linux", + buckArch: "x86_64", + wantName: "testpkg", + wantIsBinary: false, + wantDepsCount: 2, // Only non-stdlib deps }, { name: "add binary package", @@ -209,20 +142,18 @@ func TestBuckTargetsAddPackage(t *testing.T) { Imports: []string{"github.com/example/dep1", "fmt"}, EmbedFiles: []string{}, }, - buckOS: "darwin", - buckArch: "arm64", - wantName: "cmd", - wantImportPath: "github.com/example/cmd", - wantIsBinary: true, - wantEmbedFiles: []string{}, - wantDepsCount: 1, // Only non-stdlib deps + buckOS: "darwin", + buckArch: "arm64", + wantName: "cmd", + wantIsBinary: true, + wantDepsCount: 1, // Only non-stdlib deps }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { targets := make(BuckTargets) - targets.AddPackage(tt.pkg, tt.buckOS, tt.buckArch) + targets.AddPackage(tt.pkg, tt.buckOS, tt.buckArch, true) // Check if the package was added target, exists := targets[tt.pkg.ImportPath] @@ -235,16 +166,16 @@ func TestBuckTargetsAddPackage(t *testing.T) { t.Errorf("Target name = %q, want %q", target.Name, tt.wantName) } - if target.ImportPath != tt.wantImportPath { - t.Errorf("Target import path = %q, want %q", target.ImportPath, tt.wantImportPath) + if target.ImportPath != tt.pkg.ImportPath { + t.Errorf("Target import path = %q, want %q", target.ImportPath, tt.pkg.ImportPath) } if target.IsBinary != tt.wantIsBinary { t.Errorf("Target IsBinary = %v, want %v", target.IsBinary, tt.wantIsBinary) } - if !reflect.DeepEqual(target.EmbedFiles.SortedList(), tt.wantEmbedFiles) { - t.Errorf("Target EmbedFiles = %v, want %v", target.EmbedFiles.SortedList(), tt.wantEmbedFiles) + if !reflect.DeepEqual(target.EmbedFiles.SortedList(), tt.pkg.EmbedFiles) { + t.Errorf("Target EmbedFiles = %v, want %v", target.EmbedFiles.SortedList(), tt.pkg.EmbedFiles) } // Check platform deps @@ -258,18 +189,27 @@ func TestBuckTargetsAddPackage(t *testing.T) { t.Fatalf("Arch deps not found for %s", tt.buckArch) } - if archDeps.Deps.Len() != tt.wantDepsCount { - t.Errorf("Deps count = %d, want %d", archDeps.Deps.Len(), tt.wantDepsCount) + deps, exists := archDeps.CgoDeps["prelude//go/constraints:cgo_enabled[true]"] + if !exists { + t.Fatal("CGO-enabled deps not found") + } + + if deps.Len() != tt.wantDepsCount { + t.Errorf("Deps count = %d, want %d", deps.Len(), tt.wantDepsCount) } // Check TargetCompatibleWith - archList, exists := target.TargetCompatibleWith[tt.buckOS] + archMap, exists := target.TargetCompatibleWith[tt.buckOS] if !exists { t.Fatalf("TargetCompatibleWith not found for %s", tt.buckOS) } - if !slices.Contains(archList, tt.buckArch) { - t.Errorf("Arch %s not found in TargetCompatibleWith for OS %s", tt.buckArch, tt.buckOS) + cgoConstraints, exists := archMap[tt.buckArch] + if !exists { + t.Fatalf("Arch %s not found in TargetCompatibleWith for OS %s", tt.buckArch, tt.buckOS) + } + if _, exists := (*cgoConstraints)["prelude//go/constraints:cgo_enabled[true]"]; !exists { + t.Error("CGO-enabled constraint not found in TargetCompatibleWith") } }) } @@ -278,15 +218,14 @@ func TestBuckTargetsAddPackage(t *testing.T) { func TestAddPackageMultiplePlatforms(t *testing.T) { targets := make(BuckTargets) pkg := &Package{ - Name: "testpkg", ImportPath: "github.com/example/testpkg", Imports: []string{"github.com/example/dep1", "github.com/example/dep2"}, } // Add the same package for multiple platforms - targets.AddPackage(pkg, "linux", "x86_64") - targets.AddPackage(pkg, "linux", "arm64") - targets.AddPackage(pkg, "darwin", "x86_64") + targets.AddPackage(pkg, "linux", "x86_64", true) + targets.AddPackage(pkg, "linux", "arm64", true) + targets.AddPackage(pkg, "darwin", "x86_64", true) target, exists := targets[pkg.ImportPath] if !exists { @@ -329,3 +268,57 @@ func TestAddPackageMultiplePlatforms(t *testing.T) { t.Errorf("Expected 2 architectures for Linux in PlatformDeps, got %d", len(linuxDeps.ArchDeps)) } } + +func TestAddPackageMultipleCgoConfigurations(t *testing.T) { + const importPath = "github.com/example/testpkg" + targets := make(BuckTargets) + targets.AddPackage(&Package{ + ImportPath: importPath, + Imports: []string{"github.com/example/common", "github.com/example/nocgo"}, + }, "linux", "x86_64", false) + targets.AddPackage(&Package{ + ImportPath: importPath, + Imports: []string{"github.com/example/common", "github.com/example/cgo"}, + }, "linux", "x86_64", true) + + target := targets[importPath] + target.Normalise(2) + + if !reflect.DeepEqual(target.CommonDeps, []string{"github.com/example/common"}) { + t.Errorf("CommonDeps = %v", target.CommonDeps) + } + archDeps := target.PlatformDeps["linux"].ArchDeps["x86_64"] + got := archDeps.CgoDeps["prelude//go/constraints:cgo_enabled[false]"].SortedList() + if !reflect.DeepEqual(got, []string{"github.com/example/nocgo"}) { + t.Errorf("CGO-disabled deps = %v", got) + } + got = archDeps.CgoDeps["prelude//go/constraints:cgo_enabled[true]"].SortedList() + if !reflect.DeepEqual(got, []string{"github.com/example/cgo"}) { + t.Errorf("CGO-enabled deps = %v", got) + } + if target.TargetCompatibleWith != nil { + t.Errorf("TargetCompatibleWith = %v, want nil", target.TargetCompatibleWith) + } +} + +func TestAddCgoOnlyPackage(t *testing.T) { + targets := make(BuckTargets) + targets.AddPackage(&Package{ + ImportPath: "github.com/example/testpkg", + Imports: []string{"github.com/example/cgo"}, + }, "linux", "x86_64", true) + + target := targets["github.com/example/testpkg"] + target.Normalise(2) + + if len(target.CommonDeps) != 0 { + t.Errorf("CommonDeps = %v, want none", target.CommonDeps) + } + if target.TargetCompatibleWith == nil { + t.Fatal("TargetCompatibleWith = nil, want CGO-enabled compatibility") + } + cgoConstraints := target.TargetCompatibleWith["linux"]["x86_64"] + if got := cgoConstraints.SortedList(); !reflect.DeepEqual(got, []string{"prelude//go/constraints:cgo_enabled[true]"}) { + t.Errorf("CGO constraints = %v", got) + } +} diff --git a/prelude/go/tools/gobuckify/lib/golist.go b/prelude/go/tools/gobuckify/lib/golist.go index 2fd3ac3..de62076 100644 --- a/prelude/go/tools/gobuckify/lib/golist.go +++ b/prelude/go/tools/gobuckify/lib/golist.go @@ -40,7 +40,11 @@ type Package struct { Module *Module } -func QueryGoList(workDir, rootModuleName, goOS, goArch string, extraArgs ...string) (chan *Package, chan error) { +func QueryGoList( + workDir, rootModuleName, goOS, goArch string, + cgoEnabled bool, + extraArgs ...string, +) (chan *Package, chan error) { pkgChan := make(chan *Package, 1000) // 1000 is a guess, but should be enough errChan := make(chan error, 1) go func() { @@ -56,9 +60,12 @@ func QueryGoList(workDir, rootModuleName, goOS, goArch string, extraArgs ...stri // filename suffixes (e.g. `*_darwin.go`) and `//go:build` expressions (e.g. // `!linux`). Passing the platform only via `-tags` is insufficient -- the host's // own GOOS/GOARCH tags stay set, so negative and file-suffix constraints are - // mis-evaluated and platform-specific deps get silently dropped. CGO_ENABLED is - // forced on for consistent behaviour across host machines. - cmd.Env = append(os.Environ(), "CGO_ENABLED=1", "GOOS="+goOS, "GOARCH="+goArch) + // mis-evaluated and platform-specific deps get silently dropped. + cgoEnabledValue := "0" + if cgoEnabled { + cgoEnabledValue = "1" + } + cmd.Env = append(os.Environ(), "CGO_ENABLED="+cgoEnabledValue, "GOOS="+goOS, "GOARCH="+goArch) stdout, err := cmd.StdoutPipe() if err != nil { @@ -135,46 +142,46 @@ type CollectPackagesResult struct { Modules map[string]*Module } -// CollectPackages queries go list for all platforms and collects packages into BuckTargets and unique Modules +// CollectPackages queries go list for all platform and CGO configurations and +// collects packages into BuckTargets and unique Modules. func CollectPackages(cfg *Config, thirdPartyDir, rootModuleName string) (*CollectPackagesResult, error) { type result struct { - pkg *Package - buckOS string - buckArch string + pkg *Package + buckOS string + buckArch string + cgoEnabled bool } - results := make(chan *result, 1000*len(cfg.Platforms)) + results := make(chan *result, 2*1000*len(cfg.Platforms)) mainErrChan := make(chan error) // Limit concurrency to avoid OOMs as `go list` can use a lot of memory - maxConcurrency := 10 - semaphore := make(chan struct{}, maxConcurrency) + semaphore := make(chan struct{}, 10) - wg := sync.WaitGroup{} + var wg sync.WaitGroup for _, p := range cfg.Platforms { - wg.Add(1) - go func() { - defer wg.Done() - - semaphore <- struct{}{} - defer func() { <-semaphore }() - - // GOOS/GOARCH are passed via the environment (see QueryGoList), not as - // build tags, so `go list` evaluates each platform's constraints correctly. - pkgChan, errChan := QueryGoList( - thirdPartyDir, rootModuleName, p.GoOS, p.GoArch, - fmt.Sprintf("-tags=%s", strings.Join(cfg.DefaultTags, ",")), - ) - pkgCount := 0 - for pkg := range pkgChan { - pkgCount++ - results <- &result{pkg: pkg, buckOS: p.BuckOS, buckArch: p.BuckArch} - } - slog.Info("Found packages", "count", pkgCount, "os", p.GoOS, "arch", p.GoArch) - for err := range errChan { - mainErrChan <- fmt.Errorf("error querying golist for %s: %w", p, err) - } - }() + for _, cgoEnabled := range []bool{false, true} { + wg.Go(func() { + semaphore <- struct{}{} + defer func() { <-semaphore }() + + // GOOS/GOARCH and CGO_ENABLED are passed via the environment (see + // QueryGoList), so `go list` evaluates each configuration correctly. + pkgChan, errChan := QueryGoList( + thirdPartyDir, rootModuleName, p.GoOS, p.GoArch, cgoEnabled, + fmt.Sprintf("-tags=%s", strings.Join(cfg.DefaultTags, ",")), + ) + pkgCount := 0 + for pkg := range pkgChan { + pkgCount++ + results <- &result{pkg: pkg, buckOS: p.BuckOS, buckArch: p.BuckArch, cgoEnabled: cgoEnabled} + } + slog.Info("Found packages", "count", pkgCount, "os", p.GoOS, "arch", p.GoArch, "cgo_enabled", cgoEnabled) + for err := range errChan { + mainErrChan <- fmt.Errorf("error querying golist for %v with cgo_enabled=%t: %w", p, cgoEnabled, err) + } + }) + } } go func() { @@ -197,7 +204,7 @@ func CollectPackages(cfg *Config, thirdPartyDir, rootModuleName string) (*Collec resultsClosed = true continue } - buckTargets.AddPackage(res.pkg, res.buckOS, res.buckArch) + buckTargets.AddPackage(res.pkg, res.buckOS, res.buckArch, res.cgoEnabled) if res.pkg.Module != nil { modules[res.pkg.Module.Path] = res.pkg.Module } diff --git a/prelude/go/tools/gobuckify/lib/render.go b/prelude/go/tools/gobuckify/lib/render.go index be29214..54258ba 100644 --- a/prelude/go/tools/gobuckify/lib/render.go +++ b/prelude/go/tools/gobuckify/lib/render.go @@ -46,15 +46,28 @@ var TargetTemplate = ` "{{ $os }}": select({ "DEFAULT": [], {{- range $arch, $archDeps := $osDeps.ArchDeps }} - "{{ $arch }}": [ - {{- range $archDeps.Deps.SortedList }} + "{{ $arch }}": {{ if $archDeps.CommonDeps }}[ + {{- range $archDeps.CommonDeps.SortedList }} {{- if $.Config.DepsTargetLabelPrefix }} "{{TargetLabelFromImportPath $.Config.DepsTargetLabelPrefix .}}", {{- else }} "{{.}}", {{- end }} {{- end }} - ], + ]{{ else }}select({ + "DEFAULT": [], + {{- range $cgo, $deps := $archDeps.CgoDeps }} + "{{ $cgo }}": [ + {{- range $deps.SortedList }} + {{- if $.Config.DepsTargetLabelPrefix }} + "{{TargetLabelFromImportPath $.Config.DepsTargetLabelPrefix .}}", + {{- else }} + "{{.}}", + {{- end }} + {{- end }} + ], + {{- end }} + }){{ end }}, {{- end }} }), {{- end }} @@ -70,11 +83,16 @@ var TargetTemplate = ` {{- if .Target.TargetCompatibleWith }} target_compatible_with = select({ "DEFAULT": ["config//:none"], - {{- range $os, $archList := .Target.TargetCompatibleWith }} + {{- range $os, $archMap := .Target.TargetCompatibleWith }} "{{ $os }}": select({ "DEFAULT": ["config//:none"], - {{- range $archList }} - "{{.}}": [], + {{- range $arch, $cgoConstraints := $archMap }} + "{{ $arch }}": select({ + "DEFAULT": ["config//:none"], + {{- range $cgoConstraints.SortedList }} + "{{.}}": [], + {{- end }} + }), {{- end }} }), {{- end }} @@ -97,14 +115,11 @@ func RenderBuckFiles(cfg *Config, thirdPartyDir string, buckTargets BuckTargets) tmpl1 = template.Must(tmpl1.Parse(TargetTemplate)) errors := make(chan error) - wg := sync.WaitGroup{} + var wg sync.WaitGroup sem := make(chan struct{}, 50) // limit IO concurrency to some reasonable number for _, target := range buckTargets { - wg.Add(1) - go func() { - defer wg.Done() - - target.Normalise(len(cfg.Platforms)) + wg.Go(func() { + target.Normalise(2 * len(cfg.Platforms)) // Acquire semaphore before doing IO sem <- struct{}{} @@ -121,7 +136,7 @@ func RenderBuckFiles(cfg *Config, thirdPartyDir string, buckTargets BuckTargets) if err := tmpl1.Execute(f, TemplateData{Config: cfg.Buck, Target: target}); err != nil { errors <- fmt.Errorf("can't execute template: %w", err) } - }() + }) } go func() { diff --git a/prelude/go/tools/gobuckify/lib/render_test.go b/prelude/go/tools/gobuckify/lib/render_test.go index 3a76944..02cb1e6 100644 --- a/prelude/go/tools/gobuckify/lib/render_test.go +++ b/prelude/go/tools/gobuckify/lib/render_test.go @@ -11,7 +11,6 @@ package gobuckifylib import ( - "bytes" "strings" "testing" "text/template" @@ -23,7 +22,6 @@ func TestTargetTemplate(t *testing.T) { config BuckConfig target *BuckTarget expectedOutput string - notExpected []string }{ { name: "binary target with common deps and no target label prefix", @@ -59,25 +57,19 @@ go_binary( ], visibility = ["PUBLIC"], )`, - notExpected: []string{ - "go_library", - "target_compatible_with", - }, }, { - name: "library target with platform deps", + name: "library target with platform and cgo deps", config: BuckConfig{ Preambule: "# Generated by gobuckify\n", LoadGoLibraryRule: "load(\":defs.bzl\", \"go_library\")\n", GoLibraryRule: "go_library", DepsAttr: "deps", DepsTargetLabelPrefix: "//third-party/go/", - GenerateEmbedSrcs: false, }, target: &BuckTarget{ Name: "test_lib", ImportPath: "github.com/example/test_lib", - IsBinary: false, CommonDeps: []string{"github.com/example/common_dep"}, PlatformDeps: map[string]*OSDeps{ "linux": { @@ -85,11 +77,14 @@ go_binary( ArchDeps: map[string]*ArchDeps{ "x86_64": { Arch: "x86_64", - Deps: func() *StringSet { - s := NewSet() - s.Add("github.com/example/linux_dep") - return s - }(), + CgoDeps: map[string]*StringSet{ + "prelude//go/constraints:cgo_enabled[false]": NewFromList([]string{ + "github.com/example/nocgo_dep", + }), + "prelude//go/constraints:cgo_enabled[true]": NewFromList([]string{ + "github.com/example/cgo_dep", + }), + }, }, }, }, @@ -109,17 +104,19 @@ go_library( "DEFAULT": [], "linux": select({ "DEFAULT": [], - "x86_64": [ - "//third-party/go/github.com/example/linux_dep:linux_dep", - ], + "x86_64": select({ + "DEFAULT": [], + "prelude//go/constraints:cgo_enabled[false]": [ + "//third-party/go/github.com/example/nocgo_dep:nocgo_dep", + ], + "prelude//go/constraints:cgo_enabled[true]": [ + "//third-party/go/github.com/example/cgo_dep:cgo_dep", + ], + }), }), }), visibility = ["PUBLIC"], )`, - notExpected: []string{ - "go_binary", - "embed_srcs", - }, }, { name: "library target with no deps", @@ -127,12 +124,10 @@ go_library( Preambule: "# Generated by gobuckify\n", LoadGoLibraryRule: "load(\":defs.bzl\", \"go_library\")\n", GoLibraryRule: "go_library", - DepsAttr: "deps", }, target: &BuckTarget{ Name: "simple_lib", ImportPath: "github.com/example/simple_lib", - IsBinary: false, }, expectedOutput: ` # Generated by gobuckify @@ -144,14 +139,9 @@ go_library( header_namespace = "", visibility = ["PUBLIC"], )`, - notExpected: []string{ - "deps =", - "embed_srcs", - "target_compatible_with", - }, }, { - name: "library target with platform deps only", + name: "library target with multiple platforms and cgo compatibility", config: BuckConfig{ Preambule: "# Generated by gobuckify\n", LoadGoLibraryRule: "load(\":defs.bzl\", \"go_library\")\n", @@ -162,27 +152,21 @@ go_library( target: &BuckTarget{ Name: "platform_lib", ImportPath: "github.com/example/platform_lib", - IsBinary: false, - CommonDeps: []string{}, // No common deps PlatformDeps: map[string]*OSDeps{ "linux": { OS: "linux", ArchDeps: map[string]*ArchDeps{ "x86_64": { - Arch: "x86_64", - Deps: func() *StringSet { - s := NewSet() - s.Add("github.com/example/linux_x86_dep") - return s - }(), + Arch: "x86_64", + CommonDeps: NewFromList([]string{"github.com/example/linux_x86_dep"}), }, "arm64": { Arch: "arm64", - Deps: func() *StringSet { - s := NewSet() - s.Add("github.com/example/linux_arm_dep") - return s - }(), + CgoDeps: map[string]*StringSet{ + "prelude//go/constraints:cgo_enabled[true]": NewFromList([]string{ + "github.com/example/linux_arm_dep", + }), + }, }, }, }, @@ -191,18 +175,23 @@ go_library( ArchDeps: map[string]*ArchDeps{ "x86_64": { Arch: "x86_64", - Deps: func() *StringSet { - s := NewSet() - s.Add("github.com/example/darwin_dep") - return s - }(), + CgoDeps: map[string]*StringSet{ + "prelude//go/constraints:cgo_enabled[true]": NewFromList([]string{ + "github.com/example/darwin_dep", + }), + }, }, }, }, }, - TargetCompatibleWith: map[string][]string{ - "linux": {"x86_64", "arm64"}, - "darwin": {"x86_64"}, + TargetCompatibleWith: map[string]map[string]*StringSet{ + "linux": { + "x86_64": NewFromList([]string{"prelude//go/constraints:cgo_enabled[true]"}), + "arm64": NewFromList([]string{"prelude//go/constraints:cgo_enabled[true]"}), + }, + "darwin": { + "x86_64": NewFromList([]string{"prelude//go/constraints:cgo_enabled[true]"}), + }, }, }, expectedOutput: ` @@ -218,15 +207,21 @@ go_library( "DEFAULT": [], "darwin": select({ "DEFAULT": [], - "x86_64": [ - "//third-party/go/github.com/example/darwin_dep:darwin_dep", - ], + "x86_64": select({ + "DEFAULT": [], + "prelude//go/constraints:cgo_enabled[true]": [ + "//third-party/go/github.com/example/darwin_dep:darwin_dep", + ], + }), }), "linux": select({ "DEFAULT": [], - "arm64": [ - "//third-party/go/github.com/example/linux_arm_dep:linux_arm_dep", - ], + "arm64": select({ + "DEFAULT": [], + "prelude//go/constraints:cgo_enabled[true]": [ + "//third-party/go/github.com/example/linux_arm_dep:linux_arm_dep", + ], + }), "x86_64": [ "//third-party/go/github.com/example/linux_x86_dep:linux_x86_dep", ], @@ -236,36 +231,36 @@ go_library( "DEFAULT": ["config//:none"], "darwin": select({ "DEFAULT": ["config//:none"], - "x86_64": [], + "x86_64": select({ + "DEFAULT": ["config//:none"], + "prelude//go/constraints:cgo_enabled[true]": [], + }), }), "linux": select({ "DEFAULT": ["config//:none"], - "x86_64": [], - "arm64": [], + "arm64": select({ + "DEFAULT": ["config//:none"], + "prelude//go/constraints:cgo_enabled[true]": [], + }), + "x86_64": select({ + "DEFAULT": ["config//:none"], + "prelude//go/constraints:cgo_enabled[true]": [], + }), }), }), visibility = ["PUBLIC"], )`, - notExpected: []string{ - "go_binary", - "embed_srcs", - }, }, } + tmpl := template.Must(template.New("test").Funcs(template.FuncMap{ + "TargetLabelFromImportPath": TargetLabelFromImportPath, + }).Parse(TargetTemplate)) + for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - tmpl := template.New("test") - tmpl.Funcs(template.FuncMap{ - "TargetLabelFromImportPath": TargetLabelFromImportPath, - }) - tmpl, err := tmpl.Parse(TargetTemplate) - if err != nil { - t.Fatalf("Failed to parse template: %v", err) - } - - var buf bytes.Buffer - err = tmpl.Execute(&buf, TemplateData{ + var buf strings.Builder + err := tmpl.Execute(&buf, TemplateData{ Config: tt.config, Target: tt.target, }) @@ -273,18 +268,8 @@ go_library( t.Fatalf("Template execution failed: %v", err) } - result := buf.String() - - // Check for expected output - if !strings.Contains(result, strings.TrimSpace(tt.expectedOutput)) { - t.Errorf("Expected output to contain %q, but it didn't.\nOutput:\n%s", tt.expectedOutput, result) - } - - // Check for not expected output - for _, notExpected := range tt.notExpected { - if strings.Contains(result, notExpected) { - t.Errorf("Expected output to NOT contain %q, but it did.\nOutput:\n%s", notExpected, result) - } + if got, want := strings.TrimSpace(buf.String()), strings.TrimSpace(tt.expectedOutput); got != want { + t.Errorf("Template output mismatch.\nExpected:\n%s\n\nGot:\n%s", want, got) } }) } diff --git a/prelude/go/tools/gopackagesdriver/driver/buck.go b/prelude/go/tools/gopackagesdriver/driver/buck.go index e126d33..e264332 100644 --- a/prelude/go/tools/gopackagesdriver/driver/buck.go +++ b/prelude/go/tools/gopackagesdriver/driver/buck.go @@ -20,7 +20,6 @@ import ( "log/slog" "os" "os/exec" - "path/filepath" "regexp" "slices" "strings" @@ -59,26 +58,9 @@ func (s shellCommander) Exec(ctx context.Context, name string, args ...string) ( var lineRe = regexp.MustCompile(`//line\s+(.+):1:1`) -// Makes relative path absolute, if it is not already -// The compiler and golang.org/x/tools handle relative paths differently -// We have to put this hack somewhenre until it fixed -// See https://github.com/golang/go/issues/70478 -func fixupRelPathLine(projectDir, line string) string { - if filepath.IsAbs(line) { - return line - } - - newPath := filepath.Join(projectDir, line) - slog.Debug("fixed path for CGo file", "old", line, "new", newPath) - - return newPath -} - -// fixRePath updates lines like -// "//line fbcode/third-party-go/vendor/github.com/aquasecurity/libbpfgo/libbpfgo.go:1:1" -// to "//line /home/user1/fbsource/fbcode/third-party-go/vendor/github.com/aquasecurity/libbpfgo/libbpfgo.go:1:1" -// to proper full path to local checkout -func fixRePath(platform Platform, file string) error { +// fixRePath replaces CGo line directives with the absolute source path reported +// by go/packages. +func fixRePath(file, absSrcPath string) error { content, err := os.ReadFile(file) if err != nil { return err @@ -88,25 +70,21 @@ func fixRePath(platform Platform, file string) error { scanner := bufio.NewScanner(bytes.NewReader(content)) for scanner.Scan() { - l := scanner.Text() - pp := lineRe.FindStringSubmatch(l) - if len(pp) == 2 { - slog.Debug("fixing up", "file", file, "line", pp[1]) - nl := fixupRelPathLine(platform.ProjectDir(), pp[1]) - if nl != "" { - fmt.Fprintf(buf, "//line %s:1:1\n", nl) - } else { - slog.Warn("unsuccessful fixup", "line", l) - buf.WriteString(l + "\n") - } + origLine := scanner.Text() + matches := lineRe.FindStringSubmatch(origLine) + if len(matches) == 2 { + srcPath := matches[1] + slog.Debug("fixing CGo source path", "file", file, "old", srcPath, "new", absSrcPath) + fmt.Fprintf(buf, "//line %s:1:1\n", absSrcPath) } else { - buf.WriteString(l + "\n") + buf.WriteString(origLine) + buf.WriteString("\n") } } if err = scanner.Err(); err != nil { return err } - dst, err := os.OpenFile(file, os.O_RDWR, 0644) + dst, err := os.OpenFile(file, os.O_WRONLY|os.O_TRUNC, 0644) if err != nil { return err } diff --git a/prelude/go/tools/gopackagesdriver/driver/buck_test.go b/prelude/go/tools/gopackagesdriver/driver/buck_test.go index b82b9d7..42159eb 100644 --- a/prelude/go/tools/gopackagesdriver/driver/buck_test.go +++ b/prelude/go/tools/gopackagesdriver/driver/buck_test.go @@ -12,8 +12,14 @@ package driver import ( "context" - "fmt" + "maps" + "os" + "path/filepath" + "slices" + "strings" "testing" + + "golang.org/x/tools/go/packages" ) // requireNoError fails the test if err is not nil @@ -66,31 +72,106 @@ func TestBuckRoot(t *testing.T) { requireEqual(t, "/data/users/user1/fbcode", root) } -func TestFixupRelPathLine(t *testing.T) { - projectDir := "/home/user1/repo_root" - tcs := []struct { - name string - in string - want string +func TestFixRePathTruncatesFile(t *testing.T) { + path := filepath.Join(t.TempDir(), "generated.go") + original := "//line ../../../../long/path/to/source.go:1:1\npackage example\n" + requireNoError(t, os.WriteFile(path, []byte(original), 0644)) + + requireNoError(t, fixRePath(path, "/src/source.go")) + + got, err := os.ReadFile(path) + requireNoError(t, err) + want := "//line /src/source.go:1:1\npackage example\n" + requireEqual(t, want, string(got)) +} + +func TestCgoPathRewrites(t *testing.T) { + testCases := []struct { + name string + pkg *packages.Package + want map[string]string + wantErr string }{ { - name: "already absolute path", - in: "/home/user1/repo_root/foo/bar/baz.go", - want: "/home/user1/repo_root/foo/bar/baz.go", + name: "checked-in CGo source", + pkg: &packages.Package{ + ID: "cgo", + GoFiles: []string{"/repo/plain.go", "/repo/cgo_file.go"}, + CompiledGoFiles: []string{"/buck-out/cgo_gen/cgo_file.cgo1.go"}, + }, + want: map[string]string{ + "/buck-out/cgo_gen/cgo_file.cgo1.go": "/repo/cgo_file.go", + }, + }, + { + name: "generated SWIG source", + pkg: &packages.Package{ + ID: "swig", + GoFiles: []string{"/buck-out/swig/out/swig_lib.go"}, + CompiledGoFiles: []string{"/buck-out/cgo_gen/swig_lib.cgo1.go"}, + }, + want: map[string]string{ + "/buck-out/cgo_gen/swig_lib.cgo1.go": "/buck-out/swig/out/swig_lib.go", + }, + }, + { + name: "ignore CGo types file", + pkg: &packages.Package{ + ID: "cgo", + CompiledGoFiles: []string{"/buck-out/cgo_gen/_cgo_gotypes.go"}, + }, + }, + { + name: "missing source", + pkg: &packages.Package{ + ID: "cgo", + GoFiles: []string{"/repo/other.go"}, + CompiledGoFiles: []string{"/buck-out/cgo_gen/cgo_file.cgo1.go"}, + }, + wantErr: "found 0", + }, + { + name: "ambiguous source", + pkg: &packages.Package{ + ID: "cgo", + GoFiles: []string{"/repo/one/cgo_file.go", "/repo/two/cgo_file.go"}, + CompiledGoFiles: []string{"/buck-out/cgo_gen/cgo_file.cgo1.go"}, + }, + wantErr: "found 2", }, { - name: "existing path", - in: "foo/bar/baz.go", - want: "/home/user1/repo_root/foo/bar/baz.go", + name: "relative source", + pkg: &packages.Package{ + ID: "cgo", + GoFiles: []string{"cgo_file.go"}, + CompiledGoFiles: []string{"/buck-out/cgo_gen/cgo_file.cgo1.go"}, + }, + wantErr: "is not absolute", }, } - for _, tc := range tcs { - t.Run(fmt.Sprintf("%s with %q", tc.name, tc.in), func(t *testing.T) { - requireEqual(t, tc.want, fixupRelPathLine(projectDir, tc.in)) + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + got, err := cgoPathRewrites(tc.pkg) + if tc.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("Expected error containing %q, got %v", tc.wantErr, err) + } + return + } + requireNoError(t, err) + requireEqual(t, true, maps.Equal(tc.want, got)) }) } } +func TestBuildBXLArgsRequestsGoFilesForCompiledGoFiles(t *testing.T) { + req := &packages.DriverRequest{Mode: packages.NeedCompiledGoFiles} + args := buildBXLArgs(req, nil, nil) + requireEqual(t, true, slices.Contains(args, "--need_files")) + requireEqual(t, true, slices.Contains(args, "--need_compiled_go_files")) +} + func TestFixQuery(t *testing.T) { testCases := []struct { name string diff --git a/prelude/go/tools/gopackagesdriver/driver/driver.go b/prelude/go/tools/gopackagesdriver/driver/driver.go index 2a761aa..3632bae 100644 --- a/prelude/go/tools/gopackagesdriver/driver/driver.go +++ b/prelude/go/tools/gopackagesdriver/driver/driver.go @@ -68,7 +68,7 @@ func query(ctx context.Context, req *packages.DriverRequest, bucker Bucker, plat return nil, err } - resp, err = queryBXL(ctx, req, bucker, platform, targetsByType.buckPatterns, targetsByType.buckFiles) + resp, err = queryBXL(ctx, req, bucker, targetsByType.buckPatterns, targetsByType.buckFiles) if err != nil { slog.Error("error when query BXL", "args", os.Args, "err", err) return nil, err diff --git a/prelude/go/tools/gopackagesdriver/driver/query_bxl.go b/prelude/go/tools/gopackagesdriver/driver/query_bxl.go index eb23055..e4d6c47 100644 --- a/prelude/go/tools/gopackagesdriver/driver/query_bxl.go +++ b/prelude/go/tools/gopackagesdriver/driver/query_bxl.go @@ -14,9 +14,13 @@ import ( "context" "encoding/json" "errors" + "fmt" "log/slog" + "maps" "os" "os/exec" + "path/filepath" + "slices" "strings" "golang.org/x/tools/go/packages" @@ -27,11 +31,16 @@ import ( // Ideally we should not do this, but it's the only way to support cgo in gopls const LoadModeTypecheckCgo = packages.LoadMode(4096) -const cgoGoTypesFileName = "_cgo_gotypes.go" const cgoGenFileNameExt = ".cgo1.go" // queryBXL is a wrapper around query that will use BXL to resolve the targets -func queryBXL(ctx context.Context, req *packages.DriverRequest, bucker Bucker, platform Platform, patterns []string, files []string) (*packages.DriverResponse, error) { +func queryBXL( + ctx context.Context, + req *packages.DriverRequest, + bucker Bucker, + patterns []string, + files []string, +) (*packages.DriverResponse, error) { if len(patterns) == 0 && len(files) == 0 { return &packages.DriverResponse{}, nil } @@ -80,18 +89,59 @@ func queryBXL(ctx context.Context, req *packages.DriverRequest, bucker Bucker, p return nil, err } - // Fix CGO paths in compiled Go files + rewrites := make(map[string]string) for _, pkg := range response.Packages { - for _, file := range pkg.CompiledGoFiles { - if strings.HasSuffix(file, cgoGoTypesFileName) || strings.HasSuffix(file, cgoGenFileNameExt) { - _ = fixRePath(platform, file) + pkgRewrites, err := cgoPathRewrites(pkg) + if err != nil { + return nil, err + } + for generatedFile, sourceFile := range pkgRewrites { + if _, ok := rewrites[generatedFile]; ok { + return nil, fmt.Errorf("duplicate CGo path rewrite for %q", generatedFile) } + rewrites[generatedFile] = sourceFile + } + } + for generatedFile, sourceFile := range rewrites { + if err := fixRePath(generatedFile, sourceFile); err != nil { + return nil, fmt.Errorf("rewrite CGo source path for %q: %w", generatedFile, err) } } return &response, nil } +func cgoPathRewrites(pkg *packages.Package) (map[string]string, error) { + sourcesByBase := make(map[string]map[string]struct{}) + for _, sourceFile := range pkg.GoFiles { + sourceBase := filepath.Base(sourceFile) + if sourcesByBase[sourceBase] == nil { + sourcesByBase[sourceBase] = make(map[string]struct{}) + } + sourcesByBase[sourceBase][sourceFile] = struct{}{} + } + + rewrites := make(map[string]string) + for _, generatedFile := range pkg.CompiledGoFiles { + generatedBase := filepath.Base(generatedFile) + if !strings.HasSuffix(generatedBase, cgoGenFileNameExt) { + continue + } + + sourceBase := strings.TrimSuffix(generatedBase, cgoGenFileNameExt) + ".go" + matches := sourcesByBase[sourceBase] + if len(matches) != 1 { + return nil, fmt.Errorf("package %q: expected exactly one GoFiles source named %q for %q, found %d", pkg.ID, sourceBase, generatedFile, len(matches)) + } + sourceFile := slices.Collect(maps.Keys(matches))[0] + if !filepath.IsAbs(sourceFile) { + return nil, fmt.Errorf("package %q: GoFiles source %q for %q is not absolute", pkg.ID, sourceFile, generatedFile) + } + rewrites[generatedFile] = sourceFile + } + return rewrites, nil +} + func buildBXLArgs(req *packages.DriverRequest, patterns []string, files []string) []string { bxlArgs := []string{} if len(patterns) > 0 { @@ -108,15 +158,16 @@ func buildBXLArgs(req *packages.DriverRequest, patterns []string, files []string if req.Mode&packages.NeedName != 0 { bxlArgs = append(bxlArgs, "--need_name", "true") } - // we have to implicitely add some flags if syntax/types requested + // we have to implicitly add some flags if syntax/types requested // same as `go list` driver does // todo: add packages.NeedTypesSizes to the query as well, when we fix slow builds // as gopls requires it, but it can't wait very long needTypes := packages.NeedTypes | packages.NeedTypesInfo - if req.Mode&(packages.NeedFiles|packages.NeedSyntax|needTypes) != 0 { + needCompiledGoFiles := packages.NeedCompiledGoFiles | packages.NeedSyntax | needTypes + if req.Mode&(packages.NeedFiles|needCompiledGoFiles) != 0 { bxlArgs = append(bxlArgs, "--need_files", "true") } - if req.Mode&(packages.NeedCompiledGoFiles|packages.NeedSyntax|needTypes) != 0 { + if req.Mode&needCompiledGoFiles != 0 { bxlArgs = append(bxlArgs, "--need_compiled_go_files", "true") } if req.Mode&(packages.NeedImports|needTypes) != 0 { diff --git a/prelude/graphql/graphql.bzl b/prelude/graphql/graphql.bzl index 3fe7729..6ee4999 100644 --- a/prelude/graphql/graphql.bzl +++ b/prelude/graphql/graphql.bzl @@ -20,6 +20,7 @@ load( GraphQLiOSInfo = record( header_path_prefix = str | None, force_export_fragment_headers = bool, + fragment_models_use_classes = bool, use_implicit_codegen = bool, ) diff --git a/prelude/java/dex.bzl b/prelude/java/dex.bzl index c205a4d..77dc435 100644 --- a/prelude/java/dex.bzl +++ b/prelude/java/dex.bzl @@ -44,7 +44,6 @@ def get_dex_produced_from_java_library( needs_desugar: bool = False, desugar_deps: [TransitiveSetArgsProjection, None] = None, weight_factor: int = 1, - desugar_deps_file: Artifact | None = None, ) -> DexLibraryInfo: d8_cmd = cmd_args(dex_toolchain.d8_command[RunInfo]) @@ -60,18 +59,8 @@ def get_dex_produced_from_java_library( if not needs_desugar: d8_cmd.add("--no-desugar") else: - # Callers that dex many jars against one shared classpath pass desugar_deps_file so the - # list is written once instead of once per jar; writing it here would be quadratic in the - # number of jars. - # - # Invariant: desugar_deps_file only carries the jar *paths*. The jar artifacts are declared - # as action inputs via the hidden cmd_args below, sourced from desugar_deps. A caller that - # passes desugar_deps_file must therefore also pass desugar_deps, otherwise the jars are - # left untracked and may be missing when d8 runs. (Alternatively a caller could write the - # file with ctx.actions.write(..., with_inputs = True) so the artifacts ride along with the - # file, but no current caller does this.) - classpath_file = desugar_deps_file or ctx.actions.write(prefix + "_desugar_deps_file.txt", desugar_deps or [], has_content_based_path = True) - d8_cmd.add(["--classpath-files", classpath_file]) + desugar_deps_file = ctx.actions.write(prefix + "_desugar_deps_file.txt", desugar_deps or [], has_content_based_path = True) + d8_cmd.add(["--classpath-files", desugar_deps_file]) d8_cmd.add(cmd_args(hidden = desugar_deps or [])) referenced_resources_file = ctx.actions.declare_output(prefix + "_referenced_resources.txt", has_content_based_path = True) diff --git a/prelude/java/gwt_binary.bzl b/prelude/java/gwt_binary.bzl index a0a9c07..7ca3eb5 100644 --- a/prelude/java/gwt_binary.bzl +++ b/prelude/java/gwt_binary.bzl @@ -11,8 +11,8 @@ load("@prelude//java/utils:java_more_utils.bzl", "get_path_separator_for_exec_os load("@prelude//utils:expect.bzl", "expect") load( ":java_providers.bzl", - "derive_compiling_deps", "get_all_java_packaging_deps", + "get_compiling_deps_tset", ) _GWT_COMPILER_CLASS = "com.google.gwt.dev.Compiler" @@ -26,7 +26,7 @@ def gwt_binary_impl(ctx: AnalysisContext) -> list[Provider]: deploy_output = ctx.actions.declare_output("deploy", has_content_based_path = False) module_deps_classpath = [dep.gwt_module for dep in get_all_java_packaging_deps(ctx, ctx.attrs.module_deps) if dep.gwt_module] - compiling_deps_tset = derive_compiling_deps(ctx.actions, None, ctx.attrs.deps) + compiling_deps_tset = get_compiling_deps_tset(ctx.actions, ctx.attrs.deps) deps_classpath = [dep.full_library for dep in (list(compiling_deps_tset.traverse()) if compiling_deps_tset else [])] java_toolchain = ctx.attrs._java_toolchain[JavaToolchainInfo] diff --git a/prelude/java/java.bzl b/prelude/java/java.bzl index c19fd0a..74d97e4 100644 --- a/prelude/java/java.bzl +++ b/prelude/java/java.bzl @@ -13,7 +13,6 @@ load(":jar_genrule.bzl", "jar_genrule_impl") load(":java_binary.bzl", "java_binary_impl") load(":java_library.bzl", "java_library_impl") load(":java_test.bzl", "java_test_impl") -load(":keystore.bzl", "keystore_impl") load(":prebuilt_jar.bzl", "prebuilt_jar_impl") implemented_rules = { @@ -24,6 +23,5 @@ implemented_rules = { "java_library": java_library_impl, "java_plugin": java_plugin_impl, "java_test": java_test_impl, - "keystore": keystore_impl, "prebuilt_jar": prebuilt_jar_impl, } diff --git a/prelude/java/java_binary.bzl b/prelude/java/java_binary.bzl index 08acf73..8c7831d 100644 --- a/prelude/java/java_binary.bzl +++ b/prelude/java/java_binary.bzl @@ -20,7 +20,7 @@ load("@prelude//utils:expect.bzl", "expect") load( ":java_providers.bzl", "create_template_info", - "derive_compiling_deps", + "get_compiling_deps_tset", "get_java_packaging_info", ) @@ -171,7 +171,7 @@ def java_binary_impl(ctx: AnalysisContext) -> list[Provider]: packaging_info = get_java_packaging_info(ctx, ctx.attrs.deps, None) - first_order_deps = derive_compiling_deps(ctx.actions, None, ctx.attrs.deps) + first_order_deps = get_compiling_deps_tset(ctx.actions, ctx.attrs.deps) first_order_libs = [dep.full_library for dep in (list(first_order_deps.traverse()) if first_order_deps else [])] shared_library_info = merge_shared_libraries( diff --git a/prelude/java/java_library.bzl b/prelude/java/java_library.bzl index d6ddca2..40d677a 100644 --- a/prelude/java/java_library.bzl +++ b/prelude/java/java_library.bzl @@ -22,8 +22,8 @@ load( "create_abi", "create_java_library_providers", "create_native_providers", - "derive_compiling_deps", "generate_java_classpath_snapshot", + "get_compiling_deps_tset", "make_compile_outputs", "to_list", ) @@ -52,6 +52,8 @@ load( load("@prelude//jvm:cd_jar_creator_util.bzl", "postprocess_jar") load("@prelude//jvm:nullsafe.bzl", "get_nullsafe_info") load("@prelude//linking:shared_libraries.bzl", "SharedLibraryInfo") +load("@prelude//target_stats:target_stats.bzl", "CycleMode", "target_stats_providers_and_subtargets") +load("@prelude//target_stats:target_stats_config.bzl", "TARGET_STATS_ENABLED") load("@prelude//utils:expect.bzl", "expect") load("@prelude//utils:label_provider.bzl", "LabelInfo") @@ -165,7 +167,7 @@ def _build_classpath( classpath_args_projection: str, additional_classpath_entries_list: list[Artifact], ) -> [cmd_args, None]: - compiling_deps_tset = derive_compiling_deps(actions, None, deps) + compiling_deps_tset = get_compiling_deps_tset(actions, deps) if additional_classpath_entries or compiling_deps_tset or additional_classpath_entries_list: args = cmd_args() @@ -599,6 +601,28 @@ def _check_exported_deps(exported_deps: list[Dependency], attr_name: str): def _skip_java_library_dep_checks(ctx: AnalysisContext) -> bool: return "skip_buck2_java_library_dep_checks" in ctx.attrs.labels +def jvm_target_stats(ctx: AnalysisContext) -> (list[Provider], dict[str, list[Provider]]): + """target_stats for a JVM library rule (java_library, kotlin_library). + + JVM sources get package cycles, as android_library does; the tools come from + the java toolchain, which is the one both rules have. A no-op unless the + config is enabled and that toolchain carries the tools. + """ + if not TARGET_STATS_ENABLED: + return [], {} + tools = ctx.attrs._java_toolchain[JavaToolchainInfo].target_stats_tools + if tools == None: + return [], {} + return target_stats_providers_and_subtargets( + ctx, + tools = tools, + srcs = {src.short_path: src for src in ctx.attrs.srcs}, + # Target stats cover the compile graph, including non-packaged provided deps. + deps = (ctx.attrs.deps + ctx.attrs.exported_deps + ctx.attrs.runtime_deps + ctx.attrs.provided_deps + ctx.attrs.exported_provided_deps), + cycle_mode = CycleMode("package"), + module_name = ctx.label.name, + ) + def java_library_impl(ctx: AnalysisContext) -> list[Provider]: """ java_library() rule implementation @@ -635,13 +659,16 @@ def java_library_impl(ctx: AnalysisContext) -> list[Provider]: _check_dep_types(ctx.attrs.exported_provided_deps) _check_dep_types(ctx.attrs.runtime_deps) + target_stats_providers, target_stats_subtargets = jvm_target_stats(ctx) + java_providers = build_java_library( ctx = ctx, srcs = ctx.attrs.srcs, validation_deps_outputs = get_validation_deps_outputs(ctx), + extra_sub_targets = target_stats_subtargets, ) - return to_list(java_providers) + [android_packageable_info] + [LabelInfo(labels = ctx.attrs.labels)] + graphql_providers(ctx) + return to_list(java_providers) + [android_packageable_info] + [LabelInfo(labels = ctx.attrs.labels)] + graphql_providers(ctx) + target_stats_providers def build_java_library( ctx: AnalysisContext, diff --git a/prelude/java/java_providers.bzl b/prelude/java/java_providers.bzl index 3195607..b17c014 100644 --- a/prelude/java/java_providers.bzl +++ b/prelude/java/java_providers.bzl @@ -8,8 +8,9 @@ load( "@prelude//:resources.bzl", - "ResourceInfo", + "ResourceInfo", # @unused Used as a type "gather_resources", + "make_resource_info", ) load("@prelude//java:class_to_srcs.bzl", "JavaClassToSourceMapInfo") load("@prelude//java:dex.bzl", "DexLibraryInfo", "get_dex_produced_from_java_library") @@ -118,10 +119,14 @@ def _abi_snapshot_json(entry: JavaClasspathEntry): def _abi_dirs(entry: JavaClasspathEntry): return entry.abi_as_dir or [] -def _abi_to_abi_dir(entry: JavaClasspathEntry): - if entry.abi_as_dir: - return cmd_args([entry.abi, entry.abi_as_dir], delimiter = " ") - return [] +def _abi_and_dir(entry: JavaClasspathEntry): + return [entry.abi, entry.abi_as_dir or ""] + +def _source_only_abi_jars(entry: JavaClasspathEntry): + return [entry.abi] if entry.required_for_source_only_abi else [] + +def _source_only_abi_and_dir(entry: JavaClasspathEntry): + return _abi_and_dir(entry) if entry.required_for_source_only_abi else [] def _full_library_args(entry: JavaClasspathEntry): return entry.full_library @@ -137,11 +142,13 @@ JavaCompilingDepsTSetWrapper = transitive_set() JavaCompilingDepsTSet = transitive_set( args_projections = { + "abi_and_dir": _abi_and_dir, "abi_dirs": _abi_dirs, - "abi_to_abi_dir": _abi_to_abi_dir, "args_for_ast_dumper": _args_for_ast_dumper, "args_for_compiling": _args_for_compiling, "full_library_args": _full_library_args, + "source_only_abi_and_dir": _source_only_abi_and_dir, + "source_only_abi_jars": _source_only_abi_jars, }, json_projections = { "abi_snapshot_json": _abi_snapshot_json, @@ -245,14 +252,6 @@ JavaPackagingInfo = provider( }, ) -KeystoreInfo = provider( - # @unsorted-dict-items - fields = { - "store": provider_field(Artifact), - "properties": provider_field(Artifact), - }, -) - JavaCompileOutputs = record( full_library = Artifact, class_abi = Artifact | None, @@ -263,7 +262,6 @@ JavaCompileOutputs = record( preprocessed_library = Artifact, incremental_state_dir = Artifact | None, used_jars_json = Artifact | None, - kotlin_classes = Artifact | None, ) JavaProviders = record( @@ -315,7 +313,6 @@ def make_compile_outputs( incremental_state_dir: Artifact | None = None, abi_jar_snapshot: Artifact | None = None, used_jars_json: Artifact | None = None, - kotlin_classes: Artifact | None = None, ) -> JavaCompileOutputs: expect(classpath_abi != None or classpath_abi_dir == None, "A classpath_abi_dir should only be provided if a classpath_abi is provided!") return JavaCompileOutputs( @@ -334,7 +331,6 @@ def make_compile_outputs( preprocessed_library = preprocessed_library, incremental_state_dir = incremental_state_dir, used_jars_json = used_jars_json, - kotlin_classes = kotlin_classes, ) def create_abi(actions: AnalysisActions, class_abi_generator: Dependency, library: Artifact, keepSynthetic: bool = False) -> Artifact: @@ -390,24 +386,24 @@ def generate_java_classpath_snapshot( return output def single_library_compiling_deps(actions: AnalysisActions, library_output: [JavaClasspathEntry, None]) -> [JavaCompilingDepsTSet, None]: - if library_output: - return actions.tset(JavaCompilingDepsTSet, value = library_output) - else: - return None - -# Accumulate deps necessary for compilation, which consist of this library's output and compiling_deps of its exported deps -def derive_compiling_deps(actions: AnalysisActions, library_output: [JavaCompilingDepsTSet, None], children: list[Dependency]) -> [JavaCompilingDepsTSet, None]: - if children: - filtered_children = filter( - None, - [exported_dep.compiling_deps for exported_dep in filter(None, [x.get(JavaLibraryInfo) for x in children])], - ) - children = filtered_children - - if not library_output and not children: + return get_compiling_deps_tset(actions, value = library_output) + +def get_compiling_deps_tset( + actions: AnalysisActions, + deps: list[Dependency] = [], + additional_classpath_entries: list[JavaCompilingDepsTSet] = [], + value: JavaClasspathEntry | None = None, +) -> JavaCompilingDepsTSet | None: + """Create a classpath root with an optional value before its children.""" + children = [info.compiling_deps for info in filter(None, [dep.get(JavaLibraryInfo) for dep in deps]) if info.compiling_deps != None] + children += additional_classpath_entries + if value != None: + return actions.tset(JavaCompilingDepsTSet, value = value, children = children) + if not children: return None - - return actions.tset(JavaCompilingDepsTSet, children = (children or []) + ([library_output] if library_output else [])) + if len(children) == 1: + return children[0] + return actions.tset(JavaCompilingDepsTSet, children = children) def single_library_compiling_deps_wrapper(actions: AnalysisActions, compiling_deps_tset: [JavaCompilingDepsTSet, None]) -> [JavaCompilingDepsTSetWrapper, None]: if compiling_deps_tset: @@ -489,7 +485,7 @@ def get_all_java_packaging_deps_tset( ctx: AnalysisContext, java_packaging_infos: list[JavaPackagingInfo], java_packaging_dep: [JavaPackagingDep, None] = None ) -> [JavaPackagingDepTSet, None]: packaging_deps_kwargs = {} - if java_packaging_dep: + if java_packaging_dep != None: packaging_deps_kwargs["value"] = java_packaging_dep packaging_deps_children = filter(None, [info.packaging_deps for info in java_packaging_infos]) @@ -498,10 +494,14 @@ def get_all_java_packaging_deps_tset( return ctx.actions.tset(JavaPackagingDepTSet, **packaging_deps_kwargs) if packaging_deps_kwargs else None +_EMPTY_JAVA_PACKAGING_INFO = JavaPackagingInfo(packaging_deps = None) + # Accumulate deps necessary for packaging, which consist of all transitive java deps (except provided ones) def get_java_packaging_info(ctx: AnalysisContext, raw_deps: list[Dependency], java_packaging_dep: [JavaPackagingDep, None] = None) -> JavaPackagingInfo: java_packaging_infos = filter(None, [x.get(JavaPackagingInfo) for x in raw_deps]) packaging_deps = get_all_java_packaging_deps_tset(ctx, java_packaging_infos, java_packaging_dep) + if packaging_deps == None: + return _EMPTY_JAVA_PACKAGING_INFO return JavaPackagingInfo(packaging_deps = packaging_deps) def _group_global_code_children_by_name(global_code_infos: list[JavaGlobalCodeInfo]) -> dict[str, list[JavaCompilingDepsTSetWrapper]]: @@ -550,19 +550,22 @@ def get_global_code_info( global_code_infos = filter(None, [x.get(JavaGlobalCodeInfo) for x in packaging_deps]) children_by_name = _group_global_code_children_by_name(global_code_infos) - declared_deps_raw_targets = [declared_dep.label.raw_target() for declared_dep in declared_deps] + declared_deps_raw_targets = set([declared_dep.label.raw_target() for declared_dep in declared_deps]) + raw_target = ctx.label.raw_target() def declared_deps_contains_trigger(deps_triggers: set[TargetLabel]) -> TargetLabel | None: - for declared_deps_raw_target in declared_deps_raw_targets: - if declared_deps_raw_target in deps_triggers: - return declared_deps_raw_target + for trigger in deps_triggers: + if trigger in declared_deps_raw_targets: + return trigger return None global_code_map = {} for name, (config) in global_code_config.items(): - contains_trigger = declared_deps_contains_trigger(config.triggers) - target_is_global_code_dep = ctx.label.raw_target() in config.deps + target_is_global_code_dep = raw_target in config.deps + contains_trigger = None + if declared_deps_raw_targets and not target_is_global_code_dep: + contains_trigger = declared_deps_contains_trigger(config.triggers) if (contains_trigger or target_is_global_code_dep) and config.requires_first_order_classpath: global_code_library_compiling_deps = [] if single_library_dep: @@ -614,8 +617,8 @@ def create_native_providers(ctx: AnalysisContext, label: Label, packaging_deps: ctx.actions, deps = filter(None, [x.get(SharedLibraryInfo) for x in packaging_deps]), ) - cxx_resource_info = ResourceInfo( - resources = gather_resources( + cxx_resource_info = make_resource_info( + gather_resources( label, deps = packaging_deps, ) @@ -709,7 +712,11 @@ def _create_non_template_providers( global_code_config, ) - compiling_deps = derive_compiling_deps(ctx.actions, single_library, exported_deps + exported_provided_deps) + compiling_deps = get_compiling_deps_tset( + ctx.actions, + exported_deps + exported_provided_deps, + [single_library] if single_library else [], + ) return ( JavaLibraryInfo( @@ -765,7 +772,7 @@ def create_java_library_providers( first_order_classpath_deps = filter(None, [x.get(JavaLibraryInfo) for x in declared_deps + exported_deps + runtime_deps]) first_order_classpath_libs = [dep.output_for_classpath_macro for dep in first_order_classpath_deps] - compiling_deps = derive_compiling_deps(ctx.actions, None, declared_deps + exported_deps + provided_deps + exported_provided_deps) + compiling_deps = get_compiling_deps_tset(ctx.actions, declared_deps + exported_deps + provided_deps + exported_provided_deps) desugar_classpath = compiling_deps.project_as_args("full_library_args") if needs_desugar and compiling_deps != None else None library_info, packaging_info, global_code_info, shared_library_info, cxx_resource_info, linkable_graph = _create_non_template_providers( diff --git a/prelude/java/java_test.bzl b/prelude/java/java_test.bzl index f6daf52..82def15 100644 --- a/prelude/java/java_test.bzl +++ b/prelude/java/java_test.bzl @@ -11,7 +11,7 @@ load( "JavaClassToSourceMapInfo", # @unused Used as a type "merge_class_to_source_map_from_jar", ) -load("@prelude//java:java_library.bzl", "build_java_library") +load("@prelude//java:java_library.bzl", "build_java_library", "jvm_target_stats") load("@prelude//java:java_providers.bzl", "JavaLibraryInfo", "JavaPackagingInfo", "get_all_java_packaging_deps_tset") load("@prelude//java:java_toolchain.bzl", "JavaTestToolchainInfo", "JavaToolchainInfo") load("@prelude//java/utils:java_more_utils.bzl", "get_path_separator_for_exec_os") @@ -35,7 +35,12 @@ def java_test_impl(ctx: AnalysisContext) -> list[Provider]: if ctx.attrs._build_only_native_code: return [DefaultInfo()] - java_providers = build_java_library(ctx, ctx.attrs.srcs) + target_stats_providers, target_stats_subtargets = jvm_target_stats(ctx) + java_providers = build_java_library( + ctx, + ctx.attrs.srcs, + extra_sub_targets = target_stats_subtargets, + ) external_runner_test_info = build_junit_test(ctx, java_providers.java_library_info, java_providers.java_packaging_info, java_providers.class_to_src_map) providers = [ @@ -48,6 +53,7 @@ def java_test_impl(ctx: AnalysisContext) -> list[Provider]: ] if java_providers.validation_info: providers.append(java_providers.validation_info) + providers.extend(target_stats_providers) return inject_test_run_info(ctx, external_runner_test_info) + providers diff --git a/prelude/java/java_toolchain.bzl b/prelude/java/java_toolchain.bzl index 3b2481c..235cf02 100644 --- a/prelude/java/java_toolchain.bzl +++ b/prelude/java/java_toolchain.bzl @@ -72,6 +72,7 @@ JavaToolchainInfo = provider( "src_root_elements": provider_field(typing.Any, default = None), "src_root_prefixes": provider_field(typing.Any, default = None), "target_level": provider_field(typing.Any, default = None), + "target_stats_tools": provider_field(typing.Any, default = None), "track_class_usage": provider_field(bool, default = True), "uses_content_based_paths": provider_field(bool, default = True), "zip_scrubber": provider_field(typing.Any, default = None), diff --git a/prelude/java/javacd_jar_creator.bzl b/prelude/java/javacd_jar_creator.bzl index 30fbfad..363a27e 100644 --- a/prelude/java/javacd_jar_creator.bzl +++ b/prelude/java/javacd_jar_creator.bzl @@ -9,10 +9,10 @@ load( "@prelude//java:java_providers.bzl", "ClasspathSnapshotGranularity", - "JavaClasspathEntry", # @unused Used as a type "JavaCompileOutputs", # @unused Used as a type "JavaCompilingDepsTSet", # @unused Used as a type "generate_java_classpath_snapshot", + "get_compiling_deps_tset", "make_compile_outputs", ) load("@prelude//java:java_resources.bzl", "get_resources_map") @@ -46,7 +46,6 @@ load( "encode_command", "generate_abi_jars", "get_abi_generation_mode", - "get_compiling_deps_tset", "prepare_cd_exe", "prepare_final_jar", "setup_dep_files", @@ -106,7 +105,7 @@ def create_jar_artifact_javacd( output_paths = define_output_paths(actions, actions_identifier, label, uses_content_based_paths) - compiling_deps_tset = get_compiling_deps_tset(actions, deps, additional_classpath_entries) + compiling_deps_tset = get_compiling_deps_tset(actions, deps, [additional_classpath_entries] if additional_classpath_entries else []) track_class_usage = java_toolchain.track_class_usage and enable_depfiles define_javacd_action = partial( @@ -117,7 +116,6 @@ def create_jar_artifact_javacd( class_abi_jar, class_abi_output_dir, srcs, - compiling_deps_tset, track_class_usage, debug_port, uses_content_based_paths, @@ -138,13 +136,14 @@ def create_jar_artifact_javacd( abi_generation_mode = abi_generation_mode, resources_map = resources_map, extra_arguments = extra_arguments, + use_abi_dirs = bool(not is_creating_subtarget and srcs and track_class_usage and java_toolchain.dep_files == DepFiles("per_class")), ) command = command_builder( build_mode = BuildMode("LIBRARY"), target_type = TargetType("library"), output_paths = output_paths, classpath_jars_tag = library_classpath_jars_tag, - source_only_abi_compiling_deps = [], + source_only_abi_compiling_deps = None, track_class_usage = track_class_usage, ) used_jars_json = define_javacd_action( @@ -157,7 +156,6 @@ def create_jar_artifact_javacd( abi_dir = class_abi_output_dir if should_create_class_abi else None, target_type = TargetType("library"), is_creating_subtarget = is_creating_subtarget, - source_only_abi_compiling_deps = [], ) jar_postprocessor = ctx.attrs.jar_postprocessor[RunInfo] if hasattr(ctx.attrs, "jar_postprocessor") and ctx.attrs.jar_postprocessor else None final_jar_output = prepare_final_jar( @@ -242,6 +240,7 @@ def _command_builder( abi_generation_mode: AbiGenerationMode, resources_map: dict[str, Artifact], extra_arguments: cmd_args, + use_abi_dirs: bool, ): return partial( encode_command, @@ -261,6 +260,7 @@ def _command_builder( extra_arguments = extra_arguments, kotlin_extra_params = None, provide_classpath_snapshot = False, + use_abi_dirs = use_abi_dirs, ) # buildifier: disable=uninitialized @@ -272,7 +272,6 @@ def _define_javacd_action( class_abi_jar: [Artifact, None], class_abi_output_dir: [Artifact, None], srcs: list[Artifact], - compiling_deps_tset: [JavaCompilingDepsTSet, None], track_class_usage: bool, debug_port: [int, None], uses_content_based_paths: bool, @@ -286,7 +285,6 @@ def _define_javacd_action( abi_dir: Artifact | None, target_type: TargetType, is_creating_subtarget: bool = False, - source_only_abi_compiling_deps: list[JavaClasspathEntry] = [], ): expect(java_toolchain.javacd, "java_toolchain.javacd must be set for javacd protocol") compiler = java_toolchain.javacd @@ -326,17 +324,9 @@ def _define_javacd_action( and (java_toolchain.dep_files == DepFiles("per_jar") or java_toolchain.dep_files == DepFiles("per_class")) and track_class_usage ): - abi_to_abi_dir_map = None - if java_toolchain.dep_files == DepFiles("per_class"): - if target_type == TargetType("source_only_abi"): - abi_as_dir_deps = [dep for dep in source_only_abi_compiling_deps if dep.abi_as_dir] - abi_to_abi_dir_map = [cmd_args(dep.abi, dep.abi_as_dir, delimiter = " ") for dep in abi_as_dir_deps] - args.add(classpath_jars_tag.tag_artifacts(cmd_args(hidden = [dep.abi_as_dir for dep in abi_as_dir_deps]))) - elif compiling_deps_tset: - abi_to_abi_dir_map = compiling_deps_tset.project_as_args("abi_to_abi_dir") - args.add(classpath_jars_tag.tag_artifacts(cmd_args(hidden = compiling_deps_tset.project_as_args("abi_dirs")))) used_classes_json_outputs = [cmd_args(output_paths.jar.as_output(), format = "{}/used-classes.json", parent = 1)] - used_jars_json_output = declare_prefixed_output(actions, actions_identifier, "jar/used-jars.json", uses_content_based_paths) + if target_type == TargetType("library"): + used_jars_json_output = declare_prefixed_output(actions, actions_identifier, "jar/used-jars.json", uses_content_based_paths) setup_dep_files( actions, actions_identifier, @@ -344,7 +334,6 @@ def _define_javacd_action( classpath_jars_tag, used_classes_json_outputs, used_jars_json_output, - abi_to_abi_dir_map, uses_content_based_paths, ) @@ -356,23 +345,15 @@ def _define_javacd_action( ) proto = declare_prefixed_output(actions, actions_identifier, "jar_command.proto.json", uses_content_based_paths) + dep_file_fingerprints = [] if dep_files: - # This is a little bit convoluted due to the way that content-based paths affect argfiles. - # If an unused tagged input changes, we don't want to re-run the action, but if it is a - # content-based input that is written to the argfile, then the argfile will also change - # and that would cause a re-run. - # - # We therefore write the argfile twice: the "real" argfile, which is used in the action - # and tagged as unused so that it is not used for dep-file comparison, and an argfile - # that uses placeholders instead of content-based paths, which is not tagged for dep-files - # and therefore causes a dep-file miss if it changes. - proto_dep_files_placeholder = declare_prefixed_output(actions, actions_identifier, "jar_command_for_dep_files.proto.json", uses_content_based_paths) - - proto_for_args = classpath_jars_tag.tag_artifacts(actions.write_json(proto, java_build_command)) - proto_with_inputs_for_dep_files = actions.write_json( - proto_dep_files_placeholder, java_build_command, with_inputs = True, use_dep_files_placeholder_for_content_based_paths = True + proto, fingerprint = actions.write_json( + proto, + java_build_command, + dep_files_fingerprint_using_canonical_paths = True, ) - args.add(cmd_args(hidden = proto_with_inputs_for_dep_files)) + proto_for_args = classpath_jars_tag.tag_artifacts(proto) + dep_file_fingerprints.append(fingerprint) else: proto_for_args = actions.write_json(proto, java_build_command, with_inputs = True) @@ -392,6 +373,7 @@ def _define_javacd_action( category = "{}javacd_jar".format(category_prefix), identifier = actions_identifier or "", dep_files = dep_files, + dep_file_fingerprints = dep_file_fingerprints, allow_dep_file_cache_upload = True, allow_cache_upload = True, exe = exe, diff --git a/prelude/js/js_bundle.bzl b/prelude/js/js_bundle.bzl index 612a4f8..89a4d85 100644 --- a/prelude/js/js_bundle.bzl +++ b/prelude/js/js_bundle.bzl @@ -148,6 +148,7 @@ def _get_android_resource_info(ctx: AnalysisContext, js_bundle_info: JsBundleInf res = js_bundle_info.res, res_priority = RESOURCE_PRIORITY_NORMAL, text_symbols = get_text_symbols(ctx, js_bundle_info.res, [], identifier), + unused_resource_dep_validation_has_non_xml_resources = True, ) def _get_extra_providers(ctx: AnalysisContext, js_bundle_info: JsBundleInfo, identifier: str) -> list[Provider]: diff --git a/prelude/js/js_bundle_genrule.bzl b/prelude/js/js_bundle_genrule.bzl index 0304d50..043dbe9 100644 --- a/prelude/js/js_bundle_genrule.bzl +++ b/prelude/js/js_bundle_genrule.bzl @@ -7,8 +7,8 @@ # above-listed licenses. load("@prelude//:cache_mode.bzl", "CacheModeInfo") -load("@prelude//:genrule_local_labels.bzl", "genrule_labels_require_local") -load("@prelude//:genrule_prefer_local_labels.bzl", "genrule_labels_prefer_local") +load("@prelude//:genrule_local_labels.bzl", "resolved_genrule_labels_require_local") +load("@prelude//:genrule_prefer_local_labels.bzl", "resolved_genrule_labels_prefer_local") load("@prelude//:is_full_meta_repo.bzl", "is_full_meta_repo") load("@prelude//android:android_providers.bzl", "AndroidResourceInfo", "merge_android_packageable_info") load("@prelude//js:js_providers.bzl", "JsBundleInfo") @@ -37,8 +37,8 @@ def _get_cache_mode(ctx: AnalysisContext) -> CacheModeInfo: return CacheModeInfo(allow_cache_uploads = False, cache_bust_genrules = False) def _run_genrule(ctx: AnalysisContext, out_name: str, extra_env_vars: dict, identifier: str) -> Artifact: - local_only = genrule_labels_require_local(ctx.attrs.labels) - prefer_local = genrule_labels_prefer_local(ctx.attrs.labels) + local_only = resolved_genrule_labels_require_local(ctx.attrs.labels) + prefer_local = resolved_genrule_labels_prefer_local(ctx.attrs.labels) # NOTE: Eventually we shouldn't require local_only here, since we should be # fine with caching local fallbacks if necessary (or maybe that should be @@ -274,6 +274,7 @@ def _get_extra_providers( res = None if skip_resources else android_resource_info.res, res_priority = android_resource_info.res_priority, text_symbols = None if skip_resources else android_resource_info.text_symbols, + unused_resource_dep_validation_has_non_xml_resources = True, ) providers.append(new_android_resource_info) providers.append(merge_android_packageable_info(ctx.label, ctx.actions, deps = [], resource_info = new_android_resource_info)) diff --git a/prelude/js/js_library.bzl b/prelude/js/js_library.bzl index 5fb2e9e..40c3fdb 100644 --- a/prelude/js/js_library.bzl +++ b/prelude/js/js_library.bzl @@ -211,7 +211,7 @@ def _build_library_files(ctx: AnalysisContext, transform_profile: str, flavors: ) return output_path -def _build_js_library(ctx: AnalysisContext, transform_profile: str, library_files: Artifact, flavors: list[str], js_library_deps: list[Artifact]) -> Artifact: +def _build_js_library(ctx: AnalysisContext, transform_profile: str, library_files: Artifact, flavors: list[str], dep_library_files: list[Artifact]) -> Artifact: output_path = ctx.actions.declare_output( "library-dependencies-out/{}.jslib".format(transform_profile), has_content_based_path = True, @@ -219,7 +219,7 @@ def _build_js_library(ctx: AnalysisContext, transform_profile: str, library_file job_args = { "aggregatedSourceFilesFilePath": library_files, "command": "library-dependencies", - "dependencyLibraryFilePaths": js_library_deps, + "dependencyLibraryFilePaths": dep_library_files, "flavors": flavors, "outputPath": output_path.as_output(), "platform": ctx.attrs._platform, @@ -263,12 +263,17 @@ def js_library_impl(ctx: AnalysisContext) -> list[Provider]: [dep[DefaultInfo].sub_targets[transform_profile] for dep in ctx.attrs.deps], ) ) + # Dependency resolution only needs each dep's own module map (file + # paths, haste IDs and package info), all of which is present in the + # dep's `library_files` output. Depending on that instead of the dep's + # resolved `.jslib` output keeps the depth of the action graph constant + # instead of proportional to the depth of the js_library target graph. js_library = _build_js_library( ctx, transform_profile, library_files, flavors, - [js_library_dep.output for js_library_dep in js_library_deps], + [js_library_dep.library_files for js_library_dep in js_library_deps], ) transitive_outputs = get_transitive_outputs( @@ -280,6 +285,7 @@ def js_library_impl(ctx: AnalysisContext) -> list[Provider]: sub_targets[transform_profile] = [ DefaultInfo(default_output = js_library), JsLibraryInfo( + library_files = library_files, output = js_library, transitive_outputs = transitive_outputs, ), diff --git a/prelude/js/js_providers.bzl b/prelude/js/js_providers.bzl index 437771d..053f68b 100644 --- a/prelude/js/js_providers.bzl +++ b/prelude/js/js_providers.bzl @@ -13,8 +13,11 @@ TransitiveOutputsTSet = transitive_set(args_projections = {"artifacts": _artifac JsLibraryInfo = provider( fields = { - "output": provider_field(typing.Any, default = None), # "artifact" - "transitive_outputs": provider_field(typing.Any, default = None), # "TransitiveOutputsTSet" + # Output of the `library_files` action: the library's own transformed + # files, before cross-library dependency resolution. + "library_files": provider_field(Artifact | None, default = None), + "output": provider_field(Artifact | None, default = None), + "transitive_outputs": provider_field(TransitiveOutputsTSet | None, default = None), }, ) diff --git a/prelude/js/js_utils.bzl b/prelude/js/js_utils.bzl index b588a5a..fab3d48 100644 --- a/prelude/js/js_utils.bzl +++ b/prelude/js/js_utils.bzl @@ -10,7 +10,6 @@ load("@prelude//:paths.bzl", "paths") load("@prelude//apple:apple_resource_types.bzl", "AppleResourceDestination", "AppleResourceSpec") load("@prelude//apple:resource_groups.bzl", "ResourceGraphInfo", "create_resource_graph") # @unused `ResourceGraphInfo` used as a type load("@prelude//js:js_providers.bzl", "JsBundleInfo") -load("@prelude//utils:argfile.bzl", "at_argfile") load("@prelude//utils:arglike.bzl", "ArgLike") # @unused Used as a type load("@prelude//utils:expect.bzl", "expect") load(":worker_tool.bzl", "WorkerToolInfo") @@ -168,12 +167,7 @@ def run_worker_commands( worker_tool_info = worker_tool[WorkerToolInfo] worker_command = cmd_args( worker_tool_info.command.copy(), - at_argfile( - actions = ctx.actions, - name = paths.join(identifier, "{}.js_worker_argsfile".format(category)), - args = worker_args, - has_content_based_path = has_content_based_path, - ), + worker_args, hidden = [command_args_file] + hidden_artifacts, ) diff --git a/prelude/jvm/cd_jar_creator_util.bzl b/prelude/jvm/cd_jar_creator_util.bzl index e5c4889..b7a532e 100644 --- a/prelude/jvm/cd_jar_creator_util.bzl +++ b/prelude/jvm/cd_jar_creator_util.bzl @@ -9,10 +9,9 @@ load( "@prelude//java:java_providers.bzl", "JavaClasspathEntry", - "JavaCompilingDepsTSet", + "JavaCompilingDepsTSet", # @unused Used as type "JavaLibraryInfo", "create_abi", - "derive_compiling_deps", ) load("@prelude//java:java_toolchain.bzl", "AbiGenerationMode", "JavaToolchainInfo", "unsafe_memory_access_jvm_args") load("@prelude//java/plugins:java_annotation_processor.bzl", "AnnotationProcessorProperties") # @unused Used as type @@ -89,8 +88,7 @@ def encode_target_type(target_type: TargetType) -> str: OutputPaths = record( jar = Artifact, - classes = Artifact, - annotations = Artifact, + annotations = Artifact | None, ) def qualified_name_with_subtarget(label: Label) -> str: @@ -110,21 +108,25 @@ def get_qualified_name(label: Label, target_type: TargetType) -> str: TargetType("source_only_abi"): base_qualified_name(label) + "[source-only-abi]", }[target_type] -def define_output_paths(actions: AnalysisActions, prefix: [str, None], label: Label, uses_content_based_paths: bool) -> OutputPaths: +def define_output_paths( + actions: AnalysisActions, + prefix: [str, None], + label: Label, + uses_content_based_paths: bool, + declare_annotations: bool = True, +) -> OutputPaths: # currently, javacd requires that at least some outputs are in the root # output dir. so we put all of them there. If javacd is updated we # could consolidate some of these into one subdir. return OutputPaths( jar = declare_prefixed_output(actions, prefix, "jar/{}.jar".format(label.name), uses_content_based_paths), - classes = declare_prefixed_output(actions, prefix, "__classes__", uses_content_based_paths, dir = True), - annotations = declare_prefixed_output(actions, prefix, "__gen__", uses_content_based_paths, dir = True), + annotations = declare_prefixed_output(actions, prefix, "__gen__", uses_content_based_paths, dir = True) if declare_annotations else None, ) def encode_output_paths(label: Label, paths: OutputPaths, target_type: TargetType) -> struct: paths = struct( - classesDir = paths.classes.as_output(), outputJarDirPath = cmd_args(paths.jar.as_output(), parent = 1), - annotationPath = paths.annotations.as_output(), + annotationPath = paths.annotations.as_output() if paths.annotations else None, outputJarPath = paths.jar.as_output(), ) @@ -141,9 +143,7 @@ def encode_jar_params(remove_classes: list[str], output_paths: OutputPaths, mani removeEntryPredicate = struct( patterns = remove_classes, ), - entriesToJar = [output_paths.classes.as_output()], manifestFile = manifest_file, - duplicatesLogLevel = "FINE", ) def command_abi_generation_mode(target_type: TargetType, abi_generation_mode: [AbiGenerationMode, None]) -> [AbiGenerationMode, None]: @@ -155,19 +155,26 @@ def command_abi_generation_mode(target_type: TargetType, abi_generation_mode: [A return AbiGenerationMode("source_only") return abi_generation_mode -def get_compiling_deps_tset( - actions: AnalysisActions, deps: list[Dependency], additional_classpath_entries: JavaCompilingDepsTSet | None -) -> [JavaCompilingDepsTSet, None]: - compiling_deps_tset = derive_compiling_deps(actions, None, deps) - if additional_classpath_entries: - if compiling_deps_tset == None: - compiling_deps_tset = additional_classpath_entries - else: - compiling_deps_tset = actions.tset(JavaCompilingDepsTSet, children = [compiling_deps_tset, additional_classpath_entries]) +def _source_only_abi_jars(entries: list[JavaClasspathEntry]): + return [entry.abi for entry in entries] - return compiling_deps_tset +def _source_only_abi_and_dir(entries: list[JavaClasspathEntry]): + return [path for entry in entries for path in (entry.abi, entry.abi_as_dir or "")] -def get_source_only_abi_compiling_deps(compiling_deps_tset: [JavaCompilingDepsTSet, None], source_only_abi_deps: list[Dependency]) -> list[JavaClasspathEntry]: +SourceOnlyAbiCompilingDepsTSet = transitive_set( + args_projections = { + "source_only_abi_and_dir": _source_only_abi_and_dir, + "source_only_abi_jars": _source_only_abi_jars, + }, +) + +def get_source_only_abi_compiling_deps( + actions: AnalysisActions, + compiling_deps_tset: [JavaCompilingDepsTSet, None], + source_only_abi_deps: list[Dependency], +) -> JavaCompilingDepsTSet | SourceOnlyAbiCompilingDepsTSet: + if not source_only_abi_deps and compiling_deps_tset != None: + return compiling_deps_tset source_only_abi_compiling_deps = [] if compiling_deps_tset: source_only_abi_deps_filter = {} @@ -182,7 +189,7 @@ def get_source_only_abi_compiling_deps(compiling_deps_tset: [JavaCompilingDepsTS return dep.abi in source_only_abi_deps_filter or dep.required_for_source_only_abi source_only_abi_compiling_deps = [compiling_dep for compiling_dep in list(compiling_deps_tset.traverse()) if filter_compiling_deps(compiling_dep)] - return source_only_abi_compiling_deps + return actions.tset(SourceOnlyAbiCompilingDepsTSet, value = source_only_abi_compiling_deps) # buildifier: disable=unused-variable def encode_ap_params(annotation_processor_properties: AnnotationProcessorProperties, target_type: TargetType) -> [struct, None]: @@ -208,7 +215,6 @@ def encode_ap_params(annotation_processor_properties: AnnotationProcessorPropert runsOnJavaOnly = ap.runs_on_java_only, processorNames = ap.processors, classpath = ap.deps.project_as_json("javacd_json") if ap.deps else [], - pathParams = {}, ), ) return encoded_ap_params @@ -217,7 +223,6 @@ def encode_plugin_params(plugin_params: [PluginParams, None]) -> [struct, None]: encoded_plugin_params = None if plugin_params: encoded_plugin_params = struct( - parameters = [], pluginProperties = [encode_plugin_properties(processor, arguments, plugin_params) for processor, arguments in plugin_params.processors], ) return encoded_plugin_params @@ -252,40 +257,47 @@ def encode_base_jar_command( plugin_params: [PluginParams, None], manifest_file: Artifact | None, extra_arguments: cmd_args, - source_only_abi_compiling_deps: list[JavaClasspathEntry], + source_only_abi_compiling_deps: JavaCompilingDepsTSet | SourceOnlyAbiCompilingDepsTSet | None, track_class_usage: bool, provide_classpath_snapshot: bool = False, + use_abi_dirs: bool = False, + incremental_metadata_ignored_inputs_tag: ArtifactTag | None = None, ) -> struct: jar_parameters = encode_jar_params(remove_classes, output_paths, manifest_file) qualified_name = get_qualified_name(label, target_type) if target_type == TargetType("source_only_abi"): - compiling_classpath = classpath_jars_tag.tag_artifacts([dep.abi for dep in source_only_abi_compiling_deps]) + expect(source_only_abi_compiling_deps != None) + # A list-valued JSON projection would introduce an extra array level. + compiling_classpath = cmd_args(source_only_abi_compiling_deps.project_as_args("source_only_abi_and_dir" if use_abi_dirs else "source_only_abi_jars")) compiling_classpath_snapshot = [] else: - expect(len(source_only_abi_compiling_deps) == 0) + expect(source_only_abi_compiling_deps == None) + if use_abi_dirs: + compiling_classpath = cmd_args(compiling_deps_tset.project_as_args("abi_and_dir", ordering = "topological")) if compiling_deps_tset else [] + else: + compiling_classpath = compiling_deps_tset.project_as_json("javacd_json", ordering = "topological") if compiling_deps_tset else [] # The snapshot inputs are tagged for association with dep_files, but they are not marked as used, # as they serve the incremental compiler's internal needs, # which are utilized after the build system has determined whether a rebuild is necessary. - compiling_classpath = classpath_jars_tag.tag_artifacts( - compiling_deps_tset.project_as_json("javacd_json", ordering = "topological") if compiling_deps_tset else [] - ) compiling_classpath_snapshot = classpath_jars_tag.tag_artifacts( compiling_deps_tset.project_as_json("abi_snapshot_json", ordering = "topological") if provide_classpath_snapshot and compiling_deps_tset else [] ) + compiling_classpath = classpath_jars_tag.tag_artifacts(compiling_classpath) + if use_abi_dirs and incremental_metadata_ignored_inputs_tag: + compiling_classpath = incremental_metadata_ignored_inputs_tag.tag_artifacts(compiling_classpath) + build_target_value = struct( fullyQualifiedName = qualified_name, type = encode_target_type(target_type), ) - resolved_javac = {"jsr199Javac": {}} resolved_java_options = struct( bootclasspathList = bootclasspath_entries, languageLevelOptions = struct( sourceLevel = source_level, targetLevel = target_level, ), - debug = True, javaAnnotationProcessorParams = encode_ap_params(annotation_processor_properties, target_type), standardJavacPluginParams = encode_plugin_params(plugin_params), extraArguments = extra_arguments, @@ -294,27 +306,23 @@ def encode_base_jar_command( return struct( outputPathsValue = encode_output_paths(label, output_paths, target_type), - compileTimeClasspathPaths = compiling_classpath, + compileTimeClasspathPaths = [] if use_abi_dirs else compiling_classpath, + compileTimeClasspathAbiAndDirPaths = compiling_classpath if use_abi_dirs else [], compileTimeClasspathSnapshotPaths = compiling_classpath_snapshot, javaSrcs = srcs, - # We use "class" abi compatibility to match buck1 (other compatibility modes are used for abi verification. - abiCompatibilityMode = encode_abi_generation_mode(AbiGenerationMode("class")), abiGenerationMode = encode_abi_generation_mode(command_abi_generation_mode(target_type, abi_generation_mode)), trackClassUsage = track_class_usage, - configuredBuckOut = "buck-out/v2", buildTargetValue = build_target_value, resourcesMap = [ { "key": v, - "value": cmd_args([output_paths.classes.as_output(), "/", k], delimiter = ""), + "value": k, } for (k, v) in resources_map.items() ], - resolvedJavac = resolved_javac, resolvedJavacOptions = resolved_java_options, jarParameters = jar_parameters, - pathToClasses = output_paths.jar.as_output(), - annotationsPath = output_paths.annotations.as_output(), + annotationsPath = output_paths.annotations.as_output() if output_paths.annotations else None, ) def setup_dep_files( @@ -323,20 +331,15 @@ def setup_dep_files( post_build_params: dict, classpath_jars_tag: ArtifactTag, used_classes_json_outputs: list[cmd_args], - used_jars_json_output: Artifact, - abi_to_abi_dir_map: [TransitiveSetArgsProjection, list[cmd_args], None], + used_jars_json_output: Artifact | None, uses_content_based_paths: bool, ): dep_file = declare_prefixed_output(actions, actions_identifier, "jar/dep-file.txt", uses_content_based_paths) post_build_params["usedClasses"] = used_classes_json_outputs post_build_params["depFile"] = classpath_jars_tag.tag_artifacts(dep_file.as_output()) - post_build_params["usedJarsFile"] = used_jars_json_output.as_output() - - if abi_to_abi_dir_map: - abi_to_abi_dir_map_file = declare_prefixed_output(actions, actions_identifier, "abi_to_abi_dir_map", uses_content_based_paths) - actions.write(abi_to_abi_dir_map_file, abi_to_abi_dir_map) - post_build_params["jarToJarDirMap"] = classpath_jars_tag.tag_artifacts(abi_to_abi_dir_map_file) + if used_jars_json_output != None: + post_build_params["usedJarsFile"] = used_jars_json_output.as_output() FORCE_PERSISTENT_WORKERS = read_root_config("build", "require_persistent_workers", "false").lower() == "true" @@ -512,8 +515,10 @@ def encode_command( target_type: TargetType, output_paths: OutputPaths, classpath_jars_tag: ArtifactTag, - source_only_abi_compiling_deps: list[JavaClasspathEntry], + source_only_abi_compiling_deps: JavaCompilingDepsTSet | SourceOnlyAbiCompilingDepsTSet | None, track_class_usage: bool, + use_abi_dirs: bool = False, + incremental_metadata_ignored_inputs_tag: ArtifactTag | None = None, ) -> struct: base_jar_command = encode_base_jar_command( target_type, @@ -536,6 +541,8 @@ def encode_command( source_only_abi_compiling_deps = source_only_abi_compiling_deps, track_class_usage = track_class_usage, provide_classpath_snapshot = provide_classpath_snapshot, + use_abi_dirs = use_abi_dirs, + incremental_metadata_ignored_inputs_tag = incremental_metadata_ignored_inputs_tag, ) if kotlin_extra_params: @@ -569,6 +576,7 @@ def generate_abi_jars( define_action: typing.Callable, uses_content_based_paths: bool, kotlin_extra_params_builder: typing.Callable | None = None, + source_only_abi_compiling_deps: JavaCompilingDepsTSet | SourceOnlyAbiCompilingDepsTSet | None = None, ) -> tuple: class_abi = None source_abi = None @@ -584,19 +592,12 @@ def generate_abi_jars( source_abi_identifier = declare_prefixed_name("source_abi", actions_identifier) source_abi_target_type = TargetType("source_abi") source_abi_qualified_name = get_qualified_name(label, source_abi_target_type) - source_abi_output_paths = define_output_paths(actions, source_abi_identifier, label, uses_content_based_paths) + source_abi_output_paths = define_output_paths(actions, source_abi_identifier, label, uses_content_based_paths, declare_annotations = False) source_abi_classpath_jars_tag = actions.artifact_tag() source_abi_dir = declare_prefixed_output(actions, source_abi_identifier, "source-abi-dir", uses_content_based_paths, dir = True) if kotlin_extra_params_builder: - source_abi_kotlin_classes = declare_prefixed_output( - actions, - source_abi_identifier, - "__kotlin_classes__", - uses_content_based_paths, - dir = True, - ) - source_abi_kotlin_extra_params = kotlin_extra_params_builder(kotlin_classes = source_abi_kotlin_classes) + source_abi_kotlin_extra_params = kotlin_extra_params_builder() source_abi_encode_abi_command = encode_abi_command(kotlin_extra_params = source_abi_kotlin_extra_params, provide_classpath_snapshot = False) else: source_abi_encode_abi_command = encode_abi_command @@ -606,7 +607,7 @@ def generate_abi_jars( target_type = source_abi_target_type, output_paths = source_abi_output_paths, classpath_jars_tag = source_abi_classpath_jars_tag, - source_only_abi_compiling_deps = [], + source_only_abi_compiling_deps = None, track_class_usage = track_class_usage, ) define_action( @@ -629,20 +630,14 @@ def generate_abi_jars( source_only_abi_identifier = declare_prefixed_name("source_only_abi", actions_identifier) source_only_abi_target_type = TargetType("source_only_abi") source_only_abi_qualified_name = get_qualified_name(label, source_only_abi_target_type) - source_only_abi_output_paths = define_output_paths(actions, source_only_abi_identifier, label, uses_content_based_paths) + source_only_abi_output_paths = define_output_paths(actions, source_only_abi_identifier, label, uses_content_based_paths, declare_annotations = False) source_only_abi_classpath_jars_tag = actions.artifact_tag() source_only_abi_dir = declare_prefixed_output(actions, source_only_abi_identifier, "dir", uses_content_based_paths, dir = True) - source_only_abi_compiling_deps = get_source_only_abi_compiling_deps(compiling_deps_tset, source_only_abi_deps) + if source_only_abi_compiling_deps == None: + source_only_abi_compiling_deps = get_source_only_abi_compiling_deps(actions, compiling_deps_tset, source_only_abi_deps) if kotlin_extra_params_builder: - source_only_abi_kotlin_classes = declare_prefixed_output( - actions, - source_only_abi_identifier, - "__kotlin_classes__", - uses_content_based_paths, - dir = True, - ) - source_only_abi_kotlin_extra_params = kotlin_extra_params_builder(kotlin_classes = source_only_abi_kotlin_classes) + source_only_abi_kotlin_extra_params = kotlin_extra_params_builder() source_only_abi_encode_abi_command = encode_abi_command( kotlin_extra_params = source_only_abi_kotlin_extra_params, provide_classpath_snapshot = False ) @@ -666,7 +661,6 @@ def generate_abi_jars( source_only_abi_classpath_jars_tag, source_only_abi_dir, source_only_abi_target_type, - source_only_abi_compiling_deps = source_only_abi_compiling_deps, ) source_only_abi = source_only_abi_output_paths.jar diff --git a/prelude/kotlin/kotlin_library.bzl b/prelude/kotlin/kotlin_library.bzl index d7f0f71..369eede 100644 --- a/prelude/kotlin/kotlin_library.bzl +++ b/prelude/kotlin/kotlin_library.bzl @@ -12,20 +12,20 @@ load("@prelude//android:android_providers.bzl", "merge_android_packageable_info" load( "@prelude//java:java_library.bzl", "build_java_library", + "jvm_target_stats", "split_on_archives_and_plain_files", ) load( "@prelude//java:java_providers.bzl", "JavaClasspathEntry", - "JavaCompilingDepsTSet", + "JavaCompilingDepsTSet", # @unused Used as a type "JavaLibraryInfo", "JavaPackagingDepTSet", "JavaPackagingInfo", "JavaProviders", "create_java_library_providers", "create_native_providers", - "derive_compiling_deps", - "single_library_compiling_deps", + "get_compiling_deps_tset", "to_list", ) load( @@ -55,7 +55,7 @@ load( "@prelude//kotlin:kotlin_toolchain.bzl", "KotlinToolchainInfo", ) -load("@prelude//kotlin:kotlin_utils.bzl", "get_kotlinc_compatible_target") +load("@prelude//kotlin:kotlin_utils.bzl", "get_friend_paths", "get_kotlinc_compatible_target") load("@prelude//kotlin:kotlincd_jar_creator.bzl", "create_jar_artifact_kotlincd") load("@prelude//utils:argfile.bzl", "at_argfile") load("@prelude//utils:expect.bzl", "expect") @@ -110,7 +110,7 @@ def _create_kotlin_sources( # kotlic doesn't support -bootclasspath param, so adding `bootclasspath_entries` into kotlin classpath compiling_classpath.add(bootclasspath_entries) - compiling_deps_tset = derive_compiling_deps(ctx.actions, None, deps + [kotlin_toolchain.kotlin_stdlib]) + compiling_deps_tset = get_compiling_deps_tset(ctx.actions, deps + [kotlin_toolchain.kotlin_stdlib]) if compiling_deps_tset: compiling_classpath.add(compiling_deps_tset.project_as_args("args_for_compiling")) @@ -195,7 +195,7 @@ def _create_kotlin_sources( if jvm_target: compile_kotlin_cmd_args.append(["--kapt_jvm_target", jvm_target]) - friend_paths = ctx.attrs.friend_paths + friend_paths = get_friend_paths(ctx) if friend_paths: concat_friends_paths = cmd_args( [friend_path.library_output.abi for friend_path in map_idx(JavaLibraryInfo, friend_paths) if friend_path.library_output], delimiter = "," @@ -383,11 +383,14 @@ def kotlin_library_impl(ctx: AnalysisContext) -> list[Provider]: android_packageable_info, ] + target_stats_providers, target_stats_subtargets = jvm_target_stats(ctx) + java_providers = build_kotlin_library( ctx = ctx, validation_deps_outputs = get_validation_deps_outputs(ctx), + extra_sub_targets = target_stats_subtargets, ) - return to_list(java_providers) + [android_packageable_info] + return to_list(java_providers) + [android_packageable_info] + target_stats_providers def _check_exported_deps(exported_deps: list[Dependency], attr_name: str): for exported_dep in exported_deps: @@ -482,24 +485,18 @@ def build_kotlin_library( srcs.append(kapt_generated_sources) if ksp_generated_sources: srcs.append(ksp_generated_sources) - kotlinc_classes_classpath = [ - single_library_compiling_deps( - ctx.actions, - JavaClasspathEntry( - full_library = kotlinc_classes, - abi = kotlinc_classes, - abi_as_dir = None, - required_for_source_only_abi = True, - abi_jar_snapshot = None, - ), - ) - ] - children = ( - kotlinc_classes_classpath - + ([additional_classpath_entries] if additional_classpath_entries else []) - + [kotlin_toolchain.kotlin_stdlib[JavaLibraryInfo].compiling_deps] + all_additional_classpath_entries = get_compiling_deps_tset( + ctx.actions, + value = JavaClasspathEntry( + full_library = kotlinc_classes, + abi = kotlinc_classes, + abi_as_dir = None, + required_for_source_only_abi = True, + abi_jar_snapshot = None, + ), + additional_classpath_entries = ([additional_classpath_entries] if additional_classpath_entries else []) + + [kotlin_toolchain.kotlin_stdlib[JavaLibraryInfo].compiling_deps], ) - all_additional_classpath_entries = ctx.actions.tset(JavaCompilingDepsTSet, children = children) java_lib = build_java_library( ctx, srcs, @@ -542,7 +539,7 @@ def build_kotlin_library( "enable_depfiles": getattr(ctx.attrs, "enable_depfiles", True), "enable_used_classes": ctx.attrs.enable_used_classes, "extra_kotlinc_arguments": filter_out_language_version(ctx.attrs.extra_kotlinc_arguments or []), - "friend_paths": ctx.attrs.friend_paths, + "friend_paths": get_friend_paths(ctx), "is_building_android_binary": ctx.attrs._is_building_android_binary, "jar_postprocessor": ctx.attrs.jar_postprocessor[RunInfo] if hasattr(ctx.attrs, "jar_postprocessor") and ctx.attrs.jar_postprocessor else None, "java_toolchain": ctx.attrs._java_toolchain[JavaToolchainInfo], @@ -581,13 +578,6 @@ def build_kotlin_library( ] } - if outputs and outputs.kotlin_classes: - extra_sub_targets = extra_sub_targets | { - "kotlin_classes": [ - DefaultInfo(default_output = outputs.kotlin_classes), - ] - } - for subtarget_name, tracking_artifact in tracking_outputs.items(): extra_sub_targets = extra_sub_targets | { subtarget_name: [ diff --git a/prelude/kotlin/kotlin_test.bzl b/prelude/kotlin/kotlin_test.bzl index 46dff02..1571914 100644 --- a/prelude/kotlin/kotlin_test.bzl +++ b/prelude/kotlin/kotlin_test.bzl @@ -6,6 +6,7 @@ # of this source tree. You may select, at your option, one of the # above-listed licenses. +load("@prelude//java:java_library.bzl", "jvm_target_stats") load("@prelude//java:java_test.bzl", "build_junit_test") load("@prelude//kotlin:kotlin_library.bzl", "build_kotlin_library") load("@prelude//test:inject_test_run_info.bzl", "inject_test_run_info") @@ -14,14 +15,22 @@ def kotlin_test_impl(ctx: AnalysisContext) -> list[Provider]: if ctx.attrs._build_only_native_code: return [DefaultInfo()] - java_providers = build_kotlin_library(ctx) + target_stats_providers, target_stats_subtargets = jvm_target_stats(ctx) + java_providers = build_kotlin_library( + ctx, + extra_sub_targets = target_stats_subtargets, + ) external_runner_test_info = build_junit_test(ctx, java_providers.java_library_info, java_providers.java_packaging_info, java_providers.class_to_src_map) - return inject_test_run_info(ctx, external_runner_test_info) + [ - java_providers.java_library_intellij_info, - java_providers.java_library_info, - java_providers.java_packaging_info, - java_providers.template_placeholder_info, - java_providers.default_info, - java_providers.class_to_src_map, - ] + return ( + inject_test_run_info(ctx, external_runner_test_info) + + [ + java_providers.java_library_intellij_info, + java_providers.java_library_info, + java_providers.java_packaging_info, + java_providers.template_placeholder_info, + java_providers.default_info, + java_providers.class_to_src_map, + ] + + target_stats_providers + ) diff --git a/prelude/kotlin/kotlin_utils.bzl b/prelude/kotlin/kotlin_utils.bzl index ba4ba1d..5225bce 100644 --- a/prelude/kotlin/kotlin_utils.bzl +++ b/prelude/kotlin/kotlin_utils.bzl @@ -6,7 +6,34 @@ # of this source tree. You may select, at your option, one of the # above-listed licenses. +load("@prelude//java:java_providers.bzl", "JavaLibraryInfo") +load( + "@prelude//utils:build_target_pattern.bzl", + "parse_build_target_pattern", +) +load("@prelude//utils:lazy.bzl", "lazy") + # kotlinc is strict about the target that you can pass, e.g. # error: unknown JVM target version: 8. Supported versions: 1.8, 9, 10, 11, 12 def get_kotlinc_compatible_target(target: str) -> str: return "1.8" if target == "8" else target + +def get_friend_paths(ctx: AnalysisContext) -> list[Dependency]: + """ + Resolves the `friend_paths` attribute to the deps whose ABI jars kotlinc gets as `-Xfriend-paths`. + """ + friend_paths = ctx.attrs.friend_paths + explicit_friends = [dep for dep in friend_paths if not isinstance(dep, str)] + patterns = [parse_build_target_pattern(f) for f in friend_paths if isinstance(f, str)] + if not patterns: + return explicit_friends + + friends = {dep.label.raw_target(): dep for dep in explicit_friends} + for dep in ctx.attrs.deps + ctx.attrs.exported_deps + ctx.attrs.provided_deps + ctx.attrs.exported_provided_deps: + if JavaLibraryInfo not in dep: + continue + target = dep.label.raw_target() + if target not in friends and lazy.is_any(lambda pattern: pattern.matches(dep.label), patterns): + friends[target] = dep + + return list(friends.values()) diff --git a/prelude/kotlin/kotlincd_jar_creator.bzl b/prelude/kotlin/kotlincd_jar_creator.bzl index b73541b..b0e1f3f 100644 --- a/prelude/kotlin/kotlincd_jar_creator.bzl +++ b/prelude/kotlin/kotlincd_jar_creator.bzl @@ -9,11 +9,11 @@ load( "@prelude//java:java_providers.bzl", "ClasspathSnapshotGranularity", - "JavaClasspathEntry", # @unused Used as a type "JavaCompileOutputs", # @unused Used as a type "JavaCompilingDepsTSet", # @unused Used as a type "JavaLibraryInfo", "generate_java_classpath_snapshot", + "get_compiling_deps_tset", "make_compile_outputs", ) load("@prelude//java:java_resources.bzl", "get_resources_map") @@ -40,7 +40,6 @@ load( "define_output_paths", "encode_command", "generate_abi_jars", - "get_compiling_deps_tset", "get_source_only_abi_compiling_deps", "prepare_cd_exe", "prepare_final_jar", @@ -105,10 +104,14 @@ def create_jar_artifact_kotlincd( ) ) actual_abi_generation_mode = abi_generation_mode or AbiGenerationMode("class") if srcs else AbiGenerationMode("none") + kosabi_applicability_cell_root = ( + label.cell_root + if not is_creating_subtarget and actual_abi_generation_mode == AbiGenerationMode("source_only") and kotlin_toolchain.kosabi_applicability_plugin != None + else None + ) uses_content_based_paths = uses_content_based_paths or kotlin_toolchain.allow_experimental_content_based_path_hashing output_paths = define_output_paths(actions, actions_identifier, label, uses_content_based_paths) - kotlin_classes = declare_prefixed_output(actions, actions_identifier, "__kotlin_classes__", uses_content_based_paths, dir = True) # Only create class-abi inline for class-mode targets. For source_only targets, # the fallback in cd_jar_creator_util.bzl (create_abi) handles class-abi generation @@ -127,21 +130,23 @@ def create_jar_artifact_kotlincd( jvm_abi_gen = None should_use_jvm_abi_gen = False - should_kotlinc_run_incrementally = kotlin_toolchain.enable_incremental_compilation and incremental - should_ksp2_run_incrementally = kotlin_toolchain.ksp2_enable_incremental_processing and incremental + # Structured applicability must see every source in the target. + should_kotlinc_run_incrementally = kotlin_toolchain.enable_incremental_compilation and incremental and kosabi_applicability_cell_root == None + should_ksp2_run_incrementally = kotlin_toolchain.ksp2_enable_incremental_processing and incremental and kosabi_applicability_cell_root == None incremental_state_dir = declare_prefixed_output(actions, actions_identifier, "incremental_state", uses_content_based_paths, dir = True) incremental_metadata_ignored_inputs_tag = actions.artifact_tag() - compiling_deps_tset = get_compiling_deps_tset(actions, deps, additional_classpath_entries) + compiling_deps_tset = get_compiling_deps_tset(actions, deps, [additional_classpath_entries] if additional_classpath_entries else []) # Compute the reduced SO-ABI classpath for applicability checking during # library builds. The applicability plugin needs to know which deps will be # available during source-only-abi generation (only deps with # required_for_source_only_abi=True or in source_only_abi_deps). - source_only_abi_applicability_classpath = [] + source_only_abi_applicability_classpath = cmd_args() + so_abi_deps = None if actual_abi_generation_mode == AbiGenerationMode("source_only"): - so_abi_deps = get_source_only_abi_compiling_deps(compiling_deps_tset, source_only_abi_deps) - source_only_abi_applicability_classpath = [dep.abi for dep in so_abi_deps] + so_abi_deps = get_source_only_abi_compiling_deps(actions, compiling_deps_tset, source_only_abi_deps) + source_only_abi_applicability_classpath = cmd_args(so_abi_deps.project_as_args("source_only_abi_jars")) track_class_usage = enable_used_classes and enable_depfiles and kotlin_toolchain.track_class_usage_plugin != None @@ -193,6 +198,7 @@ def create_jar_artifact_kotlincd( abi_generation_mode = actual_abi_generation_mode, resources_map = resources_map, extra_arguments = extra_arguments, + use_abi_dirs = bool(not is_creating_subtarget and srcs and track_class_usage and kotlin_toolchain.dep_files == DepFiles("per_class")), ) # this is required for the Kotlin compiler to be able to use jspecify annotations @@ -207,25 +213,27 @@ def create_jar_artifact_kotlincd( friend_paths = friend_paths, target_level = target_level, should_use_jvm_abi_gen = should_use_jvm_abi_gen, - actual_abi_generation_mode = actual_abi_generation_mode, + kosabi_applicability_cell_root = kosabi_applicability_cell_root, should_kotlinc_run_incrementally = should_kotlinc_run_incrementally, should_ksp2_run_incrementally = should_ksp2_run_incrementally, incremental_state_dir = incremental_state_dir, language_version = language_version, - kotlin_classes = kotlin_classes, source_only_abi_applicability_classpath = source_only_abi_applicability_classpath, ) library_command_builder = command_builder( kotlin_extra_params = kotlin_extra_params, provide_classpath_snapshot = should_kotlinc_run_incrementally, + incremental_metadata_ignored_inputs_tag = incremental_metadata_ignored_inputs_tag + if should_kotlinc_run_incrementally or should_ksp2_run_incrementally + else None, ) command = library_command_builder( build_mode = BuildMode("LIBRARY"), target_type = TargetType("library"), output_paths = output_paths, classpath_jars_tag = library_classpath_jars_tag, - source_only_abi_compiling_deps = [], + source_only_abi_compiling_deps = None, track_class_usage = track_class_usage, ) proto, used_jars_json = define_kotlincd_action( @@ -267,12 +275,12 @@ def create_jar_artifact_kotlincd( friend_paths = friend_paths, target_level = target_level, should_use_jvm_abi_gen = should_use_jvm_abi_gen, - actual_abi_generation_mode = actual_abi_generation_mode, + kosabi_applicability_cell_root = None, should_kotlinc_run_incrementally = False, should_ksp2_run_incrementally = False, incremental_state_dir = None, language_version = language_version, - source_only_abi_applicability_classpath = [], + source_only_abi_applicability_classpath = cmd_args(), ) # kotlincd does not support source abi @@ -294,6 +302,7 @@ def create_jar_artifact_kotlincd( define_action = define_kotlincd_action, uses_content_based_paths = uses_content_based_paths, kotlin_extra_params_builder = kotlin_extra_params_builder, + source_only_abi_compiling_deps = so_abi_deps, ) abi_jar_snapshot = generate_java_classpath_snapshot( actions, java_toolchain.cp_snapshot_generator, ClasspathSnapshotGranularity("CLASS_MEMBER_LEVEL"), classpath_abi, actions_identifier @@ -316,7 +325,6 @@ def create_jar_artifact_kotlincd( incremental_state_dir = incremental_state_dir, abi_jar_snapshot = abi_jar_snapshot, used_jars_json = used_jars_json, - kotlin_classes = kotlin_classes, ), proto, tracking_outputs, @@ -347,16 +355,14 @@ def _encode_kotlin_extra_params( friend_paths: list[Dependency], target_level: int, should_use_jvm_abi_gen: bool, - actual_abi_generation_mode: AbiGenerationMode, + kosabi_applicability_cell_root, should_kotlinc_run_incrementally: bool, should_ksp2_run_incrementally: bool, incremental_state_dir: Artifact | None, language_version: str, - kotlin_classes: Artifact, - source_only_abi_applicability_classpath: list[Artifact] = [], + source_only_abi_applicability_classpath: cmd_args = cmd_args(), ): kosabiPluginOptionsMap = {} - is_source_only_abi = actual_abi_generation_mode == AbiGenerationMode("source_only") if kotlin_toolchain.kosabi_stubs_gen_k2_plugin != None: kosabiPluginOptionsMap["kosabi_stubs_gen_k2_plugin"] = kotlin_toolchain.kosabi_stubs_gen_k2_plugin @@ -364,12 +370,15 @@ def _encode_kotlin_extra_params( if kotlin_toolchain.kosabi_applicability_plugin != None: kosabiPluginOptionsMap["kosabi_applicability_plugin"] = kotlin_toolchain.kosabi_applicability_plugin + if kosabi_applicability_cell_root != None: + kosabiPluginOptionsMap["kosabi_applicability_cell_root"] = cmd_args(kosabi_applicability_cell_root, delimiter = "") + if kotlin_toolchain.kosabi_jvm_abi_gen_k2_plugin != None: kosabiPluginOptionsMap["kosabi_jvm_abi_gen_k2_plugin"] = kotlin_toolchain.kosabi_jvm_abi_gen_k2_plugin return struct( extraClassPaths = bootclasspath_entries, - extraClassPathSnapshots = bootclasspath_snapshot_entries, + extraClassPathSnapshots = bootclasspath_snapshot_entries if should_kotlinc_run_incrementally else [], standardLibraryClassPath = kotlin_toolchain.kotlin_stdlib[JavaLibraryInfo].library_output.full_library, annotationProcessingClassPath = kotlin_toolchain.annotation_processing_jar[JavaLibraryInfo].library_output.full_library, jvmAbiGenPlugin = kotlin_toolchain.jvm_abi_gen_plugin, @@ -378,17 +387,14 @@ def _encode_kotlin_extra_params( friendPaths = [friend_path.library_output.abi for friend_path in map_idx(JavaLibraryInfo, friend_paths) if friend_path.library_output], kotlinHomeLibraries = kotlin_toolchain.kotlin_home_libraries, jvmTarget = get_kotlinc_compatible_target(str(target_level)), - kosabiJvmAbiGenEarlyTerminationMessagePrefix = "exception: java.lang.RuntimeException: Terminating compilation. We're done with ABI.", shouldUseJvmAbiGen = should_use_jvm_abi_gen, - shouldVerifySourceOnlyAbiConstraints = is_source_only_abi, - shouldGenerateAnnotationProcessingStats = True, + shouldVerifySourceOnlyAbiConstraints = kosabi_applicability_cell_root != None, extraKotlincArguments = extra_kotlinc_arguments, depTrackerPlugin = kotlin_toolchain.track_class_usage_plugin, shouldKotlincRunIncrementally = should_kotlinc_run_incrementally, shouldKsp2RunIncrementally = should_ksp2_run_incrementally, incrementalStateDir = incremental_state_dir.as_output() if incremental_state_dir else None, languageVersion = language_version, - kotlinClassesDir = kotlin_classes.as_output(), javaBinary = cmd_args(kotlin_toolchain.java_binary_for_kotlincd[RunInfo], delimiter = " ") if kotlin_toolchain.java_binary_for_kotlincd else "", applicabilityClasspath = source_only_abi_applicability_classpath, ) @@ -408,6 +414,7 @@ def _command_builder( abi_generation_mode: AbiGenerationMode, resources_map: dict[str, Artifact], extra_arguments: cmd_args, + use_abi_dirs: bool, ): return partial( _encode_kotlin_command, @@ -425,6 +432,7 @@ def _command_builder( abi_generation_mode = abi_generation_mode, resources_map = resources_map, extra_arguments = extra_arguments, + use_abi_dirs = use_abi_dirs, ) def _encode_kotlin_command( @@ -444,6 +452,8 @@ def _encode_kotlin_command( extra_arguments: cmd_args, kotlin_extra_params: [struct, None], provide_classpath_snapshot: bool, + use_abi_dirs: bool, + incremental_metadata_ignored_inputs_tag: ArtifactTag | None = None, ): return partial( encode_command, @@ -463,6 +473,8 @@ def _encode_kotlin_command( extra_arguments = extra_arguments, kotlin_extra_params = kotlin_extra_params, provide_classpath_snapshot = provide_classpath_snapshot, + use_abi_dirs = use_abi_dirs, + incremental_metadata_ignored_inputs_tag = incremental_metadata_ignored_inputs_tag, ) # buildifier: disable=uninitialized @@ -495,13 +507,10 @@ def _define_kotlincd_action( classpath_jars_tag: ArtifactTag, abi_dir: Artifact | None, target_type: TargetType, - source_only_abi_compiling_deps: list[JavaClasspathEntry] = [], is_creating_subtarget: bool = False, incremental_state_dir: Artifact | None = None, should_action_run_incrementally: bool = False, ): - _unused = source_only_abi_compiling_deps - compiler = kotlin_toolchain.kotlincd[DefaultInfo].default_outputs[0] exe, local_only = prepare_cd_exe( qualified_name, @@ -560,20 +569,12 @@ def _define_kotlincd_action( cmd_args(output_paths.jar.as_output(), format = "{}/used-classes.json", parent = 1), cmd_args(output_paths.jar.as_output(), format = "{}/kotlin-used-classes.json", parent = 1), ] - used_jars_json_output = declare_prefixed_output(actions, actions_identifier, "jar/used-jars.json", uses_content_based_paths) - abi_to_abi_dir_map = None - if kotlin_toolchain.dep_files == DepFiles("per_class"): - if target_type == TargetType("source_only_abi"): - abi_as_dir_deps = [dep for dep in source_only_abi_compiling_deps if dep.abi_as_dir] - abi_to_abi_dir_map = [cmd_args(dep.abi, dep.abi_as_dir, delimiter = " ") for dep in abi_as_dir_deps] - args.add(classpath_jars_tag.tag_artifacts(cmd_args(hidden = [dep.abi_as_dir for dep in abi_as_dir_deps]))) - elif compiling_deps_tset: - abi_to_abi_dir_map = compiling_deps_tset.project_as_args("abi_to_abi_dir") - args.add( - incremental_metadata_ignored_inputs_tag.tag_artifacts( - classpath_jars_tag.tag_artifacts(cmd_args(hidden = compiling_deps_tset.project_as_args("abi_dirs"))) - ) - ) + if target_type == TargetType("library"): + used_jars_json_output = declare_prefixed_output(actions, actions_identifier, "jar/used-jars.json", uses_content_based_paths) + if should_action_run_incrementally and kotlin_toolchain.dep_files == DepFiles("per_class") and compiling_deps_tset: + # The combined projection is excluded from incremental metadata to avoid + # enumerating ABI directories; the incremental compiler still needs the jars. + args.add(classpath_jars_tag.tag_artifacts(cmd_args(hidden = compiling_deps_tset.project_as_args("args_for_compiling", ordering = "topological")))) setup_dep_files( actions, actions_identifier, @@ -581,7 +582,6 @@ def _define_kotlincd_action( classpath_jars_tag, used_classes_json_outputs, used_jars_json_output, - abi_to_abi_dir_map, uses_content_based_paths, ) @@ -593,25 +593,15 @@ def _define_kotlincd_action( ) proto = declare_prefixed_output(actions, actions_identifier, "jar_command.proto.json", uses_content_based_paths) + dep_file_fingerprints = [] if dep_files: - # This is a little bit convoluted due to the way that content-based paths affect argfiles. - # If an unused tagged input changes, we don't want to re-run the action, but if it is a - # content-based input that is written to the argfile, then the argfile will also change - # and that would cause a re-run. - # - # We therefore write the argfile twice: the "real" argfile, which is used in the action - # and tagged as unused so that it is not used for dep-file comparison, and an argfile - # that uses placeholders instead of content-based paths, which is not tagged for dep-files - # and therefore causes a dep-file miss if it changes. - proto_dep_files_placeholder = declare_prefixed_output( - actions, actions_identifier, "jar_command_dep_files_placeholder.proto.json", uses_content_based_paths - ) - - proto_for_args = classpath_jars_tag.tag_artifacts(actions.write_json(proto, kotlin_build_command)) - proto_with_inputs_for_dep_files = actions.write_json( - proto_dep_files_placeholder, kotlin_build_command, with_inputs = True, use_dep_files_placeholder_for_content_based_paths = True + proto, fingerprint = actions.write_json( + proto, + kotlin_build_command, + dep_files_fingerprint_using_canonical_paths = True, ) - args.add(cmd_args(hidden = proto_with_inputs_for_dep_files)) + proto_for_args = classpath_jars_tag.tag_artifacts(proto) + dep_file_fingerprints.append(fingerprint) else: proto_for_args = actions.write_json(proto, kotlin_build_command, with_inputs = True) @@ -647,6 +637,7 @@ def _define_kotlincd_action( category = "{}kotlincd_jar".format(category_prefix), identifier = actions_identifier, dep_files = dep_files, + dep_file_fingerprints = dep_file_fingerprints, allow_dep_file_cache_upload = True, allow_cache_upload = True, exe = exe, diff --git a/prelude/linking/add_elf_sections.bzl b/prelude/linking/add_elf_sections.bzl new file mode 100644 index 0000000..16686e7 --- /dev/null +++ b/prelude/linking/add_elf_sections.bzl @@ -0,0 +1,108 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is dual-licensed under either the MIT license found in the +# LICENSE-MIT file in the root directory of this source tree or the Apache +# License, Version 2.0 found in the LICENSE-APACHE file in the root directory +# of this source tree. You may select, at your option, one of the +# above-listed licenses. + +load("@prelude//:paths.bzl", "paths") +load("@prelude//cxx:cxx_context.bzl", "get_cxx_toolchain_info") +load( + "@prelude//cxx:cxx_library_utility.bzl", + "cxx_is_gnu", +) + +# Sections are added after BOLT but before build info stamping, so a target +# with `elf_sections` carries this suffix outside of `PRE_STAMPED_SUFFIX`. +PRE_ADD_ELF_SECTIONS_SUFFIX = "-pre_sectioned" + +def _validate_section_name(name: str): + if not name: + fail("ELF section names must not be empty") + if "=" in name: + fail("ELF section names must not contain `=`: `{}`".format(name)) + +def add_elf_sections(ctx: AnalysisContext, obj: Artifact, sections: dict[str, Artifact], output: Artifact, category: str) -> Artifact: + """ + Copy `obj` to `output`, adding each entry of `sections` as an ELF section + named by its key holding the contents of its value. + + Sections added this way are not allocated, so they cost file size but not + resident memory in the running binary. + """ + toolchain = get_cxx_toolchain_info(ctx) + + cmd = cmd_args(toolchain.binary_utilities_info.objcopy) + for name, content in sections.items(): + _validate_section_name(name) + cmd.add("--add-section", cmd_args(content, format = "{}={{}}".format(name))) + cmd.add(obj, output.as_output()) + + # objcopy rewrites the whole file. This can be run remotely, but it's often + # cheaper to do it locally for large binaries, especially on CI using + # limited hybrid. + prefer_local = not getattr(ctx.attrs, "optimize_for_action_throughput", False) + + ctx.actions.run( + cmd, + identifier = obj.short_path, + category = category, + prefer_local = prefer_local, + prefer_remote = not prefer_local, + allow_cache_upload = toolchain.cxx_compiler_info.allow_cache_upload, + ) + + return output + +def extract_elf_section(ctx: AnalysisContext, obj: Artifact, name: str) -> Artifact: + _validate_section_name(name) + toolchain = get_cxx_toolchain_info(ctx) + output = ctx.actions.declare_output( + "{}-{}.section".format(obj.short_path, name), + has_content_based_path = False, + ) + ctx.actions.run( + cmd_args( + toolchain.binary_utilities_info.objcopy, + "--dump-section", + cmd_args(output.as_output(), format = name + "={}"), + obj, + ), + category = "extract_elf_section", + identifier = obj.short_path, + ) + return output + +def get_elf_sections(ctx: AnalysisContext) -> dict[str, Artifact]: + """ + The rule's `elf_sections`, or empty for rules that do not have the attr. + """ + return getattr(ctx.attrs, "elf_sections", None) or {} + +def add_elf_sections_to_executable(ctx: AnalysisContext, obj: Artifact, has_content_based_path: bool = False) -> Artifact: + """ + Add the rule's `elf_sections` to a linked executable, if it has any. + + `obj` is expected to still carry `PRE_ADD_ELF_SECTIONS_SUFFIX`; the result + drops it, so the next stage of the link sees the name it would have seen had + the target declared no sections. + """ + sections = get_elf_sections(ctx) + if not sections: + return obj + + if not cxx_is_gnu(ctx): + fail("`elf_sections` is only supported for GNU ELF binaries") + + stem, ext = paths.split_extension(obj.short_path) + if not stem.endswith(PRE_ADD_ELF_SECTIONS_SUFFIX): + fail("expected `{}` to end in `{}`".format(obj.short_path, PRE_ADD_ELF_SECTIONS_SUFFIX)) + + return add_elf_sections( + ctx, + obj, + sections, + ctx.actions.declare_output(stem.removesuffix(PRE_ADD_ELF_SECTIONS_SUFFIX) + ext, has_content_based_path = has_content_based_path), + category = "add_elf_sections", + ) diff --git a/prelude/linking/generated_build_info.bzl b/prelude/linking/generated_build_info.bzl new file mode 100644 index 0000000..3a5e99b --- /dev/null +++ b/prelude/linking/generated_build_info.bzl @@ -0,0 +1,330 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is dual-licensed under either the MIT license found in the +# LICENSE-MIT file in the root directory of this source tree or the Apache +# License, Version 2.0 found in the LICENSE-APACHE file in the root directory +# of this source tree. You may select, at your option, one of the +# above-listed licenses. + +load("@prelude//:artifact_tset.bzl", "make_artifact_tset") +load("@prelude//cxx:cxx_context.bzl", "get_cxx_toolchain_info") +load("@prelude//cxx:cxx_library.bzl", "cxx_compile_srcs") +load("@prelude//cxx:cxx_sources.bzl", "CxxSrcWithFlags") +load("@prelude//cxx:cxx_types.bzl", "CxxRuleConstructorParams") +load("@prelude//cxx:headers.bzl", "CxxHeadersLayout", "CxxHeadersNaming") +load("@prelude//cxx:link.bzl", "cxx_link_shared_library") +load("@prelude//cxx:link_types.bzl", "link_options") +load("@prelude//cxx:shared_library_interface.bzl", "shared_library_interface") +load("@prelude//linking:execution_preference.bzl", "LinkExecutionPreference") +load( + "@prelude//linking:link_info.bzl", + "LinkArgs", + "LinkInfo", + "ObjectsLinkable", + "SharedLibLinkable", + "unpack_link_args_metadata", + "wrap_with_no_as_needed_shared_libs_flags", +) +load("@prelude//linking:shared_libraries.bzl", "SharedLibrary", "create_shlib") +load("@prelude//linking:types.bzl", "Linkage") + +GeneratedBuildInfo = record( + json = Artifact, + linker_flags = list[typing.Any], + manifest_entries = field(Artifact | None, None), + source = Artifact, +) + +GeneratedBuildInfoData = record( + json = Artifact, + manifest_entries = field(Artifact | None, None), +) + +GeneratedBuildInfoCompileOutput = record( + external_debug_info = list[Artifact], + objects = list[Artifact], +) + +# Action-output-relative dir for generated build-info artifacts. Consumers use +# it to locate the linked DSO next to the binaries that link it. +GENERATED_BUILD_INFO_OUTPUT_DIR = "__generated_build_info__" + +_REQUIRED_GENERATED_BUILD_INFO_FIELDS = [ + "allow_cache_upload", + "base_linker_flags", + "enabled", + "exported_symbols", + "final_linker_flags", + "finalize_build_info_at_link", + "local_only", +] + +GeneratedBuildInfoInvalidationInfo = provider( + fields = { + "inputs": provider_field(typing.Any), + }, +) + +GeneratedBuildInfoSharedLibrary = record( + json = Artifact, + library = SharedLibrary, + link_args = LinkArgs, + manifest_entries = field(Artifact | None, None), +) + +def compile_generated_build_info(ctx: AnalysisContext, info: GeneratedBuildInfo) -> GeneratedBuildInfoCompileOutput: + cxx_toolchain_info = get_cxx_toolchain_info(ctx) + + # `-fno-whole-program-vtables` is a Clang-only flag; GCC errors out on it. + src_flags = ["-fno-lto"] + if cxx_toolchain_info.c_compiler_info.compiler_type in ["clang", "clang_windows", "clang_cl"]: + src_flags = ["-fno-whole-program-vtables"] + src_flags + + compiled = cxx_compile_srcs( + actions = ctx.actions, + target_label = ctx.label, + cxx_toolchain_info = cxx_toolchain_info, + impl_params = CxxRuleConstructorParams( + rule_type = "generated_build_info", + headers_layout = CxxHeadersLayout( + namespace = "", + naming = CxxHeadersNaming("regular"), + ), + srcs = [CxxSrcWithFlags(file = info.source, flags = src_flags)], + _cxx_toolchain = ctx.attrs._cxx_toolchain, + ), + own_preprocessors = [], + inherited_non_exported_preprocessor_infos = [], + inherited_exported_preprocessor_infos = [], + preferred_linkage = Linkage("shared"), + add_coverage_instrumentation_compiler_flags = False, + filename_prefix = "generated_build_info_", + ) + return GeneratedBuildInfoCompileOutput( + external_debug_info = (compiled.pic.external_debug_info + (compiled.pic.objects if compiled.pic.objects_have_external_debug_info else [])), + objects = compiled.pic.objects, + ) + +# Expected `_generated_build_info_spec` shape: +# { +# "": , +# } +# Generator-owned fields remain at the top level so configured selectors are +# resolved before the specification is serialized. +def _generated_build_info_config(ctx: AnalysisContext): + spec = dict(getattr(ctx.attrs, "_generated_build_info_spec", {})) + configured_enabled = getattr(ctx.attrs, "_generated_build_info_enabled", None) + if configured_enabled != None: + if not configured_enabled: + return None + if not spec: + fail("generated build-info spec is missing") + + mode = ctx.attrs._generated_build_info_mode + spec["allow_cache_upload"] = mode != "full" + spec["build_info"] = mode + spec["enabled"] = True + spec["final_linker_flags"] = ["--build-info={}".format(mode)] + spec["local_only"] = mode == "full" + elif not spec or not spec.get("enabled", False): + return None + + missing_fields = [field for field in _REQUIRED_GENERATED_BUILD_INFO_FIELDS if field not in spec] + if missing_fields: + fail("generated build-info spec is missing required fields: {}".format(", ".join(missing_fields))) + + tool = getattr(ctx.attrs, "_gen_build_info", None) + if tool == None: + fail("_gen_build_info must be set when generated build info is enabled") + return (spec, tool[RunInfo]) + +def _generate_build_info_data( + ctx: AnalysisContext, + spec, + tool: RunInfo, + generator_args: list[typing.Any] = [], + invalidation_inputs: list[typing.Any] = [], +) -> GeneratedBuildInfoData: + output_dir = GENERATED_BUILD_INFO_OUTPUT_DIR + json = getattr(ctx.attrs, "_generated_build_info_data", None) + if json != None and spec.get("emit_manifest_entries", False): + fail("supplied generated build-info data does not provide manifest entries") + manifest_entries = None + # A caller that supplies the JSON artifact owns its invalidation edges; + # `invalidation_inputs` apply only when this rule generates the JSON. + if json == None: + generator_spec = ctx.actions.write_json( + output_dir + "/generator_spec.json", + spec, + ) + json = ctx.actions.declare_output(output_dir, "build_info.json") + manifest_entries = ctx.actions.declare_output(output_dir, "manifest_entries.json") if spec.get("emit_manifest_entries", False) else None + command = cmd_args( + tool, + generator_args, + "--spec-json", + generator_spec, + "--output-json", + json.as_output(), + ) + if manifest_entries != None: + command.add("--output-manifest-entries", manifest_entries.as_output()) + command.add(cmd_args(hidden = invalidation_inputs)) + ctx.actions.run( + command, + category = "generate_build_info_json", + local_only = spec["local_only"], + allow_cache_upload = spec["allow_cache_upload"], + ) + return GeneratedBuildInfoData( + json = json, + manifest_entries = manifest_entries, + ) + +def generate_build_info_data( + ctx: AnalysisContext, + generator_args: list[typing.Any] = [], + invalidation_inputs: list[typing.Any] = [], +) -> GeneratedBuildInfoData | None: + config = _generated_build_info_config(ctx) + if config == None: + return None + spec, tool = config + return _generate_build_info_data( + ctx, + spec, + tool, + generator_args = generator_args, + invalidation_inputs = invalidation_inputs, + ) + +# Expected `_generated_build_info_spec` shape: +# { +# "": , +# } +# Generator arguments are supplied separately so callers can reuse existing +# configurable argument lists without expanding them into this dictionary. +def generate_build_info( + ctx: AnalysisContext, + invalidation_inputs: list[typing.Any] = [], + generator_args: list[typing.Any] = [], +) -> GeneratedBuildInfo | None: + config = _generated_build_info_config(ctx) + if config == None: + return None + spec, tool = config + data = _generate_build_info_data(ctx, spec, tool, generator_args, invalidation_inputs) + + output_dir = GENERATED_BUILD_INFO_OUTPUT_DIR + source = ctx.actions.declare_output(output_dir, "__buck2_generated_build_info.c") + + ctx.actions.run( + cmd_args( + tool, + "--input-json", + data.json, + "--output-source", + source.as_output(), + ), + category = "generate_build_info_source", + ) + + json_linker_flags = ( + (spec.get("final_linker_flags", []) + [cmd_args("--build-info-json=", data.json, delimiter = "")]) + if spec.get("finalize_build_info_at_link", True) + else [] + ) + return GeneratedBuildInfo( + json = data.json, + linker_flags = json_linker_flags + spec["base_linker_flags"], + manifest_entries = data.manifest_entries, + source = source, + ) + +def generate_build_info_shared_library( + ctx: AnalysisContext, + metadata_link_args: LinkArgs, + generator_args: list[typing.Any] = [], + invalidation_inputs: list[typing.Any] = [], +) -> GeneratedBuildInfoSharedLibrary | None: + config = _generated_build_info_config(ctx) + if config == None: + return None + spec, _tool = config + + info = generate_build_info( + ctx, + generator_args = generator_args + + [ + cmd_args( + unpack_link_args_metadata(metadata_link_args), + format = "--build-info-link-metadata={}", + ), + ], + invalidation_inputs = invalidation_inputs, + ) + if info == None: + return None + + linker_info = get_cxx_toolchain_info(ctx).linker_info + soname = "libgenerated_build_info.so" + compile_output = compile_generated_build_info(ctx, info) + exported_symbols = spec["exported_symbols"] + links = [ + LinkArgs( + infos = [ + LinkInfo( + external_debug_info = make_artifact_tset( + actions = ctx.actions, + label = ctx.label, + artifacts = compile_output.external_debug_info, + ), + pre_flags = info.linker_flags + ["-Wl,--export-dynamic-symbol={}".format(symbol) for symbol in exported_symbols], + linkables = [ + ObjectsLinkable( + objects = compile_output.objects, + linker_type = linker_info.type, + link_whole = True, + ), + ], + ), + ], + ), + LinkArgs( + flags = cmd_args( + unpack_link_args_metadata(metadata_link_args), + format = "--build-info-link-metadata={}", + ), + ), + ] + link_result = cxx_link_shared_library( + ctx = ctx, + output = "{}/{}".format(GENERATED_BUILD_INFO_OUTPUT_DIR, soname), + name = soname, + opts = link_options( + links = links, + link_execution_preference = LinkExecutionPreference("any"), + link_weight = linker_info.link_weight, + ), + ) + interface = shared_library_interface(ctx, link_result.linked_object.output) + link_info = wrap_with_no_as_needed_shared_libs_flags( + linker_info.type, + LinkInfo(linkables = [SharedLibLinkable(lib = interface)]), + ) + return GeneratedBuildInfoSharedLibrary( + json = info.json, + library = create_shlib( + label = ctx.label, + lib = link_result.linked_object, + soname = soname, + ), + link_args = LinkArgs(infos = [link_info]), + manifest_entries = info.manifest_entries, + ) + +def generated_build_info_is_shared_library(ctx: AnalysisContext) -> bool: + spec = getattr(ctx.attrs, "_generated_build_info_spec", {}) + configured_enabled = getattr(ctx.attrs, "_generated_build_info_enabled", None) + enabled = configured_enabled if configured_enabled != None else spec.get("enabled", False) + return bool(enabled and spec.get("link_as_shared_library", False)) diff --git a/prelude/linking/link_groups.bzl b/prelude/linking/link_groups.bzl index c6a3209..92b2a31 100644 --- a/prelude/linking/link_groups.bzl +++ b/prelude/linking/link_groups.bzl @@ -37,6 +37,8 @@ LinkGroupLibInfo = provider( } ) +EMPTY_LINK_GROUP_LIB_INFO = LinkGroupLibInfo(libs = {}) + def gather_link_group_libs( libs: list[dict[str, LinkGroupLib]] = [], children: list[LinkGroupLibInfo] = [], deps: list[Dependency] = [] ) -> dict[str, LinkGroupLib]: @@ -67,10 +69,13 @@ def merge_link_group_lib_info( shared_libs = shared_libs, shared_link_infos = shared_link_infos, ) + merged = gather_link_group_libs( + libs = [libs], + deps = deps, + children = children, + ) + if not merged: + return EMPTY_LINK_GROUP_LIB_INFO return LinkGroupLibInfo( - libs = gather_link_group_libs( - libs = [libs], - deps = deps, - children = children, - ), + libs = merged, ) diff --git a/prelude/linking/link_info.bzl b/prelude/linking/link_info.bzl index 9261973..dce8a5b 100644 --- a/prelude/linking/link_info.bzl +++ b/prelude/linking/link_info.bzl @@ -140,13 +140,16 @@ FrameworksLinkable = record( # They get dedupped into sets as they propagate. ) -FrameworksLinkableEmpty = FrameworksLinkable() +_EMPTY_FRAMEWORKS_LINKABLE = FrameworksLinkable() SwiftmoduleLinkable = record( swiftmodules = field(ArtifactTSet, ArtifactTSet()), ) -SwiftmoduleLinkableEmpty = SwiftmoduleLinkable() +_EMPTY_SWIFTMODULE_LINKABLE = SwiftmoduleLinkable() + +_EMPTY_FRAMEWORKS_LINKABLE_BY_STRATEGY = {link_strategy: _EMPTY_FRAMEWORKS_LINKABLE for link_strategy in LinkStrategy} +_EMPTY_SWIFTMODULE_LINKABLE_BY_STRATEGY = {link_strategy: _EMPTY_SWIFTMODULE_LINKABLE for link_strategy in LinkStrategy} LinkableTypes = [ ArchiveLinkable, @@ -571,8 +574,8 @@ def create_merged_link_info( infos = {} external_debug_info = {} - frameworks = {} - swiftmodules = {} + frameworks = None + swiftmodules = None # We don't know how this target will be linked, so we generate the possible # link info given the target's preferred linkage, to be consumed by the @@ -620,8 +623,17 @@ def create_merged_link_info( if value: external_debug_info_children.append(value) - frameworks[link_strategy] = _merge_framework_linkables(framework_linkables) - swiftmodules[link_strategy] = merge_swiftmodule_linkables(ctx, swiftmodule_linkables) + merged_frameworks = _merge_framework_linkables(framework_linkables) + if merged_frameworks != _EMPTY_FRAMEWORKS_LINKABLE: + if frameworks == None: + frameworks = dict(_EMPTY_FRAMEWORKS_LINKABLE_BY_STRATEGY) + frameworks[link_strategy] = merged_frameworks + + merged_swiftmodules = merge_swiftmodule_linkables(ctx, swiftmodule_linkables) + if merged_swiftmodules != _EMPTY_SWIFTMODULE_LINKABLE: + if swiftmodules == None: + swiftmodules = dict(_EMPTY_SWIFTMODULE_LINKABLE_BY_STRATEGY) + swiftmodules[link_strategy] = merged_swiftmodules if actual_output_style in link_infos: link_info = link_infos[actual_output_style] @@ -642,8 +654,8 @@ def create_merged_link_info( return MergedLinkInfo( _infos = infos, _external_debug_info = external_debug_info, - frameworks = frameworks, - swiftmodules = swiftmodules, + frameworks = frameworks if frameworks != None else _EMPTY_FRAMEWORKS_LINKABLE_BY_STRATEGY, + swiftmodules = swiftmodules if swiftmodules != None else _EMPTY_SWIFTMODULE_LINKABLE_BY_STRATEGY, ) def create_merged_link_info_for_propagation(ctx: AnalysisContext, xs: list[MergedLinkInfo]) -> MergedLinkInfo: @@ -654,8 +666,8 @@ def create_merged_link_info_for_propagation(ctx: AnalysisContext, xs: list[Merge """ merged = {} merged_external_debug_info = {} - frameworks = {} - swiftmodules = {} + frameworks = None + swiftmodules = None for link_strategy in LinkStrategy: merged[link_strategy] = ctx.actions.tset( LinkInfosTSet, @@ -666,14 +678,23 @@ def create_merged_link_info_for_propagation(ctx: AnalysisContext, xs: list[Merge label = ctx.label, children = filter(None, [x._external_debug_info.get(link_strategy) for x in xs]), ) - frameworks[link_strategy] = _merge_framework_linkables([x.frameworks[link_strategy] for x in xs]) - swiftmodules[link_strategy] = merge_swiftmodule_linkables(ctx, [x.swiftmodules[link_strategy] for x in xs]) + merged_frameworks = _merge_framework_linkables([x.frameworks[link_strategy] for x in xs]) + if merged_frameworks != _EMPTY_FRAMEWORKS_LINKABLE: + if frameworks == None: + frameworks = dict(_EMPTY_FRAMEWORKS_LINKABLE_BY_STRATEGY) + frameworks[link_strategy] = merged_frameworks + + merged_swiftmodules = merge_swiftmodule_linkables(ctx, [x.swiftmodules[link_strategy] for x in xs]) + if merged_swiftmodules != _EMPTY_SWIFTMODULE_LINKABLE: + if swiftmodules == None: + swiftmodules = dict(_EMPTY_SWIFTMODULE_LINKABLE_BY_STRATEGY) + swiftmodules[link_strategy] = merged_swiftmodules return MergedLinkInfo( _infos = merged, _external_debug_info = merged_external_debug_info, - frameworks = frameworks, - swiftmodules = swiftmodules, + frameworks = frameworks if frameworks != None else _EMPTY_FRAMEWORKS_LINKABLE_BY_STRATEGY, + swiftmodules = swiftmodules if swiftmodules != None else _EMPTY_SWIFTMODULE_LINKABLE_BY_STRATEGY, ) def get_link_info(infos: LinkInfos, prefer_stripped: bool = False, prefer_optimized: bool = False) -> LinkInfo: @@ -1008,7 +1029,7 @@ def has_framework_linkable(linkables: list[[FrameworksLinkable, None]]) -> bool: def _merge_framework_linkables(linkables: list[[FrameworksLinkable, None]]) -> FrameworksLinkable: if not has_framework_linkable(linkables): - return FrameworksLinkableEmpty + return _EMPTY_FRAMEWORKS_LINKABLE unique_frameworks = set() unique_libraries = set() @@ -1019,22 +1040,33 @@ def _merge_framework_linkables(linkables: list[[FrameworksLinkable, None]]) -> F unique_frameworks.update(linkable.frameworks) unique_libraries.update(linkable.libraries) + # Non-None inputs can still be contentless (leaf nodes pass fresh empty + # records rather than None); collapse to the shared empty. + if not unique_frameworks and not unique_libraries: + return _EMPTY_FRAMEWORKS_LINKABLE + return FrameworksLinkable( frameworks = unique_frameworks, libraries = unique_libraries, ) def merge_swiftmodule_linkables(ctx: AnalysisContext, linkables: list[[SwiftmoduleLinkable, None]]) -> SwiftmoduleLinkable: - children = [linkable.swiftmodules for linkable in linkables if linkable != None] + # Contentless linkables (empty inner tset) contribute nothing; dropping + # them here lets the all-empty and single-contributor cases share records + # instead of re-wrapping per node. + non_empty = [linkable for linkable in linkables if linkable != None and linkable.swiftmodules._tset != None] + + if not non_empty: + return _EMPTY_SWIFTMODULE_LINKABLE - if not children: - return SwiftmoduleLinkableEmpty + if len(non_empty) == 1: + return non_empty[0] return SwiftmoduleLinkable( swiftmodules = make_artifact_tset( actions = ctx.actions, label = ctx.label, - children = children, + children = [linkable.swiftmodules for linkable in non_empty], ) ) diff --git a/prelude/linking/linkable_graph.bzl b/prelude/linking/linkable_graph.bzl index 7a0977a..b220ba7 100644 --- a/prelude/linking/linkable_graph.bzl +++ b/prelude/linking/linkable_graph.bzl @@ -6,6 +6,7 @@ # of this source tree. You may select, at your option, one of the # above-listed licenses. +load("@prelude//cxx:compile_types.bzl", "CxxSrcCompileCommand") load("@prelude//cxx:cxx_toolchain_types.bzl", "PicBehavior") load("@prelude//cxx:headers.bzl", "CPrecompiledHeaderInfo") # TODO(mattpayne): Add this back once the type is supported by dependency mgmt @@ -100,6 +101,11 @@ LinkableNode = record( ignore_force_static_follows_dependents = field(bool), # Should this library only be used for build time linkage stub = field(bool), + # Per-source C/C++ compile commands for this node, used to aggregate an app-level native + # compile-command database (the [native_build_commands][compile] sub-target). Populated only when + # compile-command capture is enabled (see cxx_library); empty otherwise and for nodes with no + # sources (e.g. prebuilt libraries). + compile_cmds = field(list[CxxSrcCompileCommand], []), # Only allow constructing within this file. _private = _DisallowConstruction, ) @@ -137,6 +143,8 @@ LinkableGraph = provider( # dynamically, at runtime (e.g. via `dlopen`). DlopenableLibraryInfo = provider(fields = {}) +DLOPENABLE_LIBRARY_INFO_MARKER = DlopenableLibraryInfo() + def _get_required_outputs_for_linkage(linkage: Linkage) -> list[LibOutputStyle]: if linkage == Linkage("shared"): return [LibOutputStyle("shared_lib")] @@ -163,6 +171,7 @@ def create_linkable_node( linker_flags: [LinkerFlags, None] = None, ignore_force_static_follows_dependents: bool = False, stub: bool = False, + compile_cmds: list[CxxSrcCompileCommand] = [], ) -> LinkableNode: for output_style in _get_required_outputs_for_linkage(preferred_linkage): expect( @@ -189,6 +198,7 @@ def create_linkable_node( linker_flags = linker_flags, ignore_force_static_follows_dependents = ignore_force_static_follows_dependents, stub = stub, + compile_cmds = compile_cmds, _private = _DisallowConstruction(), ) diff --git a/prelude/linking/shared_libraries.bzl b/prelude/linking/shared_libraries.bzl index 9cf6b06..9a1348a 100644 --- a/prelude/linking/shared_libraries.bzl +++ b/prelude/linking/shared_libraries.bzl @@ -127,6 +127,8 @@ SharedLibraryInfo = provider( } ) +EMPTY_SHARED_LIBRARY_INFO = SharedLibraryInfo(set = None) + def get_strip_non_global_flags(cxx_toolchain: CxxToolchainInfo) -> list: if cxx_toolchain.strip_flags_info and cxx_toolchain.strip_flags_info.strip_non_global_flags: return cxx_toolchain.strip_flags_info.strip_non_global_flags @@ -211,10 +213,23 @@ def create_shared_libraries( def merge_shared_libraries(actions: AnalysisActions, node: [SharedLibraries, None] = None, deps: list[SharedLibraryInfo] = []) -> SharedLibraryInfo: kwargs = {} - children = filter(None, [dep.set for dep in deps]) + contributing_deps = [] + children = [] + for dep in deps: + if dep.set != None: + contributing_deps.append(dep) + children.append(dep.set) + + if node == None: + if not children: + return EMPTY_SHARED_LIBRARY_INFO + if len(children) == 1: + # With no local value, one contributing dep is already the complete + # merged result. + return contributing_deps[0] if children: kwargs["children"] = children - if node: + if node != None: kwargs["value"] = node set = actions.tset(SharedLibrariesTSet, **kwargs) if kwargs else None diff --git a/prelude/linking/stamp_build_info.bzl b/prelude/linking/stamp_build_info.bzl index b3db33a..a963847 100644 --- a/prelude/linking/stamp_build_info.bzl +++ b/prelude/linking/stamp_build_info.bzl @@ -12,6 +12,7 @@ load( "@prelude//cxx:cxx_library_utility.bzl", "cxx_is_gnu", ) +load("@prelude//linking:add_elf_sections.bzl", "add_elf_sections") load( "@prelude//linking:link_info.bzl", "LinkArgs", # @unused Used as a type @@ -23,7 +24,19 @@ load( PRE_STAMPED_SUFFIX = "-pre_stamped" def cxx_stamp_build_info(ctx: AnalysisContext) -> bool: - return hasattr(ctx.attrs, "_build_info") and bool(ctx.attrs._build_info) and cxx_is_gnu(ctx) + if getattr(ctx.attrs, "_generated_build_info_enabled", False): + spec = ctx.attrs._generated_build_info_spec + return bool(spec.get("link_as_shared_library", False) and cxx_is_gnu(ctx)) + + generated_build_info = getattr(ctx.attrs, "_generated_build_info_spec", {}) + if generated_build_info.get("enabled", False): + return bool(generated_build_info.get("link_as_shared_library", False) and cxx_is_gnu(ctx)) + + if not hasattr(ctx.attrs, "_build_info") or not ctx.attrs._build_info or not cxx_is_gnu(ctx): + return False + + late_build_info_stamping = getattr(ctx.attrs, "_late_build_info_stamping", None) + return late_build_info_stamping == None or late_build_info_stamping def _get_library_versions(links: list[LinkArgs] | None) -> str: if not links: @@ -42,39 +55,60 @@ def _get_library_versions(links: list[LinkArgs] | None) -> str: return ";".join(versions) def stamp_build_info( - ctx: AnalysisContext, obj: Artifact, stamped_output: Artifact | None = None, has_content_based_path: bool = False, links: list[LinkArgs] | None = None + ctx: AnalysisContext, + obj: Artifact, + stamped_output: Artifact | None = None, + has_content_based_path: bool = False, + links: list[LinkArgs] | None = None, + build_info_json: Artifact | None = None, ) -> Artifact: """ If necessary, add fb_build_info section to binary via late-stamping """ if cxx_stamp_build_info(ctx): - build_info = dict(ctx.attrs._build_info) - library_versions = _get_library_versions(links) - if library_versions: - build_info["library_versions"] = library_versions - build_info_json = ctx.actions.write_json(obj.short_path + "-build-info.json", build_info, has_content_based_path = has_content_based_path) + if build_info_json == None: + build_info = dict(ctx.attrs._build_info) + library_versions = _get_library_versions(links) + if library_versions: + build_info["library_versions"] = library_versions + build_info_json = ctx.actions.write_json(obj.short_path + "-build-info.json", build_info, has_content_based_path = has_content_based_path) stem, ext = paths.split_extension(obj.short_path) if not stamped_output: name = stem.removesuffix(PRE_STAMPED_SUFFIX) if stem.endswith(PRE_STAMPED_SUFFIX) else stem + "-stamped" stamped_output = ctx.actions.declare_output(name + ext, has_content_based_path = has_content_based_path) + toolchain = get_cxx_toolchain_info(ctx) + + # elf_stamp adds the section itself when the link did not reserve + # one, so stamping has no link-side prerequisites. Exec-platform + # toolchains omit elf_stamp (to break the toolchain -> elf_stamp -> + # toolchain cycle); objcopy adds the section there instead. + elf_stamp = toolchain.binary_utilities_info.elf_stamp + if not elf_stamp: + return add_elf_sections( + ctx, + obj, + {"fb_build_info": build_info_json}, + stamped_output, + category = "stamp_build_info", + ) + + cmd = cmd_args([ + elf_stamp, + "--section", + cmd_args(build_info_json, format = "fb_build_info={}"), + obj, + stamped_output.as_output(), + ]) + # This can be run remotely, but it's often cheaper to do this locally for large # binaries, especially on CI using limited hybrid prefer_local = not getattr(ctx.attrs, "optimize_for_action_throughput", False) - toolchain = get_cxx_toolchain_info(ctx) ctx.actions.run( - cmd_args([ - toolchain.binary_utilities_info.objcopy, - "--add-section", - cmd_args(build_info_json, format = "fb_build_info={}"), - obj, - stamped_output.as_output(), - ]), + cmd, identifier = obj.short_path, category = "stamp_build_info", - # This can be run remotely, but it's often cheaper to do this locally for large - # binaries, especially on CI using limited hybrid. prefer_local = prefer_local, prefer_remote = not prefer_local, allow_cache_upload = toolchain.cxx_compiler_info.allow_cache_upload, diff --git a/prelude/platforms/apple/platforms.bzl b/prelude/platforms/apple/platforms.bzl index edcdb90..17a86bf 100644 --- a/prelude/platforms/apple/platforms.bzl +++ b/prelude/platforms/apple/platforms.bzl @@ -57,7 +57,7 @@ _SUPPORTED_WATCHOS_PLATFORMS = [ _ANALYSIS_CONSTRAINTS = ["ovr_config//bitcode/constraints:bitcode_mode[bitcode]"] _DEFAULT_ANALYSIS_IOS_PLATFORM = ios_platforms.IPHONEOS_ARM64 -_DEFAULT_ANALYSIS_MACOS_PLATFORM = mac_platforms.MACOS_X86_64 +_DEFAULT_ANALYSIS_MACOS_PLATFORM = mac_platforms.MACOS_ARM64 _DEFAULT_ANALYSIS_WATCHOS_PLATFORM = watch_platforms.WATCHOS_ARM64 _DEFAULT_ANALYSIS_APPLETVOS_PLATFORM = appletv_platforms.APPLETVOS_ARM64 diff --git a/prelude/python/compile.bzl b/prelude/python/compile.bzl index bdb6475..fb5a570 100644 --- a/prelude/python/compile.bzl +++ b/prelude/python/compile.bzl @@ -9,20 +9,8 @@ load(":manifest.bzl", "ManifestInfo") load(":toolchain.bzl", "PythonToolchainInfo") -PycInvalidationMode = enum( - "unchecked_hash", - "checked_hash", - # timestamp isn't supported at the moment - # "timestamp", -) - -def compile_manifests(ctx: AnalysisContext, manifests: list[ManifestInfo]) -> dict[PycInvalidationMode, ManifestInfo]: - return {mode: compile_manifests_for_mode(ctx, manifests, mode) for mode in [PycInvalidationMode("unchecked_hash"), PycInvalidationMode("checked_hash")]} - -def compile_manifests_for_mode( - ctx: AnalysisContext, manifests: list[ManifestInfo], invalidation_mode: PycInvalidationMode = PycInvalidationMode("unchecked_hash") -) -> ManifestInfo: - mode = invalidation_mode.value.upper() +def compile_manifests(ctx: AnalysisContext, manifests: list[ManifestInfo]) -> ManifestInfo: + mode = "UNCHECKED_HASH" has_content_based_path = ctx.attrs._python_toolchain[PythonToolchainInfo].supports_content_based_paths == True output = ctx.actions.declare_output("bytecode_{}".format(mode), dir = True, has_content_based_path = has_content_based_path) bytecode_manifest = ctx.actions.declare_output("bytecode_{}.manifest".format(mode), has_content_based_path = has_content_based_path) diff --git a/prelude/python/interface.bzl b/prelude/python/interface.bzl index f58f470..ee939fd 100644 --- a/prelude/python/interface.bzl +++ b/prelude/python/interface.bzl @@ -13,11 +13,11 @@ PythonLibraryManifestsInterface = record( # Returns the files referenced by source manifests for this Python library. # [_arglike] of source artifacts src_artifacts = field(typing.Callable), - # Returns the bytecode manifests for this Python library, given a PycInvalidationMode. - # PycInvalidationMode -> [_arglike] of bytecode manifests (compiled with that mode) + # Returns the bytecode manifests for this Python library. + # [_arglike] of bytecode manifests bytecode_manifests = field(typing.Callable), # Returns the files referenced by bytecode manifests for this Python library. - # PycInvalidationMode -> [_arglike] of bytecode artifacts + # [_arglike] of bytecode artifacts bytecode_artifacts = field(typing.Callable), # Returns the resources manifests for this Python library. # [_arglike] of resource manifests diff --git a/prelude/python/internal_tools.bzl b/prelude/python/internal_tools.bzl index b5d6cfd..2deb694 100644 --- a/prelude/python/internal_tools.bzl +++ b/prelude/python/internal_tools.bzl @@ -13,6 +13,8 @@ PythonInternalToolsInfo = provider( fields = { "default_sitecustomize": Artifact, + # An `{}` JSON file, the type-check result for targets with typing disabled. + "empty_typing_result": Artifact, "fail_with_message": RunInfo, "generate_static_extension_info": Dependency, "make_py_package_inplace": RunInfo, @@ -32,6 +34,7 @@ def _impl(ctx: AnalysisContext) -> list[Provider]: DefaultInfo(), PythonInternalToolsInfo( default_sitecustomize = ctx.attrs.default_sitecustomize, + empty_typing_result = ctx.attrs.empty_typing_result, fail_with_message = ctx.attrs.fail_with_message[RunInfo], generate_static_extension_info = ctx.attrs.generate_static_extension_info, make_source_db = ctx.attrs.make_source_db[RunInfo], @@ -49,6 +52,7 @@ python_internal_tools = rule( impl = _impl, attrs = { "default_sitecustomize": attrs.source(default = "prelude//python/tools/make_par:sitecustomize.py"), + "empty_typing_result": attrs.default_only(attrs.source(default = "prelude//python/tools:empty_typing_result.json")), "fail_with_message": attrs.exec_dep(default = "prelude//python/tools:fail_with_message", providers = [RunInfo]), "generate_static_extension_info": attrs.exec_dep(default = "prelude//python/tools:generate_static_extension_info"), "make_py_package_inplace": attrs.exec_dep(default = "prelude//python/tools:make_py_package_inplace", providers = [RunInfo]), diff --git a/prelude/python/linking/link_helper.bzl b/prelude/python/linking/link_helper.bzl index 5110416..d783404 100644 --- a/prelude/python/linking/link_helper.bzl +++ b/prelude/python/linking/link_helper.bzl @@ -11,12 +11,14 @@ load( "LINK_GROUP_MAP_ATTR", ) load("@prelude//cxx:transformation_spec.bzl", "TransformationResultProvider") +load("@prelude//decls:cxx_rules.bzl", "BUILD_INFO_ATTR") load("@prelude//python:internal_tools.bzl", "PythonInternalToolsInfo") load("@prelude//python:toolchain.bzl", "PythonToolchainInfo") load("@prelude//python/linking:native.bzl", "process_native_linking") LinkProviders = provider( fields = { + "build_info_manifest_entries": provider_field(typing.Any, default = None), "extensions": provider_field(typing.Any, default = None), "extra": provider_field(typing.Any, default = None), "extra_artifacts": provider_field(typing.Any, default = None), @@ -60,6 +62,11 @@ cxx_implicit_attrs = { "preload_deps": attrs.list(attrs.dep(), default = []), "preprocessor_flags": attrs.any(default = []), "raw_headers": attrs.set(attrs.source(), sorted = True, default = []), + "_gen_build_info": attrs.option(attrs.exec_dep(providers = [RunInfo]), default = None), + "_generated_build_info_enabled": attrs.bool(default = False), + "_generated_build_info_mode": attrs.enum(["full", "stable"], default = "stable"), + "_generated_build_info_spec": BUILD_INFO_ATTR, + "_late_build_info_stamping": attrs.any(default = None), } python_implicit_attrs = { @@ -71,16 +78,21 @@ def _process_native_linking_rule_impl(ctx): python_toolchain = ctx.attrs._python_toolchain[PythonToolchainInfo] python_internal_tools = ctx.attrs._python_internal_tools[PythonInternalToolsInfo] raw_deps = ctx.attrs.deps - shared_libs, extensions, link_args, extra, extra_artifacts, linker_map_data, gc_sections_data, runtime_files = process_native_linking( - ctx, - raw_deps, - python_toolchain, - python_internal_tools, - ctx.attrs.package_style, - ctx.attrs.allow_cache_upload, + shared_libs, extensions, link_args, extra, extra_artifacts, linker_map_data, gc_sections_data, runtime_files, build_info_manifest_entries = ( + process_native_linking( + ctx, + raw_deps, + python_toolchain, + python_internal_tools, + ctx.attrs.package_style, + ctx.attrs.allow_cache_upload, + generated_build_info_invalidation_deps = ctx.attrs.generated_build_info_invalidation_deps, + generated_build_info_invalidation_sources = ctx.attrs.generated_build_info_invalidation_sources, + ) ) return [ LinkProviders( + build_info_manifest_entries = build_info_manifest_entries, shared_libraries = shared_libs, link_args = link_args, extensions = extensions, @@ -98,6 +110,8 @@ process_native_linking_rule = rule( attrs = { "deps": attrs.list(attrs.dep()), # Note: cxx-only deps here "dlopen_deps": attrs.list(attrs.dep(), default = []), + "generated_build_info_invalidation_deps": attrs.list(attrs.dep(), default = []), + "generated_build_info_invalidation_sources": attrs.list(attrs.source(), default = []), "package_style": attrs.any(), "rpath": attrs.string(), "shared_only_deps": attrs.list(attrs.dep(), default = []), diff --git a/prelude/python/linking/native.bzl b/prelude/python/linking/native.bzl index 4c60885..d73cd59 100644 --- a/prelude/python/linking/native.bzl +++ b/prelude/python/linking/native.bzl @@ -9,7 +9,11 @@ load("@prelude//:resources.bzl", "gather_resources") load("@prelude//cxx:cxx.bzl", "create_shared_lib_link_group_specs") load("@prelude//cxx:cxx_context.bzl", "get_cxx_toolchain_info") -load("@prelude//cxx:cxx_executable.bzl", "CxxExecutableOutput", "cxx_executable") +load( + "@prelude//cxx:cxx_executable.bzl", + "CxxExecutableOutput", # @unused Used as a type + "cxx_executable", +) load("@prelude//cxx:cxx_sources.bzl", "CxxSrcWithFlags") load("@prelude//cxx:cxx_toolchain_types.bzl", "LinkerType") load( @@ -42,6 +46,7 @@ load( "@prelude//cxx:preprocessor.bzl", "cxx_inherited_preprocessor_infos", ) +load("@prelude//linking:generated_build_info.bzl", "GENERATED_BUILD_INFO_OUTPUT_DIR") load( "@prelude//linking:link_info.bzl", "LinkArgs", # @unused Used as a type @@ -67,7 +72,12 @@ load( ) load("@prelude//linking:types.bzl", "Linkage") load("@prelude//python:internal_tools.bzl", "PythonInternalToolsInfo") -load("@prelude//python:python.bzl", "python_attr_preload_deps") +load( + "@prelude//python:python.bzl", + "PythonLibraryInfo", + "PythonLibraryManifestsTSet", + "python_attr_preload_deps", +) load("@prelude//python:toolchain.bzl", "PackageStyle") load("@prelude//utils:argfile.bzl", "at_argfile") load( @@ -258,9 +268,32 @@ def _compute_cxx_extension_info(ctx, deps) -> (CxxExtensionLinkInfo, CxxExtensio def _cxx_exe_allow_cache_upload(ctx) -> bool: return hasattr(ctx.attrs, "exe_allow_cache_upload") and bool(ctx.attrs.exe_allow_cache_upload) +def _python_source_build_info_invalidation_inputs(ctx, deps, sources): + """ + Return all Python sources packaged into the binary, but only when full + build info is enabled. These are consumed solely as hidden inputs to the + build-info generator; they must not become native link inputs. + """ + if not getattr(ctx.attrs, "_generated_build_info_enabled", False) or ctx.attrs._generated_build_info_mode != "full": + return [] + + manifest_sets = [dep[PythonLibraryInfo].manifests for dep in deps if PythonLibraryInfo in dep] + # Deliberately project sources, not extension artifacts. The latter makes + # native Python build-info generation eagerly build every transitive + # extension DSO (the behavior removed in D121806144). + transitive_sources = [ctx.actions.tset(PythonLibraryManifestsTSet, children = manifest_sets).project_as_args("source_artifacts")] if manifest_sets else [] + return sources + transitive_sources + def _compute_cxx_executable_info( - ctx, extension_info_reduced, static_extension_info_out, inherited_preprocessor_info, python_toolchain, package_style, allow_cache_upload -) -> CxxExecutableOutput: + ctx, + extension_info_reduced, + static_extension_info_out, + inherited_preprocessor_info, + python_toolchain, + package_style, + allow_cache_upload, + generated_build_info_invalidation_inputs, +) -> (CxxExecutableOutput, Artifact | None): cxx_executable_srcs = [ CxxSrcWithFlags(file = ctx.attrs.cxx_main, flags = []), CxxSrcWithFlags(file = ctx.attrs.static_extension_utils, flags = ["-DOSS_PYTHON=1"] if ctx.attrs.use_oss_python else []), @@ -320,6 +353,14 @@ def _compute_cxx_executable_info( rpath_ldflag_prefix = rpath_ldflag + "{}#link-tree".format(link_tree_name) extra_binary_link_flags.append(rpath_ldflag_prefix + "/runtime/lib") extra_binary_link_flags.append(rpath_ldflag_prefix) + if use_anon_target: + # Under the anon target the ELF is materialized at a content-addressed + # path, so the link-tree RPATH entries above cannot resolve the bundled + # runtime/lib (notably libgenerated_build_info.so) when the binary is + # re-exec'd via its real path. The DSO is also materialized next to the + # ELF under __generated_build_info__/, so fall back to it. Harmless when + # the DSO is not linked: the loader ignores nonexistent RPATH dirs. + extra_binary_link_flags.append(rpath_ldflag + GENERATED_BUILD_INFO_OUTPUT_DIR) impl_params = CxxRuleConstructorParams( rule_type = "python_binary", @@ -358,13 +399,22 @@ def _compute_cxx_executable_info( error_handler = python_toolchain.python_error_handler, allow_cache_upload = cxx_attrs_get_allow_cache_upload(ctx.attrs, get_cxx_toolchain_info(ctx).cxx_compiler_info.allow_cache_upload), precompiled_header = ctx.attrs.precompiled_header, + generated_build_info_invalidation_inputs = generated_build_info_invalidation_inputs, _cxx_toolchain = ctx.attrs._cxx_toolchain, ) - return cxx_executable(ctx, impl_params) + executable_info = cxx_executable(ctx, impl_params) + return (executable_info, executable_info.build_info_manifest_entries) def process_native_linking( - ctx, deps, python_toolchain, python_internal_tools: PythonInternalToolsInfo, package_style, allow_cache_upload + ctx, + deps, + python_toolchain, + python_internal_tools: PythonInternalToolsInfo, + package_style, + allow_cache_upload, + generated_build_info_invalidation_deps = [], + generated_build_info_invalidation_sources = [], ) -> ( list[(SharedLibrary, str)], dict[str, (LinkedObject, Label)], @@ -374,11 +424,17 @@ def process_native_linking( [CxxLinkerMapData, None], [CxxGcSectionsData, None], list[typing.Any], + Artifact | None, ): extra = {} extra_artifacts = {} extension_info, extension_info_reduced = _compute_cxx_extension_info(ctx, deps) + generated_build_info_invalidation_inputs = _python_source_build_info_invalidation_inputs( + ctx, + generated_build_info_invalidation_deps, + generated_build_info_invalidation_sources, + ) executable_deps = ctx.attrs.executable_deps @@ -402,7 +458,7 @@ def process_native_linking( extra["static_extension_info"] = [DefaultInfo(default_output = static_extension_info_out)] - executable_info = _compute_cxx_executable_info( + executable_info, build_info_manifest_entries = _compute_cxx_executable_info( ctx, extension_info_reduced, static_extension_info_out, @@ -410,8 +466,11 @@ def process_native_linking( python_toolchain, package_style, allow_cache_upload, + generated_build_info_invalidation_inputs, ) extra["native-executable"] = [DefaultInfo(default_output = executable_info.binary, sub_targets = executable_info.sub_targets)] + if "generated_build_info" in executable_info.sub_targets: + extra["generated_build_info"] = executable_info.sub_targets["generated_build_info"] # Add sub-targets for libs. for shlib in executable_info.shared_libs: @@ -487,4 +546,5 @@ def process_native_linking( executable_info.linker_map_data, executable_info.gc_sections_data, executable_info.runtime_files, + build_info_manifest_entries, ) diff --git a/prelude/python/make_py_package.bzl b/prelude/python/make_py_package.bzl index f3f0017..e261edc 100644 --- a/prelude/python/make_py_package.bzl +++ b/prelude/python/make_py_package.bzl @@ -39,7 +39,6 @@ load("@prelude//python:manifest.bzl", "create_manifest_for_entries") load("@prelude//python:python.bzl", "python_attr_preload_deps") load("@prelude//unix:providers.bzl", "UnixEnv", "create_unix_env_info") load("@prelude//utils:arglike.bzl", "ArgLike") -load(":compile.bzl", "PycInvalidationMode") load(":interface.bzl", "EntryPoint", "EntryPointKind", "PythonLibraryManifestsInterface") load(":internal_tools.bzl", "PythonInternalToolsInfo") load(":manifest.bzl", "ManifestInfo") # @unused Used as a type @@ -253,6 +252,7 @@ def make_py_package( allow_cache_upload: bool, link_args: list[LinkArgs] = [], debuginfo_files: list[(str | (str, SharedLibrary, str), Artifact)] = [], + manifest_entries_overlay: Artifact | None = None, ) -> PexProviders: """ Passes a standardized set of flags to a `make_py_package` binary to create a python @@ -303,7 +303,7 @@ def make_py_package( ) generated_files.append((startup_functions_loader, "__par__/__startup_function_loader__.py")) - manifest_module = _generate_manifest_module(ctx, manifest_module_entries, python_toolchain, python_internal_tools, srcs) + manifest_module = _generate_manifest_module(ctx, manifest_module_entries, manifest_entries_overlay, python_toolchain, python_internal_tools, srcs) if manifest_module: generated_files.append((manifest_module.artifacts[1], "__manifest__.py")) generated_files.append((manifest_module.artifacts[0], "__manifest__.json")) @@ -490,6 +490,7 @@ def _make_py_package_impl( name = "{}{}".format(ctx.attrs.name, output_suffix) standalone = package_style == PackageStyle("standalone") inplace = package_style == PackageStyle("inplace") + include_bytecode = pex_modules.compile and not inplace runtime_files = [] sub_targets = {} @@ -501,14 +502,12 @@ def _make_py_package_impl( elif pex_modules.manifests.has_hidden_resources(mode = package_style.value): hidden_resources = pex_modules.manifests.hidden_resources(mode = package_style.value) - pyc_mode = PycInvalidationMode("checked_hash") if inplace else PycInvalidationMode("unchecked_hash") - # Accumulate all of the artifacts required by the build runtime_artifacts = [] runtime_artifacts.extend(dep_artifacts) runtime_artifacts.extend(pex_modules.manifests.resource_artifacts(mode = package_style.value)) - if pex_modules.compile: - runtime_artifacts.extend(pex_modules.manifests.bytecode_artifacts(pyc_mode)) + if include_bytecode: + runtime_artifacts.extend(pex_modules.manifests.bytecode_artifacts()) if manifest_module: runtime_artifacts.extend(manifest_module.artifacts) @@ -530,7 +529,7 @@ def _make_py_package_impl( runtime_artifacts, debug_artifacts, package_style, - pyc_mode, + include_bytecode, symlink_tree_path, manifest_module, pex_modules, @@ -765,12 +764,11 @@ def _make_py_package_live( cmd.add(cmd_args(resource_manifests_path, format = "--resources={}", hidden = [resources])) runtime_files.extend(resource_artifacts) - if pex_modules.compile: + if pex_modules.compile and is_outplace: # bytecode is compiled per library so the actual bytecode artifacts are directories # the compile command outputs json manifest in the form # [(src, dst, origin),] - pyc_mode = PycInvalidationMode("unchecked_hash") if is_outplace else PycInvalidationMode("checked_hash") - bytecode_manifests = pex_modules.manifests.bytecode_manifests(pyc_mode) + bytecode_manifests = pex_modules.manifests.bytecode_manifests() bytecode_manifests_path = ctx.actions.write( "__bytecode_manifests{}.txt".format(output_suffix), bytecode_manifests, @@ -778,7 +776,7 @@ def _make_py_package_live( ) cmd.add(cmd_args(bytecode_manifests_path, format = "--bytecode={}", hidden = bytecode_manifests)) - bytecode_artifacts = pex_modules.manifests.bytecode_artifacts(pyc_mode) + bytecode_artifacts = pex_modules.manifests.bytecode_artifacts() runtime_files.extend(bytecode_artifacts) # Pass resolved bytecode artifact paths so the Rust builder can replace @@ -1105,7 +1103,7 @@ def _pex_modules_args( dep_artifacts: list[ArgLike], debug_artifacts: list[(str | (str, SharedLibrary, str), ArgLike)], package_style: PackageStyle, - pyc_mode: PycInvalidationMode, + include_bytecode: bool, symlink_tree_path: Artifact | None, manifest_module: ManifestModule | None, pex_modules: PexModules, @@ -1124,8 +1122,8 @@ def _pex_modules_args( if manifest_module != None: cmd.append(cmd_args(manifest_module.manifest, format = "--module-manifest={}")) - if pex_modules.compile: - bytecode_manifests = pex_modules.manifests.bytecode_manifests(pyc_mode) + if include_bytecode: + bytecode_manifests = pex_modules.manifests.bytecode_manifests() bytecode_manifests_path = ctx.actions.write( "__bytecode_manifests{}.txt".format(output_suffix), @@ -1141,7 +1139,7 @@ def _pex_modules_args( # To support content-based path hashing, we need to pass in the actual # bytecode artifacts alongside the manifest in order to replace the # placeholder "output_artifacts" portion of the path with the resolved hash. - bytecode_artifacts = pex_modules.manifests.bytecode_artifacts(pyc_mode) + bytecode_artifacts = pex_modules.manifests.bytecode_artifacts() bytecode_artifacts_path = ctx.actions.write( "__bytecode_artifacts{}.txt".format(output_suffix), @@ -1354,6 +1352,7 @@ def load_startup_functions(): def _generate_manifest_module( ctx: AnalysisContext, manifest_module_entries: dict[str, typing.Any] | None, + manifest_entries_overlay: Artifact | None, python_toolchain: PythonToolchainInfo, python_internal_tools: PythonInternalToolsInfo, src_manifests: list[ArgLike], @@ -1368,13 +1367,14 @@ def _generate_manifest_module( if manifest_module_entries == None: return None module = ctx.actions.declare_output("manifest/__manifest__.py", has_content_based_path = False) + json_entries_output = ctx.actions.declare_output("manifest/__manifest__.json", has_content_based_path = False) entries_json = ctx.actions.write_json("manifest/entries.json", manifest_module_entries, has_content_based_path = False) src_manifests_path = ctx.actions.write( "__module_manifests.txt", src_manifests, has_content_based_path = False, ) - if ctx.attrs.use_rust_make_par: + if ctx.attrs.use_rust_make_par and manifest_entries_overlay == None: cmd = cmd_args( python_toolchain.make_py_package_live[RunInfo], "manifest-module", @@ -1383,18 +1383,19 @@ def _generate_manifest_module( ["--output", module.as_output()], hidden = src_manifests, ) + ctx.actions.run(cmd, category = "par", identifier = "manifest-module") + ctx.actions.copy_file(json_entries_output.as_output(), entries_json) else: cmd = cmd_args( python_internal_tools.make_py_package_manifest_module, ["--manifest-entries", entries_json], + ["--manifest-entries-overlay", manifest_entries_overlay] if manifest_entries_overlay else [], ["--module-manifests", src_manifests_path], ["--output", module.as_output()], + ["--output-json", json_entries_output.as_output()], hidden = src_manifests, ) - ctx.actions.run(cmd, category = "par", identifier = "manifest-module") - - json_entries_output = ctx.actions.declare_output("manifest/__manifest__.json", has_content_based_path = False) - ctx.actions.copy_file(json_entries_output.as_output(), entries_json) + ctx.actions.run(cmd, category = "par", identifier = "manifest-module") src_manifest = ctx.actions.write_json( "manifest/module_manifest.json", diff --git a/prelude/python/prebuilt_python_library.bzl b/prelude/python/prebuilt_python_library.bzl index 423bbde..d936fcb 100644 --- a/prelude/python/prebuilt_python_library.bzl +++ b/prelude/python/prebuilt_python_library.bzl @@ -8,8 +8,8 @@ load( "@prelude//:resources.bzl", - "ResourceInfo", "gather_resources", + "make_resource_info", ) load( "@prelude//cxx:omnibus.bzl", @@ -46,6 +46,7 @@ load( "gather_dep_libraries", ) load(":source_db.bzl", "create_python_source_db_info", "create_source_db_no_deps_from_manifest") +load(":toolchain.bzl", "PythonToolchainInfo") def prebuilt_python_library_impl(ctx: AnalysisContext) -> list[Provider]: providers = [] @@ -99,7 +100,9 @@ def prebuilt_python_library_impl(ctx: AnalysisContext) -> list[Provider]: deps, shared_deps = gather_dep_libraries(ctx.attrs.deps) manifest_param = "binary_src" if binary_src != None else "source_dir" src_manifest = create_manifest_for_source_dir(ctx, manifest_param, source_root, exclude = "\\.pyc$") - bytecode = compile_manifests(ctx, [src_manifest]) + bytecode = None + if ctx.attrs._python_toolchain[PythonToolchainInfo].pyc_compilation_enabled: + bytecode = compile_manifests(ctx, [src_manifest]) lazy_imports_analyzer = get_lazy_imports_analyzer(ctx) lazy_imports_cache_output = None @@ -156,12 +159,12 @@ def prebuilt_python_library_impl(ctx: AnalysisContext) -> list[Provider]: # C++ resources. providers.append( - ResourceInfo( - resources = gather_resources( + make_resource_info( + gather_resources( label = ctx.label, deps = ctx.attrs.deps, - ) - ) + ), + ), ) # Allow third-party-build rules to depend on Python rules. diff --git a/prelude/python/python.bzl b/prelude/python/python.bzl index 190ffa1..940bd39 100644 --- a/prelude/python/python.bzl +++ b/prelude/python/python.bzl @@ -9,7 +9,6 @@ load("@prelude//linking:link_info.bzl", "LinkedObject") load("@prelude//linking:shared_libraries.bzl", "SharedLibraryInfo") load("@prelude//utils:arglike.bzl", "ArgLike") -load(":compile.bzl", "PycInvalidationMode") load(":interface.bzl", "PythonLibraryManifestsInterface") load(":manifest.bzl", "ManifestInfo") load(":toolchain.bzl", "PythonToolchainInfo") @@ -29,15 +28,15 @@ PythonLibraryManifests = record( default_resources = field([(ManifestInfo, list[ArgLike]), None]), standalone_resources = field([(ManifestInfo, list[ArgLike]), None]), outplace_resources = field([(ManifestInfo, list[ArgLike]), None]), - bytecode = field([dict[PycInvalidationMode, ManifestInfo], None]), + bytecode = field([ManifestInfo, None]), extensions = field([dict[str, LinkedObject], None]), ) -def _bytecode_artifacts(invalidation_mode: PycInvalidationMode) -> typing.Callable[[PythonLibraryManifests], list[ArgLike]]: - return lambda value: [] if value.bytecode == None else ([a for a, _ in value.bytecode[invalidation_mode].artifacts]) +def _bytecode_artifacts(value: PythonLibraryManifests) -> list[ArgLike]: + return [] if value.bytecode == None else [a for a, _ in value.bytecode.artifacts] -def _bytecode_manifests(invalidation_mode: PycInvalidationMode) -> typing.Callable[[PythonLibraryManifests], list[None] | Artifact]: - return lambda value: [] if value.bytecode == None else (value.bytecode[invalidation_mode].manifest) +def _bytecode_manifests(value: PythonLibraryManifests) -> list[None] | Artifact: + return [] if value.bytecode == None else value.bytecode.manifest def _hidden_resources_for(field_name: str) -> typing.Callable[[PythonLibraryManifests], list[ArgLike]]: return lambda value: [] if getattr(value, field_name) == None else getattr(value, field_name)[1] @@ -78,10 +77,10 @@ def _source_type_artifacts(value: PythonLibraryManifests) -> list[ArgLike]: return [] return [a for a, _ in value.src_types.artifacts] -_BYTECODE_PROJ_PREFIX = { - PycInvalidationMode("checked_hash"): "checked_bytecode", - PycInvalidationMode("unchecked_hash"): "bytecode", -} +def _extension_artifacts(value: PythonLibraryManifests) -> list[Artifact]: + if value.extensions == None: + return [] + return [extension.output for extension in value.extensions.values()] # Mode strings mirror PackageStyle.value _RESOURCE_MODES = { @@ -91,6 +90,9 @@ _RESOURCE_MODES = { } args_projections = { + "bytecode_artifacts": _bytecode_artifacts, + "bytecode_manifests": _bytecode_manifests, + "extension_artifacts": _extension_artifacts, "source_artifacts": _source_artifacts, "source_manifests": _source_manifests, "source_type_artifacts": _source_type_artifacts, @@ -99,8 +101,6 @@ args_projections = { args_projections.update({"{}hidden_resources".format(prefix): _hidden_resources_for(field_name) for prefix, field_name in _RESOURCE_MODES.values()}) args_projections.update({"{}resource_manifests".format(prefix): _resource_manifests_for(field_name) for prefix, field_name in _RESOURCE_MODES.values()}) args_projections.update({"{}resource_artifacts".format(prefix): _resource_artifacts_for(field_name) for prefix, field_name in _RESOURCE_MODES.values()}) -args_projections.update({"{}_artifacts".format(prefix): _bytecode_artifacts(mode) for mode, prefix in _BYTECODE_PROJ_PREFIX.items()}) -args_projections.update({"{}_manifests".format(prefix): _bytecode_manifests(mode) for mode, prefix in _BYTECODE_PROJ_PREFIX.items()}) PythonLibraryManifestsTSet = transitive_set( args_projections = args_projections, @@ -134,8 +134,8 @@ def manifests_to_interface(manifests: PythonLibraryManifestsTSet) -> PythonLibra return PythonLibraryManifestsInterface( src_manifests = lambda: [manifests.project_as_args("source_manifests")], src_artifacts = lambda: [manifests.project_as_args("source_artifacts")], - bytecode_manifests = lambda mode: [manifests.project_as_args("{}_manifests".format(_BYTECODE_PROJ_PREFIX[mode]))], - bytecode_artifacts = lambda mode: [manifests.project_as_args("{}_artifacts".format(_BYTECODE_PROJ_PREFIX[mode]))], + bytecode_manifests = lambda: [manifests.project_as_args("bytecode_manifests")], + bytecode_artifacts = lambda: [manifests.project_as_args("bytecode_artifacts")], resource_manifests = lambda mode = "inplace": [manifests.project_as_args("{}resource_manifests".format(_RESOURCE_MODES[mode][0]))], resource_artifacts = lambda mode = "inplace": [manifests.project_as_args("{}resource_artifacts".format(_RESOURCE_MODES[mode][0]))], has_hidden_resources = lambda mode = "inplace": manifests.reduce("{}has_hidden_resources".format(_RESOURCE_MODES[mode][0])), diff --git a/prelude/python/python_binary.bzl b/prelude/python/python_binary.bzl index 37d35e3..2015225 100644 --- a/prelude/python/python_binary.bzl +++ b/prelude/python/python_binary.bzl @@ -70,7 +70,7 @@ load( "create_python_library_info", "gather_dep_libraries", "py_attr_resources", - "py_resources", + "py_resources_deduped", "qualify_srcs", ) load(":python_runtime_bundle.bzl", "PythonRuntimeBundleInfo") @@ -155,7 +155,7 @@ def python_executable( src_manifest = create_manifest_for_source_map(ctx, "srcs", srcs) python_toolchain = ctx.attrs._python_toolchain[PythonToolchainInfo] - if get_package_style(ctx) != PackageStyle("inplace") and python_toolchain.pyc_compilation_enabled: + if python_toolchain.pyc_compilation_enabled: bytecode_manifest = compile_manifests(ctx, [src_manifest]) all_default_resources = {} @@ -176,14 +176,23 @@ def python_executable( if outplace_resources: all_outplace_resources.update(outplace_resources) + binary_default_resources, binary_standalone_resources, binary_outplace_resources = py_resources_deduped( + ctx, + [ + ("", all_default_resources), + ("_standalone", all_standalone_resources), + ("_outplace", all_outplace_resources), + ], + ) + library_info = create_python_library_info( ctx.actions, ctx.label, srcs = src_manifest, src_types = src_manifest, - default_resources = py_resources(ctx, all_default_resources) if all_default_resources else None, - standalone_resources = py_resources(ctx, all_standalone_resources, "_standalone") if all_standalone_resources else None, - outplace_resources = py_resources(ctx, all_outplace_resources, "_outplace") if all_outplace_resources else None, + default_resources = binary_default_resources, + standalone_resources = binary_standalone_resources, + outplace_resources = binary_outplace_resources, bytecode = bytecode_manifest, deps = python_deps, shared_libraries = shared_deps, @@ -280,6 +289,7 @@ def _compute_pex_providers( linker_map_data = None, gc_sections_data = None, native_runtime_files = [], + build_info_manifest_entries = None, ) -> list[Provider] | Promise: dbg_source_db_output = ctx.actions.declare_output("dbg-db.json", has_content_based_path = True) dbg_source_db = create_dbg_source_db(ctx, dbg_source_db_output, src_manifest, python_deps) @@ -460,6 +470,7 @@ def _compute_pex_providers( allow_cache_upload = allow_cache_upload, debuginfo_files = debuginfo_files, link_args = link_args, + manifest_entries_overlay = build_info_manifest_entries, ) pex.sub_targets.update(extra) @@ -543,6 +554,12 @@ def _convert_python_library_to_executable( native_runtime_files = [] if link_strategy == NativeLinkStrategy("native"): + invalidate_build_info_on_python_sources = getattr(ctx.attrs, "_generated_build_info_enabled", False) and ctx.attrs._generated_build_info_mode == "full" + generated_build_info_invalidation_deps = deps if invalidate_build_info_on_python_sources else [] + generated_build_info_invalidation_sources = ( + [artifact for artifact, _ in src_manifest.artifacts] if invalidate_build_info_on_python_sources and src_manifest != None else [] + ) + use_anon_target = getattr(ctx.attrs, "use_anon_target_for_analysis", False) if use_anon_target: native_deps = {} @@ -557,6 +574,8 @@ def _convert_python_library_to_executable( "allow_cache_upload": allow_cache_upload, "deps": list(native_deps.values()), "dlopen_deps": list(dlopen_deps.values()), + "generated_build_info_invalidation_deps": generated_build_info_invalidation_deps, + "generated_build_info_invalidation_sources": generated_build_info_invalidation_sources, "name": "python_linking:" + ctx.attrs.name, "package_style": package_style, "rpath": ctx.attrs.name, @@ -592,16 +611,21 @@ def _convert_python_library_to_executable( providers[LinkProviders].linker_map_data, providers[LinkProviders].gc_sections_data, native_runtime_files = providers[LinkProviders].runtime_files, + build_info_manifest_entries = providers[LinkProviders].build_info_manifest_entries, ) ) else: - shared_libs, extensions, link_args, extra, extra_artifacts, linker_map_data, gc_sections_data, native_runtime_files = process_native_linking( - ctx, - deps, - python_toolchain, - python_internal_tools, - package_style, - allow_cache_upload, + shared_libs, extensions, link_args, extra, extra_artifacts, linker_map_data, gc_sections_data, native_runtime_files, build_info_manifest_entries = ( + process_native_linking( + ctx, + deps, + python_toolchain, + python_internal_tools, + package_style, + allow_cache_upload, + generated_build_info_invalidation_deps = generated_build_info_invalidation_deps, + generated_build_info_invalidation_sources = generated_build_info_invalidation_sources, + ) ) if ctx.attrs.runtime_bundle: runtime_bundle = ctx.attrs.runtime_bundle[PythonRuntimeBundleInfo] @@ -624,6 +648,7 @@ def _convert_python_library_to_executable( ) else: + build_info_manifest_entries = None linker_map_data = None gc_sections_data = None extensions = {} @@ -660,6 +685,7 @@ def _convert_python_library_to_executable( linker_map_data = linker_map_data if link_strategy == NativeLinkStrategy("native") else None, gc_sections_data = gc_sections_data if link_strategy == NativeLinkStrategy("native") else None, native_runtime_files = native_runtime_files, + build_info_manifest_entries = build_info_manifest_entries, ) def python_binary_impl(ctx: AnalysisContext) -> list[Provider] | Promise: diff --git a/prelude/python/python_library.bzl b/prelude/python/python_library.bzl index 7ebb81e..6a78e6d 100644 --- a/prelude/python/python_library.bzl +++ b/prelude/python/python_library.bzl @@ -15,8 +15,8 @@ load("@prelude//:attrs_validators.bzl", "get_attrs_validation_specs") load("@prelude//:paths.bzl", "paths") load( "@prelude//:resources.bzl", - "ResourceInfo", "gather_resources", + "make_resource_info", ) load("@prelude//cxx:cxx_link_utility.bzl", "shared_libs_symlink_tree_name") load( @@ -51,7 +51,7 @@ load("@prelude//unix:providers.bzl", "UnixEnv", "create_unix_env_info") load("@prelude//utils:arglike.bzl", "ArgLike") # @unused Used as a type load("@prelude//utils:expect.bzl", "expect") load("@prelude//utils:utils.bzl", "from_named_set") -load(":compile.bzl", "PycInvalidationMode", "compile_manifests") +load(":compile.bzl", "compile_manifests") load(":lazy_imports.bzl", "get_lazy_imports_analyzer", "run_lazy_imports_library_analyzer") load( ":manifest.bzl", @@ -116,7 +116,7 @@ def create_python_library_info( is_native_dep: bool, srcs: [ManifestInfo, None] = None, src_types: [ManifestInfo, None] = None, - bytecode: [dict[PycInvalidationMode, ManifestInfo], None] = None, + bytecode: [ManifestInfo, None] = None, default_resources: [(ManifestInfo, list[ArgLike]), None] = None, standalone_resources: [(ManifestInfo, list[ArgLike]), None] = None, outplace_resources: [(ManifestInfo, list[ArgLike]), None] = None, @@ -266,6 +266,30 @@ def py_attr_resources(ctx: AnalysisContext) -> (dict[str, ArtifactOutputs], dict return default_resources, standalone_resources, outplace_resources +def py_resources_deduped(ctx: AnalysisContext, keyed_resources: list[(str, dict[str, ArtifactOutputs])]) -> list[[(ManifestInfo, list[ArgLike]), None]]: + """ + Build one resource manifest per entry, aliasing entries whose resource + maps are identical to a single manifest (the maps only diverge when a + resource dep exposes style-specific subtargets). Returns one result per + entry, None for empty maps. + """ + built = [] + results = [] + for suffix, resources in keyed_resources: + if not resources: + results.append(None) + continue + result = None + for seen_resources, seen_result in built: + if resources == seen_resources: + result = seen_result + break + if result == None: + result = py_resources(ctx, resources, suffix) + built.append((resources, result)) + results.append(result) + return results + def py_resources(ctx: AnalysisContext, resources: dict[str, ArtifactOutputs], suffix: str = "") -> (ManifestInfo, list[ArgLike]): """ Generate a manifest to wrap this rules resources. @@ -342,19 +366,32 @@ def python_library_impl(ctx: AnalysisContext) -> list[Provider]: src_manifest = create_manifest_for_source_map(ctx, "srcs", qualified_srcs) if qualified_srcs else None python_toolchain = ctx.attrs._python_toolchain[PythonToolchainInfo] - src_type_manifest = create_manifest_for_source_map(ctx, "type_stubs", src_types) if src_types else None + + # For the common all-`.py`, no-stub-override case the type manifest lists + # exactly the srcs, so reuse that manifest instead of writing a copy. + if src_types == qualified_srcs: + src_type_manifest = src_manifest + elif src_types: + src_type_manifest = create_manifest_for_source_map(ctx, "type_stubs", src_types) + else: + src_type_manifest = None # Compile bytecode. bytecode = None if src_manifest != None and python_toolchain.pyc_compilation_enabled: bytecode = compile_manifests(ctx, [src_manifest]) - sub_targets["compile"] = [DefaultInfo(default_output = bytecode[PycInvalidationMode("unchecked_hash")].artifacts[0][0])] + sub_targets["compile"] = [DefaultInfo(default_output = bytecode.artifacts[0][0])] sub_targets["src-manifest"] = [DefaultInfo(default_output = src_manifest.manifest, other_outputs = [a for a, _ in src_manifest.artifacts])] raw_deps = ctx.attrs.deps - default_resource_manifest = py_resources(ctx, default_resources) if default_resources else None - standalone_resource_manifest = py_resources(ctx, standalone_resources, "_standalone") if standalone_resources else None - outplace_resource_manifest = py_resources(ctx, outplace_resources, "_outplace") if outplace_resources else None + default_resource_manifest, standalone_resource_manifest, outplace_resource_manifest = py_resources_deduped( + ctx, + [ + ("", default_resources), + ("_standalone", standalone_resources), + ("_outplace", outplace_resources), + ], + ) deps, shared_libraries = gather_dep_libraries(raw_deps, resolve_versioned_deps = False) providers.append(gather_versioned_dependencies(raw_deps)) @@ -381,7 +418,6 @@ def python_library_impl(ctx: AnalysisContext) -> list[Provider]: lazy_imports_cache = lazy_imports_cache_output, ) providers.append(library_info) - providers.append( create_unix_env_info( actions = ctx.actions, @@ -508,12 +544,12 @@ def python_library_impl(ctx: AnalysisContext) -> list[Provider]: # C++ resources. providers.append( - ResourceInfo( - resources = gather_resources( + make_resource_info( + gather_resources( label = ctx.label, deps = raw_deps, - ) - ) + ), + ), ) # Attrs validators (combined with optional pyre type-check validation spec). diff --git a/prelude/python/sourcedb/pyrefly.bxl b/prelude/python/sourcedb/pyrefly.bxl index 840cd59..a33139b 100644 --- a/prelude/python/sourcedb/pyrefly.bxl +++ b/prelude/python/sourcedb/pyrefly.bxl @@ -85,7 +85,7 @@ def _append_or_default_if_module( def _list_targets(ctx: bxl.Context) -> bxl.ConfiguredTargetSet: cquery = ctx.cquery() uquery = ctx.uquery() - files = cquery.deps(uquery.owner(ctx.cli_args.file), 0) # deps(0) to convert to configured + files = cquery.deps(ctx.target_universe(uquery.owner(ctx.cli_args.file)).target_set(), 0) targets = cquery.eval("kind('python_library|alias|genrule', %s)", ctx.cli_args.target) res = files for target in targets.values(): diff --git a/prelude/python/toolchain.bzl b/prelude/python/toolchain.bzl index c577cfd..81b3bd4 100644 --- a/prelude/python/toolchain.bzl +++ b/prelude/python/toolchain.bzl @@ -81,8 +81,9 @@ PythonToolchainInfo = provider( # must write a Pyre-compatible JSON object containing an `errors` list. # Each error must include `code`; `name` and `severity` are optional. # Errors that fail validation must also include `path`, `line`, `column`, - # and `description`. Diagnostics with code 0, name `unused-ignore`, or - # severity `info`, `ignore`, or `warn` do not fail validation. + # and `description`. Diagnostics with code 0, name `unused-ignore` or + # `unused-type-ignore`, or severity `info`, `ignore`, or `warn` do not + # fail validation. # # The checker must exit successfully whenever it writes valid result # JSON, including when that result contains type errors. A nonzero exit diff --git a/prelude/python/tools/BUCK b/prelude/python/tools/BUCK index cad65f8..ede3e97 100644 --- a/prelude/python/tools/BUCK +++ b/prelude/python/tools/BUCK @@ -81,6 +81,12 @@ prelude.python_bootstrap_binary( visibility = ["PUBLIC"], ) +prelude.export_file( + name = "gen_bytecode_bundle.py", + src = "gen_bytecode_bundle.py", + visibility = ["PUBLIC"], +) + # Main file used for native python binaries prelude.export_file( name = "embedded_main.cpp", @@ -119,6 +125,15 @@ prelude.python_bootstrap_binary( visibility = ["PUBLIC"], ) +# Shared by every target whose typing is disabled, in place of writing the +# same empty result per target. +prelude.export_file( + name = "empty_typing_result.json", + src = "empty_typing_result.json", + mode = "reference", + visibility = ["PUBLIC"], +) + prelude.python_bootstrap_binary( name = "wheel", main = "wheel.py", @@ -131,7 +146,9 @@ prelude.sh_binary( name = "tool_tests", main = "tests/main.sh", resources = [ + "gen_bytecode_bundle.py", "make_py_package_inplace.py", + "make_py_package_manifest_module.py", "run_inplace.py.in", "type_check_result_to_validation.py", "wheel.py", diff --git a/prelude/python/tools/__test_main__.py b/prelude/python/tools/__test_main__.py index 2c91822..01906fd 100644 --- a/prelude/python/tools/__test_main__.py +++ b/prelude/python/tools/__test_main__.py @@ -400,6 +400,7 @@ def __init__(self): logging.Handler.__init__(self) @property + # pyrefly: ignore [bad-override] def stream(self): return sys.stderr diff --git a/prelude/python/tools/compile.py b/prelude/python/tools/compile.py index c30bd25..aacedf3 100644 --- a/prelude/python/tools/compile.py +++ b/prelude/python/tools/compile.py @@ -144,7 +144,7 @@ def main(argv: list[str]) -> None: "--invalidation-mode", type=str, default=PycInvalidationMode.UNCHECKED_HASH.name, - choices=[m.name for m in PycInvalidationMode], + choices=[PycInvalidationMode.UNCHECKED_HASH.name], ) parser.add_argument( "--debug", diff --git a/prelude/python/tools/empty_typing_result.json b/prelude/python/tools/empty_typing_result.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/prelude/python/tools/empty_typing_result.json @@ -0,0 +1 @@ +{} diff --git a/prelude/python/tools/gen_bytecode_bundle.py b/prelude/python/tools/gen_bytecode_bundle.py new file mode 100644 index 0000000..a6d1a5b --- /dev/null +++ b/prelude/python/tools/gen_bytecode_bundle.py @@ -0,0 +1,145 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is dual-licensed under either the MIT license found in the +# LICENSE-MIT file in the root directory of this source tree or the Apache +# License, Version 2.0 found in the LICENSE-APACHE file in the root directory +# of this source tree. You may select, at your option, one of the +# above-listed licenses. + +# pyre-strict + +"""Pack the `.pyc` files produced by `compile.py` into one blob. + +Each `python_library` contributes a bytecode manifest and its matching +`py_compile` output directory. + +Format (little endian): + + header <8sII: magic, pyc_magic length, index length + pyc_magic importlib.util.MAGIC_NUMBER + index marshalled `{tree relative source path: (payload offset, size)}` + payload concatenated `.pyc` bodies with the 16-byte header stripped + +The index is keyed by path, not by module name, because +`__par__.bytecode_bundle` hangs a `FileFinder` off `sys.path_hooks`: the tree +decides which file answers an import and the blob only supplies that file's +bytecode. Keys are `/` separated on every platform. +""" + +from __future__ import annotations + +import argparse +import json +import marshal +import os +import shutil +import struct +import sys +from importlib.util import MAGIC_NUMBER, source_from_cache + +MAGIC = b"PYCBNDL4" +HEADER: struct.Struct = struct.Struct("<8sII") +PYC_HEADER_LEN = 16 +_UINT32_MAX: int = (1 << 32) - 1 + + +def source_key(cache_relpath: str) -> str: + """`pkg/__pycache__/mod.cpython-312.pyc` -> `pkg/mod.py`.""" + return source_from_cache(cache_relpath).replace(os.sep, "/") + + +def collect( + manifest_paths: list[str], bytecode_dirs: list[str] +) -> list[tuple[str, str]]: + """Return `(source path, .pyc path)` entries. + + Each manifest is paired with its `bytecode_dir`. The real `.pyc` path is + `bytecode_dir/dest`; the manifest's placeholder path is ignored. + """ + if len(manifest_paths) != len(bytecode_dirs): + raise ValueError( + "Got {} bytecode manifests but {} bytecode directories".format( + len(manifest_paths), len(bytecode_dirs) + ) + ) + + entries = [] + for manifest_path, bytecode_dir in zip(manifest_paths, bytecode_dirs): + with open(manifest_path) as f: + manifest = json.load(f) + for dest, _pyc, _source in manifest: + dest = os.path.normpath(dest) + entries.append((source_key(dest), os.path.join(bytecode_dir, dest))) + + entries.sort(key=lambda entry: entry[0]) + return entries + + +def build(entries: list[tuple[str, str]], out_path: str) -> None: + index: dict[str, tuple[int, int]] = {} + selected: list[tuple[str, int]] = [] + payload_end = 0 + for source_relpath, pyc_path in entries: + if source_relpath in index: + # Two libraries own the same path in the tree, which make_par + # tolerates for bytecode. Keep the first: `collect` fixed the order. + continue + with open(pyc_path, "rb") as f: + header = f.read(PYC_HEADER_LEN) + body_size = os.fstat(f.fileno()).st_size - PYC_HEADER_LEN + if len(header) != PYC_HEADER_LEN or body_size <= 0: + raise ValueError(f"Truncated .pyc has no bytecode payload: {pyc_path}") + if header[: len(MAGIC_NUMBER)] != MAGIC_NUMBER: + raise ValueError(f".pyc has the wrong interpreter magic: {pyc_path}") + index[source_relpath] = (payload_end, body_size) + payload_end += body_size + selected.append((pyc_path, body_size)) + + # `collect` sorted the entries and dicts keep insertion order, so the + # marshalled index is byte-identical across builds. + index_bytes = marshal.dumps(index) + if len(index_bytes) > _UINT32_MAX: + raise ValueError("Bytecode bundle index is too large") + with open(out_path, "wb") as f: + f.write(HEADER.pack(MAGIC, len(MAGIC_NUMBER), len(index_bytes))) + f.write(MAGIC_NUMBER) + f.write(index_bytes) + for pyc_path, expected_size in selected: + start = f.tell() + with open(pyc_path, "rb") as pyc: + pyc.seek(PYC_HEADER_LEN) + shutil.copyfileobj(pyc, f, length=1024 * 1024) + if f.tell() - start != expected_size: + raise OSError(f".pyc changed while bundling: {pyc_path}") + + +def _read_lines(path: str) -> list[str]: + with open(path) as f: + return [line.rstrip("\n") for line in f if line.rstrip("\n")] + + +def main(argv: list[str]) -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--output", required=True) + parser.add_argument( + "--bytecode-manifests-from", + required=True, + help="file listing py_compile bytecode manifests, one per line", + ) + parser.add_argument( + "--bytecode-artifacts-from", + required=True, + help="file listing py_compile output directories, one per line, in the " + "same order as --bytecode-manifests-from", + ) + args = parser.parse_args(argv[1:]) + + entries = collect( + _read_lines(args.bytecode_manifests_from), + _read_lines(args.bytecode_artifacts_from), + ) + build(entries, args.output) + + +if __name__ == "__main__": + main(sys.argv) diff --git a/prelude/python/tools/make_py_package_manifest_module.py b/prelude/python/tools/make_py_package_manifest_module.py index 22085cb..1ff2f5c 100755 --- a/prelude/python/tools/make_py_package_manifest_module.py +++ b/prelude/python/tools/make_py_package_manifest_module.py @@ -35,12 +35,19 @@ def parse_args() -> argparse.Namespace: type=Path, default=None, ) + parser.add_argument( + "--manifest-entries-overlay", + help="Path to JSON entries to merge into the base manifest entries.", + type=Path, + default=None, + ) parser.add_argument( "--output", type=Path, help="Output path for the generated module.", required=True, ) + parser.add_argument("--output-json", type=Path, default=None) return parser.parse_args() @@ -50,6 +57,25 @@ def path_to_module(path: str) -> str | None: return path[: -len(suffix)].replace("/", ".").replace("\\", ".") +def merge_entries(entries: dict[str, object], overlay: dict[str, object]) -> None: + for key, value in overlay.items(): + existing = entries.get(key) + if isinstance(existing, dict) and isinstance(value, dict): + existing.update(value) + else: + entries[key] = value + + +def load_entries(path: Path | None) -> dict[str, object]: + if path is None: + return {} + with open(path) as f: + entries = json.load(f) + if not isinstance(entries, dict): + raise ValueError(f"Manifest entries in {path} aren't a dictionary") + return entries + + def main() -> None: args = parse_args() output: Path = args.output @@ -79,16 +105,13 @@ def main() -> None: elif parent_module != module: break - entries = {} - if args.manifest_entries: - with open(args.manifest_entries) as f: - entries = json.load(f) - if not isinstance(entries, dict): - raise ValueError( - f"Manifest entries in {args.manifest_entries} aren't a dictionary" - ) + entries = load_entries(args.manifest_entries) + if args.manifest_entries_overlay: + merge_entries(entries, load_entries(args.manifest_entries_overlay)) if "modules" in entries: raise ValueError("'modules' can't be a key in manifest entries") + if args.output_json: + args.output_json.write_text(json.dumps(entries, sort_keys=True) + "\n") sorted_modules = sorted(modules.items()) entries["modules"] = [m[0] for m in sorted_modules] entries["origins"] = tuple(m[1] for m in sorted_modules) diff --git a/prelude/python/tools/make_py_package_modules.py b/prelude/python/tools/make_py_package_modules.py index 975623f..b3071e8 100755 --- a/prelude/python/tools/make_py_package_modules.py +++ b/prelude/python/tools/make_py_package_modules.py @@ -75,9 +75,7 @@ def create_pyc_hash_dict(args: argparse.Namespace) -> dict[str, str]: """Construct a map of bytecode artifact path prefixes to content hashes.""" pyc_hash_dict: dict[str, str] = {} - pattern = re.compile( - r"(.*)/([a-fA-F0-9]{16})/(?:bytecode_CHECKED_HASH|bytecode_UNCHECKED_HASH)" - ) + pattern = re.compile(r"(.*)/([a-fA-F0-9]{16})/bytecode_UNCHECKED_HASH") for bytecode_artifact in args.bytecode_artifacts: with open(bytecode_artifact) as f: for line in f: diff --git a/prelude/python/tools/tests/gen_bytecode_bundle_test.py b/prelude/python/tools/tests/gen_bytecode_bundle_test.py new file mode 100644 index 0000000..6f16541 --- /dev/null +++ b/prelude/python/tools/tests/gen_bytecode_bundle_test.py @@ -0,0 +1,299 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is dual-licensed under either the MIT license found in the +# LICENSE-MIT file in the root directory of this source tree or the Apache +# License, Version 2.0 found in the LICENSE-APACHE file in the root directory +# of this source tree. You may select, at your option, one of the +# above-listed licenses. + +# pyre-strict + +from __future__ import annotations + +import json +import marshal +import os +import py_compile +import tempfile +import tracemalloc +import unittest +from dataclasses import dataclass +from importlib.util import cache_from_source, MAGIC_NUMBER +from pathlib import Path +from types import CodeType + +import gen_bytecode_bundle + + +@dataclass +class Library: + """What one `python_library`'s `compile.py` action leaves behind.""" + + name: str + manifest: Path + bytecode_dir: Path + # Keyed by source path in the tree. + pycs: dict[str, Path] + + +@dataclass +class Bundle: + """A built bundle, split back into its four sections.""" + + magic: bytes + pyc_magic: bytes + index: dict[str, tuple[int, int]] + payload: bytes + + def code(self, source_relpath: str) -> CodeType: + offset, size = self.index[source_relpath] + code = marshal.loads(self.payload[offset : offset + size]) + assert isinstance(code, CodeType) + return code + + def run(self, source_relpath: str) -> dict[str, object]: + """Execute a bundled module and return its globals.""" + namespace: dict[str, object] = {} + exec(self.code(source_relpath), namespace) + return namespace + + +class BundleTestBase(unittest.TestCase): + def setUp(self) -> None: + tmpdir = tempfile.TemporaryDirectory() + self.addCleanup(tmpdir.cleanup) + self.root = Path(tmpdir.name) + + def _library( + self, name: str, sources: dict[str, str], pyc_field: str | None = None + ) -> Library: + """Compile one `python_library` the way `compile.py` does. + + Leaves a `__pycache__` tree under the action's output directory, plus a + manifest of `(destination, .pyc path, source)` for what it wrote. + `pyc_field` overrides that middle field, which is where a real build + puts a content-hash placeholder. + """ + bytecode_dir = self.root / name / "bytecode" + entries = [] + pycs = {} + for source_relpath, source in sources.items(): + src = self.root / name / "src" / source_relpath + src.parent.mkdir(parents=True, exist_ok=True) + src.write_text(source, encoding="utf8") + + dest = cache_from_source(source_relpath) + pyc = bytecode_dir / dest + pyc.parent.mkdir(parents=True, exist_ok=True) + py_compile.compile( + str(src), cfile=str(pyc), dfile=source_relpath, doraise=True + ) + entries.append([dest, pyc_field or str(pyc), str(src)]) + pycs[source_relpath] = pyc + + manifest = self.root / name / "bytecode.manifest" + manifest.write_text(json.dumps(entries), encoding="utf8") + return Library(name, manifest, bytecode_dir, pycs) + + def _build(self, *libraries: Library) -> Path: + """Bundle `libraries` and return the file the tool wrote.""" + out = self.root / "{}.bundle".format("-".join(lib.name for lib in libraries)) + gen_bytecode_bundle.build( + gen_bytecode_bundle.collect( + [str(lib.manifest) for lib in libraries], + [str(lib.bytecode_dir) for lib in libraries], + ), + str(out), + ) + return out + + def _bundle(self, *libraries: Library) -> Bundle: + return self._unpack(self._build(*libraries)) + + @staticmethod + def _unpack(path: Path) -> Bundle: + data = path.read_bytes() + magic, pyc_magic_len, index_len = gen_bytecode_bundle.HEADER.unpack_from(data) + index_start = gen_bytecode_bundle.HEADER.size + pyc_magic_len + payload_start = index_start + index_len + return Bundle( + magic, + data[gen_bytecode_bundle.HEADER.size : index_start], + marshal.loads(data[index_start:payload_start]), + data[payload_start:], + ) + + +class SourceKeyTest(unittest.TestCase): + """The inverse of the `get_pyc_path` mapping `compile.py` applies.""" + + def test_keys_are_posix_on_every_platform(self) -> None: + self.assertEqual( + gen_bytecode_bundle.source_key( + cache_from_source(os.path.join("a", "b", "c", "d.py")) + ), + "a/b/c/d.py", + ) + + +class BuildTest(BundleTestBase): + def test_header_pins_the_format_and_interpreter(self) -> None: + bundle = self._bundle(self._library("lib", {"pkg/mod.py": "VALUE = 1"})) + + self.assertEqual(bundle.magic, b"PYCBNDL4") + self.assertEqual(bundle.pyc_magic, MAGIC_NUMBER) + + def test_index_is_keyed_by_path_not_by_module_name(self) -> None: + # `pkg.py` and `pkg/__init__.py` are the same module name and both + # stay, because the finder - not the packer - is what decides which one + # an import resolves to. The top-level `__init__.py` is one + # rust_make_par synthesizes for a namespace directory; whether it is + # importable at all is likewise not the packer's business. + paths = ["top.py", "__init__.py", "pkg.py", "pkg/__init__.py", "pkg/mod.py"] + # pyre-fixme[6]: For 2nd argument expected `Dict[str, str]` but got + # `Dict[str, Literal['VALUE = 1']]`. + bundle = self._bundle(self._library("lib", dict.fromkeys(paths, "VALUE = 1"))) + + self.assertEqual(set(bundle.index), set(paths)) + + def test_payload_slices_unmarshal_to_runnable_code(self) -> None: + bundle = self._bundle(self._library("lib", {"pkg/mod.py": "VALUE = 41 + 1"})) + + self.assertEqual(bundle.run("pkg/mod.py")["VALUE"], 42) + + def test_code_keeps_the_tree_relative_filename(self) -> None: + # `BundleLoader` rewrites this to an absolute path at import time, so a + # bundle that shipped an absolute build path would leak it into every + # traceback. + bundle = self._bundle(self._library("lib", {"pkg/mod.py": "VALUE = 1"})) + + self.assertEqual(bundle.code("pkg/mod.py").co_filename, "pkg/mod.py") + + def test_bundle_is_byte_identical_regardless_of_input_order(self) -> None: + # The blob is a cached build artifact, so it may not depend on the + # order buck2 lists the libraries in, nor on the order `compile.py` + # happened to write entries within one. + modules = { + "a.py": "VALUE = 1", + "b/__init__.py": "VALUE = 2", + "b/c.py": "VALUE = 3", + } + forward = self._library("forward", modules) + backward = self._library("backward", dict(reversed(list(modules.items())))) + other = self._library("other", {"d.py": "VALUE = 4"}) + + self.assertEqual( + self._build(forward, other).read_bytes(), + self._build(other, backward).read_bytes(), + ) + + def test_duplicate_path_keeps_the_first_library(self) -> None: + # Two libraries can legitimately land the same path in the tree. + left = self._library("left", {"dup.py": "OWNER = 'left'"}) + right = self._library("right", {"dup.py": "OWNER = 'right'"}) + + bundle = self._bundle(left, right) + + self.assertEqual(list(bundle.index), ["dup.py"]) + self.assertEqual(bundle.run("dup.py")["OWNER"], "left") + + def test_the_pyc_path_in_the_manifest_is_never_read(self) -> None: + # It addresses the action's own output through an `/output_artifacts/` + # placeholder standing in for a content hash that does not exist while + # the action runs. The bytes come from the paired directory instead, so + # a manifest full of nonsense there still bundles correctly. + library = self._library( + "lib", + {"pkg/mod.py": "VALUE = 42"}, + pyc_field="buck-out/v2/gen/fbcode/output_artifacts/aaaaaaaaaaaaaaaa/x", + ) + + self.assertEqual(self._bundle(library).run("pkg/mod.py")["VALUE"], 42) + + def test_rejects_pyc_from_a_different_interpreter(self) -> None: + library = self._library("lib", {"mod.py": "VALUE = 1"}) + pyc = library.pycs["mod.py"] + pyc.write_bytes( + b"\0" * len(MAGIC_NUMBER) + pyc.read_bytes()[len(MAGIC_NUMBER) :] + ) + + with self.assertRaisesRegex(ValueError, "wrong interpreter magic"): + self._build(library) + + def test_rejects_truncated_pyc(self) -> None: + library = self._library("lib", {"mod.py": "VALUE = 1"}) + pyc = library.pycs["mod.py"] + data = pyc.read_bytes() + + for size in ( + gen_bytecode_bundle.PYC_HEADER_LEN - 1, + gen_bytecode_bundle.PYC_HEADER_LEN, + ): + with self.subTest(size=size): + pyc.write_bytes(data[:size]) + with self.assertRaisesRegex( + ValueError, "Truncated .pyc has no bytecode payload" + ): + self._build(library) + + def test_payload_is_streamed_rather_than_buffered(self) -> None: + # A binary's whole bytecode must never be resident at once. + blob = "x" * (8 * 1024 * 1024) + library = self._library("large", {"large.py": f"VALUE = '{blob}'"}) + + # `get_traced_memory` reports the whole process, and `start()` is a + # no-op while tracing is already on, so measure what `_build` adds on + # top of what is already live rather than the absolute peak. + was_tracing = tracemalloc.is_tracing() + tracemalloc.start() + tracemalloc.reset_peak() + baseline, _peak = tracemalloc.get_traced_memory() + try: + out = self._build(library) + _current, peak = tracemalloc.get_traced_memory() + finally: + if not was_tracing: + tracemalloc.stop() + + self.assertGreater(out.stat().st_size, len(blob)) + self.assertLess(peak - baseline, len(blob) // 2) + + +class CollectTest(unittest.TestCase): + def test_unequal_lengths_are_an_error(self) -> None: + # Position is the only thing tying the two lists together. If they stop + # lining up, every pair past the mismatch reads one library's manifest + # against another library's directory - silently wrong rather than + # broken, so refuse to build at all. + for manifests, dirs in ((["unused"], []), ([], ["unused"])): + with self.subTest(manifests=manifests, dirs=dirs): + with self.assertRaises(ValueError): + gen_bytecode_bundle.collect(manifests, dirs) + + +class MainTest(BundleTestBase): + def test_reads_its_inputs_from_files(self) -> None: + # Large binaries have thousands of libraries, well past the argv limit, + # so both lists arrive as files. + libraries = [ + self._library("left", {"pkg/one.py": "VALUE = 1"}), + self._library("right", {"pkg/two.py": "VALUE = 2"}), + ] + (self.root / "manifests.txt").write_text( + "".join(f"{lib.manifest}\n" for lib in libraries), encoding="utf8" + ) + (self.root / "artifacts.txt").write_text( + "".join(f"{lib.bytecode_dir}\n" for lib in libraries), encoding="utf8" + ) + out = self.root / "bundle.bin" + + gen_bytecode_bundle.main( + [ + "gen_bytecode_bundle.py", + f"--output={out}", + f"--bytecode-manifests-from={self.root / 'manifests.txt'}", + f"--bytecode-artifacts-from={self.root / 'artifacts.txt'}", + ] + ) + + self.assertEqual(set(self._unpack(out).index), {"pkg/one.py", "pkg/two.py"}) diff --git a/prelude/python/tools/tests/make_py_package_manifest_module_test.py b/prelude/python/tools/tests/make_py_package_manifest_module_test.py new file mode 100644 index 0000000..23e35c6 --- /dev/null +++ b/prelude/python/tools/tests/make_py_package_manifest_module_test.py @@ -0,0 +1,86 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is dual-licensed under either the MIT license found in the +# LICENSE-MIT file in the root directory of this source tree or the Apache +# License, Version 2.0 found in the LICENSE-APACHE file in the root directory +# of this source tree. You may select, at your option, one of the +# above-listed licenses. + +# pyre-strict + +import json +import sys +import tempfile +import unittest +from pathlib import Path + +import make_py_package_manifest_module + + +class MakePyPackageManifestModuleTest(unittest.TestCase): + def test_merge_entries(self) -> None: + entries: dict[str, object] = { + "fbmake": {"preserved": "base", "replaced": "old"}, + "replaced": "old", + } + + make_py_package_manifest_module.merge_entries( + entries, + { + "fbmake": {"replaced": "new"}, + "new": "value", + "replaced": {"nested": True}, + }, + ) + + self.assertEqual( + { + "fbmake": {"preserved": "base", "replaced": "new"}, + "new": "value", + "replaced": {"nested": True}, + }, + entries, + ) + + def test_main_writes_merged_json_before_module_entries(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + entries = root / "entries.json" + overlay = root / "overlay.json" + module_manifest = root / "module-manifest.json" + module_manifests = root / "module-manifests.txt" + output = root / "__manifest__.py" + output_json = root / "__manifest__.json" + entries.write_text(json.dumps({"fbmake": {"preserved": "base"}})) + overlay.write_text(json.dumps({"fbmake": {"revision": "fresh"}})) + module_manifest.write_text( + json.dumps([["pkg/module.py", "unused", "origin"]]) + ) + module_manifests.write_text(str(module_manifest) + "\n") + + old_argv = sys.argv + sys.argv = [ + "make_py_package_manifest_module.py", + "--manifest-entries", + str(entries), + "--manifest-entries-overlay", + str(overlay), + "--module-manifests", + str(module_manifests), + "--output", + str(output), + "--output-json", + str(output_json), + ] + try: + make_py_package_manifest_module.main() + finally: + sys.argv = old_argv + + self.assertEqual( + {"fbmake": {"preserved": "base", "revision": "fresh"}}, + json.loads(output_json.read_text()), + ) + module_text = output.read_text() + self.assertIn("'pkg.module'", module_text) + self.assertIn("'pkg.__init__'", module_text) diff --git a/prelude/python/tools/tests/type_check_result_to_validation_test.py b/prelude/python/tools/tests/type_check_result_to_validation_test.py index 9ed199b..3d9982e 100644 --- a/prelude/python/tools/tests/type_check_result_to_validation_test.py +++ b/prelude/python/tools/tests/type_check_result_to_validation_test.py @@ -73,6 +73,16 @@ def test_only_blocking_diagnostics_appear_in_failure(self) -> None: }, ) + def test_unused_ignores_never_block(self) -> None: + for name in ("unused-ignore", "unused-type-ignore"): + with self.subTest(name=name): + self.assertEqual( + self._convert( + {"errors": [{"code": -2, "name": name, "severity": "error"}]} + ), + {"version": 1, "data": {"status": "success"}}, + ) + def test_malformed_error_preserves_blocking_diagnostics(self) -> None: result = self._convert( { diff --git a/prelude/python/tools/tests/wheel_test.py b/prelude/python/tools/tests/wheel_test.py index e649a75..61a96c3 100644 --- a/prelude/python/tools/tests/wheel_test.py +++ b/prelude/python/tools/tests/wheel_test.py @@ -8,6 +8,8 @@ # pyre-strict +import csv +import io import tempfile import unittest import zipfile @@ -40,3 +42,34 @@ def test_readme_sets_metadata_body_and_content_type(self) -> None: self.assertIn("Description-Content-Type: text/markdown\n", metadata) self.assertTrue(metadata.endswith("\n# Demo\n\nLong description.\n")) + + def test_record_round_trips_paths_containing_commas(self) -> None: + # Resource filenames with commas are real: vLLM ships fused-MoE configs + # named like `E=128,N=768,device_name=NVIDIA_H100.json`. + comma_path = "demo/E=128,N=768,device_name=X.json" + + with tempfile.TemporaryDirectory() as tmpdir: + tmp = Path(tmpdir) + output = tmp / "demo-1.0-py3-none-any.whl" + plain = tmp / "demo.py" + config = tmp / "config.json" + + plain.write_text("print('ok')\n", encoding="utf-8") + config.write_text("{}\n", encoding="utf-8") + + with wheel.WheelBuilder( + name="demo", version="1.0", output=str(output) + ) as whl: + whl.write("demo.py", str(plain)) + whl.write(comma_path, str(config)) + + with zipfile.ZipFile(output) as archive: + record = archive.read("demo-1.0.dist-info/RECORD").decode("utf-8") + + rows = [row for row in csv.reader(io.StringIO(record)) if row] + self.assertTrue(all(len(row) == 3 for row in rows), record) + paths = [row[0] for row in rows] + self.assertIn(comma_path, paths) + # Paths needing no quoting stay byte-identical to the plain form, + # so this change is inert for wheels that have no such filenames. + self.assertIn("demo.py,,\n", record) diff --git a/prelude/python/tools/type_check_result_to_validation.py b/prelude/python/tools/type_check_result_to_validation.py index 4d93a70..552c19a 100644 --- a/prelude/python/tools/type_check_result_to_validation.py +++ b/prelude/python/tools/type_check_result_to_validation.py @@ -44,7 +44,7 @@ def convert_type_check_result(input_path: Path, output_path: Path) -> None: continue if ( error["code"] == 0 - or error.get("name") == "unused-ignore" + or error.get("name") in ("unused-ignore", "unused-type-ignore") or error.get("severity") in ("info", "ignore", "warn") ): continue diff --git a/prelude/python/tools/wheel.py b/prelude/python/tools/wheel.py index 549c8af..6be1aa5 100644 --- a/prelude/python/tools/wheel.py +++ b/prelude/python/tools/wheel.py @@ -13,6 +13,7 @@ import argparse import configparser import contextlib +import csv import io import json import os @@ -125,9 +126,16 @@ def writestr(self, dst: str, contents: str) -> None: def _write_record(self) -> None: record = self._dist_info("RECORD") + # RECORD is CSV, so a path containing a comma has to be quoted or it + # parses as extra fields and strict installers reject the wheel. For + # paths needing no quoting csv.writer emits exactly `path,,`. + buf = io.StringIO() + writer = csv.writer(buf, lineterminator="\n") + for path in self._record + [record]: + writer.writerow([path, "", ""]) self._outf.writestr( zinfo_or_arcname=zipfile.ZipInfo(filename=record), - data="".join([f"{f},,\n" for f in (self._record + [record])]), + data=buf.getvalue(), ) def close(self) -> None: diff --git a/prelude/python/transitions.bzl b/prelude/python/transitions.bzl index 2fb6280..dd740fd 100644 --- a/prelude/python/transitions.bzl +++ b/prelude/python/transitions.bzl @@ -35,10 +35,12 @@ def _transition_opt_by_default_impl(platform: PlatformInfo, refs: struct, attrs: mode_constraint = constraints[refs._opt_by_default__opt[ConstraintValueInfo].setting.label].label is_dev = mode_constraint == refs._opt_by_default__dev[ConstraintValueInfo].label - is_opt = mode_constraint == refs._opt_by_default__opt[ConstraintValueInfo].label - # Check if the build mode is either dev or opt. If not, cancel the transition - if not is_dev and not is_opt: + # This transition upgrades dev to opt: cancel it unless the build mode is + # dev. In particular, an opt configuration is left untouched so that python + # targets and their dependencies share the configuration -- and its + # outputs -- with everything else built in that opt configuration. + if not is_dev: return platform no_san_label = refs._opt_by_default__no_san[ConstraintValueInfo].setting.label @@ -48,30 +50,35 @@ def _transition_opt_by_default_impl(platform: PlatformInfo, refs: struct, attrs: ) # this bad boy only shows up in default dev mode 🙏 is_no_san = sanitizer_constraint == refs._opt_by_default__no_san[ConstraintValueInfo].label - if is_dev and not (is_default_dev_sanitizer or is_no_san): + if not (is_default_dev_sanitizer or is_no_san): # dev-tsan/dev-asan etc modes still appear as dev, here we check that the user has not specifically requested sanitizers # returning here preserves the original behaviour of opt-by-default, but we can likely give a opt + sanitizer config here later return platform - # opt mode comes with nosan by default, if its not default opt then cancel the transition - if is_opt and not is_no_san: - return platform - - maybe_lto_constraint = getattr(constraints.get(refs._opt_by_default__lto_none[ConstraintValueInfo].setting.label), "label", None) - is_lto_none = not maybe_lto_constraint or maybe_lto_constraint == refs._opt_by_default__lto_none[ConstraintValueInfo].label - - # if opt-lto is then cancel the transition - if is_opt and not is_lto_none: - return platform - # if opt-by-default is not enabled then cancel the transition if not attrs.opt_by_default_enabled: return platform + # Dev: apply the constraint values the opt mode sets, and nothing else, so + # the result is as close to a plain opt configuration as possible. Values + # an opt configuration derives from these (native linking, split debug info + # handling) follow the same way they do in opt. + # + # `default_opt_cxx[enabled]` is required on top of `opt`, not redundant with + # it: a mode file that sets `fbcode.build_mode_partially_selectified_test_label` + # (dev-nosan and every other partially selectified mode) pins the fbcode C/C++ + # toolchain's mode at loading time, so `core_build_mode[opt]` alone leaves the + # deps compiling with the dev mode's flags -- no `-DNDEBUG` -- while targets + # that select on the opt constraint themselves do get it, which breaks the + # link (e.g. rocksdb's `TEST_SYNC_POINT`). + # + # The default python package style is deliberately not forced to the opt + # value: the python target itself is configured with the transitioned + # configuration, so that would also repackage dev python binaries as + # standalone. opt_by_default_constraints = [ refs._opt_by_default__fbcode_build_info_mode_full[ConstraintValueInfo], refs._opt_by_default__static[ConstraintValueInfo], - refs._opt_by_default__split_dwarf_single[ConstraintValueInfo], refs._opt_by_default__opt_cxx_enabled[ConstraintValueInfo], refs._opt_by_default__no_san[ConstraintValueInfo], refs._opt_by_default__opt[ConstraintValueInfo], @@ -96,11 +103,9 @@ def _refs(): "_opt_by_default__execution_platform_marker": "@config//platform/execution/constraints:execution-platform-transitioned", "_opt_by_default__fbcode_build_info_mode_full": "@config//build_mode/constraints:fbcode-build-info-mode[full]", "_opt_by_default__linux": "@config//os/constraints:linux", - "_opt_by_default__lto_none": "@config//build_mode/constraints:lto-none", "_opt_by_default__no_san": "@config//build_mode:sanitizer_type[no-san]", "_opt_by_default__opt": "@config//build_mode/constraints:opt", "_opt_by_default__opt_cxx_enabled": "@config//build_mode/default_opt_cxx:enabled", - "_opt_by_default__split_dwarf_single": "@config//build_mode/constraints:debug_style[split-dwarf-single]", "_opt_by_default__static": "@config//build_mode/constraints:default_link_style[static]", "_opt_by_default_native_debug_enabled": "@config//build_mode/constraints:native-debugging[supported]", } diff --git a/prelude/python/typing.bzl b/prelude/python/typing.bzl index fdb16bf..80c7102 100644 --- a/prelude/python/typing.bzl +++ b/prelude/python/typing.bzl @@ -176,16 +176,11 @@ def create_per_target_type_check( ) -> DefaultInfo: if not typing_enabled: # Use empty dict to signal that no type checking was performed. - output_file = ctx.actions.write_json("type_check_result.json", {}, has_content_based_path = False) - sharded_output_file = ctx.actions.write_json( - "sharded_type_check_result.json", - {}, - has_content_based_path = False, - ) + empty_result = ctx.attrs._python_internal_tools[PythonInternalToolsInfo].empty_typing_result return DefaultInfo( - default_output = output_file, + default_output = empty_result, sub_targets = { - "shard_default": [DefaultInfo(default_output = sharded_output_file)], + "shard_default": [DefaultInfo(default_output = empty_result)], }, ) diff --git a/prelude/resources.bzl b/prelude/resources.bzl index a0806b8..cc32c24 100644 --- a/prelude/resources.bzl +++ b/prelude/resources.bzl @@ -19,6 +19,11 @@ ResourceInfo = provider( } ) +_EMPTY_RESOURCE_INFO = ResourceInfo(resources = {}) + +def make_resource_info(resources: dict[Label, dict[str, ArtifactOutputs]]) -> ResourceInfo: + return _EMPTY_RESOURCE_INFO if not resources else ResourceInfo(resources = resources) + def create_relocatable_resources_info( ctx: AnalysisContext, name: str, resources: dict[str, ArtifactOutputs], has_content_based_path: bool = False ) -> [ArgLike, Artifact]: diff --git a/prelude/rules.bzl b/prelude/rules.bzl index 9de654f..19d19ae 100644 --- a/prelude/rules.bzl +++ b/prelude/rules.bzl @@ -55,9 +55,8 @@ def _mk_rule(rule_spec: typing.Any, extra_attrs: dict[str, typing.Any] = dict(), cfg = rule_spec.cfg - extra_args = dict(kwargs) if cfg != None: - extra_args["cfg"] = cfg + kwargs["cfg"] = cfg if rule_spec.docs: doc = rule_spec.docs @@ -73,7 +72,7 @@ def _mk_rule(rule_spec: typing.Any, extra_attrs: dict[str, typing.Any] = dict(), if rule_spec.further: doc += "\n{}Additional notes:\n{}".format(" " * 8, rule_spec.further) - extra_args["doc"] = doc + kwargs["doc"] = doc impl = rule_spec.impl extra_impl = getattr(extra_implemented_rules, name, None) @@ -86,9 +85,9 @@ def _mk_rule(rule_spec: typing.Any, extra_attrs: dict[str, typing.Any] = dict(), if impl_override != None: impl = impl_override if rule_spec.uses_plugins != None: - extra_args["uses_plugins"] = rule_spec.uses_plugins + kwargs["uses_plugins"] = rule_spec.uses_plugins if rule_spec.supports_incoming_transition != None: - extra_args["supports_incoming_transition"] = rule_spec.supports_incoming_transition + kwargs["supports_incoming_transition"] = rule_spec.supports_incoming_transition is_toolchain_rule = rule_spec.is_toolchain_rule is_toolchain_rule_via_rule_name = name in toolchain_rule_names @@ -97,9 +96,9 @@ def _mk_rule(rule_spec: typing.Any, extra_attrs: dict[str, typing.Any] = dict(), elif is_toolchain_rule_via_rule_name: fail("Cannot set `is_toolchain_rule` on `prelude_rule` and also via `toolchain_rule_names`") - extra_args.setdefault("is_configuration_rule", name in _config_implemented_rules) - extra_args.setdefault("is_toolchain_rule", is_toolchain_rule) - return rule(impl = impl, attrs = attributes, **extra_args) + kwargs.setdefault("is_configuration_rule", name in _config_implemented_rules) + kwargs.setdefault("is_toolchain_rule", is_toolchain_rule) + return rule(impl = impl, attrs = attributes, **kwargs) def _categorized_decls(): grouped_decls = {} diff --git a/prelude/rules_impl.bzl b/prelude/rules_impl.bzl index e803257..919ad1d 100644 --- a/prelude/rules_impl.bzl +++ b/prelude/rules_impl.bzl @@ -18,6 +18,7 @@ load("@prelude//:sh_test.bzl", "sh_test_impl") load("@prelude//:test_suite.bzl", "test_suite_impl") load("@prelude//android:android.bzl", _android_implemented_rules = "implemented_rules") load("@prelude//android:configuration.bzl", "is_building_android_binary_attr") +load("@prelude//android:native_build_commands.bzl", "EMIT_NATIVE_BUILD_COMMANDS") load("@prelude//apple:apple_common.bzl", "apple_common") load("@prelude//apple:apple_rules_decls.bzl", "apple_rules") load("@prelude//apple:apple_rules_impl.bzl", _apple_extra_attributes = "extra_attributes", _apple_implemented_rules = "implemented_rules") @@ -26,6 +27,7 @@ load("@prelude//csharp:csharp.bzl", "csharp_library_impl", "prebuilt_dotnet_libr load("@prelude//cxx:bitcode.bzl", "llvm_link_bitcode_impl") load("@prelude//cxx:cuda.bzl", "CudaCompileStyle") load("@prelude//cxx:cxx.bzl", "cxx_binary_impl", "cxx_library_impl", "cxx_precompiled_header_impl", "cxx_test_impl", "prebuilt_cxx_library_impl") +load("@prelude//cxx:cxx_flags.bzl", "cxx_flags_impl") load("@prelude//cxx:cxx_toolchain.bzl", "cxx_toolchain_extra_attributes", "cxx_toolchain_impl") load("@prelude//cxx:cxx_toolchain_types.bzl", "CxxPlatformInfo", "CxxToolchainInfo") load("@prelude//cxx:headers.bzl", "CPrecompiledHeaderInfo", "HeaderMode") @@ -177,6 +179,7 @@ extra_implemented_rules = struct( prebuilt_dotnet_library = prebuilt_dotnet_library_impl, # c++ cxx_binary = cxx_binary_impl, + cxx_flags = cxx_flags_impl, cxx_test = cxx_test_impl, cxx_toolchain = cxx_toolchain_impl, cxx_genrule = genrule_impl, @@ -305,6 +308,7 @@ control how the dependencies of this library are linked, use `link_style` instea "third_party_project": attrs.option(attrs.string(), default = None), "_cxx_hacks": attrs.default_only(attrs.dep(default = "prelude//cxx/tools:cxx_hacks")), "_cxx_toolchain": toolchains_common.cxx(), + "_emit_native_build_commands": attrs.default_only(attrs.bool(default = EMIT_NATIVE_BUILD_COMMANDS)), "_is_building_android_binary": is_building_android_binary_attr(), } | apple_common.extra_xcode_sources() @@ -312,6 +316,13 @@ control how the dependencies of this library are linked, use `link_style` instea ) cxx_extra_attributes = { + "cxx_binary": { + "_gen_build_info": attrs.option(attrs.exec_dep(providers = [RunInfo]), default = None), + "_generated_build_info_data": attrs.option(attrs.source(), default = None), + "_generated_build_info_enabled": attrs.bool(default = False), + "_generated_build_info_mode": attrs.enum(["full", "stable"], default = "stable"), + "_generated_build_info_spec": BUILD_INFO_ATTR, + }, "cxx_genrule": genrule_attributes() | { "_cxx_toolchain": toolchains_common.cxx(), @@ -319,7 +330,14 @@ cxx_extra_attributes = { }, "cxx_library": _cxx_extra_library_attrs, "cxx_precompiled_header": _cxx_extra_library_attrs, - "cxx_test": re_test_common.test_args(), + "cxx_test": re_test_common.test_args() + | { + "_gen_build_info": attrs.option(attrs.exec_dep(providers = [RunInfo]), default = None), + "_generated_build_info_data": attrs.option(attrs.source(), default = None), + "_generated_build_info_enabled": attrs.bool(default = False), + "_generated_build_info_mode": attrs.enum(["full", "stable"], default = "stable"), + "_generated_build_info_spec": BUILD_INFO_ATTR, + }, "cxx_toolchain": cxx_toolchain_extra_attributes(is_toolchain_rule = False), "llvm_link_bitcode": { "_cxx_toolchain": toolchains_common.cxx(), @@ -487,7 +505,22 @@ _python_extra_attributes = { } _rust_extra_attributes = { - "rust_test": {}, + "rust_binary": { + "_gen_build_info": attrs.option(attrs.exec_dep(providers = [RunInfo]), default = None), + "_generated_build_info_args": attrs.list(attrs.arg(), default = []), + "_generated_build_info_data": attrs.option(attrs.source(), default = None), + "_generated_build_info_enabled": attrs.bool(default = False), + "_generated_build_info_mode": attrs.enum(["full", "stable"], default = "stable"), + "_generated_build_info_spec": BUILD_INFO_ATTR, + }, + "rust_test": { + "_gen_build_info": attrs.option(attrs.exec_dep(providers = [RunInfo]), default = None), + "_generated_build_info_args": attrs.list(attrs.arg(), default = []), + "_generated_build_info_data": attrs.option(attrs.source(), default = None), + "_generated_build_info_enabled": attrs.bool(default = False), + "_generated_build_info_mode": attrs.enum(["full", "stable"], default = "stable"), + "_generated_build_info_spec": BUILD_INFO_ATTR, + }, } _core_extra_attributes = ( diff --git a/prelude/rust/build.bzl b/prelude/rust/build.bzl index aa300bf..b56d1e5 100644 --- a/prelude/rust/build.bzl +++ b/prelude/rust/build.bzl @@ -102,7 +102,9 @@ load( ":context.bzl", "CommonArgsInfo", "CompileContext", + "DependencyArgsInfo", "output_filename", + "strip_build_info_linker_flags", ) load( ":crate_name.bzl", @@ -467,11 +469,13 @@ def generate_rustdoc_test( LinkExtraction = record( # cmd_script set via `-Clinker=`. linker_wrapper = field(typing.Any), - # The rustc-synthesized objects, listed as an argsfile. Empty when they - # are archived instead. + # The retained linker args. Contains no paths, only flags, and so is safe + # to pass along verbatim even under content-based paths. out_argsfile = field(Artifact), # Extracted link inputs directory. out_artifacts_dir = field(Artifact), + # Basenames of the extracted objects, one per line, in link order. + out_manifest = field(Artifact), # Archive of the rustc-produced objects. Only built when used by # distributed thinlto. out_archive = field(Artifact | None), @@ -508,10 +512,12 @@ def _archiver_command(ctx: AnalysisContext, compile_ctx: CompileContext, subdir: def _setup_link_extraction(ctx: AnalysisContext, compile_ctx: CompileContext, subdir: str, emit_cbp: bool, archive_objects: bool) -> LinkExtraction: out_argsfile = ctx.actions.declare_output(subdir + "/extracted-link-args.args", has_content_based_path = emit_cbp) out_artifacts_dir = ctx.actions.declare_output(subdir + "/extracted-link-artifacts", dir = True, has_content_based_path = emit_cbp) + out_manifest = ctx.actions.declare_output(subdir + "/extracted-link-manifest.txt", has_content_based_path = emit_cbp) linker_cmd = cmd_args( compile_ctx.internal_tools_info.extract_link_action, cmd_args(out_argsfile.as_output(), format = "--out_argsfile={}"), cmd_args(out_artifacts_dir.as_output(), format = "--out_artifacts={}"), + cmd_args(out_manifest.as_output(), format = "--out_manifest={}"), ) out_archive = None @@ -533,107 +539,10 @@ def _setup_link_extraction(ctx: AnalysisContext, compile_ctx: CompileContext, su linker_wrapper = linker_wrapper, out_argsfile = out_argsfile, out_artifacts_dir = out_artifacts_dir, + out_manifest = out_manifest, out_archive = out_archive, ) -def _rust_cxx_link( - ctx: AnalysisContext, - compile_ctx: CompileContext, - extraction: LinkExtraction, - crate_type: CrateType, - reloc_model: RelocModel, - dist_thin_lto_codegen_flags: list[typing.Any], - inherited_link_args: LinkArgs, - extra_link_args: list[typing.Any], - external_debug_info: ArtifactTSet, - import_library: Artifact | None, - import_library_args: list[typing.Any], - output: Artifact, - identifier: str | None, - enable_distributed_thinlto: bool, -) -> CxxLinkResult: - rust_argsfile = cmd_args(extraction.out_argsfile, format = "@{}", hidden = extraction.out_artifacts_dir) - if extraction.out_archive == None: - rust_objects = LinkArgs(flags = [rust_argsfile]) - else: - rust_objects = LinkArgs( - infos = [ - LinkInfo( - name = "rust_objects", - dist_thin_lto_codegen_flags = dist_thin_lto_codegen_flags, - pre_flags = [rust_argsfile], - linkables = [ - ArchiveLinkable( - archive = Archive(artifact = extraction.out_archive), - linker_type = compile_ctx.cxx_toolchain_info.linker_info.type, - # link_whole because everything rustc emits has to end up in the input. - link_whole = True, - ) - ], - ) - ] - ) - - is_shared = crate_type in [CrateType("dylib"), CrateType("cdylib")] - - links = [ - LinkArgs(flags = compile_ctx.linker_pre_args), - LinkArgs(flags = extra_link_args), - ] - - if compile_ctx.cxx_toolchain_info.linker_info.type == LinkerType("gnu"): - # FIXME(JakobDegen): This is here becuase rustc used to pass it, but it's not something - # we should be passing blindly on the user's behalf. Unfortunately some builds break if - # you take it out. - flags = ["-Wl,--as-needed"] - - # lld deduplicates `.debug_str` only at `-O1`, and rustc always - # passes this for links it drives, overriding toolchains that link - # at `-O0` for speed (fbcode's opt-lg). Without the deduplication, - # very large binaries overflow the 32-bit `.debug_str` offsets in - # `.debug_names`. - flags.append("-Wl,-O1") - - # The objects were codegened at the relocation model chosen in - # `_get_reloc_model`, which the toolchain's link flags cannot know - # about. Pin the PIE-ness of the executable to match, as rustc does - # for links it drives. - if not is_shared: - flags.append("-no-pie" if reloc_model == RelocModel("static") else "-pie") - - links.append(LinkArgs(flags = flags)) - - links += [ - rust_objects, - inherited_link_args, - LinkArgs(flags = import_library_args), - LinkArgs(infos = [LinkInfo(external_debug_info = external_debug_info)]), - ] - if not is_shared and link_cxx_binary_locally(ctx, compile_ctx.cxx_toolchain_info): - link_execution_preference = LinkExecutionPreference("local") - else: - link_execution_preference = LinkExecutionPreference("any") - - return cxx_link_into( - ctx = ctx, - output = output, - result_type = CxxLinkResultType("shared_library" if is_shared else "executable"), - opts = link_options( - links = links, - link_execution_preference = link_execution_preference, - # The link is local only so the fbcode linker wrapper can stamp - # build info from the repo (D115067153); dwp needs no repo and is - # memory-hungry enough that pinning it to the link's host thrashes - # small workers. Let it schedule anywhere, matching the - # non-deferred-link path, which runs dwp with no preference. - dwp_execution_preference = LinkExecutionPreference("any"), - category_suffix = "rust_dylib" if is_shared else "rust_binary", - identifier = identifier, - import_library = import_library, - enable_distributed_thinlto = enable_distributed_thinlto, - ), - ) - # Generate a compilation action. A single instance of rustc can emit # numerous output artifacts, so return an artifact object for each of # them. @@ -646,6 +555,10 @@ def rust_compile( incremental_enabled: bool, extra_link_args: list[typing.Any] = [], predeclared_output: Artifact | None = None, + # Whether `predeclared_output` was declared with a content-based path. The + # linker writes its side outputs (the `.pdb` on Windows) beside the binary, + # so they must be declared with the binary's path mode to be found. + predeclared_output_has_content_based_path: bool = False, extra_flags: list[str | ResolvedStringWithMacros | Artifact] = [], allow_cache_upload: bool = False, # Setting this to true causes the diagnostic outputs that are generated @@ -653,9 +566,9 @@ def rust_compile( # compilation fails. This should not generally be used if the "real" # output of the action is going to be depended on infallible_diagnostics: bool = False, - rust_cxx_link_group_info: RustCxxLinkGroupInfo | None = None, transformation_spec_context: TransformationSpecContext | None = None, profile_mode: ProfileMode | None = None, + precomputed_inherited_link_args: LinkArgs | None = None, ) -> RustcOutput: toolchain_info = compile_ctx.toolchain_info @@ -666,8 +579,6 @@ def rust_compile( fail("`{}` profiling uses unstable `-Z` flags and requires a toolchain with `nightly_features = True`".format(profile_mode.value)) lints = _lint_flags(compile_ctx, infallible_diagnostics, emit == Emit("clippy")) - use_cbp = getattr(ctx.attrs, "use_content_based_paths", False) - emit_cbp = use_cbp if predeclared_output == None else False common_args = _compute_common_args( ctx = ctx, @@ -683,9 +594,26 @@ def rust_compile( ) requires_linking = crate_type_linked(params.crate_type) and emit == Emit("link") + extracts_objects = emit == Emit("rlib") and params.crate_type == CrateType("bin") + + if ( + emit == Emit("link") + and compile_ctx.toolchain_info.advanced_unstable_linking + # FIXME(JakobDegen): We should probably not support cdylib or staticlib under AUL + and params.crate_type not in [CrateType("rlib"), CrateType("proc-macro"), CrateType("staticlib"), CrateType("cdylib")] + ): + fail('rustc-driven linking is not supported with `advanced_unstable_linking`: binaries compile via `Emit("rlib")` and link through cxx') + + if extracts_objects: + if not compile_ctx.toolchain_info.advanced_unstable_linking: + fail( + '`Emit("rlib")` for a bin crate requires `advanced_unstable_linking`: without it, dependency link providers do not carry the rlibs for cxx to link' + ) + if predeclared_output != None: + fail("extraction produces no linked output; the caller owns the linked artifact and must produce it via `rust_link_binary`") - deferred_link = requires_linking and deferred_link_enabled(compile_ctx, params, emit) - dist_thinlto = deferred_link and _dist_thinlto_enabled(ctx, compile_ctx) + use_cbp = getattr(ctx.attrs, "use_content_based_paths", False) + emit_cbp = use_cbp if predeclared_output == None else predeclared_output_has_content_based_path rustc_cmd = cmd_args( # Lints go first to allow other args to override them. @@ -733,7 +661,6 @@ def rust_compile( params = params, predeclared_output = predeclared_output, incremental_enabled = incremental_enabled, - deferred_link = deferred_link, profile_mode = profile_mode, ) @@ -760,13 +687,18 @@ def rust_compile( split_debug_mode = compile_ctx.cxx_toolchain_info.split_debug_mode or SplitDebugMode("none") has_split_debug = split_debug_mode != SplitDebugMode("none") + strip_dwo_members = _strip_dwo_members_args( + compile_ctx = compile_ctx, + emit = emit, + crate_type = params.crate_type, + output = emit_op.output, + has_split_debug = has_split_debug, + ) + import_library = None pdb_artifact = None dwp_inputs = [] link_extraction = None - cxx_inherited_link_args = None - cxx_import_library_args = [] - inherited_debug_info = ArtifactTSet() if requires_linking: if params.crate_type in [CrateType("cdylib"), CrateType("dylib")]: linker_info = compile_ctx.cxx_toolchain_info.linker_info @@ -785,102 +717,99 @@ def rust_compile( else: import_library_args = [] - inherited_link_args = _inherited_link_args( + subdir = common_args.subdir + tempfile = common_args.tempfile + + inherited_link_args = precomputed_inherited_link_args + if inherited_link_args == None: + inherited_link_args = get_inherited_link_args( + ctx, + compile_ctx, + params.dep_link_strategy, + None, # link group info + transformation_spec_context, + ) + + link_args_output = make_link_args( ctx, - compile_ctx, - params.dep_link_strategy, - rust_cxx_link_group_info, - transformation_spec_context, + ctx.actions, + compile_ctx.cxx_toolchain_info, + [ + LinkArgs(flags = extra_link_args), + inherited_link_args, + LinkArgs(flags = import_library_args), + ], + output_short_path = emit_op.output.short_path, + has_content_based_path = emit_cbp, ) + separate_debug_info_args = cmd_args() if has_split_debug: inherited_debug_info = inherited_external_debug_info( ctx = ctx, dep_ctx = compile_ctx.dep_ctx, dep_link_strategy = params.dep_link_strategy, ) + external_debug_infos = project_artifacts(ctx.actions, inherited_debug_info) + dwp_inputs.extend(external_debug_infos) + + # Pass to the link wrapper the paths to the .dwo/.o files to rewrite, if we are + # using split debug with content-based paths. + if ( + compile_ctx.cxx_toolchain_info.cxx_compiler_info.supports_content_based_paths + and + # Darwin does not embed paths in object files themselves, but rather + # the linker writes those paths based on the location of object files passed + # to the link. + compile_ctx.cxx_toolchain_info.linker_info.type != LinkerType("darwin") + ): + # Note: Unlike in C++, Rust does binary linking and object code generation in the same + # action. As a result, the .dwo files generated by that action are missing from the + # inputs here and so do not participate in the re-writing. Right now that doesn't matter + # anyway because Rust doesn't have content addressed artifacts. In the future that may + # be a source of bugs though. + separate_debug_info_path_file, _ = ctx.actions.write( + "{}/__{}_dwo_paths.txt".format(subdir, tempfile), + external_debug_infos, + allow_args = True, + has_content_based_path = False, + ) + separate_debug_info_args = cmd_args( + "--rewrite-content-based-dwo-paths", + separate_debug_info_path_file, + "--content-based-dwo-suffix", + ".dwo" if split_debug_mode == SplitDebugMode("split") else ".o", + ) - if deferred_link: - # The link args are passed to `cxx_link_into` below instead; rustc - # never sees them. - link_extraction = _setup_link_extraction( - ctx, - compile_ctx, - common_args.subdir, - emit_cbp, - archive_objects = dist_thinlto, - ) - linker = link_extraction.linker_wrapper - cxx_inherited_link_args = inherited_link_args - cxx_import_library_args = import_library_args - else: - subdir = common_args.subdir - tempfile = common_args.tempfile - - link_args_output = make_link_args( - ctx, - ctx.actions, - compile_ctx.cxx_toolchain_info, - [ - LinkArgs(flags = extra_link_args), - inherited_link_args, - LinkArgs(flags = import_library_args), - ], - output_short_path = emit_op.output.short_path, - has_content_based_path = emit_cbp, - ) - - separate_debug_info_args = cmd_args() - if has_split_debug: - external_debug_infos = project_artifacts(ctx.actions, inherited_debug_info) - dwp_inputs.extend(external_debug_infos) - - # Pass to the link wrapper the paths to the .dwo/.o files to rewrite, if we are - # using split debug with content-based paths. - if ( - compile_ctx.cxx_toolchain_info.cxx_compiler_info.supports_content_based_paths - and - # Darwin does not embed paths in object files themselves, but rather - # the linker writes those paths based on the location of object files passed - # to the link. - compile_ctx.cxx_toolchain_info.linker_info.type != LinkerType("darwin") - ): - # Note: Unlike in C++, Rust does binary linking and object code generation in the same - # action. As a result, the .dwo files generated by that action are missing from the - # inputs here and so do not participate in the re-writing. Right now that doesn't matter - # anyway because Rust doesn't have content addressed artifacts. In the future that may - # be a source of bugs though. - separate_debug_info_path_file, _ = ctx.actions.write( - "{}/__{}_dwo_paths.txt".format(subdir, tempfile), - external_debug_infos, - allow_args = True, - has_content_based_path = False, - ) - separate_debug_info_args = cmd_args( - "--rewrite-content-based-dwo-paths", - separate_debug_info_path_file, - "--content-based-dwo-suffix", - ".dwo" if split_debug_mode == SplitDebugMode("split") else ".o", - ) - - linker_argsfile, _ = ctx.actions.write( - "{}/__{}_linker_args.txt".format(subdir, tempfile), - cmd_args(link_args_output.link_args, separate_debug_info_args), - allow_args = True, - has_content_based_path = False, - ) - linker_argsfile = cmd_args( - linker_argsfile, - hidden = [link_args_output.hidden, separate_debug_info_args], - ) - - pdb_artifact = link_args_output.pdb_artifact - dwp_inputs.append(link_args_output.link_args) + linker_argsfile, _ = ctx.actions.write( + "{}/__{}_linker_args.txt".format(subdir, tempfile), + cmd_args(link_args_output.link_args, separate_debug_info_args), + allow_args = True, + has_content_based_path = False, + ) + linker_argsfile = cmd_args( + linker_argsfile, + hidden = [link_args_output.hidden, separate_debug_info_args], + ) - rustc_cmd.add(cmd_args(linker_argsfile, format = "-Clink-arg=@{}")) - linker = compile_ctx.linker_with_pre_args + pdb_artifact = link_args_output.pdb_artifact + dwp_inputs.append(link_args_output.link_args) - rustc_cmd.add(cmd_args(linker, format = "-Clinker={}")) + rustc_cmd.add(cmd_args(linker_argsfile, format = "-Clink-arg=@{}")) + rustc_cmd.add(cmd_args(compile_ctx.linker_with_pre_args, format = "-Clinker={}")) + elif extracts_objects: + # rustc only compiles; the caller links the extracted objects through + # cxx (see `rust_link_binary`), and the link args are constructed + # there. rustc never sees them. + link_extraction = _setup_link_extraction( + ctx, + compile_ctx, + common_args.subdir, + emit_cbp, + # FIXME(JakobDegen): Better explain why this is needed + archive_objects = _dist_thinlto_enabled(ctx, compile_ctx), + ) + rustc_cmd.add(cmd_args(link_extraction.linker_wrapper, format = "-Clinker={}")) if toolchain_info.rust_target_path != None: emit_op.env["RUST_TARGET_PATH"] = toolchain_info.rust_target_path[DefaultInfo].default_outputs[0] @@ -896,46 +825,21 @@ def rust_compile( rustc_cmd, emit_op.args, ), - required_outputs = [emit_op.output], + required_outputs = [emit_op.output] if emit_op.output else [], is_clippy = emit.value == "clippy", infallible_diagnostics = infallible_diagnostics, allow_cache_upload = allow_cache_upload and emit != Emit("clippy"), crate_map = common_args.crate_map, env = emit_op.env, incremental_enabled = incremental_enabled, - deferred_link = deferred_link, profile_mode = profile_mode, + strip_dwo_members = strip_dwo_members, ) - cxx_link_result = None - if deferred_link: - cxx_external_debug_info = make_artifact_tset( - actions = ctx.actions, - label = ctx.label, - artifacts = [emit_op.extra_out] if has_split_debug else [], - children = [inherited_debug_info], - ) - cxx_link_result = _rust_cxx_link( - ctx = ctx, - compile_ctx = compile_ctx, - extraction = link_extraction, - crate_type = params.crate_type, - reloc_model = params.reloc_model, - # The opt actions re-run codegen and take their relocation model - # from their own command line, so they must match the relocation - # model rustc's own codegen used: PIC for every strategy but - # `static` (see `_get_reloc_model`). - dist_thin_lto_codegen_flags = (compile_ctx.toolchain_info.dist_thin_lto_codegen_flags if params.dep_link_strategy != LinkStrategy("static") else []), - inherited_link_args = cxx_inherited_link_args, - extra_link_args = extra_link_args, - external_debug_info = cxx_external_debug_info, - import_library = import_library, - import_library_args = cxx_import_library_args, - output = emit_op.output, - identifier = invoke.identifier, - enable_distributed_thinlto = dist_thinlto, - ) - filtered_output = cxx_link_result.linked_object.output + if extracts_objects: + # There is no linked artifact; stand in with the manifest of extracted + # objects, which is the closest thing this compile produced. + filtered_output = link_extraction.out_manifest elif infallible_diagnostics and emit != Emit("clippy"): # This is only needed when this action's output is being used as an # input, so we only need standard diagnostics (clippy is always @@ -960,15 +864,13 @@ def rust_compile( ), ) - if emit == Emit("link") and has_split_debug: + if (emit == Emit("link") or emit == Emit("rlib")) and has_split_debug: dwo_output_directory = emit_op.extra_out dwp_inputs.append(dwo_output_directory) else: dwo_output_directory = None - if cxx_link_result != None: - dwp_output = cxx_link_result.linked_object.dwp - elif requires_linking and dwp_available(compile_ctx.cxx_toolchain_info): + if requires_linking and dwp_available(compile_ctx.cxx_toolchain_info): dwp_output = dwp( ctx, compile_ctx.cxx_toolchain_info, @@ -984,7 +886,8 @@ def rust_compile( else: dwp_output = None - if not requires_linking and emit == Emit("link"): + # FIXME(JakobDegen): What's going on with stripped objects in binaries? What is this what cxx does? + if emit in [Emit("rlib"), Emit("link")] and not extracts_objects: stripped_output = strip_debug_info( ctx.actions, paths.join( @@ -1022,12 +925,13 @@ def rust_compile( remarks_json = remarks_json, ), link_output = RustcLinkOutput( - import_library = cxx_link_result.linked_object.import_library if cxx_link_result else import_library, - pdb = cxx_link_result.linked_object.pdb if cxx_link_result else pdb_artifact, + import_library = import_library, + pdb = pdb_artifact, dwp_output = dwp_output, ) if emit == Emit("link") else None, + link_extraction = link_extraction, ) # --extern = for direct dependencies @@ -1198,11 +1102,45 @@ def _rustc_flags(flags: list[str | ResolvedStringWithMacros | Artifact], toolcha # "-Cdebuginfo=2". Rustdoc supports the latter, it just doesn't have the # "-g" shorthand for it. for i, flag in enumerate(flags): - if str(flag) == '"-g"': + flag = str(flag).strip('"') + if flag == "-g": flags[i] = "-Cdebuginfo=2" if toolchain_info.advanced_unstable_linking: - if "-Clink-arg" in str(flag): - fail("-Clink-arg is not supported with advanced_unstable_linking, use the " + "target's or toolchain's `linker_flags` instead") + # The extraction drops rustc's linker argv apart from its synthesized + # inputs, so flags injected via `-Clink-arg` would be silently lost. + # `uses_restricted_rustc_flags` does not exempt this: unlike the flags + # that attribute normally guards, these would not merely be + # unsupported, they would silently not happen. + if "-Clink-arg" in flag or (flag == "-C" and i + 1 < len(flags) and str(flags[i + 1]).strip('"').startswith("link-arg")): + fail("flags passed via `-Clink-arg` are dropped when linking through cxx; use the target's or toolchain's `linker_flags` instead") + + # Additional flags that rustc only forwards to the linker + if "-Cstrip" in flag or (flag == "-C" and i + 1 < len(flags) and str(flags[i + 1]).strip('"').startswith("strip")): + fail("`-Cstrip` has no effect when linking through cxx; use the `symbols` strip_mode, or `-Wl,--strip-all`/`-Wl,-S` in `linker_flags`, instead") + if "-Crpath" in flag or (flag == "-C" and i + 1 < len(flags) and str(flags[i + 1]).strip('"').startswith("rpath")): + fail("`-Crpath` has no effect when linking through cxx; pass `-Wl,-rpath,...` in `linker_flags` instead") + if "-Clink-dead-code" in flag or (flag == "-C" and i + 1 < len(flags) and str(flags[i + 1]).strip('"').startswith("link-dead-code")): + fail("`-Clink-dead-code` has no effect when linking through cxx; pass `-Wl,--no-gc-sections` in `linker_flags` instead") + + if toolchain_info.explicit_sysroot_deps: + # `-Clink-self-contained` selects linker inputs bundled in the + # sysroot, and explicit sysroot deps point rustc at an empty one, + # so everything it can enable is a dangling reference. Only the + # subtractive forms make sense (the toolchain itself uses them to + # keep rustc from consulting the empty sysroot). + value = None + if flag.startswith("-Clink-self-contained"): + value = flag.removeprefix("-Clink-self-contained").removeprefix("=") + elif flag == "-C" and i + 1 < len(flags): + next_flag = str(flags[i + 1]).strip('"') + if next_flag.startswith("link-self-contained"): + value = next_flag.removeprefix("link-self-contained").removeprefix("=") + if value != None: + for part in value.split(","): + if part == "" or part in ("y", "yes", "on", "true") or part.startswith("+"): + fail( + "`-Clink-self-contained` cannot enable bundled linker inputs: explicit sysroot deps leave the sysroot empty, so there is nothing there to link" + ) return flags @@ -1259,6 +1197,7 @@ def _abbreviated_subdir( Emit("llvm-ir-noopt"): "n", Emit("obj"): "o", Emit("link"): "L", + Emit("rlib"): "R", Emit("dep-info"): "d", Emit("mir"): "m", Emit("expand"): "e", @@ -1322,48 +1261,50 @@ def _compute_common_args( if compile_ctx.exec_is_windows: root = root.replace("/", "\\") - # With `advanced_unstable_linking`, we unconditionally pass the metadata - # artifacts. There are two things that work together to make this possible - # in the case of binaries: - # - # 1. The actual rlibs appear in the link providers, so they'll still be - # available for the linker to link in - # 2. The metadata artifacts aren't rmetas, but rather rlibs that just - # don't contain any generated code. Rustc can't distinguish these - # from real rlibs, and so doesn't throw an error - # - # The benefit of doing this is that there's no requirement that the - # dependency's generated code be provided to the linker via an rlib. It - # could be provided by other means, say, a link group dep_metadata_kind = dep_metadata_of_emit(emit) - if compile_ctx.dep_ctx.advanced_unstable_linking or crate_type == CrateType("rlib"): - if dep_metadata_kind == MetadataKind("link"): - dep_metadata_kind = MetadataKind("full") - - dep_args, dep_argsfiles, crate_map = dependency_args( - ctx = ctx, - internal_tools_info = compile_ctx.internal_tools_info, - transitive_dependency_dirs = compile_ctx.transitive_dependency_dirs, - toolchain_info = compile_ctx.toolchain_info, - deps = resolve_rust_deps(ctx, dep_ctx), - subdir = subdir, - dep_link_strategy = params.dep_link_strategy, - dep_metadata_kind = dep_metadata_kind, - is_rustdoc_test = is_rustdoc_test, - ) + if dep_metadata_kind == MetadataKind("link") and (is_rustdoc_test or (compile_ctx.dep_ctx.advanced_unstable_linking and crate_type_linked(crate_type))): + # AUL rlibs compile against full metadata, so Rustdoc tests and + # rustc-linked outputs need the same full metadata deps to avoid crate + # hash mismatches. The linker receives the real rlibs through inherited + # link args. + dep_metadata_kind = MetadataKind("full") + + dependency_args_key = (crate_type, params.dep_link_strategy, dep_metadata_kind, is_rustdoc_test) + cached_dependency_args = compile_ctx.dependency_args.get(dependency_args_key) + if cached_dependency_args == None: + dep_args, dep_argsfiles, crate_map = dependency_args( + ctx = ctx, + internal_tools_info = compile_ctx.internal_tools_info, + transitive_dependency_dirs = compile_ctx.transitive_dependency_dirs, + toolchain_info = compile_ctx.toolchain_info, + deps = resolve_rust_deps(ctx, dep_ctx), + subdir = subdir, + dep_link_strategy = params.dep_link_strategy, + dep_metadata_kind = dep_metadata_kind, + is_rustdoc_test = is_rustdoc_test, + ) - dep_args.add( - cmd_args( - hidden = compile_ctx.transitive_srcs.project_as_args("artifacts") if compile_ctx else [], + dep_args.add( + cmd_args( + hidden = compile_ctx.transitive_srcs.project_as_args("artifacts") if compile_ctx else [], + ) ) - ) - # Add dep_argsfiles to dep_args becuase rustc_action supports nested @argfiles - dep_args.add(dep_argsfiles) + # Add dep_argsfiles to dep_args because rustc_action supports nested @argfiles. + dep_args.add(dep_argsfiles) + + if crate_type == CrateType("proc-macro"): + dep_args.add("--extern=proc_macro") + + cached_dependency_args = DependencyArgsInfo( + args = dep_args, + crate_map = crate_map, + ) + compile_ctx.dependency_args[dependency_args_key] = cached_dependency_args - if crate_type == CrateType("proc-macro"): - dep_args.add("--extern=proc_macro") + dep_args = cached_dependency_args.args + crate_map = cached_dependency_args.crate_map toolchain_info = compile_ctx.toolchain_info edition = ctx.attrs.edition or toolchain_info.default_edition or fail("missing 'edition' attribute, and there is no 'default_edition' set by the toolchain") @@ -1507,6 +1448,27 @@ def _compute_common_args( compile_ctx.common_args[args_key] = common_args return common_args +def rust_compile_invalidation_inputs( + ctx: AnalysisContext, + compile_ctx: CompileContext, + emit: Emit, + params: BuildParams, + default_roots: list[str], + incremental_enabled: bool, +) -> cmd_args: + return _compute_common_args( + ctx = ctx, + compile_ctx = compile_ctx, + dep_ctx = compile_ctx.dep_ctx, + emit = emit, + params = params, + default_roots = default_roots, + infallible_diagnostics = False, + incremental_enabled = incremental_enabled, + is_rustdoc_test = False, + profile_mode = None, + ).args + # Returns the full label and its hash. The full label is used for `-Cmetadata` # which provided the primary disambiguator for two otherwise identically named # crates. The hash is added to the filename to give them a lower likelihood of @@ -1553,7 +1515,14 @@ def crate_root(ctx: AnalysisContext, default_roots: list[str]) -> str: + "\nOr add 'crate_root = \"src/example.rs\"' to your attributes to disambiguate. candidates={}".format(candidates) ) -def _explain(crate_type: CrateType, link_strategy: LinkStrategy, emit: Emit, infallible_diagnostics: bool, profile_mode: ProfileMode | None) -> str: +def _explain( + crate_type: CrateType, + link_strategy: LinkStrategy, + emit: Emit, + infallible_diagnostics: bool, + profile_mode: ProfileMode | None, + advanced_unstable_linking: bool, +) -> str: base = None if emit == Emit("metadata-full"): link_strategy_suffix = { @@ -1566,6 +1535,17 @@ def _explain(crate_type: CrateType, link_strategy: LinkStrategy, emit: Emit, inf if emit == Emit("metadata-fast"): base = "diag" if infallible_diagnostics else "check" + if emit == Emit("rlib"): + if crate_type == CrateType("rlib"): + base = "rlib-no-meta" if advanced_unstable_linking else "rlib" + else: + base = "rlib-from-link" + base += { + LinkStrategy("static"): "", + LinkStrategy("static_pic"): " [pic]", + LinkStrategy("shared"): " [shared]", + }[link_strategy] + if emit == Emit("link"): link_strategy_suffix = { LinkStrategy("static"): "", @@ -1609,7 +1589,9 @@ def _explain(crate_type: CrateType, link_strategy: LinkStrategy, emit: Emit, inf return base EmitOperation = record( - output = field(Artifact), + # None exactly for bin-crate `Emit("rlib")`: the caller owns the linked + # artifact, and rustc produces only the extraction outputs. + output = field(Artifact | None), args = field(cmd_args), env = field(dict[str, str]), extra_out = field(Artifact | None), @@ -1626,7 +1608,6 @@ def _rustc_emit( incremental_enabled: bool, profile_mode: ProfileMode | None, predeclared_output: Artifact | None = None, - deferred_link: bool = False, ) -> EmitOperation: simple_crate = attr_simple_crate_for_filenames(ctx) crate_type = params.crate_type @@ -1650,10 +1631,15 @@ def _rustc_emit( else: extra_hash = "-" + _metadata(compile_ctx, ctx.label, False)[1] emit_args.add("-Cextra-filename={}".format(extra_hash)) - filename = subdir + "/" + output_filename(compile_ctx, simple_crate, emit, params, extra_hash) crate_name_and_extra_for_profile = simple_crate + extra_hash - emit_output = ctx.actions.declare_output(filename, has_content_based_path = emit_cbp) + if emit == Emit("rlib") and params.crate_type == CrateType("bin"): + # The caller links the extracted objects through cxx and owns the + # linked artifact; there is no rustc output to declare. + emit_output = None + else: + filename = subdir + "/" + output_filename(compile_ctx, simple_crate, emit, params, extra_hash) + emit_output = ctx.actions.declare_output(filename, has_content_based_path = emit_cbp) if emit == Emit("expand"): emit_args.add( @@ -1685,13 +1671,22 @@ def _rustc_emit( elif emit == Emit("llvm-ir-noopt"): effective_emit = "llvm-ir" emit_args.add("-Cno-prepopulate-passes") + elif emit == Emit("rlib"): + # rlibs are `--emit=link` products of the rlib crate type. For bin + # crates there is additionally no output to bind: the extraction + # wrapper set via `-Clinker=` captures the objects instead. + effective_emit = "link" else: effective_emit = emit.value - # When using deferred link, we still want to pass `--emit` to rustc to trigger - # the correct compilation behavior, but we do not want to pass emit_output here. - # Instead, we will bind the emit output to the actual deferred link action. - if deferred_link and effective_emit == "link": + # Pipelined builds still need metadata in `metadata-full`'s hollow rlib; + # native-linkable artifacts can omit it. + if compile_ctx.toolchain_info.advanced_unstable_linking and ( + (emit == Emit("rlib") and crate_type == CrateType("rlib")) or (emit == Emit("link") and crate_type == CrateType("dylib")) + ): + emit_args.add("-Zembed-metadata=no") + + if emit_output == None: emit_args.add(cmd_args("--emit=", effective_emit, delimiter = "")) else: emit_args.add(cmd_args("--emit=", effective_emit, "=", emit_output.as_output(), delimiter = "")) @@ -1737,6 +1732,49 @@ Invoke = record( identifier = field(str | None), ) +# Under `-Csplit-debuginfo=unpacked` (see `split_debuginfo_flags` in +# `_compute_common_args`), rustc writes each codegen unit's `.dwo` to +# `--out-dir` and also packs a copy of it into the rlib or staticlib, so that a +# downstream `-Csplit-debuginfo=packed` link could build a dwp from the archive +# alone. These rules never link that way: `dwp` reads the `--out-dir` files, +# which flow to it as external debug info, and linkers never pull archive +# members that define no symbols. A toolchain that sets `strip_dwo_from_rlibs` +# has `rustc_action.py` delete those members inside the compile action, so the +# archive is never cached or handed to a consumer with them. +def _strip_dwo_members_args(compile_ctx: CompileContext, emit: Emit, crate_type: CrateType, output: Artifact | None, has_split_debug: bool) -> cmd_args | None: + if not compile_ctx.toolchain_info.strip_dwo_from_rlibs or not has_split_debug: + return None + + # Only the `--emit=link` products of the archive crate types carry `.dwo` + # members. `Emit("rlib")` of an rlib crate is that product; the hollow + # rlibs of `metadata-full` are built without codegen and have none. + if crate_type not in [CrateType("rlib"), CrateType("staticlib")]: + return None + if emit not in [Emit("link"), Emit("rlib")] or output == None: + return None + + linker_info = compile_ctx.cxx_toolchain_info.linker_info + + # rustc emits DWARF objects only for targets whose debuginfo is plain DWARF + # (`Session::target_can_use_split_dwarf`): Apple (dSYM) and Windows (PDB) + # archives never carry `.dwo` members, so skip the archiver round-trip. + if linker_info.type in [LinkerType("darwin"), LinkerType("windows")]: + return None + + # `d` is the member-deletion verb of the GNU, llvm and BSD archivers; the + # others (`lib.exe`, amdclang) have no equivalent this tool speaks, so + # toolchains using them keep the members. + if linker_info.archiver_type not in ["gnu", "llvm", "bsd"]: + return None + + return cmd_args( + cmd_args(output.as_output(), format = "--strip-dwo-members={}"), + cmd_args(linker_info.archiver, format = "--archiver={}"), + # As in `archive_flags`: GNU-style archivers need `D` to leave + # timestamps out of the archive they write. + ["--archiver-deterministic"] if linker_info.type == LinkerType("gnu") else [], + ) + # Invoke rustc and capture outputs def _rustc_invoke( ctx: AnalysisContext, @@ -1751,8 +1789,8 @@ def _rustc_invoke( incremental_enabled: bool, crate_map: list[(CrateName, Label)], env: dict[str, str | ResolvedStringWithMacros | Artifact], - deferred_link: bool, profile_mode: ProfileMode | None, + strip_dwo_members: cmd_args | None, ) -> Invoke: toolchain_info = compile_ctx.toolchain_info @@ -1801,6 +1839,9 @@ def _rustc_invoke( for out in required_outputs: compile_cmd.add("--required-output", out.short_path, out.as_output()) + if strip_dwo_members != None: + compile_cmd.add(strip_dwo_members) + compile_cmd.add(rustc_cmd) compile_cmd = _long_command( @@ -1829,6 +1870,7 @@ def _rustc_invoke( emit = common_args.emit, infallible_diagnostics = infallible_diagnostics, profile_mode = profile_mode, + advanced_unstable_linking = toolchain_info.advanced_unstable_linking, ) if incremental_enabled: @@ -1838,9 +1880,18 @@ def _rustc_invoke( # None defers the choice to the `buck2.default_allow_cache_upload` config; an # explicit False overrides it. Actions without a preference pass None. - if allow_cache_upload or deferred_link: - # Opted in, or a deferred link. In the latter rustc compiles objects - # without linking, and we always cache those. + if incremental_enabled: + # Incremental compilation should not publish any action output to a shared cache: + # 1. the incremental compilation state is not useful for any other user. unfortunately, + # there is no mechanism in buck2 that allows for uploading part of an action output + # and not another. + # 2. even if there were, the rlib is not byte-for-byte reproducible + # under `-Cincremental`, even if the source binary is unchanged, because of + # https://github.com/rust-lang/rust/pull/139453. + # For additional context, please see this zulip conversation: + # https://rust-lang.zulipchat.com/#narrow/channel/131828-t-compiler/topic/Possible.20Reproducibility.20Bug.20in.20Soundness.20Fix.3F/with/616729102. + action_allow_cache_upload = False + elif allow_cache_upload: action_allow_cache_upload = True elif is_clippy: # Clippy never uploads. @@ -1852,8 +1903,7 @@ def _rustc_invoke( ctx.actions.run( compile_cmd, local_only = local_only, - # We only want to prefer_local here if rustc is performing the link - prefer_local = prefer_local and not deferred_link, + prefer_local = prefer_local, category = category, identifier = identifier, no_outputs_cleanup = incremental_enabled, @@ -1885,6 +1935,7 @@ def _long_command(ctx: AnalysisContext, exe: RunInfo, args: cmd_args, argfile_na _DOUBLE_ESCAPED_NEWLINE_RE = regex("\\\\n") _ESCAPED_NEWLINE_RE = regex("\\n") +_ESCAPED_CARRIAGE_RETURN_RE = regex("\\r") _DIRECTORY_ENV = [ "CARGO_MANIFEST_DIR", "OUT_DIR", @@ -1921,6 +1972,7 @@ def process_env( replace_regex = [ (_DOUBLE_ESCAPED_NEWLINE_RE, "\\\n"), (_ESCAPED_NEWLINE_RE, "\\n"), + (_ESCAPED_CARRIAGE_RETURN_RE, "\\r"), ], ) else: @@ -1973,28 +2025,13 @@ def process_env( return (plain_env, path_env) -def deferred_link_enabled(compile_ctx: CompileContext, params: BuildParams, emit: Emit) -> bool: - if not compile_ctx.toolchain_info.advanced_unstable_linking or emit != Emit("link"): - return False - - # The extraction wrapper does not handle windows-style paths and flags - linker_type = compile_ctx.cxx_toolchain_info.linker_info.type - if linker_type != LinkerType("gnu") and linker_type != LinkerType("darwin"): - return False - - if compile_ctx.exec_is_windows: - return False - - # TODO: support cdylib deferred link - return params.crate_type in [CrateType("dylib"), CrateType("bin")] - def _dist_thinlto_enabled(ctx: AnalysisContext, compile_ctx: CompileContext) -> bool: if not getattr(ctx.attrs, "enable_distributed_thinlto", False): return False linker_info = compile_ctx.cxx_toolchain_info.linker_info return linker_info.supports_distributed_thinlto and linker_info.lto_mode == LtoMode("thin") -def _inherited_link_args( +def get_inherited_link_args( ctx: AnalysisContext, compile_ctx: CompileContext, dep_link_style: LinkStrategy, @@ -2036,7 +2073,7 @@ def _inherited_link_args( return inherited_link_args def rust_link_shared(ctx: AnalysisContext, compile_ctx: CompileContext, dep_link_style: LinkStrategy, static_lib: LinkInfo) -> LinkedObject: - inherited_link_args = _inherited_link_args( + inherited_link_args = get_inherited_link_args( ctx, compile_ctx, dep_link_style, @@ -2044,7 +2081,12 @@ def rust_link_shared(ctx: AnalysisContext, compile_ctx: CompileContext, dep_link transformation_spec_context = None, ) link_args = [ - LinkArgs(flags = compile_ctx.linker_pre_args), + LinkArgs( + flags = cmd_args( + compile_ctx.toolchain_info.linker_flags, + ctx.attrs.linker_flags, + ) + ), # Need link whole because otherwise the linker doesn't include anything at all (normally # you'd be passing in `.o`s but that's not really an option) # @@ -2070,6 +2112,175 @@ def rust_link_shared(ctx: AnalysisContext, compile_ctx: CompileContext, dep_link links = link_args, link_execution_preference = LinkExecutionPreference("any"), category_suffix = "rust_shared", + # `rust_library` has no `allow_cache_upload` attribute + allow_cache_upload = None, ), name = compile_ctx.soname, ).linked_object + +def rust_link_binary( + ctx: AnalysisContext, + compile_ctx: CompileContext, + extraction: LinkExtraction, + dep_link_strategy: LinkStrategy, + reloc_model: RelocModel, + extra_link_args: list[typing.Any], + binary_link_args: list[typing.Any], + inherited_link_args: LinkArgs, + dwo_output_directory: Artifact | None, + output: Artifact, + output_has_content_based_path: bool, + identifier: str | None, + allow_cache_upload: bool, +) -> CxxLinkResult: + """Link an executable from the objects that a bin-crate `Emit("rlib")` + `rust_compile` extracted, plus the link args of the dependency graph.""" + linker_flags = ctx.attrs.linker_flags + if getattr(ctx.attrs, "_generated_build_info_enabled", False): + linker_flags = strip_build_info_linker_flags(linker_flags) + dist_thinlto = extraction.out_archive != None + + retained_flags = cmd_args(extraction.out_argsfile, format = "@{}") + if not dist_thinlto: + # Which objects rustc produced is only known once it has run, so the + # argsfile listing them is written by a dynamic action after the + # manifest is available. Resolving the manifest's names against the + # artifacts directory here — rather than having the extraction write + # paths into a file itself — keeps the paths buck-rendered, which is + # what makes them correct under content-based paths. + objects_argsfile = ctx.actions.declare_output(output.short_path + ".rust_objects.args", has_content_based_path = False) + + def write_objects_argsfile(ctx: AnalysisContext, artifacts, outputs): + names = artifacts[extraction.out_manifest].read_string().splitlines() + ctx.actions.write( + outputs[objects_argsfile], + cmd_args([extraction.out_artifacts_dir.project(name) for name in names]), + ) + + ctx.actions.dynamic_output( + dynamic = [extraction.out_manifest], + inputs = [], + outputs = [objects_argsfile.as_output()], + f = write_objects_argsfile, + ) + + rust_objects = LinkArgs( + flags = [ + retained_flags, + cmd_args(objects_argsfile, format = "@{}", hidden = extraction.out_artifacts_dir), + ] + ) + else: + rust_objects = LinkArgs( + infos = [ + LinkInfo( + name = "rust_objects", + # The opt actions re-run codegen and take their relocation + # model from their own command line, so they must match the + # relocation model rustc's own codegen used: PIC for every + # strategy but `static` (see `_get_reloc_model`). + dist_thin_lto_codegen_flags = (compile_ctx.toolchain_info.dist_thin_lto_codegen_flags if dep_link_strategy != LinkStrategy("static") else []), + pre_flags = [retained_flags], + linkables = [ + ArchiveLinkable( + archive = Archive(artifact = extraction.out_archive), + linker_type = compile_ctx.cxx_toolchain_info.linker_info.type, + # link_whole because everything rustc emits has to end up in the input. + link_whole = True, + ) + ], + ) + ] + ) + + split_debug_mode = compile_ctx.cxx_toolchain_info.split_debug_mode or SplitDebugMode("none") + if split_debug_mode != SplitDebugMode("none"): + inherited_debug_info = inherited_external_debug_info( + ctx = ctx, + dep_ctx = compile_ctx.dep_ctx, + dep_link_strategy = dep_link_strategy, + ) + else: + inherited_debug_info = ArtifactTSet() + external_debug_info = make_artifact_tset( + actions = ctx.actions, + label = ctx.label, + artifacts = [dwo_output_directory] if dwo_output_directory else [], + children = [inherited_debug_info], + ) + + links = [ + # The flags this link needs beyond what `cxx_link` applies on its own; + # `binary_linker_flags` mirrors `cxx_executable`'s use of them. + LinkArgs( + flags = cmd_args( + compile_ctx.cxx_toolchain_info.linker_info.binary_linker_flags, + compile_ctx.toolchain_info.linker_flags, + linker_flags, + ) + ), + LinkArgs(flags = extra_link_args), + ] + + if compile_ctx.cxx_toolchain_info.linker_info.type == LinkerType("gnu"): + flags = [ + # FIXME(JakobDegen): This is here becuase rustc used to pass it, but it's not something + # we should be passing blindly on the user's behalf. Unfortunately some builds break if + # you take it out. + "-Wl,--as-needed", + # lld deduplicates `.debug_str` only at `-O1`, and rustc always + # passes this for links it drives, overriding toolchains that link + # at `-O0` for speed (fbcode's opt-lg). Without the deduplication, + # very large binaries overflow the 32-bit `.debug_str` offsets in + # `.debug_names`. + "-Wl,-O1", + # The objects were codegened at the relocation model chosen in + # `_get_reloc_model`, which the toolchain's link flags cannot know + # about. Pin the PIE-ness of the executable to match, as rustc does + # for links it drives. + "-no-pie" if reloc_model == RelocModel("static") else "-pie", + ] + + # On ELF, rustc-instrumented objects carry no reference to the + # profile runtime's registration object (clang-instrumented ones + # do), so nothing would pull it out of the `profiler_builtins` + # archive and the binary would silently write no profile. rustc + # compensates with this flag on links it drives; do the same, + # gated exactly like the `-Cinstrument-coverage` the compile gets. + if getattr(ctx.attrs, "coverage", False) and compile_ctx.toolchain_info.rustc_coverage_flags: + flags.append("-Wl,-u,__llvm_profile_runtime") + + links.append(LinkArgs(flags = flags)) + + links += [ + rust_objects, + inherited_link_args, + LinkArgs(infos = [LinkInfo(external_debug_info = external_debug_info)]), + ] + + if link_cxx_binary_locally(ctx, compile_ctx.cxx_toolchain_info): + link_execution_preference = LinkExecutionPreference("local") + else: + link_execution_preference = LinkExecutionPreference("any") + + return cxx_link_into( + ctx = ctx, + output = output, + result_type = CxxLinkResultType("executable"), + output_has_content_based_path = output_has_content_based_path, + opts = link_options( + binary_links = [LinkArgs(flags = binary_link_args)] if binary_link_args else [], + links = links, + link_execution_preference = link_execution_preference, + # The link is local only so the fbcode linker wrapper can stamp + # build info from the repo (D115067153); dwp needs no repo and is + # memory-hungry enough that pinning it to the link's host thrashes + # small workers. Let it schedule anywhere, matching the + # rustc-linked path, which runs dwp with no preference. + dwp_execution_preference = LinkExecutionPreference("any"), + category_suffix = "rust_binary", + identifier = identifier, + enable_distributed_thinlto = dist_thinlto, + allow_cache_upload = allow_cache_upload, + ), + ) diff --git a/prelude/rust/build_params.bzl b/prelude/rust/build_params.bzl index 52760f8..4a0b43c 100644 --- a/prelude/rust/build_params.bzl +++ b/prelude/rust/build_params.bzl @@ -56,7 +56,15 @@ Emit = enum( "llvm-ir", "llvm-ir-noopt", "obj", + # A real link, performed by rustc: non-AUL binaries and dylibs/cdylibs, + # proc macros (which rustc itself loads), and staticlib bundling (not a + # true link, but like one in that it pulls together all dependencies). "link", + # The crate's static-library-shaped artifact. For `rlib` crates this is + # the rlib itself. For `bin` crates (AUL only) rustc compiles exactly as + # for `link`, but the extraction wrapper captures the objects it would + # have handed the linker. + "rlib", "dep-info", "mir", "expand", # pseudo emit alias for -Zunpretty=expanded @@ -94,6 +102,9 @@ def dep_metadata_of_emit(emit: Emit) -> MetadataKind: Emit("llvm-ir-noopt"): MetadataKind("full"), Emit("obj"): MetadataKind("full"), Emit("link"): MetadataKind("link"), + # rlibs are consumed as archives by cxx-driven links, never loaded by + # a downstream rustc link, so their deps never need real object code. + Emit("rlib"): MetadataKind("full"), Emit("mir"): MetadataKind("full"), Emit("metadata-fast"): MetadataKind("fast"), Emit("clippy"): MetadataKind("fast"), diff --git a/prelude/rust/buildscript/buildscript_platform.bzl b/prelude/rust/buildscript/buildscript_platform.bzl index 7848021..847ff0e 100644 --- a/prelude/rust/buildscript/buildscript_platform.bzl +++ b/prelude/rust/buildscript/buildscript_platform.bzl @@ -22,6 +22,7 @@ transition_alias = rule( impl = lambda ctx: ctx.attrs.actual.providers, attrs = { "actual": attrs.dep(), + "labels": attrs.list(attrs.string(), default = []), }, supports_incoming_transition = True, ) diff --git a/prelude/rust/cargo_package.bzl b/prelude/rust/cargo_package.bzl index 5294b06..d8e192a 100644 --- a/prelude/rust/cargo_package.bzl +++ b/prelude/rust/cargo_package.bzl @@ -32,7 +32,7 @@ def set_reindeer_platforms(platforms) -> None: native.write_package_value( "rust.reindeer_platforms", - _convert_select_to_dict(platforms), + platforms, overwrite = True, ) native.write_package_value( @@ -44,7 +44,7 @@ def set_reindeer_platforms(platforms) -> None: def get_reindeer_platforms(): platforms = native.read_package_value("rust.reindeer_platforms") if platforms != None: - return _convert_dict_to_select(platforms) + return platforms return DEFAULT_REINDEER_PLATFORMS def get_reindeer_platform_names() -> set[str]: @@ -91,61 +91,6 @@ def _reindeer_platform_names(platform_select) -> set[str]: _DEFAULT_REINDEER_PLATFORM_NAMES = _reindeer_platform_names(DEFAULT_REINDEER_PLATFORMS) -# Disect the `repr` representation of a `select`, which looks like this: -# select({"DEFAULT": None, "config//os:linux": "linux-arm64", ...}) -# -# [WORKAROUND] This will be unnecessary once `write_package_value` allows -# selects, which seems to have consensus and just needs to be implemented. -def _convert_select_to_dict(select_value): - string = repr(select_value) - result = None - key = None - stack = [] - - for _ in string.elems(): - # Parse a value (non-key) - if string.startswith('"'): - value, string = string[1:].split('"', 1) - elif string.startswith("None"): - value, string = None, string.removeprefix("None") - elif string.startswith("select({"): - value, string = {}, string.removeprefix("select({") - else: - fail() - - # Insert the parsed value into the surrounding collection - if key == None: - result = value - else: - stack[-1][key] = value - - # Parse a key - if value == {}: - stack.append(value) - if string.startswith('"'): - key, string = string.removeprefix('"').split('": ', 1) - continue # Back to parsing a value - - # Pop the stack while there are selects ending - for _ in range(len(stack)): - if string.startswith("})"): - string = string.removeprefix("})") - stack.pop() - else: - break - - # Parse separator and next key - if string == "": - return result - elif string.startswith(', "'): - key, string = string.removeprefix(', "').split('": ', 1) - else: - fail() - fail() - -def _convert_dict_to_select(value): - return value if not is_dict(value) else select({k: _convert_dict_to_select(v) for k, v in value.items()}) - def apply_platform_attrs(platform_attrs, universal_attrs, platform_select = None): if platform_select == None: platform_select = get_reindeer_platforms() diff --git a/prelude/rust/context.bzl b/prelude/rust/context.bzl index 2c8e7d1..770368a 100644 --- a/prelude/rust/context.bzl +++ b/prelude/rust/context.bzl @@ -17,7 +17,7 @@ load("@prelude//linking:link_info.bzl", "LinkStrategy") load("@prelude//os_lookup:defs.bzl", "Os", "OsLookup") load("@prelude//rust/tools:attrs.bzl", "RustInternalToolsInfo") load("@prelude//utils:cmd_script.bzl", "cmd_script") -load(":build_params.bzl", "BuildParams", "CrateType", "Emit", "ProfileMode") +load(":build_params.bzl", "BuildParams", "CrateType", "Emit", "MetadataKind", "ProfileMode") load( ":crate_name.bzl", "CrateName", # @unused Used as a type @@ -47,6 +47,11 @@ CommonArgsInfo = record( crate_map = field(list[(CrateName, Label)]), ) +DependencyArgsInfo = record( + args = field(cmd_args), + crate_map = field(list[(CrateName, Label)]), +) + # Compile info which is reusable between multiple compilation command performed # by the same rule. CompileContext = record( @@ -54,6 +59,8 @@ CompileContext = record( clippy_wrapper = field(cmd_args), # Memoized common args for reuse. common_args = field(dict[(CrateType, Emit, LinkStrategy, bool, bool, bool, ProfileMode), CommonArgsInfo]), + # Memoized dependency args for reuse across emits with the same inputs. + dependency_args = field(dict[(CrateType, LinkStrategy, MetadataKind, bool), DependencyArgsInfo]), cxx_toolchain_info = field(CxxToolchainInfo), dep_ctx = field(DepCollectionContext), exec_is_windows = field(bool), @@ -65,8 +72,6 @@ CompileContext = record( # unstable `-Zpre-link-arg` for that, but until that stabilizes, we pass them from within # this script. linker_with_pre_args = field(cmd_args), - # The same pre-args, not wrapped in a script - linker_pre_args = field(cmd_args), path_sep = field(str), # Dylib name override, if any was provided by the target's `soname` attribute. soname = field(str | None), @@ -88,7 +93,7 @@ def compile_context(ctx: AnalysisContext, binary: bool = False) -> CompileContex srcs = symlinked_srcs(ctx) - linker_with_pre_args, linker_pre_args = _linker(ctx, cxx_toolchain_info.linker_info, binary = binary) + linker_with_pre_args = _linker(ctx, cxx_toolchain_info.linker_info, binary = binary) clippy_wrapper = _clippy_wrapper(ctx, toolchain_info) dep_ctx = DepCollectionContext( @@ -131,12 +136,12 @@ def compile_context(ctx: AnalysisContext, binary: bool = False) -> CompileContex return CompileContext( clippy_wrapper = clippy_wrapper, common_args = {}, + dependency_args = {}, cxx_toolchain_info = cxx_toolchain_info, dep_ctx = dep_ctx, exec_is_windows = exec_is_windows, internal_tools_info = internal_tools_info, linker_with_pre_args = linker_with_pre_args, - linker_pre_args = linker_pre_args, path_sep = path_sep, soname = _attr_soname(ctx), symlinked_srcs = srcs, @@ -171,28 +176,30 @@ def _validate_nightly_features(toolchain_info: RustToolchainInfo): ) ) -def _linker(ctx: AnalysisContext, linker_info: LinkerInfo, binary: bool = False) -> (cmd_args, cmd_args): - pre_args = cmd_args( - linker_info.linker_flags or [], - # For "binary" rules, add C++ toolchain binary-specific linker flags. - # TODO(agallagher): This feels a bit wrong -- it might be better to have - # the Rust toolchain have it's own `binary_linker_flags` instead of - # implicitly using the one from the C++ toolchain. - linker_info.binary_linker_flags if binary else [], - ctx.attrs._rust_toolchain[RustToolchainInfo].linker_flags, - ctx.attrs.linker_flags, - ) - +def _linker(ctx: AnalysisContext, linker_info: LinkerInfo, binary: bool = False) -> cmd_args: + linker_flags = ctx.attrs.linker_flags + if getattr(ctx.attrs, "_generated_build_info_enabled", False): + linker_flags = strip_build_info_linker_flags(linker_flags) return cmd_script( actions = ctx.actions, name = "linker_wrapper", cmd = cmd_args( linker_info.linker, - pre_args, + linker_info.linker_flags or [], + # For "binary" rules, add C++ toolchain binary-specific linker flags. + # TODO(agallagher): This feels a bit wrong -- it might be better to have + # the Rust toolchain have it's own `binary_linker_flags` instead of + # implicitly using the one from the C++ toolchain. + linker_info.binary_linker_flags if binary else [], + ctx.attrs._rust_toolchain[RustToolchainInfo].linker_flags, + linker_flags, ), language = ctx.attrs._exec_os_type[OsLookup].script, has_content_based_path = True, - ), pre_args + ) + +def strip_build_info_linker_flags(linker_flags): + return [flag for flag in linker_flags if not flag.startswith("--build-info")] # Return wrapper script for clippy-driver to make sure sysroot is set right # We need to make sure clippy is using the same sysroot - compiler, std libraries - @@ -286,6 +293,7 @@ _EMIT_PREFIX_SUFFIX = { Emit("metadata-fast"): ("lib", ".rmeta"), # even binaries get called 'libfoo.rmeta' Emit("metadata-full"): (None, None), # Hollow rlibs, so they get the same name Emit("link"): (None, None), # crate type and reloc model dependent + Emit("rlib"): (None, None), # crate type dependent, like `link` Emit("dep-info"): ("", ".d"), Emit("mir"): (None, ".mir"), Emit("expand"): (None, ".rs"), diff --git a/prelude/rust/extern.bzl b/prelude/rust/extern.bzl index 6f6f6ca..6b70f28 100644 --- a/prelude/rust/extern.bzl +++ b/prelude/rust/extern.bzl @@ -26,6 +26,12 @@ def extern_arg(flags: list[str], crate: CrateName, lib: Artifact) -> cmd_args: else: flags = ",".join(flags) + ":" + if type(crate.simple) == "string" and crate.dynamic == None: + return cmd_args( + lib, + format = "--extern={}{}={{}}".format(flags, crate.simple), + ) + return cmd_args( "--extern=", flags, @@ -43,7 +49,10 @@ def extern_arg(flags: list[str], crate: CrateName, lib: Artifact) -> cmd_args: # # --crate-map=$(cat path/to/REALNAME)=//path/to:target # -def crate_map_arg(crate: CrateName, label: Label) -> cmd_args: +def crate_map_arg(crate: CrateName, label: Label) -> cmd_args | str: + if type(crate.simple) == "string" and crate.dynamic == None: + return "--crate-map={}={}".format(crate.simple, str(label.raw_target())) + return cmd_args( "--crate-map=", crate_name_as_cmd_arg(crate), diff --git a/prelude/rust/outputs.bzl b/prelude/rust/outputs.bzl index 05cada4..66ff661 100644 --- a/prelude/rust/outputs.bzl +++ b/prelude/rust/outputs.bzl @@ -35,12 +35,19 @@ RustcLinkOutput = record( ) RustcOutput = record( + # For bin-crate `Emit("rlib")` compiles this is the `out_manifest` of + # `link_extraction` — the list of extracted objects — since rustc produces + # no linked artifact; the executable is produced by `rust_link_binary`. output = Artifact, singleton_tset = TransitiveDeps, compile_output = RustcCompileOutput, - # As expected, only available when the combination of params actually - # requires linking. + # Only available when the combination of params requires linking and rustc + # itself performs the link (i.e. not for bin-crate `Emit("rlib")`). link_output = RustcLinkOutput | None, + # A `LinkExtraction`, set exactly when this was a bin-crate `Emit("rlib")` + # compile: rustc compiled and its synthesized link inputs were extracted + # for the caller to link via `rust_link_binary`. + link_extraction = typing.Any, ) def output_as_diag_subtargets(o: RustcOutput, clippy: RustcOutput) -> dict[str, Artifact]: diff --git a/prelude/rust/rust_binary.bzl b/prelude/rust/rust_binary.bzl index a73d8d4..937efe7 100644 --- a/prelude/rust/rust_binary.bzl +++ b/prelude/rust/rust_binary.bzl @@ -17,6 +17,8 @@ load( "create_resource_db", "gather_resources", ) +load("@prelude//cxx:cxx_bolt.bzl", "cxx_use_bolt") +load("@prelude//cxx:cxx_executable.bzl", "get_cxx_post_link_suffix") load( "@prelude//cxx:cxx_library_utility.bzl", "cxx_attr_deps", @@ -43,12 +45,15 @@ load( "@prelude//dist:dist_info.bzl", "DistInfo", ) +load("@prelude//linking:add_elf_sections.bzl", "get_elf_sections") +load("@prelude//linking:generated_build_info.bzl", "compile_generated_build_info", "generate_build_info") load( "@prelude//linking:link_info.bzl", "LibOutputStyle", "LinkStrategy", "get_lib_output_style", "process_link_strategy_for_pic_behavior", + "unpack_link_args", ) load( "@prelude//linking:linkable_graph.bzl", @@ -59,7 +64,7 @@ load( "merge_shared_libraries", "traverse_shared_library_info", ) -load("@prelude//linking:stamp_build_info.bzl", "PRE_STAMPED_SUFFIX", "cxx_stamp_build_info", "stamp_build_info") +load("@prelude//linking:stamp_build_info.bzl", "cxx_stamp_build_info", "stamp_build_info") load("@prelude//os_lookup:defs.bzl", "OsLookup") load("@prelude//rust/rust-analyzer:provider.bzl", "rust_analyzer_provider") load("@prelude//test:inject_test_run_info.bzl", "inject_test_run_info") @@ -75,8 +80,10 @@ load("@prelude//utils:utils.bzl", "flatten_dict") load( ":build.bzl", "generate_rustdoc", + "get_inherited_link_args", "rust_compile", - _deferred_link_enabled = "deferred_link_enabled", + "rust_compile_invalidation_inputs", + "rust_link_binary", ) load( ":build_params.bzl", @@ -186,7 +193,11 @@ def _create_content_based_dist( return make_content_based_dist(ctx, name, exe, copies = copies, symlinks = symlinks) def _rust_binary_common( - ctx: AnalysisContext, compile_ctx: CompileContext, default_roots: list[str], extra_flags: list[str], allow_cache_upload: bool + ctx: AnalysisContext, + compile_ctx: CompileContext, + default_roots: list[str], + extra_flags: list[str], + allow_cache_upload: bool, ) -> (list[Provider], cmd_args): toolchain_info = compile_ctx.toolchain_info @@ -196,6 +207,12 @@ def _rust_binary_common( link_strategy = process_link_strategy_for_pic_behavior(link_strategy, compile_ctx.cxx_toolchain_info.pic_behavior) cxx_deps = cxx_attr_deps(ctx) + generated_build_info_enabled = getattr(ctx.attrs, "_generated_build_info_enabled", False) + generated_build_info_args = ctx.attrs._generated_build_info_args if generated_build_info_enabled else [] + + generated_build_info_link_args = [] + generated_build_info_link_objects = [] + resources = flatten_dict( gather_resources( label = ctx.label, @@ -211,7 +228,24 @@ def _rust_binary_common( params = strategy_param[link_strategy] name = output_filename(compile_ctx, simple_crate, Emit("link"), params) - deferred_link_enabled = _deferred_link_enabled(compile_ctx, params, Emit("link")) + # Under advanced_unstable_linking, the binary is compiled via + # `Emit("rlib")` and linked through cxx below (`rust_link_binary`), + # mirroring how rust_library produces dylibs via `rust_link_shared`. + links_via_cxx = toolchain_info.advanced_unstable_linking + bin_emit = Emit("rlib") if links_via_cxx else Emit("link") + + use_bolt = cxx_use_bolt(ctx) + if use_bolt and not links_via_cxx: + fail( + "`bolt_profile` is set, but BOLT runs as part of the cxx link. " + + "This target links via rustc; BOLT requires `advanced_unstable_linking` on the Rust toolchain." + ) + elf_sections = get_elf_sections(ctx) + if elf_sections and not links_via_cxx: + fail( + "`elf_sections` is set, but the sections are added as part of the cxx link. " + + "This target links via rustc; `elf_sections` requires `advanced_unstable_linking` on the Rust toolchain." + ) enable_late_build_info_stamping = cxx_stamp_build_info(ctx) content_based_output = getattr(ctx.attrs, "has_content_based_path", False) unstamped_name = None @@ -232,6 +266,14 @@ def _rust_binary_common( link_strategy = link_strategy, transformation_spec_context = transformation_spec_context, ) + + # Link groups repartition the dependency graph's code — including the + # Rust code — into group libraries, filtering the main link accordingly. + # Without advanced_unstable_linking, rustc links the full dep rlib set no + # matter what, so the grouped code would be linked twice; this cannot be + # made correct. + if rust_cxx_link_group_info != None and not toolchain_info.advanced_unstable_linking: + fail("link groups on Rust binaries require `advanced_unstable_linking`") if rust_cxx_link_group_info != None: link_group_mappings = rust_cxx_link_group_info.link_group_info.mappings link_group_libs = rust_cxx_link_group_info.link_group_libs @@ -247,6 +289,14 @@ def _rust_binary_common( targets_consumed_by_link_groups = {} filtered_targets = [] + native_link_args = get_inherited_link_args( + ctx, + compile_ctx, + params.dep_link_strategy, + rust_cxx_link_group_info, + transformation_spec_context, + ) + runtime_dep_handling = _get_runtime_dependency_handling(ctx, compile_ctx) shlib_deps = [] @@ -321,20 +371,17 @@ def _rust_binary_common( # RPATH that survives co-location) and get content-addressing from the # bundle dir instead. Static binaries reuse their content-based exe in the # bundle directly (no RPATH to break). - exe_content_based = content_based_output and not needs_shlib_tree - if deferred_link_enabled: - # cxx performs the terminal link and its own late build-info stamping, so - # rust_compile returns the final (stamped) binary; we must not stamp again. - if enable_late_build_info_stamping: - # Use the pre-stamp suffix so cxx's stamp strips it back to `name`. - predeclared_output = ctx.actions.declare_output( - output_filename(compile_ctx, simple_crate, Emit("link"), params, PRE_STAMPED_SUFFIX), - has_content_based_path = exe_content_based, - ) - else: - predeclared_output = ctx.actions.declare_output(name, has_content_based_path = exe_content_based) + exe_content_based = content_based_output and not needs_shlib_tree and not use_bolt + if links_via_cxx: + # cxx performs the terminal link (see `rust_link_binary` below) and its + # own post-link processing, so the output has to be named the way those + # stages expect; we must not stamp again here. + predeclared_output = ctx.actions.declare_output( + output_filename(compile_ctx, simple_crate, Emit("link"), params, get_cxx_post_link_suffix(ctx)), + has_content_based_path = exe_content_based, + ) - # final_output is whatever cxx returns, set after rust_compile below. + # final_output is whatever cxx returns, set after the link below. final_output = None # rpath and symlink-tree are computed against the pre-stamped output shlib_args_output = predeclared_output @@ -356,26 +403,89 @@ def _rust_binary_common( shlib_args_output, shared_libs, ) + if generated_build_info_enabled and not links_via_cxx: + generated_build_info = generate_build_info( + ctx, + generator_args = generated_build_info_args, + invalidation_inputs = [ + rust_compile_invalidation_inputs( + ctx = ctx, + compile_ctx = compile_ctx, + emit = bin_emit, + params = params, + default_roots = default_roots, + incremental_enabled = ctx.attrs.incremental_enabled, + ), + unpack_link_args(native_link_args), + executable_shlib_args.extra_link_args, + ], + ) + if generated_build_info: + generated_build_info_link_args.extend(generated_build_info.linker_flags) + generated_build_info_link_args.extend(compile_generated_build_info(ctx, generated_build_info).objects) + extra_link_args = executable_shlib_args.extra_link_args + generated_build_info_link_args - # Compile rust binary. + # Compile rust binary. Under `Emit("rlib")`, this only compiles: rustc's + # synthesized objects are extracted and linked below. link = rust_compile( ctx = ctx, compile_ctx = compile_ctx, - emit = Emit("link"), + emit = bin_emit, params = params, default_roots = default_roots, - extra_link_args = executable_shlib_args.extra_link_args, - predeclared_output = predeclared_output, + extra_link_args = extra_link_args, + predeclared_output = None if links_via_cxx else predeclared_output, + predeclared_output_has_content_based_path = exe_content_based, extra_flags = extra_flags, allow_cache_upload = allow_cache_upload, - rust_cxx_link_group_info = rust_cxx_link_group_info, transformation_spec_context = transformation_spec_context, incremental_enabled = ctx.attrs.incremental_enabled, + precomputed_inherited_link_args = native_link_args, ) - if deferred_link_enabled: - # Use the cxx stamped output if available - final_output = link.output + dwp_output = link.link_output.dwp_output if link.link_output else None + pdb_output = link.link_output.pdb if link.link_output else None + prebolt_output = None + if links_via_cxx: + rust_link_inputs = [ + link.link_extraction.out_argsfile, + link.link_extraction.out_artifacts_dir, + link.link_extraction.out_manifest, + ] + if link.link_extraction.out_archive != None: + rust_link_inputs.append(link.link_extraction.out_archive) + if generated_build_info_enabled: + generated_build_info = generate_build_info( + ctx, + generator_args = generated_build_info_args, + invalidation_inputs = rust_link_inputs + + [ + unpack_link_args(native_link_args), + executable_shlib_args.extra_link_args, + ], + ) + if generated_build_info: + generated_build_info_link_args.extend(generated_build_info.linker_flags) + generated_build_info_link_objects.extend(compile_generated_build_info(ctx, generated_build_info).objects) + link_result = rust_link_binary( + ctx = ctx, + compile_ctx = compile_ctx, + extraction = link.link_extraction, + dep_link_strategy = params.dep_link_strategy, + reloc_model = params.reloc_model, + extra_link_args = executable_shlib_args.extra_link_args, + binary_link_args = generated_build_info_link_objects + generated_build_info_link_args, + inherited_link_args = native_link_args, + dwo_output_directory = link.compile_output.dwo_output_directory, + output = predeclared_output, + output_has_content_based_path = exe_content_based, + identifier = name, + allow_cache_upload = allow_cache_upload, + ) + final_output = link_result.linked_object.output + dwp_output = link_result.linked_object.dwp + pdb_output = link_result.linked_object.pdb + prebolt_output = link_result.linked_object.prebolt_output elif enable_late_build_info_stamping: stamp_build_info(ctx, link.output, final_output) @@ -550,9 +660,10 @@ def _rust_binary_common( remarks = rust_compile( ctx = ctx, compile_ctx = compile_ctx, - emit = Emit("link"), + emit = bin_emit, params = params, default_roots = default_roots, + extra_link_args = extra_link_args, extra_flags = extra_flags, incremental_enabled = False, profile_mode = ProfileMode("remarks"), @@ -588,24 +699,22 @@ def _rust_binary_common( llvm_time_trace = rust_compile( ctx = ctx, compile_ctx = compile_ctx, - emit = Emit("link"), + emit = bin_emit, params = params, default_roots = default_roots, - extra_link_args = executable_shlib_args.extra_link_args, + extra_link_args = extra_link_args, extra_flags = extra_flags, - rust_cxx_link_group_info = rust_cxx_link_group_info, incremental_enabled = ctx.attrs.incremental_enabled, profile_mode = ProfileMode("llvm-time-trace"), ) self_profile = rust_compile( ctx = ctx, compile_ctx = compile_ctx, - emit = Emit("link"), + emit = bin_emit, params = params, default_roots = default_roots, - extra_link_args = executable_shlib_args.extra_link_args, + extra_link_args = extra_link_args, extra_flags = extra_flags, - rust_cxx_link_group_info = rust_cxx_link_group_info, incremental_enabled = ctx.attrs.incremental_enabled, profile_mode = ProfileMode("self-profile"), ) @@ -646,17 +755,20 @@ def _rust_binary_common( if named_deps_names: extra_compiled_targets["named_deps"] = named_deps_names - if link.link_output.dwp_output: + if dwp_output: sub_targets["dwp"] = [ DefaultInfo( - default_output = link.link_output.dwp_output, + default_output = dwp_output, other_outputs = [shlib.lib.dwp for shlib in shared_libs if shlib.lib.dwp] + ([executable_shlib_args.dwp_symlink_tree] if executable_shlib_args.dwp_symlink_tree else []), ), ] - if link.link_output.pdb: - sub_targets[PDB_SUB_TARGET] = get_pdb_providers(pdb = link.link_output.pdb, binary = final_output) + if pdb_output: + sub_targets[PDB_SUB_TARGET] = get_pdb_providers(pdb = pdb_output, binary = final_output) + + if use_bolt and prebolt_output: + sub_targets["prebolt"] = [DefaultInfo(default_output = prebolt_output)] dupmbin_toolchain = compile_ctx.cxx_toolchain_info.dumpbin_toolchain_path if dupmbin_toolchain: diff --git a/prelude/rust/rust_library.bzl b/prelude/rust/rust_library.bzl index 914199a..ebff4a1 100644 --- a/prelude/rust/rust_library.bzl +++ b/prelude/rust/rust_library.bzl @@ -11,7 +11,7 @@ load( "ArtifactTSet", "make_artifact_tset", ) -load("@prelude//:resources.bzl", "ResourceInfo", "gather_resources") +load("@prelude//:resources.bzl", "gather_resources", "make_resource_info") load( "@prelude//android:android_providers.bzl", "merge_android_packageable_info", @@ -39,6 +39,7 @@ load( "LinkInfos", "LinkStrategy", "LinkedObject", + "LinkerFlags", "MergedLinkInfo", # @unused Used as a type "SharedLibLinkable", "create_merged_link_info", @@ -48,7 +49,7 @@ load( ) load( "@prelude//linking:linkable_graph.bzl", - "DlopenableLibraryInfo", + "DLOPENABLE_LIBRARY_INFO_MARKER", "create_linkable_graph", "create_linkable_graph_node", "create_linkable_node", @@ -88,6 +89,7 @@ load( load( ":build_params.bzl", "BuildParams", # @unused Used as a type + "CrateType", "Emit", "LinkageLang", "MetadataKind", @@ -191,7 +193,7 @@ def rust_library_impl(ctx: AnalysisContext) -> list[Provider]: link = rust_compile( ctx = ctx, compile_ctx = compile_ctx, - emit = Emit("link"), + emit = Emit("rlib") if params.crate_type == CrateType("rlib") else Emit("link"), params = params, default_roots = _DEFAULT_ROOTS, incremental_enabled = ctx.attrs.incremental_enabled, @@ -200,6 +202,19 @@ def rust_library_impl(ctx: AnalysisContext) -> list[Provider]: param_subtargets.setdefault(params, {}) if LinkageLang("rust") in langs: + metadata_link = link + if toolchain_info.advanced_unstable_linking and params.crate_type == CrateType("rlib"): + # Rustc-produced staticlibs need code and metadata in one rlib. + rlib_for_staticlib = rust_compile( + ctx = ctx, + compile_ctx = compile_ctx, + emit = Emit("link"), + params = params, + default_roots = _DEFAULT_ROOTS, + incremental_enabled = ctx.attrs.incremental_enabled, + ) + metadata_link = rlib_for_staticlib + if toolchain_info.nightly_features: # Pipelined build: dependents that need full metadata compile # against the `-Zno-codegen` "hollow rlib" instead of waiting @@ -217,7 +232,7 @@ def rust_library_impl(ctx: AnalysisContext) -> list[Provider]: # wait for the real rlib instead. metadata_full = link param_metadata_outputs[params] = { - MetadataKind("link"): link, + MetadataKind("link"): metadata_link, MetadataKind("full"): metadata_full, MetadataKind("fast"): meta_fast, } @@ -258,6 +273,8 @@ def rust_library_impl(ctx: AnalysisContext) -> list[Provider]: linked_object = rust_link_shared( ctx, compile_ctx, + # Unlike `cxx_library`, `link_style` is not consulted here: the deps of a Rust DSO + # always use the shared link strategy. dep_link_style = LinkStrategy("shared"), static_lib = link_infos[LibOutputStyle("pic_archive")].default, ) @@ -352,7 +369,7 @@ def rust_library_impl(ctx: AnalysisContext) -> list[Provider]: llvm_time_trace = rust_compile( ctx = ctx, compile_ctx = compile_ctx, - emit = Emit("link"), + emit = Emit("rlib"), params = static_library_params, default_roots = _DEFAULT_ROOTS, incremental_enabled = ctx.attrs.incremental_enabled, @@ -361,7 +378,7 @@ def rust_library_impl(ctx: AnalysisContext) -> list[Provider]: self_profile = rust_compile( ctx = ctx, compile_ctx = compile_ctx, - emit = Emit("link"), + emit = Emit("rlib"), params = static_library_params, default_roots = _DEFAULT_ROOTS, incremental_enabled = ctx.attrs.incremental_enabled, @@ -396,6 +413,7 @@ def rust_library_impl(ctx: AnalysisContext) -> list[Provider]: doctests_enabled = False if toolchain_info.nightly_features: + rustdoc_test_metadata_kind = MetadataKind("full") if toolchain_info.advanced_unstable_linking and not ctx.attrs.proc_macro else MetadataKind("link") rustdoc_test_params = build_params( rule = RuleType("binary"), proc_macro = ctx.attrs.proc_macro, @@ -408,7 +426,7 @@ def rust_library_impl(ctx: AnalysisContext) -> list[Provider]: rustdoc_test = generate_rustdoc_test( ctx = ctx, compile_ctx = compile_ctx, - rlib = param_output[static_library_params].output, + rlib = param_metadata_outputs[static_library_params][rustdoc_test_metadata_kind].output, link_infos = link_infos, params = rustdoc_test_params, default_roots = _DEFAULT_ROOTS, @@ -449,7 +467,7 @@ def rust_library_impl(ctx: AnalysisContext) -> list[Provider]: remarks_artifact = rust_compile( ctx = ctx, compile_ctx = compile_ctx, - emit = Emit("link"), + emit = Emit("rlib"), params = meta_params, default_roots = _DEFAULT_ROOTS, incremental_enabled = False, @@ -509,13 +527,13 @@ def rust_library_impl(ctx: AnalysisContext) -> list[Provider]: deps = [dep.dep for dep in resolve_deps(ctx, compile_ctx.dep_ctx)] providers.append( - ResourceInfo( - resources = gather_resources( + make_resource_info( + gather_resources( label = ctx.label, resources = rust_attr_resources(ctx), deps = deps, - ) - ) + ), + ), ) providers.append(merge_android_packageable_info(ctx.label, ctx.actions, deps)) @@ -872,6 +890,13 @@ def _proc_macro_link_providers(ctx: AnalysisContext, rust_artifacts: dict[LinkSt ) ] +def _linker_flags(ctx: AnalysisContext) -> LinkerFlags: + return LinkerFlags( + flags = ctx.attrs.linker_flags, + exported_flags = ctx.attrs.exported_linker_flags, + exported_post_flags = ctx.attrs.exported_post_linker_flags, + ) + def _advanced_unstable_link_providers( ctx: AnalysisContext, compile_ctx: CompileContext, @@ -943,6 +968,7 @@ def _advanced_unstable_link_providers( exported_deps = inherited_exported_deps, link_infos = link_infos, shared_libs = shared_libs, + linker_flags = _linker_flags(ctx), default_soname = shlib_name, # Link groups have a heuristic in which they assume that a # preferred_linkage = "static" library needs to be linked @@ -972,7 +998,7 @@ def _advanced_unstable_link_providers( # Mark libraries that support `dlopen`. if getattr(ctx.attrs, "supports_python_dlopen", False): - providers.append(DlopenableLibraryInfo()) + providers.append(DLOPENABLE_LIBRARY_INFO_MARKER) # We never need to add anything to this provider because Rust libraries # cannot act as link group libs, especially given that they only support @@ -1149,7 +1175,7 @@ def _native_link_providers( # Mark libraries that support `dlopen`. if getattr(ctx.attrs, "supports_python_dlopen", False): - providers.append(DlopenableLibraryInfo()) + providers.append(DLOPENABLE_LIBRARY_INFO_MARKER) linkable_graph = create_linkable_graph( ctx, @@ -1162,6 +1188,7 @@ def _native_link_providers( exported_deps = inherited_exported_deps, link_infos = link_infos, shared_libs = shared_libs, + linker_flags = _linker_flags(ctx), default_soname = shlib_name, include_in_android_mergemap = getattr(ctx.attrs, "include_in_android_merge_map_output", True), ), diff --git a/prelude/rust/rust_toolchain.bzl b/prelude/rust/rust_toolchain.bzl index 54474be..7d21675 100644 --- a/prelude/rust/rust_toolchain.bzl +++ b/prelude/rust/rust_toolchain.bzl @@ -176,6 +176,22 @@ rust_toolchain_attrs = { # "target-feature" will block both "-Ctarget-feature=..." and the split # form "-C" followed by "target-feature=...". "restricted_rustc_flags": provider_field(list[typing.Any], default = []), + # Under `-Csplit-debuginfo=unpacked`, which the rules pass for the cxx + # toolchain's `single` and `split` debug modes, rustc writes each codegen + # unit's `.dwo` to `--out-dir` and also packs a copy of it into the rlib or + # staticlib, so that a downstream `-Csplit-debuginfo=packed` link could + # build a dwp from the archive alone. The rules never link that way: `dwp` + # reads the `--out-dir` files, which are tracked as external debug info, + # and linkers never pull archive members that define no symbols. The + # copies only add to what every dependent compile downloads and every + # link materializes, and in debug-heavy builds they are a large share of + # rlib bytes. When set, the compile action deletes the `.dwo` members from + # rlibs and staticlibs right after rustc writes them, using the cxx + # toolchain's archiver (`gnu`, `llvm` or `bsd` archiver types; others keep + # the members). No effect when the cxx toolchain's split debug mode is + # `none`, or for Apple and Windows targets, whose archives never carry + # `.dwo` members. + "strip_dwo_from_rlibs": provider_field(bool, default = False), } RustToolchainInfo = provider(fields = rust_toolchain_attrs) diff --git a/prelude/rust/tools/extract_link_action.py b/prelude/rust/tools/extract_link_action.py index c6f6551..5d716b6 100644 --- a/prelude/rust/tools/extract_link_action.py +++ b/prelude/rust/tools/extract_link_action.py @@ -18,6 +18,7 @@ import argparse import os +import re import shutil import subprocess import sys @@ -32,6 +33,7 @@ def eprint(*args: Any, **kwargs: Any) -> None: class Args(NamedTuple): out_argsfile: IO[str] out_artifacts: Path + out_manifest: IO[str] out_archive: Path | None archiver_argsfile: Path | None linker: list[str] @@ -49,8 +51,14 @@ def arg_parse() -> Args: type=Path, required=True, ) + parser.add_argument( + "--out_manifest", + type=argparse.FileType("w"), + required=True, + help="Receives the basenames of the extracted objects, one per line", + ) # When --out_archive and archiver_argsfile are set, the rustc objects are - # collected into an archive rather than in the argsfile. Distributed ThinLTO needs + # additionally collected into an archive. Distributed ThinLTO needs # the objects to arrive as an `ArchiveLinkable` so it can plan a per-object # opt action for each linkable. parser.add_argument( @@ -74,11 +82,49 @@ def arg_parse() -> Args: return Args(**vars(parser.parse_args())) +def expand_response_files(args: list[str]) -> list[str]: + """Inline the contents of any `@file` argument. + + When the argv would exceed the OS command-line limit (which a Rust link + line always does on Windows), rustc re-invokes the linker with all + arguments in a single `@file`, one argument per line. Three formats are + seen in the wild: + - UTF-16 with each argument quoted and embedded quotes doubled, for + msvc-flavored linkers; + - UTF-8 with the same quoting, for direct (non-cc-driver) linkers + invoked on a Windows host, e.g. wasm-ld; + - UTF-8 with backslashes and spaces escaped by a backslash, for + gcc-flavored linkers. + The quoted forms leave backslashes raw, so quoting is detected per line + before falling back to backslash-unescaping. A backslash-escaped line + can never begin with a bare quote (it would be written as `\\"`), so the + detection is unambiguous. + """ + expanded = [] + for arg in args: + if not arg.startswith("@"): + expanded.append(arg) + continue + data = Path(arg[1:]).read_bytes() + utf16 = data.startswith(b"\xff\xfe") or data.startswith(b"\xfe\xff") + text = data.decode("utf-16") if utf16 else data.decode("utf-8-sig") + for line in text.splitlines(): + if not line: + continue + if len(line) >= 2 and line.startswith('"') and line.endswith('"'): + expanded.append(line[1:-1].replace('""', '"')) + elif utf16: + expanded.append(line) + else: + expanded.append(re.sub(r"\\(.)", r"\1", line)) + return expanded + + def process_link_args( - args: list[str], out_artifacts: Path, collect_objects: bool + args: list[str], out_artifacts: Path ) -> tuple[list[str], list[str]]: """Extract rustc's synthesized objects from its linker argv, returning - (args, objects). + (retained args, objects). The objects are the only thing extracted: - the codegen unit objects of the crate itself, which includes the @@ -87,19 +133,17 @@ def process_link_args( `#[used]` symbols of every linked crate so that the corresponding archive members survive the link. - Nothing else is taken. In particular, the flags and libraries rustc puts - on its link line are deliberately dropped: the toolchain and the - dependency graph are the sole owners of the flags and libraries on the - cxx-driven link. A rustc-synthesized file we don't recognize is an error, - not something to pass through: it would be a dangling path by the time - the link action runs. Version scripts are one known such case, so dylib - and cdylib crates cannot currently be linked through this. - - With `collect_objects`, the objects are returned separately and left out - of the returned args, for the caller to archive instead. Otherwise they - are listed in the args and no objects are returned. + The only args retained are wasm-ld `--export` args, which carry the same + per-program symbol metadata that `symbols.o` does on ELF (see the comment + at the branch). All other flags and libraries rustc puts on its link line + are deliberately dropped: the toolchain and the dependency graph are the + sole owners of the flags and libraries on the cxx-driven link. A + rustc-synthesized file we don't recognize is an error, not something to + pass through: it would be a dangling path by the time the link action + runs. Version scripts are one known such case, so dylib and cdylib crates + cannot currently be linked through this. """ - new_args = [] + retained_args = [] objects = [] # Original argv entries we handled, and the directories rustc placed # synthesized files in, used to detect files we don't know about. @@ -115,11 +159,64 @@ def process_link_args( if path.parent.is_absolute(): temp_dirs.add(str(path.parent)) handled.add(arg) - new_path = shutil.copy(path, out_artifacts) - if collect_objects: - objects.append(new_path) - else: - new_args.append(new_path) + + objects.append(shutil.copy(path, out_artifacts)) + i += 1 + continue + + # wasm has no `symbols.o`: rustc communicates the crate graph's + # exported/kept-alive symbols to wasm-ld as `--export` arguments + # instead. Retaining these is acceptable for the same reason + # extracting `symbols.o` is — they are per-program symbol metadata, + # the analogue of what C compilers embed in object files as wasm + # export flags — NOT a precedent for preserving arbitrary flags. + elif arg in ("--export", "--export-if-defined"): + retained_args.append(arg) + if i + 1 < size: + handled.add(args[i + 1]) + retained_args.append(args[i + 1]) + i += 2 + continue + elif arg.startswith("--export=") or arg.startswith("--export-if-defined="): + retained_args.append(arg) + i += 1 + continue + + # Entry-point metadata is the same category: rustc knows whether the + # program's entry is a `_start`-style symbol or an exported `main` + # with no native entry (`--no-entry`). In C that information is + # carried by whether an object defines the entry symbol. + elif arg == "--no-entry": + retained_args.append(arg) + i += 1 + continue + elif arg == "--entry": + retained_args.append(arg) + if i + 1 < size: + handled.add(args[i + 1]) + retained_args.append(args[i + 1]) + i += 2 + continue + elif arg.startswith("--entry="): + retained_args.append(arg) + i += 1 + continue + + # Debugger-visualizer files (crate-graph natvis, embedded into the + # PDB by the linker). Deliberately dropped rather than extracted: the + # natvis set is declarable in the dependency graph — the attribute + # names an ordinary source file — and C++ treats visualizers as + # build/debugger configuration, not compiler output. Marked handled + # so the temp-file check stays quiet. + elif arg.startswith("/NATVIS:"): + handled.add(arg) + i += 1 + continue + + # The msvc-style spelling of `-o` below; the temporary output path + # is embedded in the token, so mark it handled. + elif arg.startswith("/OUT:"): + handled.add(arg) i += 1 continue @@ -151,7 +248,18 @@ def process_link_args( eprint("Teach process_link_args() about them.") sys.exit(1) - return new_args, objects + # rustc always passes at least the crate's own codegen-unit objects, so + # extracting nothing means the argv was misparsed. This also backstops the + # check above, which is blind when no extraction established `temp_dirs`. + if not objects: + eprint( + "extract_link_action.py: extracted no objects from rustc's link line; teach process_link_args() about whatever form it took:" + ) + for a in args: + eprint(f" {a!r}") + sys.exit(1) + + return retained_args, objects def archive_objects( @@ -167,21 +275,28 @@ def archive_objects( def main() -> int: args = arg_parse() + if args.out_artifacts.exists(): + shutil.rmtree(args.out_artifacts) os.mkdir(args.out_artifacts) archiver_argsfile = args.archiver_argsfile out_archive = args.out_archive archiving = archiver_argsfile is not None and out_archive is not None - filtered_args, objects = process_link_args( - args.linker[1:], + retained_args, objects = process_link_args( + expand_response_files(args.linker[1:]), out_artifacts=args.out_artifacts, - collect_objects=archiving, ) - args.out_argsfile.write("\n".join(filtered_args)) + args.out_argsfile.write("\n".join(retained_args)) args.out_argsfile.close() - if objects and archiver_argsfile is not None and out_archive is not None: + # The manifest carries only basenames: the consumer knows the artifacts + # directory as an artifact and resolves members against it, so the + # execution-time path of the directory never appears in any output. + args.out_manifest.write("\n".join(os.path.basename(o) for o in objects)) + args.out_manifest.close() + + if archiving: archive_objects(archiver_argsfile, out_archive, objects) return 0 diff --git a/prelude/rust/tools/rustc_action.py b/prelude/rust/tools/rustc_action.py index 60c1388..3ae3856 100755 --- a/prelude/rust/tools/rustc_action.py +++ b/prelude/rust/tools/rustc_action.py @@ -121,6 +121,9 @@ class Args(NamedTuple): failure_filter: Optional[IO[bytes]] required_output: Optional[list[tuple[str, str]]] echo: Optional[IO[bytes]] + strip_dwo_members: Optional[str] + archiver: Optional[list[str]] + archiver_deterministic: bool rustc: list[str] @@ -186,6 +189,23 @@ def arg_parse() -> Args: type=argparse.FileType("wb"), help="Write the input command line to this file, without running it", ) + parser.add_argument( + "--strip-dwo-members", + metavar="ARCHIVE", + help="Archive (rlib or staticlib) written by the compiler from which " + "to delete every `.dwo` member after a successful compile", + ) + parser.add_argument( + "--archiver", + action="append", + metavar="ARG", + help="Archiver command used by --strip-dwo-members, one argument per occurrence", + ) + parser.add_argument( + "--archiver-deterministic", + action="store_true", + help="Pass the `D` modifier to the archiver so the rewritten archive carries no timestamps", + ) parser.add_argument( "rustc", nargs=argparse.REMAINDER, @@ -193,7 +213,10 @@ def arg_parse() -> Args: help="Compiler command line", ) - return Args(**vars(parser.parse_args())) + args = Args(**vars(parser.parse_args())) + if args.strip_dwo_members is not None and not args.archiver: + parser.error("--strip-dwo-members requires --archiver") + return args def arg_eval(arg: str) -> str: @@ -237,6 +260,67 @@ def inherited_env() -> dict[str, str]: return env +# The archiver's `d` operation names each member to delete and rewrites the +# whole archive, so all members should go in one invocation. Keep each +# invocation's argv under the smallest command-line limit of the hosts the +# rules run on (Windows: 32767 characters). +ARCHIVER_ARGV_BUDGET = 30_000 + + +def strip_dwo_members( + archiver: list[str], archive: str, deterministic: bool, env: dict[str, str] +) -> int: + """Delete the `.dwo` members the compiler packed into an rlib or staticlib. + + Under `-Csplit-debuginfo=unpacked`, rustc writes each codegen unit's `.dwo` + to `--out-dir` and also copies it into the archive, so that a downstream + `-Csplit-debuginfo=packed` link could build a dwp from the archive alone. + The rules never link that way: `dwp` reads the `--out-dir` files, and + linkers never pull archive members that define no symbols. The copies are + dead weight for every consumer of the archive. The archive is left + untouched when it has no `.dwo` members. + """ + # Apple's archiver takes its deterministic mode from the environment. + env = {**env, "ZERO_AR_DATE": "1"} + + def members() -> list[str]: + listing = subprocess.run( + [*archiver, "t", archive], stdout=subprocess.PIPE, env=env, check=True + ) + return [m for m in listing.stdout.decode().splitlines() if m.endswith(".dwo")] + + try: + dwo_members = members() + if not dwo_members: + return 0 + + # GNU-style archivers delete one member per name given, so a name that + # occurs twice must be given twice. rustc never produces duplicates + # within one archive; the check below catches it if that changes. + delete = [*archiver, "dD" if deterministic else "d", archive] + budget = ARCHIVER_ARGV_BUDGET - sum(len(arg) + 1 for arg in delete) + chunks: list[list[str]] = [[]] + used = 0 + for member in dwo_members: + if chunks[-1] and used + len(member) + 1 > budget: + chunks.append([]) + used = 0 + chunks[-1].append(member) + used += len(member) + 1 + for chunk in chunks: + subprocess.run([*delete, *chunk], env=env, check=True) + + remaining = members() + if remaining: + eprint(f"{archive} still has .dwo members after deleting them: {remaining}") + return 1 + except subprocess.CalledProcessError as e: + eprint(f"failed to strip .dwo members from {archive}: {e}") + return e.returncode or 1 + + return 0 + + async def handle_output( # noqa: C901 proc: asyncio.subprocess.Process, args: Args, @@ -426,6 +510,12 @@ async def main() -> int: # noqa: C901 if res == 0 and got_error_diag: res = 1 + if res == 0 and args.strip_dwo_members is not None: + assert args.archiver is not None + res = strip_dwo_members( + args.archiver, args.strip_dwo_members, args.archiver_deterministic, env + ) + # Check for death by signal - this is always considered a failure if res < 0: cmdline = shlex.join(rustc_cmd + rustc_args) diff --git a/prelude/target_stats/target_stats.bzl b/prelude/target_stats/target_stats.bzl index 9f75168..a6c287b 100644 --- a/prelude/target_stats/target_stats.bzl +++ b/prelude/target_stats/target_stats.bzl @@ -6,13 +6,10 @@ # of this source tree. You may select, at your option, one of the # above-listed licenses. -# Shared implementation for the target_stats integration, reused by -# apple_library, cxx_library and android_library. A supporting rule calls -# target_stats_providers_and_subtargets(...) when the target_stats config is -# enabled and its toolchain carries a TargetStatsToolsInfo; the result is a -# TargetStatsInfo provider (carrying a per-target manifest + a transitive set -# over deps) and the [target_stats] / [all_target_stats] sub-targets. +# Shared implementation for the target_stats integration. +load("@prelude//:paths.bzl", "paths") +load(":target_stats_config.bzl", "TARGET_STATS_ENABLED") load(":target_stats_tools.bzl", "TargetStatsToolsInfo") load( ":target_stats_types.bzl", @@ -21,36 +18,50 @@ load( "TargetStatsRecord", ) -# Which cycle tool to run for a target (Apple file cycles, Android package -# cycles, or none). CycleMode = enum("file", "package", "none") _OUT_DIR = "__target_stats__" -def _file_stats_action(ctx: AnalysisContext, tools: TargetStatsToolsInfo, src: Artifact) -> Artifact: - out = ctx.actions.declare_output(_OUT_DIR, src.short_path + ".file_stats.json", has_content_based_path = False) +def _content_based_input(ctx: AnalysisContext, artifact: Artifact, name: str) -> Artifact: + """Return an input whose path can participate in cross-config dedupe. + + Source files are already eligible. Generated files are not necessarily + content-addressed, so normalize those through a cheap copy before feeding + them to the target_stats tools. + """ + if artifact.is_source: + return artifact + return ctx.actions.copy_file( + _OUT_DIR + "/inputs/" + name, + artifact, + has_content_based_path = True, + ) + +def _file_stats_action(ctx: AnalysisContext, tools: TargetStatsToolsInfo, src: Artifact, name: str) -> Artifact: + out = ctx.actions.declare_output(_OUT_DIR, name + ".file_stats.json", has_content_based_path = True) + content_based_src = _content_based_input(ctx, src, name) ctx.actions.run( - cmd_args([tools.file_stats, "--input", src, "--output", out.as_output()]), + cmd_args([tools.file_stats, "--input", content_based_src, "--output", out.as_output()]), category = "target_stats_file_stats", - identifier = src.short_path, + identifier = name, allow_cache_upload = True, + expect_eligible_for_dedupe = True, ) return out def _extract_target_data_action(ctx: AnalysisContext, tools: TargetStatsToolsInfo, file_stats_by_path: dict[str, Artifact]) -> Artifact: - # Manifest maps each source's Buck path -> its file_stats JSON; with_inputs - # carries those JSON artifacts as inputs so the tool can read them. manifest = ctx.actions.write_json( _OUT_DIR + "/extract_manifest.json", file_stats_by_path, with_inputs = True, - has_content_based_path = False, + has_content_based_path = True, ) - out = ctx.actions.declare_output(_OUT_DIR, "target_data.json", has_content_based_path = False) + out = ctx.actions.declare_output(_OUT_DIR, "target_data.json", has_content_based_path = True) ctx.actions.run( cmd_args([tools.extract_target_data, "--manifest", manifest, "--output", out.as_output()]), category = "target_stats_extract_target_data", allow_cache_upload = True, + expect_eligible_for_dedupe = True, ) return out @@ -59,7 +70,7 @@ def _cycles_action( ) -> Artifact | None: if cycle_mode == CycleMode("none"): return None - out = ctx.actions.declare_output(_OUT_DIR, "cycles.json", has_content_based_path = False) + out = ctx.actions.declare_output(_OUT_DIR, "cycles.json", has_content_based_path = True) if cycle_mode == CycleMode("file"): args = cmd_args([ tools.file_cycles, @@ -73,7 +84,13 @@ def _cycles_action( out.as_output(), ]) if swift_dot != None: - args.add("--swift-dot", swift_dot) + # Swift compilation deliberately uses configuration-based paths. + # Copy the finished graph into the content-addressed namespace so + # this analysis action can still dedupe when two configurations + # produced byte-for-byte identical dependency graphs. The cheap + # copy itself remains configuration-specific. + content_based_swift_dot = _content_based_input(ctx, swift_dot, "swift-dependencies.dot") + args.add("--swift-dot", content_based_swift_dot) else: args = cmd_args([ tools.package_cycles, @@ -86,40 +103,70 @@ def _cycles_action( args, category = "target_stats_cycles", allow_cache_upload = True, + expect_eligible_for_dedupe = True, ) return out +def _all_target_stats_subtarget(ctx: AnalysisContext, tset: TargetStatsInfoTSet) -> list[Provider]: + """A JSON object mapping every transitive target's label to its manifest.""" + manifest_by_target = {record.label: record.manifest for record in tset.traverse()} + out = ctx.actions.declare_output(_OUT_DIR, "all_target_stats.json", has_content_based_path = True) + out_with_inputs = ctx.actions.write_json(out, manifest_by_target, with_inputs = True) + return [ + DefaultInfo( + default_output = out, + other_outputs = [out_with_inputs, tset.project_as_args("manifests")], + ) + ] + +def target_stats_aggregate_providers_and_subtargets( + ctx: AnalysisContext, + *, + deps: list[Dependency], +) -> (list[Provider], dict[str, list[Provider]]): + """target_stats for a rule that contributes no sources of its own. + + A no-op when target_stats is disabled, so aggregating rules can call it + unconditionally. + """ + if not TARGET_STATS_ENABLED: + return [], {} + children = [dep[TargetStatsInfo].tset for dep in deps if dep.get(TargetStatsInfo) != None] + tset = ctx.actions.tset(TargetStatsInfoTSet, children = children) + info = TargetStatsInfo(label = str(ctx.label.raw_target()), tset = tset) + subtargets = { + "all_target_stats": _all_target_stats_subtarget(ctx, tset), + } + return [info], subtargets + def target_stats_providers_and_subtargets( ctx: AnalysisContext, *, tools: TargetStatsToolsInfo, - srcs: list[Artifact], + srcs: dict[str, Artifact], deps: list[Dependency], cycle_mode: CycleMode, module_name: str, swift_dot: Artifact | None = None, ) -> (list[Provider], dict[str, list[Provider]]): - """Build the target_stats actions, provider and sub-targets for one target. - - Returns (providers, subtargets): a single TargetStatsInfo (with a tset over - deps' TargetStatsInfo), and the [target_stats] (this target's manifest) and - [all_target_stats] (whole-graph manifests) sub-targets. - """ + """Build the target_stats actions, provider and sub-targets for one target.""" label = str(ctx.label.raw_target()) - # Per-file metrics, keyed by each source's package-relative path. file_stats_by_path = {} - for src in srcs: - file_stats_by_path[src.short_path] = _file_stats_action(ctx, tools, src) + for name, src in srcs.items(): + # A header map key need not be normalized, and it becomes an output + # path. third-party/libvpx maps each generated header under both "name" + # and "./name", so the same file arrives twice and normalizes to one + # name; measure it once rather than declaring the output twice. + name = paths.normalize(name) + if name in file_stats_by_path: + continue + file_stats_by_path[name] = _file_stats_action(ctx, tools, src, name) target_data = _extract_target_data_action(ctx, tools, file_stats_by_path) cycles = _cycles_action(ctx, tools, cycle_mode, target_data, module_name, swift_dot) - # Per-target manifest: aggregate info + cycles + file -> file_stats mapping. - # write_json(with_inputs) returns a cmd_args bundle (not an Artifact), so - # declare the output explicitly and keep the bundle to materialize the - # referenced per-file JSONs / cycles / target_data alongside the manifest. - manifest = ctx.actions.declare_output(_OUT_DIR, "target_stats.json", has_content_based_path = False) + manifest = ctx.actions.declare_output(_OUT_DIR, "target_stats.json", has_content_based_path = True) manifest_inputs = ctx.actions.write_json( manifest, { @@ -134,21 +181,17 @@ def target_stats_providers_and_subtargets( children = [dep[TargetStatsInfo].tset for dep in deps if dep.get(TargetStatsInfo) != None] tset = ctx.actions.tset( TargetStatsInfoTSet, - value = TargetStatsRecord(label = label, manifest = manifest), + value = TargetStatsRecord( + label = label, + manifest = manifest, + manifest_with_inputs = manifest_inputs, + ), children = children, ) info = TargetStatsInfo(label = label, tset = tset) - # [all_target_stats]: a file listing every transitive target's manifest. - all_manifests = ctx.actions.write( - _OUT_DIR + "/all_target_stats.txt", - tset.project_as_args("manifests"), - with_inputs = True, - has_content_based_path = False, - ) - subtargets = { - "all_target_stats": [DefaultInfo(default_output = all_manifests)], + "all_target_stats": _all_target_stats_subtarget(ctx, tset), "target_stats": [DefaultInfo(default_output = manifest, other_outputs = [manifest_inputs]), info], } return [info], subtargets diff --git a/prelude/target_stats/target_stats_config.bzl b/prelude/target_stats/target_stats_config.bzl index fcde72e..8cee0f1 100644 --- a/prelude/target_stats/target_stats_config.bzl +++ b/prelude/target_stats/target_stats_config.bzl @@ -8,9 +8,5 @@ load("@prelude//utils:buckconfig.bzl", "read_bool") -# Whether target_stats collection is enabled, from the buckconfig -# `[target_stats] enabled` (root cell). Read at load time because rule -# implementations cannot call read_config; supporting rules gate all of their -# target_stats actions, subtargets, and providers on this constant, so when it -# is False none of that work is defined at all. +# Read at load time: rule implementations cannot call read_config. TARGET_STATS_ENABLED = read_bool("target_stats", "enabled", default = False, root_cell = True) diff --git a/prelude/target_stats/target_stats_tools.bzl b/prelude/target_stats/target_stats_tools.bzl index b60e464..9cbef0f 100644 --- a/prelude/target_stats/target_stats_tools.bzl +++ b/prelude/target_stats/target_stats_tools.bzl @@ -6,22 +6,13 @@ # of this source tree. You may select, at your option, one of the # above-listed licenses. -# Reusable, rule-agnostic bundle of the target_stats tool executables. A -# concrete instance is attached (optionally) to a platform toolchain -# (cxx_toolchain, android_toolchain, ...) so any supporting rule can reach the -# tools via its toolchain when target_stats is enabled. New platforms can wire -# it onto their own toolchains without changing the rules that consume it. +# Attached to a platform toolchain so any supporting rule can reach the tools. TargetStatsToolsInfo = provider( fields = { - # Runs the intermediary that extracts cycle target-data from file_stats - # outputs. "extract_target_data": provider_field(RunInfo), - # Runs the per-library Apple file-cycle tool. "file_cycles": provider_field(RunInfo), - # Runs the per-file metrics tool. "file_stats": provider_field(RunInfo), - # Runs the per-library Android package-cycle tool. "package_cycles": provider_field(RunInfo), }, ) diff --git a/prelude/target_stats/target_stats_types.bzl b/prelude/target_stats/target_stats_types.bzl index 447752d..bb90a05 100644 --- a/prelude/target_stats/target_stats_types.bzl +++ b/prelude/target_stats/target_stats_types.bzl @@ -7,29 +7,16 @@ # above-listed licenses. # Per-target codebase statistics, propagated across the dependency graph. -# -# A supporting rule (apple_library, cxx_library, android_library, ...) that has -# target_stats enabled builds a per-target manifest JSON (aggregate metrics + -# cycles + file->json mapping) and returns a TargetStatsInfo. The provider -# carries a transitive set whose value is this target's TargetStatsRecord and -# whose children are the tsets of its dependencies, so a consumer can request -# the whole dependency graph's stats from a single top-level target. TargetStatsRecord = record( - # The (unconfigured) label of the target these stats are for. label = field(str), - # The per-target manifest JSON: aggregate info, the cycles output, and the - # mapping of source file -> its file_stats JSON. Produced with - # ctx.actions.write_json(..., with_inputs = True) so it carries the per-file - # JSONs + cycles artifact as inputs. manifest = field(Artifact), + manifest_with_inputs = field(typing.Any), ) -def _project_manifests(record: TargetStatsRecord) -> list[Artifact]: - return [record.manifest] +def _project_manifests(record: TargetStatsRecord) -> typing.Any: + return record.manifest_with_inputs -# Transitive set of TargetStatsRecord. The "manifests" projection yields every -# transitive target's manifest, used to build the [all_target_stats] subtarget. TargetStatsInfoTSet = transitive_set( args_projections = { "manifests": _project_manifests, @@ -38,10 +25,7 @@ TargetStatsInfoTSet = transitive_set( TargetStatsInfo = provider( fields = { - # The (unconfigured) label of the target. "label": provider_field(str), - # tset with this target's TargetStatsRecord as the value and its - # dependencies' tsets as the children. "tset": provider_field(TargetStatsInfoTSet), }, ) diff --git a/prelude/toolchains/android.bzl b/prelude/toolchains/android.bzl index 35433da..72f99b7 100644 --- a/prelude/toolchains/android.bzl +++ b/prelude/toolchains/android.bzl @@ -6,7 +6,7 @@ # of this source tree. You may select, at your option, one of the # above-listed licenses. -load("@prelude//android:android_toolchain.bzl", "AndroidPlatformInfo", "AndroidToolchainInfo") +load("@prelude//android:android_toolchain.bzl", "AidlToolchainInfo", "AndroidPlatformInfo", "AndroidToolchainInfo") load("@prelude//target_stats:target_stats_tools.bzl", "TargetStatsToolsInfo") load("@prelude//tests:test_listing.bzl", "TestListingInfo") @@ -83,12 +83,34 @@ android_sdk_tools = rule( }, ) +def system_android_aidl_toolchain(name, android_sdk_tools_target, **kwargs): + kwargs["aidl"] = "{}[aidl]".format(android_sdk_tools_target) + kwargs["framework_aidl_file"] = "{}[framework.aidl]".format(android_sdk_tools_target) + _system_android_aidl_toolchain_rule(name = name, **kwargs) + +def _system_android_aidl_toolchain_rule_impl(ctx): + return [ + DefaultInfo(), + AidlToolchainInfo( + aidl = ctx.attrs.aidl[RunInfo], + framework_aidl_file = ctx.attrs.framework_aidl_file, + ), + ] + +_system_android_aidl_toolchain_rule = rule( + attrs = { + "aidl": attrs.exec_dep(providers = [RunInfo]), + "framework_aidl_file": attrs.source(), + }, + impl = _system_android_aidl_toolchain_rule_impl, + is_toolchain_rule = True, +) + def system_android_toolchain(name, android_sdk_tools_target, jdk_system_image, **kwargs): kwargs["aapt2_filter_resources"] = "prelude//android/tools:filter_extra_resources" kwargs["aapt2"] = "{}[aapt2]".format(android_sdk_tools_target) kwargs["aar_builder"] = "prelude//toolchains/android/src/com/facebook/buck/android/aar:aar_builder_binary" kwargs["adb"] = "{}[adb]".format(android_sdk_tools_target) - kwargs["aidl"] = "{}[aidl]".format(android_sdk_tools_target) kwargs["android_jar"] = "{}[android.jar]".format(android_sdk_tools_target) kwargs["android_optional_jars"] = [] kwargs["apk_builder"] = "prelude//toolchains/android/src/com/facebook/buck/android/apk:apk_builder_binary" @@ -106,7 +128,6 @@ def system_android_toolchain(name, android_sdk_tools_target, jdk_system_image, * kwargs["filter_dex_class_names"] = "prelude//android/tools:filter_dex" kwargs["filter_prebuilt_native_library_dir"] = "prelude//android/tools:filter_prebuilt_native_library_dir" kwargs["filter_resources"] = "prelude//toolchains/android/src/com/facebook/buck/android/resources/filter:filter_resources_binary" - kwargs["framework_aidl_file"] = "{}[framework.aidl]".format(android_sdk_tools_target) # @oss-disable[end= ]: kwargs["gatorade_mergemap_tool"] = "prelude//android/tools/meta_only:gatorade_mergemap_tool" kwargs["generate_build_config"] = "prelude//toolchains/android/src/com/facebook/buck/android/build_config:generate_build_config_binary" kwargs["generate_manifest"] = "prelude//toolchains/android/src/com/facebook/buck/android/manifest:generate_manifest_binary" @@ -149,8 +170,6 @@ def system_android_toolchain(name, android_sdk_tools_target, jdk_system_image, * ) kwargs["secondary_dex_compression_command"] = "prelude//toolchains/android/src/com/facebook/buck/android/dex:secondary_dex_compression_binary" kwargs["secondary_dex_weight_limit"] = 1024 - kwargs["set_application_id_to_specified_package"] = True - kwargs["should_run_sanity_check_for_placeholders"] = True kwargs["unpack_aar"] = "prelude//android/tools:unpack_aar" kwargs["zipalign"] = "{}[zipalign]".format(android_sdk_tools_target) @@ -171,7 +190,6 @@ def system_android_toolchain_rule_impl(ctx): aapt2_filter_resources = ctx.attrs.aapt2_filter_resources[RunInfo], aar_builder = ctx.attrs.aar_builder[RunInfo], adb = ctx.attrs.adb[RunInfo], - aidl = ctx.attrs.aidl[RunInfo], android_bootclasspath = [ctx.attrs.android_jar], android_jar = ctx.attrs.android_jar, android_optional_jars = ctx.attrs.android_optional_jars, @@ -190,7 +208,6 @@ def system_android_toolchain_rule_impl(ctx): filter_dex_class_names = ctx.attrs.filter_dex_class_names, filter_prebuilt_native_library_dir = ctx.attrs.filter_prebuilt_native_library_dir, filter_resources = ctx.attrs.filter_resources, - framework_aidl_file = ctx.attrs.framework_aidl_file, # @oss-disable[end= ]: gatorade_mergemap_tool = ctx.attrs.gatorade_mergemap_tool[RunInfo], generate_build_config = ctx.attrs.generate_build_config, generate_manifest = ctx.attrs.generate_manifest, @@ -201,6 +218,7 @@ def system_android_toolchain_rule_impl(ctx): instrumentation_test_runner_main_class = ctx.attrs.instrumentation_test_runner_main_class, jar_splitter_command = ctx.attrs.jar_splitter_command, jdk_system_image = ctx.attrs.jdk_system_image, + jni_onload_check = ctx.attrs.jni_onload_check, manifest_utils = ctx.attrs.manifest_utils, merge_android_resource_sources = ctx.attrs.merge_android_resource_sources, merge_android_resources = ctx.attrs.merge_android_resources, @@ -219,8 +237,6 @@ def system_android_toolchain_rule_impl(ctx): replace_application_id_placeholders = ctx.attrs.replace_application_id_placeholders, secondary_dex_compression_command = ctx.attrs.secondary_dex_compression_command, secondary_dex_weight_limit = ctx.attrs.secondary_dex_weight_limit, - set_application_id_to_specified_package = ctx.attrs.set_application_id_to_specified_package, - should_run_sanity_check_for_placeholders = ctx.attrs.should_run_sanity_check_for_placeholders, sort_pre_dexed_files = ctx.attrs.sort_pre_dexed_files, unpack_aar = ctx.attrs.unpack_aar, zipalign = ctx.attrs.zipalign, @@ -233,7 +249,6 @@ system_android_toolchain_rule = rule( "aapt2_filter_resources": attrs.dep(providers = [RunInfo]), "aar_builder": attrs.dep(providers = [RunInfo]), "adb": attrs.dep(providers = [RunInfo]), - "aidl": attrs.dep(providers = [RunInfo]), "android_jar": attrs.source(), "android_optional_jars": attrs.list(attrs.source()), "apk_builder": attrs.dep(providers = [RunInfo]), @@ -252,7 +267,6 @@ system_android_toolchain_rule = rule( "filter_dex_class_names": attrs.dep(providers = [RunInfo]), "filter_prebuilt_native_library_dir": attrs.dep(providers = [RunInfo]), "filter_resources": attrs.dep(providers = [RunInfo]), - "framework_aidl_file": attrs.source(), # @oss-disable[end= ]: "gatorade_mergemap_tool": attrs.dep(providers = [RunInfo]), "generate_build_config": attrs.dep(providers = [RunInfo]), "generate_manifest": attrs.dep(providers = [RunInfo]), @@ -262,6 +276,7 @@ system_android_toolchain_rule = rule( "instrumentation_test_runner_main_class": attrs.string(), "jar_splitter_command": attrs.dep(providers = [RunInfo]), "jdk_system_image": attrs.source(), + "jni_onload_check": attrs.option(attrs.dep(providers = [RunInfo]), default = None), "list_tests": attrs.option(attrs.dep(providers = [RunInfo]), default = None), "manifest_utils": attrs.dep(providers = [RunInfo]), "merge_android_resource_sources": attrs.dep(providers = [RunInfo]), @@ -280,9 +295,7 @@ system_android_toolchain_rule = rule( "replace_application_id_placeholders": attrs.dep(providers = [RunInfo]), "secondary_dex_compression_command": attrs.dep(providers = [RunInfo]), "secondary_dex_weight_limit": attrs.int(), - "set_application_id_to_specified_package": attrs.bool(), - "should_run_sanity_check_for_placeholders": attrs.bool(), - "sort_pre_dexed_files": attrs.option(attrs.dep(providers = [RunInfo]), default = None), + "sort_pre_dexed_files": attrs.dep(providers = [RunInfo]), "target_stats_tools": attrs.option(attrs.dep(providers = [TargetStatsToolsInfo]), default = None), "unpack_aar": attrs.dep(providers = [RunInfo]), "zipalign": attrs.dep(providers = [RunInfo]), diff --git a/prelude/toolchains/android/src/com/facebook/buck/android/AdbExecutionContext.java b/prelude/toolchains/android/src/com/facebook/buck/android/AdbExecutionContext.java index e49d0b3..0921e0f 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/android/AdbExecutionContext.java +++ b/prelude/toolchains/android/src/com/facebook/buck/android/AdbExecutionContext.java @@ -10,24 +10,14 @@ package com.facebook.buck.android; -import com.facebook.buck.util.Console; import com.facebook.buck.util.environment.EnvVariablesProvider; import com.facebook.infer.annotation.Nullsafe; import com.google.common.collect.ImmutableMap; @Nullsafe(Nullsafe.Mode.LOCAL) public class AdbExecutionContext { - private final Console console; - - public AdbExecutionContext(Console console) { - this.console = console; - } public ImmutableMap getEnvironment() { return EnvVariablesProvider.getSystemEnv(); } - - public Console getConsole() { - return console; - } } diff --git a/prelude/toolchains/android/src/com/facebook/buck/android/AdbHelper.java b/prelude/toolchains/android/src/com/facebook/buck/android/AdbHelper.java index 8a008bd..317040f 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/android/AdbHelper.java +++ b/prelude/toolchains/android/src/com/facebook/buck/android/AdbHelper.java @@ -25,7 +25,6 @@ import com.facebook.buck.android.exopackage.SetDebugAppMode; import com.facebook.buck.core.filesystems.AbsPath; import com.facebook.buck.core.util.log.Logger; -import com.facebook.buck.util.Console; import com.facebook.buck.util.MoreSuppliers; import com.facebook.buck.util.Threads; import com.facebook.buck.util.environment.EnvVariablesProvider; @@ -66,6 +65,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; +import java.util.stream.Stream; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; import org.jetbrains.annotations.Nullable; @@ -94,6 +94,9 @@ public class AdbHelper implements AndroidDevicesHelper { */ static final String SERIAL_NUMBER_ENV = "ANDROID_SERIAL"; + private static final String ABI_PROPERTY = "ro.product.cpu.abi"; + private static final String ABI_LIST_PROPERTY = "ro.product.cpu.abilist"; + static final int NUM_TRIES = 5; static final int RETRY_DELAY_MS = 1000; @@ -108,7 +111,7 @@ public class AdbHelper implements AndroidDevicesHelper { private final boolean restartAdbOnFailure; // Caches the list of android devices for this execution private final Supplier devicesSupplier; - private final boolean skipMetadataIfNoInstalls; + private final Supplier>> deviceAbisSupplier; private final AndroidInstallPrinter androidPrinter; private final SetDebugAppMode setDebugAppMode; private final InstallTimings timings; @@ -124,7 +127,6 @@ public AdbHelper( AdbExecutionContext adbExecutionContext, AndroidInstallPrinter androidPrinter, boolean restartAdbOnFailure, - boolean skipMetadataIfNoInstalls, SetDebugAppMode setDebugAppMode) { this( adbUtils, @@ -133,7 +135,6 @@ public AdbHelper( adbExecutionContext, androidPrinter, restartAdbOnFailure, - skipMetadataIfNoInstalls, setDebugAppMode, InstallTimings.NONE); } @@ -145,7 +146,6 @@ public AdbHelper( AdbExecutionContext adbExecutionContext, AndroidInstallPrinter androidPrinter, boolean restartAdbOnFailure, - boolean skipMetadataIfNoInstalls, SetDebugAppMode setDebugAppMode, InstallTimings timings) { this.timings = timings; @@ -155,8 +155,8 @@ public AdbHelper( this.adbExecutionContext = adbExecutionContext; this.restartAdbOnFailure = restartAdbOnFailure; this.devicesSupplier = MoreSuppliers.memoize(this::getDevicesImpl); + this.deviceAbisSupplier = MoreSuppliers.memoize(this::deviceAbisBySerialImpl); this.androidPrinter = androidPrinter; - this.skipMetadataIfNoInstalls = skipMetadataIfNoInstalls; this.setDebugAppMode = setDebugAppMode; } @@ -166,6 +166,32 @@ public static void setDevicesSupplierForTests( AdbHelper.devicesSupplierForTests = devicesSupplierForTests; } + /** + * Of the devices this helper resolved, those adb no longer reports. + * + *

The resolved set is fixed at its first use; this asks adb afresh on every call, so it is for + * deciding once whether to go on rather than for polling. What it asks for is every serial adb + * reports, not the filtered set: a superset can only shrink the answer, and a device that never + * matched the filter was never resolved, so it can never be reported gone. + */ + public ImmutableSet departedSerials() { + ImmutableSet connected = + adbUtils.getDevices().stream() + .map(AndroidDevice::getSerialNumber) + .collect(ImmutableSet.toImmutableSet()); + if (connected.isEmpty()) { + // Adb answering with nothing is far more likely to be adb than every device at once. Reading + // it as a mass departure would fail installs that a retry would have completed, and a device + // that really has gone still fails the moment it is used. + LOG.warn("adb reported no devices at all; not treating that as a disconnection"); + return ImmutableSet.of(); + } + return devicesSupplier.get().devices.stream() + .map(AndroidDevice::getSerialNumber) + .filter(serial -> !connected.contains(serial)) + .collect(ImmutableSet.toImmutableSet()); + } + @Override public ImmutableList getDevices(boolean quiet) { GetDevicesResult result = devicesSupplier.get(); @@ -289,10 +315,9 @@ public void installApk( AbsPath rootPath, boolean installViaSd, boolean quiet, - String fullyQualifiedName) + String fullyQualifiedName, + String packageName) throws InterruptedException { - String packageName = - tryToExtractPackageNameFromManifest(isolatedApkInfo.getManifestPath().getPath()); Optional buck2BuildUuid = Optional.ofNullable(EnvVariablesProvider.getSystemEnv().get("BUCK2_UUID")); @@ -327,10 +352,9 @@ public Set getAndroidDeviceInfo(IsolatedApkInfo isolatedApkIn // Need to call both ro.product.cpu.abi and ro.product.cpu.abilist // as sticking to ro.product.cpu.abi helped fixing the issue of // exopackage install when the app was already installed in the device. - String abi = device.getProperty("ro.product.cpu.abi"); + String abi = device.getProperty(ABI_PROPERTY); Set abiList = - new HashSet<>( - Arrays.asList(device.getProperty("ro.product.cpu.abilist").split(","))); + new HashSet<>(Arrays.asList(device.getProperty(ABI_LIST_PROPERTY).split(","))); String locale = getDeviceLocale(device); String buildFingerprint = device.getProperty("ro.build.fingerprint"); String dpi = getDeviceDpi(device); @@ -365,6 +389,53 @@ public Set getAndroidDeviceInfo(IsolatedApkInfo isolatedApkIn return deviceInfos; } + /** + * What each targeted device can run, by serial, read through the configured adb rather than + * whatever `adb` is on the path. A device that will not say is left out rather than recorded as + * running nothing, so callers can tell "cannot run this" from "would not answer". + * + *

Answered once per helper: these are boot-time properties of a fixed set of devices, and the + * check that reads them is offered every time an artifact arrives. + */ + public ImmutableMap> deviceAbisBySerial() { + return deviceAbisSupplier.get(); + } + + private ImmutableMap> deviceAbisBySerialImpl() { + ImmutableMap.Builder> abis = ImmutableMap.builder(); + for (AndroidDevice device : getDevices(true)) { + // Both properties, trimmed, blanks dropped: the list is comma separated and devices do put + // spaces after the commas. Read separately so a device that answers one still contributes + // it. + ImmutableSet forDevice = + Stream.concat( + readProperty(device, ABI_PROPERTY).stream(), + readProperty(device, ABI_LIST_PROPERTY).stream() + .flatMap(list -> Arrays.stream(list.split(",")))) + .map(String::trim) + .filter(abi -> !abi.isEmpty()) + .collect(ImmutableSet.toImmutableSet()); + if (!forDevice.isEmpty()) { + abis.put(device.getSerialNumber(), forDevice); + } + } + ImmutableMap> bySerial = abis.build(); + if (bySerial.isEmpty()) { + LOG.info("No targeted device would report an ABI"); + } + return bySerial; + } + + /** One property of one device, or empty if the device will not answer for it. */ + private Optional readProperty(AndroidDevice device, String property) { + try { + return Optional.ofNullable(device.getProperty(property)); + } catch (Exception e) { + LOG.warn(e, "Could not read %s of %s", property, device.getSerialNumber()); + return Optional.empty(); + } + } + public void throwIfIncompatibleAbi( AndroidDeviceInfo androidDeviceInfo, IsolatedApkInfo isolatedApkInfo) throws IncompatibleAbiException { @@ -380,7 +451,6 @@ public void throwIfIncompatibleAbi( apk.getName(), String.format(" (CPU(s): %s)", String.join(", ", apkAbis)), String.join(", ", abis)); - getConsole().printErrorText(errorMsg); throw new IncompatibleAbiException(errorMsg); } } @@ -737,10 +807,6 @@ private GetDevicesResult getDevicesImpl() { devices.stream().collect(ImmutableList.toImmutableList())); } - private Console getConsole() { - return adbExecutionContext.getConsole(); - } - @Override public synchronized void close() { // getExecutorService() requires the context for lazy initialization, so explicitly check if it @@ -784,6 +850,33 @@ private void installApkExopackageWithRetries( } } + /** + * Pushes exopackage payloads to the matching devices ahead of the install itself. + * + *

Only pushes content. Metadata, collection of stale files and the apk all need the complete + * artifact set, so they stay in {@link ExopackageInstaller#doInstall}, which lists the directory + * again and skips whatever landed here. + */ + public void streamExopackagePayloads( + AbsPath rootPath, IsolatedExopackageInfo isolatedExopackageInfo, String packageName) + throws InterruptedException { + adbCall( + "push exopackage files", + device -> { + new ExopackageInstaller( + isolatedExopackageInfo, + androidPrinter, + rootPath, + packageName, + device, + Optional.empty(), + timings) + .streamPayloads(); + return true; + }, + /* quiet= */ true); + } + private void installApkExopackage( AbsPath rootPath, IsolatedExopackageInfo isolatedExopackageInfo, @@ -801,7 +894,6 @@ private void installApkExopackage( rootPath, packageName, device, - skipMetadataIfNoInstalls, buck2BuildUuid, timings) .doInstall(isolatedApkInfo, setDebugAppMode); diff --git a/prelude/toolchains/android/src/com/facebook/buck/android/aapt/BUCK b/prelude/toolchains/android/src/com/facebook/buck/android/aapt/BUCK index 2a02832..c419545 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/android/aapt/BUCK +++ b/prelude/toolchains/android/src/com/facebook/buck/android/aapt/BUCK @@ -1,4 +1,5 @@ -load("@prelude//toolchains/android/tools:build_rules.bzl", "SIGNED_JAR_BLOCKLIST", "buck_java_binary", "buck_java_library") +load("@prelude//toolchains/android/tools:build_rules.bzl", "SIGNED_JAR_BLOCKLIST", "buck_java_binary", "buck_java_library", "buck_kotlin_library") +load("@prelude//toolchains/android/tools/build_rules:fb_native.bzl", "fb_native") load("@prelude//utils:source_listing.bzl", "source_listing") oncall("android_devxx") @@ -66,6 +67,47 @@ buck_java_binary( ], ) +fb_native.kotlin_library( + name = "resource_source_map_format", + srcs = ["ResourceSourceMapFormat.kt"], + _kotlin_toolchain = "toolchains//:kotlin_bootstrap", + provided_deps = [ + "prelude//toolchains/android/third-party:kotlin-stdlib", + ], + # @oss-disable[end= ]: source = "8", + # @oss-disable[end= ]: target = "8", + source = "11", # @oss-enable + target = "11", # @oss-enable + visibility = ["PUBLIC"], +) + +buck_kotlin_library( + name = "resource_source_map", + srcs = ["ResourceSourceMapExecutableMain.kt"], + tests = [ + "prelude//toolchains/android/test/com/facebook/buck/android/aapt:aapt", + ], + visibility = ["PUBLIC"], + deps = [ + "prelude//toolchains/android/src/com/facebook/buck/core/filesystems:filesystems", + "prelude//toolchains/android/src/com/facebook/buck/io/pathformat:pathformat", + "prelude//toolchains/android/third-party:args4j", + "prelude//toolchains/android/third-party:guava", + ":mini_aapt", + ":r_dot_txt", + ":resource_source_map_format", + ], +) + +buck_java_binary( + name = "resource_source_map_binary", + main_class = "com.facebook.buck.android.aapt.ResourceSourceMapExecutableMain", + visibility = ["PUBLIC"], + deps = [ + ":resource_source_map", + ], +) + buck_java_library( name = "merge_android_resource_sources", srcs = MERGE_ANDROID_RESOURCE_SOURCES_SRCS, diff --git a/prelude/toolchains/android/src/com/facebook/buck/android/aapt/MiniAapt.java b/prelude/toolchains/android/src/com/facebook/buck/android/aapt/MiniAapt.java index 71398f3..5e99b67 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/android/aapt/MiniAapt.java +++ b/prelude/toolchains/android/src/com/facebook/buck/android/aapt/MiniAapt.java @@ -10,7 +10,6 @@ package com.facebook.buck.android.aapt; -import com.facebook.buck.android.aapt.RDotTxtEntry.CustomDrawableType; import com.facebook.buck.android.aapt.RDotTxtEntry.IdType; import com.facebook.buck.android.aapt.RDotTxtEntry.RType; import com.facebook.buck.core.filesystems.AbsPath; @@ -251,22 +250,14 @@ void processDrawables(Path resourceFile) throws IOException, ResourceParseExcept int dotIndex = filename.indexOf('.'); String resourceName = dotIndex != -1 ? filename.substring(0, dotIndex) : filename; - boolean isGrayscaleImage = false; - if (!filename.endsWith(".xml")) { - // .g.png is no longer an allowed filename in newer versions of aapt2. - isGrayscaleImage = filename.endsWith(".g.png") || filename.endsWith(GRAYSCALE_SUFFIX); - if (isGrayscaleImage) { - // Trim _g or .g from the resource name - resourceName = filename.substring(0, filename.length() - GRAYSCALE_SUFFIX.length()); - } + // .g.png is no longer an allowed filename in newer versions of aapt2, hence the two spellings. + if (!filename.endsWith(".xml") + && (filename.endsWith(".g.png") || filename.endsWith(GRAYSCALE_SUFFIX))) { + // Trim _g or .g from the resource name + resourceName = filename.substring(0, filename.length() - GRAYSCALE_SUFFIX.length()); } - if (isGrayscaleImage) { - resourceCollector.addCustomDrawableResourceIfNotPresent( - RType.DRAWABLE, resourceName, CustomDrawableType.GRAYSCALE_IMAGE); - } else { - resourceCollector.addIntResourceIfNotPresent(RType.DRAWABLE, resourceName); - } + resourceCollector.addIntResourceIfNotPresent(RType.DRAWABLE, resourceName); } /** @@ -576,15 +567,6 @@ public void addIntResourceIfNotPresent(RType rType, String name) { } } - public void addCustomDrawableResourceIfNotPresent( - RType rType, String name, CustomDrawableType drawableType) { - RDotTxtEntry entry = new FakeRDotTxtEntry(IdType.INT, rType, name); - if (!resources.contains(entry)) { - String idValue = getNextCustomIdValue(rType, drawableType); - resources.add(new RDotTxtEntry(IdType.INT, rType, name, idValue, drawableType)); - } - } - public void addIntArrayResourceIfNotPresent(RType rType, String name, int numValues) { addResource(rType, IdType.INT_ARRAY, name, getNextArrayIdValue(rType, numValues), null); } @@ -609,10 +591,6 @@ String getNextIdValue(RType rType) { return String.format("0x%08x", getEnumerator(rType).next()); } - String getNextCustomIdValue(RType rType, CustomDrawableType drawableType) { - return String.format("0x%08x %s", getEnumerator(rType).next(), drawableType.getIdentifier()); - } - String getNextArrayIdValue(RType rType, int numValues) { // Robolectric expects the array to be populated with the right number of values, irrespective // of what the values are. diff --git a/prelude/toolchains/android/src/com/facebook/buck/android/aapt/RDotTxtEntry.java b/prelude/toolchains/android/src/com/facebook/buck/android/aapt/RDotTxtEntry.java index 71e9136..7ddd85b 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/android/aapt/RDotTxtEntry.java +++ b/prelude/toolchains/android/src/com/facebook/buck/android/aapt/RDotTxtEntry.java @@ -25,31 +25,6 @@ @Nullsafe(Nullsafe.Mode.LOCAL) public class RDotTxtEntry implements Comparable { - public enum CustomDrawableType { - NONE("") { - @Override - public String getIdentifier() { - throw new IllegalArgumentException( - String.format("'%s' does not have a custom identifier.", this)); - } - }, - GRAYSCALE_IMAGE(GRAYSCALE_IMAGE_IDENTIFIER); - - private final String identifier; - - CustomDrawableType(String identifier) { - this.identifier = identifier; - } - - /** - * Get the string identifier (currently a single character) for the custom drawable type. Used - * in R.txt files to identify custom drawables. - */ - public String getIdentifier() { - return identifier; - } - } - // Taken from http://developer.android.com/reference/android/R.html public enum RType { ANIM, @@ -112,15 +87,10 @@ public String toString() { return entry.get(); }; - private static final String GRAYSCALE_IMAGE_IDENTIFIER = "G"; public static final String INT_ARRAY_SEPARATOR = ","; - private static final Pattern TEXT_SYMBOLS_LINE = - Pattern.compile( - "(\\S+) (\\S+) (\\S+) ([^(" - + GRAYSCALE_IMAGE_IDENTIFIER - + ")]+)( [" - + GRAYSCALE_IMAGE_IDENTIFIER - + "])?"); + // The id group is ".+" rather than "\S+" because an int[] value spans several whitespace- + // separated tokens: "int[] styleable MyView { 0x7f020001,0x7f020002 }". + private static final Pattern TEXT_SYMBOLS_LINE = Pattern.compile("(\\S+) (\\S+) (\\S+) (.+)"); // A symbols file may look like: // @@ -145,42 +115,22 @@ public String toString() { // later we can properly sort all attributes. At the moment this field only used by IdType#INT // RType#STYLEABLE attributes. To make "compareTo" logic simpler, if parent is "null" - it will be // set to "name" value - // - // Custom drawables will have an additional column to denote them. - // int drawable custom_drawable 0x07f01250 # public final IdType idType; public final RType type; public final String name; public final String idValue; public final String parent; - public final CustomDrawableType customType; public RDotTxtEntry(IdType idType, RType type, String name, String idValue) { - this(idType, type, name, idValue, CustomDrawableType.NONE); + this(idType, type, name, idValue, name); } public RDotTxtEntry( IdType idType, RType type, String name, String idValue, @Nullable String parent) { - this(idType, type, name, idValue, CustomDrawableType.NONE, parent); - } - - public RDotTxtEntry( - IdType idType, RType type, String name, String idValue, CustomDrawableType customType) { - this(idType, type, name, idValue, customType, name); - } - - public RDotTxtEntry( - IdType idType, - RType type, - String name, - String idValue, - CustomDrawableType customType, - @Nullable String parent) { this.idType = idType; this.type = type; this.name = name; this.idValue = hexDecimalStringValue(idType, type, idValue); - this.customType = customType; this.parent = parent != null ? parent : name; } @@ -200,11 +150,11 @@ private static String hexDecimalStringValue(IdType idType, RType type, String id } public RDotTxtEntry copyWithNewIdValue(String newIdValue) { - return new RDotTxtEntry(idType, type, name, newIdValue, customType, parent); + return new RDotTxtEntry(idType, type, name, newIdValue, parent); } public RDotTxtEntry copyWithNewParent(String parent) { - return new RDotTxtEntry(idType, type, name, idValue, customType, parent); + return new RDotTxtEntry(idType, type, name, idValue, parent); } public static Optional parse(String rDotTxtLine) { @@ -213,23 +163,12 @@ public static Optional parse(String rDotTxtLine) { return Optional.empty(); } - CustomDrawableType customType = CustomDrawableType.NONE; IdType idType = IdType.from(java.util.Objects.requireNonNull(matcher.group(1))); RType type = RType.valueOf(java.util.Objects.requireNonNull(matcher.group(2)).toUpperCase()); String name = java.util.Objects.requireNonNull(matcher.group(3)); String idValue = java.util.Objects.requireNonNull(matcher.group(4)); - String custom = matcher.group(5); - - if (custom != null && custom.length() > 0) { - // Remove the leading space. - custom = custom.substring(1); - } - - if (GRAYSCALE_IMAGE_IDENTIFIER.equals(custom)) { - customType = CustomDrawableType.GRAYSCALE_IMAGE; - } - return Optional.of(new RDotTxtEntry(idType, type, name, idValue, customType)); + return Optional.of(new RDotTxtEntry(idType, type, name, idValue)); } /** diff --git a/prelude/toolchains/android/src/com/facebook/buck/android/aapt/ResourceSourceMapExecutableMain.kt b/prelude/toolchains/android/src/com/facebook/buck/android/aapt/ResourceSourceMapExecutableMain.kt new file mode 100644 index 0000000..1aaa612 --- /dev/null +++ b/prelude/toolchains/android/src/com/facebook/buck/android/aapt/ResourceSourceMapExecutableMain.kt @@ -0,0 +1,246 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is dual-licensed under either the MIT license found in the + * LICENSE-MIT file in the root directory of this source tree or the Apache + * License, Version 2.0 found in the LICENSE-APACHE file in the root directory + * of this source tree. You may select, at your option, one of the + * above-listed licenses. + */ + +package com.facebook.buck.android.aapt + +import com.facebook.buck.android.aapt.RDotTxtEntry.RType +import com.facebook.buck.core.filesystems.AbsPath +import com.facebook.buck.core.filesystems.RelPath +import com.facebook.buck.io.pathformat.PathFormatter +import com.google.common.collect.ImmutableMap +import com.google.common.collect.ImmutableSet +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths +import java.util.Locale +import java.util.TreeSet +import java.util.concurrent.Callable +import java.util.concurrent.Executors +import org.kohsuke.args4j.CmdLineParser +import org.kohsuke.args4j.Option + +/** Builds source provenance for the resources packaged into a Robolectric test. */ +object ResourceSourceMapExecutableMain { + private const val MAX_RESOURCE_WORKERS = 8 + + @JvmStatic + fun main(args: Array) { + val fallbackOutput = outputPath(args) + try { + run(args) + } catch (failure: Throwable) { + if (failure is VirtualMachineError || failure is ThreadDeath) throw failure + System.err.println( + "Robolectric resource source-map generation failed; " + + "continuing coverage with resource attribution disabled.", + ) + failure.printStackTrace(System.err) + val output = fallbackOutput ?: throw failure + try { + output.parent?.let(Files::createDirectories) + Files.writeString(output, "", StandardCharsets.UTF_8) + } catch (writeFailure: Throwable) { + writeFailure.addSuppressed(failure) + throw writeFailure + } + } + } + + private fun run(args: Array) { + val options = Options() + CmdLineParser(options).parseArgument(*args) + generate(options.resourceDirs, options.assetDirs, options.output) + } + + private fun outputPath(args: Array): Path? { + val index = args.indexOf("--output") + if (index >= 0 && index + 1 < args.size) return Paths.get(args[index + 1]) + val inlineValue = args.firstOrNull { it.startsWith("--output=") }?.substringAfter('=') + return inlineValue?.takeIf(String::isNotEmpty)?.let(Paths::get) + } + + internal fun generate(resourceDirsFile: Path, assetDirsFile: Path, output: Path) { + val root = Paths.get(".").toAbsolutePath().normalize() + val resourceRoots = readRoots(resourceDirsFile) + val assetRoots = readRoots(assetDirsFile) + val lines = TreeSet() + + val resourceFiles = resourceRoots.flatMapIndexed { priority, input -> + val resourceDir = resolve(root, input.path) + filesUnder(resourceDir).mapNotNull { (relative, file) -> + if (relative.nameCount < 2) { + null + } else { + ResourceFile(relative, file, input.ownerBuildFile, priority) + } + } + } + collectResources(root, resourceFiles, lines) + assetRoots.forEachIndexed { priority, input -> + collectAssets(root, input, priority, lines) + } + output.parent?.let(Files::createDirectories) + val contents = lines.joinToString(separator = "\n", postfix = if (lines.isEmpty()) "" else "\n") + Files.writeString(output, contents, StandardCharsets.UTF_8) + } + + private fun readRoots(path: Path): List = + Files.readAllLines(path, StandardCharsets.UTF_8).mapNotNull { line -> + if (line.isBlank()) { + null + } else { + val parts = line.split('\t', limit = 2) + InputRoot(Paths.get(parts[0]), parts.getOrElse(1) { "" }) + } + } + + private fun collectResources( + root: Path, + files: List, + output: MutableSet, + ) { + val (valuesFiles, otherFiles) = + files.partition { + val directory = it.relative.getName(0).toString() + directory == "values" || directory.startsWith("values-") + } + // MiniAapt uses shared XPathExpression instances for non-values XML, and XPathExpression is + // not thread-safe. Values parsing does not use those expressions, so only values files run in + // parallel. + val workerCount = minOf(MAX_RESOURCE_WORKERS, Runtime.getRuntime().availableProcessors()) + if (valuesFiles.size > 1 && workerCount > 1) { + val executor = Executors.newFixedThreadPool(workerCount) + try { + executor + .invokeAll(valuesFiles.map { file -> Callable { resourceLinesOrEmpty(root, file) } }) + .forEach { output.addAll(it.get()) } + } finally { + executor.shutdownNow() + } + } else { + valuesFiles.forEach { output.addAll(resourceLinesOrEmpty(root, it)) } + } + otherFiles.forEach { output.addAll(resourceLinesOrEmpty(root, it)) } + } + + private fun resourceLinesOrEmpty(root: Path, file: ResourceFile): List { + try { + return resourceLines(root, file) + } catch (failure: Throwable) { + if (failure is VirtualMachineError || failure is ThreadDeath) throw failure + System.err.println( + "Failed to map Robolectric resource file ${file.path}; skipping this file.", + ) + failure.printStackTrace(System.err) + return emptyList() + } + } + + private fun resourceLines(root: Path, file: ResourceFile): List { + val miniAapt = MiniAapt(ImmutableSet.of()) + miniAapt.processAllFiles(ImmutableMap.of(file.relative, file.path)) + val source = sourcePath(root, file.path, file.ownerBuildFile) + val qualifiers = qualifiers(file.relative.getName(0).toString()) + return miniAapt.resourceCollector.resources.mapNotNull { entry -> + // Styleables are generated arrays and indices, not runtime resource-table entries. MiniAapt + // emits their underlying attributes separately as R.attr entries. + if (entry.type == RType.STYLEABLE) { + null + } else { + ResourceSourceMapFormat.ResourceRow( + file.priority, + entry.type.name.lowercase(Locale.ROOT), + entry.name, + qualifiers, + source, + file.ownerBuildFile, + ) + .serialize() + } + } + } + + private fun collectAssets( + root: Path, + input: InputRoot, + priority: Int, + output: MutableSet, + ) { + val assetDir = resolve(root, input.path) + val assetFiles = + try { + filesUnder(assetDir) + } catch (failure: Throwable) { + if (failure is VirtualMachineError || failure is ThreadDeath) throw failure + System.err.println( + "Failed to enumerate Robolectric asset directory $assetDir; skipping this directory.", + ) + failure.printStackTrace(System.err) + return + } + for ((relative, file) in assetFiles) { + try { + output.add( + ResourceSourceMapFormat.AssetRow( + priority, + PathFormatter.pathWithUnixSeparators(relative), + sourcePath(root, file, input.ownerBuildFile), + input.ownerBuildFile, + ) + .serialize(), + ) + } catch (failure: Throwable) { + if (failure is VirtualMachineError || failure is ThreadDeath) throw failure + System.err.println( + "Failed to map Robolectric asset file $file; skipping this file.", + ) + failure.printStackTrace(System.err) + } + } + } + + private fun filesUnder(directory: Path): List> = + MiniAapt.getAllResourceFiles(AbsPath.of(directory), RelPath.get("")).entries.sortedBy { + PathFormatter.pathWithUnixSeparators(it.key) + } + + private fun resolve(root: Path, path: Path): Path = + if (path.isAbsolute) path.normalize() else root.resolve(path).normalize() + + private fun sourcePath(root: Path, path: Path, ownerBuildFile: String): String { + val realPath = path.toRealPath() + if (realPath.startsWith(root)) { + val relative = PathFormatter.pathWithUnixSeparators(root.relativize(realPath)) + if (!relative.startsWith("buck-out/")) return relative + } + return ownerBuildFile + } + + private fun qualifiers(resourceDirectory: String): String = + resourceDirectory.substringAfter('-', missingDelimiterValue = "") + + private data class InputRoot(val path: Path, val ownerBuildFile: String) + + private data class ResourceFile( + val relative: Path, + val path: Path, + val ownerBuildFile: String, + val priority: Int, + ) + + private class Options { + @field:Option(name = "--resource-dirs", required = true) lateinit var resourceDirs: Path + + @field:Option(name = "--asset-dirs", required = true) lateinit var assetDirs: Path + + @field:Option(name = "--output", required = true) lateinit var output: Path + } +} diff --git a/prelude/toolchains/android/src/com/facebook/buck/android/aapt/ResourceSourceMapFormat.kt b/prelude/toolchains/android/src/com/facebook/buck/android/aapt/ResourceSourceMapFormat.kt new file mode 100644 index 0000000..8c6a85c --- /dev/null +++ b/prelude/toolchains/android/src/com/facebook/buck/android/aapt/ResourceSourceMapFormat.kt @@ -0,0 +1,121 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is dual-licensed under either the MIT license found in the + * LICENSE-MIT file in the root directory of this source tree or the Apache + * License, Version 2.0 found in the LICENSE-APACHE file in the root directory + * of this source tree. You may select, at your option, one of the + * above-listed licenses. + */ + +package com.facebook.buck.android.aapt + +import java.nio.charset.StandardCharsets +import java.util.Base64 + +/** Serialization contract for Robolectric resource source-map rows. */ +object ResourceSourceMapFormat { + private const val DELIMITER = '\t' + private val fieldEncoder = Base64.getUrlEncoder().withoutPadding() + private val fieldDecoder = Base64.getUrlDecoder() + + interface Row { + fun serialize(): String + } + + /** Format: `R\tpriority\ttype\tname\tqualifiers\tsource\townerBuildFile`. */ + data class ResourceRow( + val priority: Int, + val type: String, + val name: String, + val qualifiers: String, + val source: String, + val ownerBuildFile: String, + ) : Row { + override fun serialize(): String = + listOf(TAG, priority.toString(), type, name, qualifiers, source, ownerBuildFile) + .joinToString(DELIMITER.toString()) + + companion object { + private const val TAG = "R" + + private enum class Attribute { + TAG, + PRIORITY, + TYPE, + NAME, + QUALIFIERS, + SOURCE, + OWNER_BUILD_FILE, + } + + internal fun parse(fields: List): ResourceRow? { + if (fields.size != Attribute.entries.size || fields[Attribute.TAG.ordinal] != TAG) + return null + return ResourceRow( + fields[Attribute.PRIORITY.ordinal].toIntOrNull() ?: return null, + fields[Attribute.TYPE.ordinal], + fields[Attribute.NAME.ordinal], + fields[Attribute.QUALIFIERS.ordinal], + fields[Attribute.SOURCE.ordinal], + fields[Attribute.OWNER_BUILD_FILE.ordinal], + ) + } + } + } + + /** Format: `A\tpriority\tencodedPath\tencodedSource\tencodedOwnerBuildFile`. */ + data class AssetRow( + val priority: Int, + val path: String, + val source: String, + val ownerBuildFile: String, + ) : Row { + override fun serialize(): String = listOf( + TAG, + priority.toString(), + encodeField(path), + encodeField(source), + encodeField(ownerBuildFile), + ) + .joinToString(DELIMITER.toString()) + + companion object { + private const val TAG = "A" + + private enum class Attribute { + TAG, + PRIORITY, + PATH, + SOURCE, + OWNER_BUILD_FILE, + } + + internal fun parse(fields: List): AssetRow? { + if (fields.size != Attribute.entries.size || fields[Attribute.TAG.ordinal] != TAG) + return null + return try { + AssetRow( + fields[Attribute.PRIORITY.ordinal].toIntOrNull() ?: return null, + decodeField(fields[Attribute.PATH.ordinal]), + decodeField(fields[Attribute.SOURCE.ordinal]), + decodeField(fields[Attribute.OWNER_BUILD_FILE.ordinal]), + ) + } catch (_: IllegalArgumentException) { + null + } + } + } + } + + fun parse(line: String): Row? { + val fields = line.split(DELIMITER) + return ResourceRow.parse(fields) ?: AssetRow.parse(fields) + } + + fun encodeField(value: String): String = + fieldEncoder.encodeToString(value.toByteArray(StandardCharsets.UTF_8)) + + fun decodeField(value: String): String = + String(fieldDecoder.decode(value), StandardCharsets.UTF_8) +} diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/cd/command/kotlin/AnnotationProcessingTool.kt b/prelude/toolchains/android/src/com/facebook/buck/android/dex/D8OutputOption.java similarity index 50% rename from prelude/toolchains/android/src/com/facebook/buck/jvm/cd/command/kotlin/AnnotationProcessingTool.kt rename to prelude/toolchains/android/src/com/facebook/buck/android/dex/D8OutputOption.java index 3dd4247..b18047c 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/jvm/cd/command/kotlin/AnnotationProcessingTool.kt +++ b/prelude/toolchains/android/src/com/facebook/buck/android/dex/D8OutputOption.java @@ -8,18 +8,12 @@ * above-listed licenses. */ -package com.facebook.buck.jvm.cd.command.kotlin +package com.facebook.buck.android.dex; -enum class AnnotationProcessingTool { - /** - * Default tool for Kotlin modules. Allows to run Java annotation processors against Kotlin - * sources while backporting it for Java sources too. - */ - KAPT, +import com.facebook.infer.annotation.Nullsafe; - /** - * Works only against Java sources, Kotlin sources won't have access to generated classes at - * compile time. - */ - JAVAC, +/** Additional metadata to capture while running D8. */ +@Nullsafe(Nullsafe.Mode.LOCAL) +public enum D8OutputOption { + CLASS_DESCRIPTORS, } diff --git a/prelude/toolchains/android/src/com/facebook/buck/android/dex/D8Utils.java b/prelude/toolchains/android/src/com/facebook/buck/android/dex/D8Utils.java index 2558af3..12334a0 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/android/dex/D8Utils.java +++ b/prelude/toolchains/android/src/com/facebook/buck/android/dex/D8Utils.java @@ -105,7 +105,7 @@ public static D8Output runD8Command( classpathFiles, minSdkVersion, threadCount, - false); + ImmutableSet.of()); } public static D8Output runD8CommandWithOutputClassDescriptors( @@ -129,7 +129,7 @@ public static D8Output runD8CommandWithOutputClassDescriptors( classpathFiles, minSdkVersion, threadCount, - true); + ImmutableSet.of(D8OutputOption.CLASS_DESCRIPTORS)); } private static D8Output runD8Command( @@ -142,7 +142,7 @@ private static D8Output runD8Command( Collection classpathFiles, Optional minSdkVersion, OptionalInt threadCount, - boolean captureOutputClassDescriptors) + Set outputOptions) throws CompilationFailedException, IOException { Set inputs = new HashSet<>(); for (Path toDex : filesToDex) { @@ -226,7 +226,7 @@ public void finished() {} }); OutputClassDescriptorConsumer outputClassDescriptorConsumer = null; - if (captureOutputClassDescriptors) { + if (outputOptions.contains(D8OutputOption.CLASS_DESCRIPTORS)) { outputClassDescriptorConsumer = new OutputClassDescriptorConsumer(recordingConsumer); builder.setProgramConsumer(outputClassDescriptorConsumer); } else { @@ -298,7 +298,7 @@ static void writeSecondaryDexJarAndMetadataFile( new BufferedInputStream(new FileInputStream(rawSecondaryDexPath.toFile()))) { ZipEntry customEntry = new ZipEntry("classes.dex"); - if (compression.equals("xz") || compression.equals("xzs")) { + if (compression.equals("xzs")) { try (ByteArrayOutputStream bos = new ByteArrayOutputStream()) { ByteStreams.copy(secondaryDexInputStream, bos); byte[] bytes = bos.toByteArray(); @@ -343,10 +343,10 @@ static void writeSecondaryDexMetadata( } long jarSize = Files.size(secondaryDexOutputJarPath); - if (compression.equals("xz") || compression.equals("xzs")) { + if (compression.equals("xzs")) { Preconditions.checkState( uncompressedSize + 120 == jarSize, - "For xz and xzs compression, we expect the .dex to be stored uncompressed and the " + "For xzs compression, we expect the .dex to be stored uncompressed and the " + "overhead of the .jar itself to be 120 bytes!"); } diff --git a/prelude/toolchains/android/src/com/facebook/buck/android/dex/MultiDexExecutableMain.java b/prelude/toolchains/android/src/com/facebook/buck/android/dex/MultiDexExecutableMain.java index be975f1..566c556 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/android/dex/MultiDexExecutableMain.java +++ b/prelude/toolchains/android/src/com/facebook/buck/android/dex/MultiDexExecutableMain.java @@ -215,8 +215,13 @@ private void run() throws IOException { Predicate matchesAllFiles = f -> true; PrimaryDexClassNamesHolder primaryDexClassNamesHolder = getPrimaryDexClassNames(primaryDexFilesToDex, matchesAllFiles, deobfuscateFunction); - Preconditions.checkState(classpath.isEmpty()); - classpath = ImmutableSet.copyOf(primaryDexFilesToDex); + ImmutableSet primaryDexClasspath = + ImmutableSet.copyOf( + Sets.difference( + ImmutableSet.builder().addAll(filesToDex).addAll(classpath).build(), + ImmutableSet.copyOf(primaryDexFilesToDex))); + classpath = + ImmutableSet.builder().addAll(primaryDexFilesToDex).addAll(classpath).build(); if (enableBootstrapDexes) { Preconditions.checkNotNull( bootstrapDexOutputDir, @@ -236,7 +241,7 @@ private void run() throws IOException { noOptimize ? EnumSet.of(D8Options.NO_OPTIMIZE) : EnumSet.noneOf(D8Options.class), Optional.of(hackMainDexListForBootstrapRun), Paths.get(androidJar), - filesToDex, + primaryDexClasspath, minSdkVersion, D8_THREAD_COUNT); } catch (CompilationFailedException e) { @@ -272,7 +277,7 @@ private void run() throws IOException { noOptimize ? EnumSet.of(D8Options.NO_OPTIMIZE) : EnumSet.noneOf(D8Options.class), Optional.empty(), Paths.get(androidJar), - filesToDex, + primaryDexClasspath, minSdkVersion, D8_THREAD_COUNT); } catch (CompilationFailedException e) { diff --git a/prelude/toolchains/android/src/com/facebook/buck/android/dex/SecondaryDexCompressionExecutableMain.java b/prelude/toolchains/android/src/com/facebook/buck/android/dex/SecondaryDexCompressionExecutableMain.java index b6f0798..1e5301d 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/android/dex/SecondaryDexCompressionExecutableMain.java +++ b/prelude/toolchains/android/src/com/facebook/buck/android/dex/SecondaryDexCompressionExecutableMain.java @@ -85,14 +85,14 @@ public static void main(String[] args) throws IOException { private void run() throws IOException { Path rawSecondaryDexesDirPath = Paths.get(rawSecondaryDexesDir); Preconditions.checkState( - ImmutableList.of("raw", "raw_subdir", "jar", "xz", "xzs").contains(compression), - "Only raw, raw_subdir, jar, xz and xzs compression is supported!"); + ImmutableList.of("raw", "raw_subdir", "jar", "xzs").contains(compression), + "Only raw, raw_subdir, jar and xzs compression is supported!"); Preconditions.checkState( compression.equals("raw") || compression.equals("raw_subdir") || compression.equals("jar") || xzCompressionLevel != -1, - "Must specify a valid compression level when xz or xzs compression is used!"); + "Must specify a valid compression level when xzs compression is used!"); if (bootstrapDexDirString != null) { Path bootstrapDexDir = Paths.get(bootstrapDexDirString); @@ -156,16 +156,9 @@ private void run() throws IOException { D8Utils.writeSecondaryDexJarAndMetadataFile( secondaryDexOutputJarPath, metadataPath, rawSecondaryDexPath, compression); - Path secondaryDexOutput; - if (compression.equals("xz")) { - secondaryDexOutput = doXzCompression(secondaryDexOutputJarPath); - } else { - secondaryDexOutput = secondaryDexOutputJarPath; - } - metadataLines.add( D8Utils.getSecondaryDexMetadataString( - secondaryDexOutput, + secondaryDexOutputJarPath, CanaryUtils.getFullyQualifiedCanaryClassName(canaryClassName, i))); } @@ -200,24 +193,6 @@ private String getSecondaryDexName(String module, int index, String suffix) { "%s-%d.dex%s", APKModule.isRootModule(module) ? "secondary" : module, index + 1, suffix); } - private Path doXzCompression(Path secondaryDexOutputJarPath) throws IOException { - Path xzCompressedOutputJarPath = - secondaryDexOutputJarPath.resolveSibling(secondaryDexOutputJarPath.getFileName() + ".xz"); - - try (InputStream in = - new BufferedInputStream(new FileInputStream(secondaryDexOutputJarPath.toFile())); - OutputStream out = - new BufferedOutputStream(new FileOutputStream(xzCompressedOutputJarPath.toFile())); - XZOutputStream xzOut = - new XZOutputStream(out, new LZMA2Options(xzCompressionLevel), XZ.CHECK_CRC32)) { - ByteStreams.copy(in, xzOut); - } - - Files.delete(secondaryDexOutputJarPath); - - return xzCompressedOutputJarPath; - } - private void doXzsCompression(Path secondaryDexSubdir, ImmutableList secondaryDexJarPaths) throws IOException { try (OutputStream secondaryDexOutput = diff --git a/prelude/toolchains/android/src/com/facebook/buck/android/exopackage/AndroidDevice.java b/prelude/toolchains/android/src/com/facebook/buck/android/exopackage/AndroidDevice.java index 9785107..23fcb9c 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/android/exopackage/AndroidDevice.java +++ b/prelude/toolchains/android/src/com/facebook/buck/android/exopackage/AndroidDevice.java @@ -129,8 +129,16 @@ boolean installApexOnDevice( */ void rmStaleFiles(String packageName) throws Exception; - AutoCloseable createForward() throws Exception; - + /** + * Pushes {@code installPaths}, a map of device destination to local source. + * + *

Called concurrently for the same package: an install pushes its payload as several shards at + * once. Whatever an implementation stages under has to be unique per call, or concurrent shards + * overwrite each other on the way in. + * + * @param packageName the app these files belong to. Scopes whatever scratch the device needs, so + * that installs of different apps cannot disturb each other's transfers. + */ void installFiles(String filesType, Map installPaths, String packageName) throws Exception; diff --git a/prelude/toolchains/android/src/com/facebook/buck/android/exopackage/AndroidDeviceImpl.kt b/prelude/toolchains/android/src/com/facebook/buck/android/exopackage/AndroidDeviceImpl.kt index ba6f1b4..f234a2a 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/android/exopackage/AndroidDeviceImpl.kt +++ b/prelude/toolchains/android/src/com/facebook/buck/android/exopackage/AndroidDeviceImpl.kt @@ -16,6 +16,7 @@ import com.google.common.base.Splitter import com.google.common.collect.ImmutableSortedSet import com.google.common.collect.Sets import java.io.File +import java.io.IOException import java.lang.Thread.sleep import java.nio.file.Files import java.nio.file.Path @@ -24,11 +25,15 @@ import java.nio.file.StandardCopyOption import java.security.MessageDigest import java.util.Optional import java.util.UUID +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.ConcurrentMap import java.util.regex.Pattern import kotlin.system.measureTimeMillis class AndroidDeviceImpl(val serial: String, val adbUtils: AdbUtils) : AndroidDevice { + private val properties: ConcurrentMap = ConcurrentHashMap() + override fun installApkOnDevice( apk: File, installViaSd: Boolean, @@ -498,11 +503,6 @@ class AndroidDeviceImpl(val serial: String, val adbUtils: AdbUtils) : AndroidDev LOG.info("Deleted ${filesToDelete.count()} files from $dirPath in ${elapsed/1000.0} seconds.") } - @Throws(Exception::class) - override fun createForward(): AutoCloseable { - return AutoCloseable {} - } - @Throws(Exception::class) override fun installFiles( filesType: String, @@ -539,16 +539,18 @@ class AndroidDeviceImpl(val serial: String, val adbUtils: AdbUtils) : AndroidDev .forEach { tempFolders[it] = Files.createTempDirectory("${it.fileName}_") } installPaths.forEach { (destination, source) -> val targetPath = tempFolders[destination.parent]?.resolve(destination.fileName) - Files.copy(source, targetPath, StandardCopyOption.REPLACE_EXISTING) + stageForPush(source, checkNotNull(targetPath)) } // push the temp folder to the device mkDirP(stagingDir) tempFolders.forEach { (destination, source) -> try { executeAdbCommand("push -z brotli $source $stagingDir") + // In staging, where the glob covers this shard's files and nothing else. The + // destination holds every shard's, so chmodding there costs the whole directory + // once per shard. The app will not start if its dex files are writable. + executeAdbShellCommand("chmod 644 $stagingDir/${source.fileName}/*") executeAdbShellCommand("mv $stagingDir/${source.fileName}/* $destination") - // instagram will fail to star if dex files are writable - executeAdbShellCommand("chmod 644 $destination/*") } catch (e: AdbCommandFailedException) { throw AndroidInstallException.adbCommandFailedException( "Failed to push $source to $destination.", @@ -583,6 +585,12 @@ class AndroidDeviceImpl(val serial: String, val adbUtils: AdbUtils) : AndroidDev "push $source $destination", "Failed to push $source to $destination.", ) + // As for the payloads above: nothing the app reads is left writable. One named file + // rather than a glob, since these are pushed one at a time. + executeAdbShellCommandCatching( + "chmod 644 $destination", + "Failed to set permissions on $destination.", + ) } } } @@ -616,7 +624,32 @@ class AndroidDeviceImpl(val serial: String, val adbUtils: AdbUtils) : AndroidDev @Throws(Exception::class) override fun getProperty(name: String): String { - return executeAdbShellCommandCatching("getprop $name", "Failed to get property $name.") + val read = { executeAdbShellCommandCatching("getprop $name", "Failed to get property $name.") } + // Only `ro.` properties are fixed at boot and so safe to hold on to; anything else can change + // under us mid-install. Caching is worth it because a single exopackage install otherwise + // re-queries ro.product.cpu.abilist five times over adb. + return if (name.startsWith("ro.")) properties.computeIfAbsent(name) { read() } else read() + } + + /** + * Hardlinks an artifact into the staging directory, copying only if it cannot be linked -- a + * different filesystem, typically, when the temp directory is on another volume. + * + * Buck materialises exopackage payloads as symlink farms and adb will not follow symlinks, so + * something has to resolve them; staging also renames each file to the hash-based name it takes + * on the device. Neither needs the bytes copied, and a payload is several GB. + * + * The link shares an inode with the artifact in buck-out, so nothing may modify a staged file. + * The `chmod` after the push deliberately runs on the device, not here. + */ + private fun stageForPush(source: Path, target: Path) { + try { + Files.createLink(target, source.toRealPath()) + } catch (e: IOException) { + Files.copy(source, target, StandardCopyOption.REPLACE_EXISTING) + } catch (e: UnsupportedOperationException) { + Files.copy(source, target, StandardCopyOption.REPLACE_EXISTING) + } } @Throws(Exception::class) @@ -732,8 +765,16 @@ class AndroidDeviceImpl(val serial: String, val adbUtils: AdbUtils) : AndroidDev ): Boolean { val destinationPath: String = dataRoot.resolve(packageName).toString() try { - executeAdbShellCommand("umask 022 && mkdir -p $destinationPath") - executeAdbShellCommand("echo $buildUuid > $destinationPath/build_uuid.txt") + // One `adb shell`, not two, because `umask` is per-process. Split across two shells the + // second one never runs the umask, so the file its redirect creates takes adbd's default + // mode rather than 0644. The directory is unaffected either way — the FIRST shell is what + // creates it — and that asymmetry is what makes this easy to miss: on a host whose default + // umask is already 022 both land 0644 and the split looks harmless. The file is read back + // as build provenance, so it should not be writable by anything but the installer. + executeAdbShellCommand( + "umask 022 && mkdir -p $destinationPath && " + + "echo $buildUuid > $destinationPath/build_uuid.txt", + ) } catch (e: Exception) { // we don't want to fail the install if we can't install the build_uuid.txt file LOG.warn("Failed to install build_uuid.txt file on $serial: ${e.message}") diff --git a/prelude/toolchains/android/src/com/facebook/buck/android/exopackage/ExopackageInstaller.java b/prelude/toolchains/android/src/com/facebook/buck/android/exopackage/ExopackageInstaller.java index 4fad54f..34b85db 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/android/exopackage/ExopackageInstaller.java +++ b/prelude/toolchains/android/src/com/facebook/buck/android/exopackage/ExopackageInstaller.java @@ -31,22 +31,39 @@ import com.google.common.collect.ImmutableSortedSet; import com.google.common.collect.Ordering; import com.google.common.io.Closer; +import com.google.common.util.concurrent.ThreadFactoryBuilder; import java.io.File; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.HashMap; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.OptionalLong; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.function.Function; -import java.util.stream.Collectors; -import org.jetbrains.annotations.Nullable; -/** ExopackageInstaller manages the installation of apps with the "exopackage" flag set to true. */ +/** + * ExopackageInstaller manages the installation of apps with the "exopackage" flag set to true. + * + *

Two ways in, one way down. {@link #doInstall} is the whole install; {@link #streamPayloads} + * sends a subset of the payloads while the build is still running. Both reach the device through + * {@code pushMissingFiles}, the only thing here that moves payload bytes. + * + *

Three names for sets of files: + * + *

    + *
  • {@code filesOnDevice} -- what listing the data root found already there. + *
  • {@code filesToDelete} -- on the device from previous installs and not wanted anymore. + *
  • {@code filesToPush} -- wanted by one payload and not on the device. + *
+ */ @Nullsafe(Nullsafe.Mode.LOCAL) public class ExopackageInstaller { @@ -68,6 +85,15 @@ public class ExopackageInstaller { private static final long BYTES_PER_BLOCK = 1024L; + private static final int MAX_CONCURRENT_PUSHES = 8; + + private static final long TARGET_SHARD_BYTES = 32L * 1024L * 1024L; + + private static final ExecutorService PUSH_EXECUTOR = + Executors.newFixedThreadPool( + MAX_CONCURRENT_PUSHES, + new ThreadFactoryBuilder().setNameFormat("exopackage-push-%d").setDaemon(true).build()); + public static final Path EXOPACKAGE_INSTALL_ROOT = Paths.get("/data/local/tmp/exopackage/"); private final IsolatedExopackageInfo exoInfo; @@ -77,7 +103,6 @@ public class ExopackageInstaller { private final String packageName; private final Optional buck2BuildUuid; private final Path dataRoot; - private final boolean skipMetadataIfNoInstalls; private final InstallTimings timings; public ExopackageInstaller( @@ -86,7 +111,6 @@ public ExopackageInstaller( AbsPath rootPath, String packageName, AndroidDevice device, - boolean skipMetadataIfNoInstalls, Optional buck2BuildUuid) { this( exoInfo, @@ -94,7 +118,6 @@ public ExopackageInstaller( rootPath, packageName, device, - skipMetadataIfNoInstalls, buck2BuildUuid, InstallTimings.NONE); } @@ -105,7 +128,6 @@ public ExopackageInstaller( AbsPath rootPath, String packageName, AndroidDevice device, - boolean skipMetadataIfNoInstalls, Optional buck2BuildUuid, InstallTimings timings) { this.timings = timings; @@ -115,7 +137,6 @@ public ExopackageInstaller( this.device = device; this.packageName = packageName; this.dataRoot = EXOPACKAGE_INSTALL_ROOT.resolve(packageName); - this.skipMetadataIfNoInstalls = skipMetadataIfNoInstalls; this.buck2BuildUuid = buck2BuildUuid; Preconditions.checkArgument(AdbHelper.PACKAGE_NAME_PATTERN.matcher(packageName).matches()); @@ -129,27 +150,32 @@ public void doInstall(IsolatedApkInfo isolatedApkInfo, SetDebugAppMode setDebugA } if (exopackageEnabled()) { long setupStart = System.currentTimeMillis(); - device.mkDirP(dataRoot.toString()); - device.fixRootDir(dataRoot.toString()); - ImmutableSortedSet presentFiles = device.listDirRecursive(dataRoot); + prepareDataRoot(); + ImmutableSortedSet filesOnDevice = device.listDirRecursive(dataRoot); timings.recordDeviceSetup(setupStart, System.currentTimeMillis()); ImmutableList payloads = resolveExoPayloads(); // Reclaim space before pushing so a device that is already full can free room for the // payload. Best effort on purpose: neither step is needed for the install to be correct. - // Scratch is read by nothing, and the unwanted set is disjoint from the push set, so the new - // app gets every file it needs either way -- a failure here leaves unreferenced files on the - // device and costs disk, not correctness. Whether there is still room to proceed is the - // preflight's answer to give, from the space the device actually has. + // Scratch is read by nothing, and filesToDelete is disjoint from filesToPush, so the new app + // gets every file it needs either way -- a failure here leaves unreferenced files on the + // device and costs disk, not correctness. That disjointness is also why the push below still + // reads the pre-delete listing: nothing deleted is a file any payload asks about. Whether + // there is still room to proceed is the preflight's answer to give, from the space the + // device actually has. try { device.rmStaleFiles(packageName); - deleteUnwantedFiles(presentFiles, wantedPaths(payloads)); + deleteFiles(filesToDelete(filesOnDevice, payloads)); } catch (Exception e) { LOG.warn(e, "Could not reclaim exopackage space for %s; continuing", packageName); } - checkEnoughFreeSpace(presentFiles, payloads); + pushMissingFiles(filesOnDevice, payloads); - installMissingExopackageFiles(presentFiles, payloads); + // Metadata is what the app reads to find these files, so it must not land before them. + installMetadata( + payloads.stream() + .flatMap(payload -> payload.metadataToInstall.entrySet().stream()) + .collect(ImmutableMap.toImmutableMap(Map.Entry::getKey, Map.Entry::getValue))); } if (buck2BuildUuid.isPresent()) { device.installBuildUuidFile( @@ -232,25 +258,189 @@ private List getDeviceAbis() { } /** Every path this install wants on the device; anything else under the data root is stale. */ - private static ImmutableSet wantedPaths(ImmutableList payloads) { - ImmutableSet.Builder wantedPaths = ImmutableSet.builder(); + /** + * Pushes whatever of {@code payloads} the device does not already have. + * + *

Every payload byte reaches a device through here, whether it goes while the build is still + * running or as part of the install. Free space is checked first, so a device without room says + * so rather than filling up partway through. + */ + private void pushMissingFiles( + ImmutableSortedSet filesOnDevice, ImmutableList payloads) + throws Exception { + ImmutableMap.Builder> transfers = + ImmutableMap.builder(); for (ResolvedExoPayload payload : payloads) { - wantedPaths.addAll(payload.filesToInstall.keySet()); - wantedPaths.addAll(payload.metadataToInstall.keySet()); + transfers.put(payload, filesToPush(filesOnDevice, payload.filesToInstall)); } - return wantedPaths.build(); + ImmutableMap> filesToTransfer = + transfers.build(); + + checkEnoughFreeSpace(filesToTransfer); + + ImmutableList.Builder shards = ImmutableList.builder(); + filesToTransfer.forEach( + (payload, files) -> + shards.addAll( + splitIntoShards(payload.type, files, rootPath, dataRoot, TARGET_SHARD_BYTES))); + pushShards(shards.build()); } - /** Installs missing exo package files */ - private void installMissingExopackageFiles( - ImmutableSortedSet presentFiles, ImmutableList payloads) - throws Exception { - ImmutableMap.Builder metadata = ImmutableMap.builder(); - for (ResolvedExoPayload payload : payloads) { - installMissingFiles(presentFiles, payload, metadata); + /** + * Pushes payload content ahead of the install proper. + * + *

Writes no metadata, collects no stale files and does not touch the apk: all three need the + * complete artifact set, and until metadata names them the pushed files are inert. The install + * lists the directory afterwards, so anything landed here is seen as already present and skipped + * -- which is what makes this safe to run more than once, and safe to fail. + */ + public void streamPayloads() throws Exception { + if (!exopackageEnabled()) { + return; + } + // Not recorded as device setup: this happens while the build is still running, so charging it + // to the install would report time the install never spent. + prepareDataRoot(); + pushMissingFiles(device.listDirRecursive(dataRoot), resolveExoPayloads()); + } + + /** Makes the data root usable. */ + private void prepareDataRoot() throws Exception { + device.mkDirP(dataRoot.toString()); + device.fixRootDir(dataRoot.toString()); + } + + @VisibleForTesting + static ImmutableSortedMap filesToPush( + ImmutableSortedSet filesOnDevice, ImmutableMap filesToInstall) { + return filesToInstall.entrySet().stream() + .filter(entry -> !filesOnDevice.contains(entry.getKey())) + .collect( + ImmutableSortedMap.toImmutableSortedMap( + Ordering.natural(), Map.Entry::getKey, Map.Entry::getValue)); + } + + /** + * Splits a payload into roughly equal chunks by size. + * + *

A single `adb push` is limited by per-stream round trips rather than by bandwidth or by the + * host, so concurrent pushes scale close to linearly. Sharding by bytes rather than by payload + * matters because native libs alone are over half the total, and pushing one payload per stream + * leaves that stream setting the wall time on its own. + */ + @VisibleForTesting + static ImmutableList splitIntoShards( + String filesType, + ImmutableSortedMap filesToInstall, + AbsPath rootPath, + Path dataRoot, + long targetShardBytes) { + ImmutableList.Builder shards = ImmutableList.builder(); + ImmutableMap.Builder current = ImmutableMap.builder(); + long currentBytes = 0L; + boolean currentIsEmpty = true; + + for (Map.Entry file : filesToInstall.entrySet()) { + Path localPath = rootPath.resolve(file.getValue()).getPath(); + current.put(dataRoot.resolve(file.getKey()), localPath); + currentBytes += localPath.toFile().length(); + currentIsEmpty = false; + // A shard can never be smaller than a single file, so a payload of one huge file stays whole. + if (currentBytes >= targetShardBytes) { + shards.add(new PushShard(filesType, current.build())); + current = ImmutableMap.builder(); + currentBytes = 0L; + currentIsEmpty = true; + } + } + if (!currentIsEmpty) { + shards.add(new PushShard(filesType, current.build())); + } + return shards.build(); + } + + /** Pushes every shard, up to {@link #MAX_CONCURRENT_PUSHES} at a time. */ + private void pushShards(ImmutableList shards) throws Exception { + if (shards.isEmpty()) { + return; + } + // Create every destination directory up front: shards from one payload share a directory, and + // concurrent mkdir -p of the same path is pointless work at best. + ImmutableSet destinationDirs = + shards.stream() + .flatMap(shard -> shard.installPaths.keySet().stream()) + .map(Path::getParent) + .filter(Objects::nonNull) + .collect(ImmutableSet.toImmutableSet()); + for (Path destinationDir : destinationDirs) { + device.mkDirP(destinationDir.toString()); + } + + List> pushes = new ArrayList<>(shards.size()); + for (PushShard shard : shards) { + pushes.add( + PUSH_EXECUTOR.submit( + () -> { + pushShard(shard); + return null; + })); + } + awaitAll(pushes); + } + + private void pushShard(PushShard shard) throws Exception { + long start = System.currentTimeMillis(); + device.installFiles(shard.filesType, shard.installPaths, packageName); + // Each shard records its own window; the group's transfer time is their union. + timings.recordPush(shard.filesType, start, System.currentTimeMillis()); + } + + /** + * Waits for every push, reporting the first failure with any others attached. + * + *

Deliberately does not cancel the rest once one fails. A thread blocked reading an {@code + * adb} subprocess does not observe an interrupt, so cancelling would not stop the transfer -- it + * would only stop us waiting for it, leaving shards writing to the device after the install has + * been called failed, and skipping the scratch each one removes on its way out. The cost is that + * a failing install takes as long as its slowest shard. + * + *

Interruption is the exception: it means the process is going away, so it stops waiting and + * leaves whatever is in flight to die with the JVM. + */ + private static void awaitAll(List> pushes) throws Exception { + Exception failure = null; + for (Future push : pushes) { + try { + push.get(); + } catch (ExecutionException e) { + Exception cause = e.getCause() instanceof Exception ? (Exception) e.getCause() : e; + if (failure == null) { + failure = cause; + } else { + // Every shard that failed, not just the first: they fail independently, and which one + // arrives first says nothing about which one explains the install. + failure.addSuppressed(cause); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw e; + } + } + if (failure != null) { + throw failure; + } + } + + /** One concurrently pushable chunk of a payload. */ + @VisibleForTesting + static final class PushShard { + final String filesType; + final ImmutableMap installPaths; + + PushShard(String filesType, ImmutableMap installPaths) { + this.filesType = filesType; + this.installPaths = installPaths; } - // Metadata is what the app reads to find these files, so it must not land before them. - installMetadata(metadata.build()); } /** @@ -258,24 +448,22 @@ private void installMissingExopackageFiles( * with a bare ENOSPC. */ private void checkEnoughFreeSpace( - ImmutableSortedSet presentFiles, ImmutableList payloads) { + ImmutableMap> filesToTransfer) { OptionalLong availableBytes = availableBytesOnDevice(); if (availableBytes.isEmpty()) { return; } long requiredBytes = 0L; - for (ResolvedExoPayload payload : payloads) { - for (Map.Entry file : payload.filesToInstall.entrySet()) { - if (!presentFiles.contains(file.getKey())) { - File source = rootPath.resolve(file.getValue()).toFile(); - if (!source.isFile()) { - // Zero is what length() would answer, which would quietly shrink the estimate and let - // the check pass. The install cannot succeed without the file either way, so say which - // one is missing while there is still somewhere useful to say it. - throw AndroidInstallException.Companion.artifactMissing(source.toString()); - } - requiredBytes += source.length(); + for (ImmutableSortedMap files : filesToTransfer.values()) { + for (Path source : files.values()) { + File file = rootPath.resolve(source).toFile(); + if (!file.isFile()) { + // Zero is what length() would answer, which would quietly shrink the estimate and let + // the check pass. The install cannot succeed without the file either way, so say which + // one is missing while there is still somewhere useful to say it. + throw AndroidInstallException.Companion.artifactMissing(file.toString()); } + requiredBytes += file.length(); } } if (requiredBytes > availableBytes.getAsLong()) { @@ -308,7 +496,7 @@ private OptionalLong availableBytesOnDevice() { } /** One exopackage payload class, with its contents resolved exactly once. */ - private static final class ResolvedExoPayload { + static final class ResolvedExoPayload { private final String type; private final ImmutableMap filesToInstall; private final ImmutableMap metadataToInstall; @@ -372,46 +560,33 @@ private String getInstalledAppManifestDigest(String packagePath) throws Exceptio return result; } - private void installMissingFiles( - ImmutableSortedSet presentFiles, - ResolvedExoPayload payload, - @Nullable ImmutableMap.Builder metadataToInstall) - throws Exception { - ImmutableSortedMap filesToInstall = - payload.filesToInstall.entrySet().stream() - .filter(entry -> !presentFiles.contains(entry.getKey())) - .collect( - ImmutableSortedMap.toImmutableSortedMap( - Ordering.natural(), Map.Entry::getKey, Map.Entry::getValue)); - - installFiles(payload.type, filesToInstall); - - if (metadataToInstall != null && (!skipMetadataIfNoInstalls || !filesToInstall.isEmpty())) { - metadataToInstall.putAll(payload.metadataToInstall); + /** What the device holds that no payload wants. The lock file belongs to no payload and stays. */ + @VisibleForTesting + static ImmutableSortedSet filesToDelete( + ImmutableSortedSet filesOnDevice, ImmutableList payloads) { + ImmutableSet.Builder wanted = ImmutableSet.builder(); + for (ResolvedExoPayload payload : payloads) { + wanted.addAll(payload.filesToInstall.keySet()); + wanted.addAll(payload.metadataToInstall.keySet()); } + ImmutableSet wantedFiles = wanted.build(); + return filesOnDevice.stream() + .filter(p -> !p.getFileName().toString().equals("lock") && !wantedFiles.contains(p)) + .collect(ImmutableSortedSet.toImmutableSortedSet(Ordering.natural())); } - private void deleteUnwantedFiles( - ImmutableSortedSet presentFiles, ImmutableSet wantedFiles) { - ImmutableSortedSet filesToDelete = - presentFiles.stream() - .filter(p -> !p.getFileName().toString().equals("lock") && !wantedFiles.contains(p)) - .collect(ImmutableSortedSet.toImmutableSortedSet(Ordering.natural())); - deleteFiles(filesToDelete); - } - - private void deleteFiles(ImmutableSortedSet filesToDelete) { + private void deleteFiles(ImmutableSortedSet toDelete) { Function toRootDirFn = - filesToDelete.size() <= RM_GROUPING_THRESHOLD + toDelete.size() <= RM_GROUPING_THRESHOLD ? path -> dataRoot : path -> dataRoot.resolve(path).getParent(); Function toFileFn = - filesToDelete.size() <= RM_GROUPING_THRESHOLD + toDelete.size() <= RM_GROUPING_THRESHOLD ? Path::toString : path -> path.getFileName().toString(); try { - filesToDelete.stream() + toDelete.stream() .collect(ImmutableListMultimap.toImmutableListMultimap(toRootDirFn, toFileFn)) .asMap() .forEach((dir, files) -> device.rmFiles(dir.toString(), files)); @@ -422,42 +597,20 @@ private void deleteFiles(ImmutableSortedSet filesToDelete) { } } - private void installFiles(String filesType, ImmutableMap filesToInstall) - throws Exception { - try (AutoCloseable ignored = device.createForward()) { - // Make sure all the directories exist. - for (Path parent : - filesToInstall.keySet().stream() - .map(p -> dataRoot.resolve(p).getParent()) - .distinct() - .collect(Collectors.toList())) { - device.mkDirP(parent.toString()); - } - // Plan the installation. - Map installPaths = - filesToInstall.entrySet().stream() - .collect( - Collectors.toMap( - entry -> dataRoot.resolve(entry.getKey()), - entry -> rootPath.resolve(entry.getValue()).getPath())); - // Install the files. - long pushStart = System.currentTimeMillis(); - device.installFiles(filesType, installPaths, packageName); - timings.recordPush(filesType, pushStart, System.currentTimeMillis()); - } - } - private void installMetadata(ImmutableMap metadataToInstall) throws Exception { try (Closer closer = Closer.create()) { - Map filesToInstall = new HashMap<>(); + ImmutableMap.Builder filesToInstall = ImmutableMap.builder(); for (Map.Entry entry : metadataToInstall.entrySet()) { NamedTemporaryFile temp = Objects.requireNonNull(closer.register(new NamedTemporaryFile("metadata", "tmp"))); com.google.common.io.Files.write( entry.getValue().getBytes(StandardCharsets.UTF_8), temp.get().toFile()); - filesToInstall.put(entry.getKey(), temp.get()); + filesToInstall.put( + dataRoot.resolve(entry.getKey()), rootPath.resolve(temp.get()).getPath()); } - installFiles("metadata", ImmutableMap.copyOf(filesToInstall)); + // Pushed as one shard, through the same path as every payload. It stays inside the closer: + // the temporary files it names are deleted when that closes. + pushShards(ImmutableList.of(new PushShard("metadata", filesToInstall.build()))); } } diff --git a/prelude/toolchains/android/src/com/facebook/buck/android/resources/BUCK b/prelude/toolchains/android/src/com/facebook/buck/android/resources/BUCK index cca2a2f..052bb7c 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/android/resources/BUCK +++ b/prelude/toolchains/android/src/com/facebook/buck/android/resources/BUCK @@ -51,7 +51,6 @@ buck_java_library( "prelude//toolchains/android/third-party:guava", "prelude//toolchains/android/third-party:infer-annotations", "prelude//toolchains/android/third-party:kotlin-annotations", - ":resources", ], ) @@ -100,7 +99,6 @@ buck_java_library( java_version = "11", visibility = ["PUBLIC"], deps = [ - "prelude//toolchains/android/src/com/facebook/buck/android/zipalign:zipalign", "prelude//toolchains/android/src/com/facebook/buck/core/filesystems:filesystems", "prelude//toolchains/android/third-party:args4j", "prelude//toolchains/android/third-party:guava", diff --git a/prelude/toolchains/android/src/com/facebook/buck/android/resources/ExoResourcesRewriter.java b/prelude/toolchains/android/src/com/facebook/buck/android/resources/ExoResourcesRewriter.java index a8dd9f0..b95754f 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/android/resources/ExoResourcesRewriter.java +++ b/prelude/toolchains/android/src/com/facebook/buck/android/resources/ExoResourcesRewriter.java @@ -44,7 +44,6 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.zip.CRC32; -import java.util.zip.Deflater; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; import javax.annotation.Nullable; @@ -141,7 +140,7 @@ static ReferenceMapper rewriteResources( // Write the full (rearranged) resources to the exo resources. try (ResourcesZipBuilder zipBuilder = new ResourcesZipBuilder(exoResources)) { for (ZipEntry entry : apkZip.getEntries()) { - addEntryOptimized(zipBuilder, apkZip, entry, getCompressionLevel()); + addEntryWithRawPassThrough(zipBuilder, apkZip, entry, COMPRESSION_LEVEL); } } // Then, slice out the resources needed for the primary apk. @@ -158,11 +157,11 @@ static ReferenceMapper rewriteResources( primaryResourceTable.serialize(), apkZip.getEntry("resources.arsc").getMethod() == ZipEntry.STORED ? 0 - : getCompressionLevel(), + : COMPRESSION_LEVEL, false); for (String path : closure.files.stream().sorted().collect(Collectors.toList())) { ZipEntry entry = apkZip.getEntry(path); - addEntryOptimized(zipBuilder, apkZip, entry, getCompressionLevel()); + addEntryWithRawPassThrough(zipBuilder, apkZip, entry, COMPRESSION_LEVEL); } } return resMapping; @@ -243,16 +242,9 @@ static void rewriteRDotTxt( } } - /** JDK's Deflater uses level 6 internally when DEFAULT_COMPRESSION (-1) is specified. */ - private static final int OPTIMIZED_COMPRESSION_LEVEL = 6; - - private static int getCompressionLevel() { - // DEFAULT_COMPRESSION (-1) is not accepted by CustomZipEntry.setCompressionLevel(), - // so we use the JDK's actual default level (6) for optimized builds. - return ResourceProcessingConfig.areOptimizationsEnabled() - ? OPTIMIZED_COMPRESSION_LEVEL - : Deflater.BEST_COMPRESSION; - } + // DEFAULT_COMPRESSION (-1) is not accepted by CustomZipEntry.setCompressionLevel(), so use the + // JDK's actual default level. + private static final int COMPRESSION_LEVEL = 6; private static void addEntry( ResourcesZipBuilder zipBuilder, @@ -274,15 +266,10 @@ private static void addEntry( isDirectory); } - /** - * Adds a zip entry, using raw pass-through for unmodified DEFLATED entries when optimizations are - * enabled, or falling back to decompress/recompress otherwise. - */ - private static void addEntryOptimized( + private static void addEntryWithRawPassThrough( ResourcesZipBuilder zipBuilder, ApkZip apkZip, ZipEntry entry, int compressionLevel) throws IOException { - if (ResourceProcessingConfig.areOptimizationsEnabled() - && !apkZip.isModifiedEntry(entry.getName()) + if (!apkZip.isModifiedEntry(entry.getName()) && entry.getMethod() == ZipEntry.DEFLATED && entry.getCompressedSize() >= 0) { byte[] rawBytes = apkZip.readRawCompressedBytes(entry); @@ -307,7 +294,6 @@ private static class ApkZip implements Closeable, UsedResourcesFinder.ApkContent private final ZipFile zipFile; private final AbsPath inputPath; private final SortedMap entries; - private final Map entryContents; private final Map xmlEntries; private final Supplier resourceTable; private @Nullable RandomAccessFile randomAccessFile; @@ -320,20 +306,9 @@ public ApkZip(AbsPath inputPath) throws IOException { .collect( ImmutableSortedMap.toImmutableSortedMap( Ordering.natural(), ZipEntry::getName, e -> e)); - this.entryContents = new HashMap<>(); this.xmlEntries = new HashMap<>(); - if (ResourceProcessingConfig.areOptimizationsEnabled()) { - // Read raw bytes eagerly to avoid circular dependency with getContent(), which - // calls resourceTable.get().serialize() for resources.arsc in the optimized path. - byte[] arscBytes = readEntryBytes("resources.arsc"); - entryContents.put("resources.arsc", arscBytes); - this.resourceTable = - MoreSuppliers.memoize(() -> ResourceTable.get(ResChunk.wrap(arscBytes))); - } else { - this.resourceTable = - MoreSuppliers.memoize( - () -> ResourceTable.get(ResChunk.wrap(getContent("resources.arsc")))); - } + byte[] arscBytes = readEntryBytes("resources.arsc"); + this.resourceTable = MoreSuppliers.memoize(() -> ResourceTable.get(ResChunk.wrap(arscBytes))); } @Override @@ -379,27 +354,15 @@ Iterable getResourcesXmls() { .collect(ImmutableList.toImmutableList()); } - /** - * Gets the content for an entry. When optimizations are enabled, transformed entries (arsc, - * XML) are serialized from in-memory objects and pass-through entries are read directly from - * the zip. When disabled, all entries are cached in memory (original behavior). - */ byte[] getContent(String path) { - if (ResourceProcessingConfig.areOptimizationsEnabled()) { - // In the optimized path, serialize() is called on each invocation rather than caching, - // because getContent() is only called once per entry during zip writing. Avoiding the - // cache reduces peak memory by not holding all entry bytes simultaneously. - if (path.equals("resources.arsc")) { - return resourceTable.get().serialize(); - } - ResourcesXml xml = xmlEntries.get(path); - if (xml != null) { - return xml.serialize(); - } - return readEntryBytes(path); - } else { - return entryContents.computeIfAbsent(path, this::readEntryBytes); + if (path.equals("resources.arsc")) { + return resourceTable.get().serialize(); + } + ResourcesXml xml = xmlEntries.get(path); + if (xml != null) { + return xml.serialize(); } + return readEntryBytes(path); } private byte[] readEntryBytes(String path) { @@ -414,15 +377,7 @@ private byte[] readEntryBytes(String path) { private ResourcesXml extractXml(String path) { try { - // In the optimized path, skip the entryContents cache because the raw bytes are only - // needed here to construct the ResourcesXml object. The XML is then accessed via - // xmlEntries, and getContent() serializes from the in-memory object. Caching the raw - // bytes would waste memory since they're not reused. - byte[] bytes = - ResourceProcessingConfig.areOptimizationsEnabled() - ? readEntryBytes(path) - : entryContents.computeIfAbsent(path, this::readEntryBytes); - return ResourcesXml.get(ResChunk.wrap(bytes)); + return ResourcesXml.get(ResChunk.wrap(readEntryBytes(path))); } catch (Exception e) { throw new RuntimeException("When extracting " + path, e); } diff --git a/prelude/toolchains/android/src/com/facebook/buck/android/resources/ExoResourcesRewriterExecutableMain.java b/prelude/toolchains/android/src/com/facebook/buck/android/resources/ExoResourcesRewriterExecutableMain.java index 3c59b03..3eed96e 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/android/resources/ExoResourcesRewriterExecutableMain.java +++ b/prelude/toolchains/android/src/com/facebook/buck/android/resources/ExoResourcesRewriterExecutableMain.java @@ -10,14 +10,12 @@ package com.facebook.buck.android.resources; -import com.facebook.buck.android.zipalign.ZipAlign; import com.facebook.buck.core.filesystems.AbsPath; import com.facebook.buck.core.filesystems.RelPath; import com.facebook.infer.annotation.Nullsafe; import com.google.common.hash.Hashing; import java.io.IOException; import java.nio.file.Files; -import java.nio.file.Path; import java.nio.file.Paths; import org.kohsuke.args4j.CmdLineException; import org.kohsuke.args4j.CmdLineParser; @@ -44,18 +42,11 @@ public class ExoResourcesRewriterExecutableMain { @Option(name = "--exo-resources-hash", required = true) private String exoResourcesHash; - @Option(name = "--zipalign-tool", required = true) - private String zipalignTool; - - @Option(name = "--optimized-processing", usage = "enable optimized resource processing") - private boolean optimizedProcessing = false; - public static void main(String[] args) throws IOException { ExoResourcesRewriterExecutableMain main = new ExoResourcesRewriterExecutableMain(); CmdLineParser parser = new CmdLineParser(main); try { parser.parseArgument(args); - ResourceProcessingConfig.setOptimizationsEnabled(main.optimizedProcessing); main.run(); System.exit(0); } catch (CmdLineException e) { @@ -68,38 +59,13 @@ public static void main(String[] args) throws IOException { private void run() throws IOException { AbsPath root = AbsPath.of(Paths.get(".").normalize().toAbsolutePath()); - if (ResourceProcessingConfig.areOptimizationsEnabled()) { - // Optimized path: write directly to final output, skip zipalign. - // Exo resources are loaded by the app for exopackage — they don't need - // Play Store alignment. All DEFLATED entries are unaffected by zipalign anyway. - ExoResourcesRewriter.rewrite( - root, - RelPath.get(originalPrimaryApkResources), - RelPath.get(originalRDotTxt), - RelPath.get(newPrimaryApkResources), - Paths.get(exoResources), - RelPath.get(newRDotTxt)); - } else { - Path unalignedExoResources = Files.createTempFile("unalignedExoResources", "apk"); - - ExoResourcesRewriter.rewrite( - root, - RelPath.get(originalPrimaryApkResources), - RelPath.get(originalRDotTxt), - RelPath.get(newPrimaryApkResources), - unalignedExoResources, - RelPath.get(newRDotTxt)); - - try { - ZipAlign zipAlign = - new ZipAlign(zipalignTool, unalignedExoResources.toString(), exoResources.toString()); - zipAlign.run(); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } finally { - Files.deleteIfExists(unalignedExoResources); - } - } + ExoResourcesRewriter.rewrite( + root, + RelPath.get(originalPrimaryApkResources), + RelPath.get(originalRDotTxt), + RelPath.get(newPrimaryApkResources), + Paths.get(exoResources), + RelPath.get(newRDotTxt)); Files.writeString( Paths.get(exoResourcesHash), diff --git a/prelude/toolchains/android/src/com/facebook/buck/android/resources/MergeAndroidResources.java b/prelude/toolchains/android/src/com/facebook/buck/android/resources/MergeAndroidResources.java index 017e081..52a4d8e 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/android/resources/MergeAndroidResources.java +++ b/prelude/toolchains/android/src/com/facebook/buck/android/resources/MergeAndroidResources.java @@ -34,9 +34,7 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; -import java.util.ArrayList; import java.util.Collection; -import java.util.Comparator; import java.util.EnumSet; import java.util.HashMap; import java.util.Iterator; @@ -65,7 +63,6 @@ public static void mergeAndroidResources( boolean forceFinalResourceIds, EnumSet bannedDuplicateResourceTypes, Optional duplicateResourceWhitelistPath, - Optional unionPackage, ImmutableList overrideSymbolsPath, Path outputDir, Optional stringsOutputDir, @@ -122,26 +119,8 @@ public static void mergeAndroidResources( ImmutableSet.Builder requiredPackages = ImmutableSet.builder(); - // Create a temporary list as the multimap - // will be concurrently modified below. - ArrayList> entries = - new ArrayList<>(rDotJavaPackageToResources.entries()); - requiredPackages.addAll(symbolsFileToRDotJavaPackage.values()); - // If a resource_union_package was specified, copy all resource into that package, - // unless they are already present. - if (unionPackage.isPresent()) { - String unionPackageName = unionPackage.get(); - requiredPackages.add(unionPackageName); - - for (Entry entry : entries) { - if (!rDotJavaPackageToResources.containsEntry(unionPackageName, entry.getValue())) { - rDotJavaPackageToResources.put(unionPackageName, entry.getValue()); - } - } - } - Preconditions.checkState( stringsOutputDir.isPresent() == idsOutputDir.isPresent(), "Expect either both stringOutputDir and idsOutputDir, or neither"); @@ -240,23 +219,6 @@ private static void writeEmptyRDotJavaForPackages(Path outputDir, Set rD } } - private static void writePackagePrivateArrayHolderClass( - Path outputFile, - String packageName, - String className, - String fieldName, - ImmutableList arrayContents) - throws IOException { - try (ThrowingPrintWriter writer = - new ThrowingPrintWriter(new FileOutputStream(outputFile.toFile()))) { - writer.format("package %s;\n\n", packageName); - writer.format("final class %s {\n", className); - writer.format(" static final int[] %s = ", fieldName); - writer.format("{ %s };\n", Joiner.on(",").join(arrayContents)); - writer.println("}"); - } - } - private static void writePerPackageRDotJava( Path outputDir, SortedSetMultimap packageToResources, @@ -264,7 +226,6 @@ private static void writePerPackageRDotJava( ImmutableSet referencedResources) throws IOException { Files.createDirectories(outputDir); - ImmutableList.Builder allGrayscaleImagesBuilder = ImmutableList.builder(); for (String rDotJavaPackage : packageToResources.keySet()) { Path outputFile = getPathToRDotJava(outputDir, rDotJavaPackage); Files.createDirectories(Objects.requireNonNull(outputFile.getParent())); @@ -273,15 +234,14 @@ private static void writePerPackageRDotJava( writer.format("package %s;\n\n", rDotJavaPackage); writer.write("public class R {\n"); - ImmutableList.Builder grayscaleImagesBuilder = ImmutableList.builder(); RType lastType = null; for (RDotTxtEntry res : packageToResources.get(rDotJavaPackage)) { boolean isUsed = referencedResources.isEmpty() || referencedResources.contains(rDotJavaPackage + "." + res.name); - RType type = res.type; if (isUsed) { + RType type = res.type; if (!Objects.equals(type, lastType)) { // If the previous type needs to be closed, close it. if (lastType != null) { @@ -299,11 +259,6 @@ private static void writePerPackageRDotJava( " public static%s%s %s=%s;\n", forceFinalResourceIds ? " final " : " ", res.idType, res.name, res.idValue); } - - if (type == RType.DRAWABLE - && res.customType == RDotTxtEntry.CustomDrawableType.GRAYSCALE_IMAGE) { - grayscaleImagesBuilder.add(Integer.decode(res.idValue)); - } } // If some type was written (e.g., the for loop was entered), then the last type needs to be @@ -312,86 +267,10 @@ private static void writePerPackageRDotJava( writer.println(" }\n"); } - ImmutableList grayscaleImages = grayscaleImagesBuilder.build(); - if (grayscaleImages.size() > 0) { - Path grayscaleDrawablesAuxFile = - getPathToJavaFile(outputDir, rDotJavaPackage, "RGrayscale.java"); - writePackagePrivateArrayHolderClass( - grayscaleDrawablesAuxFile, - rDotJavaPackage, - "RGrayscale", - "grayscale_images", - grayscaleImages); - // Add a new field for the grayscale drawables. - writer.format( - " public static final int[] grayscale_images = %s.RGrayscale.grayscale_images;", - rDotJavaPackage); - writer.format("\n"); - } - - allGrayscaleImagesBuilder.addAll(grayscaleImages); - // Close the class definition. writer.println("}"); } } - - ImmutableList allGrayscaleImages = allGrayscaleImagesBuilder.build(); - if (!allGrayscaleImages.isEmpty()) { - String drawablesPackage = "com.facebook.buck.android.drawables"; - String drawablesPackageReplaced = drawablesPackage.replace('.', '/'); - - // To lessen the likelihood of generating too large static initializer, - // generate 1 class per array. Further class splitting into chunks holding - // subarrays could be done if problems continue. Please note that int[] - // array construction may have ramifications on dead resource elimination. - String arrayHolderClassGrayscale = "GrayscaleDrawablesAux"; - Path grayscaleOutputFile = - outputDir - .resolve(drawablesPackageReplaced) - .resolve(String.format("%s.java", arrayHolderClassGrayscale)); - Files.createDirectories(Objects.requireNonNull(grayscaleOutputFile.getParent())); - writePackagePrivateArrayHolderClass( - grayscaleOutputFile, - drawablesPackage, - arrayHolderClassGrayscale, - "grayscaleDrawables", - ImmutableList.sortedCopyOf(Comparator.naturalOrder(), allGrayscaleImages)); - - String drawablesClass = "CustomDrawables"; - Path outputFile = - outputDir - .resolve(drawablesPackageReplaced) - .resolve(String.format("%s.java", drawablesClass)); - try (ThrowingPrintWriter writer = - new ThrowingPrintWriter(new FileOutputStream(outputFile.toFile()))) { - writer.format("package %s;\n\n", drawablesPackage); - writer.format("import java.util.Arrays;\n\n"); - writer.format("public final class %s {\n", drawablesClass); - - writer.format(" private %s() {\n", drawablesClass); - writer.format(" throw new RuntimeException();\n"); - writer.format(" }\n"); - writer.format("\n"); - - writer.format( - " private static final int[] grayscaleDrawables = %s.%s.grayscaleDrawables;", - drawablesPackage, arrayHolderClassGrayscale); - writer.format("\n"); - - writer.format(" public static boolean isGrayscaleDrawable(int resourceId) {\n"); - writer.format(" return Arrays.binarySearch(grayscaleDrawables, resourceId) >= 0;\n"); - writer.format(" }\n"); - writer.format("\n"); - - writer.format(" public static int[] getGrayscaleDrawableIds() {\n"); - writer.format(" return grayscaleDrawables;\n"); - writer.format(" }\n"); - writer.format("\n"); - - writer.println("}"); - } - } } @VisibleForTesting diff --git a/prelude/toolchains/android/src/com/facebook/buck/android/resources/MergeAndroidResourcesExecutableMain.java b/prelude/toolchains/android/src/com/facebook/buck/android/resources/MergeAndroidResourcesExecutableMain.java index 84c5744..b99d268 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/android/resources/MergeAndroidResourcesExecutableMain.java +++ b/prelude/toolchains/android/src/com/facebook/buck/android/resources/MergeAndroidResourcesExecutableMain.java @@ -69,21 +69,14 @@ public class MergeAndroidResourcesExecutableMain { @Option(name = "--duplicate-resource-allowlist-path") private @Nullable String duplicateResourceAllowlist = null; - @Option(name = "--union-package") - private @Nullable String unionPackageString = null; - @Option(name = "--referenced-resources-lists") private @Nullable String referencedResourcesLists = null; - @Option(name = "--optimized-processing", usage = "enable optimized resource processing") - private boolean optimizedProcessing = false; - public static void main(String[] args) throws IOException { MergeAndroidResourcesExecutableMain main = new MergeAndroidResourcesExecutableMain(); CmdLineParser parser = new CmdLineParser(main); try { parser.parseArgument(args); - ResourceProcessingConfig.setOptimizationsEnabled(main.optimizedProcessing); main.run(); System.exit(0); } catch (CmdLineException e) { @@ -128,8 +121,6 @@ private void run() throws IOException { : ImmutableList.of(); Optional duplicateResourceAllowlistPath = Optional.ofNullable(duplicateResourceAllowlist).map(Paths::get); - Optional unionPackage = Optional.ofNullable(unionPackageString); - Path outputDir = Paths.get(outputDirString); Optional stringsOutputDirPath = Optional.ofNullable(stringsOutputDirString).map(Paths::get); @@ -151,7 +142,6 @@ private void run() throws IOException { forceFinalResourceIds, bannedDuplicateResourceTypes, duplicateResourceAllowlistPath, - unionPackage, overrideSymbols, outputDir, stringsOutputDirPath, diff --git a/prelude/toolchains/android/src/com/facebook/buck/android/resources/MergeAssetsExecutableMain.java b/prelude/toolchains/android/src/com/facebook/buck/android/resources/MergeAssetsExecutableMain.java index 5743337..813cc05 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/android/resources/MergeAssetsExecutableMain.java +++ b/prelude/toolchains/android/src/com/facebook/buck/android/resources/MergeAssetsExecutableMain.java @@ -68,15 +68,11 @@ public class MergeAssetsExecutableMain { @Option(name = "--binary-type", usage = "either 'apk' or 'aab'") private @Nullable String binaryType = null; - @Option(name = "--optimized-processing", usage = "enable optimized resource processing") - private boolean optimizedProcessing = false; - public static void main(String[] args) throws IOException { MergeAssetsExecutableMain main = new MergeAssetsExecutableMain(); CmdLineParser parser = new CmdLineParser(main); try { parser.parseArgument(args); - ResourceProcessingConfig.setOptimizationsEnabled(main.optimizedProcessing); main.run(); System.exit(0); } catch (CmdLineException e) { diff --git a/prelude/toolchains/android/src/com/facebook/buck/android/resources/MergeAssetsUtils.java b/prelude/toolchains/android/src/com/facebook/buck/android/resources/MergeAssetsUtils.java index bbf9f79..ae3a4d4 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/android/resources/MergeAssetsUtils.java +++ b/prelude/toolchains/android/src/com/facebook/buck/android/resources/MergeAssetsUtils.java @@ -18,9 +18,6 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import com.google.common.hash.HashCode; -import com.google.common.hash.Hashing; -import com.google.common.io.ByteSource; import com.google.common.io.Files; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -242,34 +239,17 @@ private static void addAllAssets( isNoCompress(packagingPathForAsset.toString(), allNoCompressExtensions, noCompressPattern) ? 0 : Deflater.BEST_COMPRESSION; - if (ResourceProcessingConfig.areOptimizationsEnabled()) { - byte[] assetBytes = java.nio.file.Files.readAllBytes(fullPathToAsset); - CRC32 crc32 = new CRC32(); - crc32.update(assetBytes); - try (InputStream assetStream = new ByteArrayInputStream(assetBytes)) { - output.addEntry( - assetStream, - assetBytes.length, - crc32.getValue(), - assetsZipRoot.resolve(packagingPathForAsset).toString(), - compression, - false); - } - } else { - ByteSource assetSource = Files.asByteSource(fullPathToAsset.toFile()); - HashCode assetCrc32 = assetSource.hash(Hashing.crc32()); - try (InputStream assetStream = assetSource.openStream()) { - output.addEntry( - assetStream, - assetSource.size(), - // CRC32s are only 32 bits, but setCrc() takes a - // long. Avoid sign-extension here during the - // conversion to long by masking off the high 32 bits. - assetCrc32.asInt() & 0xFFFFFFFFL, - assetsZipRoot.resolve(packagingPathForAsset).toString(), - compression, - false); - } + byte[] assetBytes = java.nio.file.Files.readAllBytes(fullPathToAsset); + CRC32 crc32 = new CRC32(); + crc32.update(assetBytes); + try (InputStream assetStream = new ByteArrayInputStream(assetBytes)) { + output.addEntry( + assetStream, + assetBytes.length, + crc32.getValue(), + assetsZipRoot.resolve(packagingPathForAsset).toString(), + compression, + false); } } } diff --git a/prelude/toolchains/android/src/com/facebook/buck/android/resources/ResTablePackage.java b/prelude/toolchains/android/src/com/facebook/buck/android/resources/ResTablePackage.java index 7cb8e41..9ee289d 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/android/resources/ResTablePackage.java +++ b/prelude/toolchains/android/src/com/facebook/buck/android/resources/ResTablePackage.java @@ -25,7 +25,6 @@ import java.util.Map; import java.util.function.Supplier; import java.util.stream.IntStream; -import javax.annotation.Nullable; /** * A Package consists of a header: ResTable_header u16 chunk_type u16 header_size u32 chunk_size u32 @@ -48,7 +47,7 @@ public class ResTablePackage extends ResChunk { private final StringPool types; private final StringPool keys; private final List typeSpecs; - private final @Nullable ResTableTypeSpec[] typeSpecByType; + private final ResTableTypeSpec[] typeSpecByType; public void reassignIds(ReferenceMapper refMapping) { for (ResTableTypeSpec spec : typeSpecs) { @@ -183,18 +182,13 @@ public ResTablePackage( this.keys = keys; this.typeSpecs = typeSpecs; - // Build O(1) lookup array indexed by type id (only when optimizations are enabled). - if (ResourceProcessingConfig.areOptimizationsEnabled()) { - int maxType = 0; - for (ResTableTypeSpec spec : typeSpecs) { - maxType = Math.max(maxType, spec.getResourceType()); - } - this.typeSpecByType = new ResTableTypeSpec[maxType + 1]; - for (ResTableTypeSpec spec : typeSpecs) { - typeSpecByType[spec.getResourceType()] = spec; - } - } else { - this.typeSpecByType = null; + int maxType = 0; + for (ResTableTypeSpec spec : typeSpecs) { + maxType = Math.max(maxType, spec.getResourceType()); + } + this.typeSpecByType = new ResTableTypeSpec[maxType + 1]; + for (ResTableTypeSpec spec : typeSpecs) { + typeSpecByType[spec.getResourceType()] = spec; } name = @@ -228,18 +222,10 @@ public StringPool getKeys() { } public ResTableTypeSpec getTypeSpec(int type) { - if (typeSpecByType != null) { - if (type >= 0 && type < typeSpecByType.length) { - ResTableTypeSpec spec = typeSpecByType[type]; - if (spec != null) { - return spec; - } - } - } else { - for (ResTableTypeSpec spec : typeSpecs) { - if (spec.getResourceType() == type) { - return spec; - } + if (type >= 0 && type < typeSpecByType.length) { + ResTableTypeSpec spec = typeSpecByType[type]; + if (spec != null) { + return spec; } } throw new IllegalArgumentException(); diff --git a/prelude/toolchains/android/src/com/facebook/buck/android/resources/ResTableType.java b/prelude/toolchains/android/src/com/facebook/buck/android/resources/ResTableType.java index 1434118..5d61fc8 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/android/resources/ResTableType.java +++ b/prelude/toolchains/android/src/com/facebook/buck/android/resources/ResTableType.java @@ -363,14 +363,6 @@ public void transformKeyReferences(RefTransformer visitor) { } public void visitKeyReferences(RefVisitor visitor) { - if (!ResourceProcessingConfig.areOptimizationsEnabled()) { - transformKeyReferences( - i -> { - visitor.visit(i); - return i; - }); - return; - } for (int i = 0; i < entryCount; i++) { int offset = getEntryValueOffset(i); if (offset != -1) { @@ -431,14 +423,6 @@ public void transformStringReferences(int[] idsToVisit, RefTransformer visitor) } public void visitStringReferences(RefVisitor visitor) { - if (!ResourceProcessingConfig.areOptimizationsEnabled()) { - transformStringReferences( - i -> { - visitor.visit(i); - return i; - }); - return; - } for (int i = 0; i < entryCount; i++) { int offset = getEntryValueOffset(i); if (offset != -1) { @@ -448,15 +432,6 @@ public void visitStringReferences(RefVisitor visitor) { } public void visitStringReferences(int[] idsToVisit, RefVisitor visitor) { - if (!ResourceProcessingConfig.areOptimizationsEnabled()) { - transformStringReferences( - idsToVisit, - i -> { - visitor.visit(i); - return i; - }); - return; - } for (int i : idsToVisit) { int offset = getEntryValueOffset(i); if (offset != -1) { @@ -525,36 +500,19 @@ public int compareTo(AttrRef other) { } } - if (ResourceProcessingConfig.areOptimizationsEnabled()) { - AttrRef[] refs = new AttrRef[attrCount]; - int entryOffset = attrStart; - for (int j = 0; j < attrCount; j++) { - refs[j] = new AttrRef(entryOffset); - entryOffset += refs[j].size; - } - Arrays.sort(refs); - byte[] newData = new byte[entryOffset - attrStart]; - ByteBuffer newBuf = wrap(newData); - for (AttrRef ref : refs) { - newBuf.put(slice(entryData, ref.offset, ref.size)); - } - slice(entryData, attrStart).put(newData); - } else { - java.util.stream.Stream.Builder builder = java.util.stream.Stream.builder(); - int entryOffset = attrStart; - for (int j = 0; j < attrCount; j++) { - AttrRef ref = new AttrRef(entryOffset); - builder.add(ref); - entryOffset += ref.size; - } - byte[] newData = new byte[entryOffset - attrStart]; - ByteBuffer newBuf = wrap(newData); - builder - .build() - .sorted() - .forEachOrdered(ref -> newBuf.put(slice(entryData, ref.offset, ref.size))); - slice(entryData, attrStart).put(newData); + AttrRef[] refs = new AttrRef[attrCount]; + int entryOffset = attrStart; + for (int j = 0; j < attrCount; j++) { + refs[j] = new AttrRef(entryOffset); + entryOffset += refs[j].size; } + Arrays.sort(refs); + byte[] newData = new byte[entryOffset - attrStart]; + ByteBuffer newBuf = wrap(newData); + for (AttrRef ref : refs) { + newBuf.put(slice(entryData, ref.offset, ref.size)); + } + slice(entryData, attrStart).put(newData); } public void transformReferences(RefTransformer visitor) { @@ -576,15 +534,6 @@ public void transformReferences(int[] ids, RefTransformer visitor) { } public void visitReferences(int[] ids, RefVisitor visitor) { - if (!ResourceProcessingConfig.areOptimizationsEnabled()) { - transformReferences( - ids, - i -> { - visitor.visit(i); - return i; - }); - return; - } for (int i : ids) { int offset = getEntryValueOffset(i); if (offset != -1) { diff --git a/prelude/toolchains/android/src/com/facebook/buck/android/resources/ResourceProcessingConfig.java b/prelude/toolchains/android/src/com/facebook/buck/android/resources/ResourceProcessingConfig.java deleted file mode 100644 index d50e45e..0000000 --- a/prelude/toolchains/android/src/com/facebook/buck/android/resources/ResourceProcessingConfig.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) Meta Platforms, Inc. and affiliates. - * - * This source code is dual-licensed under either the MIT license found in the - * LICENSE-MIT file in the root directory of this source tree or the Apache - * License, Version 2.0 found in the LICENSE-APACHE file in the root directory - * of this source tree. You may select, at your option, one of the - * above-listed licenses. - */ - -package com.facebook.buck.android.resources; - -import com.facebook.infer.annotation.Nullsafe; - -/** - * Controls whether optimized code paths are used in the Android resource processing pipeline. - * - *

When disabled (the default), original code paths are used. When enabled, optimized code paths - * are used for improved performance. This allows A/B testing of optimizations via: - * - * 

buck build ig4a -c android.optimized_resource_processing=true
- */ -@Nullsafe(Nullsafe.Mode.LOCAL) -public class ResourceProcessingConfig { - private static volatile boolean optimizationsEnabled = false; - - private ResourceProcessingConfig() {} - - public static void setOptimizationsEnabled(boolean enabled) { - optimizationsEnabled = enabled; - } - - public static boolean areOptimizationsEnabled() { - return optimizationsEnabled; - } -} diff --git a/prelude/toolchains/android/src/com/facebook/buck/android/resources/ResourcesXml.java b/prelude/toolchains/android/src/com/facebook/buck/android/resources/ResourcesXml.java index 076e88e..e07b9c7 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/android/resources/ResourcesXml.java +++ b/prelude/toolchains/android/src/com/facebook/buck/android/resources/ResourcesXml.java @@ -48,9 +48,7 @@ */ @Nullsafe(Nullsafe.Mode.LOCAL) public class ResourcesXml extends ResChunk { - private static boolean debugEnabled() { - return !ResourceProcessingConfig.areOptimizationsEnabled(); - } + private static final boolean DEBUG = false; public static final int HEADER_SIZE = 8; @@ -127,7 +125,7 @@ public void transformReferences(RefTransformer visitor) { int extOffset = offset + nodeHeaderSize; int attrStart = extOffset + nodeBuf.getShort(extOffset + 8); Preconditions.checkState(attrStart == extOffset + 20); - if (debugEnabled()) { + if (DEBUG) { int attrSize = nodeBuf.getShort(extOffset + 10); Preconditions.checkState(attrSize == ATTRIBUTE_SIZE); } @@ -212,29 +210,17 @@ public int compareTo(AttrRef other) { } } - if (ResourceProcessingConfig.areOptimizationsEnabled()) { - AttrRef[] refs = new AttrRef[attrCount]; - for (int i = 0; i < attrCount; i++) { - refs[i] = new AttrRef(attrStart + ATTRIBUTE_SIZE * i); - } - Arrays.sort(refs); - byte[] newData = new byte[ATTRIBUTE_SIZE * attrCount]; - ByteBuffer newBuf = wrap(newData); - for (AttrRef ref : refs) { - newBuf.put(slice(nodeBuf, ref.offset, ATTRIBUTE_SIZE)); - } - slice(nodeBuf, attrStart).put(newData); - } else { - byte[] newData = new byte[ATTRIBUTE_SIZE * attrCount]; - ByteBuffer newBuf = wrap(newData); - int finalAttrStart = attrStart; - int finalAttrCount = attrCount; - java.util.stream.IntStream.range(0, finalAttrCount) - .mapToObj(i -> new AttrRef(finalAttrStart + ATTRIBUTE_SIZE * i)) - .sorted() - .forEachOrdered(ref -> newBuf.put(slice(nodeBuf, ref.offset, ATTRIBUTE_SIZE))); - slice(nodeBuf, attrStart).put(newData); + AttrRef[] refs = new AttrRef[attrCount]; + for (int i = 0; i < attrCount; i++) { + refs[i] = new AttrRef(attrStart + ATTRIBUTE_SIZE * i); + } + Arrays.sort(refs); + byte[] newData = new byte[ATTRIBUTE_SIZE * attrCount]; + ByteBuffer newBuf = wrap(newData); + for (AttrRef ref : refs) { + newBuf.put(slice(nodeBuf, ref.offset, ATTRIBUTE_SIZE)); } + slice(nodeBuf, attrStart).put(newData); } public void visitReferences(RefVisitor visitor) { diff --git a/prelude/toolchains/android/src/com/facebook/buck/android/resources/UsedResourcesFinder.java b/prelude/toolchains/android/src/com/facebook/buck/android/resources/UsedResourcesFinder.java index 156f0a6..49f8d7f 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/android/resources/UsedResourcesFinder.java +++ b/prelude/toolchains/android/src/com/facebook/buck/android/resources/UsedResourcesFinder.java @@ -111,15 +111,8 @@ void addIdToProcess(int id) { int k = id & 0xFFFF; Set processedIdsForType = Objects.requireNonNull(processedIds.computeIfAbsent(type, v -> new TreeSet<>())); - if (ResourceProcessingConfig.areOptimizationsEnabled()) { - if (processedIdsForType.add(k)) { - Objects.requireNonNull(idsToProcess.computeIfAbsent(type, v -> new TreeSet<>())).add(k); - } - } else { - if (!processedIdsForType.contains(k)) { - processedIdsForType.add(k); - Objects.requireNonNull(idsToProcess.computeIfAbsent(type, v -> new TreeSet<>())).add(k); - } + if (processedIdsForType.add(k)) { + Objects.requireNonNull(idsToProcess.computeIfAbsent(type, v -> new TreeSet<>())).add(k); } } } diff --git a/prelude/toolchains/android/src/com/facebook/buck/cd/resources/proto/common.proto b/prelude/toolchains/android/src/com/facebook/buck/cd/resources/proto/common.proto index 50de778..729de50 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/cd/resources/proto/common.proto +++ b/prelude/toolchains/android/src/com/facebook/buck/cd/resources/proto/common.proto @@ -39,7 +39,8 @@ message PostBuildParams { string abiOutputDir = 5; repeated string usedClasses = 6; string depFile = 7; - string jarToJarDirMap = 8; + reserved 8; + reserved "jarToJarDirMap"; repeated string optionalDirs = 9; string incrementalStateDir = 10; bool shouldCreateClassAbi = 11; diff --git a/prelude/toolchains/android/src/com/facebook/buck/cd/resources/proto/javacd.proto b/prelude/toolchains/android/src/com/facebook/buck/cd/resources/proto/javacd.proto index 6615f7d..964c5c2 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/cd/resources/proto/javacd.proto +++ b/prelude/toolchains/android/src/com/facebook/buck/cd/resources/proto/javacd.proto @@ -51,11 +51,13 @@ enum BuildMode { // Common parameters for build jar commands (abi and library jar ones) message BaseJarCommand { - AbiGenerationMode abiCompatibilityMode = 1; + reserved 1; + reserved "abiCompatibilityMode"; AbiGenerationMode abiGenerationMode = 2; bool isRequiredForSourceOnlyAbi = 3; bool trackClassUsage = 4; - string configuredBuckOut = 6; + reserved 6; + reserved "configuredBuckOut"; BuildTargetValue buildTargetValue = 7; OutputPathsValue outputPathsValue = 8; repeated string compileTimeClasspathPaths = 9; @@ -63,11 +65,18 @@ message BaseJarCommand { repeated RelPathMapEntry resourcesMap = 13; JarParameters jarParameters = 15; string buildCellRootPath = 16; - ResolvedJavac resolvedJavac = 17; + reserved 17; + reserved "resolvedJavac"; ResolvedJavacOptions resolvedJavacOptions = 18; repeated string compileTimeClasspathSnapshotPaths = 19; - string pathToClasses = 20; + reserved 20; + reserved "pathToClasses"; + // Optional. When omitted, generated sources stay in BUCK_SCRATCH_PATH. string annotationsPath = 23; + // Alternating ABI jar and directory paths, in classpath order. An empty + // directory keeps jar-level dep-file tracking. Mutually exclusive with + // compileTimeClasspathPaths. + repeated string compileTimeClasspathAbiAndDirPaths = 24; } enum AbiGenerationMode { @@ -107,9 +116,11 @@ message OutputPathsValue { string libraryTargetFullyQualifiedName = 4; message OutputPaths { - string classesDir = 1; + reserved 1; + reserved "classesDir"; string outputJarDirPath = 2; string abiJarPath = 3; + // Optional. When omitted, generated sources stay in BUCK_SCRATCH_PATH. string annotationPath = 4; string pathToSourcesList = 5; string workingDirectory = 6; @@ -121,7 +132,8 @@ message ResolvedJavacOptions { // reserved = 1; repeated string bootclasspathList = 2; JavacLanguageLevelOptions languageLevelOptions = 3; - bool debug = 4; + reserved 4; + reserved "debug"; bool verbose = 5; JavacPluginParams javaAnnotationProcessorParams = 6; JavacPluginParams standardJavacPluginParams = 7; @@ -150,8 +162,8 @@ message ResolvedJavacOptions { bool runsOnJavaOnly = 4; repeated string processorNames = 5; repeated string classpath = 6; - // Appears to only be used for the internal NullSafe. - map pathParams = 7; + reserved 7; + reserved "pathParams"; repeated string arguments = 8; } } @@ -165,36 +177,10 @@ message JarParameters { repeated string overrideEntriesToJar = 6; string mainClass = 7; string manifestFile = 8; - LogLevel duplicatesLogLevel = 9; - - enum LogLevel { - UNKNOWN = 0; - OFF = 1; - SEVERE = 2; - WARNING = 3; - INFO = 4; - CONFIG = 5; - FINE = 6; - FINER = 7; - FINEST = 8; - ALL = 9; - } + reserved 9; + reserved "duplicatesLogLevel"; message RemoveClassesPatternsMatcher { repeated string patterns = 1; } } - -message ResolvedJavac { - oneof javac { - ExternalJavac externalJavac = 1; - JSR199Javac jsr199Javac = 2; - } - - message ExternalJavac { - string shortName = 1; - repeated string commandPrefix = 2; - } - - message JSR199Javac {} -} diff --git a/prelude/toolchains/android/src/com/facebook/buck/cd/resources/proto/kotlincd.proto b/prelude/toolchains/android/src/com/facebook/buck/cd/resources/proto/kotlincd.proto index 017b56a..a9023a4 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/cd/resources/proto/kotlincd.proto +++ b/prelude/toolchains/android/src/com/facebook/buck/cd/resources/proto/kotlincd.proto @@ -35,17 +35,18 @@ message KotlinExtraParams { repeated string extraClassPaths = 2; string standardLibraryClassPath = 3; string annotationProcessingClassPath = 4; - AnnotationProcessingTool annotationProcessingTool = 5; + reserved 5; repeated string extraKotlincArguments = 6; // kotlinCompilerPlugin keys are strings encoded as strings due to limitations // in proto3. map kotlinCompilerPlugins = 7; map kosabiPluginOptions = 8; - string kosabiJvmAbiGenEarlyTerminationMessagePrefix = 9; + reserved 9; + reserved "kosabiJvmAbiGenEarlyTerminationMessagePrefix"; repeated string friendPaths = 10; repeated string kotlinHomeLibraries = 11; string jvmTarget = 12; - bool shouldGenerateAnnotationProcessingStats = 13; + reserved 13; bool shouldUseJvmAbiGen = 14; string jvmAbiGenPlugin = 15; bool shouldVerifySourceOnlyAbiConstraints = 16; @@ -58,7 +59,8 @@ message KotlinExtraParams { bool shouldKsp2RunIncrementally = 26; reserved 27; // was shouldKosabiJvmAbiGenUseK2, unused (kotlincd derives K2 // from LanguageVersion.getSupportsK2()) - string kotlinClassesDir = 28; + reserved 28; + reserved "kotlinClassesDir"; reserved 29; // was skipClasspathRemovalRebuild, removed after KT-85740 fix string javaBinary = 30; // Reduced SO-ABI classpath for the applicability plugin (only deps with @@ -72,12 +74,6 @@ message PluginParams { map params = 1; } -/** Model for KotlinLibraryDescription.AnnotationProcessingTool */ -enum AnnotationProcessingTool { - KAPT = 0; - JAVAC = 1; -} - /** Model for metadata provided by Buck2 incremental action see: * https://buck2.build/docs/rule_authors/incremental_actions */ message ActionMetadata { diff --git a/prelude/toolchains/android/src/com/facebook/buck/installer/InstallCommand.java b/prelude/toolchains/android/src/com/facebook/buck/installer/InstallCommand.java index b59fcb8..1b5d56b 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/installer/InstallCommand.java +++ b/prelude/toolchains/android/src/com/facebook/buck/installer/InstallCommand.java @@ -11,15 +11,30 @@ package com.facebook.buck.installer; import java.nio.file.Path; +import java.util.Set; /** * Install Command interface. Support for a specific installer such as iOS or Android is implemented * as subclasses of this interface. * */ public interface InstallCommand { + /** + * Names this implementation, e.g. {@code android}. Reported with every install so telemetry can + * tell one installer's installs from another's. + */ + String name(); + /** Installs an artifact from a given path/location. */ InstallResult fileReady(String artifact, Path artifactPath, InstallId installId); /** Indicate that all files have been received by the installer */ InstallResult allFilesReady(InstallId installId); + + /** + * Notifies which artifacts the client intends to send, before any of them arrive. + * + *

Lets an implementation tell "this build has no such artifact" apart from "it has not turned + * up yet", which is not otherwise decidable while artifacts are still in flight. + */ + default void onInstallStarted(InstallId installId, Set expectedArtifacts) {} } diff --git a/prelude/toolchains/android/src/com/facebook/buck/installer/InstallerService.java b/prelude/toolchains/android/src/com/facebook/buck/installer/InstallerService.java index 1f33d1e..c58c875 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/installer/InstallerService.java +++ b/prelude/toolchains/android/src/com/facebook/buck/installer/InstallerService.java @@ -110,13 +110,20 @@ private InstallResponse handleInstallRequest(InstallInfoRequest request) { LOG.info( String.format( "Received install id: %s with %d file names", installId.getValue(), fileNames.size())); + // Before the install is registered: a file is only accepted for an install that is in the map, + // so declaring first leaves no window where an artifact can arrive for an install that has not + // been told what to expect. + installer.onInstallStarted(installId, fileNames); synchronized (installIdToFilesMap) { installIdToFilesMap.put( installId, fileNames.stream() .collect(Collectors.toMap(Function.identity(), ignore -> Optional.empty()))); } - return InstallResponse.newBuilder().setInstallId(installId.getValue()).build(); + return InstallResponse.newBuilder() + .setInstallId(installId.getValue()) + .setInstallerName(installer.name()) + .build(); } @Override diff --git a/prelude/toolchains/android/src/com/facebook/buck/installer/android/AdbHelperFactory.java b/prelude/toolchains/android/src/com/facebook/buck/installer/android/AdbHelperFactory.java new file mode 100644 index 0000000..0546e4f --- /dev/null +++ b/prelude/toolchains/android/src/com/facebook/buck/installer/android/AdbHelperFactory.java @@ -0,0 +1,75 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is dual-licensed under either the MIT license found in the + * LICENSE-MIT file in the root directory of this source tree or the Apache + * License, Version 2.0 found in the LICENSE-APACHE file in the root directory + * of this source tree. You may select, at your option, one of the + * above-listed licenses. + */ + +package com.facebook.buck.installer.android; + +import com.facebook.buck.android.AdbExecutionContext; +import com.facebook.buck.android.AdbHelper; +import com.facebook.buck.android.AdbOptions; +import com.facebook.buck.android.device.TargetDeviceOptions; +import com.facebook.buck.android.exopackage.AdbUtils; +import com.facebook.buck.android.exopackage.InstallTimings; +import com.facebook.buck.android.exopackage.SetDebugAppMode; +import java.util.Optional; +import java.util.logging.Logger; // NOPMD + +/** + * Builds the {@link AdbHelper} for an install. + * + *

The one place the command line and the install options are turned into the objects adb needs, + * so a second construction cannot drift from it and reach a different set of devices. + */ +class AdbHelperFactory { + + private AdbHelperFactory() {} + + static AdbHelper create( + Logger logger, + AndroidCommandLineOptions cliOptions, + AndroidInstallApkOptions apkOptions, + SetDebugAppMode setDebugAppMode, + InstallTimings timings) { + AdbOptions adbOptions = + new AdbOptions( + cliOptions.adbExecutablePath, + cliOptions.adbThreadCount, + cliOptions.adbServerPort, + cliOptions.multiInstallMode, + apkOptions.stagedInstallMode, + cliOptions.ignoreMissingDevices, + apkOptions.apexMode, + cliOptions.restartMode.name(), + cliOptions.waitForDeviceReady); + logger.info("adbOptions: " + adbOptions); + + TargetDeviceOptions targetDeviceOptions = + new TargetDeviceOptions( + cliOptions.useEmulatorsOnlyMode, + cliOptions.useRealDevicesOnlyMode, + Optional.ofNullable(cliOptions.serialNumber)); + logger.info("targetDeviceOptions: " + targetDeviceOptions); + + AdbUtils adbUtils = + new AdbUtils( + Optional.ofNullable(apkOptions.adbExecutable) + .orElseThrow(AndroidInstallException.Companion::adbPathNotFound), + adbOptions.getAdbServerPort()); + + return new AdbHelper( + adbUtils, + adbOptions, + targetDeviceOptions, + new AdbExecutionContext(), + new IsolatedAndroidInstallerPrinter(logger), + apkOptions.restartAdbOnFailure, + setDebugAppMode, + timings); + } +} diff --git a/prelude/toolchains/android/src/com/facebook/buck/installer/android/AndroidArtifacts.java b/prelude/toolchains/android/src/com/facebook/buck/installer/android/AndroidArtifacts.java index 767cebd..c44a81a 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/installer/android/AndroidArtifacts.java +++ b/prelude/toolchains/android/src/com/facebook/buck/installer/android/AndroidArtifacts.java @@ -10,26 +10,29 @@ package com.facebook.buck.installer.android; -import com.facebook.buck.android.exopackage.InstallTimings; import com.facebook.buck.core.filesystems.AbsPath; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.Comparator; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; import java.util.EnumMap; import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Locale; +import java.util.HashSet; import java.util.Map; import java.util.Optional; +import java.util.Set; import javax.annotation.Nullable; -/** Holds android install related artifacts (apk options, manifest path, etc) */ -class AndroidArtifacts implements InstallTimings { +/** + * Holds android install related artifacts (apk options, manifest path, etc) + * + *

Shared across the gRPC handler threads that deliver artifacts, the threads that push them, and + * the install that reads them, so every member is guarded by this object's monitor. + */ +class AndroidArtifacts { private AbsPath androidManifestPath; private AndroidInstallApkOptions apkOptions; private AbsPath apk; + private ImmutableSet apkAbis; private Optional secondaryDexExopackageInfoDirectory = Optional.empty(); private Optional secondaryDexExopackageInfoMetadata = Optional.empty(); private Optional nativeLibraryExopackageInfoDirectory = Optional.empty(); @@ -38,189 +41,76 @@ class AndroidArtifacts implements InstallTimings { private Optional resourcesExopackageInfoAssetsHash = Optional.empty(); private Optional resourcesExopackageInfoRes = Optional.empty(); private Optional resourcesExopackageInfoResHash = Optional.empty(); - // Artifact name -> wall-clock arrival. Written from concurrent gRPC handler threads. + // Artifact name -> wall-clock arrival. private final Map fileArrivalMillis = new HashMap<>(); - - /** - * Records that {@code artifactName} was delivered by buck at {@code timestampMillis}. Must be - * called when the file is actually received, not when the installer gets around to consuming it, - * or every artifact is stamped with the same instant and the spans below all collapse to zero. - */ - public void recordFileArrival(String artifactName, long timestampMillis) { - synchronized (fileArrivalMillis) { - fileArrivalMillis.putIfAbsent(artifactName, timestampMillis); + // What buck said it would send, split by payload. A class with no entry is one this build does + // not produce, which is why it can never be complete. + private final Map> expectedByClass = + new EnumMap<>(ArtifactClass.class); + + /** Records which artifacts buck said it would send, before any of them arrive. */ + public synchronized void setExpectedArtifacts(Set expectedArtifacts) { + expectedByClass.clear(); + for (String artifactName : expectedArtifacts) { + expectedByClass + .computeIfAbsent(ArtifactClass.of(artifactName), unused -> new HashSet<>()) + .add(artifactName); } } - // Stage timings, written from the install thread and read once the install is over. - private final Map pushWindows = new LinkedHashMap<>(); - private long deviceSetupMillis; - private long apkInstallMillis; - private long deviceWorkMillis; - - @Override - public synchronized void recordDeviceSetup(long startMillis, long endMillis) { - deviceSetupMillis += endMillis - startMillis; - } - - @Override - public synchronized void recordPush(String group, long startMillis, long endMillis) { - // A group may be pushed as several concurrent shards; the group spans all of them. - pushWindows.merge( - group, - new long[] {startMillis, endMillis}, - (a, b) -> new long[] {Math.min(a[0], b[0]), Math.max(a[1], b[1])}); - } - - @Override - public synchronized void recordApkInstall(long startMillis, long endMillis) { - apkInstallMillis += endMillis - startMillis; - } - - @Override - public synchronized void recordDeviceWork(long startMillis, long endMillis) { - deviceWorkMillis += endMillis - startMillis; - } - /** - * How the install went, as one timeline. Every value is seconds.milliseconds, measured from the - * first artifact arriving. - * - *

Each class of artifact reports when buck finished delivering it ({@code _arrival_s}) and how - * long moving it to the device took ({@code _transfer_s}), in arrival order. Control artifacts - * are read on the host, so they never transfer. + * True once every artifact this build declared for {@code artifactClass} has arrived, and false + * if it declared none. * - *

{@code critical_path_s} replays this same install against these same durations, but starts - * each payload the moment its own artifacts were there instead of waiting for all of them. {@code - * potential_saving_s} is {@code total_s} minus that: time the device spent waiting for an - * artifact it did not yet need. Neither says anything about transferring faster, only about - * waiting less. + *

Checked against what buck said it would send, not against what happens to be on disk: assets + * are optional for a build, so their absence is otherwise indistinguishable from their not having + * turned up yet. */ - public synchronized Map getInstallMetrics(long installCompleteMillis) { - Map arrivals; - synchronized (fileArrivalMillis) { - if (fileArrivalMillis.isEmpty()) { - return Map.of(); - } - arrivals = new HashMap<>(fileArrivalMillis); - } - long first = Collections.min(arrivals.values()); - - Map readyByClass = new EnumMap<>(ArtifactClass.class); - for (Map.Entry arrival : arrivals.entrySet()) { - readyByClass.merge(ArtifactClass.of(arrival.getKey()), arrival.getValue(), Math::max); - } - long controlReady = readyByClass.getOrDefault(ArtifactClass.CONTROL, first); - long apkReady = readyByClass.getOrDefault(ArtifactClass.APK, first); - - // Replay: one pusher, each payload started as soon as it was available. - List payloads = new ArrayList<>(); // {availableAt, durationMillis} - long metadataMillis = 0L; - for (Map.Entry push : pushWindows.entrySet()) { - long duration = push.getValue()[1] - push.getValue()[0]; - ArtifactClass pushed = ArtifactClass.forPushGroup(push.getKey()); - if (pushed == null) { - metadataMillis += duration; // metadata is derived, so it can only follow every payload - continue; - } - long availableAt = readyByClass.getOrDefault(pushed, controlReady); - payloads.add(new long[] {Math.max(availableAt, controlReady), duration}); - } - payloads.sort(Comparator.comparingLong(payload -> payload[0])); - - // Replay onto as many pushers as the install actually used, so the estimate tracks the - // implementation. With one pusher this is a queue; with several it is a list schedule. - long[] pushers = new long[observedConcurrency()]; - Arrays.fill(pushers, controlReady + deviceSetupMillis); - for (long[] payload : payloads) { - int earliest = 0; - for (int i = 1; i < pushers.length; i++) { - if (pushers[i] < pushers[earliest]) { - earliest = i; - } - } - pushers[earliest] = Math.max(pushers[earliest], payload[0]) + payload[1]; - } - long clock = controlReady + deviceSetupMillis; - for (long pusher : pushers) { - clock = Math.max(clock, pusher); - } - clock += metadataMillis; - clock = Math.max(clock, apkReady) + apkInstallMillis; - // Device work that is not one of the stages above still has to happen, so charge it too. - // Without this the replay looks faster than anything achievable and every install appears to - // have a saving. - long modelled = deviceSetupMillis + apkInstallMillis; - for (long[] window : pushWindows.values()) { - modelled += window[1] - window[0]; - } - clock += Math.max(0L, deviceWorkMillis - modelled); - - long total = installCompleteMillis - first; - long criticalPath = clock - first; - - Map metrics = new LinkedHashMap<>(); - readyByClass.entrySet().stream() - .sorted(Map.Entry.comparingByValue()) - .forEach( - ready -> { - String artifactClass = ready.getKey().metricName; - metrics.put(artifactClass + "_arrival_s", seconds(ready.getValue() - first)); - metrics.put(artifactClass + "_transfer_s", seconds(transferMillis(ready.getKey()))); - }); - metrics.put("device_setup_s", seconds(deviceSetupMillis)); - if (metadataMillis > 0L) { - metrics.put("metadata_transfer_s", seconds(metadataMillis)); - } - metrics.put("critical_path_s", seconds(criticalPath)); - metrics.put("total_s", seconds(total)); - metrics.put("potential_saving_s", seconds(Math.max(0L, total - criticalPath))); - return metrics; + public synchronized boolean hasAllArtifactsFor(ArtifactClass artifactClass) { + Set declared = expectedByClass.get(artifactClass); + return declared != null && fileArrivalMillis.keySet().containsAll(declared); } - /** How long this class of artifact took to reach the device, or zero if it is not transferred. */ - private long transferMillis(ArtifactClass artifactClass) { - if (artifactClass == ArtifactClass.APK) { - return apkInstallMillis; - } - for (Map.Entry push : pushWindows.entrySet()) { - if (ArtifactClass.forPushGroup(push.getKey()) == artifactClass) { - return push.getValue()[1] - push.getValue()[0]; - } - } - return 0L; + /** + * True once every artifact buck declared has arrived, and false if it declared none. + * + *

Independent of how names bucket into classes: it asks whether anything is still coming, not + * which payload it belongs to. + */ + public synchronized boolean allArtifactsArrived() { + return !expectedByClass.isEmpty() && undeliveredArtifacts().isEmpty(); } - private static String seconds(long millis) { - return String.format(Locale.ROOT, "%.3f", millis / 1000.0); + /** + * Artifacts buck declared but never delivered. + * + *

Only meaningful once buck says it has sent everything; before that an artifact is missing + * simply because it has not arrived. A name here means the two sides disagree about what an + * artifact is called, which leaves its class permanently incomplete and its payload unstreamed. + */ + public synchronized ImmutableSet undeliveredArtifacts() { + return expectedByClass.values().stream() + .flatMap(Set::stream) + .filter(artifactName -> !fileArrivalMillis.containsKey(artifactName)) + .collect(ImmutableSet.toImmutableSet()); } /** - * How many payloads the install had in flight at once, from their observed windows. One if the - * pushes did not overlap. + * Records that {@code artifactName} was delivered by buck at {@code timestampMillis}. Must be + * called when the file is actually received, not when the installer gets around to consuming it, + * or every artifact is stamped with the same instant and the spans below all collapse to zero. */ - private int observedConcurrency() { - List edges = new ArrayList<>(); - for (long[] window : pushWindows.values()) { - if (window[1] > window[0]) { - edges.add(new long[] {window[0], 1L}); - edges.add(new long[] {window[1], -1L}); - } - } - // Ends before starts at the same instant, so touching windows are not counted as overlapping. - edges.sort( - Comparator.comparingLong(edge -> edge[0]).thenComparingLong(edge -> edge[1])); - int concurrent = 0; - int peak = 0; - for (long[] edge : edges) { - concurrent += (int) edge[1]; - peak = Math.max(peak, concurrent); - } - return Math.max(1, peak); + public synchronized void recordFileArrival(String artifactName, long timestampMillis) { + fileArrivalMillis.putIfAbsent(artifactName, timestampMillis); + } + + /** When each artifact was delivered, for whoever is measuring the install. */ + public synchronized ImmutableMap arrivals() { + return ImmutableMap.copyOf(fileArrivalMillis); } /** Groups install artifacts by the exopackage payload they belong to. */ - private enum ArtifactClass { + enum ArtifactClass { SECONDARY_DEX("dex"), NATIVE_LIBRARY("native"), RESOURCES("resources"), @@ -233,6 +123,10 @@ private enum ArtifactClass { this.metricName = metricName; } + /** The classes that are exopackage payloads, each pushable to the device on its own. */ + static final ImmutableSet EXOPACKAGE_PAYLOADS = + Sets.immutableEnumSet(SECONDARY_DEX, NATIVE_LIBRARY, RESOURCES); + /** The class whose arrival gates pushing {@code pushGroup}, or null if it is derived. */ @Nullable static ArtifactClass forPushGroup(String pushGroup) { @@ -268,96 +162,112 @@ static ArtifactClass of(String artifactName) { } } - public void setAndroidManifestPath(AbsPath androidManifestPath) { + public synchronized void setAndroidManifestPath(AbsPath androidManifestPath) { this.androidManifestPath = androidManifestPath; } - public AbsPath getAndroidManifestPath() { + public synchronized AbsPath getAndroidManifestPath() { return this.androidManifestPath; } - public void setApkOptions(AndroidInstallApkOptions apkOptions) { + public synchronized void setApkOptions(AndroidInstallApkOptions apkOptions) { this.apkOptions = apkOptions; } - public AndroidInstallApkOptions getApkOptions() { + public synchronized AndroidInstallApkOptions getApkOptions() { return this.apkOptions; } - public AbsPath getApk() { + /** + * The ABIs the apk carries native code for, or null until buck has sent the cpu filters. Empty + * means the filters named nothing this installer recognises. + */ + @Nullable + public synchronized ImmutableSet getApkAbis() { + return apkAbis; + } + + public synchronized void setApkAbis(ImmutableSet apkAbis) { + this.apkAbis = apkAbis; + } + + public synchronized AbsPath getApk() { return apk; } - public void setApk(AbsPath apk) { + public synchronized void setApk(AbsPath apk) { this.apk = apk; } - public Optional getSecondaryDexExopackageInfoDirectory() { + public synchronized Optional getSecondaryDexExopackageInfoDirectory() { return secondaryDexExopackageInfoDirectory; } - public void setSecondaryDexExopackageInfoDirectory( + public synchronized void setSecondaryDexExopackageInfoDirectory( Optional secondaryDexExopackageInfoDirectory) { this.secondaryDexExopackageInfoDirectory = secondaryDexExopackageInfoDirectory; } - public Optional getSecondaryDexExopackageInfoMetadata() { + public synchronized Optional getSecondaryDexExopackageInfoMetadata() { return secondaryDexExopackageInfoMetadata; } - public void setSecondaryDexExopackageInfoMetadata( + public synchronized void setSecondaryDexExopackageInfoMetadata( Optional secondaryDexExopackageInfoMetadata) { this.secondaryDexExopackageInfoMetadata = secondaryDexExopackageInfoMetadata; } - public Optional getNativeLibraryExopackageInfoDirectory() { + public synchronized Optional getNativeLibraryExopackageInfoDirectory() { return nativeLibraryExopackageInfoDirectory; } - public void setNativeLibraryExopackageInfoDirectory( + public synchronized void setNativeLibraryExopackageInfoDirectory( Optional nativeLibraryExopackageInfoDirectory) { this.nativeLibraryExopackageInfoDirectory = nativeLibraryExopackageInfoDirectory; } - public Optional getNativeLibraryExopackageInfoMetadata() { + public synchronized Optional getNativeLibraryExopackageInfoMetadata() { return nativeLibraryExopackageInfoMetadata; } - public void setNativeLibraryExopackageInfoMetadata( + public synchronized void setNativeLibraryExopackageInfoMetadata( Optional nativeLibraryExopackageInfoMetadata) { this.nativeLibraryExopackageInfoMetadata = nativeLibraryExopackageInfoMetadata; } - public Optional getResourcesExopackageInfoAssets() { + public synchronized Optional getResourcesExopackageInfoAssets() { return resourcesExopackageInfoAssets; } - public void setResourcesExopackageInfoAssets(Optional resourcesExopackageInfoAssets) { + public synchronized void setResourcesExopackageInfoAssets( + Optional resourcesExopackageInfoAssets) { this.resourcesExopackageInfoAssets = resourcesExopackageInfoAssets; } - public Optional getResourcesExopackageInfoAssetsHash() { + public synchronized Optional getResourcesExopackageInfoAssetsHash() { return resourcesExopackageInfoAssetsHash; } - public void setResourcesExopackageInfoAssetsHash( + public synchronized void setResourcesExopackageInfoAssetsHash( Optional resourcesExopackageInfoAssetsHash) { this.resourcesExopackageInfoAssetsHash = resourcesExopackageInfoAssetsHash; } - public Optional getResourcesExopackageInfoRes() { + public synchronized Optional getResourcesExopackageInfoRes() { return resourcesExopackageInfoRes; } - public void setResourcesExopackageInfoRes(Optional resourcesExopackageInfoRes) { + public synchronized void setResourcesExopackageInfoRes( + Optional resourcesExopackageInfoRes) { this.resourcesExopackageInfoRes = resourcesExopackageInfoRes; } - public Optional getResourcesExopackageInfoResHash() { + public synchronized Optional getResourcesExopackageInfoResHash() { return resourcesExopackageInfoResHash; } - public void setResourcesExopackageInfoResHash(Optional resourcesExopackageInfoResHash) { + public synchronized void setResourcesExopackageInfoResHash( + Optional resourcesExopackageInfoResHash) { this.resourcesExopackageInfoResHash = resourcesExopackageInfoResHash; } } diff --git a/prelude/toolchains/android/src/com/facebook/buck/installer/android/AndroidInstall.java b/prelude/toolchains/android/src/com/facebook/buck/installer/android/AndroidInstall.java index 176565b..81d035e 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/installer/android/AndroidInstall.java +++ b/prelude/toolchains/android/src/com/facebook/buck/installer/android/AndroidInstall.java @@ -10,26 +10,18 @@ package com.facebook.buck.installer.android; -import com.facebook.buck.android.AdbExecutionContext; import com.facebook.buck.android.AdbHelper; -import com.facebook.buck.android.AdbOptions; import com.facebook.buck.android.IsolatedApkInfo; -import com.facebook.buck.android.device.TargetDeviceOptions; -import com.facebook.buck.android.exopackage.AdbUtils; import com.facebook.buck.android.exopackage.AndroidDeviceInfo; import com.facebook.buck.android.exopackage.ExopackageInstaller; import com.facebook.buck.android.exopackage.IsolatedExopackageInfo; -import com.facebook.buck.android.exopackage.SetDebugAppMode; import com.facebook.buck.core.filesystems.AbsPath; import com.facebook.buck.installer.InstallId; import com.facebook.buck.installer.InstallResult; -import com.facebook.buck.util.Console; -import com.facebook.buck.util.Verbosity; -import com.google.common.io.ByteStreams; -import java.io.ByteArrayOutputStream; -import java.io.PrintStream; import java.time.Duration; import java.time.Instant; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; @@ -42,6 +34,8 @@ /** Installs an Android Apk */ class AndroidInstall { private static final Logger LOG = Logger.getLogger(AndroidInstall.class.getName()); + private static final DateTimeFormatter INSTALL_COMPLETION_TIME_FORMAT = + DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); private static final Set ENABLE_APP_LINKS_ALLOWLIST = Set.of("com.facebook.wakizashi", "com.facebook.lite", "com.instagram.lite"); @@ -53,73 +47,25 @@ class AndroidInstall { private final boolean installViaSd = false; private final Logger logger; private final AdbHelper adbHelper; - private final ByteArrayOutputStream stderr; - private final AndroidArtifacts artifacts; + private final InstallState state; public AndroidInstall( Logger logger, AbsPath rootPath, AndroidCommandLineOptions cliOptions, - AndroidInstallApkOptions apkOptions, IsolatedApkInfo apkInfo, Optional exopackageInfo, InstallId installId, - AndroidArtifacts artifacts) { + InstallState state, + AdbHelper adbHelper) { this.logger = logger; this.rootPath = rootPath; this.apkInfo = apkInfo; this.exopackageInfo = exopackageInfo; this.installId = installId; this.cliOptions = cliOptions; - this.artifacts = artifacts; - - // Set-up adbOptions - AdbOptions adbOptions = - new AdbOptions( - cliOptions.adbExecutablePath, - cliOptions.adbThreadCount, - cliOptions.adbServerPort, - cliOptions.multiInstallMode, - apkOptions.stagedInstallMode, - cliOptions.ignoreMissingDevices, - apkOptions.apexMode, - cliOptions.restartMode.name(), - cliOptions.waitForDeviceReady); - LOG.info("adbOptions: " + adbOptions); - - TargetDeviceOptions targetDeviceOptions = - new TargetDeviceOptions( - cliOptions.useEmulatorsOnlyMode, - cliOptions.useRealDevicesOnlyMode, - Optional.ofNullable(cliOptions.serialNumber)); - LOG.info("targetDeviceOptions: " + targetDeviceOptions); - - this.stderr = new ByteArrayOutputStream(); - Console console = - new Console( - Verbosity.STANDARD_INFORMATION, - new PrintStream(ByteStreams.nullOutputStream()), - new PrintStream(stderr)); - SetDebugAppMode setDebugAppMode = SetDebugAppMode.SET; - if (cliOptions.skipSetDebugApp) { - setDebugAppMode = SetDebugAppMode.SKIP; - } - AdbUtils adbUtils = - new AdbUtils( - Optional.of(apkOptions.adbExecutable) - .orElseThrow(AndroidInstallException.Companion::adbPathNotFound), - adbOptions.getAdbServerPort()); - this.adbHelper = - new AdbHelper( - adbUtils, - adbOptions, - targetDeviceOptions, - new AdbExecutionContext(console), - new IsolatedAndroidInstallerPrinter(logger), - apkOptions.restartAdbOnFailure, - apkOptions.skipInstallMetadata, - setDebugAppMode, - artifacts); + this.state = state; + this.adbHelper = adbHelper; } /** Uses AdbHelper to do actual install with APK */ @@ -127,14 +73,10 @@ public synchronized InstallResult installApk() { List> deviceInfos = new ArrayList(); try { if (cliOptions.cleanUp) { - String appId = - AdbHelper.tryToExtractPackageNameFromManifest(apkInfo.getManifestPath().getPath()); - adbHelper.uninstallApp(appId, cliOptions.keepUserData); + adbHelper.uninstallApp(state.packageName(), cliOptions.keepUserData); } else { if (cliOptions.uninstallFirst) { - String appId = - AdbHelper.tryToExtractPackageNameFromManifest(apkInfo.getManifestPath().getPath()); - adbHelper.uninstallApp(appId, cliOptions.keepUserData); + adbHelper.uninstallApp(state.packageName(), cliOptions.keepUserData); } logger.info(String.format("Attempting install of %s", apkInfo.getApkPath())); Instant start = Instant.now(); @@ -167,21 +109,23 @@ public synchronized InstallResult installApk() { rootPath, installViaSd, /* quiet= */ false, - installId.getValue()); - artifacts.recordDeviceWork(deviceWorkStart, System.currentTimeMillis()); + installId.getValue(), + state.packageName()); + state.metrics().recordDeviceWork(deviceWorkStart, System.currentTimeMillis()); // Only now are the stage timings complete, so the metrics cannot be gathered any earlier. Map installMetrics = - artifacts.getInstallMetrics(System.currentTimeMillis()); + state.metrics().summarise(System.currentTimeMillis(), state.artifacts().arrivals()); deviceInfos.forEach(infoMap -> infoMap.putAll(installMetrics)); + Instant completedAt = Instant.now(); logger.info( - String.format( - "Install of %s finished in %d seconds", - apkInfo.getApkPath().getFileName(), - Duration.between(start, Instant.now()).getSeconds())); + formatCompletionMessage( + apkInfo.getApkPath().getFileName().toString(), + start, + completedAt, + ZoneId.systemDefault())); - String packageName = - AdbHelper.tryToExtractPackageNameFromManifest(apkInfo.getManifestPath().getPath()); + String packageName = state.packageName(); // Determine if app links should be enabled based on command line option or allowlist boolean shouldEnableAppLinks = false; @@ -221,11 +165,7 @@ public synchronized InstallResult installApk() { } catch (AndroidInstallException exc) { return new InstallResult(deviceInfos, Optional.of(exc.getInstallError())); } catch (Exception err) { - String errMsg = - Optional.ofNullable(stderr.toString()) - .filter(s -> !s.isEmpty()) - .map(s -> "stderr message: " + s) - .orElseGet(err::getMessage); + String errMsg = err.getMessage(); logger.log( Level.SEVERE, String.format("Error while installing %s. Error message: %s", installId, errMsg), @@ -235,4 +175,13 @@ public synchronized InstallResult installApk() { Optional.of(errMsg).map(AndroidInstallErrorClassifier.INSTANCE::fromErrorMessage)); } } + + static String formatCompletionMessage( + String apkName, Instant start, Instant completedAt, ZoneId zoneId) { + return String.format( + "Install of %s finished in %d seconds at %s", + apkName, + Duration.between(start, completedAt).getSeconds(), + INSTALL_COMPLETION_TIME_FORMAT.withZone(zoneId).format(completedAt)); + } } diff --git a/prelude/toolchains/android/src/com/facebook/buck/installer/android/AndroidInstallApkOptions.java b/prelude/toolchains/android/src/com/facebook/buck/installer/android/AndroidInstallApkOptions.java index fdfaff0..1ea0f67 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/installer/android/AndroidInstallApkOptions.java +++ b/prelude/toolchains/android/src/com/facebook/buck/installer/android/AndroidInstallApkOptions.java @@ -33,7 +33,6 @@ public class AndroidInstallApkOptions { public final String adbExecutable; public final boolean restartAdbOnFailure; public final boolean stagedInstallMode; - public final boolean skipInstallMetadata; public final boolean apexMode; AndroidInstallApkOptions(Path jsonArtifactPath, String adbExecutablePath) @@ -45,7 +44,6 @@ public class AndroidInstallApkOptions { this.restartAdbOnFailure = readBoolean(jsonData, "adb_restart_on_failure"); this.stagedInstallMode = readBoolean(jsonData, "staged_install_mode"); this.apexMode = readBoolean(jsonData, "apex_mode"); - this.skipInstallMetadata = readBoolean(jsonData, "skip_install_metadata"); } private boolean readBoolean(Map jsonData, String name) { @@ -103,8 +101,6 @@ public String toString() { + restartAdbOnFailure + ", stagedInstallMode=" + stagedInstallMode - + ", skipInstallMetadata=" - + skipInstallMetadata + ", apexMode=" + apexMode + '}'; diff --git a/prelude/toolchains/android/src/com/facebook/buck/installer/android/AndroidInstallException.kt b/prelude/toolchains/android/src/com/facebook/buck/installer/android/AndroidInstallException.kt index 1421d5b..a3aa166 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/installer/android/AndroidInstallException.kt +++ b/prelude/toolchains/android/src/com/facebook/buck/installer/android/AndroidInstallException.kt @@ -87,6 +87,14 @@ class AndroidInstallException(val installError: InstallError) : ), ) + fun devicesDeparted(serials: Collection) = AndroidInstallException( + InstallError( + "Device(s) disconnected while the build was running, so the install could not " + + "reach them: ${serials.sorted().joinToString(", ")}", + AndroidInstallErrorTag.DEVICE_NOT_FOUND, + ), + ) + fun adbPathNotFound() = AndroidInstallException( InstallError("Adb path not found.", AndroidInstallErrorTag.ADB_NOT_FOUND), ) diff --git a/prelude/toolchains/android/src/com/facebook/buck/installer/android/AndroidInstallerManager.java b/prelude/toolchains/android/src/com/facebook/buck/installer/android/AndroidInstallerManager.java index 93c85c9..824e928 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/installer/android/AndroidInstallerManager.java +++ b/prelude/toolchains/android/src/com/facebook/buck/installer/android/AndroidInstallerManager.java @@ -10,32 +10,37 @@ package com.facebook.buck.installer.android; +import com.facebook.buck.android.AdbHelper; import com.facebook.buck.android.IsolatedApkInfo; import com.facebook.buck.android.exopackage.IsolatedExopackageInfo; +import com.facebook.buck.android.exopackage.SetDebugAppMode; import com.facebook.buck.core.filesystems.AbsPath; import com.facebook.buck.installer.InstallCommand; import com.facebook.buck.installer.InstallError; import com.facebook.buck.installer.InstallId; import com.facebook.buck.installer.InstallResult; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.base.Throwables; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; -import java.io.BufferedReader; +import com.google.common.util.concurrent.ThreadFactoryBuilder; import java.io.IOException; -import java.io.InputStreamReader; +import java.io.UncheckedIOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.logging.Level; import java.util.logging.Logger; // NOPMD -import java.util.stream.Collectors; /** * Coordinates an Android Install of an APK. We need three artifacts: the apk, @@ -47,18 +52,27 @@ class AndroidInstallerManager implements InstallCommand { private final AndroidCommandLineOptions options; private final AndroidInstallErrorClassifier errorClassifier = AndroidInstallErrorClassifier.INSTANCE; - private final Map installIdToFutureMap = new HashMap<>(); + private final Map installs = new ConcurrentHashMap<>(); - private final Map SHORT_TO_FULL_ABI_MAP = - new HashMap<>( - Map.of("armv7", "armeabi-v7a", "arm64", "arm64-v8a", "x86", "x86", "x86_64", "x86_64")); + private static final AbsPath ROOT_PATH = AbsPath.of(Paths.get(".").normalize().toAbsolutePath()); - private final Set ABI_64s = new HashSet<>(Arrays.asList("arm64-v8a", "x86_64")); + private static final ExecutorService STREAM_EXECUTOR = + Executors.newCachedThreadPool( + new ThreadFactoryBuilder().setNameFormat("exopackage-stream-%d").setDaemon(true).build()); + + private static final ImmutableMap SHORT_TO_FULL_ABI_MAP = + ImmutableMap.of( + "armv7", "armeabi-v7a", "arm64", "arm64-v8a", "x86", "x86", "x86_64", "x86_64"); AndroidInstallerManager(AndroidCommandLineOptions options) { this.options = options; } + @Override + public String name() { + return "android"; + } + /** * Coordinates the install artifacts needed for an install. The install_android_options.json is * parsed into a AndroidInstallApkOptions and the manifest is later set in the apkInstallOptions @@ -67,60 +81,123 @@ class AndroidInstallerManager implements InstallCommand { @Override public InstallResult fileReady(String artifactName, Path artifactPath, InstallId installId) { try { - AndroidArtifacts androidArtifacts = getOrMakeAndroidArtifacts(installId); - androidArtifacts.recordFileArrival(artifactName, System.currentTimeMillis()); + AndroidArtifacts androidArtifacts = installState(installId).artifacts(); + long arrivedAt = System.currentTimeMillis(); - if (artifactName.equals("cpu_filters")) { - return checkAbiCompatibility(AbsPath.of(artifactPath)); - } + // Path before arrival: an artifact counted as arrived while its path is still unset reads as + // usable to anyone judging readiness from arrivals. Delivery is all this records -- what the + // install then makes of the artifact is not the artifact's business. + recordArtifactPath(installState(installId), artifactName, artifactPath); + androidArtifacts.recordFileArrival(artifactName, arrivedAt); if (artifactName.equals("options")) { - androidArtifacts.setApkOptions( - new AndroidInstallApkOptions(artifactPath, options.adbExecutablePath)); + resolveDevices(installId, androidArtifacts); + } + Optional incompatible = checkAbiCompatibility(installId); + if (incompatible.isPresent()) { + return InstallResult.error(incompatible.get()); + } + maybeStreamReadyPayloads(installId, androidArtifacts); + return InstallResult.success(); + } catch (Exception err) { + String errMsg = Throwables.getStackTraceAsString(err); + LOG.log( + Level.SEVERE, + String.format( + "Error installing %s from %s due to %s", artifactName, artifactPath, errMsg), + err); + return InstallResult.error(errorClassifier.fromErrorMessage(errMsg)); + } + } + + @Override + public void onInstallStarted(InstallId installId, Set expectedArtifacts) { + installState(installId).artifacts().setExpectedArtifacts(expectedArtifacts); + } + + /** + * Records an artifact as it arrives. The install_android_options.json is parsed into an + * AndroidInstallApkOptions and the manifest is later set in the apkInstallOptions as a separate + * field. + */ + private void recordArtifactPath( + InstallState installState, String artifactName, Path artifactPath) { + AndroidArtifacts androidArtifacts = installState.artifacts(); + switch (artifactName) { + case "cpu_filters": + androidArtifacts.setApkAbis(readApkAbis(artifactPath)); + break; + case "options": + try { + androidArtifacts.setApkOptions( + new AndroidInstallApkOptions(artifactPath, options.adbExecutablePath)); + } catch (IOException e) { + // Surfaces as an install failure: InstallerService turns this into an error response. + throw new UncheckedIOException("Could not read install options " + artifactPath, e); + } LOG.log(Level.INFO, androidArtifacts.getApkOptions().toString()); - } else if (artifactName.equals("manifest")) { + break; + case "manifest": androidArtifacts.setAndroidManifestPath(AbsPath.of(artifactPath)); - } else if (artifactName.equals("secondary_dex_exopackage_info_directory")) { + // Read now rather than when a later stage wants it: a manifest that cannot be parsed names + // no package, and no part of the install can proceed without one. + installState.setPackageName(AdbHelper.tryToExtractPackageNameFromManifest(artifactPath)); + break; + case "secondary_dex_exopackage_info_directory": androidArtifacts.setSecondaryDexExopackageInfoDirectory( Optional.of(AbsPath.of(artifactPath))); - } else if (artifactName.equals("secondary_dex_exopackage_info_metadata")) { + break; + case "secondary_dex_exopackage_info_metadata": androidArtifacts.setSecondaryDexExopackageInfoMetadata( Optional.of(AbsPath.of(artifactPath))); - } else if (artifactName.equals("native_library_exopackage_info_directory")) { + break; + case "native_library_exopackage_info_directory": androidArtifacts.setNativeLibraryExopackageInfoDirectory( Optional.of(AbsPath.of(artifactPath))); - } else if (artifactName.equals("native_library_exopackage_info_metadata")) { + break; + case "native_library_exopackage_info_metadata": androidArtifacts.setNativeLibraryExopackageInfoMetadata( Optional.of(AbsPath.of(artifactPath))); - } else if (artifactName.equals("resources_exopackage_assets")) { + break; + case "resources_exopackage_assets": androidArtifacts.setResourcesExopackageInfoAssets(Optional.of(AbsPath.of(artifactPath))); - } else if (artifactName.equals("resources_exopackage_assets_hash")) { + break; + case "resources_exopackage_assets_hash": androidArtifacts.setResourcesExopackageInfoAssetsHash( Optional.of(AbsPath.of(artifactPath))); - } else if (artifactName.equals("resources_exopackage_res")) { + break; + case "resources_exopackage_res": androidArtifacts.setResourcesExopackageInfoRes(Optional.of(AbsPath.of(artifactPath))); - } else if (artifactName.equals("resources_exopackage_res_hash")) { + break; + case "resources_exopackage_res_hash": androidArtifacts.setResourcesExopackageInfoResHash(Optional.of(AbsPath.of(artifactPath))); - } else { + break; + default: androidArtifacts.setApk(AbsPath.of(artifactPath)); - } - - return InstallResult.success(); - } catch (Exception err) { - String errMsg = Throwables.getStackTraceAsString(err); - LOG.log( - Level.SEVERE, - String.format( - "Error installing %s from %s due to %s", artifactName, artifactPath, errMsg), - err); - return InstallResult.error(errorClassifier.fromErrorMessage(errMsg)); + break; } } @Override public InstallResult allFilesReady(InstallId installId) { try { - AndroidArtifacts androidArtifacts = getOrMakeAndroidArtifacts(installId); + AndroidArtifacts androidArtifacts = installState(installId).artifacts(); + + // Before anything else, and before any path can return: a push cannot be stopped once it has + // started, so the install waits it out rather than leaving one writing into the directories + // it is about to write itself. + installState(installId).streamedPushes().sealAndAwait(); + + ImmutableSet undelivered = androidArtifacts.undeliveredArtifacts(); + if (!undelivered.isEmpty()) { + LOG.log( + Level.WARNING, + String.format( + "Install %s: buck declared these artifacts and never sent them, so their payload" + + " never counted as complete and was pushed by the install rather than" + + " ahead of it: %s", + installId.getValue(), undelivered)); + } String adbPath = androidArtifacts.getApkOptions().adbExecutable; if (!Files.exists(Paths.get(adbPath))) { @@ -132,16 +209,142 @@ public InstallResult allFilesReady(InstallId installId) { adbPath))); } - Optional secondaryDexExopackageInfoDirectory = - androidArtifacts.getSecondaryDexExopackageInfoDirectory(); - Optional secondaryDexExopackageInfoMetadata = - androidArtifacts.getSecondaryDexExopackageInfoMetadata(); - Optional nativeLibraryExopackageInfoDirectory = - androidArtifacts.getNativeLibraryExopackageInfoDirectory(); - Optional nativeLibraryExopackageInfoMetadata = - androidArtifacts.getNativeLibraryExopackageInfoMetadata(); - ImmutableList.Builder - pathAndHashBuilder = ImmutableList.builder(); + AdbHelper adbHelper = resolveDevices(installId, androidArtifacts); + ImmutableSet departed = adbHelper.departedSerials(); + if (!departed.isEmpty()) { + return InstallResult.error( + AndroidInstallException.Companion.devicesDeparted(departed).getInstallError()); + } + + Optional isolatedExopackageInfo = + buildExopackageInfo(androidArtifacts, AndroidArtifacts.ArtifactClass.EXOPACKAGE_PAYLOADS); + + AndroidInstall androidInstaller = + new AndroidInstall( + LOG, + ROOT_PATH, + options, + IsolatedApkInfo.of( + androidArtifacts.getAndroidManifestPath(), androidArtifacts.getApk()), + isolatedExopackageInfo, + installId, + installState(installId), + adbHelper); + return androidInstaller.installApk(); + + } catch (Exception err) { + String errMsg = Throwables.getStackTraceAsString(err); + LOG.log(Level.SEVERE, String.format("Install error due to %s", errMsg), err); + return InstallResult.error(errorClassifier.fromErrorMessage(errMsg)); + } + } + + /** + * Starts pushing exopackage payloads as soon as they are complete, rather than waiting for the + * apk. + * + *

Payloads finish at different points in the build, so the device would otherwise sit idle + * until the last of them. Readiness is judged per payload, so one that finishes early does not + * wait behind one that finishes late; payloads that are ready together go in a single push, which + * pays for the device setup once rather than once each. + */ + private void maybeStreamReadyPayloads(InstallId installId, AndroidArtifacts androidArtifacts) { + if (options.cleanUp) { + // This install only uninstalls, so nothing it sends would ever be read -- and the payload + // would outlive the app it belongs to. + return; + } + // Check if it is too early to start streaming. + AdbHelper adbHelper = installState(installId).adbHelper(); + if (androidArtifacts.getAndroidManifestPath() == null || adbHelper == null) { + // No package name, or no devices to reach yet. Before anything is claimed, because a claim + // is permanent: claiming a payload that cannot be sent leaves it to the install to push. + return; + } + // Check if it is too late to start streaming. + if (androidArtifacts.allArtifactsArrived()) { + // Nothing left to wait for, so there is no idle device time to fill. + return; + } + Set ready = + AndroidArtifacts.ArtifactClass.EXOPACKAGE_PAYLOADS.stream() + .filter(androidArtifacts::hasAllArtifactsFor) + .collect(ImmutableSet.toImmutableSet()); + installState(installId) + .streamedPushes() + .dispatch( + ready, + pending -> + STREAM_EXECUTOR.submit( + () -> streamPayloads(installState(installId), adbHelper, pending))); + } + + private void streamPayloads( + InstallState state, AdbHelper adbHelper, Set payloads) { + try { + Optional exopackageInfo = + buildExopackageInfo(state.artifacts(), payloads); + if (exopackageInfo.isEmpty()) { + return; + } + // Read here rather than where the push is dispatched: an unreadable manifest is this + // method's problem to swallow, not a reason to fail the artifact that triggered it. + String packageName = state.packageName(); + if (packageName == null) { + LOG.log(Level.WARNING, "No package name in the manifest; leaving the payloads to install"); + return; + } + // The devices the install itself will use, so a payload never streams somewhere the install + // is not going. Counted like any other push: whatever lands here the install then finds + // already present, so no payload is recorded twice. + adbHelper.streamExopackagePayloads(ROOT_PATH, exopackageInfo.get(), packageName); + } catch (InterruptedException e) { + // Only reachable through adbCall's signature; nothing interrupts these threads. Rethrown so + // the future carries the failure rather than reporting a push that did not happen. + LOG.log(Level.WARNING, String.format("Streaming %s was interrupted", payloads), e); + throw new RuntimeException(e); + } catch (Exception e) { + // Best effort. Whatever did not make it is pushed by the install, which lists the directory + // and so sees exactly what is missing. + LOG.log( + Level.WARNING, + String.format("Could not stream %s; the install will push instead", payloads), + e); + } + } + + /** + * Assembles the payloads named in {@code payloads} from the artifacts that have arrived. + * + *

A payload left out is reported absent rather than empty, so the caller pushes only what it + * asked for. Only payloads known to be complete may be named: a half-delivered one fails the + * pairing checks below. + */ + @VisibleForTesting + static Optional buildExopackageInfo( + AndroidArtifacts androidArtifacts, Set payloads) { + boolean includeSecondaryDex = payloads.contains(AndroidArtifacts.ArtifactClass.SECONDARY_DEX); + boolean includeNativeLibrary = payloads.contains(AndroidArtifacts.ArtifactClass.NATIVE_LIBRARY); + boolean includeResources = payloads.contains(AndroidArtifacts.ArtifactClass.RESOURCES); + Optional secondaryDexExopackageInfoDirectory = + includeSecondaryDex + ? androidArtifacts.getSecondaryDexExopackageInfoDirectory() + : Optional.empty(); + Optional secondaryDexExopackageInfoMetadata = + includeSecondaryDex + ? androidArtifacts.getSecondaryDexExopackageInfoMetadata() + : Optional.empty(); + Optional nativeLibraryExopackageInfoDirectory = + includeNativeLibrary + ? androidArtifacts.getNativeLibraryExopackageInfoDirectory() + : Optional.empty(); + Optional nativeLibraryExopackageInfoMetadata = + includeNativeLibrary + ? androidArtifacts.getNativeLibraryExopackageInfoMetadata() + : Optional.empty(); + ImmutableList.Builder pathAndHashBuilder = + ImmutableList.builder(); + if (includeResources) { // Assets are optional for a build, but a resource and its hash always ship together. Without // this, a half-delivered pair is indistinguishable from a build that has no assets at all. Preconditions.checkState( @@ -166,167 +369,145 @@ public InstallResult allFilesReady(InstallId installId) { pathAndHashBuilder.add( new IsolatedExopackageInfo.IsolatedExopackagePathAndHash( res, androidArtifacts.getResourcesExopackageInfoResHash().get()))); - ImmutableList exopackageResources = - pathAndHashBuilder.build(); - - Optional isolatedExopackageInfo = Optional.empty(); - if (secondaryDexExopackageInfoDirectory.isPresent() - || secondaryDexExopackageInfoMetadata.isPresent() - || nativeLibraryExopackageInfoDirectory.isPresent() - || nativeLibraryExopackageInfoMetadata.isPresent() - || !exopackageResources.isEmpty()) { - Preconditions.checkState( - secondaryDexExopackageInfoDirectory.isPresent() - == secondaryDexExopackageInfoMetadata.isPresent()); - Optional dexInfo = - secondaryDexExopackageInfoDirectory.map( - directory -> - new IsolatedExopackageInfo.IsolatedDexInfo( - secondaryDexExopackageInfoMetadata.get(), directory)); - - Preconditions.checkState( - nativeLibraryExopackageInfoDirectory.isPresent() - == nativeLibraryExopackageInfoMetadata.isPresent()); - Optional nativeLibsInfo = - nativeLibraryExopackageInfoDirectory.map( - absPath -> - new IsolatedExopackageInfo.IsolatedNativeLibsInfo( - nativeLibraryExopackageInfoMetadata.get(), absPath)); - - Optional resourcesInfo; - if (!exopackageResources.isEmpty()) { - resourcesInfo = - Optional.of(new IsolatedExopackageInfo.IsolatedResourcesInfo(exopackageResources)); - } else { - resourcesInfo = Optional.empty(); - } + } + ImmutableList exopackageResources = + pathAndHashBuilder.build(); + + Optional isolatedExopackageInfo = Optional.empty(); + if (secondaryDexExopackageInfoDirectory.isPresent() + || secondaryDexExopackageInfoMetadata.isPresent() + || nativeLibraryExopackageInfoDirectory.isPresent() + || nativeLibraryExopackageInfoMetadata.isPresent() + || !exopackageResources.isEmpty()) { + Preconditions.checkState( + secondaryDexExopackageInfoDirectory.isPresent() + == secondaryDexExopackageInfoMetadata.isPresent()); + Optional dexInfo = + secondaryDexExopackageInfoDirectory.map( + directory -> + new IsolatedExopackageInfo.IsolatedDexInfo( + secondaryDexExopackageInfoMetadata.get(), directory)); + + Preconditions.checkState( + nativeLibraryExopackageInfoDirectory.isPresent() + == nativeLibraryExopackageInfoMetadata.isPresent()); + Optional nativeLibsInfo = + nativeLibraryExopackageInfoDirectory.map( + absPath -> + new IsolatedExopackageInfo.IsolatedNativeLibsInfo( + nativeLibraryExopackageInfoMetadata.get(), absPath)); - isolatedExopackageInfo = - Optional.of(new IsolatedExopackageInfo(dexInfo, nativeLibsInfo, resourcesInfo)); + Optional resourcesInfo; + if (!exopackageResources.isEmpty()) { + resourcesInfo = + Optional.of(new IsolatedExopackageInfo.IsolatedResourcesInfo(exopackageResources)); + } else { + resourcesInfo = Optional.empty(); } - AndroidInstall androidInstaller = - new AndroidInstall( - LOG, - AbsPath.of(Paths.get(".").normalize().toAbsolutePath()), - options, - androidArtifacts.getApkOptions(), - IsolatedApkInfo.of( - androidArtifacts.getAndroidManifestPath(), androidArtifacts.getApk()), - isolatedExopackageInfo, - installId, - androidArtifacts); - return androidInstaller.installApk(); - } catch (Exception err) { - String errMsg = Throwables.getStackTraceAsString(err); - LOG.log(Level.SEVERE, String.format("Install error due to %s", errMsg), err); - return InstallResult.error(errorClassifier.fromErrorMessage(errMsg)); + isolatedExopackageInfo = + Optional.of(new IsolatedExopackageInfo(dexInfo, nativeLibsInfo, resourcesInfo)); } + return isolatedExopackageInfo; } - private AndroidArtifacts getOrMakeAndroidArtifacts(InstallId install_id) { - synchronized (installIdToFutureMap) { - return installIdToFutureMap.computeIfAbsent(install_id, ignore -> new AndroidArtifacts()); - } + /** + * Fixes the devices this install targets, as soon as there are options to reach them with. + * + *

Pinned here rather than at install time so that everything the install does reaches one set + * of devices. A device connected after this point is not installed to; one that disconnects + * before the install fails it, rather than being dropped silently. + */ + private AdbHelper resolveDevices(InstallId installId, AndroidArtifacts androidArtifacts) { + InstallState state = installState(installId); + // Per install, not process wide: resolution asks adb, and two installs arriving together have + // no reason to wait for each other. + return state.resolveDevices( + () -> { + AdbHelper adbHelper = + AdbHelperFactory.create( + LOG, + options, + androidArtifacts.getApkOptions(), + options.skipSetDebugApp ? SetDebugAppMode.SKIP : SetDebugAppMode.SET, + state.metrics()); + // Asking now is what fixes the set: the helper resolves its devices once, on + // first use, and answers from that for the rest of the install. Leaving it to + // whoever asks first would move the instant this install is pinned to. + ImmutableSet serials = + adbHelper.getDevices(true).stream() + .map(device -> device.getSerialNumber()) + .collect(ImmutableSet.toImmutableSet()); + LOG.log(Level.INFO, "Install targets " + serials); + return adbHelper; + }); + } + + private InstallState installState(InstallId installId) { + return installs.computeIfAbsent(installId, ignore -> new InstallState()); } - private InstallResult checkAbiCompatibility(AbsPath apkAbiPath) { + /** + * The ABIs the apk carries native code for, from the cpu filters buck sent. + * + *

Empty when the filters name nothing this installer recognises, or cannot be read at all. + * Either way the check has nothing to compare and the install goes ahead -- an ABI we cannot + * determine is not treated as incompatible. + */ + private static ImmutableSet readApkAbis(Path cpuFiltersPath) { try { - Set apkAbis = - Files.readAllLines(apkAbiPath.getPath()).stream() - .map(abi -> SHORT_TO_FULL_ABI_MAP.get(abi.trim())) - .collect(Collectors.toSet()); - Set deviceAbis = getDeviceAbis(); - if (deviceAbis.isEmpty()) { - // we couldn't determine the device ABI, so returning success. - return InstallResult.success(); - } - Set commonAbis = - apkAbis.stream().filter(deviceAbis::contains).collect(Collectors.toSet()); - if (commonAbis.isEmpty()) { - String message = - String.format( - "You are trying to install an APK with incompatible native libraries - " - + "the APK has native libraries built for %s, but the device has CPU %s.", - String.join(",", apkAbis), String.join(",", deviceAbis)); - return InstallResult.error( - new InstallError(message, AndroidInstallErrorTag.INCOMPATIBLE_NATIVE_LIB)); - } + // A filter this installer does not know is dropped rather than carried through as null: it + // would never match a device ABI, and would reach the user as the word "null". + return Files.readAllLines(cpuFiltersPath).stream() + .map(abi -> SHORT_TO_FULL_ABI_MAP.get(abi.trim())) + .filter(Objects::nonNull) + .collect(ImmutableSet.toImmutableSet()); } catch (IOException e) { - // unable to determine if we should fail early. no-op + LOG.log(Level.WARNING, "Could not read the apk cpu filters; skipping the ABI check", e); + return ImmutableSet.of(); } - return InstallResult.success(); } - private Set getDeviceAbis() { - // Unless serial is specified, - // use first device from -d or -e flag - Set abis = new HashSet<>(); - String serialNumber = options.serialNumber; - if (serialNumber == null) { - try { - String command = "adb devices"; - Process process = Runtime.getRuntime().exec(command); - BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream())); - String line; - while ((line = reader.readLine()) != null) { - String[] output = line.split("\t"); - String currentSerialNumber = output[0].trim(); - if (output.length == 2 && output[1].trim().equals("device")) { - if (!options.useEmulatorsOnlyMode && !options.useRealDevicesOnlyMode) { - serialNumber = currentSerialNumber; - break; - } - if (options.useEmulatorsOnlyMode && currentSerialNumber.startsWith("emulator")) { - serialNumber = currentSerialNumber; - break; - } - if (options.useRealDevicesOnlyMode && !currentSerialNumber.startsWith("emulator")) { - serialNumber = currentSerialNumber; - break; - } - } - } - } catch (Exception _e) { - // no op - } + /** + * Fails early when the apk has no native code the targeted devices can run. + * + *

Offered every arrival and answers for itself: it needs the cpu filters and the devices, + * which turn up in either order, and does nothing until both are there. Empty means compatible, + * or that no device would report an ABI -- unknown is not treated as incompatible. + */ + private Optional checkAbiCompatibility(InstallId installId) { + InstallState state = installState(installId); + ImmutableSet apkAbis = state.artifacts().getApkAbis(); + AdbHelper adbHelper = state.adbHelper(); + if (apkAbis == null || adbHelper == null) { + return Optional.empty(); } - - try { - // get default abi - String abiCommand = String.format("adb -s %s shell getprop ro.product.cpu.abi", serialNumber); - Process abiProcess = Runtime.getRuntime().exec(abiCommand); - BufferedReader abiReader = - new BufferedReader(new InputStreamReader(abiProcess.getInputStream())); - String abi = abiReader.readLine(); - if (abi != null) { - abis.add(abi.trim()); - } - } catch (Exception _e) { - // no op + if (apkAbis.isEmpty()) { + // No native code this installer recognises, so there is nothing a device could be wrong for. + return Optional.empty(); } - - if (!Sets.intersection(abis, ABI_64s).isEmpty()) { - // Need to call both ro.product.cpu.abi and ro.product.cpu.abilist - // as sticking to ro.product.cpu.abi helped fixing the issue of - // exopackage install when the app was already installed in the device. - try { - // add 32bit abi if device supports it - String abiCommand = - String.format("adb -s %s shell getprop ro.product.cpu.abilist", serialNumber); - Process abiProcess = Runtime.getRuntime().exec(abiCommand); - BufferedReader abiReader = - new BufferedReader(new InputStreamReader(abiProcess.getInputStream())); - Set abiList = - Arrays.stream(abiReader.readLine().split(",")) - .map(String::trim) - .collect(Collectors.toSet()); - abis.addAll(abiList); - } catch (Exception _e) { - // no op - } + ImmutableMap> abisByDevice = adbHelper.deviceAbisBySerial(); + if (abisByDevice.isEmpty()) { + return Optional.empty(); } - - return abis; + // Every device, not their union: an install reaches all of them, so one that cannot run the + // apk fails it, and failing here is the point -- the alternative is finding out after the + // build. + ImmutableList incompatible = + abisByDevice.entrySet().stream() + .filter(device -> Sets.intersection(apkAbis, device.getValue()).isEmpty()) + .map(device -> device.getKey() + " (" + String.join(",", device.getValue()) + ")") + .sorted() + .collect(ImmutableList.toImmutableList()); + if (incompatible.isEmpty()) { + return Optional.empty(); + } + return Optional.of( + new InstallError( + String.format( + "You are trying to install an APK with native libraries built for %s, onto " + + "device(s) that cannot run any of them: %s.", + String.join(",", apkAbis), String.join("; ", incompatible)), + AndroidInstallErrorTag.INCOMPATIBLE_NATIVE_LIB)); } } diff --git a/prelude/toolchains/android/src/com/facebook/buck/installer/android/InstallMetrics.java b/prelude/toolchains/android/src/com/facebook/buck/installer/android/InstallMetrics.java new file mode 100644 index 0000000..35ae022 --- /dev/null +++ b/prelude/toolchains/android/src/com/facebook/buck/installer/android/InstallMetrics.java @@ -0,0 +1,242 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is dual-licensed under either the MIT license found in the + * LICENSE-MIT file in the root directory of this source tree or the Apache + * License, Version 2.0 found in the LICENSE-APACHE file in the root directory + * of this source tree. You may select, at your option, one of the + * above-listed licenses. + */ + +package com.facebook.buck.installer.android; + +import com.facebook.buck.android.exopackage.InstallTimings; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.EnumMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * How long each stage of one install took, and what that says about the time it spent waiting. + * + *

Written from the push threads and read once the install is over, so every member is guarded by + * this object's monitor. Arrival times live with the artifacts that arrived; {@link #summarise} + * takes them as an argument rather than tracking them twice. + */ +final class InstallMetrics implements InstallTimings { + // Stage timings, written from the push threads and read once the install is over. + // Every shard's window, per push group. Kept separately rather than spanned: shards of a group + // overlap each other and queue behind other groups, so only the union of these is time the group + // was actually transferring. + private final Map> pushWindows = new LinkedHashMap<>(); + private long deviceSetupMillis; + private long apkInstallMillis; + private long deviceWorkMillis; + private long deviceWorkStartMillis; + + @Override + public synchronized void recordDeviceSetup(long startMillis, long endMillis) { + deviceSetupMillis += endMillis - startMillis; + } + + @Override + public synchronized void recordPush(String group, long startMillis, long endMillis) { + pushWindows + .computeIfAbsent(group, unused -> new ArrayList<>()) + .add(new long[] {startMillis, endMillis}); + } + + @Override + public synchronized void recordApkInstall(long startMillis, long endMillis) { + apkInstallMillis += endMillis - startMillis; + } + + @Override + public synchronized void recordDeviceWork(long startMillis, long endMillis) { + deviceWorkStartMillis = + deviceWorkStartMillis == 0L ? startMillis : Math.min(deviceWorkStartMillis, startMillis); + deviceWorkMillis += endMillis - startMillis; + } + + /** + * How the install went, as one timeline. Every value is seconds.milliseconds, measured from the + * first artifact arriving. + * + *

Each class of artifact reports when buck finished delivering it ({@code _arrival_s}) and how + * long moving it to the device took ({@code _transfer_s}), in arrival order. Control artifacts + * are read on the host, so they never transfer. + * + *

{@code critical_path_s} replays this same install against these same durations, but starts + * each payload the moment its own artifacts were there instead of waiting for all of them. {@code + * potential_saving_s} is {@code total_s} minus that: time the device spent waiting for an + * artifact it did not yet need. Neither says anything about transferring faster, only about + * waiting less. + */ + public synchronized Map summarise( + long installCompleteMillis, Map fileArrivalMillis) { + if (fileArrivalMillis.isEmpty()) { + return Map.of(); + } + long first = Collections.min(fileArrivalMillis.values()); + + Map readyByClass = + new EnumMap<>(AndroidArtifacts.ArtifactClass.class); + for (Map.Entry arrival : fileArrivalMillis.entrySet()) { + readyByClass.merge( + AndroidArtifacts.ArtifactClass.of(arrival.getKey()), arrival.getValue(), Math::max); + } + long controlReady = readyByClass.getOrDefault(AndroidArtifacts.ArtifactClass.CONTROL, first); + long apkReady = readyByClass.getOrDefault(AndroidArtifacts.ArtifactClass.APK, first); + + // Replay: one pusher, each payload started as soon as it was available. + List payloads = new ArrayList<>(); // {availableAt, durationMillis} + long metadataMillis = 0L; + for (Map.Entry> push : pushWindows.entrySet()) { + long duration = unionMillis(push.getValue()); + AndroidArtifacts.ArtifactClass pushed = + AndroidArtifacts.ArtifactClass.forPushGroup(push.getKey()); + if (pushed == null) { + metadataMillis += duration; // metadata is derived, so it can only follow every payload + continue; + } + long availableAt = readyByClass.getOrDefault(pushed, controlReady); + payloads.add(new long[] {Math.max(availableAt, controlReady), duration}); + } + payloads.sort(Comparator.comparingLong(payload -> payload[0])); + + // Replay onto as many pushers as the install actually used, so the estimate tracks the + // implementation. With one pusher this is a queue; with several it is a list schedule. + long[] pushers = new long[observedConcurrency()]; + Arrays.fill(pushers, controlReady + deviceSetupMillis); + for (long[] payload : payloads) { + int earliest = 0; + for (int i = 1; i < pushers.length; i++) { + if (pushers[i] < pushers[earliest]) { + earliest = i; + } + } + pushers[earliest] = Math.max(pushers[earliest], payload[0]) + payload[1]; + } + long clock = controlReady + deviceSetupMillis; + for (long pusher : pushers) { + clock = Math.max(clock, pusher); + } + clock += metadataMillis; + clock = Math.max(clock, apkReady) + apkInstallMillis; + // Device work that is not one of the stages above still has to happen, so charge it too. + // Without this the replay looks faster than anything achievable and every install appears to + // have a saving. + long modelled = deviceSetupMillis + apkInstallMillis; + for (List windows : pushWindows.values()) { + // Only the pushes the install itself made. A streamed push finished before the device phase + // began, so it is not part of deviceWorkMillis, and subtracting it would cancel the whole + // correction out for any install that streamed anything. + modelled += + unionMillis( + windows.stream() + .filter(window -> window[0] >= deviceWorkStartMillis) + .collect(Collectors.toList())); + } + clock += Math.max(0L, deviceWorkMillis - modelled); + + long total = installCompleteMillis - first; + long criticalPath = clock - first; + + Map metrics = new LinkedHashMap<>(); + readyByClass.entrySet().stream() + .sorted(Map.Entry.comparingByValue()) + .forEach( + ready -> { + String artifactClass = ready.getKey().metricName; + metrics.put(artifactClass + "_arrival_s", seconds(ready.getValue() - first)); + metrics.put(artifactClass + "_transfer_s", seconds(transferMillis(ready.getKey()))); + }); + metrics.put("device_setup_s", seconds(deviceSetupMillis)); + if (metadataMillis > 0L) { + metrics.put("metadata_transfer_s", seconds(metadataMillis)); + } + metrics.put("critical_path_s", seconds(criticalPath)); + metrics.put("total_s", seconds(total)); + metrics.put("potential_saving_s", seconds(Math.max(0L, total - criticalPath))); + return metrics; + } + + /** How long this class of artifact took to reach the device, or zero if it is not transferred. */ + private long transferMillis(AndroidArtifacts.ArtifactClass artifactClass) { + if (artifactClass == AndroidArtifacts.ArtifactClass.APK) { + return apkInstallMillis; + } + for (Map.Entry> push : pushWindows.entrySet()) { + if (AndroidArtifacts.ArtifactClass.forPushGroup(push.getKey()) == artifactClass) { + return unionMillis(push.getValue()); + } + } + return 0L; + } + + /** + * Time a group spent transferring: the union of its shards' windows. Overlapping shards count + * once, and a gap where every shard was queued behind another group counts for nothing -- the + * span between the first start and the last end would charge the group for both. + */ + private static long unionMillis(List windows) { + long total = 0L; + for (long[] window : coalesce(windows)) { + total += window[1] - window[0]; + } + return total; + } + + /** The windows in order, with overlapping ones fused so each instant appears once. */ + private static List coalesce(List windows) { + List sorted = new ArrayList<>(windows); + sorted.sort(Comparator.comparingLong(window -> window[0])); + List fused = new ArrayList<>(); + for (long[] window : sorted) { + long[] open = fused.isEmpty() ? null : fused.get(fused.size() - 1); + if (open != null && window[0] <= open[1]) { + open[1] = Math.max(open[1], window[1]); + } else { + fused.add(new long[] {window[0], window[1]}); + } + } + return fused; + } + + private static String seconds(long millis) { + return String.format(Locale.ROOT, "%.3f", millis / 1000.0); + } + + /** + * How many payloads the install had in flight at once, from their observed windows. One if the + * pushes did not overlap. + */ + private int observedConcurrency() { + List edges = new ArrayList<>(); + for (List windows : pushWindows.values()) { + // Fused first: a group's own shards overlapping is not two payloads in flight. + for (long[] window : coalesce(windows)) { + if (window[1] > window[0]) { + edges.add(new long[] {window[0], 1L}); + edges.add(new long[] {window[1], -1L}); + } + } + } + // Ends before starts at the same instant, so touching windows are not counted as overlapping. + edges.sort( + Comparator.comparingLong(edge -> edge[0]).thenComparingLong(edge -> edge[1])); + int concurrent = 0; + int peak = 0; + for (long[] edge : edges) { + concurrent += (int) edge[1]; + peak = Math.max(peak, concurrent); + } + return Math.max(1, peak); + } +} diff --git a/prelude/toolchains/android/src/com/facebook/buck/installer/android/InstallState.java b/prelude/toolchains/android/src/com/facebook/buck/installer/android/InstallState.java new file mode 100644 index 0000000..78f4946 --- /dev/null +++ b/prelude/toolchains/android/src/com/facebook/buck/installer/android/InstallState.java @@ -0,0 +1,76 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is dual-licensed under either the MIT license found in the + * LICENSE-MIT file in the root directory of this source tree or the Apache + * License, Version 2.0 found in the LICENSE-APACHE file in the root directory + * of this source tree. You may select, at your option, one of the + * above-listed licenses. + */ + +package com.facebook.buck.installer.android; + +import com.facebook.buck.android.AdbHelper; +import java.util.function.Supplier; +import javax.annotation.Nullable; + +/** + * Everything one install accumulates, so that something new to track is a field here rather than + * another map keyed by install id. + */ +final class InstallState { + private final AndroidArtifacts artifacts = new AndroidArtifacts(); + private final InstallMetrics metrics = new InstallMetrics(); + private final StreamedPushes streamedPushes = new StreamedPushes(); + + // Written once, under this object's monitor; read without it, from the threads delivering + // artifacts. Those reads happen on every arrival, so they must not queue behind a resolution + // that is talking to adb. + @Nullable private volatile AdbHelper adbHelper; + + // Read when the manifest arrives, because the manifest does not change during an install and + // several stages of one ask for the package. + @Nullable private volatile String packageName; + + AndroidArtifacts artifacts() { + return artifacts; + } + + InstallMetrics metrics() { + return metrics; + } + + /** The package this install is for, or null until the manifest has arrived. */ + @Nullable + String packageName() { + return packageName; + } + + void setPackageName(String packageName) { + this.packageName = packageName; + } + + /** What has already been sent ahead of this install, so no payload is sent twice. */ + StreamedPushes streamedPushes() { + return streamedPushes; + } + + /** Bound to the devices this install targets, or null until its options have arrived. */ + @Nullable + AdbHelper adbHelper() { + return adbHelper; + } + + /** + * Resolves the devices if they are not resolved already, and answers with them. + * + *

Resolution happens once however many arrivals ask for it: the answer is what pins this + * install to a set of devices, so a second resolution would move that instant. + */ + synchronized AdbHelper resolveDevices(Supplier resolver) { + if (adbHelper == null) { + adbHelper = resolver.get(); + } + return adbHelper; + } +} diff --git a/prelude/toolchains/android/src/com/facebook/buck/installer/android/StreamedPushes.java b/prelude/toolchains/android/src/com/facebook/buck/installer/android/StreamedPushes.java new file mode 100644 index 0000000..c370bc8 --- /dev/null +++ b/prelude/toolchains/android/src/com/facebook/buck/installer/android/StreamedPushes.java @@ -0,0 +1,107 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is dual-licensed under either the MIT license found in the + * LICENSE-MIT file in the root directory of this source tree or the Apache + * License, Version 2.0 found in the LICENSE-APACHE file in the root directory + * of this source tree. You may select, at your option, one of the + * above-listed licenses. + */ + +package com.facebook.buck.installer.android; + +import com.google.common.annotations.VisibleForTesting; +import java.util.ArrayList; +import java.util.EnumSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.function.Function; +import java.util.logging.Level; +import java.util.logging.Logger; // NOPMD + +/** + * The payloads sent ahead of an install, and the pushes carrying them. + * + *

Guards itself. Claiming a payload and recording the push that carries it have to happen + * together, so that pair is one method rather than a contract callers are trusted to keep, and the + * pushes are waited on and stopped from here rather than handed out. + */ +final class StreamedPushes { + private static final Logger LOG = Logger.getLogger(StreamedPushes.class.getName()); + + private final Set dispatched = + EnumSet.noneOf(AndroidArtifacts.ArtifactClass.class); + private final List> pushes = new ArrayList<>(); + private boolean sealed; + + /** + * Sends whatever of {@code ready} has not gone yet, and records the push. Does nothing when + * everything ready has already gone, or once the install has sealed them. + */ + synchronized void dispatch( + Set ready, + Function, Future> push) { + if (sealed) { + return; + } + Set pending = + EnumSet.noneOf(AndroidArtifacts.ArtifactClass.class); + pending.addAll(ready); + pending.removeAll(dispatched); + if (pending.isEmpty()) { + return; + } + // Recorded only once the push exists: a payload marked sent with nothing carrying it would + // never be offered again. + Future started = push.apply(pending); + dispatched.addAll(pending); + pushes.add(started); + } + + /** + * Hands the pushes to the install and waits for them, so nothing is still writing into the + * directories it is about to. Sealing and waiting are one method because waiting without sealing + * first lets a dispatch slip in behind the wait. + */ + void sealAndAwait() throws InterruptedException { + seal(); + await(); + } + + private synchronized void seal() { + sealed = true; + } + + /** Waits for every push to finish. A push that failed is left for the install to redo. */ + private void await() throws InterruptedException { + List> streaming = started(); + for (Future streamed : streaming) { + try { + streamed.get(); + } catch (InterruptedException e) { + // Interrupting does not stop a push: the pool hands each task straight to a thread, which + // then blocks on adb's monitor and inside a subprocess read, neither of which is + // interruptible. So the install gives up rather than pretending the device is quiet. + Thread.currentThread().interrupt(); + throw e; + } catch (ExecutionException e) { + LOG.log(Level.WARNING, "A streamed push failed; the install will push instead", e); + } + } + } + + /** + * The pushes so far. Copied under the monitor and used outside it: waiting on a push while + * holding it would block the arrivals that are still trying to dispatch. + */ + private synchronized List> started() { + return List.copyOf(pushes); + } + + @VisibleForTesting + synchronized int pushCount() { + return pushes.size(); + } +} diff --git a/prelude/toolchains/android/src/com/facebook/buck/installer/proto/install.proto b/prelude/toolchains/android/src/com/facebook/buck/installer/proto/install.proto index 1ecddf5..2d84cd0 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/installer/proto/install.proto +++ b/prelude/toolchains/android/src/com/facebook/buck/installer/proto/install.proto @@ -46,6 +46,10 @@ message InstallInfoRequest { message InstallResponse { string install_id = 1; + // Names the installer implementation, e.g. `android`. Reported at the + // handshake rather than alongside a device, so an install that reaches the + // installer says which one ran it even when it resolves no device. + string installer_name = 2; } message FileReadyRequest { diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/cd/DepFileUtils.java b/prelude/toolchains/android/src/com/facebook/buck/jvm/cd/DepFileUtils.java index 62acd95..83fcce3 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/jvm/cd/DepFileUtils.java +++ b/prelude/toolchains/android/src/com/facebook/buck/jvm/cd/DepFileUtils.java @@ -24,7 +24,6 @@ import java.util.Comparator; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Set; import java.util.TreeMap; import java.util.stream.Collectors; @@ -36,22 +35,9 @@ public class DepFileUtils { public static void usedClassesToDepFile( List usedClassesMapPaths, Path depFileOutput, - Optional jarToJarDirMapPath, + Map jarToJarDirMap, boolean append) throws IOException { - ImmutableMap jarToJarDirMap; - if (jarToJarDirMapPath.isPresent()) { - jarToJarDirMap = - Files.readAllLines(jarToJarDirMapPath.get()).stream() - // TODO(ianc) fix this, we shouldn't be adding the same jar to the classpath multiple - // times - .distinct() - .map(line -> line.split(" ")) - .collect(ImmutableMap.toImmutableMap(x -> Paths.get(x[0]), x -> Paths.get(x[1]))); - } else { - jarToJarDirMap = ImmutableMap.of(); - } - List allUsedPaths = new ArrayList<>(); for (Path usedClassesMapPath : usedClassesMapPaths) { ImmutableMap> usedClassesMap = diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/cd/command/BaseJarCommand.kt b/prelude/toolchains/android/src/com/facebook/buck/jvm/cd/command/BaseJarCommand.kt index df41b0e..5792d05 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/jvm/cd/command/BaseJarCommand.kt +++ b/prelude/toolchains/android/src/com/facebook/buck/jvm/cd/command/BaseJarCommand.kt @@ -20,15 +20,17 @@ import com.facebook.buck.jvm.cd.serialization.java.BuildTargetValueSerializer import com.facebook.buck.jvm.cd.serialization.java.CompilerOutputPathsValueSerializer import com.facebook.buck.jvm.cd.serialization.java.JarParametersSerializer import com.facebook.buck.jvm.cd.serialization.java.ResolvedJavacOptionsSerializer -import com.facebook.buck.jvm.cd.serialization.java.ResolvedJavacSerializer import com.facebook.buck.jvm.core.BuildTargetValue import com.facebook.buck.jvm.java.CompilerOutputPathsValue import com.facebook.buck.jvm.java.JarParameters +import com.facebook.buck.jvm.java.JdkProvidedInMemoryJavac import com.facebook.buck.jvm.java.ResolvedJavac import com.facebook.buck.jvm.java.ResolvedJavacOptions import com.google.common.collect.ImmutableList import com.google.common.collect.ImmutableMap import com.google.common.collect.ImmutableSortedSet +import java.nio.file.Path +import java.nio.file.Paths import java.util.Optional class BaseJarCommand( @@ -47,31 +49,82 @@ class BaseJarCommand( val resolvedJavacOptions: ResolvedJavacOptions, val buildTargetValue: BuildTargetValue, val buckOut: RelPath, - val pathToClasses: RelPath?, val annotationPath: RelPath?, + val jarToJarDirMap: ImmutableMap, ) { companion object { fun fromProto(model: ProtoBaseJarCommand, scratchDir: Optional): BaseJarCommand { + val paths = model.compileTimeClasspathAbiAndDirPathsList + require(paths.size % 2 == 0) { "Expected alternating ABI jar and directory paths" } + require(paths.isEmpty() || model.compileTimeClasspathPathsCount == 0) { + "Specify either compileTimeClasspathPaths or compileTimeClasspathAbiAndDirPaths" + } + val classpath = ImmutableList.builder() + val jarToJarDirMap = mutableMapOf() + for (i in paths.indices step 2) { + val jar = RelPath.get(paths[i]) + classpath.add(jar) + if (paths[i + 1].isNotEmpty()) { + val dir = Paths.get(paths[i + 1]) + // TODO(ianc) fix this, we shouldn't be adding the same jar to the classpath multiple + // times + val previous = jarToJarDirMap.putIfAbsent(jar.path, dir) + require(previous == null || previous == dir) { "Conflicting ABI directories for $jar" } + } + } + if (paths.isEmpty()) { + classpath.addAll(RelPathSerializer.toListOfRelPath(model.compileTimeClasspathPathsList)) + } + val buildTarget = BuildTargetValueSerializer.deserialize(model.buildTargetValue) + val outputPathsValue = + CompilerOutputPathsValueSerializer.deserialize(model.outputPathsValue, scratchDir) + val outputPaths = + when { + buildTarget.isSourceOnlyAbi -> outputPathsValue.sourceOnlyAbiCompilerOutputPath + buildTarget.isSourceAbi -> outputPathsValue.sourceAbiCompilerOutputPath + else -> outputPathsValue.libraryCompilerOutputPath + } + val resources = RelPathSerializer.toResourceMap(model.resourcesMapList) + val jarParameters = + if (model.hasJarParameters()) { + val parameters = JarParametersSerializer.deserialize(model.jarParameters) + if (parameters.entriesToJar.isEmpty()) { + parameters.copy( + entriesToJar = + ImmutableSortedSet.orderedBy(RelPath.comparator()) + .add(outputPaths.classesDir) + .build(), + ) + } else { + parameters + } + } else { + null + } return BaseJarCommand( - model.abiCompatibilityMode, + AbiGenerationMode.CLASS, model.abiGenerationMode, model.trackClassUsage, model.trackClassUsage, - CompilerOutputPathsValueSerializer.deserialize(model.outputPathsValue, scratchDir), - RelPathSerializer.toListOfRelPath(model.compileTimeClasspathPathsList), + outputPathsValue, + classpath.build(), RelPathSerializer.toListOfRelPath(model.compileTimeClasspathSnapshotPathsList), RelPathSerializer.toSortedSetOfRelPath(model.getJavaSrcsList()), - RelPathSerializer.toResourceMap(model.resourcesMapList), - if (model.hasJarParameters()) JarParametersSerializer.deserialize(model.jarParameters) - else null, + ImmutableMap.copyOf( + resources.mapValues { (_, destination) -> + outputPaths.classesDir.resolveRel(destination.toString()) + }, + ), + jarParameters, AbsPathSerializer.deserialize(""), - ResolvedJavacSerializer.deserialize(model.resolvedJavac), + JdkProvidedInMemoryJavac.createJsr199Javac(), ResolvedJavacOptionsSerializer.deserialize(model.resolvedJavacOptions), - BuildTargetValueSerializer.deserialize(model.buildTargetValue), - RelPathSerializer.deserialize(model.configuredBuckOut), - RelPathSerializer.deserialize(model.pathToClasses), - RelPathSerializer.deserialize(model.annotationsPath), + buildTarget, + RelPath.get("buck-out/v2"), + if (model.annotationsPath.isEmpty()) scratchDir.get().resolveRel("__gen__") + else RelPathSerializer.deserialize(model.annotationsPath), + ImmutableMap.copyOf(jarToJarDirMap), ) } } diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/cd/command/PostBuildParams.kt b/prelude/toolchains/android/src/com/facebook/buck/jvm/cd/command/PostBuildParams.kt index dac3602..2a8a900 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/jvm/cd/command/PostBuildParams.kt +++ b/prelude/toolchains/android/src/com/facebook/buck/jvm/cd/command/PostBuildParams.kt @@ -21,7 +21,6 @@ class PostBuildParams( val abiOutputDir: Path?, val usedClassesPaths: List, val depFile: Path?, - val jarToJarDirMap: Path?, val optionalDirsPaths: List, val incrementalStateDir: Path?, val shouldCreateClassAbi: Boolean, @@ -38,7 +37,6 @@ class PostBuildParams( model.abiOutputDir.takeIf { it.isNotEmpty() }?.let(Paths::get), model.usedClassesList?.map(Paths::get).orEmpty(), model.depFile.takeIf { it.isNotEmpty() }?.let(Paths::get), - model.jarToJarDirMap.takeIf { it.isNotEmpty() }?.let(Paths::get), model.optionalDirsList?.map(Paths::get).orEmpty(), model.incrementalStateDir.takeIf { it.isNotEmpty() }?.let(Paths::get), model.shouldCreateClassAbi, diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/cd/command/kotlin/BUCK b/prelude/toolchains/android/src/com/facebook/buck/jvm/cd/command/kotlin/BUCK index 7d4a830..b98beb8 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/jvm/cd/command/kotlin/BUCK +++ b/prelude/toolchains/android/src/com/facebook/buck/jvm/cd/command/kotlin/BUCK @@ -15,7 +15,6 @@ buck_kotlin_library( buck_kotlin_library( name = "support", srcs = [ - "AnnotationProcessingTool.kt", "KotlinExtraParams.kt", ], labels = ["codegen_rule"], diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/cd/command/kotlin/BuildKotlinCommand.kt b/prelude/toolchains/android/src/com/facebook/buck/jvm/cd/command/kotlin/BuildKotlinCommand.kt index e9748b6..e4f3033 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/jvm/cd/command/kotlin/BuildKotlinCommand.kt +++ b/prelude/toolchains/android/src/com/facebook/buck/jvm/cd/command/kotlin/BuildKotlinCommand.kt @@ -33,6 +33,7 @@ class BuildKotlinCommand( KotlinExtraParamsSerializer.deserialize( model.baseJarCommand.resolvedJavacOptions, model.kotlinExtraParams, + scratchDir, ) return BuildKotlinCommand(kotlinExtraParams, baseJarCommand, buildMode) diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/cd/command/kotlin/KotlinExtraParams.kt b/prelude/toolchains/android/src/com/facebook/buck/jvm/cd/command/kotlin/KotlinExtraParams.kt index b4af782..55312b4 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/jvm/cd/command/kotlin/KotlinExtraParams.kt +++ b/prelude/toolchains/android/src/com/facebook/buck/jvm/cd/command/kotlin/KotlinExtraParams.kt @@ -23,7 +23,6 @@ data class KotlinExtraParams( val extraClassPathSnapshots: ImmutableList, val standardLibraryClassPath: AbsPath, val annotationProcessingClassPath: AbsPath, - val annotationProcessingTool: AnnotationProcessingTool, val extraKotlincArguments: ImmutableList, val kotlinCompilerPlugins: ImmutableMap>, val kosabiPluginOptions: ImmutableMap, diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/cd/serialization/java/CompilerOutputPathsSerializer.java b/prelude/toolchains/android/src/com/facebook/buck/jvm/cd/serialization/java/CompilerOutputPathsSerializer.java index 0affd4c..73fbf72 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/jvm/cd/serialization/java/CompilerOutputPathsSerializer.java +++ b/prelude/toolchains/android/src/com/facebook/buck/jvm/cd/serialization/java/CompilerOutputPathsSerializer.java @@ -34,10 +34,11 @@ public static CompilerOutputPaths deserialize(OutputPathsValue.OutputPaths outpu public static CompilerOutputPaths deserialize( OutputPathsValue.OutputPaths outputPaths, Optional tmpDir) { return new CompilerOutputPaths( - toRelPath(outputPaths.getClassesDir()), + tmpDir.get().resolveRel("__classes__"), toRelPath(outputPaths.getOutputJarDirPath()), toOptionalRelPath(outputPaths.getAbiJarPath()), - toRelPath(outputPaths.getAnnotationPath()), + toOptionalRelPath(outputPaths.getAnnotationPath()) + .orElseGet(() -> tmpDir.get().resolveRel("__gen__")), outputPaths.getPathToSourcesList().isEmpty() ? tmpDir.map(p -> p.resolveRel("__srcs__")).get() : toRelPath(outputPaths.getPathToSourcesList()), diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/cd/serialization/java/JarParametersSerializer.java b/prelude/toolchains/android/src/com/facebook/buck/jvm/cd/serialization/java/JarParametersSerializer.java index 43ef03a..751fbdf 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/jvm/cd/serialization/java/JarParametersSerializer.java +++ b/prelude/toolchains/android/src/com/facebook/buck/jvm/cd/serialization/java/JarParametersSerializer.java @@ -16,6 +16,7 @@ import com.facebook.infer.annotation.Nullsafe; import com.google.common.collect.ImmutableSortedSet; import java.util.Optional; +import java.util.logging.Level; /** {@link JarParameters} to protobuf serializer */ @Nullsafe(Nullsafe.Mode.LOCAL) @@ -49,6 +50,6 @@ public static JarParameters deserialize( .collect(ImmutableSortedSet.toImmutableSortedSet(RelPath.comparator())), mainClass, manifestFile, - LogLevelSerializer.deserialize(jarParameters.getDuplicatesLogLevel())); + Level.FINE); } } diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/cd/serialization/java/LogLevelSerializer.java b/prelude/toolchains/android/src/com/facebook/buck/jvm/cd/serialization/java/LogLevelSerializer.java deleted file mode 100644 index e962b29..0000000 --- a/prelude/toolchains/android/src/com/facebook/buck/jvm/cd/serialization/java/LogLevelSerializer.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright (c) Meta Platforms, Inc. and affiliates. - * - * This source code is dual-licensed under either the MIT license found in the - * LICENSE-MIT file in the root directory of this source tree or the Apache - * License, Version 2.0 found in the LICENSE-APACHE file in the root directory - * of this source tree. You may select, at your option, one of the - * above-listed licenses. - */ - -package com.facebook.buck.jvm.cd.serialization.java; - -import com.facebook.buck.cd.model.java.JarParameters; -import com.facebook.buck.jvm.cd.serialization.SerializationUtil; -import com.facebook.infer.annotation.Nullsafe; -import java.util.logging.Level; - -/** {@link Level} to protobuf serializer */ -@Nullsafe(Nullsafe.Mode.LOCAL) -class LogLevelSerializer { - - private LogLevelSerializer() {} - - /** Deserializes javacd model's {@link JarParameters.LogLevel} into {@link Level}. */ - public static Level deserialize(JarParameters.LogLevel level) { - switch (level) { - case ALL: - return Level.ALL; - case OFF: - return Level.OFF; - case CONFIG: - return Level.CONFIG; - - case SEVERE: - return Level.SEVERE; - case WARNING: - return Level.WARNING; - case INFO: - return Level.INFO; - - case FINE: - return Level.FINE; - case FINER: - return Level.FINER; - case FINEST: - return Level.FINEST; - - case UNRECOGNIZED: - case UNKNOWN: - default: - throw SerializationUtil.createNotSupportedException(level); - } - } -} diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/cd/serialization/java/ResolvedJavacOptionsSerializer.java b/prelude/toolchains/android/src/com/facebook/buck/jvm/cd/serialization/java/ResolvedJavacOptionsSerializer.java index 9dd17b3..61ca9b2 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/jvm/cd/serialization/java/ResolvedJavacOptionsSerializer.java +++ b/prelude/toolchains/android/src/com/facebook/buck/jvm/cd/serialization/java/ResolvedJavacOptionsSerializer.java @@ -38,7 +38,7 @@ public static ResolvedJavacOptions deserialize( return new ResolvedJavacOptions( bootclasspathList, JavacLanguageLevelOptionsSerializer.deserialize(options.getLanguageLevelOptions()), - options.getDebug(), + true, options.getVerbose(), JavacPluginParamsSerializer.deserialize(options.getJavaAnnotationProcessorParams()), JavacPluginParamsSerializer.deserialize(options.getStandardJavacPluginParams()), diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/cd/serialization/java/ResolvedJavacPluginPropertiesSerializer.java b/prelude/toolchains/android/src/com/facebook/buck/jvm/cd/serialization/java/ResolvedJavacPluginPropertiesSerializer.java index 30a3976..f1d91ed 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/jvm/cd/serialization/java/ResolvedJavacPluginPropertiesSerializer.java +++ b/prelude/toolchains/android/src/com/facebook/buck/jvm/cd/serialization/java/ResolvedJavacPluginPropertiesSerializer.java @@ -11,14 +11,12 @@ package com.facebook.buck.jvm.cd.serialization.java; import com.facebook.buck.cd.model.java.ResolvedJavacOptions; -import com.facebook.buck.core.filesystems.RelPath; import com.facebook.buck.jvm.cd.serialization.RelPathSerializer; import com.facebook.buck.jvm.java.ResolvedJavacPluginProperties; import com.facebook.infer.annotation.Nullsafe; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSortedSet; -import java.util.Map; /** {@link ResolvedJavacPluginProperties} to protobuf serializer */ @Nullsafe(Nullsafe.Mode.LOCAL) @@ -41,16 +39,7 @@ public static ResolvedJavacPluginProperties deserialize( pluginProperties.getClasspathList().stream() .map(RelPathSerializer::deserialize) .collect(ImmutableList.toImmutableList()), - toPathParams(pluginProperties.getPathParamsMap()), + ImmutableMap.of(), ImmutableList.copyOf(pluginProperties.getArgumentsList())); } - - private static ImmutableMap toPathParams(Map pathParamsMap) { - ImmutableMap.Builder pathParamsBuilder = - ImmutableMap.builderWithExpectedSize(pathParamsMap.size()); - for (Map.Entry entry : pathParamsMap.entrySet()) { - pathParamsBuilder.put(entry.getKey(), RelPathSerializer.deserialize(entry.getValue())); - } - return pathParamsBuilder.build(); - } } diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/cd/serialization/java/ResolvedJavacSerializer.java b/prelude/toolchains/android/src/com/facebook/buck/jvm/cd/serialization/java/ResolvedJavacSerializer.java deleted file mode 100644 index abfb0fa..0000000 --- a/prelude/toolchains/android/src/com/facebook/buck/jvm/cd/serialization/java/ResolvedJavacSerializer.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright (c) Meta Platforms, Inc. and affiliates. - * - * This source code is dual-licensed under either the MIT license found in the - * LICENSE-MIT file in the root directory of this source tree or the Apache - * License, Version 2.0 found in the LICENSE-APACHE file in the root directory - * of this source tree. You may select, at your option, one of the - * above-listed licenses. - */ - -package com.facebook.buck.jvm.cd.serialization.java; - -import com.facebook.buck.jvm.cd.serialization.SerializationUtil; -import com.facebook.buck.jvm.java.JdkProvidedInMemoryJavac; -import com.facebook.buck.jvm.java.ResolvedJavac; -import com.facebook.infer.annotation.Nullsafe; -import com.google.common.collect.ImmutableList; -import com.google.protobuf.ProtocolStringList; - -/** {@link ResolvedJavac} to protobuf serializer */ -@Nullsafe(Nullsafe.Mode.LOCAL) -public class ResolvedJavacSerializer { - - private ResolvedJavacSerializer() {} - - /** - * Deserializes javacd model's {@link com.facebook.buck.cd.model.java.ResolvedJavac} into {@link - * ResolvedJavac}. - */ - public static ResolvedJavac deserialize(com.facebook.buck.cd.model.java.ResolvedJavac javac) { - var javacCase = javac.getJavacCase(); - switch (javacCase) { - case EXTERNALJAVAC: - throw new IllegalArgumentException("ExternalJavac in CD worker is not supported"); - - case JSR199JAVAC: - return JdkProvidedInMemoryJavac.createJsr199Javac(); - - case JAVAC_NOT_SET: - default: - throw SerializationUtil.createNotSupportedException(javacCase); - } - } - - private static ImmutableList toImmutableList(ProtocolStringList list) { - return ImmutableList.copyOf(list); - } -} diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/cd/serialization/kotlin/AnnotationProcessingToolSerializer.java b/prelude/toolchains/android/src/com/facebook/buck/jvm/cd/serialization/kotlin/AnnotationProcessingToolSerializer.java deleted file mode 100644 index deffdee..0000000 --- a/prelude/toolchains/android/src/com/facebook/buck/jvm/cd/serialization/kotlin/AnnotationProcessingToolSerializer.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright (c) Meta Platforms, Inc. and affiliates. - * - * This source code is dual-licensed under either the MIT license found in the - * LICENSE-MIT file in the root directory of this source tree or the Apache - * License, Version 2.0 found in the LICENSE-APACHE file in the root directory - * of this source tree. You may select, at your option, one of the - * above-listed licenses. - */ - -package com.facebook.buck.jvm.cd.serialization.kotlin; - -import com.facebook.buck.jvm.cd.command.kotlin.AnnotationProcessingTool; -import com.facebook.infer.annotation.Nullsafe; - -/** - * Marshalling between: - * - *

    - *
  • {@link com.facebook.buck.jvm.cd.command.kotlin.AnnotationProcessingTool}, and - *
  • {@link com.facebook.buck.cd.model.kotlin.AnnotationProcessingTool} (part of the protocol - * buffer model). - *
- */ -@Nullsafe(Nullsafe.Mode.LOCAL) -public class AnnotationProcessingToolSerializer { - - private AnnotationProcessingToolSerializer() {} - - /** Protocol buffer model to internal buck representation. */ - public static AnnotationProcessingTool deserialize( - com.facebook.buck.cd.model.kotlin.AnnotationProcessingTool annotationProcessingTool) { - switch (annotationProcessingTool) { - case KAPT: - return AnnotationProcessingTool.KAPT; - - case JAVAC: - return AnnotationProcessingTool.JAVAC; - - case UNRECOGNIZED: - default: - throw new IllegalArgumentException( - "Unrecognised annotation processing tool: " + annotationProcessingTool); - } - } -} diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/cd/serialization/kotlin/KotlinExtraParamsSerializer.java b/prelude/toolchains/android/src/com/facebook/buck/jvm/cd/serialization/kotlin/KotlinExtraParamsSerializer.java index c6ead0b..c133e95 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/jvm/cd/serialization/kotlin/KotlinExtraParamsSerializer.java +++ b/prelude/toolchains/android/src/com/facebook/buck/jvm/cd/serialization/kotlin/KotlinExtraParamsSerializer.java @@ -11,6 +11,7 @@ package com.facebook.buck.jvm.cd.serialization.kotlin; import com.facebook.buck.core.filesystems.AbsPath; +import com.facebook.buck.core.filesystems.RelPath; import com.facebook.buck.jvm.cd.command.kotlin.KotlinExtraParams; import com.facebook.buck.jvm.cd.serialization.AbsPathSerializer; import com.facebook.buck.jvm.cd.serialization.java.ResolvedJavacOptionsSerializer; @@ -39,7 +40,8 @@ private KotlinExtraParamsSerializer() {} /** Protocol buffer model to internal buck representation. */ public static KotlinExtraParams deserialize( com.facebook.buck.cd.model.java.ResolvedJavacOptions resolvedJavacOptions, - com.facebook.buck.cd.model.kotlin.KotlinExtraParams kotlinExtraParams) { + com.facebook.buck.cd.model.kotlin.KotlinExtraParams kotlinExtraParams, + Optional scratchDir) { return new KotlinExtraParams( kotlinExtraParams.getExtraClassPathsList().stream() .map(AbsPathSerializer::deserialize) @@ -49,8 +51,6 @@ public static KotlinExtraParams deserialize( .collect(ImmutableList.toImmutableList()), AbsPathSerializer.deserialize(kotlinExtraParams.getStandardLibraryClassPath()), AbsPathSerializer.deserialize(kotlinExtraParams.getAnnotationProcessingClassPath()), - AnnotationProcessingToolSerializer.deserialize( - kotlinExtraParams.getAnnotationProcessingTool()), kotlinExtraParams.getExtraKotlincArgumentsList().stream() .collect(ImmutableList.toImmutableList()), kotlinExtraParams.getKotlinCompilerPluginsMap().entrySet().stream() @@ -62,7 +62,8 @@ public static KotlinExtraParams deserialize( .collect( ImmutableMap.toImmutableMap( Map.Entry::getKey, e -> AbsPathSerializer.deserialize(e.getValue()))), - Optional.ofNullable(kotlinExtraParams.getKosabiJvmAbiGenEarlyTerminationMessagePrefix()), + Optional.of( + "exception: java.lang.RuntimeException: Terminating compilation. We're done with ABI."), kotlinExtraParams.getFriendPathsList().stream() .map(AbsPathSerializer::deserialize) .collect(ImmutableSortedSet.toImmutableSortedSet(AbsPath.comparator())), @@ -87,7 +88,11 @@ public static KotlinExtraParams deserialize( .map(AbsPathSerializer::deserialize), kotlinExtraParams.getShouldKsp2RunIncrementally(), kotlinExtraParams.getLanguageVersion(), - AbsPathSerializer.deserialize(kotlinExtraParams.getKotlinClassesDir()), + (kotlinExtraParams.getShouldKotlincRunIncrementally() + || kotlinExtraParams.getShouldKsp2RunIncrementally() + ? AbsPathSerializer.deserialize(kotlinExtraParams.getIncrementalStateDir()) + : scratchDir.get().toAbsolutePath()) + .resolve("__kotlin_classes__"), Optional.of(kotlinExtraParams.getJavaBinary()).filter(s -> !s.isEmpty()), kotlinExtraParams.getApplicabilityClasspathList().stream() .map(AbsPathSerializer::deserialize) diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/java/BaseJavacToJarStepFactory.java b/prelude/toolchains/android/src/com/facebook/buck/jvm/java/BaseJavacToJarStepFactory.java index a4329f8..c4a1ece 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/jvm/java/BaseJavacToJarStepFactory.java +++ b/prelude/toolchains/android/src/com/facebook/buck/jvm/java/BaseJavacToJarStepFactory.java @@ -58,6 +58,7 @@ public void createCompileStep( buckOut, compilerOutputPathsValue, parameters, + extraParams.getSkipIfNoCompilationUnits(), abiJarParameters, null, mixedCompilation)); diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/java/DaemonJavacToJarStepFactory.java b/prelude/toolchains/android/src/com/facebook/buck/jvm/java/DaemonJavacToJarStepFactory.java index 8c89dd6..4deddba 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/jvm/java/DaemonJavacToJarStepFactory.java +++ b/prelude/toolchains/android/src/com/facebook/buck/jvm/java/DaemonJavacToJarStepFactory.java @@ -66,6 +66,8 @@ public void createCompileToJarStepImpl( buckOut, compilerOutputPathsValue, compilerParameters, + // javac writes libraryJarParameters here, so it must run even with nothing to compile + /* skipIfNoCompilationUnits */ false, abiJarParameters, libraryJarParameters, false)); diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/java/JavaExtraParams.kt b/prelude/toolchains/android/src/com/facebook/buck/jvm/java/JavaExtraParams.kt index 30d4aef..6671533 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/jvm/java/JavaExtraParams.kt +++ b/prelude/toolchains/android/src/com/facebook/buck/jvm/java/JavaExtraParams.kt @@ -16,6 +16,11 @@ import com.facebook.buck.jvm.java.CompileToJarStepFactory.ExtraParams data class JavaExtraParams( val resolvedJavacOptions: ResolvedJavacOptions, val addAnnotationPath: Boolean, + /** + * Only safe where javac does not own the rule's jar, since a skipped invocation writes no + * output. + */ + val skipIfNoCompilationUnits: Boolean = false, ) : ExtraParams { companion object { @JvmStatic @@ -30,5 +35,14 @@ data class JavaExtraParams( ): JavaExtraParams { return JavaExtraParams(resolvedJavacOptions, addAnnotationPath) } + + @JvmStatic + fun of( + resolvedJavacOptions: ResolvedJavacOptions, + addAnnotationPath: Boolean, + skipIfNoCompilationUnits: Boolean, + ): JavaExtraParams { + return JavaExtraParams(resolvedJavacOptions, addAnnotationPath, skipIfNoCompilationUnits) + } } } diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/java/JavaPaths.java b/prelude/toolchains/android/src/com/facebook/buck/jvm/java/JavaPaths.java index f8d38bb..0f18e32 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/jvm/java/JavaPaths.java +++ b/prelude/toolchains/android/src/com/facebook/buck/jvm/java/JavaPaths.java @@ -59,6 +59,32 @@ static ImmutableList extractArchivesAndGetPaths( return sources.build(); } + /** + * Whether the given source paths expand to at least one {@code .java} compilation unit. A + * SRC_ZIP/SRC_JAR staged by an annotation processor may hold no Java sources at all, so the + * archives are inspected rather than counted. + * + *

A path that is neither a Java source nor an archive counts as a compilation unit, leaving it + * to javac to reject. + */ + static boolean hasJavaCompilationUnits(Iterable javaSourceFilePaths) throws IOException { + for (Path path : javaSourceFilePaths) { + String pathString = path.toString(); + if (!pathString.endsWith(SRC_ZIP) && !pathString.endsWith(SRC_JAR)) { + return true; + } + try (ZipFile zipFile = new ZipFile(path.toFile())) { + Enumeration entries = zipFile.entries(); + while (entries.hasMoreElements()) { + if (entries.nextElement().getName().endsWith(".java")) { + return true; + } + } + } + } + return false; + } + /** * Traverses a list of java inputs return the list of all found files (for SRC_ZIP/SRC_JAR, it * returns the paths from within the archive). diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/java/JavacStep.java b/prelude/toolchains/android/src/com/facebook/buck/jvm/java/JavacStep.java index e9c492c..58d02e6 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/jvm/java/JavacStep.java +++ b/prelude/toolchains/android/src/com/facebook/buck/jvm/java/JavacStep.java @@ -11,6 +11,7 @@ package com.facebook.buck.jvm.java; import com.facebook.buck.core.build.execution.context.IsolatedExecutionContext; +import com.facebook.buck.core.filesystems.AbsPath; import com.facebook.buck.core.filesystems.RelPath; import com.facebook.buck.jvm.core.BuildTargetValue; import com.facebook.buck.step.StepExecutionResult; @@ -33,6 +34,7 @@ public class JavacStep implements IsolatedStep { private final RelPath configuredBuckOut; private final boolean ownsPipelineObject; private final CompilerOutputPathsValue compilerOutputPathsValue; + private final boolean skipIfNoCompilationUnits; private final boolean mixedModule; @@ -43,6 +45,7 @@ public JavacStep( RelPath configuredBuckOut, CompilerOutputPathsValue compilerOutputPathsValue, CompilerParameters compilerParameters, + boolean skipIfNoCompilationUnits, @Nullable JarParameters abiJarParameters, @Nullable JarParameters libraryJarParameters, boolean mixedCompilation) { @@ -58,6 +61,7 @@ public JavacStep( configuredBuckOut, true, compilerOutputPathsValue, + skipIfNoCompilationUnits, mixedCompilation); } @@ -66,7 +70,7 @@ public JavacStep( BuildTargetValue invokingRule, RelPath configuredBuckOut, CompilerOutputPathsValue compilerOutputPathsValue) { - this(state, invokingRule, configuredBuckOut, false, compilerOutputPathsValue, false); + this(state, invokingRule, configuredBuckOut, false, compilerOutputPathsValue, false, false); } private JavacStep( @@ -75,12 +79,14 @@ private JavacStep( RelPath configuredBuckOut, boolean ownsPipelineObject, CompilerOutputPathsValue compilerOutputPathsValue, + boolean skipIfNoCompilationUnits, boolean mixedModule) { this.state = state; this.invokingRule = invokingRule; this.configuredBuckOut = configuredBuckOut; this.ownsPipelineObject = ownsPipelineObject; this.compilerOutputPathsValue = compilerOutputPathsValue; + this.skipIfNoCompilationUnits = skipIfNoCompilationUnits; this.mixedModule = mixedModule; } @@ -88,6 +94,10 @@ private JavacStep( public final StepExecutionResult executeIsolatedStep(IsolatedExecutionContext context) throws IOException, InterruptedException { + if (skipIfNoCompilationUnits && !hasCompilationUnits(context.getRuleCellRoot())) { + return StepExecutionResults.SUCCESS; + } + int exitCode; Optional stderr = Optional.empty(); try { @@ -115,6 +125,17 @@ public final StepExecutionResult executeIsolatedStep(IsolatedExecutionContext co return new StepExecutionResult(exitCode, stderr); } + /** + * KSP and KAPT stage a {@code .src.zip} into the Java source list before their processors have + * run, so whether javac has anything to compile is not known until this step executes. + */ + private boolean hasCompilationUnits(AbsPath ruleCellRoot) throws IOException { + return JavaPaths.hasJavaCompilationUnits( + state.getCompilerParameters().getSourceFilePaths().stream() + .map(path -> ruleCellRoot.resolve(path).getPath()) + .collect(ImmutableList.toImmutableList())); + } + @VisibleForTesting ResolvedJavac getResolvedJavac() { return state.getResolvedJavac(); diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/java/abi/BUCK b/prelude/toolchains/android/src/com/facebook/buck/jvm/java/abi/BUCK index 1be4590..035673b 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/jvm/java/abi/BUCK +++ b/prelude/toolchains/android/src/com/facebook/buck/jvm/java/abi/BUCK @@ -1,3 +1,4 @@ +load("@prelude//:is_full_meta_repo.bzl", "is_full_meta_repo") load("@prelude//toolchains/android/tools:build_rules.bzl", "buck_java_binary", "buck_java_library", "buck_prebuilt_artifact", "buck_prebuilt_jar") load("@prelude//toolchains/android/tools/build_rules:fb_native.bzl", "fb_native") load("@prelude//utils:source_listing.bzl", "source_listing") @@ -51,7 +52,7 @@ buck_java_library( fb_native.alias( name = "api-stubber", - actual = ":api-stubber_prebuilt", + actual = ":api-stubber_prebuilt" if is_full_meta_repo() else ":api-stubber-main", visibility = ["PUBLIC"], ) diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/java/stepsbuilder/javacd/main/JavaCDCommand.java b/prelude/toolchains/android/src/com/facebook/buck/jvm/java/stepsbuilder/javacd/main/JavaCDCommand.java index 68d4f19..3e18f97 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/jvm/java/stepsbuilder/javacd/main/JavaCDCommand.java +++ b/prelude/toolchains/android/src/com/facebook/buck/jvm/java/stepsbuilder/javacd/main/JavaCDCommand.java @@ -45,6 +45,7 @@ public class JavaCDCommand implements JvmCDCommand { private int loggingLevel = 0; private final JavaStepsBuilder stepsBuilder; + private final ImmutableMap jarToJarDirMap; private final PostBuildParams postBuildParams; public JavaCDCommand(String[] args, ImmutableMap env) @@ -69,6 +70,7 @@ public JavaCDCommand(String[] args, ImmutableMap env) BuildJavaCommand.Companion.fromProto(proto.getBuildCommand(), Optional.of(buckScratchPath)); this.postBuildParams = PostBuildParams.Companion.fromProto(proto.getPostBuildParams()); this.stepsBuilder = new JavaStepsBuilder(buildJavaCommand); + this.jarToJarDirMap = buildJavaCommand.getBaseJarCommand().getJarToJarDirMap(); } public void maybeWriteClassAbi() throws IOException { @@ -104,7 +106,7 @@ public void maybeWriteDepFile() throws IOException { DepFileUtils.usedClassesToDepFile( postBuildParams.getUsedClassesPaths(), postBuildParams.getDepFile(), - Optional.ofNullable(postBuildParams.getJarToJarDirMap()), + jarToJarDirMap, false); } } diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/java/stepsbuilder/javacd/main/JavaCDMain.java b/prelude/toolchains/android/src/com/facebook/buck/jvm/java/stepsbuilder/javacd/main/JavaCDMain.java index d005453..f8db181 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/jvm/java/stepsbuilder/javacd/main/JavaCDMain.java +++ b/prelude/toolchains/android/src/com/facebook/buck/jvm/java/stepsbuilder/javacd/main/JavaCDMain.java @@ -43,10 +43,7 @@ public static void main(String[] args) throws IOException { })); CompilerDaemonRunner.run(command); logger.info(String.format("Starting JavaCDWorkerTool %s", command)); - command.maybeWriteClassAbi(); - command.maybeWriteAbiDir(); - command.maybeWriteDepFile(); - command.maybeWriteUsedJarsFile(); + command.postExecute(); System.err.println("JavaCDWorkerTool succeeded!"); System.exit(0); } catch (Exception e) { diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/ClasspathUtils.java b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/ClasspathUtils.java index 811237c..3178880 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/ClasspathUtils.java +++ b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/ClasspathUtils.java @@ -60,8 +60,7 @@ public String getFriendPathArgs(ImmutableList.Builder steps) { .map(AbsPath::toString) .collect(ImmutableSortedSet.toImmutableSortedSet(Ordering.natural())); - return "-Xfriend-paths=" - + absoluteFriendPaths.stream().reduce("", (path1, path2) -> path1 + "," + path2); + return "-Xfriend-paths=" + String.join(",", absoluteFriendPaths); } private ImmutableSortedSet getFriendAbsPaths(ImmutableList.Builder steps) { diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/DaemonKotlincToJarStepFactory.java b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/DaemonKotlincToJarStepFactory.java index 44b208a..b8ce7f3 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/DaemonKotlincToJarStepFactory.java +++ b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/DaemonKotlincToJarStepFactory.java @@ -20,7 +20,6 @@ import com.facebook.buck.io.file.GlobPatternMatcher; import com.facebook.buck.io.file.PathMatcher; import com.facebook.buck.io.filesystem.CopySourceMode; -import com.facebook.buck.jvm.cd.command.kotlin.AnnotationProcessingTool; import com.facebook.buck.jvm.cd.command.kotlin.KotlinExtraParams; import com.facebook.buck.jvm.core.BuildTargetValue; import com.facebook.buck.jvm.core.BuildTargetValueExtraParams; @@ -92,7 +91,11 @@ public void createCompileStep( ImmutableSortedSet sourceFilePaths = parameters.getSourceFilePaths(); RelPath outputDirectory = compilerOutputPaths.getClassesDir(); RelPath kotlinOutputDirectory = buildCellRootPath.relativize(extraParams.getKotlinClassesDir()); - steps.add(new MkdirIsolatedStep(kotlinOutputDirectory)); + if (!extraParams.getShouldActionRunIncrementally()) { + steps.addAll(MakeCleanDirectoryIsolatedStep.of(kotlinOutputDirectory)); + } else { + steps.add(new MkdirIsolatedStep(kotlinOutputDirectory)); + } RelPath annotationGenFolder = compilerOutputPaths.getAnnotationPath(); Path pathToSrcsList = compilerOutputPaths.getPathToSourcesList().getPath(); @@ -163,7 +166,6 @@ public void createCompileStep( ImmutableList kotlinHomeLibraries = extraParams.getKotlinHomeLibraries(); KaptStepsBuilder.prepareKaptProcessorsIfNeeded( - extraParams.getAnnotationProcessingTool(), invokingRule, buildCellRootPath, steps, @@ -279,8 +281,7 @@ public void createCompileStep( } ResolvedJavacOptions resolvedJavacOptions = extraParams.getResolvedJavacOptions(); - if (hasKotlinSources - && extraParams.getAnnotationProcessingTool() == AnnotationProcessingTool.KAPT) { + if (hasKotlinSources) { // Most of the time, KotlinC have ran annotation processing, // so only run "java on mix" processors (very uncommon) on Javac resolvedJavacOptions = diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/JavacStepsBuilder.java b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/JavacStepsBuilder.java index 218be3e..2ae7270 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/JavacStepsBuilder.java +++ b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/JavacStepsBuilder.java @@ -84,7 +84,10 @@ public static void prepareJavaCompilationIfNeeded( steps, resolvedJavac, null, - JavaExtraParams.of(resolvedJavacOptions, /* addAnnotationPath */ false), + JavaExtraParams.of( + resolvedJavacOptions, + /* addAnnotationPath */ false, + /* skipIfNoCompilationUnits */ true), abiJarParameter, true); } diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/KaptStepsBuilder.java b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/KaptStepsBuilder.java index 4826cf7..cb3f891 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/KaptStepsBuilder.java +++ b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/KaptStepsBuilder.java @@ -22,7 +22,6 @@ import com.facebook.buck.core.filesystems.AbsPath; import com.facebook.buck.core.filesystems.RelPath; import com.facebook.buck.io.filesystem.CopySourceMode; -import com.facebook.buck.jvm.cd.command.kotlin.AnnotationProcessingTool; import com.facebook.buck.jvm.cd.command.kotlin.KotlinSupportedLanguageVersion; import com.facebook.buck.jvm.cd.command.kotlin.LanguageVersion; import com.facebook.buck.jvm.core.BuildTargetValue; @@ -85,7 +84,6 @@ public class KaptStepsBuilder { *

This method will do nothing if there are no relevant annotation processors to run. */ public static void prepareKaptProcessorsIfNeeded( - AnnotationProcessingTool annotationProcessingTool, BuildTargetValue invokingRule, AbsPath rootCellPath, ImmutableList.Builder steps, @@ -120,10 +118,6 @@ public static void prepareKaptProcessorsIfNeeded( return; } - if (!annotationProcessingTool.equals(AnnotationProcessingTool.KAPT)) { - return; - } - // We need to generate the KAPT generation folder anyway, to help IntelliJ with red symbols. RelPath kaptAnnotationGenFolder = buildTargetValueExtraParams.getKaptAnnotationGenPath(); steps.addAll(MakeCleanDirectoryIsolatedStep.of(kaptAnnotationGenFolder)); diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/KosabiConfig.java b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/KosabiConfig.java index f79bc80..9b93cd7 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/KosabiConfig.java +++ b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/KosabiConfig.java @@ -14,6 +14,9 @@ public abstract class KosabiConfig { public static final String PROPERTY_KOSABI_APPLICABILITY_PLUGIN = "kosabi_applicability_plugin"; + public static final String PROPERTY_KOSABI_APPLICABILITY_CELL_ROOT = + "kosabi_applicability_cell_root"; + public static final String PROPERTY_KOSABI_STUBS_GEN_K2_PLUGIN = "kosabi_stubs_gen_k2_plugin"; public static final String PROPERTY_KOSABI_JVM_ABI_GEN_K2_PLUGIN = "kosabi_jvm_abi_gen_k2_plugin"; diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/KotlincStep.java b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/KotlincStep.java index 4f13828..13a87d1 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/KotlincStep.java +++ b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/KotlincStep.java @@ -52,6 +52,8 @@ public class KotlincStep implements IsolatedStep { private static final String DESTINATION_FLAG = "-d"; private static final String X_PLUGIN_ARG = "-Xplugin="; private static final String PLUGIN = "-P"; + private static final String APPLICABILITY_PLUGIN_ID = + "com.facebook.kotlin.compilerplugins.kosabiapplicability"; private static final int EXPECTED_SOURCE_ONLY_ABI_EXIT_CODE = 2; @@ -179,6 +181,7 @@ public StepExecutionResult executeIsolatedStep(IsolatedExecutionContext context) loggingContext.addExtras( this.getClass().getSimpleName(), "Kotlinc step duration: " + compilationDuration.toMillis() + " ms"); + loggingContext.setDurationMs(compilationDuration.toMillis()); kotlinCDAnalytics.log(loggingContext); String firstOrderStderr = stderr.getContentsAsString(StandardCharsets.UTF_8); @@ -276,37 +279,7 @@ ImmutableList getOptions( addClasspath(builder, buildClasspathEntries); } - // We expect Kosabi/Applicability to generate a compilation error if - // a library target verification fails. - // User will see a broken compilation with the following message: - // Kosabi/Applicability FAILED on this target ... - if (verifySourceOnlyAbiConstraints && invokingRule.isLibraryJar()) { - if (resolvedKosabiPluginOptionPath.containsKey( - KosabiConfig.PROPERTY_KOSABI_APPLICABILITY_PLUGIN)) { - AbsPath applicabilityPlugin = - resolvedKosabiPluginOptionPath.get(KosabiConfig.PROPERTY_KOSABI_APPLICABILITY_PLUGIN); - builder.add(X_PLUGIN_ARG + applicabilityPlugin); - - // Pass the reduced source-only-abi classpath to the applicability plugin - // so checkers can detect types that won't be available during SO-ABI - // generation. applicabilityClasspath contains only deps with - // required_for_source_only_abi=True or in source_only_abi_deps — an - // empty list is valid (no deps on SO-ABI classpath, checker flags all - // external type refs). Never fall back to compilationClasspath here - // as it contains the full library classpath during library builds. - if (!applicabilityClasspath.isEmpty()) { - String classpathValue = - Joiner.on(File.pathSeparator) - .join(transform(applicabilityClasspath, path -> path.getPath().toString())); - builder.add(PLUGIN); - builder.add( - "plugin:" - + "com.facebook.kotlin.compilerplugins.kosabiapplicability" - + ":source-only-abi-classpath=" - + classpathValue); - } - } - } + configureKosabiApplicability(builder, ruleCellRoot); if (trackClassUsage) { depTrackerPath.ifPresentOrElse( @@ -346,6 +319,77 @@ ImmutableList getOptions( return builder.build(); } + private void configureKosabiApplicability( + ImmutableList.Builder builder, AbsPath sourceRoot) { + if (!verifySourceOnlyAbiConstraints || !invokingRule.isLibraryJar()) return; + + builder.add( + X_PLUGIN_ARG + getRequiredKosabiApplicabilityPlugin(resolvedKosabiPluginOptionPath)); + builder.addAll( + getKosabiApplicabilityPluginOptions( + invokingRule.getFullyQualifiedName(), + sourceRoot, + applicabilityClasspath, + Optional.ofNullable( + resolvedKosabiPluginOptionPath.get( + KosabiConfig.PROPERTY_KOSABI_APPLICABILITY_CELL_ROOT)))); + } + + @VisibleForTesting + static AbsPath getRequiredKosabiApplicabilityPlugin( + ImmutableMap resolvedPluginPaths) { + AbsPath plugin = resolvedPluginPaths.get(KosabiConfig.PROPERTY_KOSABI_APPLICABILITY_PLUGIN); + if (plugin == null) { + throw new IllegalStateException( + "Structured Kosabi applicability was enabled, but the plugin path is missing"); + } + return plugin; + } + + @VisibleForTesting + static ImmutableList getKosabiApplicabilityPluginOptions( + String targetLabel, + AbsPath sourceRoot, + ImmutableList classpath, + Optional cellRoot) { + int cellSeparator = targetLabel.indexOf("//"); + if (cellSeparator <= 0) { + throw new IllegalStateException( + "Kosabi applicability requires a fully-qualified target label: " + targetLabel); + } + AbsPath resolvedCellRoot = + cellRoot.orElseThrow( + () -> new IllegalStateException("Kosabi applicability cell root path is missing")); + if (!resolvedCellRoot.startsWith(sourceRoot)) { + throw new IllegalStateException( + "Kosabi applicability cell root path must be within the project source root"); + } + String sourceRootPrefix = + sourceRoot + .relativize(resolvedCellRoot) + .getPath() + .toString() + .replace(File.separatorChar, '/'); + if (sourceRootPrefix.isEmpty()) sourceRootPrefix = "."; + + ImmutableList.Builder builder = ImmutableList.builder(); + addApplicabilityPluginOption(builder, "target-label", targetLabel); + addApplicabilityPluginOption(builder, "source-root", sourceRoot.getPath().toString()); + addApplicabilityPluginOption(builder, "source-root-prefix", sourceRootPrefix); + addApplicabilityPluginOption( + builder, + "source-only-abi-classpath", + Joiner.on(File.pathSeparator) + .join(transform(classpath, path -> path.getPath().toString()))); + return builder.build(); + } + + private static void addApplicabilityPluginOption( + ImmutableList.Builder builder, String name, String value) { + builder.add(PLUGIN); + builder.add("plugin:" + APPLICABILITY_PLUGIN_ID + ":" + name + "=" + value); + } + protected void configureSourceOnlyOptions( ImmutableList.Builder builder, LanguageVersion languageVersion, diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/KspStepsBuilder.java b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/KspStepsBuilder.java index 90d32f1..d8add86 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/KspStepsBuilder.java +++ b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/KspStepsBuilder.java @@ -17,7 +17,6 @@ import com.facebook.buck.core.filesystems.AbsPath; import com.facebook.buck.core.filesystems.RelPath; import com.facebook.buck.io.filesystem.CopySourceMode; -import com.facebook.buck.jvm.cd.command.kotlin.AnnotationProcessingTool; import com.facebook.buck.jvm.cd.command.kotlin.KotlinExtraParams; import com.facebook.buck.jvm.core.BuildTargetValue; import com.facebook.buck.jvm.core.BuildTargetValueExtraParams; @@ -32,6 +31,7 @@ import com.facebook.buck.step.isolatedsteps.common.CopyIsolatedStep; import com.facebook.buck.step.isolatedsteps.common.MakeCleanDirectoryIsolatedStep; import com.facebook.buck.step.isolatedsteps.common.MkdirIsolatedStep; +import com.facebook.buck.step.isolatedsteps.common.RmIsolatedStep; import com.facebook.buck.step.isolatedsteps.common.ZipIsolatedStep; import com.facebook.buck.util.zip.ZipCompressionLevel; import com.google.common.collect.ImmutableList; @@ -46,6 +46,12 @@ public class KspStepsBuilder { private static final String KSP_PLUGIN_ID = "plugin:com.google.devtools.ksp.symbol-processing:"; private static final String MODULE_NAME = "-module-name"; private static final String PLUGIN = "-P"; + private static final ImmutableSet PROCESSORS_USING_FINAL_ROUND_PLACEHOLDER = + ImmutableSet.of( + "KSP:com.facebook.annotationprocessors.inject.ksp.InjectorKspProcessorProvider", + "KSP:com.facebook.metagen.processor.kspmetagen.MetagenKspProcessorProvider", + "KSP:com.facebook.annotationprocessors.gatekeepers.ksp.GatekeeperDeclarationKspProcessor", + "KSP:com.facebook.annotationprocessors.qe.ksp.QEKspProcessorProvider"); /** Initialize all the folders, steps and parameters needed to run KSP plugins for this rule. */ public static KSPInvocationStatus prepareKspProcessorsIfNeeded( @@ -82,11 +88,6 @@ public static KSPInvocationStatus prepareKspProcessorsIfNeeded( KSPInvocationStatus kspInvocationStatus = KSPInvocationStatus.NOT_INVOKED; - // The other option is to use JAVAC, and we don't want to use KSP in that case. - if (!extraParams.getAnnotationProcessingTool().equals(AnnotationProcessingTool.KAPT)) { - return kspInvocationStatus; - } - // We need to generate the KSP generation folder anyway, to help IntelliJ with red // symbols. RelPath kspAnnotationGenFolder = buildTargetValueExtraParams.getKspAnnotationGenPath(); @@ -173,17 +174,64 @@ public static KSPInvocationStatus prepareKspProcessorsIfNeeded( extraParams, actionMetadata.orElse(null))); steps.add(ksp2Step); - steps.add( + steps.addAll( + createKspOutputStagingSteps( + rootPath, + kspKotlinOutput, + kspJavaOutput, + kspClassesOutput, + kspAnnotationGenFolder, + kspGenOutput, + annotationGenFolder, + kspAnnotationProcessors)); + + // Generated classes should be part of the output. This way generated files such as + // META-INF dirs will also be added to the final jar. + postKotlinCompilationSteps.add( + CopyIsolatedStep.forDirectory( + kspClassesOutput.getPath(), + outputDirectory.getPath(), + CopySourceMode.DIRECTORY_CONTENTS_ONLY)); + + sourceBuilderWithKspOutputs.add(kspGenOutput); + + javacSourceBuilder.add(kspGenOutput); + + return kspInvocationStatus; + } + + static ImmutableList createKspOutputStagingSteps( + AbsPath rootPath, + RelPath kspKotlinOutput, + RelPath kspJavaOutput, + RelPath kspClassesOutput, + RelPath kspAnnotationGenFolder, + RelPath kspGenOutput, + RelPath annotationGenFolder, + ImmutableList kspAnnotationProcessors) { + ImmutableList.Builder stagingSteps = ImmutableList.builder(); + stagingSteps.add( CopyIsolatedStep.forDirectory( kspKotlinOutput, kspAnnotationGenFolder, CopySourceMode.DIRECTORY_CONTENTS_ONLY)); - steps.add( + stagingSteps.add( CopyIsolatedStep.forDirectory( kspJavaOutput, kspAnnotationGenFolder, CopySourceMode.DIRECTORY_CONTENTS_ONLY)); - steps.add( + stagingSteps.add( CopyIsolatedStep.forDirectory( kspClassesOutput, kspAnnotationGenFolder, CopySourceMode.DIRECTORY_CONTENTS_ONLY)); - steps.add( + if (usesFinalRoundPlaceholder(kspAnnotationProcessors)) { + // The KSP adapter generates this empty source only to force KSP's final processing round. + // Keep it in KSP's own output for round and incremental bookkeeping, but exclude it from the + // staged generated sources consumed by Kotlin and javac. + stagingSteps.add( + new RmIsolatedStep( + kspAnnotationGenFolder.resolveRel("com/facebook/Dummy.java"), + false, + ImmutableSet.of())); + } + + stagingSteps.add( new ZipIsolatedStep( rootPath, kspGenOutput.getPath(), @@ -192,24 +240,17 @@ public static KSPInvocationStatus prepareKspProcessorsIfNeeded( false, ZipCompressionLevel.DEFAULT, kspAnnotationGenFolder.getPath())); - - steps.add( + stagingSteps.add( CopyIsolatedStep.forDirectory( kspAnnotationGenFolder, annotationGenFolder, CopySourceMode.DIRECTORY_CONTENTS_ONLY)); + return stagingSteps.build(); + } - // Generated classes should be part of the output. This way generated files such as - // META-INF dirs will also be added to the final jar. - postKotlinCompilationSteps.add( - CopyIsolatedStep.forDirectory( - kspClassesOutput.getPath(), - outputDirectory.getPath(), - CopySourceMode.DIRECTORY_CONTENTS_ONLY)); - - sourceBuilderWithKspOutputs.add(kspGenOutput); - - javacSourceBuilder.add(kspGenOutput); - - return kspInvocationStatus; + static boolean usesFinalRoundPlaceholder( + ImmutableList kspAnnotationProcessors) { + return kspAnnotationProcessors.stream() + .flatMap(processor -> processor.getProcessorNames().stream()) + .anyMatch(PROCESSORS_USING_FINAL_ROUND_PLACEHOLDER::contains); } private static ImmutableList getKspPluginsArgs( diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/buildtools/BuckKotlinLogger.kt b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/buildtools/BuckKotlinLogger.kt index c6e8bfa..f251990 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/buildtools/BuckKotlinLogger.kt +++ b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/buildtools/BuckKotlinLogger.kt @@ -61,6 +61,14 @@ internal class BuckKotlinLogger( stdErr.println(msg) } + // OSS-only: the internal `KotlinLoggerCompat` supertype lacks this Kotlin 2.2 + // overload on some toolchain configurations. + override fun warn(msg: String, throwable: Throwable?) { // @oss-enable + if (!LOG.isLoggable(Level.WARNING)) return // @oss-enable + stdErr.println(msg) // @oss-enable + throwable?.printStackTrace(stdErr) // @oss-enable + } // @oss-enable + companion object { private val LOG: Logger = Logger.get(BuckKotlinLogger::class.java) } diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/cd/analytics/KotlinCDLoggingContext.kt b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/cd/analytics/KotlinCDLoggingContext.kt index 3b178eb..eb692d3 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/cd/analytics/KotlinCDLoggingContext.kt +++ b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/cd/analytics/KotlinCDLoggingContext.kt @@ -24,6 +24,9 @@ class KotlinCDLoggingContext( var numKotlinTokens: Long = 0L var numJavaTokens: Long = 0L + /** Wall-clock duration of the step, in milliseconds; null when the step does not time itself. */ + var durationMs: Long? = null + private val _extras: MutableMap> = mutableMapOf() val extras: Map> get() = _extras diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/cd/analytics/logger/KotlinCDLoggerAnalytics.kt b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/cd/analytics/logger/KotlinCDLoggerAnalytics.kt index 7ff951e..7a45d93 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/cd/analytics/logger/KotlinCDLoggerAnalytics.kt +++ b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/cd/analytics/logger/KotlinCDLoggerAnalytics.kt @@ -106,6 +106,7 @@ constructor( removedFiles = removedFiles, numKotlinTokens = context.numKotlinTokens.takeIf { it > 0 }, numJavaTokens = context.numJavaTokens.takeIf { it > 0 }, + durationMs = context.durationMs, ) } diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/cd/analytics/logger/model/KotlinCDLogEntry.kt b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/cd/analytics/logger/model/KotlinCDLogEntry.kt index a45153d..46aa04d 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/cd/analytics/logger/model/KotlinCDLogEntry.kt +++ b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/cd/analytics/logger/model/KotlinCDLogEntry.kt @@ -29,4 +29,13 @@ data class KotlinCDLogEntry( val removedFiles: Set?, val numKotlinTokens: Long? = null, val numJavaTokens: Long? = null, + /** + * Wall-clock duration of the step this entry describes, in milliseconds. + * + * Null when the step does not time itself. Without this there is no step-level timing anywhere + * in KotlinCD - `IsolatedStepsRunner.runStep` does not time steps, and + * `buck2_action_command_perf` only sees the whole `kotlincd_jar` action, which bundles ksp2, + * kotlinc, kosabi and copy/zip together. + */ + val durationMs: Long? = null, ) diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/cd/workertool/KotlinCDCommand.java b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/cd/workertool/KotlinCDCommand.java index 2a6efac..dad24cc 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/cd/workertool/KotlinCDCommand.java +++ b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/cd/workertool/KotlinCDCommand.java @@ -286,7 +286,7 @@ public void maybeWriteDepFile() throws IOException { DepFileUtils.usedClassesToDepFile( usedClassesMapPaths, postBuildParams.getDepFile(), - Optional.ofNullable(postBuildParams.getJarToJarDirMap()), + buildKotlinCommand.getBaseJarCommand().getJarToJarDirMap(), buildKotlinCommand.getKotlinExtraParams().getShouldActionRunIncrementally()); } } diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/common/FullTypeQualifier.kt b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/common/FullTypeQualifier.kt index 85ac822..c9b830b 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/common/FullTypeQualifier.kt +++ b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/common/FullTypeQualifier.kt @@ -43,7 +43,7 @@ class FullTypeQualifier : Comparable { } else { pkg = segments.take(firstUpperCaseIndex) val lastSegment = segments.last() - if (NonConventionalImports.interfaces.contains(segments)) { + if (NonConventionalImports.classLikeDeclarations.contains(segments)) { names = listOf(lastSegment) member = null } else if (lastSegment.isStaticConstQualifier() || lastSegment.isStaticMethodQualifier()) { diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/common/NonConventionalImports.kt b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/common/NonConventionalImports.kt index 25ea4bc..0cdc1d4 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/common/NonConventionalImports.kt +++ b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/common/NonConventionalImports.kt @@ -11,13 +11,14 @@ package com.facebook.kotlin.compilerplugins.kosabi.common object NonConventionalImports { - /** [interfaces] is exclude list from Constant Value Naming convention in kosabi. */ - val interfaces: Set> = setOf( + /** Exact FQNs parsed as class-like declarations despite their non-conventional names. */ + val classLikeDeclarations: Set> = setOf( listOf("javax", "microedition", "khronos", "egl", "EGL"), listOf("javax", "microedition", "khronos", "egl", "EGL10"), listOf("javax", "microedition", "khronos", "egl", "EGL11"), listOf("javax", "microedition", "khronos", "opengles", "GL"), listOf("javax", "microedition", "khronos", "opengles", "GL10"), listOf("javax", "microedition", "khronos", "opengles", "GL11"), + listOf("com", "whatsapp", "infra", "smax", "common", "RPC"), ) } diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/common/stub/model/KFunStub.kt b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/common/stub/model/KFunStub.kt index 5aea27c..fe222e6 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/common/stub/model/KFunStub.kt +++ b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/common/stub/model/KFunStub.kt @@ -22,6 +22,8 @@ open class KFunStub(val name: String, val namedArgs: List) { // Top level functions are always static var static: Boolean = false var private: Boolean = false + var protected: Boolean = false + var isOverride: Boolean = false var abstract: Boolean = false var isConstructor: Boolean = false var isOperator: Boolean = false diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/common/stub/render/StubBytecodeRender.kt b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/common/stub/render/StubBytecodeRender.kt index d220a43..cf902e8 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/common/stub/render/StubBytecodeRender.kt +++ b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/common/stub/render/StubBytecodeRender.kt @@ -66,7 +66,12 @@ object StubBytecodeRender { var flag = when (type) { - KStub.Type.ANNOTATION -> Opcodes.ACC_ANNOTATION + // ACC_ANNOTATION alone is not a valid annotation type: JVMS 4.1 requires ACC_INTERFACE + // (and therefore ACC_ABSTRACT) alongside it, and the class must implement + // java/lang/annotation/Annotation. A class carrying only ACC_ANNOTATION does not resolve + // in an annotation position, so the use site keeps failing even once the stub exists. + KStub.Type.ANNOTATION -> + Opcodes.ACC_ANNOTATION or Opcodes.ACC_INTERFACE or Opcodes.ACC_ABSTRACT KStub.Type.INTERFACE -> Opcodes.ACC_INTERFACE else -> Opcodes.ACC_SUPER } @@ -119,6 +124,10 @@ object StubBytecodeRender { return "java/lang/Object" } + private fun KStub.interfaces(): Array? { + return if (type == KStub.Type.ANNOTATION) arrayOf("java/lang/annotation/Annotation") else null + } + fun KStub.signature(): String? { if (genericTypes == 0) return null return "<${(0 until genericTypes).map { "T$it:Ljava/lang/Object;" }.joinToString("")}>" @@ -141,6 +150,9 @@ object StubBytecodeRender { private fun KStub.renderConstructor(visitor: ClassVisitor) { if (ctor == null) return + // An annotation type is an interface, and an interface has no : emitting one makes the + // class file unverifiable, which would trade the unresolved use site for a worse failure. + if (type == KStub.Type.ANNOTATION) return visitor.visitMethod( Opcodes.ACC_PUBLIC, "", @@ -249,7 +261,7 @@ object StubBytecodeRender { stub.internalName(), stub.signature(), stub.superClass(), - null, + stub.interfaces(), ) stub.apply { diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/common/stub/render/StubRender.kt b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/common/stub/render/StubRender.kt index e536679..711134c 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/common/stub/render/StubRender.kt +++ b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/common/stub/render/StubRender.kt @@ -61,7 +61,12 @@ fun KStub.render(): String { } when (type) { KStub.Type.TOP_LEVEL_DECLARATION -> renderTopLevelDeclaration(this@render) - else -> addType(renderType(returnObjectTypeIfNeeded())) + else -> + addType( + renderType( + if (doNotRenderAsKotlinObject) type else returnObjectTypeIfNeeded(), + ), + ) } } .build() @@ -115,7 +120,13 @@ private fun KStub.renderType(stubType: KStub.Type): TypeSpec { else superclass(it.asTypeName()) } implements.map { it.asTypeName() }.forEach { addSuperinterface(it) } - propertyStubs.map { addProperty(it.render()) } + // A static property on a CLASS only becomes a static JVM field from inside the companion; + // rendered on the class body it is an instance field. Scoped to the opted-in shape so the + // OBJECT-coercion path below keeps emitting the layout every other stub already relies on. + val companionProperties = + if (stubType == CLASS && doNotRenderAsKotlinObject) propertyStubs.filter { it.static } + else emptyList() + propertyStubs.filterNot { it in companionProperties }.forEach { addProperty(it.render()) } innerStubs .map { it.renderType( @@ -124,9 +135,12 @@ private fun KStub.renderType(stubType: KStub.Type): TypeSpec { } .forEach { addType(it) } val staticFunctions = funStubs.filter { it.static } - if (stubType == CLASS && staticFunctions.isNotEmpty()) { + if ( + stubType == CLASS && (staticFunctions.isNotEmpty() || companionProperties.isNotEmpty()) + ) { val companionObject = TypeSpec.companionObjectBuilder() staticFunctions.map { it.renderFunction() }.forEach { companionObject.addFunction(it) } + companionProperties.map { it.render() }.forEach { companionObject.addProperty(it) } addType(companionObject.build()) funStubs.filterNot { it.static }.forEach { addFunction(it.renderFunction()) } } else { @@ -165,6 +179,12 @@ private fun KFunStub.renderFunction(isOpen: Boolean = true): FunSpec { } } else { FunSpec.builder(name).apply { + if (protected) { + addModifiers(KModifier.PROTECTED) + } + if (isOverride) { + addModifiers(KModifier.OVERRIDE) + } // Operator function couldn't be open if (isOperator) { addModifiers(KModifier.OPERATOR) @@ -172,7 +192,7 @@ private fun KFunStub.renderFunction(isOpen: Boolean = true): FunSpec { addModifiers(KModifier.ABSTRACT) } else if (isFinal) { addModifiers(KModifier.FINAL) - } else if (isOpen) { + } else if (isOpen && !isOverride) { addModifiers(KModifier.OPEN) } diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/k2-jvm-abi-gen/BUCK b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/k2-jvm-abi-gen/BUCK index 0a60214..0215326 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/k2-jvm-abi-gen/BUCK +++ b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/k2-jvm-abi-gen/BUCK @@ -5,7 +5,13 @@ oncall("android_devxx") source_listing() -KOTLIN_VERSION = read_config("kotlin", "kotlin_version_for_upgrade_only_do_not_use", "2.2.0") +# compilercompat shims are bucketed by Kotlin minor version: every 2.3.x compiler +# uses the compilercompat/2.3.0 folder. Key off kotlin_version_for_upgrade_only_do_not_use +# (the same config the kotlincd toolchain selects the compiler jar from) so the shims +# always match the compiler actually run -- not kotlin_version, which can differ. +_KOTLIN_VERSION_PARTS = read_root_config("kotlin", "kotlin_version_for_upgrade_only_do_not_use", "2.2.0").split(".") + +KOTLIN_VERSION = _KOTLIN_VERSION_PARTS[0] + "." + _KOTLIN_VERSION_PARTS[1] + ".0" buck_kotlin_library( name = "k2-jvm-abi-gen-lib", diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/k2-jvm-abi-gen/src/main/kotlin/K2JvmAbiCommandLineProcessor.kt b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/k2-jvm-abi-gen/src/main/kotlin/K2JvmAbiCommandLineProcessor.kt index c370639..63615f3 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/k2-jvm-abi-gen/src/main/kotlin/K2JvmAbiCommandLineProcessor.kt +++ b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/k2-jvm-abi-gen/src/main/kotlin/K2JvmAbiCommandLineProcessor.kt @@ -58,6 +58,17 @@ class K2JvmAbiCommandLineProcessor : CommandLineProcessor { "Enable Compose ABI emulation. Default is false.", false, ) + + val ABI_VALIDATION_MODE_OPTION: CliOption = CliOption( + "abiValidationMode", + "off|warn|error", + "How to report ABI content that had to be repaired because a dependency was not on " + + "the source-only ABI classpath. `warn` reports each repair as a compiler warning, " + + "`error` fails the compilation instead of emitting a knowingly-wrong ABI. " + + "Default is off: fbsource builds Kotlin with `-Werror`, so `warn` also fails the " + + "compile and has to be opted into per target.", + false, + ) } override val pluginId: String @@ -70,6 +81,7 @@ class K2JvmAbiCommandLineProcessor : CommandLineProcessor { ENABLE_MIXED_COMPILATION, DEBUG_OUTPUT_OPTION, ENABLE_COMPOSE_ABI_EMULATION_OPTION, + ABI_VALIDATION_MODE_OPTION, ) override fun processOption( @@ -90,6 +102,8 @@ class K2JvmAbiCommandLineProcessor : CommandLineProcessor { K2JvmAbiConfigurationKeys.ENABLE_COMPOSE_ABI_EMULATION, value == "true", ) + ABI_VALIDATION_MODE_OPTION -> + configuration.put(K2JvmAbiConfigurationKeys.ABI_VALIDATION_MODE, value) else -> throw CliOptionProcessingException("Unknown option: ${option.optionName}") } } diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/k2-jvm-abi-gen/src/main/kotlin/K2JvmAbiConfigurationKeys.kt b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/k2-jvm-abi-gen/src/main/kotlin/K2JvmAbiConfigurationKeys.kt index e2546b4..ee9a5f5 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/k2-jvm-abi-gen/src/main/kotlin/K2JvmAbiConfigurationKeys.kt +++ b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/k2-jvm-abi-gen/src/main/kotlin/K2JvmAbiConfigurationKeys.kt @@ -28,4 +28,7 @@ object K2JvmAbiConfigurationKeys { val ENABLE_COMPOSE_ABI_EMULATION: CompilerConfigurationKey = CompilerConfigurationKey.create("enable Compose ABI emulation for K2 JVM ABI generation") + + val ABI_VALIDATION_MODE: CompilerConfigurationKey = + CompilerConfigurationKey.create("ABI validation mode for K2 JVM ABI generation") } diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/k2-jvm-abi-gen/src/main/kotlin/K2JvmAbiFirAnalysisHandlerExtension.kt b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/k2-jvm-abi-gen/src/main/kotlin/K2JvmAbiFirAnalysisHandlerExtension.kt index 14baa9b..5c1fa1e 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/k2-jvm-abi-gen/src/main/kotlin/K2JvmAbiFirAnalysisHandlerExtension.kt +++ b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/k2-jvm-abi-gen/src/main/kotlin/K2JvmAbiFirAnalysisHandlerExtension.kt @@ -30,6 +30,7 @@ import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment import org.jetbrains.kotlin.cli.jvm.compiler.VfsBasedProjectEnvironment import org.jetbrains.kotlin.cli.jvm.compiler.createContextForIncrementalCompilation import org.jetbrains.kotlin.cli.jvm.compiler.report +import org.jetbrains.kotlin.cli.jvm.config.jvmClasspathRoots import org.jetbrains.kotlin.com.intellij.openapi.diagnostic.Logger import org.jetbrains.kotlin.com.intellij.openapi.project.Project import org.jetbrains.kotlin.com.intellij.openapi.util.Disposer @@ -50,6 +51,7 @@ import org.jetbrains.kotlin.extensions.PreprocessedFileCreator import org.jetbrains.kotlin.fir.FirElement import org.jetbrains.kotlin.fir.FirSession import org.jetbrains.kotlin.fir.backend.jvm.JvmFir2IrExtensions +import org.jetbrains.kotlin.fir.declarations.FirConstructor import org.jetbrains.kotlin.fir.declarations.FirEnumEntry import org.jetbrains.kotlin.fir.declarations.FirRegularClass import org.jetbrains.kotlin.fir.declarations.FirTypeAlias @@ -80,6 +82,7 @@ import org.jetbrains.kotlin.fir.symbols.impl.FirNamedFunctionSymbol import org.jetbrains.kotlin.fir.symbols.impl.FirPropertySymbol import org.jetbrains.kotlin.fir.types.ConeClassLikeType import org.jetbrains.kotlin.fir.types.ConeKotlinType +import org.jetbrains.kotlin.fir.types.ConeKotlinTypeProjection import org.jetbrains.kotlin.fir.types.FirResolvedTypeRef import org.jetbrains.kotlin.fir.types.coneType import org.jetbrains.kotlin.fir.visitors.FirDefaultVisitorVoid @@ -95,6 +98,7 @@ import org.jetbrains.kotlin.name.ClassId import org.jetbrains.kotlin.name.FqName import org.jetbrains.kotlin.name.Name import org.jetbrains.kotlin.psi.KtAnnotationEntry +import org.jetbrains.kotlin.psi.KtClass import org.jetbrains.kotlin.psi.KtFile import org.jetbrains.kotlin.psi.KtTreeVisitorVoid import org.jetbrains.kotlin.resolve.multiplatform.hmppModuleName @@ -518,6 +522,7 @@ class K2JvmAbiFirAnalysisHandlerExtension(private val outputPath: String) : if (composeAbiEnabled) { add(pipeline.composeAbi.createExtension()) } + add(ParcelizeAbiEmulationExtension()) add(pipeline.irSanitizer.createExtension(sourceFiles)) } @@ -578,22 +583,34 @@ class K2JvmAbiFirAnalysisHandlerExtension(private val outputPath: String) : // Write bytecode from generationState.factory to disk. // Apply bytecode post-processing (strip @Throws annotations and private metadata) // in-memory before writing to avoid a separate read-back pass. + val outputFiles = + result.classFileFactoryCompat.asList().map { outputFile -> + val originalBytes = outputFile.asByteArray() + val bytes = + if (outputFile.relativePath.endsWith(".class")) { + pipeline.bytecodeSanitizer.transform(originalBytes) + } else { + originalBytes + } + AbiValidationOutputFile(outputFile.relativePath, bytes) + } val outputDir = configuration[JVMConfigurationKeys.OUTPUT_DIRECTORY] if (outputDir != null) { - val outputFiles = result.classFileFactoryCompat.asList() outputFiles.forEach { outputFile -> val file = File(outputDir, outputFile.relativePath) file.parentFile?.mkdirs() - var bytes = outputFile.asByteArray() - if (file.extension == "class") { - bytes = pipeline.bytecodeSanitizer.transform(bytes) - } - file.writeBytes(bytes) + file.writeBytes(outputFile.bytes) } } - // Phase 6: Validation - pipeline.validator.validate(irInput.irModuleFragment, messageCollector) + // Phase 5: Validation. Needs the written output files and the classpath, so it runs after + // code generation rather than before it. + pipeline.validator.validate( + irInput.irModuleFragment, + messageCollector, + AbiRepairPolicy.parse(configuration.get(K2JvmAbiConfigurationKeys.ABI_VALIDATION_MODE)), + AbiValidationInputs(outputFiles, configuration.jvmClasspathRoots), + ) // Generate .kotlin_module file generateKotlinModuleFile(irInput.irModuleFragment, module.getModuleName(), configuration) @@ -789,41 +806,157 @@ class K2JvmAbiFirAnalysisHandlerExtension(private val outputPath: String) : return result } - private class MissingConstantsVisitor(private val usedConstants: MutableSet) : - KtTreeVisitorVoid() { + /** + * A place where a constant is passed to an annotation parameter. + * + * The parameter's declared type is the only remaining evidence of a missing constant's type once + * the declaring class is off the classpath, so it is captured alongside the expression text. + */ + private data class ConstUsageSite( + val expressionText: String, + val annotationShortName: String?, + val parameterName: String?, + val parameterIndex: Int, + ) + + private class MissingConstantsVisitor( + private val usedConstants: MutableSet, + private val usageSites: MutableList, + ) : KtTreeVisitorVoid() { override fun visitAnnotationEntry(annotationEntry: KtAnnotationEntry) { + val annotationShortName = annotationEntry.shortName?.asString() // Use getArgumentExpression()?.text to get just the value, not the parameter name // For @Annotation(param = VALUE), we want "VALUE", not "param = VALUE" // For array expressions like [A, B], extract individual elements - annotationEntry.valueArgumentList - ?.arguments - ?.mapNotNull { it.getArgumentExpression()?.text } - ?.forEach { expressionText -> - if (expressionText.startsWith("[") && expressionText.endsWith("]")) { - // Array expression: parse and extract individual elements - val arrayContent = expressionText.substring(1, expressionText.length - 1).trim() - if (arrayContent.isNotEmpty()) { - // Simple comma split - for annotation arguments, we don't expect complex nesting - arrayContent.split(",").forEach { element -> - val trimmed = element.trim() - if (trimmed.isNotEmpty()) { - usedConstants.add(trimmed) - } - } + annotationEntry.valueArgumentList?.arguments?.forEachIndexed { index, argument -> + val expressionText = argument.getArgumentExpression()?.text ?: return@forEachIndexed + val parameterName = argument.getArgumentName()?.asName?.asString() + fun record(text: String) { + usedConstants.add(text) + usageSites.add(ConstUsageSite(text, annotationShortName, parameterName, index)) + } + if (expressionText.startsWith("[") && expressionText.endsWith("]")) { + // Array expression: parse and extract individual elements + val arrayContent = expressionText.substring(1, expressionText.length - 1).trim() + if (arrayContent.isNotEmpty()) { + // Simple comma split - for annotation arguments, we don't expect complex nesting + arrayContent.split(",").forEach { element -> + val trimmed = element.trim() + if (trimmed.isNotEmpty()) { + record(trimmed) } - } else { - // Regular expression: add as-is - usedConstants.add(expressionText) } } + } else { + // Regular expression: add as-is + record(expressionText) + } + } return super.visitAnnotationEntry(annotationEntry) } } + /** + * Resolves the declared type of the annotation parameter [site] is passed to. + * + * This runs before `buildFirFromKtFiles`, so annotations declared in the module being compiled + * are only available as PSI while annotations from dependencies are only available as FIR; both + * are tried. Returns null whenever the answer is not certain - a wrong type here would be exactly + * the defect this exists to fix. + */ + private fun resolveAnnotationParameterType( + site: ConstUsageSite, + sourceFile: KtFile, + sourceFiles: List, + session: FirSession, + ): ConstTypeHint? { + val annotationShortName = site.annotationShortName ?: return null + + val importedFqName = + sourceFile.importDirectives + .mapNotNull { it.importedFqName } + .firstOrNull { it.shortName().asString() == annotationShortName } + val candidates = listOfNotNull( + importedFqName, + sourceFile.packageFqName.child(Name.identifier(annotationShortName)), + ) + + // Recovering the type is best-effort: without it the old assumed-String behaviour still + // applies. It must therefore never be able to turn a working compilation red, which querying + // half-resolved declarations at this stage otherwise can. + return try { + candidates.firstNotNullOfOrNull { candidate -> + annotationParameterTypeFromPsi(candidate, site, sourceFiles) + ?: annotationParameterTypeFromFir(candidate, site, session) + } + } catch (e: Exception) { + pipeline.repairLog.recordFailedRepair( + annotationShortName, + "could not read annotation parameter type: ${e.javaClass.simpleName}: ${e.message}", + ) + null + } + } + + private fun annotationParameterTypeFromPsi( + annotationFqName: FqName, + site: ConstUsageSite, + sourceFiles: List, + ): ConstTypeHint? { + val ktClass = + sourceFiles + .flatMap { it.declarations } + .filterIsInstance() + .firstOrNull { it.isAnnotation() && it.fqName == annotationFqName } ?: return null + val parameters = ktClass.primaryConstructor?.valueParameters ?: return null + val parameter = + if (site.parameterName != null) { + parameters.firstOrNull { it.name == site.parameterName } + } else { + parameters.getOrNull(site.parameterIndex) + } ?: return null + return constTypeHintFromTypeText(parameter.typeReference?.text) + } + + /** + * This runs before source declarations exist in FIR, so it deliberately consults only + * `dependenciesSymbolProvider` - the same provider the surrounding missing-constant logic uses. + * Reaching into the full symbol provider here resolves source and Java declarations too early. + */ + @OptIn(SymbolInternals::class) + private fun annotationParameterTypeFromFir( + annotationFqName: FqName, + site: ConstUsageSite, + session: FirSession, + ): ConstTypeHint? { + val classSymbol = + session.dependenciesSymbolProvider.getClassLikeSymbolByClassId( + ClassId.topLevel(annotationFqName), + ) as? FirClassSymbol<*> ?: return null + val firClass = classSymbol.fir as? FirRegularClass ?: return null + if (firClass.classKind != ClassKind.ANNOTATION_CLASS) return null + val primaryConstructor = + firClass.declarations.filterIsInstance().firstOrNull { it.isPrimary } + ?: return null + val parameter = + if (site.parameterName != null) { + primaryConstructor.valueParameters.firstOrNull { + it.name.asString() == site.parameterName + } + } else { + primaryConstructor.valueParameters.getOrNull(site.parameterIndex) + } ?: return null + // Java-declared annotations carry a FirJavaTypeRef at this point; asking such a ref for its + // cone type throws. No type is better than a crash, and better than a guessed one. + val typeRef = parameter.returnTypeRef as? FirResolvedTypeRef ?: return null + return constTypeHintFromConeType(typeRef.coneType) + } + // Collect missing constants from source files by analyzing imports private fun collectMissingConstantsFromSourceFiles( sourceFiles: List, session: FirSession, + state: AbiGenState, ): Map> { val missingConstants = mutableMapOf>() @@ -833,7 +966,29 @@ class K2JvmAbiFirAnalysisHandlerExtension(private val outputPath: String) : for (sourceFile in sourceFiles) { val usedConstants = mutableSetOf() - sourceFile.accept(MissingConstantsVisitor(usedConstants)) + val usageSites = mutableListOf() + sourceFile.accept(MissingConstantsVisitor(usedConstants, usageSites)) + + // Records the annotation-parameter type for a constant we are about to synthesise. Two + // annotations disagreeing about the type means the evidence is unusable, so the constant is + // marked ambiguous and falls back rather than picking one arbitrarily. + fun recordTypeHint(classId: ClassId, propertyName: String, usage: String) { + val key = constHintKey(classId, propertyName) + if (key in state.ambiguousConstantTypeHints) return + val hint = + usageSites + .asSequence() + .filter { it.expressionText == usage } + .mapNotNull { resolveAnnotationParameterType(it, sourceFile, sourceFiles, session) } + .firstOrNull() ?: return + val existing = state.constantTypeHints[key] + if (existing == null) { + state.constantTypeHints[key] = hint + } else if (existing != hint) { + state.constantTypeHints.remove(key) + state.ambiguousConstantTypeHints.add(key) + } + } for (importDirective in sourceFile.importDirectives) { val importPath = importDirective.importedFqName ?: continue @@ -878,6 +1033,7 @@ class K2JvmAbiFirAnalysisHandlerExtension(private val outputPath: String) : ) if (classId != null) { missingConstants.getOrPut(classId) { mutableSetOf() }.add(propertyName) + recordTypeHint(classId, propertyName, usage) } } else if (usage == importedName) { // Constant import: import pkg.Class.CONSTANT, used as CONSTANT @@ -891,6 +1047,7 @@ class K2JvmAbiFirAnalysisHandlerExtension(private val outputPath: String) : ) if (classId != null) { missingConstants.getOrPut(classId) { mutableSetOf() }.add(importedName) + recordTypeHint(classId, importedName, usage) } } } @@ -1150,17 +1307,52 @@ class K2JvmAbiFirAnalysisHandlerExtension(private val outputPath: String) : ) val outputs = sessionsWithSources.map { (session, sources) -> - val missingConstants = collectMissingConstantsFromSourceFiles(sources, session) - session.jvmAbiGenService.state.missingConstants.putAll(missingConstants) + val state = session.jvmAbiGenService.state + val missingConstants = collectMissingConstantsFromSourceFiles(sources, session, state) + state.missingConstants.putAll(missingConstants) // Skip checkers - ABI generation only needs resolved types, and third-party // plugin checkers (like Litho K2) crash on unresolved references from stubs. val firFiles = session.buildFirFromKtFiles(sources) val (scopeSession, fir) = session.runResolution(firFiles) + // Constants are synthesised lazily during resolution, so this is the earliest point at + // which the full set is known. Reporting here rather than in ValidationStage keeps the + // signal even when a later phase fails. + reportFabricatedConstants(state, configuration) moduleFrontendOutputCompat(session, scopeSession, fir) } return frontendOutputCompat(outputs) } + + private fun reportFabricatedConstants(state: AbiGenState, configuration: CompilerConfiguration) { + if (state.fabricatedConstants.isEmpty()) return + pipeline.repairLog.fabricatedConstants.addAll(state.fabricatedConstants) + + val policy = + AbiRepairPolicy.parse(configuration.get(K2JvmAbiConfigurationKeys.ABI_VALIDATION_MODE)) + if (policy == AbiRepairPolicy.OFF) return + + val severity = + if (policy == AbiRepairPolicy.ERROR) CompilerMessageSeverity.ERROR + else CompilerMessageSeverity.WARNING + for (constant in state.fabricatedConstants) { + configuration.messageCollector.report( + severity, + "Kosabi source-only ABI: the value of constant `${constant.classId}.${constant.name}` " + + "is not available, because no target on the source-only ABI classpath declares it. " + + "A placeholder of type `${constant.emittedType}` " + + "(${if (constant.typeSource == ConstTypeSource.ANNOTATION_PARAMETER) + "type taken from the annotation parameter it is passed to" + else "type unknown, assumed String"}) " + + "was emitted instead, so this ABI does not match a class-ABI build. " + + "To fix, add the target that declares the constant to THIS target's " + + "`source_only_abi_deps`. Reach for `required_for_source_only_abi = True` on the " + + "declaring target only when many consumers need it: that is a global edge which " + + "puts its ABI on the source-only ABI classpath of every consumer, not just this " + + "one.", + ) + } + } } class AbiGenFirExtensionRegistrar() : FirExtensionRegistrar() { @@ -1365,17 +1557,35 @@ class MissingConstantDeclarationGenerationExtension( return missingConstants.keys.any { classId -> classId.packageFqName == packageFqName } } + /** + * Synthesises a constant whose declaring class is not on the source-only ABI classpath. + * + * The real value is unrecoverable here by construction: the constant is being synthesised + * precisely because nothing on the classpath declares it, and Kosabi never reads the provider's + * sources. Only a placeholder can be emitted, so any constant reaching this function makes the + * resulting ABI differ from a class-ABI build. The correct repair is to put the declaring target + * on the consuming target's source-only ABI classpath, by adding it to that target's + * `source_only_abi_deps`; this function only limits the damage in the meantime and records it for + * [AbiGenRepairLog]. + * + * The type, unlike the value, is often recoverable - see [resolveAnnotationParameterType]. Using + * it matters: emitting `String` for an `Int` constant makes constant evaluation fail outright + * (`ClassCastException` in `IrConstAnnotationTransformer`) rather than merely produce a wrong + * number. + */ private fun generateConstantProperty( constantName: String, owner: FirClassSymbol<*>, ): FirPropertySymbol { - // Default to String type with empty string value - // Only constants that are not available in dependencies are generated + val state = session.jvmAbiGenService.state + val key = constHintKey(owner.classId, constantName) + val hint = state.constantTypeHints[key] + val property = createMemberProperty( owner, JvmAbiGenPlugin, Name.identifier(constantName), - session.builtinTypes.stringType.coneType, + hint?.coneType(session) ?: session.builtinTypes.stringType.coneType, ) property.replaceStatus( FirResolvedDeclarationStatusImpl( @@ -1388,15 +1598,107 @@ class MissingConstantDeclarationGenerationExtension( property.replaceInitializer( buildLiteralExpression( source = null, - kind = ConstantValueKind.String, - value = "", + kind = hint?.constantValueKind ?: ConstantValueKind.String, + value = hint?.placeholderValue ?: "", setType = true, ), ) + + state.fabricatedConstants.add( + FabricatedConstant( + classId = owner.classId.asString(), + name = constantName, + emittedType = hint?.typeName ?: "String", + typeSource = + if (hint != null) ConstTypeSource.ANNOTATION_PARAMETER + else ConstTypeSource.ASSUMED_STRING, + ), + ) return property.symbol } } +/** A constant type Kosabi is able to synthesise a placeholder literal for. */ +enum class ConstTypeHint( + val typeName: String, + val constantValueKind: ConstantValueKind, + val placeholderValue: Any, +) { + BOOLEAN("Boolean", ConstantValueKind.Boolean, false), + BYTE("Byte", ConstantValueKind.Byte, 0.toByte()), + SHORT("Short", ConstantValueKind.Short, 0.toShort()), + INT("Int", ConstantValueKind.Int, 0), + LONG("Long", ConstantValueKind.Long, 0L), + FLOAT("Float", ConstantValueKind.Float, 0.0f), + DOUBLE("Double", ConstantValueKind.Double, 0.0), + CHAR("Char", ConstantValueKind.Char, '\u0000'), + STRING("String", ConstantValueKind.String, ""); + + fun coneType(session: FirSession): ConeKotlinType = + when (this) { + BOOLEAN -> session.builtinTypes.booleanType.coneType + BYTE -> session.builtinTypes.byteType.coneType + SHORT -> session.builtinTypes.shortType.coneType + INT -> session.builtinTypes.intType.coneType + LONG -> session.builtinTypes.longType.coneType + FLOAT -> session.builtinTypes.floatType.coneType + DOUBLE -> session.builtinTypes.doubleType.coneType + CHAR -> session.builtinTypes.charType.coneType + STRING -> session.builtinTypes.stringType.coneType + } + + companion object { + fun bySimpleName(name: String): ConstTypeHint? = entries.firstOrNull { it.typeName == name } + } +} + +internal fun constHintKey(classId: ClassId, constantName: String): String = + "${classId.asString()}.$constantName" + +/** + * Maps a type as written in source to a constant kind. + * + * Array types are unwrapped once because no constant can itself be an array - an array-typed + * annotation parameter is always fed element-wise. Anything not recognised (a typealias, an enum, a + * generic) yields null so the caller falls back instead of guessing. + */ +internal fun constTypeHintFromTypeText(typeText: String?): ConstTypeHint? { + var text = typeText?.trim()?.removeSuffix("?")?.trim() ?: return null + if (text.endsWith(">")) { + val open = text.indexOf('<') + if (open <= 0) return null + if (text.take(open).substringAfterLast('.') != "Array") return null + text = text.substring(open + 1, text.length - 1).removePrefix("out ").trim() + } + val simpleName = text.substringAfterLast('.').removeSuffix("?").trim() + ConstTypeHint.bySimpleName(simpleName)?.let { + return it + } + return when (simpleName) { + "BooleanArray" -> ConstTypeHint.BOOLEAN + "ByteArray" -> ConstTypeHint.BYTE + "ShortArray" -> ConstTypeHint.SHORT + "IntArray" -> ConstTypeHint.INT + "LongArray" -> ConstTypeHint.LONG + "FloatArray" -> ConstTypeHint.FLOAT + "DoubleArray" -> ConstTypeHint.DOUBLE + "CharArray" -> ConstTypeHint.CHAR + else -> null + } +} + +/** FIR counterpart of [constTypeHintFromTypeText], for annotations coming from dependencies. */ +internal fun constTypeHintFromConeType(type: ConeKotlinType): ConstTypeHint? { + val classId = (type as? ConeClassLikeType)?.lookupTag?.classId ?: return null + val fqName = classId.asFqNameString() + if (fqName == "kotlin.Array") { + val elementType = + (type.typeArguments.firstOrNull() as? ConeKotlinTypeProjection)?.type ?: return null + return constTypeHintFromConeType(elementType) + } + return constTypeHintFromTypeText(fqName.takeIf { it.startsWith("kotlin.") }) +} + class JvmAbiGenService(session: FirSession, state: AbiGenState) : FirExtensionSessionComponent(session) { companion object { @@ -1410,6 +1712,18 @@ class JvmAbiGenService(session: FirSession, state: AbiGenState) : class AbiGenState { val missingConstants: MutableMap> = mutableMapOf() + + /** + * Declared type of each synthesised constant, keyed by [constHintKey], recovered from the + * annotation parameter it is passed to. Absent means the type could not be established. + */ + val constantTypeHints: MutableMap = mutableMapOf() + + /** Constants whose usages disagreed about the type; treated the same as having no hint. */ + val ambiguousConstantTypeHints: MutableSet = mutableSetOf() + + /** Every constant synthesised during this compilation. See [AbiGenRepairLog]. */ + val fabricatedConstants: MutableList = mutableListOf() // Track methods from internal interfaces that need to be generated for classes // Key: owning class's ClassId, Value: List of interface method details val internalInterfaceMethods: MutableMap> = diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/k2-jvm-abi-gen/src/main/kotlin/compilercompat/2.2.0/IrNonAbiStripper.kt b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/k2-jvm-abi-gen/src/main/kotlin/compilercompat/2.2.0/IrNonAbiStripper.kt index 68bb15f..6eecba7 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/k2-jvm-abi-gen/src/main/kotlin/compilercompat/2.2.0/IrNonAbiStripper.kt +++ b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/k2-jvm-abi-gen/src/main/kotlin/compilercompat/2.2.0/IrNonAbiStripper.kt @@ -20,12 +20,14 @@ package com.facebook import com.facebook.buck.jvm.kotlin.compilerplugins.common.isStub import org.jetbrains.kotlin.backend.common.extensions.IrGenerationExtension import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext +import org.jetbrains.kotlin.backend.common.lower.DeclarationIrBuilder import org.jetbrains.kotlin.descriptors.ClassKind import org.jetbrains.kotlin.descriptors.DescriptorVisibilities import org.jetbrains.kotlin.descriptors.Modality import org.jetbrains.kotlin.ir.IrBuiltIns import org.jetbrains.kotlin.ir.IrElement import org.jetbrains.kotlin.ir.IrStatement +import org.jetbrains.kotlin.ir.builders.irCallConstructor import org.jetbrains.kotlin.ir.declarations.* import org.jetbrains.kotlin.ir.expressions.* import org.jetbrains.kotlin.ir.expressions.impl.IrConstImpl @@ -33,12 +35,17 @@ import org.jetbrains.kotlin.ir.expressions.impl.IrReturnImpl import org.jetbrains.kotlin.ir.symbols.IrReturnTargetSymbol import org.jetbrains.kotlin.ir.symbols.UnsafeDuringIrConstructionAPI import org.jetbrains.kotlin.ir.types.* +import org.jetbrains.kotlin.ir.util.constructors import org.jetbrains.kotlin.ir.util.kotlinFqName +import org.jetbrains.kotlin.ir.util.parentAsClass +import org.jetbrains.kotlin.ir.util.primaryConstructor import org.jetbrains.kotlin.name.Name import org.jetbrains.kotlin.psi.KtFile -internal class NonAbiDeclarationsStrippingIrExtension(private val sourceFiles: List) : - IrGenerationExtension { +internal class NonAbiDeclarationsStrippingIrExtension( + private val sourceFiles: List, + private val repairLog: AbiGenRepairLog, +) : IrGenerationExtension { private fun shouldStripAnnotation(annotation: IrConstructorCall): Boolean { val annotationClass = annotation.symbol.owner.parent as? IrClass ?: return false @@ -204,7 +211,12 @@ internal class NonAbiDeclarationsStrippingIrExtension(private val sourceFiles: L stripSourceRetentionAnnotations(moduleFragment) moduleFragment.transform( - NonAbiDeclarationsStrippingIrVisitor(pluginContext.irFactory, pluginContext.irBuiltIns), + NonAbiDeclarationsStrippingIrVisitor( + pluginContext.irFactory, + pluginContext.irBuiltIns, + pluginContext, + repairLog, + ), null, ) } @@ -214,6 +226,8 @@ internal class NonAbiDeclarationsStrippingIrExtension(private val sourceFiles: L internal class NonAbiDeclarationsStrippingIrVisitor( private val irFactory: IrFactory, private val irBuiltins: IrBuiltIns, + private val pluginContext: IrPluginContext, + private val repairLog: AbiGenRepairLog, ) : IrElementTransformerVoidCompat() { override fun visitFile(declaration: IrFile): IrFile { @@ -229,23 +243,20 @@ internal class NonAbiDeclarationsStrippingIrVisitor( } override fun visitClass(declaration: IrClass): IrStatement { - // Strip PRIVATE supertypes from the class's implemented interfaces. - // Internal supertypes are kept because source-only ABI is consumed within the same + // Strip supertypes whose class file the ABI jar will not contain, since the reference would + // dangle. Internal supertypes are kept because source-only ABI is consumed within the same // module, where internal types are accessible. Stripping them would cause Java consumers // to see "incompatible types" errors when a public class implements an internal interface. - - // First, collect the supertypes that will be stripped (private only) val strippedSupertypes = declaration.superTypes.filter { superType -> val superClass = superType.classOrNull?.owner ?: return@filter false - isClassPrivate(superClass) + isClassAbsentFromAbi(superClass) } - // Strip the private supertypes declaration.superTypes = declaration.superTypes.filter { superType -> val superClass = superType.classOrNull?.owner ?: return@filter true - !isClassPrivate(superClass) + !isClassAbsentFromAbi(superClass) } // For each stripped supertype that was an interface, convert fake override methods @@ -273,13 +284,13 @@ internal class NonAbiDeclarationsStrippingIrVisitor( if (!decl.isFakeOverride) continue if (!decl.visibility.isPublicAPI) continue - // Check if any of the overridden symbols is from a class that implements a non-public - // interface + // Materialize only when the declaring class is gone from the ABI. If it survives -- any + // nested private class does -- the fake override still resolves, and emitting a body here + // would add a method the library jar does not have. val shouldMaterialize = decl.overriddenSymbols.any { overriddenSymbol -> - val overridden = overriddenSymbol.owner - val overriddenParent = overridden.parent as? IrClass - overriddenParent != null && !isClassPubliclyAccessible(overriddenParent) + val overriddenParent = overriddenSymbol.owner.parent as? IrClass + overriddenParent != null && isClassAbsentFromAbi(overriddenParent) } if (shouldMaterialize) { @@ -352,33 +363,22 @@ internal class NonAbiDeclarationsStrippingIrVisitor( } } - // Check if a class is publicly accessible (it and all its containing classes are public) - private fun isClassPubliclyAccessible(irClass: IrClass): Boolean { - var current: IrClass? = irClass - while (current != null) { - if (!current.visibility.isPublicAPI) { - return false - } - // Get the containing class, if any - current = current.parent as? IrClass - } - return true - } - - // Check if a class or any of its containing classes is private/local (not internal or public). - // Internal classes are accessible within the same module (source-only ABI scope). - private fun isClassPrivate(irClass: IrClass): Boolean { - var current: IrClass? = irClass - while (current != null) { - val visibility = current.visibility - if ( - visibility == DescriptorVisibilities.PRIVATE || visibility == DescriptorVisibilities.LOCAL - ) { + // True when removeNonPublicApi will drop this class from the ABI, which is the only reason a + // supertype has to leave a supertype list. Only top-level private classes are dropped; a nested + // private class is kept, so a `private sealed class`/`sealed interface` and everything nested + // under it stays referenceable and must keep appearing as a supertype. + private fun isClassAbsentFromAbi(irClass: IrClass): Boolean { + var current: IrClass = irClass + while (true) { + if (current.visibility == DescriptorVisibilities.LOCAL) { return true } - current = current.parent as? IrClass + val outer = current.parent as? IrClass + if (outer == null) { + return current.visibility == DescriptorVisibilities.PRIVATE + } + current = outer } - return false } override fun visitField(declaration: IrField): IrStatement { @@ -397,6 +397,14 @@ internal class NonAbiDeclarationsStrippingIrVisitor( val defaultExpressionBody = generateDefaultExpressionBody(declaration.type) if (defaultExpressionBody != null) { declaration.initializer = defaultExpressionBody + // For a non-const field this only affects the initializer, which is not part of the + // ABI. For a const val it rewrites the ConstantValue attribute consumers inline. + val ownerPrefix = (declaration.parent as? IrClass)?.kotlinFqName?.asString()?.plus(".") + repairLog.recordReplacedFieldInitializer( + (ownerPrefix ?: "") + declaration.name.asString(), + "initializer contained a call that source-only ABI cannot evaluate; " + + "replaced with the default value for ${declaration.type.classFqName?.asString()}", + ) } } } @@ -427,43 +435,85 @@ internal class NonAbiDeclarationsStrippingIrVisitor( // For primitive types, create a default constant value // The code handles properties like val DEFAULT_SHADOW_COLOR: Int = Color.argb(128, 0, 0, 0) // If we don't set a default value, compiler crashes because it expects a ConstExpression + // Primitives are matched by classifier (isInt/isLong/...) rather than by instance equality with + // irBuiltins: a value class deserialized from another module carries a non-canonical underlying + // primitive type that is not `==` to irBuiltins.longType, and matching it by instance used to + // drop through to the null branch -- storing ACONST_NULL into an unboxed primitive slot. The + // predicates are non-null-only, so a nullable primitive (a boxed, reference slot) still falls + // through to null, which is correct there. var defaultValue: IrExpression? = when { - constructedType == irBuiltins.intType -> IrConstImpl.int(-1, -1, irBuiltins.intType, 0) - constructedType == irBuiltins.booleanType -> - IrConstImpl.boolean(-1, -1, irBuiltins.booleanType, false) - constructedType == irBuiltins.stringType -> - IrConstImpl.string(-1, -1, irBuiltins.stringType, "") - constructedType == irBuiltins.doubleType -> - IrConstImpl.double(-1, -1, irBuiltins.doubleType, 0.0) - constructedType == irBuiltins.floatType -> - IrConstImpl.float(-1, -1, irBuiltins.floatType, 0.0f) - constructedType == irBuiltins.longType -> - IrConstImpl.long(-1, -1, irBuiltins.longType, 0L) - constructedType == irBuiltins.charType -> - IrConstImpl.char(-1, -1, irBuiltins.charType, '\u0000') - constructedType == irBuiltins.byteType -> IrConstImpl.byte(-1, -1, irBuiltins.byteType, 0) - constructedType == irBuiltins.shortType -> - IrConstImpl.short(-1, -1, irBuiltins.shortType, 0) + constructedType.isInt() -> IrConstImpl.int(-1, -1, irBuiltins.intType, 0) + constructedType.isBoolean() -> IrConstImpl.boolean(-1, -1, irBuiltins.booleanType, false) + constructedType.isString() -> IrConstImpl.string(-1, -1, irBuiltins.stringType, "") + constructedType.isDouble() -> IrConstImpl.double(-1, -1, irBuiltins.doubleType, 0.0) + constructedType.isFloat() -> IrConstImpl.float(-1, -1, irBuiltins.floatType, 0.0f) + constructedType.isLong() -> IrConstImpl.long(-1, -1, irBuiltins.longType, 0L) + constructedType.isChar() -> IrConstImpl.char(-1, -1, irBuiltins.charType, '\u0000') + constructedType.isByte() -> IrConstImpl.byte(-1, -1, irBuiltins.byteType, 0) + constructedType.isShort() -> IrConstImpl.short(-1, -1, irBuiltins.shortType, 0) else -> - IrConstImpl.constNull( - -1, - -1, - constructedType.makeNullable(), - ) // If we can't handle this type, return null + // For value/inline classes (e.g. Compose's `Color`, which wraps `ULong`/`long`) a + // null default is wrong: the JVM slot is the unboxed primitive, so the + // compiler-generated `$default` method would emit ACONST_NULL into e.g. a `long` + // slot and fail bytecode verification ("Expected J, but found R"). Build a boxed + // default from the underlying primitive so inline-class lowering unboxes it to the + // correct primitive zero. Falls back to null for everything else. + generateInlineClassDefaultValue(constructedType) + ?: IrConstImpl.constNull( + -1, + -1, + constructedType.makeNullable(), + ) // If we can't handle this type, return null } return defaultValue } + // Build a default value for a non-null value/inline class over a primitive, whose `$default` + // slot is the *unboxed* primitive. Returns null (leaving the caller's ACONST_NULL fallback) for a + // non-value class or a nullable value class -- which is boxed, so its slot is a reference and + // null + // is correct. + // + // The underlying primitive is read from the class's inline-class representation, which survives + // deserialization even when the value class's constructor does not: a value class reached through + // a source-only-ABI dependency deserializes with zero constructors (e.g. WA + // `TranscriptionStatus`, + // `ctors=0`). When a constructor *is* materialized we invoke it, so inline-class lowering unboxes + // the call to the primitive zero (ICONST_0/LCONST_0). When it is not, we emit the underlying + // primitive directly -- the slot is already that primitive, so a bare zero verifies -- rather + // than + // dropping through to ACONST_NULL and failing bytecode verification ("Expected I, but found R"). + private fun generateInlineClassDefaultValue(type: IrSimpleType): IrExpression? { + val irClass = type.classOrNull?.owner ?: return null + if (!irClass.isValue) return null + if (type.isNullable()) return null + val constructor = irClass.primaryConstructor ?: irClass.constructors.singleOrNull() + val underlyingType = + irClass.inlineClassRepresentation?.underlyingType + ?: constructor?.valueParameters?.singleOrNull()?.type + ?: return null + val underlyingDefault = generateDefaultValue(underlyingType) ?: return null + return if (constructor != null && constructor.valueParameters.size == 1) { + DeclarationIrBuilder(pluginContext, constructor.symbol) + .irCallConstructor(constructor.symbol, emptyList()) + .apply { putValueArgument(0, underlyingDefault) } + } else { + underlyingDefault + } + } + private fun IrDeclarationContainer.removeNonPublicApi() { // For inline/value classes, we need to keep the backing field even if it's private val inlineClassBackingFieldName = (this as? IrClass)?.inlineClassRepresentation?.underlyingPropertyName + this.declarations.filterIsInstance().forEach { property -> + if (property.isDelegated) property.backingField = null + } + this.declarations.removeAll { declaration -> - // Keep synthetic declarations (generated by compiler) - if (declaration.origin.isSynthetic) return@removeAll false // Keep constructors (needed for instantiation) if (declaration is IrConstructor) return@removeAll false // Keep companion objects (may contain public members) @@ -482,6 +532,11 @@ internal class NonAbiDeclarationsStrippingIrVisitor( if (declaration.isInlineClassBackingMember(inlineClassBackingFieldName)) return@removeAll false + if (declaration.isPropertyDelegateBackingField()) return@removeAll true + + // Keep synthetic declarations (generated by compiler) + if (declaration.origin.isSynthetic) return@removeAll false + // Remove private/local members only (NOT internal - K1 kept internal in ABI) val visibility = (declaration as? IrDeclarationWithVisibility)?.visibility visibility == DescriptorVisibilities.PRIVATE || @@ -490,6 +545,11 @@ internal class NonAbiDeclarationsStrippingIrVisitor( } } + private fun IrDeclaration.isPropertyDelegateBackingField(): Boolean = + this is IrField && + visibility == DescriptorVisibilities.PRIVATE && + origin == IrDeclarationOrigin.PROPERTY_DELEGATE + private fun IrDeclaration.isInlineClassBackingMember(backingFieldName: Name?): Boolean { if (backingFieldName == null) return false return (this is IrField || this is IrProperty) && @@ -514,6 +574,21 @@ internal class NonAbiDeclarationsStrippingIrVisitor( return irFactory.createExpressionBody(-1, -1, defaultValue) } + // Replace each defaulted value parameter's default with a fabricated, type-correct default. + // For source-only ABI only the presence and type of a default matter, not its value. This + // turns a value-class-over-primitive default into a boxed constructor call, which inline-class + // lowering unboxes to the primitive zero, rather than the ACONST_NULL codegen would otherwise + // store into the synthetic `$default` overload's unboxed slot ("Expected I, but found R"). + // Shared by visitSimpleFunction and visitConstructor -- the latter is where the WA + // TranscriptionViewModel.VmState `$default` crash lived. + private fun regenerateDefaultParameterValues(function: IrFunction) { + function.valueParameters + .filter { it.defaultValue != null } + .forEach { parameter -> + generateDefaultExpressionBody(parameter.type)?.let { parameter.defaultValue = it } + } + } + override fun visitSimpleFunction(declaration: IrSimpleFunction): IrStatement { if (!declaration.origin.isSynthetic) { if (declaration.parent is IrProperty) { @@ -528,17 +603,26 @@ internal class NonAbiDeclarationsStrippingIrVisitor( } else { declaration.body = irFactory.createBlockBody(-1, -1) } - val parametersWithDefaultValues = - declaration.valueParameters.filter { it.defaultValue != null } - for (parameter in parametersWithDefaultValues) { - // if we can - we resolve the default value to a constant - // handles default values in functions like fun foo(x: Int = Something.SomeValue) - generateDefaultExpressionBody(parameter.type)?.let { parameter.defaultValue = it } - } + // handles default values in functions like fun foo(x: Int = Something.SomeValue) + regenerateDefaultParameterValues(declaration) } return super.visitSimpleFunction(declaration) } + override fun visitConstructor(declaration: IrConstructor): IrStatement { + // A defaulted value-class-over-primitive constructor parameter reaches the synthetic + // `$default` with an unboxed slot; regenerate its default as visitSimpleFunction does + // so the slot gets the unboxed primitive zero rather than ACONST_NULL. Bodies are left to the + // default traversal -- unlike 2.3.0, this compiler path needs no constructor body stubbing. + // Skip annotation classes: their constructor parameter defaults ARE the ABI (the + // `AnnotationDefault` attribute), so regenerating them drops the element default and breaks + // consumers ("annotation @X is missing a default value for the element ..."). + if (declaration.parentAsClass.kind != ClassKind.ANNOTATION_CLASS) { + regenerateDefaultParameterValues(declaration) + } + return super.visitConstructor(declaration) + } + override fun visitAnonymousInitializer( declaration: IrAnonymousInitializer, ): IrStatement { diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/k2-jvm-abi-gen/src/main/kotlin/compilercompat/2.3.0/IrNonAbiStripper.kt b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/k2-jvm-abi-gen/src/main/kotlin/compilercompat/2.3.0/IrNonAbiStripper.kt index 4db8a1d..749915f 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/k2-jvm-abi-gen/src/main/kotlin/compilercompat/2.3.0/IrNonAbiStripper.kt +++ b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/k2-jvm-abi-gen/src/main/kotlin/compilercompat/2.3.0/IrNonAbiStripper.kt @@ -45,8 +45,10 @@ import org.jetbrains.kotlin.ir.util.primaryConstructor import org.jetbrains.kotlin.name.Name import org.jetbrains.kotlin.psi.KtFile -internal class NonAbiDeclarationsStrippingIrExtension(private val sourceFiles: List) : - IrGenerationExtension { +internal class NonAbiDeclarationsStrippingIrExtension( + private val sourceFiles: List, + private val repairLog: AbiGenRepairLog, +) : IrGenerationExtension { private fun shouldStripAnnotation(annotation: IrConstructorCall): Boolean { val annotationClass = annotation.symbol.owner.parent as? IrClass ?: return false @@ -216,6 +218,7 @@ internal class NonAbiDeclarationsStrippingIrExtension(private val sourceFiles: L pluginContext.irFactory, pluginContext.irBuiltIns, pluginContext, + repairLog, ), null, ) @@ -227,6 +230,7 @@ internal class NonAbiDeclarationsStrippingIrVisitor( private val irFactory: IrFactory, private val irBuiltins: IrBuiltIns, private val pluginContext: IrPluginContext, + private val repairLog: AbiGenRepairLog, ) : IrElementTransformerVoidCompat() { override fun visitFile(declaration: IrFile): IrFile { @@ -242,23 +246,20 @@ internal class NonAbiDeclarationsStrippingIrVisitor( } override fun visitClass(declaration: IrClass): IrStatement { - // Strip PRIVATE supertypes from the class's implemented interfaces. - // Internal supertypes are kept because source-only ABI is consumed within the same + // Strip supertypes whose class file the ABI jar will not contain, since the reference would + // dangle. Internal supertypes are kept because source-only ABI is consumed within the same // module, where internal types are accessible. Stripping them would cause Java consumers // to see "incompatible types" errors when a public class implements an internal interface. - - // First, collect the supertypes that will be stripped (private only) val strippedSupertypes = declaration.superTypes.filter { superType -> val superClass = superType.classOrNull?.owner ?: return@filter false - isClassPrivate(superClass) + isClassAbsentFromAbi(superClass) } - // Strip the private supertypes declaration.superTypes = declaration.superTypes.filter { superType -> val superClass = superType.classOrNull?.owner ?: return@filter true - !isClassPrivate(superClass) + !isClassAbsentFromAbi(superClass) } // For each stripped supertype that was an interface, convert fake override methods @@ -286,13 +287,13 @@ internal class NonAbiDeclarationsStrippingIrVisitor( if (!decl.isFakeOverride) continue if (!decl.visibility.isPublicAPI) continue - // Check if any of the overridden symbols is from a class that implements a non-public - // interface + // Materialize only when the declaring class is gone from the ABI. If it survives -- any + // nested private class does -- the fake override still resolves, and emitting a body here + // would add a method the library jar does not have. val shouldMaterialize = decl.overriddenSymbols.any { overriddenSymbol -> - val overridden = overriddenSymbol.owner - val overriddenParent = overridden.parent as? IrClass - overriddenParent != null && !isClassPubliclyAccessible(overriddenParent) + val overriddenParent = overriddenSymbol.owner.parent as? IrClass + overriddenParent != null && isClassAbsentFromAbi(overriddenParent) } if (shouldMaterialize) { @@ -365,33 +366,22 @@ internal class NonAbiDeclarationsStrippingIrVisitor( } } - // Check if a class is publicly accessible (it and all its containing classes are public) - private fun isClassPubliclyAccessible(irClass: IrClass): Boolean { - var current: IrClass? = irClass - while (current != null) { - if (!current.visibility.isPublicAPI) { - return false - } - // Get the containing class, if any - current = current.parent as? IrClass - } - return true - } - - // Check if a class or any of its containing classes is private/local (not internal or public). - // Internal classes are accessible within the same module (source-only ABI scope). - private fun isClassPrivate(irClass: IrClass): Boolean { - var current: IrClass? = irClass - while (current != null) { - val visibility = current.visibility - if ( - visibility == DescriptorVisibilities.PRIVATE || visibility == DescriptorVisibilities.LOCAL - ) { + // True when removeNonPublicApi will drop this class from the ABI, which is the only reason a + // supertype has to leave a supertype list. Only top-level private classes are dropped; a nested + // private class is kept, so a `private sealed class`/`sealed interface` and everything nested + // under it stays referenceable and must keep appearing as a supertype. + private fun isClassAbsentFromAbi(irClass: IrClass): Boolean { + var current: IrClass = irClass + while (true) { + if (current.visibility == DescriptorVisibilities.LOCAL) { return true } - current = current.parent as? IrClass + val outer = current.parent as? IrClass + if (outer == null) { + return current.visibility == DescriptorVisibilities.PRIVATE + } + current = outer } - return false } override fun visitField(declaration: IrField): IrStatement { @@ -410,6 +400,14 @@ internal class NonAbiDeclarationsStrippingIrVisitor( val defaultExpressionBody = generateDefaultExpressionBody(declaration.type) if (defaultExpressionBody != null) { declaration.initializer = defaultExpressionBody + // For a non-const field this only affects the initializer, which is not part of the + // ABI. For a const val it rewrites the ConstantValue attribute consumers inline. + val ownerPrefix = (declaration.parent as? IrClass)?.kotlinFqName?.asString()?.plus(".") + repairLog.recordReplacedFieldInitializer( + (ownerPrefix ?: "") + declaration.name.asString(), + "initializer contained a call that source-only ABI cannot evaluate; " + + "replaced with the default value for ${declaration.type.classFqName?.asString()}", + ) } } } @@ -474,24 +472,23 @@ internal class NonAbiDeclarationsStrippingIrVisitor( // For primitive types, create a default constant value // The code handles properties like val DEFAULT_SHADOW_COLOR: Int = Color.argb(128, 0, 0, 0) // If we don't set a default value, compiler crashes because it expects a ConstExpression + // Primitives are matched by classifier (isInt/isLong/...) rather than by instance equality with + // irBuiltins: a value class deserialized from another module carries a non-canonical underlying + // primitive type that is not `==` to irBuiltins.longType, and matching it by instance used to + // drop through to the null branch -- storing ACONST_NULL into an unboxed primitive slot. The + // predicates are non-null-only, so a nullable primitive (a boxed, reference slot) still falls + // through to null, which is correct there. var defaultValue: IrExpression? = when { - constructedType == irBuiltins.intType -> IrConstImpl.int(-1, -1, irBuiltins.intType, 0) - constructedType == irBuiltins.booleanType -> - IrConstImpl.boolean(-1, -1, irBuiltins.booleanType, false) - constructedType == irBuiltins.stringType -> - IrConstImpl.string(-1, -1, irBuiltins.stringType, "") - constructedType == irBuiltins.doubleType -> - IrConstImpl.double(-1, -1, irBuiltins.doubleType, 0.0) - constructedType == irBuiltins.floatType -> - IrConstImpl.float(-1, -1, irBuiltins.floatType, 0.0f) - constructedType == irBuiltins.longType -> - IrConstImpl.long(-1, -1, irBuiltins.longType, 0L) - constructedType == irBuiltins.charType -> - IrConstImpl.char(-1, -1, irBuiltins.charType, '\u0000') - constructedType == irBuiltins.byteType -> IrConstImpl.byte(-1, -1, irBuiltins.byteType, 0) - constructedType == irBuiltins.shortType -> - IrConstImpl.short(-1, -1, irBuiltins.shortType, 0) + constructedType.isInt() -> IrConstImpl.int(-1, -1, irBuiltins.intType, 0) + constructedType.isBoolean() -> IrConstImpl.boolean(-1, -1, irBuiltins.booleanType, false) + constructedType.isString() -> IrConstImpl.string(-1, -1, irBuiltins.stringType, "") + constructedType.isDouble() -> IrConstImpl.double(-1, -1, irBuiltins.doubleType, 0.0) + constructedType.isFloat() -> IrConstImpl.float(-1, -1, irBuiltins.floatType, 0.0f) + constructedType.isLong() -> IrConstImpl.long(-1, -1, irBuiltins.longType, 0L) + constructedType.isChar() -> IrConstImpl.char(-1, -1, irBuiltins.charType, '\u0000') + constructedType.isByte() -> IrConstImpl.byte(-1, -1, irBuiltins.byteType, 0) + constructedType.isShort() -> IrConstImpl.short(-1, -1, irBuiltins.shortType, 0) else -> // For value/inline classes (e.g. Compose's `Color`, which wraps `ULong`/`long`) a // null default is wrong: the JVM slot is the unboxed primitive, so the @@ -510,18 +507,38 @@ internal class NonAbiDeclarationsStrippingIrVisitor( return defaultValue } - // Build a default value for a value/inline class by invoking its primary constructor with the - // (recursively derived) default of its single underlying field. Returns null when the type is not - // a value class or its constructor is unavailable. + // Build a default value for a non-null value/inline class over a primitive, whose `$default` + // slot is the *unboxed* primitive. Returns null (leaving the caller's ACONST_NULL fallback) for a + // non-value class or a nullable value class -- which is boxed, so its slot is a reference and + // null + // is correct. + // + // The underlying primitive is read from the class's inline-class representation, which survives + // deserialization even when the value class's constructor does not: a value class reached through + // a source-only-ABI dependency deserializes with zero constructors (e.g. WA + // `TranscriptionStatus`, + // `ctors=0`). When a constructor *is* materialized we invoke it, so inline-class lowering unboxes + // the call to the primitive zero (ICONST_0/LCONST_0). When it is not, we emit the underlying + // primitive directly -- the slot is already that primitive, so a bare zero verifies -- rather + // than + // dropping through to ACONST_NULL and failing bytecode verification ("Expected I, but found R"). private fun generateInlineClassDefaultValue(type: IrSimpleType): IrExpression? { val irClass = type.classOrNull?.owner ?: return null if (!irClass.isValue) return null - val constructor = irClass.primaryConstructor ?: return null - val underlyingParam = constructor.valueParameters.singleOrNull() ?: return null - val underlyingDefault = generateDefaultValue(underlyingParam.type) ?: return null - return DeclarationIrBuilder(pluginContext, constructor.symbol) - .irCallConstructor(constructor.symbol, emptyList()) - .apply { putValueArgument(0, underlyingDefault) } + if (type.isNullable()) return null + val constructor = irClass.primaryConstructor ?: irClass.constructors.singleOrNull() + val underlyingType = + irClass.inlineClassRepresentation?.underlyingType + ?: constructor?.valueParameters?.singleOrNull()?.type + ?: return null + val underlyingDefault = generateDefaultValue(underlyingType) ?: return null + return if (constructor != null && constructor.valueParameters.size == 1) { + DeclarationIrBuilder(pluginContext, constructor.symbol) + .irCallConstructor(constructor.symbol, emptyList()) + .apply { putValueArgument(0, underlyingDefault) } + } else { + underlyingDefault + } } private fun IrDeclarationContainer.removeNonPublicApi() { @@ -529,9 +546,11 @@ internal class NonAbiDeclarationsStrippingIrVisitor( val inlineClassBackingFieldName = (this as? IrClass)?.inlineClassRepresentation?.underlyingPropertyName + this.declarations.filterIsInstance().forEach { property -> + if (property.isDelegated) property.backingField = null + } + this.declarations.removeAll { declaration -> - // Keep synthetic declarations (generated by compiler) - if (declaration.origin.isSynthetic) return@removeAll false // Keep constructors (needed for instantiation) if (declaration is IrConstructor) return@removeAll false // Keep companion objects (may contain public members) @@ -550,6 +569,11 @@ internal class NonAbiDeclarationsStrippingIrVisitor( if (declaration.isInlineClassBackingMember(inlineClassBackingFieldName)) return@removeAll false + if (declaration.isPropertyDelegateBackingField()) return@removeAll true + + // Keep synthetic declarations (generated by compiler) + if (declaration.origin.isSynthetic) return@removeAll false + // Remove private/local members only (NOT internal - K1 kept internal in ABI) val visibility = (declaration as? IrDeclarationWithVisibility)?.visibility visibility == DescriptorVisibilities.PRIVATE || @@ -558,6 +582,11 @@ internal class NonAbiDeclarationsStrippingIrVisitor( } } + private fun IrDeclaration.isPropertyDelegateBackingField(): Boolean = + this is IrField && + visibility == DescriptorVisibilities.PRIVATE && + origin == IrDeclarationOrigin.PROPERTY_DELEGATE + private fun IrDeclaration.isInlineClassBackingMember(backingFieldName: Name?): Boolean { if (backingFieldName == null) return false return (this is IrField || this is IrProperty) && @@ -582,6 +611,21 @@ internal class NonAbiDeclarationsStrippingIrVisitor( return irFactory.createExpressionBody(-1, -1, defaultValue) } + // Replace each defaulted value parameter's default with a fabricated, type-correct default. + // For source-only ABI only the presence and type of a default matter, not its value. This + // turns a value-class-over-primitive default into a boxed constructor call, which inline-class + // lowering unboxes to the primitive zero, rather than the ACONST_NULL codegen would otherwise + // store into the synthetic `$default` overload's unboxed slot ("Expected I, but found R"). + // Shared by visitSimpleFunction and visitConstructor -- the latter is where the WA + // TranscriptionViewModel.VmState `$default` crash lived. + private fun regenerateDefaultParameterValues(function: IrFunction) { + function.valueParameters + .filter { it.defaultValue != null } + .forEach { parameter -> + generateDefaultExpressionBody(parameter.type)?.let { parameter.defaultValue = it } + } + } + override fun visitSimpleFunction(declaration: IrSimpleFunction): IrStatement { if (!declaration.origin.isSynthetic) { if (declaration.parent is IrProperty) { @@ -596,13 +640,8 @@ internal class NonAbiDeclarationsStrippingIrVisitor( } else { declaration.body = irFactory.createBlockBody(-1, -1) } - val parametersWithDefaultValues = - declaration.valueParameters.filter { it.defaultValue != null } - for (parameter in parametersWithDefaultValues) { - // if we can - we resolve the default value to a constant - // handles default values in functions like fun foo(x: Int = Something.SomeValue) - generateDefaultExpressionBody(parameter.type)?.let { parameter.defaultValue = it } - } + // handles default values in functions like fun foo(x: Int = Something.SomeValue) + regenerateDefaultParameterValues(declaration) } else if (declaration.body?.containsErrorExpression() == true) { // Synthetic dispatchers -- most commonly the `$default` overload generated for a // function/data-class member with default parameters -- keep a when/IrErrorExpression @@ -630,25 +669,50 @@ internal class NonAbiDeclarationsStrippingIrVisitor( // the `(..., int mask, DefaultConstructorMarker)` overload) reaches codegen with a // body that still holds Kosabi's `skipBodies` IrErrorExpression placeholders, which // ExpressionCodegen cannot emit. - // For most classes a stub (empty) body is sufficient -- ABI jars are compile-classpath-only, - // only the signature matters, and ordinary constructors keep the delegating call fir2ir - // still emits for them. + // A stub body must still carry a delegating constructor call: an empty body compiles to a + // bare RETURN and the JVM verifier rejects a constructor that returns without first calling + // super()/this() ("Constructor must call super() or this() before return"). ABI jars are + // compile-classpath-only, so only a well-formed delegating call matters, not its arguments + // (see createSuperDelegatingConstructorBody). Ordinary constructors whose delegating call + // fir2ir still emits are left untouched by the default traversal. // - // Inner classes are the exception: the JVM pipeline runs InnerClassesLowering, which asserts + // Inner classes need extra care: the JVM pipeline runs InnerClassesLowering, which asserts // that every inner-class constructor body contains an IrDelegatingConstructorCall (it // rewrites that call to thread the outer `this`). A bodyless inner constructor is skipped by // that lowering but then fails codegen; an *empty* stub body makes the lowering run and trip - // its "Delegating constructor call expected" assertion. So for inner-class constructors that - // have lost their delegating call we synthesize a trivial delegation to the superclass - // constructor. + // its "Delegating constructor call expected" assertion. So for inner-class constructors whose + // stripped body is not well-formed for that lowering we synthesize a trivial delegation to the + // superclass constructor (see the predicate below for exactly which shapes qualify). val parentClass = declaration.parentAsClass + // A defaulted value-class-over-primitive constructor parameter reaches the synthetic + // `$default` with an unboxed slot; regenerate its default as visitSimpleFunction does + // so the slot gets the unboxed primitive zero rather than ACONST_NULL. Skip annotation classes: + // their constructor parameter defaults ARE the ABI (the `AnnotationDefault` attribute), so + // regenerating them drops the element default and breaks consumers ("annotation @X is missing a + // default value for the element ..."). + if (parentClass.kind != ClassKind.ANNOTATION_CLASS) { + regenerateDefaultParameterValues(declaration) + } sanitizeErrorDefaultValues(declaration) if (parentClass.isInner) { - val existingBody = declaration.body - val hasDelegatingCall = - existingBody is IrBlockBody && - existingBody.statements.any { it is IrDelegatingConstructorCall } - if (!hasDelegatingCall) { + // A stripped inner-class constructor body is well-formed for InnerClassesLowering only if it + // has an IrInstanceInitializerCall (super-delegation) or an IrDelegatingConstructorCall to + // this same class (this()-delegation). The lowering treats a body WITHOUT an instance + // initializer as a this()-delegation and threads the outer instance onto the delegating + // call's dispatch receiver; for a super-delegation -- e.g. silverstonedgw's `private inner + // class InternalListener(..) : SomeInterface`, whose only supertype is an interface so it + // super-delegates to kotlin.Any -- that crashes with "no argument slot for the corresponding + // dispatch receiver parameter". Any other shape (no delegating call, or a super-delegation + // that lost its instance initializer) is replaced with a synthesized super-delegation. + val existingBody = declaration.body as? IrBlockBody + val isBodyWellFormed = + existingBody != null && + existingBody.statements.any { + it is IrInstanceInitializerCall || + (it is IrDelegatingConstructorCall && + it.symbol.owner.parentAsClass == parentClass) + } + if (!isBodyWellFormed) { declaration.body = createSuperDelegatingConstructorBody(declaration) ?: irFactory.createBlockBody(-1, -1) } @@ -659,24 +723,56 @@ internal class NonAbiDeclarationsStrippingIrVisitor( declaration.body == null || declaration.origin == IrDeclarationOrigin.FUNCTION_FOR_DEFAULT_PARAMETER ) { - declaration.body = irFactory.createBlockBody(-1, -1) + // SKIP_BODIES dropped the delegating constructor call for a bodyless constructor (e.g. a + // primary constructor whose body was stripped) and left the synthetic $default overload + // holding skipBodies error expressions. An empty replacement body is a bare RETURN that the + // JVM verifier rejects, so synthesize a well-formed delegation. Value-class secondary + // constructors must delegate to their primary constructor; other constructors delegate to + // their superclass as above. + declaration.body = + if (parentClass.isValue && parentClass.primaryConstructor != declaration) { + createValueClassDelegatingConstructorBody(declaration) + ?: irFactory.createBlockBody(-1, -1) + } else { + createSuperDelegatingConstructorBody(declaration) ?: irFactory.createBlockBody(-1, -1) + } } return super.visitConstructor(declaration) } /** - * Builds a constructor body with a delegating call to [innerClass]'s superclass constructor - * (defaulting to kotlin.Any) plus an instance-initializer, so InnerClassesLowering can find and - * rewrite it after SKIP_BODIES stripped the original delegating call. When the superclass - * constructor takes value parameters, fabricated default constants are passed -- for a - * compile-classpath-only ABI stub the argument values are irrelevant, only a well-formed call is. - * Returns null only when the superclass has no usable constructor. + * Builds a body that delegates a value-class secondary constructor to its primary constructor. + * Kotlin 2.3 lowers a super-delegating stub to a STATIC_INLINE_CLASS_CONSTRUCTOR containing an + * INSTANCE_INITIALIZER_CALL, which JVM codegen cannot emit. A this()-delegation lowers to the + * expected constructor-impl that returns the underlying value. + */ + private fun createValueClassDelegatingConstructorBody(constructor: IrConstructor): IrBody? { + val primaryConstructor = constructor.parentAsClass.primaryConstructor ?: return null + val builder = DeclarationIrBuilder(pluginContext, constructor.symbol) + val delegatingCall = builder.irDelegatingConstructorCall(primaryConstructor) + primaryConstructor.valueParameters.forEachIndexed { index, parameter -> + val defaultArgument = + generateDefaultValue(parameter.type) + ?: IrConstImpl.constNull(-1, -1, parameter.type.makeNullable()) + delegatingCall.putValueArgument(index, defaultArgument) + } + return irFactory.createBlockBody(-1, -1).apply { statements.add(delegatingCall) } + } + + /** + * Builds a constructor body with a delegating call to the owner class's superclass constructor + * (defaulting to kotlin.Any) plus an instance-initializer. SKIP_BODIES stripped the original + * delegating call, so without this the stub would be a bare RETURN (rejected by the JVM verifier) + * and, for an inner class, InnerClassesLowering would have no delegating call to find and + * rewrite. When the superclass constructor takes value parameters, fabricated default constants + * are passed -- for a compile-classpath-only ABI stub the argument values are irrelevant, only a + * well-formed call is. Returns null only when the superclass has no usable constructor. */ @OptIn(org.jetbrains.kotlin.DeprecatedForRemovalCompilerApi::class) private fun createSuperDelegatingConstructorBody(constructor: IrConstructor): IrBody? { - val innerClass = constructor.parentAsClass + val ownerClass = constructor.parentAsClass val superClass = - innerClass.superTypes + ownerClass.superTypes .mapNotNull { it.classOrNull?.owner } .firstOrNull { it.kind == ClassKind.CLASS } ?: irBuiltins.anyClass.owner val superConstructor = superClass.primaryConstructor ?: superClass.constructors.firstOrNull() @@ -691,7 +787,7 @@ internal class NonAbiDeclarationsStrippingIrVisitor( // super-delegations), so populate it here with the current inner class's enclosing `this`; // otherwise codegen sees a null dispatch receiver ("Null argument ... kind:DispatchReceiver"). if (superClass.isInner) { - innerClass.parentAsClass.thisReceiver?.let { outerThis -> + ownerClass.parentAsClass.thisReceiver?.let { outerThis -> delegatingCall.dispatchReceiver = IrGetValueImpl(-1, -1, outerThis.type, outerThis.symbol) } } @@ -712,7 +808,7 @@ internal class NonAbiDeclarationsStrippingIrVisitor( // instance-initializer marks it as a super-delegation, so the lowering initializes the // outer-this field instead of rewriting the delegating call. val instanceInitializer = - IrInstanceInitializerCallImpl(-1, -1, innerClass.symbol, irBuiltins.unitType) + IrInstanceInitializerCallImpl(-1, -1, ownerClass.symbol, irBuiltins.unitType) return irFactory.createBlockBody(-1, -1).apply { statements.add(delegatingCall) statements.add(instanceInitializer) diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/k2-jvm-abi-gen/src/main/kotlin/compose/ClassStabilityTransformer.kt b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/k2-jvm-abi-gen/src/main/kotlin/compose/ClassStabilityTransformer.kt index 99be4de..fdb3de1 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/k2-jvm-abi-gen/src/main/kotlin/compose/ClassStabilityTransformer.kt +++ b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/k2-jvm-abi-gen/src/main/kotlin/compose/ClassStabilityTransformer.kt @@ -25,6 +25,7 @@ import org.jetbrains.kotlin.ir.declarations.IrDeclarationOrigin import org.jetbrains.kotlin.ir.declarations.IrField import org.jetbrains.kotlin.ir.declarations.IrModuleFragment import org.jetbrains.kotlin.ir.declarations.IrProperty +import org.jetbrains.kotlin.ir.expressions.IrConst import org.jetbrains.kotlin.ir.expressions.impl.IrConstImpl import org.jetbrains.kotlin.ir.symbols.IrClassSymbol import org.jetbrains.kotlin.ir.symbols.UnsafeDuringIrConstructionAPI @@ -45,6 +46,7 @@ import org.jetbrains.kotlin.ir.types.isString import org.jetbrains.kotlin.ir.types.isUnit import org.jetbrains.kotlin.ir.types.makeNotNull import org.jetbrains.kotlin.ir.util.constructors +import org.jetbrains.kotlin.ir.util.getAnnotation import org.jetbrains.kotlin.ir.util.isEnumClass import org.jetbrains.kotlin.ir.util.isEnumEntry import org.jetbrains.kotlin.ir.util.isFileClass @@ -67,8 +69,12 @@ import org.jetbrains.kotlin.name.Name * - val properties of stable types: stable * - var properties (non-delegated): unstable * - Primitives, String, Unit, enums, function types: stable - * - Type parameters: encoded in the bitmask - * - Everything else: conservatively unstable (0) + * - A class is no more stable than its superclass + * - Unrecognised types: conservatively not-known-stable + * + * The two outputs use different encodings. $stable carries StabilityBits: 0 when known stable, + * UNSTABLE.bitsForSlot(0) = 8 otherwise. @StabilityInferred(parameters) carries the type-parameter + * bitmask plus a known-stable high bit at (1 shl typeParameters.size). */ @OptIn(UnsafeDuringIrConstructionAPI::class) internal class ClassStabilityTransformer(private val pluginContext: IrPluginContext) { @@ -78,6 +84,7 @@ internal class ClassStabilityTransformer(private val pluginContext: IrPluginCont ClassId(FqName("androidx.compose.runtime"), Name.identifier("Composer")) private val STABILITY_INFERRED_CLASS_ID = ClassId(FqName("androidx.compose.runtime.internal"), Name.identifier("StabilityInferred")) + private val STABILITY_INFERRED_FQ_NAME = STABILITY_INFERRED_CLASS_ID.asSingleFqName() // Known stable FQNs beyond primitives/String/Unit private val KNOWN_STABLE_FQNS = setOf( @@ -97,6 +104,11 @@ internal class ClassStabilityTransformer(private val pluginContext: IrPluginCont "kotlinx.collections.immutable.PersistentMap", "dagger.Lazy", ) + + // androidx StabilityBits: STABLE 0b000, UNSTABLE 0b100, bitsForSlot(n) = bits shl (1 + n*3). + // Written as the derivation rather than 8 so the slot-0 substitution stays visible. + private const val STABLE_BITS = 0 + private const val UNSTABLE_BITS_SLOT0 = 0b100 shl 1 } // Classes currently being analyzed (cycle detection) @@ -105,13 +117,33 @@ internal class ClassStabilityTransformer(private val pluginContext: IrPluginCont // Lazily resolved — null if compose-runtime is not on the classpath private var stabilityInferredClass: IrClassSymbol? = null + // FQNs declared in the module under compilation. A superclass in this set has not had its + // @StabilityInferred attached yet (this transformer is what attaches it, and visit order does not + // follow the inheritance graph), so its stability must be recomputed rather than read back. + private val moduleClassFqns = mutableSetOf() + fun transform(moduleFragment: IrModuleFragment) { // No-op when Compose runtime is not on the classpath. pluginContext.referenceClass(COMPOSER_CLASS_ID) ?: return stabilityInferredClass = pluginContext.referenceClass(STABILITY_INFERRED_CLASS_ID) + // Carrying FQNs across fragments would route a compiled dependency down the recompute branch + // instead of the external-annotation one. + moduleClassFqns.clear() + moduleFragment.accept(ModuleClassCollector(), null) moduleFragment.accept(StabilityVisitor(), null) } + private inner class ModuleClassCollector : IrElementVisitorVoidCompat() { + override fun visitElement(element: IrElement) { + element.acceptChildren(this, null) + } + + override fun visitClass(declaration: IrClass) { + declaration.classFqName()?.let { moduleClassFqns.add(it) } + super.visitClass(declaration) + } + } + private inner class StabilityVisitor : IrElementVisitorVoidCompat() { override fun visitElement(element: IrElement) { @@ -159,7 +191,8 @@ internal class ClassStabilityTransformer(private val pluginContext: IrPluginCont } private fun addStabilityField(irClass: IrClass) { - val stableValue = computeStabilityValue(irClass) + val stability = computeStability(irClass) + val stableValue = if (stability.knownStable) STABLE_BITS else UNSTABLE_BITS_SLOT0 // Add static final $stable: Int field val stableField = @@ -180,7 +213,19 @@ internal class ClassStabilityTransformer(private val pluginContext: IrPluginCont stableField.initializer = pluginContext.irFactory.createExpressionBody(-1, -1, constExpr) irClass.declarations.add(stableField) - addStabilityInferredAnnotation(irClass, stableValue) + // Not the same encoding as $stable: a bitmask over the class's type parameters, plus a high + // bit at (1 shl typeParameters.size) meaning "known stable regardless of its parameters". + // The `< 32` bound mirrors Compose exactly (ClassStabilityTransformer.kt:161 emits it, + // Stability.kt:246 reads it). At 31 params both sides use `1 shl 31` = Int.MIN_VALUE and + // compare with `and`, so the sign bit is correct; tightening to `< 31` would emit 0 here + // while the consumer still looks for that bit, silently losing "known stable". + val knownStableBit = + if (stability.knownStable && irClass.typeParameters.size < 32) { + 1 shl irClass.typeParameters.size + } else { + 0 + } + addStabilityInferredAnnotation(irClass, stability.typeParamMask or knownStableBit) } private fun addStabilityInferredAnnotation(irClass: IrClass, parametersValue: Int) { @@ -195,45 +240,113 @@ internal class ClassStabilityTransformer(private val pluginContext: IrPluginCont ) } } - irClass.annotations += annotation + // Register via metadataDeclarationRegistrar so HAS_ANNOTATIONS is set in Kotlin metadata. + // Without it, consumers cannot see the emulated annotation and fold stability constants + // instead of deferring to runtime. + pluginContext.metadataDeclarationRegistrar.addMetadataVisibleAnnotationsToElement( + irClass, + annotation, + ) } - // Compute the $stable field value. - // 0 = all stable, non-zero encodes which type parameters affect stability. - // For simplicity in ABI emulation, we compute based on property analysis. - private fun computeStabilityValue(irClass: IrClass): Int { + /** Type-parameter bitmask plus whether the class itself is known stable. */ + private data class Stability(val typeParamMask: Int, val knownStable: Boolean) + + // Deliberately conservative: an unrecognised member type yields not-known-stable, so consumers + // resolve stability at runtime rather than assume the wrong answer. + private fun computeStability(irClass: IrClass): Stability { val classFqn = irClass.classFqName() if (classFqn != null && analyzing.contains(classFqn)) { - return 0 // Cycle — treat as stable to avoid infinite recursion + // Cycle — do not claim knowledge we do not have. + return Stability(0, knownStable = false) } if (classFqn != null) analyzing.add(classFqn) - var result = 0 + var mask = 0 + var knownStable = true for (declaration in irClass.declarations) { when (declaration) { is IrProperty -> { - // var (non-delegated) → unstable if (declaration.isVar && declaration.isDelegated != true) { if (classFqn != null) analyzing.remove(classFqn) - return 0 // Unstable class gets $stable = 0 (correct — means "evaluate at runtime") + return Stability(0, knownStable = false) } - // Check backing field type for type parameter dependency + // Only stored state counts. A computed getter has no backing field and is skipped, + // which is what the real StabilityInferencer does - it reaches members through + // `member.backingField?.let { ... }`. val backingField = declaration.backingField if (backingField != null) { - result = result or typeParamBits(backingField.type, irClass) + mask = mask or typeParamBits(backingField.type, irClass) + if ( + !isStabilityDelegatedToTypeParam(backingField.type, irClass) && + !isKnownStableType(backingField.type) + ) { + knownStable = false + } } } is IrField -> { if (!declaration.isStatic) { - result = result or typeParamBits(declaration.type, irClass) + mask = mask or typeParamBits(declaration.type, irClass) + if ( + !isStabilityDelegatedToTypeParam(declaration.type, irClass) && + !isKnownStableType(declaration.type) + ) { + knownStable = false + } } } else -> {} } } + // A subclass holds every field its superclass declares, so it can be no more stable than that + // superclass. Skipping this let a class with no stored state of its own be called stable while + // the real compiler called it unstable — e.g. `object EmptyPainter : Painter()`, where all the + // mutable state lives in Painter. Over-asserting stability makes a consumer skip recomposition + // for a value that did change, so this is deliberately the pessimistic direction. + if (knownStable && !superclassKnownStable(irClass)) { + knownStable = false + } + if (classFqn != null) analyzing.remove(classFqn) - return result + return Stability(mask, knownStable) + } + + /** + * Whether [irClass]'s superclass permits it to be known stable. Interfaces and [Any] are ignored: + * neither contributes stored state. + */ + private fun superclassKnownStable(irClass: IrClass): Boolean { + val superClass = + irClass.superTypes.mapNotNull { it.classOrNull?.owner }.firstOrNull { !it.isInterface } + ?: return true + val superFqn = superClass.classFqName() ?: return false + if (superFqn == "kotlin.Any") return true + if (superFqn in moduleClassFqns) { + return computeStability(superClass).knownStable + } + // Compiled dependency: trust its @StabilityInferred, and treat an absent one as unstable. A + // class outside a Compose-enabled compilation carries no stability information, and its private + // state may be stripped from the ABI jar, so its declarations cannot be inspected instead. + return externalClassKnownStable(superClass) + } + + private fun externalClassKnownStable(irClass: IrClass): Boolean { + val annotation = irClass.getAnnotation(STABILITY_INFERRED_FQ_NAME) ?: return false + val parameters = (annotation.getValueArgument(0) as? IrConst)?.value as? Int ?: return false + val typeParamCount = irClass.typeParameters.size + if (typeParamCount >= 32) return false + return (parameters and (1 shl typeParamCount)) != 0 + } + + // True only when the type IS one of the class's own type parameters, e.g. `val value: T`. Such a + // member's stability is carried by its bit in the mask and resolved by the consumer, so it must + // not clear knownStable. A container that merely mentions T, e.g. `List`, is not delegated: + // List is unstable whatever T is, and must clear knownStable. + private fun isStabilityDelegatedToTypeParam(type: IrType, irClass: IrClass): Boolean { + val classifier = (type as? IrSimpleType)?.classifier ?: return false + return irClass.typeParameters.any { classifier == it.symbol } } // Determine which type parameter bits are affected by a given type. @@ -302,8 +415,16 @@ internal class ClassStabilityTransformer(private val pluginContext: IrPluginCont if (owner.isEnumClass) return true } - // Known stable types - if (fqn in KNOWN_STABLE_FQNS) return true + // Known stable types. The container-shaped ones are only as stable as their arguments: + // `Pair>` matches on FQN alone but is not stable, and claiming it is + // makes a consumer skip a recomposition it needed. A star projection or a type-parameter + // argument is not known stable here either — that stability is carried by the type-parameter + // mask, which is a separate signal from this one. + if (fqn in KNOWN_STABLE_FQNS) { + return (type as? IrSimpleType)?.arguments.orEmpty().all { + it is org.jetbrains.kotlin.ir.types.IrTypeProjection && isKnownStableType(it.type) + } + } // SDK types (java.*, kotlin.* excluding collections interfaces) // Kotlin collections interfaces (List, Set, Map) are NOT stable — they're interfaces diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/k2-jvm-abi-gen/src/main/kotlin/compose/ComposableTypeRewriter.kt b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/k2-jvm-abi-gen/src/main/kotlin/compose/ComposableTypeRewriter.kt index 15873d3..96bb97e 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/k2-jvm-abi-gen/src/main/kotlin/compose/ComposableTypeRewriter.kt +++ b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/k2-jvm-abi-gen/src/main/kotlin/compose/ComposableTypeRewriter.kt @@ -16,6 +16,8 @@ package com.facebook import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext import org.jetbrains.kotlin.ir.IrElement import org.jetbrains.kotlin.ir.declarations.IrClass +import org.jetbrains.kotlin.ir.declarations.IrConstructor +import org.jetbrains.kotlin.ir.declarations.IrField import org.jetbrains.kotlin.ir.declarations.IrFunction import org.jetbrains.kotlin.ir.declarations.IrModuleFragment import org.jetbrains.kotlin.ir.declarations.IrProperty @@ -25,6 +27,7 @@ import org.jetbrains.kotlin.ir.types.IrType import org.jetbrains.kotlin.ir.types.classFqName import org.jetbrains.kotlin.ir.types.classOrNull import org.jetbrains.kotlin.ir.types.defaultType +import org.jetbrains.kotlin.ir.types.isMarkedNullable import org.jetbrains.kotlin.ir.types.makeNullable import org.jetbrains.kotlin.ir.types.typeWith import org.jetbrains.kotlin.ir.util.hasAnnotation @@ -57,6 +60,18 @@ internal class ComposableTypeRewriter(private val pluginContext: IrPluginContext ClassId(FqName("androidx.compose.runtime"), Name.identifier("Composer")) private val FUNCTION_FQ_PREFIX = "kotlin.Function" + /** + * Under K2 the Compose plugin registers its own function-type *kinds* + * (ComposeFirExtensions.registerKinds), so a `@Composable (P..) -> R` is classified as the + * synthetic class `androidx.compose.runtime.internal.(K)ComposableFunctionN` and carries no + * `@Composable` annotation on the type itself. Only deserialized dependency metadata still uses + * the legacy `kotlin.FunctionN` + annotation form. + */ + private const val COMPOSABLE_FUNCTION_FQ_PREFIX = + "androidx.compose.runtime.internal.ComposableFunction" + private const val K_COMPOSABLE_FUNCTION_FQ_PREFIX = + "androidx.compose.runtime.internal.KComposableFunction" + /** Each parameter uses 3 bits. 32 / 3 = 10 slots per Int. */ private const val SLOTS_PER_INT = 10 } @@ -84,12 +99,38 @@ internal class ComposableTypeRewriter(private val pluginContext: IrPluginContext super.visitSimpleFunction(declaration) } + /** + * Constructors need their own override: [IrConstructor] extends IrFunction but NOT + * IrSimpleFunction, so [visitSimpleFunction] never sees one. Without this, a `class Foo(val + * content: @Composable () -> Unit)` keeps ComposableFunction0 in its `` descriptor while + * its generated getter is rewritten correctly -- and every consumer of the class calls a + * constructor. + */ + override fun visitConstructor(declaration: IrConstructor) { + rewriteFunctionSignatureTypes(declaration) + super.visitConstructor(declaration) + } + override fun visitProperty(declaration: IrProperty) { declaration.getter?.let { rewriteFunctionSignatureTypes(it) } declaration.setter?.let { rewriteFunctionSignatureTypes(it) } super.visitProperty(declaration) } + /** + * Backing fields are not reached via [visitSimpleFunction] or [visitProperty] either. These are + * private and so do not affect consumers, but leaving the synthetic ComposableFunctionN type in + * the stub jar leaves a dangling reference to a class that exists only inside a Compose-enabled + * compilation. + */ + override fun visitField(declaration: IrField) { + val rewritten = rewriteTypeIfComposable(declaration.type) + if (rewritten !== declaration.type) { + declaration.type = rewritten + } + super.visitField(declaration) + } + override fun visitClass(declaration: IrClass) { // Rewrite @Composable function types in supertypes. declaration.superTypes = declaration.superTypes.map { rewriteTypeIfComposable(it) } @@ -158,18 +199,36 @@ internal class ComposableTypeRewriter(private val pluginContext: IrPluginContext } /** - * Check if an IrSimpleType is a @Composable function type. - * - * A composable function type has: - * 1. A @Composable annotation on the type - * 2. A classifier that is kotlin.FunctionN + * Two representations reach IR: the K2 kind `(K)ComposableFunctionN` with no annotation + * (in-module source), and `kotlin.FunctionN` plus `@Composable` (read back from metadata). */ private fun isComposableFunctionType(type: IrSimpleType): Boolean { + if (isSyntheticComposableFunction(type) || isKComposableFunction(type)) return true if (!type.hasAnnotation(COMPOSABLE_FQ_NAME)) return false val classFqn = type.classFqName?.asString() ?: return false return classFqn.startsWith(FUNCTION_FQ_PREFIX) } + /** The non-reflect Compose kind, `androidx.compose.runtime.internal.ComposableFunctionN`. */ + private fun isSyntheticComposableFunction(type: IrSimpleType): Boolean = + hasNumberedPrefix(type, COMPOSABLE_FUNCTION_FQ_PREFIX) + + /** The reflect Compose kind, `androidx.compose.runtime.internal.KComposableFunctionN`. */ + private fun isKComposableFunction(type: IrSimpleType): Boolean = + hasNumberedPrefix(type, K_COMPOSABLE_FUNCTION_FQ_PREFIX) + + /** + * These classes are synthetic (declared nowhere) and unbounded in arity, so match on prefix + + * an all-digit suffix rather than enumerating. The two prefixes are disjoint: + * `...internal.KComposableFunction0` does not start with `...internal.ComposableFunction`. + */ + private fun hasNumberedPrefix(type: IrSimpleType, prefix: String): Boolean { + val fqn = type.classFqName?.asString() ?: return false + if (!fqn.startsWith(prefix)) return false + val suffix = fqn.substring(prefix.length) + return suffix.isNotEmpty() && suffix.all { it.isDigit() } + } + // Rewrite a @Composable function type. // // @Composable FunctionN(P1, ..., Pn, R) becomes @@ -194,23 +253,30 @@ internal class ComposableTypeRewriter(private val pluginContext: IrPluginContext val newArity = userParamCount + 1 + changedCount // Build new type arguments: [P1, ..., Pn, Composer, Int, ..., R] + // Recurse: a composable lambda nested inside another composable lambda's arguments is + // itself a composable function type and must be rewritten too. rewriteTypeIfComposable + // returns early on the composable branch, so this is the only place that reaches them. val newTypeArgs = mutableListOf() for (arg in paramTypeArgs) { - newTypeArgs.add(extractTypeFromArgument(arg)) + newTypeArgs.add(rewriteTypeIfComposable(extractTypeFromArgument(arg))) } // Add Composer? type argument. newTypeArgs.add(composerIrType) // Add Int type arguments for $changed params. repeat(changedCount) { newTypeArgs.add(pluginContext.irBuiltIns.intType) } // Add return type. - newTypeArgs.add(extractTypeFromArgument(returnTypeArg)) - - // Resolve the new FunctionN class. - val functionClassId = ClassId(FqName("kotlin"), Name.identifier("Function$newArity")) - val functionClass = pluginContext.referenceClass(functionClassId) ?: return type - - // Build the new type without @Composable annotation. - return functionClass.typeWith(newTypeArgs) + newTypeArgs.add(rewriteTypeIfComposable(extractTypeFromArgument(returnTypeArg))) + + // irBuiltIns.functionN/kFunctionN cannot fail, unlike referenceClass, whose `?: return type` + // silently left the type un-rewritten when the synthetic class was not in the symbol table. + val functionClass = + if (isKComposableFunction(type)) pluginContext.irBuiltIns.kFunctionN(newArity) + else pluginContext.irBuiltIns.functionN(newArity) + + // typeWith() constructs NOT_SPECIFIED, so a declared-nullable type would come back non-null + // and be emitted @NotNull; Java interop and NullAway read those JVM annotations. + val rewritten = functionClass.typeWith(newTypeArgs) + return if (type.isMarkedNullable()) rewritten.makeNullable() else rewritten } private fun extractTypeFromArgument( diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/k2-jvm-abi-gen/src/main/kotlin/compose/ComposerParamInjector.kt b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/k2-jvm-abi-gen/src/main/kotlin/compose/ComposerParamInjector.kt index b9dba0d..41ce0f0 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/k2-jvm-abi-gen/src/main/kotlin/compose/ComposerParamInjector.kt +++ b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/k2-jvm-abi-gen/src/main/kotlin/compose/ComposerParamInjector.kt @@ -14,18 +14,24 @@ package com.facebook import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext +import org.jetbrains.kotlin.backend.common.lower.createIrBuilder import org.jetbrains.kotlin.ir.IrElement +import org.jetbrains.kotlin.ir.builders.irCallConstructor import org.jetbrains.kotlin.ir.declarations.IrClass import org.jetbrains.kotlin.ir.declarations.IrDeclarationOrigin import org.jetbrains.kotlin.ir.declarations.IrModuleFragment import org.jetbrains.kotlin.ir.declarations.IrProperty import org.jetbrains.kotlin.ir.declarations.IrSimpleFunction import org.jetbrains.kotlin.ir.declarations.IrValueParameter +import org.jetbrains.kotlin.ir.expressions.impl.IrConstImpl +import org.jetbrains.kotlin.ir.symbols.IrClassSymbol import org.jetbrains.kotlin.ir.symbols.UnsafeDuringIrConstructionAPI import org.jetbrains.kotlin.ir.types.IrType import org.jetbrains.kotlin.ir.types.defaultType import org.jetbrains.kotlin.ir.types.makeNullable +import org.jetbrains.kotlin.ir.util.constructors import org.jetbrains.kotlin.ir.util.hasAnnotation +import org.jetbrains.kotlin.load.java.JvmAbi import org.jetbrains.kotlin.name.ClassId import org.jetbrains.kotlin.name.FqName import org.jetbrains.kotlin.name.Name @@ -36,12 +42,13 @@ import org.jetbrains.kotlin.name.Name * For every @Composable function (excluding constructors and expect functions), this adds: * - `$composer: Composer?` — the Composer instance threaded through composable calls * - `$changed: Int` (1 or more) — bitmask tracking parameter change state - * - `$default: Int` (0 or more) — bitmask for default parameter values, only when defaults exist + * - `$default: Int` (0 or more) — bitmask for default parameter values, only on a concrete function + * that has defaults * * Parameter counting follows the Compose compiler spec: - * - changedParamCount = max(1, ceil((realValueParams + thisParams + 1) / SLOTS_PER_INT)) where +1 - * accounts for the force bit in slot 0 - * - defaultParamCount = ceil(valueParams / BITS_PER_DEFAULT_INT), only if any param has a default + * - changedParamCount = max(1, ceil((realValueParams + thisParams) / SLOTS_PER_INT)) + * - defaultParamCount = ceil(valueParams / BITS_PER_DEFAULT_INT), only if the function is concrete + * and any param has a default * - thisParams = count of dispatch receiver + extension receiver (context receivers are value * params) */ @@ -52,6 +59,8 @@ internal class ComposerParamInjector(private val pluginContext: IrPluginContext) val COMPOSABLE_FQ_NAME = FqName("androidx.compose.runtime.Composable") private val COMPOSER_CLASS_ID = ClassId(FqName("androidx.compose.runtime"), Name.identifier("Composer")) + private val JVM_NAME_CLASS_ID = ClassId(FqName("kotlin.jvm"), Name.identifier("JvmName")) + private val JVM_NAME_FQ_NAME = FqName("kotlin.jvm.JvmName") /** Each parameter uses 3 bits in the $changed bitmask. 32 / 3 = 10 slots per Int. */ private const val SLOTS_PER_INT = 10 @@ -65,6 +74,10 @@ internal class ComposerParamInjector(private val pluginContext: IrPluginContext) pluginContext.referenceClass(COMPOSER_CLASS_ID)?.defaultType?.makeNullable() } + private val jvmNameClass: IrClassSymbol? by lazy { + pluginContext.referenceClass(JVM_NAME_CLASS_ID) + } + /** Run the transform on the entire module. */ fun transform(moduleFragment: IrModuleFragment) { val composerIrType = composerType ?: return // Compose runtime not on classpath; skip. @@ -72,6 +85,9 @@ internal class ComposerParamInjector(private val pluginContext: IrPluginContext) ComposerParamVisitor(pluginContext, composerIrType), null, ) + // This must run after every in-module composable has been transformed, so links between two + // transformed declarations are preserved while stale links to dependency declarations are not. + moduleFragment.accept(OverrideLinkRepairVisitor(), null) } private inner class ComposerParamVisitor( @@ -117,6 +133,12 @@ internal class ComposerParamInjector(private val pluginContext: IrPluginContext) private fun shouldTransform(function: IrSimpleFunction): Boolean { if (!function.hasAnnotation(COMPOSABLE_FQ_NAME)) return false if (function.isExpect) return false + // Idempotence guard. A @Composable property getter is reached TWICE: once explicitly via + // visitProperty below, and again when super.visitProperty descends into the getter as an + // IrSimpleFunction. Without this check the synthetic params are injected twice, producing + // (..., Composer, int, Composer, int) -- a descriptor no consumer can link against. Only + // fires when @Composable is on the getter rather than the property, which is why it is rare. + if (function.valueParameters.any { it.name.asString() == "\$composer" }) return false return true } @@ -137,13 +159,19 @@ internal class ComposerParamInjector(private val pluginContext: IrPluginContext) val realValueParamCount = existingValueParams.size val totalSlottedParams = realValueParamCount + thisParamCount - // $changed count: ceil((totalSlottedParams + 1) / SLOTS_PER_INT), minimum 1. - // The +1 accounts for the force bit in slot 0. - val changedCount = maxOf(1, ceilDiv(totalSlottedParams + 1, SLOTS_PER_INT)) + // Deliberately no "+1": the force bit shares int 0 with slots 0..9 rather than consuming a + // slot, and adding one emits a trailing `I` at every exact multiple of SLOTS_PER_INT. + val changedCount = maxOf(1, ceilDiv(totalSlottedParams, SLOTS_PER_INT)) - // $default count: only present if any parameter has a default value. + // An abstract @Composable member gets no $default: Compose puts defaults handling in a + // nested ComposeDefaultImpls, so injecting it here adds a trailing `I` and changes the + // value-class mangled name, leaving a cross-target implementer overriding a signature the + // real ABI does not have. Keyed on having a body, not on modality, so bodiless fake + // overrides are treated alike. + val hasBody = function.body != null val hasDefaults = existingValueParams.any { it.defaultValue != null } - val defaultCount = if (hasDefaults) ceilDiv(realValueParamCount, BITS_PER_DEFAULT_INT) else 0 + val defaultCount = + if (hasDefaults && hasBody) ceilDiv(realValueParamCount, BITS_PER_DEFAULT_INT) else 0 // Build the new parameter list: existing + $composer + $changed[N] + $default[N] val newParams = mutableListOf() @@ -221,8 +249,67 @@ internal class ComposerParamInjector(private val pluginContext: IrPluginContext) } function.valueParameters = newParams + stampAccessorJvmName(function) + } + + // A @Composable PROPERTY ACCESSOR must keep the plain JVM name of the accessor. + // + // ComposerParamTransformer.copyWithComposerParam stamps @JvmName(getterName(property)) on every + // accessor it rewrites, and @JvmName suppresses both the value-class mangle and the $module + // suffix the JVM backend would otherwise apply. Injecting the synthetic params in place leaves + // neither suppressed, so the stub advertises a name the real jar does not have. Measured on + // MdsButton$Type: + // + // real: getMinHeight(Composer, int)F + // stub: getMinHeight-chRvn1I$fbandroid_java_com_facebook_mds_compose_button_button( + // Composer, int)F + // + // A consumer compiled against the stub links the mangled name -> NoSuchMethodError. A + // @Composable FUNCTION has no corresponding property, gets no @JvmName from the real + // transformer either, and must keep both manglings -- which is why this is scoped to accessors. + private fun stampAccessorJvmName(function: IrSimpleFunction) { + val property = function.correspondingPropertySymbol?.owner ?: return + if (function.hasAnnotation(JVM_NAME_FQ_NAME)) return + val constructor = jvmNameClass?.constructors?.singleOrNull() ?: return + val propertyName = property.name.identifier + val accessorName = + if (property.setter == function) JvmAbi.setterName(propertyName) + else JvmAbi.getterName(propertyName) + val annotation = + pluginContext.irBuiltIns.createIrBuilder(function.symbol).run { + irCallConstructor(constructor, emptyList()).apply { + putValueArgument( + 0, + IrConstImpl.string(-1, -1, pluginContext.irBuiltIns.stringType, accessorName), + ) + } + } + function.annotations = function.annotations + annotation } } + + /** + * Removes override links whose source-level dependency declarations lack Compose's synthetic + * parameters. The transformed JVM descriptor still matches the dependency's real Compose-lowered + * descriptor, so no bridge is needed to preserve the override in bytecode. + */ + private inner class OverrideLinkRepairVisitor : IrElementVisitorVoidCompat() { + override fun visitElement(element: IrElement) { + element.acceptChildren(this, null) + } + + override fun visitSimpleFunction(declaration: IrSimpleFunction) { + if (declaration.hasInjectedComposerParam() && declaration.overriddenSymbols.isNotEmpty()) { + declaration.overriddenSymbols = + declaration.overriddenSymbols.filter { it.owner.hasInjectedComposerParam() } + } + super.visitSimpleFunction(declaration) + } + } + + private fun IrSimpleFunction.hasInjectedComposerParam(): Boolean = valueParameters.any { + it.name.asString() == "\$composer" + } } /** Integer ceiling division: ceil(a / b) for positive a, b. */ diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/k2-jvm-abi-gen/src/main/kotlin/parcelize/ParcelableCreatorEmulator.kt b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/k2-jvm-abi-gen/src/main/kotlin/parcelize/ParcelableCreatorEmulator.kt new file mode 100644 index 0000000..fa4acdb --- /dev/null +++ b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/k2-jvm-abi-gen/src/main/kotlin/parcelize/ParcelableCreatorEmulator.kt @@ -0,0 +1,97 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is dual-licensed under either the MIT license found in the + * LICENSE-MIT file in the root directory of this source tree or the Apache + * License, Version 2.0 found in the LICENSE-APACHE file in the root directory + * of this source tree. You may select, at your option, one of the + * above-listed licenses. + */ + +@file:SuppressWarnings("PackageLocationMismatch") +@file:Suppress("OPT_IN_USAGE_ERROR") + +package com.facebook + +import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext +import org.jetbrains.kotlin.descriptors.DescriptorVisibilities +import org.jetbrains.kotlin.descriptors.Modality +import org.jetbrains.kotlin.ir.IrElement +import org.jetbrains.kotlin.ir.declarations.IrClass +import org.jetbrains.kotlin.ir.declarations.IrDeclarationOrigin +import org.jetbrains.kotlin.ir.declarations.IrField +import org.jetbrains.kotlin.ir.declarations.IrModuleFragment +import org.jetbrains.kotlin.ir.declarations.IrProperty +import org.jetbrains.kotlin.ir.symbols.UnsafeDuringIrConstructionAPI +import org.jetbrains.kotlin.ir.symbols.impl.IrFieldSymbolImpl +import org.jetbrains.kotlin.ir.types.starProjectedType +import org.jetbrains.kotlin.ir.types.typeWith +import org.jetbrains.kotlin.ir.util.companionObject +import org.jetbrains.kotlin.ir.util.hasAnnotation +import org.jetbrains.kotlin.ir.util.isInterface +import org.jetbrains.kotlin.ir.visitors.IrVisitorVoid +import org.jetbrains.kotlin.ir.visitors.acceptChildrenVoid +import org.jetbrains.kotlin.name.ClassId +import org.jetbrains.kotlin.name.FqName +import org.jetbrains.kotlin.name.Name + +internal class ParcelableCreatorEmulator(private val pluginContext: IrPluginContext) { + + fun transform(moduleFragment: IrModuleFragment) { + val creatorClass = pluginContext.referenceClass(PARCELABLE_CREATOR_CLASS_ID) ?: return + moduleFragment.acceptChildrenVoid( + object : IrVisitorVoid() { + override fun visitElement(element: IrElement) { + element.acceptChildrenVoid(this) + } + + override fun visitClass(declaration: IrClass) { + if (declaration.needsCreator()) { + declaration.addCreatorField(creatorClass.owner) + } + declaration.acceptChildrenVoid(this) + } + }, + ) + } + + @OptIn(UnsafeDuringIrConstructionAPI::class) + private fun IrClass.needsCreator(): Boolean { + if (!hasAnnotation(PARCELIZE_FQ_NAME)) return false + if (isInterface || modality == Modality.ABSTRACT || modality == Modality.SEALED) return false + return !declaresCreator() && companionObject()?.declaresCreator() != true + } + + @OptIn(UnsafeDuringIrConstructionAPI::class) + private fun IrClass.declaresCreator(): Boolean = declarations.any { + (it as? IrField)?.name == CREATOR_NAME || + (it as? IrProperty)?.name == CREATOR_NAME || + (it as? IrClass)?.name == CREATOR_NAME + } + + @OptIn(UnsafeDuringIrConstructionAPI::class) + private fun IrClass.addCreatorField(creatorClass: IrClass) { + val field = + pluginContext.irFactory.createField( + startOffset = -1, + endOffset = -1, + origin = IrDeclarationOrigin.DEFINED, + name = CREATOR_NAME, + type = creatorClass.symbol.typeWith(symbol.starProjectedType), + visibility = DescriptorVisibilities.PUBLIC, + symbol = IrFieldSymbolImpl(), + isFinal = true, + isExternal = false, + isStatic = true, + ) + field.parent = this + declarations.add(field) + } + + private companion object { + val CREATOR_NAME = Name.identifier("CREATOR") + val PARCELIZE_FQ_NAME = FqName("kotlinx.parcelize.Parcelize") + val PARCELABLE_CREATOR_CLASS_ID = + ClassId(FqName("android.os"), FqName("Parcelable.Creator"), isLocal = false) + } +} diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/k2-jvm-abi-gen/src/main/kotlin/parcelize/ParcelizeAbiEmulationExtension.kt b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/k2-jvm-abi-gen/src/main/kotlin/parcelize/ParcelizeAbiEmulationExtension.kt new file mode 100644 index 0000000..7ed1762 --- /dev/null +++ b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/k2-jvm-abi-gen/src/main/kotlin/parcelize/ParcelizeAbiEmulationExtension.kt @@ -0,0 +1,24 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is dual-licensed under either the MIT license found in the + * LICENSE-MIT file in the root directory of this source tree or the Apache + * License, Version 2.0 found in the LICENSE-APACHE file in the root directory + * of this source tree. You may select, at your option, one of the + * above-listed licenses. + */ + +@file:SuppressWarnings("PackageLocationMismatch") + +package com.facebook + +import org.jetbrains.kotlin.backend.common.extensions.IrGenerationExtension +import org.jetbrains.kotlin.backend.common.extensions.IrPluginContext +import org.jetbrains.kotlin.ir.declarations.IrModuleFragment + +internal class ParcelizeAbiEmulationExtension : IrGenerationExtension { + + override fun generate(moduleFragment: IrModuleFragment, pluginContext: IrPluginContext) { + ParcelableCreatorEmulator(pluginContext).transform(moduleFragment) + } +} diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/k2-jvm-abi-gen/src/main/kotlin/pipeline/AbiGenPipeline.kt b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/k2-jvm-abi-gen/src/main/kotlin/pipeline/AbiGenPipeline.kt index 536338c..c87c660 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/k2-jvm-abi-gen/src/main/kotlin/pipeline/AbiGenPipeline.kt +++ b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/k2-jvm-abi-gen/src/main/kotlin/pipeline/AbiGenPipeline.kt @@ -17,8 +17,12 @@ package com.facebook +import com.facebook.kotlin.compilercompat.callableNameOrNullCompat +import com.facebook.kotlin.compilercompat.packageNameOrNullCompat import java.io.File +import java.util.jar.JarFile import org.jetbrains.kotlin.backend.common.extensions.IrGenerationExtension +import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity import org.jetbrains.kotlin.cli.common.messages.MessageCollector import org.jetbrains.kotlin.descriptors.ClassKind import org.jetbrains.kotlin.descriptors.Visibilities @@ -28,7 +32,9 @@ import org.jetbrains.kotlin.fir.backend.FirMetadataSource import org.jetbrains.kotlin.fir.declarations.FirCallableDeclaration import org.jetbrains.kotlin.fir.declarations.FirDeclaration import org.jetbrains.kotlin.fir.declarations.FirDeclarationOrigin +import org.jetbrains.kotlin.fir.declarations.FirFile import org.jetbrains.kotlin.fir.declarations.FirFunction +import org.jetbrains.kotlin.fir.declarations.FirProperty import org.jetbrains.kotlin.fir.declarations.FirRegularClass import org.jetbrains.kotlin.fir.declarations.utils.isConst import org.jetbrains.kotlin.fir.expressions.FirAnnotation @@ -62,14 +68,53 @@ import org.jetbrains.kotlin.ir.IrElement import org.jetbrains.kotlin.ir.declarations.IrClass import org.jetbrains.kotlin.ir.declarations.IrConstructor import org.jetbrains.kotlin.ir.declarations.IrDeclarationBase +import org.jetbrains.kotlin.ir.declarations.IrFile import org.jetbrains.kotlin.ir.declarations.IrMetadataSourceOwner import org.jetbrains.kotlin.ir.declarations.IrModuleFragment import org.jetbrains.kotlin.ir.declarations.IrProperty import org.jetbrains.kotlin.ir.declarations.IrSimpleFunction +import org.jetbrains.kotlin.ir.util.kotlinFqName import org.jetbrains.kotlin.name.CallableId import org.jetbrains.kotlin.name.ClassId import org.jetbrains.kotlin.name.FqName import org.jetbrains.kotlin.psi.KtFile +import org.jetbrains.org.objectweb.asm.AnnotationVisitor +import org.jetbrains.org.objectweb.asm.ClassReader +import org.jetbrains.org.objectweb.asm.ClassVisitor +import org.jetbrains.org.objectweb.asm.ConstantDynamic +import org.jetbrains.org.objectweb.asm.FieldVisitor +import org.jetbrains.org.objectweb.asm.Handle +import org.jetbrains.org.objectweb.asm.Label +import org.jetbrains.org.objectweb.asm.MethodVisitor +import org.jetbrains.org.objectweb.asm.ModuleVisitor +import org.jetbrains.org.objectweb.asm.Opcodes +import org.jetbrains.org.objectweb.asm.RecordComponentVisitor +import org.jetbrains.org.objectweb.asm.Type +import org.jetbrains.org.objectweb.asm.TypePath +import org.jetbrains.org.objectweb.asm.signature.SignatureReader +import org.jetbrains.org.objectweb.asm.signature.SignatureVisitor + +fun errorTypedApiPositions( + isProperty: Boolean, + returnTypeHasError: Boolean, + propertyTypeHasError: Boolean, + receiverHasError: Boolean, + contextParameterErrors: List, + valueParameterErrors: List, +): List = buildList { + if (isProperty) { + if (propertyTypeHasError) add("property type") + } else if (returnTypeHasError) { + add("return type") + } + if (receiverHasError) add("receiver") + contextParameterErrors.forEachIndexed { index, hasError -> + if (hasError) add("context parameter $index") + } + valueParameterErrors.forEachIndexed { index, hasError -> + if (hasError) add("parameter $index") + } +} /** * FIR metadata sanitization stage. @@ -80,7 +125,7 @@ import org.jetbrains.kotlin.psi.KtFile * - Strip annotations with errors from FIR metadata sources (post-IR) * - Strip private supertypes + fake override conversion (post-IR) */ -internal class FirMetadataSanitizerStage : AbiGenStage { +internal class FirMetadataSanitizerStage(private val repairLog: AbiGenRepairLog) : AbiGenStage { override val name = "FirMetadataSanitizer" /** @@ -117,6 +162,11 @@ internal class FirMetadataSanitizerStage : AbiGenStage { element.acceptChildren(this, null) } + override fun visitFile(declaration: IrFile) { + degradeErrorTypedFileMemberTypes(declaration) + super.visitFile(declaration) + } + override fun visitClass(declaration: IrClass) { stripThrowsAndErrorAnnotationsFromDeclaration(declaration) stripPrivateSupertypesFromDeclaration(declaration) @@ -126,20 +176,19 @@ internal class FirMetadataSanitizerStage : AbiGenStage { override fun visitSimpleFunction(declaration: IrSimpleFunction) { stripThrowsAndErrorAnnotationsFromDeclaration(declaration) - replaceErrorReturnTypeWithAny(declaration, session) - replaceErrorValueParameterTypesWithAny(declaration, session) + degradeErrorTypedMetadataSource(declaration, session) super.visitSimpleFunction(declaration) } override fun visitProperty(declaration: IrProperty) { stripThrowsAndErrorAnnotationsFromDeclaration(declaration) - replaceErrorReturnTypeWithAny(declaration, session) + degradeErrorTypedMetadataSource(declaration, session) super.visitProperty(declaration) } override fun visitConstructor(declaration: IrConstructor) { stripThrowsAndErrorAnnotationsFromDeclaration(declaration) - replaceErrorValueParameterTypesWithAny(declaration, session) + degradeErrorTypedMetadataSource(declaration, session) super.visitConstructor(declaration) } @@ -177,46 +226,45 @@ internal class FirMetadataSanitizerStage : AbiGenStage { val firMetadataSource = metadataSourceOwner.metadata as? FirMetadataSource ?: return val firClass = firMetadataSource.fir as? FirRegularClass ?: return + // A member of a private/local class is not ABI surface even when it is declared + // public: no consumer can reference it, so degrading its unserializable type cannot + // make the ABI lie to anyone. This covers the common + // `private object Utils { fun get() = Dep.somethingUnresolvable() }` shape, where the + // member's own visibility is public but the enclosing object is private. + val classIsNonApi = isClassEffectivelyPrivate(declaration) + firClass.symbol.declarationSymbols.forEach { symbol -> val decl = (symbol as? FirCallableSymbol<*>)?.fir ?: return@forEach - if (!isNonApiVisibility(decl)) return@forEach - if (hasErrorReturnType(decl)) { - runCatching { decl.replaceReturnTypeRef(session.builtinTypes.nullableAnyType) } + if (!classIsNonApi && !isNonApiVisibility(decl)) { + recordErrorTypedApiMember(firClass.symbol.classId.asString(), decl) + return@forEach } // The metadata serializer collects functions/constructors from the class member - // SCOPE, which references these same symbols' FIR. An error-typed value parameter - // (e.g. a parameter typed by a nested enum of a stubbed dependency such as - // `CdsNavigationBar.Action`) crashes FirElementSerializer.valueParameterProto the - // same - // way an error return type does, so degrade those to `Any?` here as well. The - // IR-visitor pass only reaches parameters via firClass.declarations, which can be - // missing members that are served only from the member scope. - if (decl is FirFunction) { - decl.valueParameters.forEach { param -> - if (hasErrorReturnType(param)) { - runCatching { - param.replaceReturnTypeRef(session.builtinTypes.nullableAnyType) - } - } - } - } + // SCOPE, which references these same symbols' FIR. An error type in any position the + // serializer reads — the return type, a value parameter (e.g. a parameter typed by a + // nested enum of a stubbed dependency such as `CdsNavigationBar.Action`), or a + // property's accessors/backing field — crashes FirElementSerializer the same way, so + // degrade all of them to `Any?` here. The IR-visitor pass only reaches these via + // firClass.declarations, which can be missing members served only from the scope. + degradeErrorTypedPositions(decl, session) } // Belt-and-suspenders: also cover anything present only in firClass.declarations. + // Mirror the symbol loop's asymmetry - degrade non-API members, record API ones - so an + // error-typed API member reachable only via this path is not silently missed by + // Assertion 5. recordErrorTypedApiMember de-duplicates, so members served from both + // paths are recorded once. firClass.declarations.forEach { decl -> - if ( - decl is FirCallableDeclaration && - isNonApiVisibility(decl) && - hasErrorReturnType(decl) - ) { - runCatching { decl.replaceReturnTypeRef(session.builtinTypes.nullableAnyType) } + if (decl !is FirCallableDeclaration) return@forEach + if (!classIsNonApi && !isNonApiVisibility(decl)) { + recordErrorTypedApiMember(firClass.symbol.classId.asString(), decl) + return@forEach } + degradeErrorTypedPositions(decl, session) } } - // Same degradation as stripErrorTypedPrivateMembersFromDeclaration, for non-API members - // the class-scoped pass does not reach: top-level (file-facade) and companion members. - private fun replaceErrorReturnTypeWithAny( + private fun degradeErrorTypedMetadataSource( declaration: IrDeclarationBase, session: FirSession, ) { @@ -224,29 +272,114 @@ internal class FirMetadataSanitizerStage : AbiGenStage { val firMetadataSource = metadataSourceOwner.metadata as? FirMetadataSource ?: return val fir = firMetadataSource.fir as? FirCallableDeclaration ?: return if (!isNonApiVisibility(fir)) return - if (!hasErrorReturnType(fir)) return - runCatching { fir.replaceReturnTypeRef(session.builtinTypes.nullableAnyType) } + degradeErrorTypedPositions(fir, session) } - // For a non-API function/constructor with a value parameter whose type resolved to an - // error type in source-only ABI, replace that parameter type with `Any?` so - // FirElementSerializer can serialize it instead of crashing in valueParameterProto with - // "Cannot serialize error type". In source-only ABI a parameter typed by a symbol absent - // from a stubbed dependency (e.g. a nested enum like `CdsNavigationBar.Action`) resolves - // to an error type. Mirrors replaceErrorReturnTypeWithAny for parameters. - private fun replaceErrorValueParameterTypesWithAny( - declaration: IrDeclarationBase, + // The file facade's @Metadata is serialized by FirElementSerializer.packagePartProto, + // which iterates firFile.declarations. A private top-level member is deleted from the IR + // by the IR sanitizer before this pass runs, so visitSimpleFunction/visitProperty never + // reach it, yet the serializer still reads it from the FIR file and crashes on its error + // type (e.g. `private fun f() = Dep.unresolved()`). + private fun degradeErrorTypedFileMemberTypes(declaration: IrFile) { + val firFile = (declaration.metadata as? FirMetadataSource)?.fir as? FirFile ?: return + firFile.declarations.forEach { decl -> + if (decl !is FirCallableDeclaration) return@forEach + if (isNonApiVisibility(decl)) { + degradeErrorTypedPositions(decl, session) + } else { + recordErrorTypedApiMember( + // Null callableId (2.3+) degrades to ""; a legitimate + // root package stays "". Version detail lives in the buckets + // via packageNameOrNullCompat(). + decl.symbol.packageNameOrNullCompat() ?: "", + decl, + ) + } + } + } + + // Degrade every error-typed position the serializer reads for a callable, not just its + // own return type. propertyProto serializes the setter's value parameter whenever the + // accessors are non-default, which is the case for a delegated property + // (`private var x by AtomicReference(...)` with an unresolved `getValue`), so degrading + // only the property return type leaves the crash in place. + private fun degradeErrorTypedPositions( + decl: FirCallableDeclaration, session: FirSession, ) { - val metadataSourceOwner = declaration as? IrMetadataSourceOwner ?: return - val firMetadataSource = metadataSourceOwner.metadata as? FirMetadataSource ?: return - val fir = firMetadataSource.fir as? FirFunction ?: return - if (!isNonApiVisibility(fir)) return - fir.valueParameters.forEach { param -> - if (hasErrorReturnType(param)) { - runCatching { param.replaceReturnTypeRef(session.builtinTypes.nullableAnyType) } + decl.receiverParameter?.let { receiver -> + if ( + runCatching { receiver.typeRef.coneType.containsErrorType() }.getOrDefault(true) + ) { + runCatching { receiver.replaceTypeRef(session.builtinTypes.nullableAnyType) } + .onFailure { failure -> + repairLog.recordFailedRepair( + decl.symbol.callableId.toString(), + "could not degrade unresolved receiver: " + + "${failure.javaClass.simpleName}: ${failure.message}", + ) + } + } + } + decl.contextParameters.forEachIndexed { index, parameter -> + if (hasErrorReturnType(parameter)) { + runCatching { + parameter.replaceReturnTypeRef(session.builtinTypes.nullableAnyType) + } + .onFailure { failure -> + repairLog.recordFailedRepair( + decl.symbol.callableId.toString(), + "could not degrade unresolved context parameter $index: " + + "${failure.javaClass.simpleName}: ${failure.message}", + ) + } + } + } + + if (decl is FirProperty) { + // The property declaration, getter return and setter value parameter describe the + // same JVM property type and must be degraded together. A delegated property's + // backing field has the delegate type instead, so handle it independently. + val accessorTypePositions = + buildList { + add(decl) + decl.getter?.let { add(it) } + decl.setter?.let { addAll(it.valueParameters) } + } + if (accessorTypePositions.any { hasErrorReturnType(it) }) { + accessorTypePositions.forEach { replaceReturnTypeWithAny(it, session) } } + decl.backingField?.let { degradeReturnTypeIfError(it, session) } + return } + // A function's return type and each of its value parameters are independent, so they + // are degraded one by one to keep the degradation as narrow as possible. + degradeReturnTypeIfError(decl, session) + if (decl is FirFunction) { + decl.valueParameters.forEach { degradeReturnTypeIfError(it, session) } + } + } + + private fun degradeReturnTypeIfError( + decl: FirCallableDeclaration, + session: FirSession, + ) { + if (!hasErrorReturnType(decl)) return + replaceReturnTypeWithAny(decl, session) + } + + private fun replaceReturnTypeWithAny( + decl: FirCallableDeclaration, + session: FirSession, + ) { + runCatching { decl.replaceReturnTypeRef(session.builtinTypes.nullableAnyType) } + .onFailure { failure -> + repairLog.recordFailedRepair( + decl.symbol.callableId.toString(), + "could not degrade unresolved type: " + + "${failure.javaClass.simpleName}: ${failure.message}", + ) + } } // --- @Throws stripping helpers --- @@ -467,13 +600,23 @@ internal class FirMetadataSanitizerStage : AbiGenStage { if (toRemove.isNotEmpty()) { superTypeRefs.removeAll(toRemove) + for (classId in strippedSupertypeClassIds) { + repairLog.recordStrippedSupertype( + firClass.symbol.classId.asString(), + "private supertype ${classId.asString()} removed from ABI", + ) + } + convertFirFakeOverridesFromStrippedPrivateSupertypes( firClass, strippedSupertypeClassIds, ) } } catch (e: Exception) { - // Reflection failures are silently ignored + repairLog.recordFailedRepair( + firClass.symbol.classId.asString(), + "could not strip private supertypes: ${e.javaClass.simpleName}: ${e.message}", + ) } } @@ -519,8 +662,17 @@ internal class FirMetadataSanitizerStage : AbiGenStage { firClass.moduleData.session.providedDeclarationsForMetadataService .registerDeclaration(copiedMethod) } catch (e: Exception) { - // Registration failure is silently ignored + repairLog.recordFailedRepair( + "${firClass.symbol.classId.asString()}.$methodName", + "could not register materialized interface method: " + + "${e.javaClass.simpleName}: ${e.message}", + ) } + } else { + repairLog.recordFailedRepair( + "${firClass.symbol.classId.asString()}.$methodName", + "could not copy method from stripped private interface", + ) } } } @@ -594,27 +746,52 @@ internal class FirMetadataSanitizerStage : AbiGenStage { session.symbolProvider.getClassLikeSymbolByClassId(classId) as? FirClassSymbol<*> ?: return false - if (isClassPrivate(classSymbol)) { + if (classSymbol.resolvedStatus.visibility == Visibilities.Local) { return true } - var outerClassId = classId.outerClassId - while (outerClassId != null) { - val outerSymbol = - session.symbolProvider.getClassLikeSymbolByClassId(outerClassId) - as? FirClassSymbol<*> - if (outerSymbol != null && isClassPrivate(outerSymbol)) { - return true - } - outerClassId = outerClassId.outerClassId + // Only a top-level private class is dropped from the ABI; nested private classes + // are kept so their InnerClasses references resolve. So what matters is whether the + // outermost enclosing class is private, not whether any enclosing class is. Walk the + // classId chain by name -- which needs no symbol resolution -- and resolve only the + // outermost class. Resolving each intermediate enclosing symbol would let one that + // fails to resolve mask a top-level-private outermost class and leave the supertype + // dangling. + var outermostClassId = classId + while (outermostClassId.outerClassId != null) { + outermostClassId = outermostClassId.outerClassId!! } - return false + val outermost = + if (outermostClassId == classId) { + classSymbol + } else { + session.symbolProvider.getClassLikeSymbolByClassId(outermostClassId) + as? FirClassSymbol<*> ?: return false + } + + return isClassPrivate(outermost) } catch (e: Exception) { return false } } + private fun isClassEffectivelyPrivate(irClass: IrClass): Boolean { + var current: IrClass? = irClass + while (current != null) { + if ( + current.visibility == + org.jetbrains.kotlin.descriptors.DescriptorVisibilities.PRIVATE || + current.visibility == + org.jetbrains.kotlin.descriptors.DescriptorVisibilities.LOCAL + ) { + return true + } + current = current.parent as? IrClass + } + return false + } + private fun isClassPrivate(classSymbol: FirClassSymbol<*>): Boolean { val visibility = classSymbol.resolvedStatus.visibility return visibility == Visibilities.Private || visibility == Visibilities.Local @@ -706,6 +883,50 @@ internal class FirMetadataSanitizerStage : AbiGenStage { // --- FIR tree sanitizing visitor for cleanupFirTree --- + // An API member with an unresolved type reaches the descriptor as `error/NonExistentClass`, + // which no consumer can link against. Degrading it would only move the lie from the descriptor + // into the metadata, so it is recorded for ValidationStage to report instead. + @OptIn(SymbolInternals::class) + private fun recordErrorTypedApiMember(owner: String, decl: FirCallableDeclaration) { + val member = + runCatching { + decl.symbol.callableNameOrNullCompat()?.asString() + } + .getOrNull() ?: "" + + val isProperty = decl is FirProperty + val propertyTypeHasError = + if (decl is FirProperty) { + // A property's declaration, getter return type and setter value parameter describe the + // same JVM type. The backing field is deliberately excluded: for a delegated property + // it has the delegate type and is stripped before consumer ABI emission. + buildList { + add(decl) + decl.getter?.let { add(it) } + decl.setter?.let { addAll(it.valueParameters) } + } + .any { hasErrorReturnType(it) } + } else { + false + } + val positions = errorTypedApiPositions( + isProperty = isProperty, + returnTypeHasError = !isProperty && hasErrorReturnType(decl), + propertyTypeHasError = propertyTypeHasError, + receiverHasError = + decl.receiverParameter?.let { receiver -> + runCatching { receiver.typeRef.coneType.containsErrorType() }.getOrDefault(true) + } ?: false, + contextParameterErrors = decl.contextParameters.map(::hasErrorReturnType), + valueParameterErrors = + if (decl is FirFunction) decl.valueParameters.map(::hasErrorReturnType) + else emptyList(), + ) + positions.forEach { where -> + repairLog.recordErrorTypedApiMember(decl, owner, member, where) + } + } + // A type ref left in an unresolved/inconsistent state after failed inference can throw on // coneType access; treat that as an error too. private fun hasErrorReturnType(decl: FirCallableDeclaration): Boolean = runCatching { @@ -748,12 +969,30 @@ internal class FirMetadataSanitizerStage : AbiGenStage { override fun visitProperty(property: org.jetbrains.kotlin.fir.declarations.FirProperty) { val initializer = property.initializer if (initializer != null && hasErrorExpressionRecursive(initializer)) { + val owner = property.symbol.callableId.toString() try { val initializerField = property.javaClass.getDeclaredField("initializer") initializerField.isAccessible = true initializerField.set(property, null) - } catch (_: Exception) { - // If reflection fails, skip this property + // A const val that loses its initializer is emitted without a ConstantValue attribute, + // so consumers that constant-fold it fail during their own compile rather than here. + // That reasoning only applies to a constant a consumer can actually see, hence the + // visibility, which [ValidationStage] filters on. + repairLog.recordClearedPropertyInitializer( + owner, + if (property.isConst) "const val, no ConstantValue will be emitted" + else "val initializer discarded", + isConst = property.isConst, + consumerVisible = + property.status.visibility != Visibilities.Private && + property.status.visibility != Visibilities.PrivateToThis && + property.status.visibility != Visibilities.Local, + ) + } catch (e: Exception) { + repairLog.recordFailedRepair( + owner, + "could not clear unresolvable initializer: ${e.javaClass.simpleName}: ${e.message}", + ) } } super.visitProperty(property) @@ -792,6 +1031,13 @@ internal class FirMetadataSanitizerStage : AbiGenStage { decl is FirCallableDeclaration && isNonApiVisibility(decl) && hasErrorReturnType(decl) ) { runCatching { decl.replaceReturnTypeRef(session.builtinTypes.nullableAnyType) } + .onFailure { failure -> + repairLog.recordFailedRepair( + decl.symbol.callableId.toString(), + "could not degrade unresolved pre-IR return type: " + + "${failure.javaClass.simpleName}: ${failure.message}", + ) + } } } } @@ -941,12 +1187,12 @@ internal class FirMetadataSanitizerStage : AbiGenStage { * - Strips private supertypes and converts fake overrides * - Stubs method bodies */ -internal class IrSanitizerStage : AbiGenStage { +internal class IrSanitizerStage(private val repairLog: AbiGenRepairLog) : AbiGenStage { override val name = "IrSanitizer" /** Create the IR generation extension to be registered during FIR-to-IR conversion. */ fun createExtension(sourceFiles: List): IrGenerationExtension { - return NonAbiDeclarationsStrippingIrExtension(sourceFiles) + return NonAbiDeclarationsStrippingIrExtension(sourceFiles, repairLog) } } @@ -996,22 +1242,1036 @@ internal class BytecodeSanitizerStage : AbiGenStage { } } +internal data class AbiValidationInputs( + val outputFiles: List, + val classpathRoots: List, +) + +internal class AbiValidationOutputFile( + val relativePath: String, + val bytes: ByteArray, +) + +private data class EmittedTypeReference( + val owner: String, + val site: String, + val internalName: String, + val eligibleForStubOnlyDetection: Boolean, +) + +private const val NON_EXISTENT_CLASS_INTERNAL_NAME = "error/NonExistentClass" + /** * Validation stage. * - * Checks for leftover references to private supertypes or error annotations that might have slipped - * through the sanitization stages. + * Answers a single question: does the ABI about to be published differ from what a class-ABI build + * would have produced, in a way the jar itself cannot reveal? + * + * The failure mode this guards against is a *green* build that emits a subtly wrong ABI - a + * constant holding a placeholder instead of its real value, or a `const val` emitted with no + * `ConstantValue` attribute at all. Both produce well-formed bytecode, so nothing downstream of + * here can detect them; the only evidence is what the earlier stages recorded in [AbiGenRepairLog]. + * + * Assertion 6 extends that to fabricated stub-only types. If stubgen fabricates a class to keep the + * producer compiling, but no emitted class and no non-stub classpath root actually ships that type, + * then any surviving descriptor that mentions it is a broken ABI even though the descriptor itself + * is well-formed. Same-package phantom classes are the motivating case. + * + * Rollout: [AbiRepairPolicy.OFF] is the default, so this stage emits nothing unless a target asks + * for it via `abiValidationMode`. Warnings are not a softer setting here - fbsource builds Kotlin + * with `-Werror`, so `warn` fails the compile just as `error` does. Enabling it repo-wide therefore + * has to wait until the repairs it names have been driven out, not the other way round. + * + * The cleared-initializer check carries that visibility filter (Assertion 2 below): a `private + * const val` is not part of any consumer's constant folding, so repairing one silently is not the + * defect this stage is looking for. [unsoundConstants] needs no equivalent filter - a fabricated + * constant only ever reaches that path as an annotation argument, which is by construction + * consumer-visible. */ -internal class ValidationStage : AbiGenStage { +internal class ValidationStage(private val repairLog: AbiGenRepairLog) : AbiGenStage { override val name = "Validation" - fun validate(moduleFragment: IrModuleFragment, messageCollector: MessageCollector) { - // Placeholder for future validation checks. - // Potential checks: - // - Verify no IrClass has private supertypes remaining - // - Verify no FirMetadataSource has annotations with error expressions - // - Verify no bytecode references to stripped classes + fun validate( + moduleFragment: IrModuleFragment, + messageCollector: MessageCollector, + policy: AbiRepairPolicy, + inputs: AbiValidationInputs, + ) { + if (policy == AbiRepairPolicy.OFF) return + + // Always emitted, including at zero, so that "no repairs happened" is distinguishable from + // "validation did not run" when aggregating across a build. + messageCollector.report(CompilerMessageSeverity.INFO, repairLog.counterLine()) + + val severity = + if (policy == AbiRepairPolicy.ERROR) CompilerMessageSeverity.ERROR + else CompilerMessageSeverity.WARNING + + // The final class bytes are authoritative: source-stage repair bookkeeping and stub classpath + // candidates can both miss a literal error type that survives a transform or hides in metadata. + val emittedTypeReferences = collectEmittedTypeReferences(inputs.outputFiles) + for (reference in emittedTypeReferences) { + if (reference.internalName != NON_EXISTENT_CLASS_INTERNAL_NAME) continue + messageCollector.report( + CompilerMessageSeverity.ERROR, + "Kosabi ABI validation: `${reference.owner}` emits `${reference.site}` referencing " + + "literal `$NON_EXISTENT_CLASS_INTERNAL_NAME`. The final ABI bytecode contains an " + + "unresolved type and no consumer can link against it.", + ) + } + + // Assertion 1: every synthesised constant has a type consistent with a real declaration. + // Constants that reached ASSUMED_STRING have a fabricated type, not merely a fabricated value. + for (constant in repairLog.unsoundConstants()) { + messageCollector.report( + severity, + "Kosabi ABI validation: constant `${constant.classId}.${constant.name}` was emitted " + + "with an assumed type of `String`. Neither its type nor its value could be " + + "established, so if the real constant is not a String this ABI is wrong in a way " + + "that will not surface until a consumer compiles against it.", + ) + } + + // Assertion 2: no *consumer-visible* const val silently lost its ConstantValue attribute. + // + // The visibility filter is not a convenience. Every one of the 12 constants this assertion + // named on its first repo-wide run was a `private const val` initialised from a constant + // declared in another buck target. A private companion constant takes part in no consumer's + // constant folding, so the message's own justification - "consumers that constant-fold it fail + // during their own compile" - cannot apply to it. Reporting those is a false positive, and + // since `warn` is fatal under `-Werror` a false positive here is a broken build. What they do + // expose is real but narrower, and belongs to a different check: Kosabi's const resolver is + // source-local and cannot read a constant's value off a dependency's ABI. + for (cleared in repairLog.clearedPropertyInitializers) { + if (!cleared.isConst) continue + if (!cleared.consumerVisible) continue + messageCollector.report( + severity, + "Kosabi ABI validation: `${cleared.owner}` is a const val whose initializer could not " + + "be resolved and was discarded. It will be emitted without a ConstantValue " + + "attribute, so consumers that constant-fold it fail during their own compile.", + ) + } + + // Assertion 3: a repair that threw leaves the tree in an unknown state. Unlike a placeholder + // value there is no claim that can be made about the result at all, which makes this the first + // check that should be promoted to a hard error once the rate is known to be zero. + for (failure in repairLog.failedRepairs) { + messageCollector.report( + severity, + "Kosabi ABI validation: repair of `${failure.owner}` failed and was previously " + + "swallowed: ${failure.detail}. The emitted ABI cannot be trusted.", + ) + } + + // Assertion 4: no supertype survived that the ABI jar will not contain a class file for. + // Nested private classes are kept in the ABI, so only a top-level private supertype dangles. + // Unlike the checks above this one is verifiable from the module itself, so it is checked + // directly rather than trusted. + val leakedSupertypes = mutableListOf() + moduleFragment.accept( + object : IrElementVisitorVoidCompat() { + override fun visitElement(element: IrElement) { + element.acceptChildren(this, null) + } + + override fun visitClass(declaration: IrClass) { + for (superType in declaration.superTypes) { + val superClass = + (superType as? org.jetbrains.kotlin.ir.types.IrSimpleType)?.classifier?.owner + as? IrClass ?: continue + if ( + superClass.parent !is IrClass && + superClass.visibility == + org.jetbrains.kotlin.descriptors.DescriptorVisibilities.PRIVATE + ) { + leakedSupertypes.add( + "${declaration.kotlinFqName.asString()} -> ${superClass.kotlinFqName.asString()}", + ) + } + } + super.visitClass(declaration) + } + }, + null, + ) + for (leaked in leakedSupertypes) { + messageCollector.report( + severity, + "Kosabi ABI validation: dangling private supertype survived stripping: $leaked", + ) + } + + // Assertion 5: no consumer-visible member reached the descriptor with an unresolved type. A + // non-API member in that state is degraded to `Any?`; an API member cannot be, so it ships as + // `Lerror/NonExistentClass;` and no consumer can link against it. + for (m in repairLog.errorTypedApiMembers) { + messageCollector.report( + severity, + "Kosabi ABI validation: `${m.owner}.${m.member}` has an unresolved ${m.where}, so its " + + "descriptor was emitted with `error/NonExistentClass` and no consumer can link " + + "against it. The referenced type is not on the source-only ABI classpath. Add the " + + "target that provides it to this target's `source_only_abi_deps`, or declare that " + + "target `required_for_source_only_abi = True`.", + ) + } + + // Assertion 6: no descriptor in the emitted ABI may mention a stub-only class that is absent + // from both the emitted jar and the non-stub classpath. That pattern means stubgen preserved + // compilation by fabricating a class, but source-only ABI is about to publish a phantom type. + for (leak in + findStubOnlyDescriptorLeaks( + inputs, + emittedTypeReferences, + messageCollector, + severity, + )) { + messageCollector.report( + severity, + "Kosabi ABI validation: `${leak.owner}` emits `${leak.site}` with stub-only type " + + "`${leak.internalName.replace('/', '.')}`. Kosabi generated a stub for that class, " + + "but neither this ABI jar nor any non-stub classpath root provides it. This usually " + + "means stubgen fabricated a same-package phantom from an unresolved simple name. " + + "Add the real provider to THIS target's `source_only_abi_deps`, or fix the unresolved " + + "type so source-only ABI does not publish a phantom descriptor.", + ) + } + } + + private fun findStubOnlyDescriptorLeaks( + inputs: AbiValidationInputs, + emittedTypeReferences: Set, + messageCollector: MessageCollector, + severity: CompilerMessageSeverity, + ): List { + val (stubRoots, realRoots) = inputs.classpathRoots.partition { it.isStubClasspathRoot() } + if (stubRoots.isEmpty()) { + messageCollector.report( + severity, + "Kosabi ABI validation: same-package phantom detection could not complete because no " + + "stub classpath root (stubgen_stubs.jar / stubs.jar) was found.", + ) + return emptyList() + } + val stubDeclaredClasses = + collectClassesFromClasspath(stubRoots, messageCollector, severity) ?: return emptyList() + if (stubDeclaredClasses.isEmpty()) return emptyList() + + val emittedClasses = + inputs.outputFiles + .asSequence() + .filter { it.relativePath.endsWith(".class") } + .map { classNameFromRelativePath(it.relativePath) } + .toSet() + + // Subtract the emitted classes first: a stub-declared class that this jar ships itself is never + // a phantom, and that subtraction needs no classpath I/O. Only if candidates survive do we pay + // to enumerate the (potentially large) non-stub classpath, keeping the added compile cost + // proportional to the number of stub-declared candidates rather than the whole classpath. + val stubOnlyCandidates = stubDeclaredClasses.filterTo(linkedSetOf()) { it !in emittedClasses } + if (stubOnlyCandidates.isEmpty()) return emptyList() + + // Materialize the non-stub classpath once so each jar is opened at most one time; the phantom + // check below is then an O(1) set lookup per candidate rather than a jar reopen per candidate. + val realClasspathClasses = + collectClassesFromClasspath(realRoots, messageCollector, severity) ?: return emptyList() + + val phantomClasses = stubOnlyCandidates.filterTo(linkedSetOf()) { it !in realClasspathClasses } + if (phantomClasses.isEmpty()) return emptyList() + + // The shared walk also covers code/debug-only references for the literal NEC invariant. + // Preserve + // the existing phantom check's ABI-surface scope so implementation details cannot become new + // policy failures. + return emittedTypeReferences.filter { reference -> + reference.eligibleForStubOnlyDetection && + reference.internalName.replace('/', '.') in phantomClasses + } + } + + private fun collectEmittedTypeReferences( + outputFiles: List, + ): Set { + val references = linkedSetOf() + outputFiles + .asSequence() + .filter { it.relativePath.endsWith(".class") } + .forEach { outputFile -> + val owner = classNameFromRelativePath(outputFile.relativePath) + ClassReader(outputFile.bytes) + .accept( + object : ClassVisitor(Opcodes.ASM9) { + override fun visit( + version: Int, + access: Int, + name: String?, + signature: String?, + superName: String?, + interfaces: Array?, + ) { + collectInternalName( + references, + owner, + "class identity", + name, + eligibleForStubOnlyDetection = false, + ) + collectInternalName(references, owner, "supertype", superName) + interfaces.orEmpty().forEach { iface -> + collectInternalName(references, owner, "interface", iface) + } + collectSignature(references, owner, "class signature", signature) + } + + override fun visitModule( + name: String?, + access: Int, + version: String?, + ): ModuleVisitor = + object : ModuleVisitor(Opcodes.ASM9) { + override fun visitMainClass(mainClass: String?) { + collectInternalName( + references, + owner, + "module main class", + mainClass, + eligibleForStubOnlyDetection = false, + ) + } + + override fun visitUse(service: String?) { + collectInternalName( + references, + owner, + "module service", + service, + eligibleForStubOnlyDetection = false, + ) + } + + override fun visitProvide( + service: String?, + providers: Array?, + ) { + collectInternalName( + references, + owner, + "module service", + service, + eligibleForStubOnlyDetection = false, + ) + providers.orEmpty().forEach { provider -> + collectInternalName( + references, + owner, + "module service provider", + provider, + eligibleForStubOnlyDetection = false, + ) + } + } + } + + override fun visitOuterClass( + outerClassOwner: String?, + name: String?, + descriptor: String?, + ) { + collectInternalName(references, owner, "outer class", outerClassOwner) + if (descriptor != null) { + collectDescriptor(references, owner, "outer method", descriptor) + } + } + + override fun visitInnerClass( + name: String?, + outerName: String?, + innerName: String?, + access: Int, + ) { + collectInternalName(references, owner, "inner class", name) + collectInternalName(references, owner, "inner class outer", outerName) + } + + override fun visitNestHost(nestHost: String?) { + collectInternalName(references, owner, "nest host", nestHost) + } + + override fun visitNestMember(nestMember: String?) { + collectInternalName(references, owner, "nest member", nestMember) + } + + override fun visitPermittedSubclass(permittedSubclass: String?) { + collectInternalName( + references, + owner, + "permitted subclass", + permittedSubclass, + ) + } + + override fun visitRecordComponent( + name: String, + descriptor: String, + signature: String?, + ): RecordComponentVisitor { + val site = "record component `$name`" + collectDescriptor(references, owner, site, descriptor) + collectTypeSignature(references, owner, "$site signature", signature) + return object : RecordComponentVisitor(Opcodes.ASM9) { + override fun visitAnnotation( + descriptor: String, + visible: Boolean, + ): AnnotationVisitor = annotationVisitor( + references, + owner, + "$site annotation", + descriptor, + ) + + override fun visitTypeAnnotation( + typeRef: Int, + typePath: TypePath?, + descriptor: String, + visible: Boolean, + ): AnnotationVisitor = annotationVisitor( + references, + owner, + "$site type annotation", + descriptor, + ) + } + } + + override fun visitAnnotation( + descriptor: String, + visible: Boolean, + ): AnnotationVisitor = + annotationVisitor(references, owner, "class annotation", descriptor) + + override fun visitTypeAnnotation( + typeRef: Int, + typePath: TypePath?, + descriptor: String, + visible: Boolean, + ): AnnotationVisitor = + annotationVisitor(references, owner, "class type annotation", descriptor) + + override fun visitField( + access: Int, + name: String, + descriptor: String, + signature: String?, + value: Any?, + ): FieldVisitor { + val site = "field `$name`" + collectDescriptor(references, owner, site, descriptor) + collectTypeSignature(references, owner, "$site signature", signature) + collectConstantValue(references, owner, "$site value", value) + return object : FieldVisitor(Opcodes.ASM9) { + override fun visitAnnotation( + descriptor: String, + visible: Boolean, + ): AnnotationVisitor = annotationVisitor( + references, + owner, + "$site annotation", + descriptor, + ) + + override fun visitTypeAnnotation( + typeRef: Int, + typePath: TypePath?, + descriptor: String, + visible: Boolean, + ): AnnotationVisitor = annotationVisitor( + references, + owner, + "$site type annotation", + descriptor, + ) + } + } + + override fun visitMethod( + access: Int, + name: String, + descriptor: String, + signature: String?, + exceptions: Array?, + ): MethodVisitor { + val site = "method `$name$descriptor`" + collectDescriptor(references, owner, site, descriptor) + collectSignature(references, owner, "$site signature", signature) + exceptions.orEmpty().forEach { exceptionInternalName -> + collectInternalName( + references, + owner, + "$site throws", + exceptionInternalName, + ) + } + return object : MethodVisitor(Opcodes.ASM9) { + override fun visitAnnotationDefault(): AnnotationVisitor = + annotationValueVisitor(references, owner, "$site annotation default") + + override fun visitAnnotation( + descriptor: String, + visible: Boolean, + ): AnnotationVisitor = annotationVisitor( + references, + owner, + "$site annotation", + descriptor, + ) + + override fun visitParameterAnnotation( + parameter: Int, + descriptor: String, + visible: Boolean, + ): AnnotationVisitor = annotationVisitor( + references, + owner, + "$site parameter annotation", + descriptor, + ) + + override fun visitTypeAnnotation( + typeRef: Int, + typePath: TypePath?, + descriptor: String, + visible: Boolean, + ): AnnotationVisitor = annotationVisitor( + references, + owner, + "$site type annotation", + descriptor, + ) + + override fun visitInsnAnnotation( + typeRef: Int, + typePath: TypePath?, + descriptor: String, + visible: Boolean, + ): AnnotationVisitor = annotationVisitor( + references, + owner, + "$site instruction type annotation", + descriptor, + eligibleForStubOnlyDetection = false, + ) + + override fun visitTryCatchAnnotation( + typeRef: Int, + typePath: TypePath?, + descriptor: String, + visible: Boolean, + ): AnnotationVisitor = annotationVisitor( + references, + owner, + "$site try/catch type annotation", + descriptor, + eligibleForStubOnlyDetection = false, + ) + + override fun visitLocalVariableAnnotation( + typeRef: Int, + typePath: TypePath?, + start: Array?, + end: Array?, + index: IntArray?, + descriptor: String, + visible: Boolean, + ): AnnotationVisitor = annotationVisitor( + references, + owner, + "$site local variable type annotation", + descriptor, + eligibleForStubOnlyDetection = false, + ) + + override fun visitTypeInsn(opcode: Int, type: String?) { + collectInternalName( + references, + owner, + "$site instruction", + type, + eligibleForStubOnlyDetection = false, + ) + } + + override fun visitFieldInsn( + opcode: Int, + instructionOwner: String?, + name: String?, + descriptor: String?, + ) { + collectInternalName( + references, + owner, + "$site field instruction owner", + instructionOwner, + eligibleForStubOnlyDetection = false, + ) + if (descriptor != null) { + collectDescriptor( + references, + owner, + "$site field instruction", + descriptor, + eligibleForStubOnlyDetection = false, + ) + } + } + + override fun visitMethodInsn( + opcode: Int, + instructionOwner: String?, + name: String?, + descriptor: String?, + isInterface: Boolean, + ) { + collectInternalName( + references, + owner, + "$site method instruction owner", + instructionOwner, + eligibleForStubOnlyDetection = false, + ) + if (descriptor != null) { + collectDescriptor( + references, + owner, + "$site method instruction", + descriptor, + eligibleForStubOnlyDetection = false, + ) + } + } + + override fun visitInvokeDynamicInsn( + name: String?, + descriptor: String?, + bootstrapMethodHandle: Handle?, + vararg bootstrapMethodArguments: Any?, + ) { + if (descriptor != null) { + collectDescriptor( + references, + owner, + "$site invokedynamic", + descriptor, + eligibleForStubOnlyDetection = false, + ) + } + collectConstantValue( + references, + owner, + "$site invokedynamic bootstrap", + bootstrapMethodHandle, + eligibleForStubOnlyDetection = false, + ) + bootstrapMethodArguments.forEach { argument -> + collectConstantValue( + references, + owner, + "$site invokedynamic bootstrap argument", + argument, + eligibleForStubOnlyDetection = false, + ) + } + } + + override fun visitLdcInsn(value: Any?) { + collectConstantValue( + references, + owner, + "$site constant", + value, + eligibleForStubOnlyDetection = false, + ) + } + + override fun visitMultiANewArrayInsn( + descriptor: String?, + numDimensions: Int, + ) { + if (descriptor != null) { + collectDescriptor( + references, + owner, + "$site multi-dimensional array", + descriptor, + eligibleForStubOnlyDetection = false, + ) + } + } + + override fun visitTryCatchBlock( + start: Label?, + end: Label?, + handler: Label?, + type: String?, + ) { + collectInternalName( + references, + owner, + "$site catch type", + type, + eligibleForStubOnlyDetection = false, + ) + } + + override fun visitLocalVariable( + name: String?, + descriptor: String?, + signature: String?, + start: Label?, + end: Label?, + index: Int, + ) { + if (descriptor != null) { + collectDescriptor( + references, + owner, + "$site local variable", + descriptor, + eligibleForStubOnlyDetection = false, + ) + } + collectTypeSignature( + references, + owner, + "$site local variable signature", + signature, + eligibleForStubOnlyDetection = false, + ) + } + + override fun visitFrame( + type: Int, + numLocal: Int, + local: Array?, + numStack: Int, + stack: Array?, + ) { + local.orEmpty().filterIsInstance().forEach { internalName -> + collectInternalName( + references, + owner, + "$site stack map frame", + internalName, + eligibleForStubOnlyDetection = false, + ) + } + stack.orEmpty().filterIsInstance().forEach { internalName -> + collectInternalName( + references, + owner, + "$site stack map frame", + internalName, + eligibleForStubOnlyDetection = false, + ) + } + } + } + } + }, + 0, + ) + } + return references + } + + private fun collectDescriptor( + references: MutableSet, + owner: String, + site: String, + descriptor: String, + eligibleForStubOnlyDetection: Boolean = true, + ) { + collectType( + references, + owner, + site, + Type.getType(descriptor), + eligibleForStubOnlyDetection, + ) + } + + private fun collectType( + references: MutableSet, + owner: String, + site: String, + type: Type, + eligibleForStubOnlyDetection: Boolean = true, + ) { + when (type.sort) { + Type.ARRAY -> + collectType(references, owner, site, type.elementType, eligibleForStubOnlyDetection) + Type.OBJECT -> + collectInternalName( + references, + owner, + site, + type.internalName, + eligibleForStubOnlyDetection, + ) + Type.METHOD -> { + type.argumentTypes.forEach { argumentType -> + collectType(references, owner, site, argumentType, eligibleForStubOnlyDetection) + } + collectType(references, owner, site, type.returnType, eligibleForStubOnlyDetection) + } + } + } + + private fun collectInternalName( + references: MutableSet, + owner: String, + site: String, + internalName: String?, + eligibleForStubOnlyDetection: Boolean = true, + ) { + if (internalName == null) return + if (internalName.startsWith("[")) { + collectDescriptor(references, owner, site, internalName, eligibleForStubOnlyDetection) + } else { + references.add( + EmittedTypeReference(owner, site, internalName, eligibleForStubOnlyDetection), + ) + } + } + + private fun collectConstantValue( + references: MutableSet, + owner: String, + site: String, + value: Any?, + eligibleForStubOnlyDetection: Boolean = true, + ) { + when (value) { + is Type -> collectType(references, owner, site, value, eligibleForStubOnlyDetection) + is Handle -> { + collectInternalName( + references, + owner, + "$site owner", + value.owner, + eligibleForStubOnlyDetection, + ) + collectDescriptor( + references, + owner, + site, + value.desc, + eligibleForStubOnlyDetection, + ) + } + is ConstantDynamic -> { + collectDescriptor( + references, + owner, + site, + value.descriptor, + eligibleForStubOnlyDetection, + ) + collectConstantValue( + references, + owner, + "$site bootstrap", + value.bootstrapMethod, + eligibleForStubOnlyDetection, + ) + for (index in 0 until value.bootstrapMethodArgumentCount) { + collectConstantValue( + references, + owner, + "$site bootstrap argument", + value.getBootstrapMethodArgument(index), + eligibleForStubOnlyDetection, + ) + } + } + } + } + + private fun annotationVisitor( + references: MutableSet, + owner: String, + site: String, + descriptor: String, + eligibleForStubOnlyDetection: Boolean = true, + ): AnnotationVisitor { + collectDescriptor(references, owner, site, descriptor, eligibleForStubOnlyDetection) + return annotationValueVisitor(references, owner, site, eligibleForStubOnlyDetection) + } + + private fun annotationValueVisitor( + references: MutableSet, + owner: String, + site: String, + eligibleForStubOnlyDetection: Boolean = true, + ): AnnotationVisitor = + object : AnnotationVisitor(Opcodes.ASM9) { + override fun visit(name: String?, value: Any?) { + collectConstantValue(references, owner, site, value, eligibleForStubOnlyDetection) + } + + override fun visitEnum(name: String?, descriptor: String, value: String?) { + collectDescriptor(references, owner, site, descriptor, eligibleForStubOnlyDetection) + } + + override fun visitAnnotation( + name: String?, + descriptor: String, + ): AnnotationVisitor = annotationVisitor( + references, + owner, + site, + descriptor, + eligibleForStubOnlyDetection, + ) + + override fun visitArray(name: String?): AnnotationVisitor = + annotationValueVisitor(references, owner, site, eligibleForStubOnlyDetection) + } + + private fun collectSignature( + references: MutableSet, + owner: String, + site: String, + signature: String?, + eligibleForStubOnlyDetection: Boolean = true, + ) { + if (signature == null) return + SignatureReader(signature) + .accept(signatureVisitor(references, owner, site, eligibleForStubOnlyDetection)) } + + private fun collectTypeSignature( + references: MutableSet, + owner: String, + site: String, + signature: String?, + eligibleForStubOnlyDetection: Boolean = true, + ) { + if (signature == null) return + SignatureReader(signature) + .acceptType(signatureVisitor(references, owner, site, eligibleForStubOnlyDetection)) + } + + private fun signatureVisitor( + references: MutableSet, + owner: String, + site: String, + eligibleForStubOnlyDetection: Boolean = true, + ): SignatureVisitor = + object : SignatureVisitor(Opcodes.ASM9) { + private var currentClassInternalName: String? = null + + override fun visitClassType(name: String?) { + currentClassInternalName = name + collectInternalName(references, owner, site, name, eligibleForStubOnlyDetection) + } + + override fun visitInnerClassType(name: String?) { + val enclosing = currentClassInternalName + val nestedName = + when { + name == null -> null + enclosing == null -> name + else -> enclosing + "$" + name + } + currentClassInternalName = nestedName + collectInternalName( + references, + owner, + site, + nestedName, + eligibleForStubOnlyDetection, + ) + } + + override fun visitClassBound(): SignatureVisitor = + signatureVisitor(references, owner, site, eligibleForStubOnlyDetection) + + override fun visitInterfaceBound(): SignatureVisitor = + signatureVisitor(references, owner, site, eligibleForStubOnlyDetection) + + override fun visitSuperclass(): SignatureVisitor = + signatureVisitor(references, owner, site, eligibleForStubOnlyDetection) + + override fun visitInterface(): SignatureVisitor = + signatureVisitor(references, owner, site, eligibleForStubOnlyDetection) + + override fun visitParameterType(): SignatureVisitor = + signatureVisitor(references, owner, site, eligibleForStubOnlyDetection) + + override fun visitReturnType(): SignatureVisitor = + signatureVisitor(references, owner, site, eligibleForStubOnlyDetection) + + override fun visitExceptionType(): SignatureVisitor = + signatureVisitor(references, owner, site, eligibleForStubOnlyDetection) + + override fun visitArrayType(): SignatureVisitor = + signatureVisitor(references, owner, site, eligibleForStubOnlyDetection) + + override fun visitTypeArgument(wildcard: Char): SignatureVisitor = + signatureVisitor(references, owner, site, eligibleForStubOnlyDetection) + } + + private fun classNameFromRelativePath(relativePath: String): String { + return relativePath.removeSuffix(".class").replace('/', '.') + } + + private fun collectClassesFromClasspath( + classpathRoots: List, + messageCollector: MessageCollector, + severity: CompilerMessageSeverity, + ): Set? { + val classes = linkedSetOf() + for (root in classpathRoots) { + try { + when { + root.isDirectory -> { + root + .walkTopDown() + .filter { it.isFile && it.extension == "class" } + .forEach { classFile -> + val relativePath = classFile.relativeTo(root).invariantSeparatorsPath + classes.add(classNameFromRelativePath(relativePath)) + } + } + root.isFile && root.extension == "jar" -> { + JarFile(root).use { jarFile -> + jarFile + .entries() + .asSequence() + .filter { entry -> !entry.isDirectory && entry.name.endsWith(".class") } + .forEach { entry -> classes.add(classNameFromRelativePath(entry.name)) } + } + } + } + } catch (failure: Exception) { + messageCollector.report( + severity, + "Kosabi ABI validation: same-package phantom detection could not complete because " + + "classpath root `$root` could not be read: $failure", + ) + return null + } + } + return classes + } + + // KosabiStubgenStepsBuilder emits the stub classpath as `__%s_stubgen_stubs.jar`, or as the + // corresponding `__%s_stubgen_stubs` directory before packaging. `stubs.jar` stays an equality + // check: as a suffix it would also swallow `core-lambda-stubs.jar` and treat the Android lambda + // stubs as fabricated classes. + private fun File.isStubClasspathRoot(): Boolean = + (isDirectory && name.endsWith("stubgen_stubs")) || + (isFile && (name.endsWith("stubgen_stubs.jar") || name == "stubs.jar")) } /** @@ -1028,12 +2288,15 @@ internal class ValidationStage : AbiGenStage { * compiler infrastructure (FIR-to-IR conversion, code generation). */ internal class AbiGenPipeline( - val firMetadataSanitizer: FirMetadataSanitizerStage = FirMetadataSanitizerStage(), + /** Shared by every stage; the only record of repairs the emitted jar cannot reveal. */ + val repairLog: AbiGenRepairLog = AbiGenRepairLog(), val composeAbi: ComposeAbiEmulationStage = ComposeAbiEmulationStage(), - val irSanitizer: IrSanitizerStage = IrSanitizerStage(), val bytecodeSanitizer: BytecodeSanitizerStage = BytecodeSanitizerStage(), - val validator: ValidationStage = ValidationStage(), ) { + val firMetadataSanitizer: FirMetadataSanitizerStage = FirMetadataSanitizerStage(repairLog) + val irSanitizer: IrSanitizerStage = IrSanitizerStage(repairLog) + val validator: ValidationStage = ValidationStage(repairLog) + val stages: List get() = listOf(firMetadataSanitizer, composeAbi, irSanitizer, bytecodeSanitizer, validator) } diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/k2-jvm-abi-gen/src/main/kotlin/pipeline/AbiGenRepairLog.kt b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/k2-jvm-abi-gen/src/main/kotlin/pipeline/AbiGenRepairLog.kt new file mode 100644 index 0000000..d73d8d5 --- /dev/null +++ b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/k2-jvm-abi-gen/src/main/kotlin/pipeline/AbiGenRepairLog.kt @@ -0,0 +1,201 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is dual-licensed under either the MIT license found in the + * LICENSE-MIT file in the root directory of this source tree or the Apache + * License, Version 2.0 found in the LICENSE-APACHE file in the root directory + * of this source tree. You may select, at your option, one of the + * above-listed licenses. + */ + +@file:SuppressWarnings("PackageLocationMismatch") + +package com.facebook + +import java.util.IdentityHashMap + +/** + * How the ABI generator reacts when it has to emit content it knows is not faithful to the real + * declaration. + * + * `ERROR` is the only setting under which a green build implies a correct ABI. + * + * [WARN] is not a "keeps existing targets building" setting in this repository. fbsource compiles + * Kotlin with `-Werror`, so a `CompilerMessageSeverity.WARNING` terminates the compile exactly as + * [ERROR] would; the two differ only in the message text. Any rollout that treats warnings as the + * safe intermediate step will break every target that trips a repair. + */ +enum class AbiRepairPolicy { + OFF, + WARN, + ERROR; + + companion object { + /** + * Absent or unrecognised values fall back to [OFF]. + * + * The plugin cannot see whether the enclosing compile passes `-Werror`, so it cannot pick a + * non-fatal severity on its own. Reporting therefore has to be requested explicitly by whoever + * knows the target is prepared for it. + */ + fun parse(value: String?): AbiRepairPolicy = + when (value?.lowercase()) { + "warn" -> WARN + "error", + "fail" -> ERROR + else -> OFF + } + } +} + +/** Where the type of a fabricated constant came from. */ +enum class ConstTypeSource { + /** Taken from the declared type of the annotation parameter the constant is passed to. */ + ANNOTATION_PARAMETER, + /** + * No type information was available anywhere, so `String` was assumed. Any non-`String` constant + * reaching this state produces an ABI whose type does not match the real declaration. + */ + ASSUMED_STRING, +} + +/** + * A constant that had to be synthesised because its declaring class was not on the source-only ABI + * classpath. + * + * The value is never recoverable in this situation - only the type sometimes is. See + * [AbiGenRepairLog]. + */ +data class FabricatedConstant( + val classId: String, + val name: String, + val emittedType: String, + val typeSource: ConstTypeSource, +) + +/** + * An initializer that was discarded because it could not be resolved. + * + * [isConst] and [consumerVisible] exist so [ValidationStage] can tell the defect it is looking for + * - a `const val` a consumer will constant-fold - from a `private const val`, which no consumer can + * see and which therefore takes part in no consumer's constant folding. Both default to the + * conservative answer for the record kinds that do not set them. + */ +data class DiscardedInitializer( + val owner: String, + val detail: String, + val isConst: Boolean = false, + val consumerVisible: Boolean = true, +) + +/** + * A consumer-visible member whose signature did not resolve, so the emitted descriptor carries + * `error/NonExistentClass` in the slot named by [where]. + * + * A non-API member in this state is degraded to `Any?`; an API member cannot be, because the + * descriptor IS the ABI. The only honest outcomes are to fail or to ship a stub no consumer can + * link against. + */ +data class ErrorTypedApiMember(val owner: String, val member: String, val where: String) + +/** + * Record of every place the ABI generator knowingly emitted something other than what the source + * says, plus every place a best-effort repair failed outright. + * + * Kosabi cannot see the value of a constant whose declaring target is absent from the source-only + * ABI classpath, so it substitutes a placeholder. That substitution is not detectable in the + * resulting jar - the bytecode is well formed, it just holds the wrong number. This log is the only + * signal that it happened, so nothing here may be dropped on the floor. + * + * One instance is shared by all pipeline stages for a single compilation. + */ +class AbiGenRepairLog { + val fabricatedConstants: MutableList = mutableListOf() + val clearedPropertyInitializers: MutableList = mutableListOf() + val replacedFieldInitializers: MutableList = mutableListOf() + val strippedSupertypes: MutableList = mutableListOf() + + /** + * Repairs that did not even complete - a reflective mutation threw, so the FIR tree is in + * whatever state the failure left it in. Previously these were swallowed by empty catch blocks. + */ + val failedRepairs: MutableList = mutableListOf() + + private val errorTypedApiPositionsByDeclaration: IdentityHashMap> = + IdentityHashMap() + + val errorTypedApiMembers: MutableList = mutableListOf() + + fun recordErrorTypedApiMember( + declarationIdentity: Any, + owner: String, + member: String, + where: String, + ) { + val positions = + errorTypedApiPositionsByDeclaration.getOrPut(declarationIdentity) { mutableSetOf() } + if (positions.add(where)) { + errorTypedApiMembers.add(ErrorTypedApiMember(owner, member, where)) + } + } + + fun recordFabricatedConstant(constant: FabricatedConstant) { + fabricatedConstants.add(constant) + } + + fun recordClearedPropertyInitializer( + owner: String, + detail: String, + isConst: Boolean = false, + consumerVisible: Boolean = true, + ) { + clearedPropertyInitializers.add( + DiscardedInitializer(owner, detail, isConst, consumerVisible), + ) + } + + fun recordReplacedFieldInitializer(owner: String, detail: String) { + replacedFieldInitializers.add(DiscardedInitializer(owner, detail)) + } + + fun recordStrippedSupertype(owner: String, detail: String) { + strippedSupertypes.add(DiscardedInitializer(owner, detail)) + } + + fun recordFailedRepair(owner: String, detail: String) { + failedRepairs.add(DiscardedInitializer(owner, detail)) + } + + /** + * Count of repairs the generator actually performed. [errorTypedApiMembers] and [failedRepairs] + * are deliberately excluded: neither is a repair. An error-typed API member is left untouched by + * design (degrading it would move the lie from the descriptor into the metadata), and a failed + * repair is one that threw. Both are unresolved defects that [ValidationStage] reports on their + * own; do not fold them in here. + */ + val totalRepairs: Int + get() = + fabricatedConstants.size + + clearedPropertyInitializers.size + + replacedFieldInitializers.size + + strippedSupertypes.size + + /** + * Constants whose emitted type is known not to match the real declaration, or is only a guess. + * These are the ones that corrupt a consumer's constant folding rather than merely degrading it. + */ + fun unsoundConstants(): List = fabricatedConstants.filter { + it.typeSource == ConstTypeSource.ASSUMED_STRING + } + + /** Single machine-greppable line so the repair rate can be aggregated across a build. */ + fun counterLine(): String = + "KOSABI_ABI_REPAIR_COUNTERS" + + " fabricated_constants=${fabricatedConstants.size}" + + " fabricated_constants_type_unknown=${unsoundConstants().size}" + + " cleared_property_initializers=${clearedPropertyInitializers.size}" + + " replaced_field_initializers=${replacedFieldInitializers.size}" + + " stripped_supertypes=${strippedSupertypes.size}" + + " error_typed_api_members=${errorTypedApiMembers.size}" + + " failed_repairs=${failedRepairs.size}" +} diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/BUCK b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/BUCK index 55433d5..a2ffe0f 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/BUCK +++ b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/BUCK @@ -38,15 +38,15 @@ buck_kotlin_library( name = "stubsgen-k2-plugin", srcs = glob( [ - "plugin/stubsgen_k2/*.kt", + "plugin/*.kt", ], ), labels = ["codegen_rule"], provided_deps = [ "prelude//toolchains/android/third-party:kotlin-compiler-embeddable", ], - resources = glob(["plugin/stubsgen_k2/resources/**"]), - resources_root = "plugin/stubsgen_k2/resources", + resources = glob(["plugin/resources/**"]), + resources_root = "plugin/resources", tests = [ "fbsource//fbandroid/javatests/com/facebook/kotlin/compilerplugins/kosabi/integrationtest:integration-test", "fbsource//fbandroid/javatests/com/facebook/kotlin/compilerplugins/kosabi/stubsgen:stubsgen", diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/cli/StubsGenCli.kt b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/cli/StubsGenCli.kt index 3152c99..4ab4636 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/cli/StubsGenCli.kt +++ b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/cli/StubsGenCli.kt @@ -40,6 +40,7 @@ object StubsGenCli { stubsDumpDir = stubsDumpDir, stubsClassOutputDir = stubsClassOutputDir, classPaths = classpath, + // @oss-disable: knownSources = ktFiles, ) .generateStubs(ktFiles, CompilerConfiguration(), env.project) } diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/generators/AnnotationStubsGenerator.kt b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/generators/AnnotationStubsGenerator.kt index f0c5515..ffbae3c 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/generators/AnnotationStubsGenerator.kt +++ b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/generators/AnnotationStubsGenerator.kt @@ -10,6 +10,7 @@ package com.facebook.kotlin.compilerplugins.kosabi.stubsgen.generators +import com.facebook.kotlin.compilerplugins.kosabi.common.FullTypeQualifier import com.facebook.kotlin.compilerplugins.kosabi.common.Logger import com.facebook.kotlin.compilerplugins.kosabi.common.filterDifferentOuterClassIn import com.facebook.kotlin.compilerplugins.kosabi.common.outerClassOnlyQualifier @@ -30,40 +31,79 @@ class AnnotationStubsGenerator : StubsGenerator { context.importedTypes.filterDifferentOuterClassIn(context.declaredTypes).filterNot { context.externalTypeReferences.contains(it.outerClassOnlyQualifier()) } + val modulePkg = context.packageName()?.split(".").orEmpty() context.annotationEntries .mapNotNull { it.typeReference?.getChildOfType() } .forEach { annotationType -> val genFullQualifier = annotationType.calculateQualifierList() - val imp = candidates.find { it.names.last() == genFullQualifier.first() } + val imp = context.resolveImportedType(candidates, genFullQualifier.first()) + val pkg: String + val name: String + val inners: List if (imp != null) { - val pkg = imp.pkgAsString() - val name = imp.names.first() - val inners: List = imp.names.drop(1) + genFullQualifier.drop(1) - - val stub = context.stubsContainer.find(pkg, name, inners) - if (stub != null) { - stub.type = KStub.Type.ANNOTATION - } else { - Logger.log( - """ - | [Warning] stub not found - | - name: $pkg:$name - | - inners: $inners - """ - .trimMargin(), - ) - } + pkg = imp.pkgAsString() + name = imp.names.first() + inners = imp.names.drop(1) + genFullQualifier.drop(1) } else { + // No import matches. A qualifier carrying its own package is written out in full; one + // that carries none names a type of the module's own package. Either way this only + // retypes a stub that already exists, so it cannot invent an annotation: an annotation + // left as a plain class is what makes the use site unresolvable. + val written = FullTypeQualifier(genFullQualifier) + val qualifier = + if (written.pkg.isEmpty() && modulePkg.isNotEmpty()) { + FullTypeQualifier(modulePkg + genFullQualifier) + } else { + written + } + if (qualifier.names.isEmpty()) return@forEach + pkg = qualifier.pkgAsString() + name = qualifier.names.first() + inners = qualifier.names.drop(1) + } + val stub = + context.stubsContainer.find(pkg, name, inners) + ?: nestUnderStubbedOuter(context, pkg, name, inners) + if (stub != null) { + stub.type = KStub.Type.ANNOTATION + } else { Logger.log( """ - | [Warning] ImportTypes not found - | - name: $genFullQualifier - """, + | [Warning] stub not found + | - name: $pkg:$name + | - inners: $inners + """ + .trimMargin(), ) } } } + + // A nested annotation written only as `@Outer.Inner`, same-package and unimported, reaches no + // other generator: there is no import for InnerClassStubsGenerator to walk, and + // SamePackageClassStubsGenerator counts the annotation's own referencedName as a known symbol, so + // nothing fabricates the inner. Nesting it is only safe when the outer is itself a stub: an outer + // that is stubbed is off the reduced classpath, so everything nested in it is too, and the inner + // cannot shadow a type that would otherwise resolve. + private fun nestUnderStubbedOuter( + context: GenerationContext, + pkg: String, + name: String, + inners: List, + ): KStub? { + if (inners.isEmpty()) return null + var stubToEdit = context.stubsContainer.find(pkg, name) ?: return null + var innerPkg = "$pkg.$name" + for (innerName in inners) { + val innerStub = + stubToEdit.innerStubs.find { it.name == innerName } + ?: KStub(innerPkg, innerName).also { stubToEdit.innerStubs += it } + innerPkg = "$innerPkg.$innerName" + stubToEdit = innerStub + } + return stubToEdit + } } diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/generators/ClassLevelFunctionStubsGenerator.kt b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/generators/ClassLevelFunctionStubsGenerator.kt index 8640b6e..aed99a1 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/generators/ClassLevelFunctionStubsGenerator.kt +++ b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/generators/ClassLevelFunctionStubsGenerator.kt @@ -27,7 +27,11 @@ class ClassLevelFunctionStubsGenerator : StubsGenerator { override fun generateStubs(context: GenerationContext) { val classLevelFunctionImportsCandidate = context.importedDeclarations - context.declaredTypes - val stubbedFuns = classLevelFunctionImportsCandidate.filter { it.isTopLevelDeclaration() } + // A member import proven to name a class (an all-caps outer used as a type) is stubbed as a + // class elsewhere; stubbing it as a fun too would invent a bogus top-level declaration. + val stubbedFuns = classLevelFunctionImportsCandidate.filter { + it.isTopLevelDeclaration() && !context.isProvenClassImport(it) + } stubbedFuns .groupBy { it.pkg } diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/generators/GenerationContext.kt b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/generators/GenerationContext.kt index 0822d31..9a57efa 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/generators/GenerationContext.kt +++ b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/generators/GenerationContext.kt @@ -38,6 +38,7 @@ import org.jetbrains.kotlin.psi.KtSuperTypeListEntry import org.jetbrains.kotlin.psi.KtTreeVisitorVoid import org.jetbrains.kotlin.psi.KtTypeAlias import org.jetbrains.kotlin.psi.KtTypeParameter +import org.jetbrains.kotlin.psi.KtTypeParameterListOwner import org.jetbrains.kotlin.psi.KtTypeReference import org.jetbrains.kotlin.psi.KtUserType import org.jetbrains.kotlin.psi.psiUtil.getChildOfType @@ -50,17 +51,34 @@ class GenerationContext { val importedDeclarations: Set val importedTypes: Set + /** + * [importedTypes] split by the file that wrote the import. A usage must be attributed against its + * own file's imports: the pooled set resolves a simple name to whichever file imported it first, + * which is the wrong type as soon as two files import different types of the same name. + */ + val importedTypesByFile: Map> + /** * This contains external types outside from the local source codes, etc: classpath, code * generation */ val externalTypeReferences: Set + val knownGeneratedTypes: Set val pkgsInClasspath: Set> val interfaceTypes: List + // Each entry is one multi-bound type parameter's bounds, in source order. + val multiBoundGroups: List> val annotationEntries: List val declaredTypes: Set val fullQualifierTypes: Set val importAlias: Set + + /** + * Alias name -> the type `import a.B as Alias` names. An import's [FullTypeQualifier] carries the + * imported FQN, whose last name is always the real name `B`, so a usage written `Alias` matches + * no candidate by name and would otherwise resolve to nothing. + */ + val importAliasQualifiers: Map val typeAliasSymbol: Set val parameterNames: Set @@ -76,26 +94,39 @@ class GenerationContext { ) { this.projectFiles = projectFiles this.stubsContainer = StubsContainerImpl() + this.knownGeneratedTypes = knownGeneratedTypes if (lightweight) { this.externalTypeReferences = emptySet() this.pkgsInClasspath = emptySet() this.importedDeclarations = emptySet() this.importedTypes = emptySet() + this.importedTypesByFile = emptyMap() this.interfaceTypes = emptyList() + this.multiBoundGroups = emptyList() this.annotationEntries = emptyList() this.declaredTypes = emptySet() this.typeValueArgs = emptyMap() this.importAlias = emptySet() + this.importAliasQualifiers = emptyMap() this.fullQualifierTypes = emptySet() this.usedUserTypes = emptySet() this.typeAliasSymbol = emptySet() this.parameterNames = emptySet() } else { - val importDirectives = projectFiles.flatMap { it.importList?.imports ?: emptyList() } + val importsByFile = projectFiles.associateWith { it.importList?.imports ?: emptyList() } + val importDirectives = importsByFile.values.flatten() this.importedDeclarations = importDirectives.mapNotNull { it.toImportedClass() }.toSet() this.importAlias = importDirectives.mapNotNull { it.aliasName }.toSet() - this.importedTypes = importedDeclarations.filterNot { it.isTopLevelDeclaration() }.toSet() + val rawImportedTypes = importedDeclarations.filterNot { it.isTopLevelDeclaration() }.toSet() + // `importedTypes` and `importedTypesByFile` are assigned after the traversal below: both + // gain the usage-proven class imports, which need the collected type usages. + this.importAliasQualifiers = + importDirectives + .mapNotNull { directive -> + directive.aliasName?.let { alias -> alias to directive.toImportedClass() } + } + .toMap() val dataInClasspath: Pair, Set>> = parseClasspathFileClassesAndPackages(classpath) @@ -116,6 +147,7 @@ class GenerationContext { val usedTypes = mutableSetOf() val multiSegQualifiers = mutableListOf>() val interfaceUserTypes = mutableListOf() + val multiBoundUserTypes = mutableListOf>() val typeValueArgPairs = mutableListOf>() for (ktFile in projectFiles) { val superCallPairs = mutableListOf>() @@ -148,6 +180,9 @@ class GenerationContext { element.getInterfaceTypes().mapNotNull { it.typeAsUserType }, ) } + if (element is KtTypeParameterListOwner) { + multiBoundUserTypes.addAll(element.multiBoundGroups()) + } if (element is KtSuperTypeCallEntry) { element.typeValueArgs()?.let { superCallPairs.add(it) } } @@ -173,12 +208,23 @@ class GenerationContext { this.parameterNames = paramNames this.usedUserTypes = usedTypes this.interfaceTypes = interfaceUserTypes + this.multiBoundGroups = multiBoundUserTypes this.fullQualifierTypes = multiSegQualifiers .distinct() .map { FullTypeQualifier(it) } .filterNot { it.isSdkQualifier() } .toSet() + // An all-caps simple name (`IABJSOTA`, `OTA`, `URI`) parses as a static-const member, so + // its import carries no class name and drops out of the raw type sets above. A member used + // in type position is really a class: promote those imports, module-wide and per file, so + // every consumer resolves the owner. A member never used as a type stays out (top-level + // consts and funs). + this.importedTypes = rawImportedTypes + promoteProvenClassImports(importedDeclarations) + this.importedTypesByFile = importsByFile.mapValues { (_, imports) -> + val quals = imports.mapNotNull { it.toImportedClass() } + quals.filterNot { it.isTopLevelDeclaration() }.toSet() + promoteProvenClassImports(quals) + } // TODO: We might have multiple ctors for each type. // For now we're just skipping to have one (the default) this.typeValueArgs = typeValueArgPairs.toMap() @@ -291,6 +337,83 @@ class GenerationContext { return (nullableTypeWrapper ?: this).getChildOfType() } + // Grouped rather than flattened: the consumer decides per parameter, from the first bound. + private fun KtTypeParameterListOwner.multiBoundGroups(): List> { + val boundsByParameter = mutableMapOf>() + typeParameters.forEach { parameter -> + val name = parameter.name ?: return@forEach + parameter.extendsBound?.let { boundsByParameter.getOrPut(name) { mutableListOf() }.add(it) } + } + typeConstraints.forEach { constraint -> + val name = constraint.subjectTypeParameterName?.getReferencedName() ?: return@forEach + constraint.boundTypeReference?.let { + boundsByParameter.getOrPut(name) { mutableListOf() }.add(it) + } + } + return boundsByParameter.values + .filter { it.size > 1 } + .mapNotNull { bounds -> + val userTypes = bounds.map { it.userTypeForQualifier() } + // The consumer keys the whole-parameter decision off the source-order first bound. + // Compacting nulls away would shift a later bound into first position, so a first bound + // that doesn't resolve to a KtUserType (e.g. a function-type bound) drops the group. + if (userTypes.first() == null) null else userTypes.filterNotNull() + } + .filter { it.size > 1 } + } + + /** + * A member-only import (`import a.b.OTA`) whose name is used in type position names a class: an + * all-caps simple name parses as a static-const member, so the import carries no class name until + * usage proves it. A member never used as a type stays a declaration (a top-level const or fun). + */ + fun isProvenClassImport(imp: FullTypeQualifier): Boolean { + val name = + imp.member?.takeIf { imp.names.isEmpty() && it.first().isUpperCase() } ?: return false + return usedUserTypes.any { it.referencedName == name } + } + + private fun promoteProvenClassImports( + quals: Collection, + ): Set = + quals + .mapNotNull { imp -> + imp.member + ?.takeIf { isProvenClassImport(imp) } + ?.let { name -> + FullTypeQualifier.unsafeBuildQualifier(imp.segments, imp.pkg, listOf(name)) + } + } + .toSet() + + /** + * The candidate an unqualified [simpleName] refers to. + * + * `import a.B as Alias` binds `Alias` and does not bind `B`, so matching an import by the last + * name of its FQN is only correct for an import that carries no alias. An alias resolves to the + * type it names; every other name prefers an unaliased import and only then falls back to the + * historical last-name match, so a name that resolves today keeps resolving. + * + * An alias naming a type the caller's own filter excluded stays unresolved: [candidates] is the + * authority on what this pass may act on. + */ + fun resolveImportedType( + candidates: Collection, + simpleName: String, + ): FullTypeQualifier? { + importAliasQualifiers[simpleName]?.let { aliased -> + candidates + .find { it.segments == aliased.segments } + ?.let { + return it + } + } + val byLastName = candidates.filter { it.names.last() == simpleName } + return byLastName.firstOrNull { candidate -> + importAliasQualifiers.values.none { it.segments == candidate.segments } + } ?: byLastName.firstOrNull() + } + fun packageName(): String? { return projectFiles .map { it -> it.packageFqName.asString() } diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/generators/GenericStubsGenerator.kt b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/generators/GenericStubsGenerator.kt index df3d67e..cd0ccc7 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/generators/GenericStubsGenerator.kt +++ b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/generators/GenericStubsGenerator.kt @@ -28,18 +28,34 @@ class GenericStubsGenerator : StubsGenerator { } // check if this is generic one .filter { it.anyDescendantOfType() } - .distinctBy { it.text } + // Deduplication is per file: the same text in two files can name two different types, + // and each file's usage has to reach its own stub. + .distinctBy { it.containingKtFile to it.text } // TODO: Do not apply for SDK classes - val candidates = context.importedTypes - context.declaredTypes + // A usage resolves against the imports of ITS OWN file. Pooling every file's imports attributes + // a simple name to whichever file imported it first, so two files importing different types of + // the same name give one stub both arities and leave the other bare. + val candidatesByFile = + context.importedTypesByFile.mapValues { (_, imports) -> imports - context.declaredTypes } + val pooledCandidates = context.importedTypes - context.declaredTypes + val modulePkg = context.packageName()?.split(".").orEmpty() for (genType in usedGenericTypes) { + val candidates = candidatesByFile[genType.containingKtFile] ?: pooledCandidates val genFullQualifier = genType.calculateQualifierList() val imp = - candidates.find { it.names.last() == genFullQualifier.first() } - ?: if (genFullQualifier.size > 1) FullTypeQualifier(genFullQualifier) else continue - val pkg = imp.pkgAsString() + context.resolveImportedType(candidates, genFullQualifier.first()) + ?: when { + genFullQualifier.size > 1 -> FullTypeQualifier(genFullQualifier) + modulePkg.isNotEmpty() -> FullTypeQualifier(modulePkg + genFullQualifier) + else -> continue + } val name = imp.names + // An all-caps simple name reads as a static-const member, so the qualifier carries no class + // name to look up. + if (name.isEmpty()) continue + val pkg = imp.pkgAsString() val innerClassNames = if (genFullQualifier == imp.segments) emptyList() else (name.drop(1) + genFullQualifier.drop(1)) diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/generators/InnerClassStubsGenerator.kt b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/generators/InnerClassStubsGenerator.kt index f20b190..9e31fc9 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/generators/InnerClassStubsGenerator.kt +++ b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/generators/InnerClassStubsGenerator.kt @@ -42,7 +42,7 @@ class InnerClassStubsGenerator : StubsGenerator { // imp: com.foo.T1.T2 // segment: T2.T3... val imp: FullTypeQualifier? = - context.importedTypes.find { it.names.last() == segment.first() } + context.resolveImportedType(context.importedTypes, segment.first()) if (imp == null) { Logger.log(" [Warning] ImportedType matches $segment not found!") } diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/generators/InterfaceStubsGenerator.kt b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/generators/InterfaceStubsGenerator.kt index ce0ec2c..7369e42 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/generators/InterfaceStubsGenerator.kt +++ b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/generators/InterfaceStubsGenerator.kt @@ -15,15 +15,46 @@ import com.facebook.kotlin.compilerplugins.kosabi.common.Logger import com.facebook.kotlin.compilerplugins.kosabi.common.filterDifferentOuterClassIn import com.facebook.kotlin.compilerplugins.kosabi.common.stub.model.KStub import com.facebook.kotlin.compilerplugins.kosabi.stubsgen.util.calculateQualifierList +import org.jetbrains.kotlin.psi.KtUserType /** [InnerClassStubsGenerator] should be after [CtorStubsGenerator] in the generation pipeline. */ class InterfaceStubsGenerator : StubsGenerator { override fun generateStubs(context: GenerationContext) { - val candidates = context.importedTypes.filterDifferentOuterClassIn(context.declaredTypes) + // A usage resolves against the imports of ITS OWN file. Pooling every file's imports attributes + // a simple name to whichever file imported it first, so when two files import different types + // of the same name the wrong stub is retyped and the real supertype is left a class. + val candidatesByFile = + context.importedTypesByFile.mapValues { (_, imports) -> + imports.filterDifferentOuterClassIn(context.declaredTypes) + } + val pooledCandidates = context.importedTypes.filterDifferentOuterClassIn(context.declaredTypes) + fun candidatesFor(type: KtUserType): List = + candidatesByFile[type.containingKtFile] ?: pooledCandidates + + // Every stubbed bound of a multi-bound type parameter is emitted as an interface. Kotlin + // permits at most one non-interface bound, so at most one bound of a group can be a class, and + // the rules stacked with this one require that bound to be written first and to be on the + // reduced classpath -- where it is the real type, is never stubbed, and holds the class slot on + // its own. Interface is therefore the sound kind for every bound this pass can reach. + // + // A stub is born a class and carries no ClassKind, so leaving stubbed bounds alone corrupts a + // group whose bounds are all off the classpath: kotlinc hands the single class slot to + // whichever stub comes first and discards the other class-kinded bounds, which moves the + // erasure of the parameter. + // + // A nested bound cannot be retyped either way -- its enclosing stub would stay a class whose + // InnerClasses entry then describes an interface member, which javac rejects -- and the + // decision is whole-group rather than per bound: on facecast's 36-bound clause, retyping the + // group's OTHER tails still broke resolution of the nested bound at position 13, and dropping + // any one earlier retype cleared it. + val multiBoundStubs = + context.multiBoundGroups + .filterNot { group -> group.any { nested(context, candidatesFor(it), it) } } + .flatMap { group -> group.filter { isStubbed(context, candidatesFor(it), it) } } - for (iType in context.interfaceTypes) { + for (iType in context.interfaceTypes + multiBoundStubs) { val qualifierList = iType.calculateQualifierList() - val imp = candidates.find { it.names.last() == qualifierList.first() } + val imp = context.resolveImportedType(candidatesFor(iType), qualifierList.first()) var pkg: String var name: String var inners: List @@ -66,4 +97,59 @@ class InterfaceStubsGenerator : StubsGenerator { } } } + + // Retyping a nested stub would leave its enclosing stub a class whose InnerClasses entry + // describes an interface member, and javac rejects that pair while completing the outer. + private fun nested( + context: GenerationContext, + candidates: Collection, + type: KtUserType, + ): Boolean { + val qualifierList = type.calculateQualifierList() + val imp = context.resolveImportedType(candidates, qualifierList.first()) + return when { + imp != null -> + (imp.names.drop(1) + qualifierList.drop(1)).isNotEmpty() || + imp.pkgAsString().isNestedOwner() + qualifierList.size > 1 -> + FullTypeQualifier(qualifierList).let { + it.names.drop(1).isNotEmpty() || it.pkgAsString().isNestedOwner() + } + else -> false + } + } + + // A stub exists exactly when the type is off the reduced classpath. A bound that is not stubbed + // is the real type and already carries its own kind, so retyping it is neither needed nor safe. + private fun isStubbed( + context: GenerationContext, + candidates: Collection, + first: KtUserType, + ): Boolean { + val qualifierList = first.calculateQualifierList() + val imp = context.resolveImportedType(candidates, qualifierList.first()) + val pkg: String + val name: String + val inners: List + if (imp != null) { + pkg = imp.pkgAsString() + name = imp.names.first() + inners = imp.names.drop(1) + qualifierList.drop(1) + } else if (qualifierList.size > 1) { + val full = FullTypeQualifier(qualifierList) + pkg = full.pkgAsString() + name = full.names.first() + inners = full.names.drop(1) + } else { + // A single segment with no import is either declared in this module -- never stubbed -- or + // a builtin, which is treated as stubbed because that only costs the retyping. + return context.declaredTypes.none { it.names.last() == qualifierList.first() } + } + return context.stubsContainer.find(pkg, name, inners) != null + } } + +// StubBytecodeRender.internalName treats an uppercase trailing package segment as an outer +// class, so a stub owned by one renders as `Outer$Inner`. +private fun String.isNestedOwner(): Boolean = + substringAfterLast('.').firstOrNull()?.isUpperCase() == true diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/generators/PrimitiveStubsGenerator.kt b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/generators/PrimitiveStubsGenerator.kt index a3527ea..1947ac8 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/generators/PrimitiveStubsGenerator.kt +++ b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/generators/PrimitiveStubsGenerator.kt @@ -45,6 +45,15 @@ class PrimitiveStubsGenerator(private val postfixToSkip: String?) : StubsGenerat private const val KOTLIN_STDLIB_PREFIX = "kotlin" private const val JAVA_STDLIB_PREFIX = "java" private const val JAVA_LANG_PREFIX = "java.lang" +// `jdk.compiler` classes are loaded from JRT rather than `jvmClasspathRoots`. Generated stubs would +// shadow the real classes and erase inherited member contracts used for JVM descriptor lowering. +private val JDK_COMPILER_PACKAGE_PREFIXES = listOf( + "com.sun.source", + "com.sun.tools.javac", + "javax.annotation.processing", + "javax.lang.model", + "javax.tools", +) // kotlin.Unit - is an SDK class // kotlinx.Parcelize - is not @@ -52,12 +61,18 @@ private fun isKotlinSdk(pkg: String): Boolean = pkg == KOTLIN_STDLIB_PREFIX || pkg.startsWith("$KOTLIN_STDLIB_PREFIX.") private fun isJavaSdk(pkg: String): Boolean = - pkg == JAVA_STDLIB_PREFIX || pkg.startsWith("$JAVA_STDLIB_PREFIX.") + pkg == JAVA_STDLIB_PREFIX || + pkg.startsWith("$JAVA_STDLIB_PREFIX.") || + JDK_COMPILER_PACKAGE_PREFIXES.any { prefix -> + pkg == prefix || pkg.startsWith("$prefix.") + } private fun String.isSdkImport(): Boolean = isKotlinSdk(this) || isJavaSdk(this) -fun FullTypeQualifier.isSdkQualifier(): Boolean = - isKotlinSdk(segments.first()) || isJavaSdk(segments.first()) +fun FullTypeQualifier.isSdkQualifier(): Boolean { + val pkg = pkgAsString() + return isKotlinSdk(pkg) || isJavaSdk(pkg) +} // Types that don't require imports (auto-imported) fun FullTypeQualifier.isAutoImported(): Boolean { diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/generators/SamePackageClassStubsGenerator.kt b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/generators/SamePackageClassStubsGenerator.kt index 5daac5c..3b1ac92 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/generators/SamePackageClassStubsGenerator.kt +++ b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/generators/SamePackageClassStubsGenerator.kt @@ -46,13 +46,26 @@ class SamePackageClassStubsGenerator : StubsGenerator { // to avoid name collisions with non-auto-imported types that have the same simple name. // Types like android.*, javax.*, etc. require explicit imports and should not be included here. val autoImportedExternalTypes = context.externalTypeReferences.filter { it.isAutoImported() } + val generatedTypesInModulePackage = + context.knownGeneratedTypes.filter { it.pkgAsString() == modulePkgName } + // A qualifier that carries no package (`Outer.Inner`) and whose outer no import, + // declaration, or classpath entry owns names a nested type of the module's own + // package. FullQualifiedClassStubsGenerator drops it for having no package and + // InnerClassStubsGenerator can only reach it through an import, so nothing creates its outer: + // counting that outer as known here is what leaves the usage with no stub at all. + val nestedSamePackageQualifiers = + context.fullQualifierTypes.filter { + it.pkg.isEmpty() && it.member == null && it.names.size > 1 + } allKnownSymbols.addAll( (context.importedTypes + context.declaredTypes + autoImportedExternalTypes + - context.fullQualifierTypes) + generatedTypesInModulePackage + + (context.fullQualifierTypes - nestedSamePackageQualifiers.toSet())) .flatMap { it.segments + it.names }, ) + allKnownSymbols.addAll(nestedSamePackageQualifiers.flatMap { it.names.drop(1) }) // Alias & type parameter names allKnownSymbols.addAll(context.typeAliasSymbol + context.importAlias + context.parameterNames) @@ -76,5 +89,24 @@ class SamePackageClassStubsGenerator : StubsGenerator { context.stubsContainer.add(KStub(modulePkgName, typeName)) } } + + // The usage is `Outer.Inner`, so the outer stub fabricated above still has to own an Inner for + // the reference to resolve. Only outers fabricated here are nested into: an outer resolved any + // other way already has its own owner, and its nesting belongs to that owner's generator. + val fabricatedNames = maybeUnknownClasses.toSet() + nestedSamePackageQualifiers + .filter { it.names.first() in fabricatedNames } + .forEach { qualifier -> + var stubToEdit = + context.stubsContainer.find(modulePkgName, qualifier.names.first()) ?: return@forEach + var innerPkg = "$modulePkgName.${stubToEdit.name}" + for (innerName in qualifier.names.drop(1)) { + val innerStub = + stubToEdit.innerStubs.find { it.name == innerName } + ?: KStub(innerPkg, innerName).also { stubToEdit.innerStubs += it } + innerPkg = "$innerPkg.$innerName" + stubToEdit = innerStub + } + } } } diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/plugin/stubsgen_k2/StubsCodegenCommandLineProcessor.kt b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/plugin/StubsCodegenCommandLineProcessor.kt similarity index 100% rename from prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/plugin/stubsgen_k2/StubsCodegenCommandLineProcessor.kt rename to prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/plugin/StubsCodegenCommandLineProcessor.kt diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/plugin/stubsgen_k2/StubsCodegenCompilerPluginRegistrar.kt b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/plugin/StubsCodegenCompilerPluginRegistrar.kt similarity index 100% rename from prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/plugin/stubsgen_k2/StubsCodegenCompilerPluginRegistrar.kt rename to prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/plugin/StubsCodegenCompilerPluginRegistrar.kt diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/plugin/stubsgen_k2/StubsCodegenK2FirAnalysisHandlerExtension.kt b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/plugin/StubsCodegenK2FirAnalysisHandlerExtension.kt similarity index 65% rename from prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/plugin/stubsgen_k2/StubsCodegenK2FirAnalysisHandlerExtension.kt rename to prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/plugin/StubsCodegenK2FirAnalysisHandlerExtension.kt index aa09be6..f237bf4 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/plugin/stubsgen_k2/StubsCodegenK2FirAnalysisHandlerExtension.kt +++ b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/plugin/StubsCodegenK2FirAnalysisHandlerExtension.kt @@ -31,23 +31,22 @@ class StubsCodegenK2FirAnalysisHandlerExtension( } override fun doAnalysis(project: Project, configuration: CompilerConfiguration): Boolean { - try { - // Use the compiler's canonical source collection, which recognizes Kotlin sources by file - // type (including .kts) and reports an error for missing source roots. Mirrors the approach - // used by K2JvmAbiFirAnalysisHandlerExtension and replaces the deprecated - // KotlinCoreEnvironment-based K1 source lookup. - val ktFiles = - createSourceFilesFromSourceRoots(configuration, project, configuration.kotlinSourceRoots) - StubsGenAPI( // @oss-enable - // @oss-disable: StubsGenApiImpl( - stubsDumpDir, - stubsClassOutputDir, - classPaths, - ) - .generateStubs(ktFiles, configuration, project) - } finally { - throw RuntimeException("Terminating compilation. We're done with Stubgen.") - } + // Use the compiler's canonical source collection, which recognizes Kotlin sources by file + // type (including .kts) and reports an error for missing source roots. Mirrors the approach + // used by K2JvmAbiFirAnalysisHandlerExtension and replaces the deprecated + // KotlinCoreEnvironment-based K1 source lookup. + val ktFiles = + createSourceFilesFromSourceRoots(configuration, project, configuration.kotlinSourceRoots) + StubsGenAPI( // @oss-enable + // @oss-disable: StubsGenApiImpl( + stubsDumpDir, + stubsClassOutputDir, + classPaths, + // @oss-disable: ktFiles, + ) + .generateStubs(ktFiles, configuration, project) + + throw RuntimeException("Terminating compilation. We're done with Stubgen.") return true } diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/plugin/stubsgen_k2/resources/META-INF/services/org.jetbrains.kotlin.compiler.plugin.CommandLineProcessor b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/plugin/resources/META-INF/services/org.jetbrains.kotlin.compiler.plugin.CommandLineProcessor similarity index 100% rename from prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/plugin/stubsgen_k2/resources/META-INF/services/org.jetbrains.kotlin.compiler.plugin.CommandLineProcessor rename to prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/plugin/resources/META-INF/services/org.jetbrains.kotlin.compiler.plugin.CommandLineProcessor diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/plugin/stubsgen_k2/resources/META-INF/services/org.jetbrains.kotlin.compiler.plugin.CompilerPluginRegistrar b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/plugin/resources/META-INF/services/org.jetbrains.kotlin.compiler.plugin.CompilerPluginRegistrar similarity index 100% rename from prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/plugin/stubsgen_k2/resources/META-INF/services/org.jetbrains.kotlin.compiler.plugin.CompilerPluginRegistrar rename to prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/compilerplugins/kosabi/stubsgen/plugin/resources/META-INF/services/org.jetbrains.kotlin.compiler.plugin.CompilerPluginRegistrar diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/ksp/Factory.kt b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/ksp/Factory.kt index d91c7f3..dc95807 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/ksp/Factory.kt +++ b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/ksp/Factory.kt @@ -20,12 +20,14 @@ import com.facebook.buck.jvm.kotlin.ksp.incremental.Ksp2Mode internal fun KotlinCDLoggingContext( languageVersion: LanguageVersion, ksp2Mode: Ksp2Mode, + durationMs: Long? = null, ): KotlinCDLoggingContext = KotlinCDLoggingContext( step = StepParam.KSP2, languageVersion = languageVersion, mode = ModeParam(ksp2Mode), ) .apply { + this.durationMs = durationMs (ksp2Mode as? Ksp2Mode.Incremental)?.reprocessReason?.message?.let { message -> addExtras( "KotlinCDLoggingContextFactory", diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/ksp/Ksp2NoOpDetector.kt b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/ksp/Ksp2NoOpDetector.kt new file mode 100644 index 0000000..4bb9042 --- /dev/null +++ b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/ksp/Ksp2NoOpDetector.kt @@ -0,0 +1,97 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is dual-licensed under either the MIT license found in the + * LICENSE-MIT file in the root directory of this source tree or the Apache + * License, Version 2.0 found in the LICENSE-APACHE file in the root directory + * of this source tree. You may select, at your option, one of the + * above-listed licenses. + */ + +package com.facebook.buck.jvm.kotlin.ksp + +import com.google.devtools.ksp.processing.CodeGenerator +import com.google.devtools.ksp.processing.Resolver +import com.google.devtools.ksp.processing.SymbolProcessor +import com.google.devtools.ksp.processing.SymbolProcessorProvider +import com.google.devtools.ksp.symbol.KSAnnotated + +/** + * Counts the files each native KSP processor generates, so that a processor attached to a target + * but generating nothing for it becomes visible. + * + * `CodeGenerator.generatedFile` is shared by every processor in an invocation and holds only files + * whose output streams are still open, which KSP closes at each round boundary. Output is therefore + * attributable only as a delta taken around each call and summed - exact because KSP runs + * processors sequentially: `process()` per round, then `finish()` once after the last round. + * + * Processors running under the javax adapter are not counted. They emit a placeholder file to force + * a final processing round, and do so precisely when a round generated nothing, so for them + * "generated no files" is not observable this way. They are left uncounted rather than counted + * wrongly. + * + * Zero is evidence of a no-op only under non-incremental processing: a processor with nothing to do + * in an incremental round legitimately generates nothing. + */ +class Ksp2NoOpDetector { + + private val generatedCounts = mutableMapOf() + + /** Files generated per native provider class name. A provider whose processor never ran is 0. */ + val countsByProcessor: Map + get() = generatedCounts.toMap() + + fun wrap(providers: List): List = + providers.map { provider -> + SymbolProcessorProvider { environment -> + val processor = provider.create(environment) + if (isAdapterProcessor(processor)) { + processor + } else { + val name = provider::class.java.name + // Seed the entry so a provider whose processor never generates is distinguishable from + // one that was never attached. That distinction is the point of this class. + generatedCounts.putIfAbsent(name, 0) + Counting(processor, name, environment.codeGenerator) + } + } + } + + /** Matched by name: the adapter is loaded from the per-target processor classloader. */ + private fun isAdapterProcessor(processor: SymbolProcessor): Boolean = + generateSequence>(processor::class.java) { it.superclass } + .any { it.name == ADAPTER_BASE_CLASS } + + private inner class Counting( + private val delegate: SymbolProcessor, + private val name: String, + private val codeGenerator: CodeGenerator, + ) : SymbolProcessor by delegate { + + override fun process(resolver: Resolver): List = counted { + delegate.process(resolver) + } + + override fun finish() { + counted { delegate.finish() } + } + + private fun counted(call: () -> T): T { + val before = codeGenerator.generatedFile.size + return call().also { + generatedCounts[name] = + generatedCounts.getValue(name) + (codeGenerator.generatedFile.size - before) + } + } + } + + private companion object { + /** + * Matched against processor superclasses by name: the adapter lives on the per-target processor + * classloader, so it cannot be referenced directly. Pinned to the real class by + * `AdapterBaseClassFqnTest` in fbandroid and to the same-named fixture by + * `Ksp2NoOpDetectorTest` — update all three together on a rename or move. + */ + const val ADAPTER_BASE_CLASS = "com.facebook.kotlin.ksp.kspadapter.SymbolProcessorBase" + } +} diff --git a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/ksp/Ksp2Step.kt b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/ksp/Ksp2Step.kt index 5de61d4..97ebaa2 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/ksp/Ksp2Step.kt +++ b/prelude/toolchains/android/src/com/facebook/buck/jvm/kotlin/ksp/Ksp2Step.kt @@ -13,6 +13,7 @@ package com.facebook.buck.jvm.kotlin.ksp import com.facebook.buck.core.build.execution.context.IsolatedExecutionContext import com.facebook.buck.core.filesystems.AbsPath import com.facebook.buck.core.filesystems.RelPath +import com.facebook.buck.core.util.log.Logger import com.facebook.buck.io.file.GlobPatternMatcher import com.facebook.buck.jvm.cd.command.kotlin.LanguageVersion import com.facebook.buck.jvm.core.BuildTargetValue @@ -40,6 +41,7 @@ import java.nio.charset.StandardCharsets import java.nio.file.Path import java.util.Optional import java.util.ServiceLoader +import kotlin.time.measureTimedValue class Ksp2Step( private val invokingRule: BuildTargetValue, @@ -65,12 +67,43 @@ class Ksp2Step( private val ksp2Mode: Ksp2Mode, ) : IsolatedStep { + private val noOpDetector = Ksp2NoOpDetector() + @Throws(IOException::class, InterruptedException::class) override fun executeIsolatedStep(context: IsolatedExecutionContext): StepExecutionResult { CapturingPrintStream().use { stderr -> try { - val exitCode: KotlinSymbolProcessing.ExitCode = executeKsp2(stderr, context) - kotlinCDAnalytics.log(KotlinCDLoggingContext(languageVersion, ksp2Mode)) + val (exitCode, elapsed) = measureTimedValue { executeKsp2(stderr, context) } + val durationMs = elapsed.inWholeMilliseconds + // Same shape KotlincStep already emits, so the two steps are greppable together. + LOG.info( + "KOTLINCD_STEP_DURATION|%s|%s|%d|%d", + invokingRule.fullyQualifiedName, + this::class.java.simpleName, + durationMs, + sourceFilePaths.size, + ) + kotlinCDAnalytics.log( + KotlinCDLoggingContext(languageVersion, ksp2Mode, durationMs).apply { + addExtras( + this@Ksp2Step::class.java.simpleName, + "Ksp2 step duration: $durationMs ms", + ) + val counts = noOpDetector.countsByProcessor + if ( + shouldRecordProcessorCounts( + exitCode == KotlinSymbolProcessing.ExitCode.OK, + counts.isNotEmpty(), + ksp2Mode, + ) + ) { + addExtras( + PROCESSOR_OUTPUT_EXTRAS_KEY, + counts.entries.joinToString(",") { "${it.key}=${it.value}" }, + ) + } + }, + ) return when (exitCode) { KotlinSymbolProcessing.ExitCode.OK -> StepExecutionResults.SUCCESS KotlinSymbolProcessing.ExitCode.PROCESSING_ERROR -> @@ -202,7 +235,8 @@ class Ksp2Step( .trimMargin(), ) // Run! - val kotlinSymbolProcessing = KotlinSymbolProcessing(kspConfig, processorProviders, logger) + val kotlinSymbolProcessing = + KotlinSymbolProcessing(kspConfig, noOpDetector.wrap(processorProviders), logger) return kotlinSymbolProcessing.execute() } @@ -262,6 +296,21 @@ class Ksp2Step( } companion object { + private val LOG: Logger = Logger.get(Ksp2Step::class.java) + + private const val PROCESSOR_OUTPUT_EXTRAS_KEY = "ksp2_processor_generated_files" + + /** + * Telemetry records only successful non-incremental runs that produced counts: an aborted run + * leaves never-ran processors reading zero, and in an incremental round a zero mixes genuine + * no-ops with legitimately idle processors. + */ + fun shouldRecordProcessorCounts( + succeeded: Boolean, + hasCounts: Boolean, + ksp2Mode: Ksp2Mode, + ): Boolean = succeeded && hasCounts && ksp2Mode is Ksp2Mode.NonIncremental + private val jdkHomeCache = java.util.concurrent.ConcurrentHashMap() private val JAVA_HOME_REGEX = Regex("""java\.home\s*=\s*(.+)""") diff --git a/prelude/toolchains/android/src/com/facebook/buck/testrunner/BUCK b/prelude/toolchains/android/src/com/facebook/buck/testrunner/BUCK index 1bdfedb..5d54290 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/testrunner/BUCK +++ b/prelude/toolchains/android/src/com/facebook/buck/testrunner/BUCK @@ -19,9 +19,11 @@ buck_java_library( source = "11", # @oss-enable target = "11", # @oss-enable tests = [ + "prelude//toolchains/android/test/com/facebook/buck/testrunner:junit_runner_test", "prelude//toolchains/android/test/com/facebook/buck/testrunner:testrunner", ], visibility = [ + "prelude//toolchains/android/test/com/facebook/buck/testrunner:junit_runner_test", "prelude//toolchains/android/test/com/facebook/buck/testrunner:testrunner", ], deps = [ @@ -55,7 +57,11 @@ buck_kotlin_library( fb_native.kotlin_library( name = "per_test_coverage_base", - srcs = ["BasePerTestCoverageRunListener.kt"], + srcs = [ + "BasePerTestCoverageRunListener.kt", + "PerTestCoverageExtension.kt", + "PerTestCoverageExtensionManager.kt", + ], # Uses fb_native.kotlin_library (not buck_kotlin_library) with kotlin-stdlib in provided_deps: # both consumers (prelude :junit, fbandroid :instrumentation) already package kotlin-stdlib at # different Buck target paths, and shipping ours too breaks Android APK D8 dexing with @@ -76,6 +82,7 @@ fb_native.kotlin_library( ], visibility = [ "fbsource//fbandroid/java/com/facebook/testing/instrumentation/...", + "fbsource//fbandroid/java/com/facebook/testing/robolectric/resourceusage:resourceusage", "fbsource//fbandroid/javatests/com/facebook/testing/instrumentation/...", "prelude//toolchains/android/src/com/facebook/buck/testrunner:junit", "prelude//toolchains/android/test/com/facebook/buck/testrunner:per_test_coverage_base_test", @@ -114,11 +121,13 @@ buck_kotlin_library( source = "11", # @oss-enable target = "11", # @oss-enable tests = [ + "prelude//toolchains/android/test/com/facebook/buck/testrunner:junit_runner_test", "prelude//toolchains/android/test/com/facebook/buck/testrunner:per_test_coverage_listener_test", "prelude//toolchains/android/test/com/facebook/buck/testrunner:testrunner", "prelude//toolchains/android/test/com/facebook/buck/testrunner:tpx_timeout_buffer_manager_test", ], visibility = [ + "prelude//toolchains/android/test/com/facebook/buck/testrunner:junit_runner_test", "prelude//toolchains/android/test/com/facebook/buck/testrunner:per_test_coverage_listener_test", "prelude//toolchains/android/test/com/facebook/buck/testrunner:testrunner", "prelude//toolchains/android/test/com/facebook/buck/testrunner:tpx_timeout_buffer_manager_test", @@ -193,7 +202,6 @@ buck_java_library( "InstrumentationTestRunnerForClout.java", "RunShellCommand.java", ], - required_for_source_only_abi = True, # @oss-disable[end= ]: source = "8", # @oss-disable[end= ]: target = "8", source = "11", # @oss-enable diff --git a/prelude/toolchains/android/src/com/facebook/buck/testrunner/BasePerTestCoverageRunListener.kt b/prelude/toolchains/android/src/com/facebook/buck/testrunner/BasePerTestCoverageRunListener.kt index c821431..c93825b 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/testrunner/BasePerTestCoverageRunListener.kt +++ b/prelude/toolchains/android/src/com/facebook/buck/testrunner/BasePerTestCoverageRunListener.kt @@ -78,14 +78,17 @@ abstract class BasePerTestCoverageRunListener( } override fun testFinished(description: Description) { - val className = description.className ?: return - val methodName = description.methodName ?: return - // TPX-format test name — matches what the standard output listener emits, so TPX can attach - // this entry to the right test row without a translation step. - val testName = "$methodName ($className)" + val testName = coverageTestName(description) ?: return dumpExec(testName = testName, baseFileName = testName) } + /** TPX-format name shared with coverage extensions and `manifest.jsonl`. */ + protected fun coverageTestName(description: Description): String? { + val className = description.className ?: return null + val methodName = description.methodName ?: return null + return "$methodName ($className)" + } + override fun testRunFinished(result: Result?) { // Probes that fire AFTER the last testFinished (framework teardown, deferred class loading) — // same bundle-level attribution as setup. diff --git a/prelude/toolchains/android/src/com/facebook/buck/testrunner/JUnitRunner.java b/prelude/toolchains/android/src/com/facebook/buck/testrunner/JUnitRunner.java index 3b5af29..c0429dc 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/testrunner/JUnitRunner.java +++ b/prelude/toolchains/android/src/com/facebook/buck/testrunner/JUnitRunner.java @@ -48,6 +48,7 @@ import org.junit.runner.manipulation.Filter; import org.junit.runner.notification.Failure; import org.junit.runner.notification.RunListener; +import org.junit.runners.ParentRunner; import org.junit.runners.model.RunnerBuilder; /** @@ -151,7 +152,6 @@ public void run() throws Throwable { tpxListener = new JUnitTpxStandardOutputListener(testResultsOutputSender.get()); jUnitCore.addListener(tpxListener); - // Add Robolectric timeout enforcement listener if this is a Robolectric test if (isRobolectricTest(suite) && "true".equals(System.getProperty("android.per.test.timeout.enabled"))) { RobolectricTimeoutEnforcingRunListener timeoutListener = @@ -420,68 +420,46 @@ protected JUnit4Builder junit4Builder() { }; } - /** - * Checks if a test class is a Robolectric test by examining its runner. - * - *

This method handles two cases: - * - *

    - *
  1. Direct Robolectric runners: The runner class directly extends RobolectricTestRunner - *
  2. Suite-based Robolectric runners: The runner extends Suite (e.g., - * WhatsAppParameterizedRobolectricTestRunner) but its children extend RobolectricTestRunner - *
- * - * @param runner The instantiated runner for this test class - */ + /** Checks if a test class is a Robolectric test by examining its runner tree. */ private boolean isRobolectricTest(Runner runner) { - Class runnerClass = runner.getClass(); try { Class robolectricTestRunner = Class.forName("org.robolectric.RobolectricTestRunner"); - - // Case 1: Runner directly extends RobolectricTestRunner - if (robolectricTestRunner.isAssignableFrom(runnerClass)) { - return true; - } - - // Case 2: Runner extends Suite - check if children extend RobolectricTestRunner - Class suiteClass = Class.forName("org.junit.runners.Suite"); - if (suiteClass.isAssignableFrom(runnerClass)) { - return isRobolectricSuiteRunner(runner, robolectricTestRunner); - } + Class suiteRunner = Class.forName("org.junit.runners.Suite"); + return isRunnerOfTypeOrSuiteOfType(runner, robolectricTestRunner, suiteRunner); } catch (ClassNotFoundException e) { - // Not a Robolectric test + return false; } - return false; } - /** - * Checks if a Suite-based runner has children that extend RobolectricTestRunner. - * - *

This handles parameterized Robolectric test runners like - * WhatsAppParameterizedRobolectricTestRunner which extend Suite but have child runners that - * extend RobolectricTestRunner. - * - * @param runner The instantiated runner - * @param robolectricTestRunner The RobolectricTestRunner class to check against - */ - private boolean isRobolectricSuiteRunner(Runner runner, Class robolectricTestRunner) { + static boolean isRunnerOfTypeOrSuiteOfType( + Runner runner, Class runnerType, Class suiteType) { + if (runnerType.isAssignableFrom(runner.getClass())) { + return true; + } + + if (!suiteType.isAssignableFrom(runner.getClass())) { + return false; + } + try { - // getChildren() is a protected method defined in ParentRunner - Class parentRunner = Class.forName("org.junit.runners.ParentRunner"); - Method getChildrenMethod = parentRunner.getDeclaredMethod("getChildren"); + Method getChildrenMethod = ParentRunner.class.getDeclaredMethod("getChildren"); getChildrenMethod.setAccessible(true); - @SuppressWarnings("unchecked") - List children = (List) getChildrenMethod.invoke(runner); + List children = (List) getChildrenMethod.invoke(runner); - // Check if first child extends RobolectricTestRunner (we only ever deal with one child) - if (!children.isEmpty()) { - Runner firstChild = children.get(0); - return robolectricTestRunner.isAssignableFrom(firstChild.getClass()); + if (children.isEmpty()) { + return false; } + + for (Object child : children) { + if (!(child instanceof Runner) + || !isRunnerOfTypeOrSuiteOfType((Runner) child, runnerType, suiteType)) { + return false; + } + } + return true; } catch (ReflectiveOperationException e) { - // Not a Robolectric suite runner + return false; } - return false; } /** diff --git a/prelude/toolchains/android/src/com/facebook/buck/testrunner/PerTestCoverageExtension.kt b/prelude/toolchains/android/src/com/facebook/buck/testrunner/PerTestCoverageExtension.kt new file mode 100644 index 0000000..da2bb60 --- /dev/null +++ b/prelude/toolchains/android/src/com/facebook/buck/testrunner/PerTestCoverageExtension.kt @@ -0,0 +1,25 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is dual-licensed under either the MIT license found in the + * LICENSE-MIT file in the root directory of this source tree or the Apache + * License, Version 2.0 found in the LICENSE-APACHE file in the root directory + * of this source tree. You may select, at your option, one of the + * above-listed licenses. + */ + +package com.facebook.buck.testrunner + +import java.io.Closeable +import java.io.File + +/** Optional coverage collector discovered by [PerTestCoverageExtensionManager]. */ +interface PerTestCoverageExtension : Closeable { + fun initialize(outputDir: File) + + fun testStarted(testName: String) {} + + fun testFinished(testName: String) {} + + override fun close() {} +} diff --git a/prelude/toolchains/android/src/com/facebook/buck/testrunner/PerTestCoverageExtensionManager.kt b/prelude/toolchains/android/src/com/facebook/buck/testrunner/PerTestCoverageExtensionManager.kt new file mode 100644 index 0000000..d12788d --- /dev/null +++ b/prelude/toolchains/android/src/com/facebook/buck/testrunner/PerTestCoverageExtensionManager.kt @@ -0,0 +1,112 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is dual-licensed under either the MIT license found in the + * LICENSE-MIT file in the root directory of this source tree or the Apache + * License, Version 2.0 found in the LICENSE-APACHE file in the root directory + * of this source tree. You may select, at your option, one of the + * above-listed licenses. + */ + +package com.facebook.buck.testrunner + +import java.io.File +import java.util.ServiceLoader +import java.util.concurrent.ConcurrentHashMap + +/** Manages discovery and lifecycle callbacks for optional per-test coverage extensions. */ +class PerTestCoverageExtensionManager( + private val extensions: List, +) : PerTestCoverageExtension { + + private val reportedFailures = ConcurrentHashMap.newKeySet() + + override fun initialize(outputDir: File) { + extensions.forEach { extension -> + runExtension(extension, "initialization") { extension.initialize(outputDir) } + } + } + + override fun testStarted(testName: String) { + extensions.forEach { extension -> + runExtension(extension, "test start for $testName") { extension.testStarted(testName) } + } + } + + override fun testFinished(testName: String) { + extensions.forEach { extension -> + runExtension(extension, "test finish for $testName") { extension.testFinished(testName) } + } + } + + override fun close() { + extensions.asReversed().forEach { extension -> + runExtension(extension, "close") { extension.close() } + } + } + + private inline fun runExtension( + extension: PerTestCoverageExtension, + context: String, + block: () -> Unit, + ) { + try { + block() + } catch (failure: Throwable) { + if (isFatal(failure)) throw failure + if (reportedFailures.add(extension.javaClass.name)) { + logFailure(extension.javaClass.name, context, failure) + } + } + } + + companion object { + fun load(): PerTestCoverageExtensionManager = + try { + load(ServiceLoader.load(PerTestCoverageExtension::class.java).iterator()) + } catch (failure: Throwable) { + if (isFatal(failure)) throw failure + logFailure( + PerTestCoverageExtension::class.java.name, + "provider discovery", + failure, + ) + PerTestCoverageExtensionManager(emptyList()) + } + + internal fun load( + providers: Iterator, + ): PerTestCoverageExtensionManager { + val extensions = ArrayList() + while (true) { + try { + if (!providers.hasNext()) break + extensions.add(providers.next()) + } catch (failure: Throwable) { + if (isFatal(failure)) throw failure + logFailure( + PerTestCoverageExtension::class.java.name, + "provider discovery", + failure, + ) + } + } + return PerTestCoverageExtensionManager(extensions) + } + + private fun logFailure( + extensionName: String, + context: String, + failure: Throwable, + ) { + System.err.println( + "Per-test coverage extension $extensionName failed during $context; " + + "continuing primary coverage without failing the test run.", + ) + failure.printStackTrace(System.err) + } + + private fun isFatal(failure: Throwable): Boolean = + failure is VirtualMachineError || failure is ThreadDeath + } +} diff --git a/prelude/toolchains/android/src/com/facebook/buck/testrunner/PerTestJUnitCoverageRunListener.kt b/prelude/toolchains/android/src/com/facebook/buck/testrunner/PerTestJUnitCoverageRunListener.kt index da5e2c6..eb47131 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/testrunner/PerTestJUnitCoverageRunListener.kt +++ b/prelude/toolchains/android/src/com/facebook/buck/testrunner/PerTestJUnitCoverageRunListener.kt @@ -30,8 +30,14 @@ class PerTestJUnitCoverageRunListener constructor( outputDir: File, agent: CoverageAgent = ReflectiveCoverageAgent(), + private val extensionManager: PerTestCoverageExtensionManager = + PerTestCoverageExtensionManager.load(), ) : BasePerTestCoverageRunListener(outputDir, agent) { + init { + extensionManager.initialize(outputDir) + } + /** Default implementation that calls JaCoCo's `RT.getAgent()` via reflection. */ class ReflectiveCoverageAgent : CoverageAgent { private val jacocoAgent: Any @@ -68,4 +74,23 @@ constructor( val base = sanitizeFileName(description?.displayName ?: "anon") return "_${base}_$runCount" } + + override fun testStarted(description: Description) { + super.testStarted(description) + val testName = coverageTestName(description) ?: return + extensionManager.testStarted(testName) + } + + override fun testFinished(description: Description) { + val testName = coverageTestName(description) + if (testName != null) { + extensionManager.testFinished(testName) + } + super.testFinished(description) + } + + override fun close() { + extensionManager.close() + super.close() + } } diff --git a/prelude/toolchains/android/src/com/facebook/buck/util/environment/Platform.java b/prelude/toolchains/android/src/com/facebook/buck/util/environment/Platform.java index a03b36c..8e09e8d 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/util/environment/Platform.java +++ b/prelude/toolchains/android/src/com/facebook/buck/util/environment/Platform.java @@ -19,19 +19,11 @@ * Unix (MACOS, LINUX, ...). */ public enum Platform { - LINUX("Linux", "Linux", "linux", "linux", PlatformType.UNIX), - MACOS("OS X", "Mac OS X", "darwin", "osx", PlatformType.UNIX), - WINDOWS("Windows", "Windows", "windows", "windows", PlatformType.WINDOWS), - FREEBSD("FreeBSD", "FreeBSD", "freebsd", "freebsd", PlatformType.UNIX), - UNKNOWN("Unknown", "Unknown", "unknown", "unknown", PlatformType.UNKNOWN); - - /** - * Name that is used for debugging purposes. This is different from {@link #platformName} due to - * historical reasons. - * - *

TODO: remove this field is favor of {@link #platformName} - */ - private final String printableName; + LINUX("Linux", "linux", "linux", PlatformType.UNIX), + MACOS("Mac OS X", "darwin", "osx", PlatformType.UNIX), + WINDOWS("Windows", "windows", "windows", PlatformType.WINDOWS), + FREEBSD("FreeBSD", "freebsd", "freebsd", PlatformType.UNIX), + UNKNOWN("Unknown", "unknown", "unknown", PlatformType.UNKNOWN); private final String autoconfName; private final String canonicalName; @@ -39,12 +31,7 @@ public enum Platform { private final PlatformType platformType; Platform( - String printableName, - String platformName, - String autoconfName, - String canonicalName, - PlatformType platformType) { - this.printableName = printableName; + String platformName, String autoconfName, String canonicalName, PlatformType platformType) { this.platformName = platformName; this.autoconfName = autoconfName; this.canonicalName = canonicalName; @@ -58,10 +45,6 @@ public String getAutoconfName() { return autoconfName; } - public String getPrintableName() { - return printableName; - } - public PlatformType getType() { return platformType; } diff --git a/prelude/toolchains/android/src/com/facebook/buck/util/zip/ZipScrubber.java b/prelude/toolchains/android/src/com/facebook/buck/util/zip/ZipScrubber.java index 76c2f59..2d5952c 100644 --- a/prelude/toolchains/android/src/com/facebook/buck/util/zip/ZipScrubber.java +++ b/prelude/toolchains/android/src/com/facebook/buck/util/zip/ZipScrubber.java @@ -233,14 +233,6 @@ private static void scrubExtraFields(LargeByteBuffer data) throws IOException { } } - /** Read the name of a zip file from a local entry. Useful for debugging. */ - @SuppressWarnings("unused") - private static String localEntryName(LargeByteBuffer entry) throws IOException { - byte[] nameBytes = new byte[entry.getShort(ZipEntry.LOCNAM)]; - entry.slice().position(ZipEntry.LOCHDR).get(nameBytes); - return new String(nameBytes); - } - @Option(name = "--paths-to-scrub") private Path pathsToScrubFile; diff --git a/prelude/toolchains/android/test/com/facebook/buck/android/AdbHelperTest.java b/prelude/toolchains/android/test/com/facebook/buck/android/AdbHelperTest.java index 57a1e55..cfbde88 100644 --- a/prelude/toolchains/android/test/com/facebook/buck/android/AdbHelperTest.java +++ b/prelude/toolchains/android/test/com/facebook/buck/android/AdbHelperTest.java @@ -26,7 +26,6 @@ import com.facebook.buck.installer.android.IsolatedAndroidInstallerPrinter; import com.facebook.buck.testutil.MoreAsserts; import com.facebook.buck.testutil.TestLogSink; -import com.facebook.buck.util.Console; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import java.util.ArrayList; @@ -55,7 +54,6 @@ public class AdbHelperTest { @Rule public ExpectedException exceptionRule = ExpectedException.none(); @Rule public TestLogSink testLogSink = new TestLogSink(AdbHelperTest.class); - private Console testConsole; private TestAdbExecutionContext adbExecutionContext; private AdbHelper basicAdbHelper; @@ -63,8 +61,7 @@ public class AdbHelperTest { @Before public void setUp() { - adbExecutionContext = new TestAdbExecutionContext(Console.createNullConsole()); - testConsole = adbExecutionContext.getConsole(); + adbExecutionContext = new TestAdbExecutionContext(); basicAdbHelper = createAdbHelper(createAdbOptions(), new TargetDeviceOptions()); } @@ -106,7 +103,6 @@ private AdbHelper createAdbHelper( adbExecutionContext, new IsolatedAndroidInstallerPrinter(LOGGER), /* restartAdbOnFailure */ true, - /* skipMetadataIfNoInstalls */ false, SetDebugAppMode.SET); } @@ -230,7 +226,6 @@ public void whenSerialNumberSetInEnvironmentThenCorrectDeviceFound() { AdbHelper myAdbHelper = createAdbHelper( new TestAdbExecutionContext( - testConsole, ImmutableMap.of(AdbHelper.SERIAL_NUMBER_ENV, device.getSerialNumber())), createAdbOptions(), new TargetDeviceOptions()); @@ -427,7 +422,6 @@ private AdbHelper createAdbHelper(AdbOptions options) { adbExecutionContext, new IsolatedAndroidInstallerPrinter(LOGGER), /* restartAdbOnFailure */ true, - /* skipMetadataIfNoInstalls= */ false, SetDebugAppMode.SET) {}; } @@ -446,7 +440,6 @@ private AdbHelper createAdbHelper(List deviceList) { adbExecutionContext, new IsolatedAndroidInstallerPrinter(LOGGER), /* restartAdbOnFailure */ true, - /* skipMetadataIfNoInstalls= */ false, SetDebugAppMode.SET); } diff --git a/prelude/toolchains/android/test/com/facebook/buck/android/TestAdbExecutionContext.java b/prelude/toolchains/android/test/com/facebook/buck/android/TestAdbExecutionContext.java index 6519db7..047da3f 100644 --- a/prelude/toolchains/android/test/com/facebook/buck/android/TestAdbExecutionContext.java +++ b/prelude/toolchains/android/test/com/facebook/buck/android/TestAdbExecutionContext.java @@ -10,19 +10,17 @@ package com.facebook.buck.android; -import com.facebook.buck.util.Console; import com.google.common.collect.ImmutableMap; public class TestAdbExecutionContext extends AdbExecutionContext { private final ImmutableMap extraEnvironment; - public TestAdbExecutionContext(Console console) { - this(console, ImmutableMap.of()); + public TestAdbExecutionContext() { + this(ImmutableMap.of()); } - public TestAdbExecutionContext(Console console, ImmutableMap environment) { - super(console); + public TestAdbExecutionContext(ImmutableMap environment) { extraEnvironment = environment; } diff --git a/prelude/toolchains/android/test/com/facebook/buck/android/TestAndroidDevice.java b/prelude/toolchains/android/test/com/facebook/buck/android/TestAndroidDevice.java index d585bde..4d04003 100644 --- a/prelude/toolchains/android/test/com/facebook/buck/android/TestAndroidDevice.java +++ b/prelude/toolchains/android/test/com/facebook/buck/android/TestAndroidDevice.java @@ -93,11 +93,6 @@ public void rmStaleFiles(String packageName) { throw new UnsupportedOperationException(); } - @Override - public AutoCloseable createForward() throws Exception { - throw new UnsupportedOperationException(); - } - @Override public void installFiles(String filesType, Map installPaths, String packageName) throws Exception { diff --git a/prelude/toolchains/android/test/com/facebook/buck/android/aapt/BUCK b/prelude/toolchains/android/test/com/facebook/buck/android/aapt/BUCK index a6a2b7c..58d1269 100644 --- a/prelude/toolchains/android/test/com/facebook/buck/android/aapt/BUCK +++ b/prelude/toolchains/android/test/com/facebook/buck/android/aapt/BUCK @@ -1,4 +1,4 @@ -load("@prelude//toolchains/android/tools:build_rules.bzl", "buck_java_library", "buck_java_test") +load("@prelude//toolchains/android/tools:build_rules.bzl", "buck_java_library", "buck_java_test", "buck_kotlin_test") load("@prelude//utils:source_listing.bzl", "source_listing") oncall("android_devxx") @@ -35,3 +35,16 @@ buck_java_test( ":testutil", ], ) + +buck_kotlin_test( + name = "resource_source_map_test", + srcs = ["ResourceSourceMapExecutableMainTest.kt"], + friend_paths = [ + "prelude//toolchains/android/src/com/facebook/buck/android/aapt:resource_source_map", + ], + deps = [ + "prelude//toolchains/android/src/com/facebook/buck/android/aapt:resource_source_map", + "prelude//toolchains/android/src/com/facebook/buck/android/aapt:resource_source_map_format", + "prelude//toolchains/android/third-party:junit", + ], +) diff --git a/prelude/toolchains/android/test/com/facebook/buck/android/aapt/MiniAaptTest.java b/prelude/toolchains/android/test/com/facebook/buck/android/aapt/MiniAaptTest.java index 5928a77..f701805 100644 --- a/prelude/toolchains/android/test/com/facebook/buck/android/aapt/MiniAaptTest.java +++ b/prelude/toolchains/android/test/com/facebook/buck/android/aapt/MiniAaptTest.java @@ -18,7 +18,6 @@ import static org.junit.Assert.fail; import com.facebook.buck.android.aapt.MiniAapt.ResourceParseException; -import com.facebook.buck.android.aapt.RDotTxtEntry.CustomDrawableType; import com.facebook.buck.android.aapt.RDotTxtEntry.IdType; import com.facebook.buck.android.aapt.RDotTxtEntry.RType; import com.facebook.buck.core.filesystems.RelPath; @@ -76,11 +75,6 @@ private static Set createTestingFakesWithParents(Set return createTestingFakes(entries, RDotTxtEntryUtil::matchParent); } - private static Set createTestingFakesWithCustomDrawables( - Set entries) { - return createTestingFakes(entries, RDotTxtEntryUtil::matchCustomDrawables); - } - @Before public void setUp() {} @@ -320,15 +314,12 @@ private void testParsingGrayscaleImageImpl(String normalFilename, String graysca Set definitions = aapt.getResourceCollector().getResources(); + // Both spellings of the grayscale suffix are trimmed off the resource name. assertThat( - createTestingFakesWithCustomDrawables(definitions), + createTestingFakes(definitions), IsEqual.equalToObject( ImmutableSet.of( - FakeEntry.createWithCustomDrawable( - IdType.INT, - RType.DRAWABLE, - "fbui_tomato", - CustomDrawableType.GRAYSCALE_IMAGE)))); + FakeEntry.create(IdType.INT, RType.DRAWABLE, "fbui_tomato")))); } @Test diff --git a/prelude/toolchains/android/test/com/facebook/buck/android/aapt/RDotTxtEntryUtil.java b/prelude/toolchains/android/test/com/facebook/buck/android/aapt/RDotTxtEntryUtil.java index 61b217e..12c6484 100644 --- a/prelude/toolchains/android/test/com/facebook/buck/android/aapt/RDotTxtEntryUtil.java +++ b/prelude/toolchains/android/test/com/facebook/buck/android/aapt/RDotTxtEntryUtil.java @@ -31,12 +31,6 @@ public static RDotTxtEntry matchDefault(RDotTxtEntry entry) { } /** Return an entry that checks equality against the id + the default fields. */ - public static RDotTxtEntry matchCustomDrawables(RDotTxtEntry entry) { - return FakeEntry.createWithCustomDrawable( - entry.idType, entry.type, entry.name, entry.customType); - } - - /** Return an entry that checks equality against the custom type + the default fields. */ public static RDotTxtEntry matchId(RDotTxtEntry entry) { return FakeEntry.createWithId(entry.idType, entry.type, entry.name, entry.idValue); } @@ -48,47 +42,24 @@ public static RDotTxtEntry matchParent(RDotTxtEntry entry) { public static class FakeEntry extends RDotTxtEntry { - private final boolean customTypeDefined; - public static FakeEntry create( RDotTxtEntry.IdType idType, RDotTxtEntry.RType type, String name) { - return new FakeEntry(idType, type, name, FAKE_ID, null, null); + return new FakeEntry(idType, type, name, FAKE_ID, null); } public static FakeEntry createWithId( RDotTxtEntry.IdType idType, RDotTxtEntry.RType type, String name, String idValue) { - return new FakeEntry(idType, type, name, idValue, null, null); - } - - public static FakeEntry createWithCustomDrawable( - RDotTxtEntry.IdType idType, - RDotTxtEntry.RType type, - String name, - RDotTxtEntry.CustomDrawableType customType) { - return new FakeEntry(idType, type, name, null, customType, null); + return new FakeEntry(idType, type, name, idValue, null); } public static FakeEntry createWithParent( RDotTxtEntry.IdType idType, RDotTxtEntry.RType type, String name, String parent) { - return new FakeEntry(idType, type, name, FAKE_ID, null, parent); + return new FakeEntry(idType, type, name, FAKE_ID, parent); } private FakeEntry( - IdType idType, - RType type, - String name, - @Nullable String idValue, - @Nullable CustomDrawableType customType, - @Nullable String parent) { - super( - idType, - type, - name, - idValue == null ? FAKE_ID : idValue, - customType == null ? CustomDrawableType.NONE : customType, - parent); - - customTypeDefined = customType != null; + IdType idType, RType type, String name, @Nullable String idValue, @Nullable String parent) { + super(idType, type, name, idValue == null ? FAKE_ID : idValue, parent); } @Override @@ -102,17 +73,15 @@ public boolean equals(Object obj) { } FakeEntry that = (FakeEntry) obj; - return customTypeDefined == that.customTypeDefined - && Objects.equal(type, that.type) + return Objects.equal(type, that.type) && Objects.equal(name, that.name) && Objects.equal(idValue, that.idValue) - && Objects.equal(customType, that.customType) && Objects.equal(parent, that.parent); } @Override public int hashCode() { - return Objects.hashCode(customTypeDefined, type, name, idValue, customType, parent); + return Objects.hashCode(type, name, idValue, parent); } } } diff --git a/prelude/toolchains/android/test/com/facebook/buck/android/aapt/ResourceSourceMapExecutableMainTest.kt b/prelude/toolchains/android/test/com/facebook/buck/android/aapt/ResourceSourceMapExecutableMainTest.kt new file mode 100644 index 0000000..cd85519 --- /dev/null +++ b/prelude/toolchains/android/test/com/facebook/buck/android/aapt/ResourceSourceMapExecutableMainTest.kt @@ -0,0 +1,205 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is dual-licensed under either the MIT license found in the + * LICENSE-MIT file in the root directory of this source tree or the Apache + * License, Version 2.0 found in the LICENSE-APACHE file in the root directory + * of this source tree. You may select, at your option, one of the + * above-listed licenses. + */ + +package com.facebook.buck.android.aapt + +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import org.junit.Assert.assertEquals +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder + +class ResourceSourceMapExecutableMainTest { + @get:Rule val temporaryFolder = TemporaryFolder() + + @Test + fun `resource entries preserve qualifiers and overlay priority`() { + val firstResources = temporaryFolder.newFolder("first-resources").toPath() + val secondResources = temporaryFolder.newFolder("second-resources").toPath() + Files.createDirectories(firstResources.resolve("values")) + Files.createDirectories(firstResources.resolve("layout")) + Files.createDirectories(secondResources.resolve("values-fr")) + Files.writeString( + firstResources.resolve("values/strings.xml"), + "First", + StandardCharsets.UTF_8, + ) + Files.writeString( + secondResources.resolve("values-fr/strings.xml"), + "Second", + StandardCharsets.UTF_8, + ) + Files.writeString( + firstResources.resolve("layout/screen.xml"), + """ + + + + """ + .trimIndent(), + StandardCharsets.UTF_8, + ) + + val resourceDirs = temporaryFolder.newFile("resource-dirs").toPath() + Files.write( + resourceDirs, + listOf( + "$firstResources\tfirst/BUCK", + " ", + "$secondResources\tsecond/BUCK", + ), + StandardCharsets.UTF_8, + ) + val assetDirs = temporaryFolder.newFile("asset-dirs").toPath() + val output = temporaryFolder.newFile("source-map").toPath() + + ResourceSourceMapExecutableMain.generate(resourceDirs, assetDirs, output) + + val expectedLines = listOf( + "R\t0\tid\ttitle_view\t\tfirst/BUCK\tfirst/BUCK", + "R\t0\tlayout\tscreen\t\tfirst/BUCK\tfirst/BUCK", + "R\t0\tstring\ttitle\t\tfirst/BUCK\tfirst/BUCK", + "R\t1\tstring\ttitle\tfr\tsecond/BUCK\tsecond/BUCK", + ) + assertEquals( + expectedLines.joinToString(separator = "\n", postfix = "\n"), + Files.readString(output, StandardCharsets.UTF_8), + ) + } + + @Test + fun `entry point writes an empty map when generation fails`() { + val resourceDirs = temporaryFolder.newFile("resource-dirs").toPath() + Files.writeString( + resourceDirs, + "${temporaryFolder.root.toPath().resolve("missing-resources")}\tmissing/BUCK", + StandardCharsets.UTF_8, + ) + val assetDirs = temporaryFolder.newFile("asset-dirs").toPath() + val output = temporaryFolder.root.toPath().resolve("fallback/source-map") + + ResourceSourceMapExecutableMain.main( + arrayOf( + "--resource-dirs", + resourceDirs.toString(), + "--asset-dirs", + assetDirs.toString(), + "--output=$output", + ), + ) + + assertEquals(emptyList(), Files.readAllLines(output, StandardCharsets.UTF_8)) + } + + @Test + fun `generation skips malformed files without discarding valid resources`() { + val resources = temporaryFolder.newFolder("resources").toPath() + Files.createDirectories(resources.resolve("values")) + Files.createDirectories(resources.resolve("layout")) + Files.writeString( + resources.resolve("values/strings.xml"), + "Title", + StandardCharsets.UTF_8, + ) + Files.writeString( + resources.resolve("values/broken.xml"), + "", + StandardCharsets.UTF_8, + ) + Files.writeString( + resources.resolve("layout/broken.xml"), + "", + StandardCharsets.UTF_8, + ) + + val resourceDirs = temporaryFolder.newFile("resource-dirs").toPath() + Files.writeString(resourceDirs, "$resources\tresources/BUCK", StandardCharsets.UTF_8) + val assetDirs = temporaryFolder.newFile("asset-dirs").toPath() + Files.writeString( + assetDirs, + "${temporaryFolder.root.toPath().resolve("missing-assets")}\tassets/BUCK", + StandardCharsets.UTF_8, + ) + val output = temporaryFolder.newFile("source-map").toPath() + + ResourceSourceMapExecutableMain.generate(resourceDirs, assetDirs, output) + + assertEquals( + listOf("R\t0\tstring\ttitle\t\tresources/BUCK\tresources/BUCK"), + Files.readAllLines(output, StandardCharsets.UTF_8), + ) + } + + @Test + fun `asset entries preserve paths and overlay priority`() { + val firstAssets = temporaryFolder.newFolder("first-assets").toPath() + val secondAssets = temporaryFolder.newFolder("second-assets").toPath() + Files.createDirectories(firstAssets.resolve("images")) + Files.createDirectories(secondAssets.resolve("images")) + val assetPath = "images/icon.webp" + Files.writeString(firstAssets.resolve(assetPath), "first", StandardCharsets.UTF_8) + Files.writeString(secondAssets.resolve(assetPath), "second", StandardCharsets.UTF_8) + + val resourceDirs = temporaryFolder.newFile("resource-dirs").toPath() + val assetDirs = temporaryFolder.newFile("asset-dirs").toPath() + Files.write( + assetDirs, + listOf( + "$firstAssets\tfirst/BUCK", + "$secondAssets\tsecond/BUCK", + ), + StandardCharsets.UTF_8, + ) + val output = temporaryFolder.newFile("source-map").toPath() + + ResourceSourceMapExecutableMain.generate(resourceDirs, assetDirs, output) + + assertEquals( + listOf( + assetSourceMapRow(0, assetPath, "first/BUCK", "first/BUCK"), + assetSourceMapRow(1, assetPath, "second/BUCK", "second/BUCK"), + ), + Files.readAllLines(output, StandardCharsets.UTF_8), + ) + } + + @Test + fun `asset row serialization preserves delimiter characters`() { + val assetRow = + ResourceSourceMapFormat.AssetRow( + 2, + "images/icon\tvariant.webp", + "asset\nsource", + "asset\tBUCK", + ) + + assertEquals(assetRow, ResourceSourceMapFormat.parse(assetRow.serialize())) + } + + private fun assetSourceMapRow( + priority: Int, + path: String, + source: String, + ownerBuildFile: String, + ): String = listOf( + "A", + priority.toString(), + encodeField(path), + encodeField(source), + encodeField(ownerBuildFile), + ) + .joinToString("\t") + + private fun encodeField(value: String): String = + java.util.Base64.getUrlEncoder() + .withoutPadding() + .encodeToString(value.toByteArray(StandardCharsets.UTF_8)) +} diff --git a/prelude/toolchains/android/test/com/facebook/buck/android/exopackage/AndroidDeviceImplTest.kt b/prelude/toolchains/android/test/com/facebook/buck/android/exopackage/AndroidDeviceImplTest.kt index 6970329..0250a92 100644 --- a/prelude/toolchains/android/test/com/facebook/buck/android/exopackage/AndroidDeviceImplTest.kt +++ b/prelude/toolchains/android/test/com/facebook/buck/android/exopackage/AndroidDeviceImplTest.kt @@ -15,6 +15,7 @@ import com.facebook.buck.installer.android.AndroidInstallErrorTag import com.facebook.buck.installer.android.AndroidInstallException import java.io.File import java.nio.file.Files +import java.nio.file.Paths import java.security.MessageDigest import org.junit.Assert.assertEquals import org.junit.Assert.assertFalse @@ -30,6 +31,7 @@ import org.mockito.kotlin.eq import org.mockito.kotlin.inOrder import org.mockito.kotlin.mock import org.mockito.kotlin.never +import org.mockito.kotlin.times import org.mockito.kotlin.verify import org.mockito.kotlin.whenever @@ -257,17 +259,18 @@ class AndroidDeviceImplTest { @Test fun testIsEmulator() { + // A device caches the read-only properties it reads, so each case needs its own instance. // Setup for non-emulator whenever(mockAdbUtils.executeAdbShellCommand("getprop ro.kernel.qemu", serialNumber)) .thenReturn("0") - assertFalse(androidDevice.isEmulator) + assertFalse(AndroidDeviceImpl(serialNumber, mockAdbUtils).isEmulator) // Setup for emulator whenever(mockAdbUtils.executeAdbShellCommand("getprop ro.kernel.qemu", serialNumber)) .thenReturn("1") - assertTrue(androidDevice.isEmulator) + assertTrue(AndroidDeviceImpl(serialNumber, mockAdbUtils).isEmulator) // Setup for Genymotion device whenever(mockAdbUtils.executeAdbShellCommand("getprop ro.kernel.qemu", serialNumber)) @@ -275,6 +278,32 @@ class AndroidDeviceImplTest { assertTrue(AndroidDeviceImpl("192.168.57.101:5555", mockAdbUtils).isEmulator) } + @Test + fun testReadOnlyPropertiesAreOnlyQueriedOnce() { + whenever(mockAdbUtils.executeAdbShellCommand("getprop ro.product.cpu.abilist", serialNumber)) + .thenReturn("arm64-v8a") + + androidDevice.getDeviceAbis() + androidDevice.getDeviceAbis() + androidDevice.getProperty("ro.product.cpu.abilist") + + verify(mockAdbUtils, times(1)) + .executeAdbShellCommand("getprop ro.product.cpu.abilist", serialNumber) + } + + /** Anything outside `ro.` can change mid-install, so it must be read through every time. */ + @Test + fun testMutablePropertiesAreNotCached() { + whenever(mockAdbUtils.executeAdbShellCommand("getprop sys.boot_completed", serialNumber)) + .thenReturn("1") + + androidDevice.getProperty("sys.boot_completed") + androidDevice.getProperty("sys.boot_completed") + + verify(mockAdbUtils, times(2)) + .executeAdbShellCommand("getprop sys.boot_completed", serialNumber) + } + @Test fun testGetDeviceAbis() { whenever(mockAdbUtils.executeAdbShellCommand("getprop ro.product.cpu.abilist", serialNumber)) @@ -834,4 +863,43 @@ class AndroidDeviceImplTest { .executeAdbCommand(eq("uninstall com.meta.ar.helixserver"), eq(serialNumber), any()) inOrder.verify(mockAdbUtils).executeAdbCommand(eq(plainInstall), eq(serialNumber), any()) } + + @Test + fun testInstallBuildUuidFileSetsTheUmaskInTheShellThatWritesTheFile() { + val result = + androidDevice.installBuildUuidFile( + Paths.get("/data/local/tmp/build_metadata"), + packageName, + "some-build-uuid", + ) + + // One command, not two: `umask` is per-process, so one set in its own `adb shell` is gone by + // the time a second shell's redirect creates the file, which then takes adbd's default mode + // rather than 0644. Pinning the single chained command is what holds that. + assertTrue(result) + verify(mockAdbUtils) + .executeAdbShellCommand( + "umask 022 && mkdir -p /data/local/tmp/build_metadata/$packageName && " + + "echo some-build-uuid > /data/local/tmp/build_metadata/$packageName/build_uuid.txt", + serialNumber, + false, + ) + } + + @Test + fun testInstallBuildUuidFileDoesNotFailTheInstallWhenTheDeviceRefusesTheWrite() { + // doAnswer, not thenThrow: Kotlin emits no `throws` clause, so Mockito rejects a checked + // exception as a stubbed one even though the code under test can observe it. + doAnswer { throw AdbCommandFailedException("read-only file system") } + .whenever(mockAdbUtils) + .executeAdbShellCommand(any(), eq(serialNumber), any()) + + assertTrue( + androidDevice.installBuildUuidFile( + Paths.get("/data/local/tmp/build_metadata"), + packageName, + "some-build-uuid", + ), + ) + } } diff --git a/prelude/toolchains/android/test/com/facebook/buck/android/exopackage/ExopackageShardingTest.java b/prelude/toolchains/android/test/com/facebook/buck/android/exopackage/ExopackageShardingTest.java new file mode 100644 index 0000000..4481310 --- /dev/null +++ b/prelude/toolchains/android/test/com/facebook/buck/android/exopackage/ExopackageShardingTest.java @@ -0,0 +1,196 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is dual-licensed under either the MIT license found in the + * LICENSE-MIT file in the root directory of this source tree or the Apache + * License, Version 2.0 found in the LICENSE-APACHE file in the root directory + * of this source tree. You may select, at your option, one of the + * above-listed licenses. + */ + +package com.facebook.buck.android.exopackage; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import com.facebook.buck.core.filesystems.AbsPath; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSortedMap; +import com.google.common.collect.ImmutableSortedSet; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +public class ExopackageShardingTest { + + private static final Path DATA_ROOT = Paths.get("/data/local/tmp/exopackage/com.example"); + + @Rule public TemporaryFolder tmp = new TemporaryFolder(); + + private AbsPath root() throws Exception { + return AbsPath.of(tmp.getRoot().toPath().toRealPath()); + } + + /** A file of exactly {@code bytes}, named by its path relative to the project root. */ + private Path sized(String name, int bytes) throws Exception { + Files.write(tmp.getRoot().toPath().resolve(name), new byte[bytes]); + return Paths.get(name); + } + + private ImmutableList shard(long targetBytes, String... names) + throws Exception { + ImmutableSortedMap.Builder files = ImmutableSortedMap.naturalOrder(); + for (String name : names) { + files.put(Paths.get(name), Paths.get(name)); + } + return ExopackageInstaller.splitIntoShards( + "native_library", files.build(), root(), DATA_ROOT, targetBytes); + } + + /** The names each shard carries, in order, for readable assertions. */ + private static List> contents(ImmutableList shards) { + List> byShard = new ArrayList<>(); + for (ExopackageInstaller.PushShard shard : shards) { + List names = new ArrayList<>(); + shard.installPaths.keySet().forEach(path -> names.add(path.getFileName().toString())); + byShard.add(names); + } + return byShard; + } + + @Test + public void aPayloadUnderTheTargetStaysOneShard() throws Exception { + sized("a.so", 10); + sized("b.so", 10); + + assertEquals(List.of(List.of("a.so", "b.so")), contents(shard(1_000L, "a.so", "b.so"))); + } + + /** The file that crosses the target closes its own shard rather than opening the next one. */ + @Test + public void theFileThatCrossesTheTargetEndsThatShard() throws Exception { + sized("a.so", 60); + sized("b.so", 60); + sized("c.so", 10); + + assertEquals( + List.of(List.of("a.so", "b.so"), List.of("c.so")), + contents(shard(100L, "a.so", "b.so", "c.so"))); + } + + /** Whatever is left when the files run out still ships. */ + @Test + public void theTrailingFilesFormAShardOfTheirOwn() throws Exception { + sized("a.so", 200); + sized("b.so", 5); + + assertEquals(List.of(List.of("a.so"), List.of("b.so")), contents(shard(100L, "a.so", "b.so"))); + } + + /** A shard is never smaller than one file, so an oversized file is not split. */ + @Test + public void aFileLargerThanTheTargetIsNotSplit() throws Exception { + sized("huge.so", 500); + + assertEquals(List.of(List.of("huge.so")), contents(shard(100L, "huge.so"))); + } + + /** The lock file belongs to no payload, so reclaiming must never take it. */ + @Test + public void theLockFileIsNeverUnwanted() { + ImmutableSortedSet present = + ImmutableSortedSet.of(Paths.get("lock"), Paths.get("resources/old.apk")); + + assertEquals( + ImmutableSortedSet.of(Paths.get("resources/old.apk")), + ExopackageInstaller.filesToDelete(present, ImmutableList.of())); + } + + @Test + public void whatAPayloadWantsIsNotUnwanted() throws Exception { + ImmutableSortedSet present = + ImmutableSortedSet.of(Paths.get("resources/keep.apk"), Paths.get("resources/drop.apk")); + + assertEquals( + ImmutableSortedSet.of(Paths.get("resources/drop.apk")), + ExopackageInstaller.filesToDelete( + present, + ImmutableList.of( + payload( + ImmutableMap.of(Paths.get("resources/keep.apk"), Paths.get("src/keep.apk")), + ImmutableMap.of())))); + } + + @Test + public void anEmptyPayloadProducesNoShards() throws Exception { + assertEquals(List.of(), contents(shard(100L))); + } + + /** Destinations are resolved against the device data root, sources against the project root. */ + @Test + public void pathsAreResolvedAgainstTheirOwnRoots() throws Exception { + sized("a.so", 10); + + ExopackageInstaller.PushShard only = shard(1_000L, "a.so").get(0); + + assertEquals("native_library", only.filesType); + assertEquals(DATA_ROOT.resolve("a.so"), only.installPaths.keySet().iterator().next()); + assertEquals(root().resolve("a.so").getPath(), only.installPaths.values().iterator().next()); + } + + /** A payload with the given content and metadata, for the delete-set arithmetic below. */ + private static ExopackageInstaller.ResolvedExoPayload payload( + ImmutableMap filesToInstall, ImmutableMap metadataToInstall) + throws IOException { + return new ExopackageInstaller.ResolvedExoPayload( + new ExoHelper() { + @Override + public ImmutableMap getFilesToInstall() { + return filesToInstall; + } + + @Override + public ImmutableMap getMetadataToInstall() { + return metadataToInstall; + } + + @Override + public String getType() { + return "resources"; + } + }); + } + + /** Only what the device is missing is sent; what it already holds is left alone. */ + @Test + public void onlyFilesTheDeviceDoesNotHaveArePushed() { + ImmutableSortedSet present = ImmutableSortedSet.of(Paths.get("resources/have.apk")); + + assertEquals( + ImmutableSortedMap.of(Paths.get("resources/want.apk"), Paths.get("src/want.apk")), + ExopackageInstaller.filesToPush( + present, + ImmutableMap.of( + Paths.get("resources/have.apk"), Paths.get("src/have.apk"), + Paths.get("resources/want.apk"), Paths.get("src/want.apk")))); + } + + /** A payload the device already has in full sends nothing. */ + @Test + public void aPayloadAlreadyOnTheDevicePushesNothing() { + ImmutableSortedSet present = ImmutableSortedSet.of(Paths.get("resources/have.apk")); + + assertTrue( + ExopackageInstaller.filesToPush( + present, + ImmutableMap.of(Paths.get("resources/have.apk"), Paths.get("src/have.apk"))) + .isEmpty()); + } +} diff --git a/prelude/toolchains/android/test/com/facebook/buck/android/resources/ExoResourcesRewriterTest.java b/prelude/toolchains/android/test/com/facebook/buck/android/resources/ExoResourcesRewriterTest.java index a5f96b8..a3d7a13 100644 --- a/prelude/toolchains/android/test/com/facebook/buck/android/resources/ExoResourcesRewriterTest.java +++ b/prelude/toolchains/android/test/com/facebook/buck/android/resources/ExoResourcesRewriterTest.java @@ -152,76 +152,4 @@ public void rewrite(int type, IntBuffer buf) { assertEquals(expectedOutput, Files.readString(tmp.getRoot().resolve(outputRTxt).getPath())); } - - private ZipInspector baselinePrimaryInspector; - private ZipInspector optPrimaryInspector; - private ZipInspector baselineExoInspector; - private ZipInspector optExoInspector; - - private void runBaselineAndOptimizedRewrites() throws IOException { - if (baselinePrimaryInspector != null) { - return; // Already ran - } - - // Run without optimizations (baseline) - ResourceProcessingConfig.setOptimizationsEnabled(false); - AbsPath baselinePrimary = tmp.getRoot().resolve("baseline_primary.apk"); - AbsPath baselineExo = tmp.getRoot().resolve("baseline_exo.apk"); - ExoResourcesRewriter.rewriteResources( - tmp.getRoot(), - tmp.getRoot().relativize(apkPath), - tmp.getRoot().relativize(baselinePrimary), - baselineExo.getPath()); - - // Run with optimizations (uses raw pass-through for unmodified entries) - ResourceProcessingConfig.setOptimizationsEnabled(true); - AbsPath optPrimary = tmp.getRoot().resolve("opt_primary.apk"); - AbsPath optExo = tmp.getRoot().resolve("opt_exo.apk"); - try { - ExoResourcesRewriter.rewriteResources( - tmp.getRoot(), - tmp.getRoot().relativize(apkPath), - tmp.getRoot().relativize(optPrimary), - optExo.getPath()); - } finally { - ResourceProcessingConfig.setOptimizationsEnabled(false); - } - - baselinePrimaryInspector = new ZipInspector(baselinePrimary); - optPrimaryInspector = new ZipInspector(optPrimary); - baselineExoInspector = new ZipInspector(baselineExo); - optExoInspector = new ZipInspector(optExo); - } - - @Test - public void testOptimizedPrimaryApkHasSameEntries() throws IOException { - runBaselineAndOptimizedRewrites(); - assertEquals( - baselinePrimaryInspector.getZipFileEntries(), optPrimaryInspector.getZipFileEntries()); - } - - @Test - public void testOptimizedPrimaryResourcesArscIsIdentical() throws IOException { - runBaselineAndOptimizedRewrites(); - assertArrayEquals( - baselinePrimaryInspector.getFileContents("resources.arsc"), - optPrimaryInspector.getFileContents("resources.arsc")); - } - - @Test - public void testOptimizedExoApkHasSameEntries() throws IOException { - runBaselineAndOptimizedRewrites(); - assertEquals(baselineExoInspector.getZipFileEntries(), optExoInspector.getZipFileEntries()); - } - - @Test - public void testOptimizedExoEntriesHaveIdenticalContent() throws IOException { - runBaselineAndOptimizedRewrites(); - for (String entry : baselineExoInspector.getZipFileEntries()) { - assertArrayEquals( - "Decompressed content mismatch for entry: " + entry, - baselineExoInspector.getFileContents(entry), - optExoInspector.getFileContents(entry)); - } - } } diff --git a/prelude/toolchains/android/test/com/facebook/buck/installer/InstallerServiceTest.java b/prelude/toolchains/android/test/com/facebook/buck/installer/InstallerServiceTest.java index 9b57e53..0f2be3d 100644 --- a/prelude/toolchains/android/test/com/facebook/buck/installer/InstallerServiceTest.java +++ b/prelude/toolchains/android/test/com/facebook/buck/installer/InstallerServiceTest.java @@ -47,6 +47,11 @@ private static class FakeInstaller implements InstallCommand { final AtomicInteger installs = new AtomicInteger(); final AtomicInteger handedOver = new AtomicInteger(); + @Override + public String name() { + return "fake"; + } + @Override public InstallResult fileReady(String artifact, Path artifactPath, InstallId installId) { handedOver.incrementAndGet(); diff --git a/prelude/toolchains/android/test/com/facebook/buck/installer/android/AndroidArtifactsTest.java b/prelude/toolchains/android/test/com/facebook/buck/installer/android/AndroidArtifactsTest.java index ed00ba3..082c04b 100644 --- a/prelude/toolchains/android/test/com/facebook/buck/installer/android/AndroidArtifactsTest.java +++ b/prelude/toolchains/android/test/com/facebook/buck/installer/android/AndroidArtifactsTest.java @@ -14,225 +14,129 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; +import java.util.Set; import org.junit.Test; public class AndroidArtifactsTest { - private static final long T0 = 1_700_000_000_000L; + /** A payload is complete only once every artifact buck declared for it has turned up. */ @Test - public void noArrivalsProducesNoMetrics() { - assertTrue(new AndroidArtifacts().getInstallMetrics(T0).isEmpty()); - } - - /** Every class reports both halves of its story, and classes read in the order they landed. */ - @Test - public void eachClassReportsWhenItLandedAndHowLongItTookToTransfer() { + public void aPayloadIsCompleteOnlyWhenAllOfItsArtifactsHaveArrived() { AndroidArtifacts artifacts = new AndroidArtifacts(); - artifacts.recordFileArrival("options", T0); - artifacts.recordFileArrival("resources_exopackage_res", T0 + 2_000L); - artifacts.recordFileArrival("native_library_exopackage_info_directory", T0 + 1_000L); - artifacts.recordFileArrival("secondary_dex_exopackage_info_metadata", T0 + 4_000L); - // Any name the installer does not recognise is the apk. - artifacts.recordFileArrival("fbandroid_arm64_exo-native", T0 + 3_000L); - - artifacts.recordPush("native_library", 0L, 8_000L); - artifacts.recordPush("resources", 8_000L, 9_500L); - artifacts.recordPush("secondary_dex", 9_500L, 12_000L); - artifacts.recordApkInstall(12_000L, 13_250L); + artifacts.setExpectedArtifacts( + Set.of( + "native_library_exopackage_info_directory", "native_library_exopackage_info_metadata")); - Map metrics = artifacts.getInstallMetrics(T0 + 20_000L); - - assertEquals( - List.of( - "control_arrival_s", - "control_transfer_s", - "native_arrival_s", - "native_transfer_s", - "resources_arrival_s", - "resources_transfer_s", - "apk_arrival_s", - "apk_transfer_s", - "dex_arrival_s", - "dex_transfer_s"), - new ArrayList<>(metrics.keySet()).subList(0, 10)); - assertEquals("1.000", metrics.get("native_arrival_s")); - assertEquals("8.000", metrics.get("native_transfer_s")); - assertEquals("2.000", metrics.get("resources_arrival_s")); - assertEquals("1.500", metrics.get("resources_transfer_s")); - assertEquals("4.000", metrics.get("dex_arrival_s")); - assertEquals("2.500", metrics.get("dex_transfer_s")); - assertEquals("3.000", metrics.get("apk_arrival_s")); - assertEquals("1.250", metrics.get("apk_transfer_s")); - // Control artifacts are read on the host. - assertEquals("0.000", metrics.get("control_transfer_s")); - } - - /** A class is only ready once its last artifact lands. */ - @Test - public void classArrivalUsesTheLatestArtifactInThatClass() { - AndroidArtifacts artifacts = new AndroidArtifacts(); - artifacts.recordFileArrival("resources_exopackage_res", T0); - artifacts.recordFileArrival("resources_exopackage_res_hash", T0 + 500L); - artifacts.recordFileArrival("resources_exopackage_assets", T0 + 7_000L); - artifacts.recordFileArrival("resources_exopackage_assets_hash", T0 + 1_000L); + artifacts.recordFileArrival("native_library_exopackage_info_directory", T0); + assertFalse(artifacts.hasAllArtifactsFor(AndroidArtifacts.ArtifactClass.NATIVE_LIBRARY)); - assertEquals("7.000", artifacts.getInstallMetrics(T0 + 9_000L).get("resources_arrival_s")); + artifacts.recordFileArrival("native_library_exopackage_info_metadata", T0 + 1L); + assertTrue(artifacts.hasAllArtifactsFor(AndroidArtifacts.ArtifactClass.NATIVE_LIBRARY)); } - /** Retries must not move an artifact's recorded arrival later. */ + /** + * A payload the build does not produce is never complete, so the early push does not treat it as + * something to send. + */ @Test - public void firstArrivalWinsForARepeatedArtifact() { + public void anUndeclaredPayloadIsNeverComplete() { AndroidArtifacts artifacts = new AndroidArtifacts(); + artifacts.setExpectedArtifacts(Set.of("options", "manifest", "my_app")); + artifacts.recordFileArrival("options", T0); artifacts.recordFileArrival("manifest", T0); - artifacts.recordFileArrival("manifest", T0 + 6_000L); + artifacts.recordFileArrival("my_app", T0); - assertEquals("0.000", artifacts.getInstallMetrics(T0 + 9_000L).get("control_arrival_s")); + assertFalse(artifacts.hasAllArtifactsFor(AndroidArtifacts.ArtifactClass.NATIVE_LIBRARY)); + assertFalse(artifacts.hasAllArtifactsFor(AndroidArtifacts.ArtifactClass.RESOURCES)); + assertTrue(artifacts.hasAllArtifactsFor(AndroidArtifacts.ArtifactClass.APK)); } - /** Metadata is derived from the payloads, so it belongs to no class of its own. */ + /** Before buck says what it will send, nothing is complete, however much has arrived. */ @Test - public void metadataIsReportedSeparatelyFromTheArtifactClasses() { + public void nothingIsCompleteBeforeBuckDeclaresWhatItWillSend() { AndroidArtifacts artifacts = new AndroidArtifacts(); - artifacts.recordFileArrival("options", T0); - artifacts.recordFileArrival("my_app", T0); - artifacts.recordPush("native_library", 0L, 5_000L); - artifacts.recordPush("metadata", 5_000L, 5_400L); - - Map metrics = artifacts.getInstallMetrics(T0 + 6_000L); + artifacts.recordFileArrival("native_library_exopackage_info_directory", T0); + artifacts.recordFileArrival("native_library_exopackage_info_metadata", T0); - assertEquals("0.400", metrics.get("metadata_transfer_s")); - assertFalse(metrics.containsKey("metadata_arrival_s")); + assertFalse(artifacts.hasAllArtifactsFor(AndroidArtifacts.ArtifactClass.NATIVE_LIBRARY)); } + /** Payloads complete independently, which is what lets each be pushed on its own. */ @Test - public void anInstallWithoutMetadataDoesNotReportIt() { + public void oneIncompletePayloadDoesNotHoldBackAnother() { AndroidArtifacts artifacts = new AndroidArtifacts(); - artifacts.recordFileArrival("my_app", T0); - artifacts.recordApkInstall(0L, 3_000L); + artifacts.setExpectedArtifacts( + Set.of( + "resources_exopackage_res", + "resources_exopackage_res_hash", + "secondary_dex_exopackage_info_directory", + "secondary_dex_exopackage_info_metadata")); + artifacts.recordFileArrival("resources_exopackage_res", T0); + artifacts.recordFileArrival("resources_exopackage_res_hash", T0); + artifacts.recordFileArrival("secondary_dex_exopackage_info_directory", T0); - assertFalse(artifacts.getInstallMetrics(T0 + 4_000L).containsKey("metadata_transfer_s")); + assertTrue(artifacts.hasAllArtifactsFor(AndroidArtifacts.ArtifactClass.RESOURCES)); + assertFalse(artifacts.hasAllArtifactsFor(AndroidArtifacts.ArtifactClass.SECONDARY_DEX)); } - /** - * The trace this work started from: native libs and resources are ready immediately, dex and the - * apk only 37s later, and payloads are pushed one at a time after everything has landed. - */ + /** Names buck declared but never sent, which is how a naming drift becomes visible. */ @Test - public void criticalPathReplaysTheInstallWithoutTheWait() { + public void undeliveredArtifactsAreThoseDeclaredButNeverArrived() { AndroidArtifacts artifacts = new AndroidArtifacts(); - for (String control : new String[] {"options", "manifest", "cpu_filters"}) { - artifacts.recordFileArrival(control, T0); - } - for (String resource : - new String[] { - "resources_exopackage_assets", "resources_exopackage_assets_hash", - "resources_exopackage_res", "resources_exopackage_res_hash" - }) { - artifacts.recordFileArrival(resource, T0); - } - artifacts.recordFileArrival("native_library_exopackage_info_metadata", T0); - artifacts.recordFileArrival("native_library_exopackage_info_directory", T0 + 1_000L); - artifacts.recordFileArrival("secondary_dex_exopackage_info_directory", T0 + 36_000L); - artifacts.recordFileArrival("secondary_dex_exopackage_info_metadata", T0 + 37_000L); - artifacts.recordFileArrival("my_app", T0 + 37_000L); - - // Pushed one payload at a time, so the windows do not overlap. - artifacts.recordDeviceSetup(0L, 1_000L); - artifacts.recordPush("secondary_dex", 1_000L, 28_019L); - artifacts.recordPush("native_library", 28_019L, 71_259L); - artifacts.recordPush("resources", 71_259L, 89_741L); - artifacts.recordPush("metadata", 89_741L, 90_055L); - artifacts.recordApkInstall(90_055L, 102_191L); - - Map metrics = artifacts.getInstallMetrics(T0 + 140_000L); - - // 1s setup, then resources (available at 0), native (1s), dex (37s), metadata, then the apk. - assertEquals("1.000", metrics.get("device_setup_s")); - assertEquals("102.191", metrics.get("critical_path_s")); - assertEquals("140.000", metrics.get("total_s")); - assertEquals("37.809", metrics.get("potential_saving_s")); + artifacts.setExpectedArtifacts( + Set.of( + "resources_exopackage_res", + "resources_exopackage_res_hash", + "secondary_dex_exopackage_info_directory")); + artifacts.recordFileArrival("resources_exopackage_res", T0); + + assertEquals( + Set.of("resources_exopackage_res_hash", "secondary_dex_exopackage_info_directory"), + artifacts.undeliveredArtifacts()); } - /** - * Payloads arriving apart is the case a per-payload cap gets wrong: resources is pushable from - * the start even though native does not turn up until much later. - */ + /** Nothing outstanding once every declared artifact has turned up. */ @Test - public void staggeredArrivalsAreCreditedIndependently() { + public void nothingIsUndeliveredOnceEverythingHasArrived() { AndroidArtifacts artifacts = new AndroidArtifacts(); - artifacts.recordFileArrival("options", T0); - artifacts.recordFileArrival("manifest", T0); + artifacts.setExpectedArtifacts( + Set.of("resources_exopackage_res", "resources_exopackage_res_hash")); artifacts.recordFileArrival("resources_exopackage_res", T0); artifacts.recordFileArrival("resources_exopackage_res_hash", T0); - artifacts.recordFileArrival("native_library_exopackage_info_directory", T0 + 50_000L); - artifacts.recordFileArrival("native_library_exopackage_info_metadata", T0 + 50_000L); - artifacts.recordFileArrival("secondary_dex_exopackage_info_directory", T0 + 60_000L); - artifacts.recordFileArrival("my_app", T0 + 60_000L); - - artifacts.recordPush("resources", 0L, 10_000L); - artifacts.recordPush("native_library", 10_000L, 20_000L); - artifacts.recordPush("secondary_dex", 20_000L, 40_000L); - artifacts.recordApkInstall(40_000L, 50_000L); - Map metrics = artifacts.getInstallMetrics(T0 + 110_000L); - - // resources 0-10s, idle until native at 50s, native 50-60s, dex 60-80s, apk 80-90s. - assertEquals("90.000", metrics.get("critical_path_s")); - assertEquals("20.000", metrics.get("potential_saving_s")); + assertTrue(artifacts.undeliveredArtifacts().isEmpty()); } + /** An arrival buck never declared is not undelivered; only the declared set is tracked. */ @Test - public void anInstallThatNeverWaitedHasNothingToSave() { + public void anUndeclaredArrivalIsNotReportedAsUndelivered() { AndroidArtifacts artifacts = new AndroidArtifacts(); - artifacts.recordFileArrival("options", T0); - artifacts.recordFileArrival("manifest", T0); - artifacts.recordFileArrival("my_app", T0 + 5_000L); - artifacts.recordApkInstall(0L, 3_000L); - - Map metrics = artifacts.getInstallMetrics(T0 + 8_000L); + artifacts.setExpectedArtifacts(Set.of("resources_exopackage_res")); + artifacts.recordFileArrival("resources_exopackage_res", T0); + artifacts.recordFileArrival("secondary_dex_exopackage_info_directory", T0); - assertEquals("8.000", metrics.get("critical_path_s")); - assertEquals("0.000", metrics.get("potential_saving_s")); + assertTrue(artifacts.undeliveredArtifacts().isEmpty()); } - /** A group pushed as several concurrent shards spans all of them. */ + /** Nothing left to wait for once every declared artifact has turned up. */ @Test - public void shardsOfOneGroupAreMergedIntoOneWindow() { + public void allArtifactsArrivedOnceEveryDeclaredOneHas() { AndroidArtifacts artifacts = new AndroidArtifacts(); - artifacts.recordFileArrival("options", T0); - artifacts.recordFileArrival("native_library_exopackage_info_directory", T0); - artifacts.recordPush("native_library", 1_000L, 9_000L); - artifacts.recordPush("native_library", 2_000L, 6_000L); + artifacts.setExpectedArtifacts(Set.of("resources_exopackage_res", "apk")); + artifacts.recordFileArrival("resources_exopackage_res", T0); + assertFalse(artifacts.allArtifactsArrived()); - assertEquals("8.000", artifacts.getInstallMetrics(T0 + 10_000L).get("native_transfer_s")); + artifacts.recordFileArrival("apk", T0); + assertTrue(artifacts.allArtifactsArrived()); } - /** - * Once payloads are pushed concurrently, replaying them one at a time would total more than the - * install actually took and report a negative saving. The replay uses however many pushers the - * install itself used. - */ + /** Before buck declares anything, nothing has arrived as far as this is concerned. */ @Test - public void concurrentPushesAreReplayedConcurrently() { + public void nothingHasArrivedBeforeBuckDeclaresWhatItWillSend() { AndroidArtifacts artifacts = new AndroidArtifacts(); - artifacts.recordFileArrival("options", T0); - artifacts.recordFileArrival("manifest", T0); - artifacts.recordFileArrival("native_library_exopackage_info_directory", T0); - artifacts.recordFileArrival("native_library_exopackage_info_metadata", T0); - artifacts.recordFileArrival("secondary_dex_exopackage_info_directory", T0); - artifacts.recordFileArrival("my_app", T0); - - // Both payloads in flight over the same window. - artifacts.recordPush("native_library", 0L, 30_000L); - artifacts.recordPush("secondary_dex", 0L, 30_000L); - - Map metrics = artifacts.getInstallMetrics(T0 + 30_000L); + artifacts.recordFileArrival("apk", T0); - assertEquals("30.000", metrics.get("critical_path_s")); - assertEquals("0.000", metrics.get("potential_saving_s")); + assertFalse(artifacts.allArtifactsArrived()); } } diff --git a/prelude/toolchains/android/test/com/facebook/buck/installer/android/AndroidInstallTest.java b/prelude/toolchains/android/test/com/facebook/buck/installer/android/AndroidInstallTest.java new file mode 100644 index 0000000..a63a4a7 --- /dev/null +++ b/prelude/toolchains/android/test/com/facebook/buck/installer/android/AndroidInstallTest.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is dual-licensed under either the MIT license found in the + * LICENSE-MIT file in the root directory of this source tree or the Apache + * License, Version 2.0 found in the LICENSE-APACHE file in the root directory + * of this source tree. You may select, at your option, one of the + * above-listed licenses. + */ + +package com.facebook.buck.installer.android; + +import static org.junit.Assert.assertEquals; + +import java.time.Instant; +import java.time.ZoneId; +import org.junit.Test; + +public class AndroidInstallTest { + + @Test + public void completionMessageIncludesTimestamp() { + assertEquals( + "Install of basel_split_arm64_exo-native.apk finished in 5 seconds at 2026-08-27 14:05:09", + AndroidInstall.formatCompletionMessage( + "basel_split_arm64_exo-native.apk", + Instant.parse("2026-08-27T21:05:04Z"), + Instant.parse("2026-08-27T21:05:09.987654321Z"), + ZoneId.of("America/Los_Angeles"))); + } +} diff --git a/prelude/toolchains/android/test/com/facebook/buck/installer/android/BUCK b/prelude/toolchains/android/test/com/facebook/buck/installer/android/BUCK index e844461..447b89f 100644 --- a/prelude/toolchains/android/test/com/facebook/buck/installer/android/BUCK +++ b/prelude/toolchains/android/test/com/facebook/buck/installer/android/BUCK @@ -17,6 +17,7 @@ buck_kotlin_test( ), deps = [ "prelude//toolchains/android/src/com/facebook/buck/android:utils_api", + "prelude//toolchains/android/src/com/facebook/buck/core/filesystems:filesystems", "prelude//toolchains/android/src/com/facebook/buck/installer:installer_server", "prelude//toolchains/android/src/com/facebook/buck/installer/android:installer_android_lib", "prelude//toolchains/android/third-party:junit", diff --git a/prelude/toolchains/android/test/com/facebook/buck/installer/android/BuildExopackageInfoTest.java b/prelude/toolchains/android/test/com/facebook/buck/installer/android/BuildExopackageInfoTest.java new file mode 100644 index 0000000..60c3dfb --- /dev/null +++ b/prelude/toolchains/android/test/com/facebook/buck/installer/android/BuildExopackageInfoTest.java @@ -0,0 +1,130 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is dual-licensed under either the MIT license found in the + * LICENSE-MIT file in the root directory of this source tree or the Apache + * License, Version 2.0 found in the LICENSE-APACHE file in the root directory + * of this source tree. You may select, at your option, one of the + * above-listed licenses. + */ + +package com.facebook.buck.installer.android; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +import com.facebook.buck.android.exopackage.IsolatedExopackageInfo; +import com.facebook.buck.core.filesystems.AbsPath; +import java.nio.file.Paths; +import java.util.Optional; +import java.util.Set; +import org.junit.Test; + +/** + * A streamed push carries only the payloads that are complete, so assembling a subset has to name + * exactly those. Getting it wrong is invisible at runtime: the pairing checks throw, streaming + * treats that as best effort, and the install pushes everything anyway. + */ +public class BuildExopackageInfoTest { + + private static AbsPath path(String name) { + return AbsPath.of(Paths.get("/tmp").resolve(name).toAbsolutePath()); + } + + /** Every payload populated, so any subset asked for can be assembled. */ + private static AndroidArtifacts allPayloads() { + AndroidArtifacts artifacts = new AndroidArtifacts(); + artifacts.setSecondaryDexExopackageInfoDirectory(Optional.of(path("dex"))); + artifacts.setSecondaryDexExopackageInfoMetadata(Optional.of(path("dex.metadata"))); + artifacts.setNativeLibraryExopackageInfoDirectory(Optional.of(path("libs"))); + artifacts.setNativeLibraryExopackageInfoMetadata(Optional.of(path("libs.metadata"))); + artifacts.setResourcesExopackageInfoAssets(Optional.of(path("assets"))); + artifacts.setResourcesExopackageInfoAssetsHash(Optional.of(path("assets.hash"))); + artifacts.setResourcesExopackageInfoRes(Optional.of(path("res"))); + artifacts.setResourcesExopackageInfoResHash(Optional.of(path("res.hash"))); + return artifacts; + } + + @Test + public void secondaryDexAloneBringsOnlySecondaryDex() { + IsolatedExopackageInfo info = + AndroidInstallerManager.buildExopackageInfo( + allPayloads(), Set.of(AndroidArtifacts.ArtifactClass.SECONDARY_DEX)) + .get(); + + assertTrue(info.getDexInfo().isPresent()); + assertFalse(info.getNativeLibsInfo().isPresent()); + assertFalse(info.getResourcesInfo().isPresent()); + } + + @Test + public void nativeLibrariesAloneBringOnlyNativeLibraries() { + IsolatedExopackageInfo info = + AndroidInstallerManager.buildExopackageInfo( + allPayloads(), Set.of(AndroidArtifacts.ArtifactClass.NATIVE_LIBRARY)) + .get(); + + assertFalse(info.getDexInfo().isPresent()); + assertTrue(info.getNativeLibsInfo().isPresent()); + assertFalse(info.getResourcesInfo().isPresent()); + } + + @Test + public void resourcesAloneBringOnlyResources() { + IsolatedExopackageInfo info = + AndroidInstallerManager.buildExopackageInfo( + allPayloads(), Set.of(AndroidArtifacts.ArtifactClass.RESOURCES)) + .get(); + + assertFalse(info.getDexInfo().isPresent()); + assertFalse(info.getNativeLibsInfo().isPresent()); + assertTrue(info.getResourcesInfo().isPresent()); + assertEquals(2, info.getResourcesInfo().get().getResourcesPaths().size()); + } + + /** The install asks for all of them, and gets all of them. */ + @Test + public void everyPayloadTogetherBringsAllThree() { + IsolatedExopackageInfo info = + AndroidInstallerManager.buildExopackageInfo( + allPayloads(), AndroidArtifacts.ArtifactClass.EXOPACKAGE_PAYLOADS) + .get(); + + assertTrue(info.getDexInfo().isPresent()); + assertTrue(info.getNativeLibsInfo().isPresent()); + assertTrue(info.getResourcesInfo().isPresent()); + } + + /** Nothing complete yet means nothing to send, rather than an empty push. */ + @Test + public void noPayloadsAtAllIsAbsentRatherThanEmpty() { + assertFalse(AndroidInstallerManager.buildExopackageInfo(allPayloads(), Set.of()).isPresent()); + } + + /** A build with no exopackage payloads has nothing to assemble however much is asked for. */ + @Test + public void aBuildWithoutPayloadsIsAbsent() { + assertFalse( + AndroidInstallerManager.buildExopackageInfo( + new AndroidArtifacts(), AndroidArtifacts.ArtifactClass.EXOPACKAGE_PAYLOADS) + .isPresent()); + } + + /** + * A payload named while only half delivered is a bug in the readiness check, not something to + * paper over: assembling it would push resources the metadata does not describe. + */ + @Test + public void aHalfDeliveredPayloadIsRejected() { + AndroidArtifacts artifacts = allPayloads(); + artifacts.setResourcesExopackageInfoResHash(Optional.empty()); + + assertThrows( + IllegalStateException.class, + () -> + AndroidInstallerManager.buildExopackageInfo( + artifacts, Set.of(AndroidArtifacts.ArtifactClass.RESOURCES))); + } +} diff --git a/prelude/toolchains/android/test/com/facebook/buck/installer/android/InstallMetricsTest.java b/prelude/toolchains/android/test/com/facebook/buck/installer/android/InstallMetricsTest.java new file mode 100644 index 0000000..1500096 --- /dev/null +++ b/prelude/toolchains/android/test/com/facebook/buck/installer/android/InstallMetricsTest.java @@ -0,0 +1,314 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is dual-licensed under either the MIT license found in the + * LICENSE-MIT file in the root directory of this source tree or the Apache + * License, Version 2.0 found in the LICENSE-APACHE file in the root directory + * of this source tree. You may select, at your option, one of the + * above-listed licenses. + */ + +package com.facebook.buck.installer.android; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import com.google.common.collect.ImmutableMap; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import org.junit.Test; + +public class InstallMetricsTest { + + private static final long T0 = 1_700_000_000_000L; + + @Test + public void noArrivalsProducesNoMetrics() { + assertTrue(new InstallMetrics().summarise(T0, ImmutableMap.of()).isEmpty()); + } + + /** Every class reports both halves of its story, and classes read in the order they landed. */ + @Test + public void eachClassReportsWhenItLandedAndHowLongItTookToTransfer() { + AndroidArtifacts artifacts = new AndroidArtifacts(); + InstallMetrics timings = new InstallMetrics(); + artifacts.recordFileArrival("options", T0); + artifacts.recordFileArrival("resources_exopackage_res", T0 + 2_000L); + artifacts.recordFileArrival("native_library_exopackage_info_directory", T0 + 1_000L); + artifacts.recordFileArrival("secondary_dex_exopackage_info_metadata", T0 + 4_000L); + // Any name the installer does not recognise is the apk. + artifacts.recordFileArrival("fbandroid_arm64_exo-native", T0 + 3_000L); + + timings.recordPush("native_library", 0L, 8_000L); + timings.recordPush("resources", 8_000L, 9_500L); + timings.recordPush("secondary_dex", 9_500L, 12_000L); + timings.recordApkInstall(12_000L, 13_250L); + + Map metrics = timings.summarise(T0 + 20_000L, artifacts.arrivals()); + + assertEquals( + List.of( + "control_arrival_s", + "control_transfer_s", + "native_arrival_s", + "native_transfer_s", + "resources_arrival_s", + "resources_transfer_s", + "apk_arrival_s", + "apk_transfer_s", + "dex_arrival_s", + "dex_transfer_s"), + new ArrayList<>(metrics.keySet()).subList(0, 10)); + assertEquals("1.000", metrics.get("native_arrival_s")); + assertEquals("8.000", metrics.get("native_transfer_s")); + assertEquals("2.000", metrics.get("resources_arrival_s")); + assertEquals("1.500", metrics.get("resources_transfer_s")); + assertEquals("4.000", metrics.get("dex_arrival_s")); + assertEquals("2.500", metrics.get("dex_transfer_s")); + assertEquals("3.000", metrics.get("apk_arrival_s")); + assertEquals("1.250", metrics.get("apk_transfer_s")); + // Control artifacts are read on the host. + assertEquals("0.000", metrics.get("control_transfer_s")); + } + + /** A class is only ready once its last artifact lands. */ + @Test + public void classArrivalUsesTheLatestArtifactInThatClass() { + AndroidArtifacts artifacts = new AndroidArtifacts(); + InstallMetrics timings = new InstallMetrics(); + artifacts.recordFileArrival("resources_exopackage_res", T0); + artifacts.recordFileArrival("resources_exopackage_res_hash", T0 + 500L); + artifacts.recordFileArrival("resources_exopackage_assets", T0 + 7_000L); + artifacts.recordFileArrival("resources_exopackage_assets_hash", T0 + 1_000L); + + assertEquals( + "7.000", timings.summarise(T0 + 9_000L, artifacts.arrivals()).get("resources_arrival_s")); + } + + /** Retries must not move an artifact's recorded arrival later. */ + @Test + public void firstArrivalWinsForARepeatedArtifact() { + AndroidArtifacts artifacts = new AndroidArtifacts(); + InstallMetrics timings = new InstallMetrics(); + artifacts.recordFileArrival("manifest", T0); + artifacts.recordFileArrival("manifest", T0 + 6_000L); + + assertEquals( + "0.000", timings.summarise(T0 + 9_000L, artifacts.arrivals()).get("control_arrival_s")); + } + + /** Metadata is derived from the payloads, so it belongs to no class of its own. */ + @Test + public void metadataIsReportedSeparatelyFromTheArtifactClasses() { + AndroidArtifacts artifacts = new AndroidArtifacts(); + InstallMetrics timings = new InstallMetrics(); + artifacts.recordFileArrival("options", T0); + artifacts.recordFileArrival("my_app", T0); + timings.recordPush("native_library", 0L, 5_000L); + timings.recordPush("metadata", 5_000L, 5_400L); + + Map metrics = timings.summarise(T0 + 6_000L, artifacts.arrivals()); + + assertEquals("0.400", metrics.get("metadata_transfer_s")); + assertFalse(metrics.containsKey("metadata_arrival_s")); + } + + @Test + public void anInstallWithoutMetadataDoesNotReportIt() { + AndroidArtifacts artifacts = new AndroidArtifacts(); + InstallMetrics timings = new InstallMetrics(); + artifacts.recordFileArrival("my_app", T0); + timings.recordApkInstall(0L, 3_000L); + + assertFalse( + timings.summarise(T0 + 4_000L, artifacts.arrivals()).containsKey("metadata_transfer_s")); + } + + /** + * The trace this work started from: native libs and resources are ready immediately, dex and the + * apk only 37s later, and payloads are pushed one at a time after everything has landed. + */ + @Test + public void criticalPathReplaysTheInstallWithoutTheWait() { + AndroidArtifacts artifacts = new AndroidArtifacts(); + InstallMetrics timings = new InstallMetrics(); + for (String control : new String[] {"options", "manifest", "cpu_filters"}) { + artifacts.recordFileArrival(control, T0); + } + for (String resource : + new String[] { + "resources_exopackage_assets", "resources_exopackage_assets_hash", + "resources_exopackage_res", "resources_exopackage_res_hash" + }) { + artifacts.recordFileArrival(resource, T0); + } + artifacts.recordFileArrival("native_library_exopackage_info_metadata", T0); + artifacts.recordFileArrival("native_library_exopackage_info_directory", T0 + 1_000L); + artifacts.recordFileArrival("secondary_dex_exopackage_info_directory", T0 + 36_000L); + artifacts.recordFileArrival("secondary_dex_exopackage_info_metadata", T0 + 37_000L); + artifacts.recordFileArrival("my_app", T0 + 37_000L); + + // Pushed one payload at a time, so the windows do not overlap. + timings.recordDeviceSetup(0L, 1_000L); + timings.recordPush("secondary_dex", 1_000L, 28_019L); + timings.recordPush("native_library", 28_019L, 71_259L); + timings.recordPush("resources", 71_259L, 89_741L); + timings.recordPush("metadata", 89_741L, 90_055L); + timings.recordApkInstall(90_055L, 102_191L); + + Map metrics = timings.summarise(T0 + 140_000L, artifacts.arrivals()); + + // 1s setup, then resources (available at 0), native (1s), dex (37s), metadata, then the apk. + assertEquals("1.000", metrics.get("device_setup_s")); + assertEquals("102.191", metrics.get("critical_path_s")); + assertEquals("140.000", metrics.get("total_s")); + assertEquals("37.809", metrics.get("potential_saving_s")); + } + + /** + * Payloads arriving apart is the case a per-payload cap gets wrong: resources is pushable from + * the start even though native does not turn up until much later. + */ + @Test + public void staggeredArrivalsAreCreditedIndependently() { + AndroidArtifacts artifacts = new AndroidArtifacts(); + InstallMetrics timings = new InstallMetrics(); + artifacts.recordFileArrival("options", T0); + artifacts.recordFileArrival("manifest", T0); + artifacts.recordFileArrival("resources_exopackage_res", T0); + artifacts.recordFileArrival("resources_exopackage_res_hash", T0); + artifacts.recordFileArrival("native_library_exopackage_info_directory", T0 + 50_000L); + artifacts.recordFileArrival("native_library_exopackage_info_metadata", T0 + 50_000L); + artifacts.recordFileArrival("secondary_dex_exopackage_info_directory", T0 + 60_000L); + artifacts.recordFileArrival("my_app", T0 + 60_000L); + + timings.recordPush("resources", 0L, 10_000L); + timings.recordPush("native_library", 10_000L, 20_000L); + timings.recordPush("secondary_dex", 20_000L, 40_000L); + timings.recordApkInstall(40_000L, 50_000L); + + Map metrics = timings.summarise(T0 + 110_000L, artifacts.arrivals()); + + // resources 0-10s, idle until native at 50s, native 50-60s, dex 60-80s, apk 80-90s. + assertEquals("90.000", metrics.get("critical_path_s")); + assertEquals("20.000", metrics.get("potential_saving_s")); + } + + @Test + public void anInstallThatNeverWaitedHasNothingToSave() { + AndroidArtifacts artifacts = new AndroidArtifacts(); + InstallMetrics timings = new InstallMetrics(); + artifacts.recordFileArrival("options", T0); + artifacts.recordFileArrival("manifest", T0); + artifacts.recordFileArrival("my_app", T0 + 5_000L); + timings.recordApkInstall(0L, 3_000L); + + Map metrics = timings.summarise(T0 + 8_000L, artifacts.arrivals()); + + assertEquals("8.000", metrics.get("critical_path_s")); + assertEquals("0.000", metrics.get("potential_saving_s")); + } + + /** Shards of one group overlapping is one payload transferring, not two. */ + @Test + public void overlappingShardsOfOneGroupCountOnce() { + AndroidArtifacts artifacts = new AndroidArtifacts(); + InstallMetrics timings = new InstallMetrics(); + artifacts.recordFileArrival("options", T0); + artifacts.recordFileArrival("native_library_exopackage_info_directory", T0); + timings.recordPush("native_library", 1_000L, 9_000L); + timings.recordPush("native_library", 2_000L, 6_000L); + + assertEquals( + "8.000", timings.summarise(T0 + 10_000L, artifacts.arrivals()).get("native_transfer_s")); + } + + /** + * Shards are recorded when they finish, so a long one started first is recorded last. Coalescing + * has to order them itself; taking them as given would drop everything before the first record. + */ + @Test + public void shardsRecordedOutOfOrderStillCoalesce() { + AndroidArtifacts artifacts = new AndroidArtifacts(); + InstallMetrics timings = new InstallMetrics(); + artifacts.recordFileArrival("options", T0); + artifacts.recordFileArrival("native_library_exopackage_info_directory", T0); + // The short shard finishes first, so it is recorded before the long one that started earlier. + timings.recordPush("native_library", 1_000L, 5_000L); + timings.recordPush("native_library", 0L, 30_000L); + + assertEquals( + "30.000", timings.summarise(T0 + 40_000L, artifacts.arrivals()).get("native_transfer_s")); + } + + /** + * Shards queue behind other groups, so a group can be idle between its own shards. That gap is + * not transfer time -- the span from first start to last end would charge the group for it. + */ + @Test + public void aGapBetweenShardsOfOneGroupIsNotTransferTime() { + AndroidArtifacts artifacts = new AndroidArtifacts(); + InstallMetrics timings = new InstallMetrics(); + artifacts.recordFileArrival("options", T0); + artifacts.recordFileArrival("native_library_exopackage_info_directory", T0); + timings.recordPush("native_library", 0L, 10_000L); + // Queued behind another group for ten seconds, then transferring again. + timings.recordPush("native_library", 20_000L, 30_000L); + + assertEquals( + "20.000", timings.summarise(T0 + 40_000L, artifacts.arrivals()).get("native_transfer_s")); + } + + /** + * Once payloads are pushed concurrently, replaying them one at a time would total more than the + * install actually took and report a negative saving. The replay uses however many pushers the + * install itself used. + */ + @Test + public void concurrentPushesAreReplayedConcurrently() { + AndroidArtifacts artifacts = new AndroidArtifacts(); + InstallMetrics timings = new InstallMetrics(); + artifacts.recordFileArrival("options", T0); + artifacts.recordFileArrival("manifest", T0); + artifacts.recordFileArrival("native_library_exopackage_info_directory", T0); + artifacts.recordFileArrival("native_library_exopackage_info_metadata", T0); + artifacts.recordFileArrival("secondary_dex_exopackage_info_directory", T0); + artifacts.recordFileArrival("my_app", T0); + + // Both payloads in flight over the same window. + timings.recordPush("native_library", 0L, 30_000L); + timings.recordPush("secondary_dex", 0L, 30_000L); + + Map metrics = timings.summarise(T0 + 30_000L, artifacts.arrivals()); + + assertEquals("30.000", metrics.get("critical_path_s")); + assertEquals("0.000", metrics.get("potential_saving_s")); + } + + /** + * A streamed push finished before the install's device phase began, so it is not part of the + * device work already accounted for. Counting it would cancel out the correction that charges + * whatever the replay does not model, and every streaming install would look free. + */ + @Test + public void aPushMadeBeforeTheDevicePhaseIsNotCountedAgainstDeviceWork() { + AndroidArtifacts artifacts = new AndroidArtifacts(); + InstallMetrics timings = new InstallMetrics(); + artifacts.recordFileArrival("resources_exopackage_res", T0); + artifacts.recordFileArrival("apk", T0); + + // Streamed while the build ran, then the install's own device phase. + timings.recordPush("resources", T0, T0 + 10_000L); + timings.recordDeviceWork(T0 + 20_000L, T0 + 40_000L); + timings.recordPush("secondary_dex", T0 + 22_000L, T0 + 27_000L); + timings.recordApkInstall(T0 + 30_000L, T0 + 35_000L); + + Map metrics = timings.summarise(T0 + 40_000L, artifacts.arrivals()); + + // modelled counts the apk install and the in-phase push only: 5s + 5s. The other 10s of device + // work is unmodelled and must still be charged, so it survives on the critical path. + assertEquals("30.000", metrics.get("critical_path_s")); + } +} diff --git a/prelude/toolchains/android/test/com/facebook/buck/installer/android/StreamedPushesTest.java b/prelude/toolchains/android/test/com/facebook/buck/installer/android/StreamedPushesTest.java new file mode 100644 index 0000000..6b47370 --- /dev/null +++ b/prelude/toolchains/android/test/com/facebook/buck/installer/android/StreamedPushesTest.java @@ -0,0 +1,111 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is dual-licensed under either the MIT license found in the + * LICENSE-MIT file in the root directory of this source tree or the Apache + * License, Version 2.0 found in the LICENSE-APACHE file in the root directory + * of this source tree. You may select, at your option, one of the + * above-listed licenses. + */ + +package com.facebook.buck.installer.android; + +import static com.facebook.buck.installer.android.AndroidArtifacts.ArtifactClass.NATIVE_LIBRARY; +import static com.facebook.buck.installer.android.AndroidArtifacts.ArtifactClass.RESOURCES; +import static com.facebook.buck.installer.android.AndroidArtifacts.ArtifactClass.SECONDARY_DEX; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +import java.util.EnumSet; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.Test; + +/** What stops a payload being streamed twice, or streamed after the install has started. */ +public class StreamedPushesTest { + + @Test + public void everyPushThatStartedIsRecorded() { + StreamedPushes pushes = new StreamedPushes(); + pushes.dispatch(EnumSet.of(RESOURCES), pending -> CompletableFuture.completedFuture(null)); + pushes.dispatch(EnumSet.of(NATIVE_LIBRARY), pending -> CompletableFuture.completedFuture(null)); + + assertEquals(2, pushes.pushCount()); + } + + /** Nothing is sent when everything ready has already gone. */ + @Test + public void aPayloadAlreadySentIsNotDispatchedAgain() { + StreamedPushes pushes = new StreamedPushes(); + pushes.dispatch(EnumSet.of(RESOURCES), pending -> CompletableFuture.completedFuture(null)); + pushes.dispatch(EnumSet.of(RESOURCES), pending -> CompletableFuture.completedFuture(null)); + + assertEquals(1, pushes.pushCount()); + } + + /** + * Once the install has taken the pushes over it writes the same directories, so a payload that + * only now becomes ready is left to the install rather than started alongside it. + */ + @Test + public void nothingIsDispatchedOnceSealed() throws Exception { + StreamedPushes pushes = new StreamedPushes(); + pushes.sealAndAwait(); + + AtomicBoolean pushed = new AtomicBoolean(); + pushes.dispatch( + EnumSet.of(NATIVE_LIBRARY), + pending -> { + pushed.set(true); + return CompletableFuture.completedFuture(null); + }); + + assertFalse(pushed.get()); + assertEquals(0, pushes.pushCount()); + } + + /** Only what has not gone yet: an overlapping offer sends the difference, not the whole set. */ + @Test + public void anOverlappingOfferSendsOnlyWhatIsNew() { + StreamedPushes pushes = new StreamedPushes(); + pushes.dispatch( + EnumSet.of(RESOURCES, SECONDARY_DEX), pending -> CompletableFuture.completedFuture(null)); + + AtomicReference> second = new AtomicReference<>(); + pushes.dispatch( + EnumSet.of(SECONDARY_DEX, NATIVE_LIBRARY), + pending -> { + second.set(pending); + return CompletableFuture.completedFuture(null); + }); + + assertEquals(EnumSet.of(NATIVE_LIBRARY), second.get()); + } + + /** + * A failed push is the install's problem to redo, not a reason to fail: whatever did not land is + * pushed again once the install lists the device. + */ + @Test + public void aFailedPushDoesNotFailTheInstall() throws Exception { + StreamedPushes pushes = new StreamedPushes(); + pushes.dispatch( + EnumSet.of(RESOURCES), + pending -> CompletableFuture.failedFuture(new IllegalStateException("adb died"))); + + pushes.sealAndAwait(); + } + + /** Waiting on pushes that all worked is uneventful. */ + @Test + public void awaitingSuccessfulPushesReturns() throws Exception { + StreamedPushes pushes = new StreamedPushes(); + pushes.dispatch(EnumSet.of(RESOURCES), pending -> CompletableFuture.completedFuture(null)); + + pushes.sealAndAwait(); + + assertEquals(1, pushes.pushCount()); + } +} diff --git a/prelude/toolchains/android/test/com/facebook/buck/jvm/cd/DepFileUtilsTest.kt b/prelude/toolchains/android/test/com/facebook/buck/jvm/cd/DepFileUtilsTest.kt index 4031de5..dff044d 100644 --- a/prelude/toolchains/android/test/com/facebook/buck/jvm/cd/DepFileUtilsTest.kt +++ b/prelude/toolchains/android/test/com/facebook/buck/jvm/cd/DepFileUtilsTest.kt @@ -15,7 +15,6 @@ import com.fasterxml.jackson.core.type.TypeReference import java.nio.file.Files import java.nio.file.Path import java.nio.file.Paths -import java.util.Optional import org.junit.Assert.assertEquals import org.junit.Rule import org.junit.Test @@ -39,7 +38,7 @@ class DepFileUtilsTest { DepFileUtils.usedClassesToDepFile( listOf(usedClassesMapPath), depFileOutput, - Optional.empty(), + emptyMap(), false, ) @@ -62,7 +61,7 @@ class DepFileUtilsTest { DepFileUtils.usedClassesToDepFile( listOf(prevUsedClassesMapPath), depFileOutput, - Optional.empty(), + emptyMap(), false, ) @@ -77,7 +76,7 @@ class DepFileUtilsTest { DepFileUtils.usedClassesToDepFile( listOf(usedClassesMapPath), depFileOutput, - Optional.empty(), + emptyMap(), true, ) @@ -96,17 +95,16 @@ class DepFileUtilsTest { "/path/to/B.jar" to setOf("ClassB.class"), ), ) - val jarToJarDirMapPath = tempFolder.newFile("jar-to-dir-map.txt").toPath() - Files.write( - jarToJarDirMapPath, - listOf("/path/to/C.jar /expanded/C", "/path/to/A.jar /expanded/A"), + val jarToJarDirMap = mapOf( + Paths.get("/path/to/C.jar") to Paths.get("/expanded/C"), + Paths.get("/path/to/A.jar") to Paths.get("/expanded/A"), ) val depFileOutput = tempFolder.newFile("dep-file.txt").toPath() DepFileUtils.usedClassesToDepFile( listOf(usedClassesMapPath), depFileOutput, - Optional.of(jarToJarDirMapPath), + jarToJarDirMap, false, ) diff --git a/prelude/toolchains/android/test/com/facebook/buck/jvm/cd/serialization/kotlin/BUCK b/prelude/toolchains/android/test/com/facebook/buck/jvm/cd/serialization/kotlin/BUCK index 7875da1..362f466 100644 --- a/prelude/toolchains/android/test/com/facebook/buck/jvm/cd/serialization/kotlin/BUCK +++ b/prelude/toolchains/android/test/com/facebook/buck/jvm/cd/serialization/kotlin/BUCK @@ -7,11 +7,13 @@ source_listing() standard_java_test( name = "kotlin", + env = {"JAVACD_ABSOLUTE_PATHS_ARE_RELATIVE_TO_CWD": "1"}, deps = [ "prelude//toolchains/android/src/com/facebook/buck/cd/resources/proto:cd-model", "prelude//toolchains/android/src/com/facebook/buck/cd/resources/proto:javacd-model", "prelude//toolchains/android/src/com/facebook/buck/cd/resources/proto:kotlincd-model", "prelude//toolchains/android/src/com/facebook/buck/core/filesystems:filesystems", + "prelude//toolchains/android/src/com/facebook/buck/jvm/cd/command:command", "prelude//toolchains/android/src/com/facebook/buck/jvm/cd/command/kotlin:language-version", "prelude//toolchains/android/src/com/facebook/buck/jvm/cd/command/kotlin:support", "prelude//toolchains/android/src/com/facebook/buck/jvm/cd/serialization/java:java", diff --git a/prelude/toolchains/android/test/com/facebook/buck/jvm/cd/serialization/kotlin/CompilerCommandDefaultsTest.java b/prelude/toolchains/android/test/com/facebook/buck/jvm/cd/serialization/kotlin/CompilerCommandDefaultsTest.java new file mode 100644 index 0000000..30fb43f --- /dev/null +++ b/prelude/toolchains/android/test/com/facebook/buck/jvm/cd/serialization/kotlin/CompilerCommandDefaultsTest.java @@ -0,0 +1,227 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is dual-licensed under either the MIT license found in the + * LICENSE-MIT file in the root directory of this source tree or the Apache + * License, Version 2.0 found in the LICENSE-APACHE file in the root directory + * of this source tree. You may select, at your option, one of the + * above-listed licenses. + */ + +package com.facebook.buck.jvm.cd.serialization.kotlin; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import com.facebook.buck.cd.model.java.AbiGenerationMode; +import com.facebook.buck.cd.model.java.ResolvedJavacOptions; +import com.facebook.buck.core.filesystems.RelPath; +import com.facebook.buck.jvm.cd.command.BaseJarCommand; +import com.facebook.buck.jvm.cd.serialization.java.JarParametersSerializer; +import com.facebook.buck.jvm.cd.serialization.java.ResolvedJavacOptionsSerializer; +import com.facebook.buck.jvm.java.Jsr199Javac.ResolvedJsr199Javac; +import java.nio.file.Paths; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.logging.Level; +import org.junit.Test; + +public class CompilerCommandDefaultsTest { + @Test + public void baseCommandUsesCompilerDefaults() { + var model = + com.facebook.buck.cd.model.java.BaseJarCommand.newBuilder() + .setAbiGenerationMode(AbiGenerationMode.SOURCE_ONLY) + .setTrackClassUsage(true) + .setResolvedJavacOptions(javaOptions()) + .addCompileTimeClasspathPaths("dep.jar") + .build(); + + var command = BaseJarCommand.Companion.fromProto(model, Optional.of(RelPath.get("scratch"))); + + assertEquals(AbiGenerationMode.CLASS, command.getAbiCompatibilityMode()); + assertEquals(AbiGenerationMode.SOURCE_ONLY, command.getAbiGenerationMode()); + assertTrue(command.getTrackClassUsage()); + assertEquals(RelPath.get("dep.jar"), command.getCompileTimeClasspathPaths().get(0)); + assertTrue(command.getJarToJarDirMap().isEmpty()); + assertEquals(RelPath.get("buck-out/v2"), command.getBuckOut()); + assertTrue(command.getResolvedJavac() instanceof ResolvedJsr199Javac); + assertTrue(command.getResolvedJavacOptions().getDebug()); + } + + @Test + public void abiDirectoriesPreserveClasspathOrderAndDuplicateJars() { + var model = + com.facebook.buck.cd.model.java.BaseJarCommand.newBuilder() + .setResolvedJavacOptions(javaOptions()) + .addAllCompileTimeClasspathAbiAndDirPaths( + List.of("b jar.jar", "b dir", "a.jar", "", "b jar.jar", "b dir")) + .build(); + + var command = BaseJarCommand.Companion.fromProto(model, Optional.of(RelPath.get("scratch"))); + + assertEquals( + List.of(RelPath.get("b jar.jar"), RelPath.get("a.jar"), RelPath.get("b jar.jar")), + command.getCompileTimeClasspathPaths()); + assertEquals(Map.of(Paths.get("b jar.jar"), Paths.get("b dir")), command.getJarToJarDirMap()); + } + + @Test(expected = IllegalArgumentException.class) + public void abiDirectoriesRejectIncompletePairs() { + var model = + com.facebook.buck.cd.model.java.BaseJarCommand.newBuilder() + .setResolvedJavacOptions(javaOptions()) + .addCompileTimeClasspathAbiAndDirPaths("dep.jar") + .build(); + BaseJarCommand.Companion.fromProto(model, Optional.empty()); + } + + @Test(expected = IllegalArgumentException.class) + public void abiDirectoriesRejectConflictingDirectories() { + var model = + com.facebook.buck.cd.model.java.BaseJarCommand.newBuilder() + .setResolvedJavacOptions(javaOptions()) + .addAllCompileTimeClasspathAbiAndDirPaths( + List.of("dep.jar", "first-dir", "dep.jar", "second-dir")) + .build(); + BaseJarCommand.Companion.fromProto(model, Optional.empty()); + } + + @Test(expected = IllegalArgumentException.class) + public void abiDirectoriesRejectTwoClasspaths() { + var model = + com.facebook.buck.cd.model.java.BaseJarCommand.newBuilder() + .setResolvedJavacOptions(javaOptions()) + .addCompileTimeClasspathPaths("dep.jar") + .addAllCompileTimeClasspathAbiAndDirPaths(List.of("dep.jar", "dep-dir")) + .build(); + BaseJarCommand.Companion.fromProto(model, Optional.empty()); + } + + @Test + public void jarParametersUseFineLogging() { + var model = + com.facebook.buck.cd.model.java.JarParameters.newBuilder() + .setJarPath("out.jar") + .addEntriesToJar("classes") + .build(); + + var parameters = JarParametersSerializer.deserialize(model); + + assertEquals(Level.FINE, parameters.getDuplicatesLogLevel()); + assertEquals(RelPath.get("out.jar"), parameters.getJarPath()); + assertTrue(parameters.getEntriesToJar().contains(RelPath.get("classes"))); + } + + @Test + public void annotationProcessorsKeepVariableFlagsWithoutPathParameters() { + var model = + javaOptions().toBuilder() + .setJavaAnnotationProcessorParams( + ResolvedJavacOptions.JavacPluginParams.newBuilder() + .addPluginProperties( + ResolvedJavacOptions.ResolvedJavacPluginProperties.newBuilder() + .setCanReuseClassLoader(true) + .setDoesNotAffectAbi(true) + .setSupportsAbiGenerationFromSource(true) + .setRunsOnJavaOnly(true) + .addProcessorNames("Processor") + .addClasspath("processor.jar"))) + .build(); + + var processor = + ResolvedJavacOptionsSerializer.deserialize(model) + .getJavaAnnotationProcessorParams() + .getPluginProperties() + .get(0); + + assertTrue(processor.getCanReuseClassLoader()); + assertTrue(processor.getDoesNotAffectAbi()); + assertTrue(processor.getSupportAbiGenerationFromSource()); + assertTrue(processor.getRunsOnJavaOnly()); + assertTrue(processor.getPathParams().isEmpty()); + assertEquals(RelPath.get("processor.jar"), processor.getClasspath().get(0)); + } + + @Test + public void kotlinParametersUseExactAbiTerminationMessage() { + var model = + com.facebook.buck.cd.model.kotlin.KotlinExtraParams.newBuilder() + .setStandardLibraryClassPath("stdlib.jar") + .setAnnotationProcessingClassPath("kapt.jar") + .setLanguageVersion("2.2") + .build(); + + var parameters = + KotlinExtraParamsSerializer.deserialize( + javaOptions(), model, Optional.of(RelPath.get("scratch"))); + + assertEquals( + Optional.of( + "exception: java.lang.RuntimeException: Terminating compilation. We're done with ABI."), + parameters.getKosabiJvmAbiGenEarlyTerminationMessagePrefix()); + assertTrue(parameters.getResolvedJavacOptions().getDebug()); + } + + private static ResolvedJavacOptions javaOptions() { + return ResolvedJavacOptions.newBuilder() + .setLanguageLevelOptions( + ResolvedJavacOptions.JavacLanguageLevelOptions.newBuilder() + .setSourceLevel("8") + .setTargetLevel("8")) + .build(); + } + + @Test + public void classesUseScratchForResourcesAndJarEntries() { + for (var type : + List.of( + com.facebook.buck.cd.model.java.BuildTargetValue.Type.LIBRARY, + com.facebook.buck.cd.model.java.BuildTargetValue.Type.SOURCE_ABI, + com.facebook.buck.cd.model.java.BuildTargetValue.Type.SOURCE_ONLY_ABI)) { + var model = + com.facebook.buck.cd.model.java.BaseJarCommand.newBuilder() + .setBuildTargetValue( + com.facebook.buck.cd.model.java.BuildTargetValue.newBuilder().setType(type)) + .setResolvedJavacOptions(javaOptions()) + .setJarParameters( + com.facebook.buck.cd.model.java.JarParameters.newBuilder().setJarPath("out.jar")) + .addResourcesMap( + com.facebook.buck.cd.model.common.RelPathMapEntry.newBuilder() + .setKey("input/resource.txt") + .setValue("pkg/resource.txt")) + .build(); + var command = BaseJarCommand.Companion.fromProto(model, Optional.of(RelPath.get("scratch"))); + var classes = RelPath.get("scratch/__classes__"); + assertEquals(classes, command.getCompilerOutputPathsValue().getByType(type).getClassesDir()); + assertEquals(List.of(classes), List.copyOf(command.getJarParameters().getEntriesToJar())); + assertEquals( + Map.of( + RelPath.get("input/resource.txt"), + RelPath.get("scratch/__classes__/pkg/resource.txt")), + command.getResourcesMap()); + } + } + + @Test + public void kotlinClassesUseScratchOrPersistentIncrementalState() { + for (int mode = 0; mode < 3; mode++) { + var model = + com.facebook.buck.cd.model.kotlin.KotlinExtraParams.newBuilder() + .setStandardLibraryClassPath("stdlib.jar") + .setAnnotationProcessingClassPath("kapt.jar") + .setLanguageVersion("2.2") + .setIncrementalStateDir("state") + .setShouldKotlincRunIncrementally(mode == 1) + .setShouldKsp2RunIncrementally(mode == 2) + .build(); + var parameters = + KotlinExtraParamsSerializer.deserialize( + javaOptions(), model, Optional.of(RelPath.get("scratch"))); + assertEquals( + RelPath.get((mode == 0 ? "scratch" : "state") + "/__kotlin_classes__").toAbsolutePath(), + parameters.getKotlinClassesDir()); + } + } +} diff --git a/prelude/toolchains/android/test/com/facebook/buck/jvm/java/JavacStepTest.java b/prelude/toolchains/android/test/com/facebook/buck/jvm/java/JavacStepTest.java index 0f61474..e87abfe 100644 --- a/prelude/toolchains/android/test/com/facebook/buck/jvm/java/JavacStepTest.java +++ b/prelude/toolchains/android/test/com/facebook/buck/jvm/java/JavacStepTest.java @@ -29,9 +29,13 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSortedSet; import java.io.File; +import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Paths; import java.util.List; import java.util.Optional; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; import org.hamcrest.Matchers; import org.junit.Before; import org.junit.Rule; @@ -77,6 +81,7 @@ public void successfulCompileDoesNotSendStdoutAndStderrToConsole() throws Except configuredBuckOut, getCompilerOutputPathsValue(), compilerParameters, + /* skipIfNoCompilationUnits */ false, null, null, false); @@ -101,6 +106,7 @@ public void failedCompileSendsStdoutAndStderrToConsole() throws Exception { configuredBuckOut, getCompilerOutputPathsValue(), compilerParameters, + /* skipIfNoCompilationUnits */ false, null, null, false); @@ -131,6 +137,7 @@ public void existingBootclasspathDirSucceeds() throws Exception { configuredBuckOut, getCompilerOutputPathsValue(), compilerParameters, + /* skipIfNoCompilationUnits */ false, null, null, false); @@ -155,6 +162,7 @@ public void bootclasspathResolvedToAbsolutePath() { configuredBuckOut, getCompilerOutputPathsValue(), compilerParameters, + /* skipIfNoCompilationUnits */ false, null, null, false); @@ -176,6 +184,68 @@ public void bootclasspathResolvedToAbsolutePath() { } } + @Test + public void sourceZipWithoutJavaEntriesSkipsCompilation() throws Exception { + JavacStep step = skippableJavacStep(writeSourceZip("com/facebook/resource.txt")); + + StepExecutionResult result = + step.executeIsolatedStep(TestExecutionContext.newInstance(tmp.getRoot())); + + // FakeJavac would have reported exit code 3 had javac been invoked. + assertThat(result, equalTo(StepExecutionResults.SUCCESS)); + } + + @Test + public void sourceZipWithJavaEntriesRunsCompilation() throws Exception { + JavacStep step = skippableJavacStep(writeSourceZip("com/facebook/Dummy.java")); + + StepExecutionResult result = + step.executeIsolatedStep(TestExecutionContext.newInstance(tmp.getRoot())); + + assertThat( + result, + equalTo( + new StepExecutionResult( + StepExecutionResults.ERROR_EXIT_CODE, Optional.of("javac stderr\n")))); + } + + private RelPath writeSourceZip(String... entryNames) throws IOException { + RelPath zipPath = RelPath.get("generated.src.zip"); + try (ZipOutputStream out = + new ZipOutputStream(Files.newOutputStream(tmp.getRoot().resolve(zipPath).getPath()))) { + for (String entryName : entryNames) { + out.putNextEntry(new ZipEntry(entryName)); + out.closeEntry(); + } + } + return zipPath; + } + + private JavacStep skippableJavacStep(RelPath sourceFilePath) { + CompilerParameters parameters = + new CompilerParameters( + ImmutableSortedSet.orderedBy(RelPath.comparator()).add(sourceFilePath).build(), + ImmutableList.of(), + ImmutableList.of(), + getCompilerOutputPaths(), + AbiGenerationMode.CLASS, + AbiGenerationMode.CLASS, + false, + null); + + return new JavacStep( + new FakeJavac(3, "javac stderr\n"), + getResolvedJavacOptions(), + buildTargetValue, + configuredBuckOut, + getCompilerOutputPathsValue(), + parameters, + /* skipIfNoCompilationUnits */ true, + null, + null, + false); + } + private static ResolvedJavacOptions getResolvedJavacOptions() { return getResolvedJavacOptions(ImmutableList.of()); } diff --git a/prelude/toolchains/android/test/com/facebook/buck/jvm/java/abi/StubJarTest.java b/prelude/toolchains/android/test/com/facebook/buck/jvm/java/abi/StubJarTest.java index 623ef88..03eb620 100644 --- a/prelude/toolchains/android/test/com/facebook/buck/jvm/java/abi/StubJarTest.java +++ b/prelude/toolchains/android/test/com/facebook/buck/jvm/java/abi/StubJarTest.java @@ -12,7 +12,6 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.fail; import com.facebook.buck.cd.model.java.AbiGenerationMode; import com.facebook.buck.core.filesystems.AbsPath; @@ -97,11 +96,33 @@ public static Object[][] getParameters() { } public boolean isKotlin22() { - return EnvVariablesProvider.getSystemEnv().get("KOTLIN_VERSION").equals("2.2.0"); + return isKotlinVersion("2.2"); + } + + public boolean isKotlin23() { + return isKotlinVersion("2.3"); } public boolean isKotlin20() { - return EnvVariablesProvider.getSystemEnv().get("KOTLIN_VERSION").equals("2.0.20"); + return isKotlinVersion("2.0"); + } + + private boolean isKotlinVersion(String majorMinor) { + return EnvVariablesProvider.getSystemEnv().get("KOTLIN_VERSION").startsWith(majorMinor + "."); + } + + private String metadataForKotlinVersion(String metadata22, String metadata20) { + if (isKotlin23()) { + String kotlin22Version = "mv={2, 2, 0}"; + if (!metadata22.contains(kotlin22Version)) { + throw new IllegalArgumentException("Missing Kotlin 2.2 metadata version: " + metadata22); + } + return metadata22.replace(kotlin22Version, "mv={2, 3, 0}"); + } + if (isKotlin22()) { + return metadata22; + } + return metadata20; } private static final ImmutableSortedSet EMPTY_CLASSPATH = ImmutableSortedSet.of(); @@ -169,7 +190,7 @@ public void emptyKotlinClass() throws IOException { "", " // compiled from: A.kt", "", - isKotlin22() ? metadata22 : metadata20, + metadataForKotlinVersion(metadata22, metadata20), "", " // access flags 0x1", " public ()V", @@ -300,7 +321,7 @@ public void kotlinClassWithInlineMethod() throws IOException { "", " // compiled from: A.kt", "", - isKotlin22() ? metadata22 : metadata20, + metadataForKotlinVersion(metadata22, metadata20), "", " // access flags 0x1", " public ()V", @@ -384,7 +405,7 @@ public void kotlinClassWithInlineMethodThatUsesInternalMethodWithPublishedApi() "", " // compiled from: A.kt", "", - isKotlin22() ? metadata22 : metadata20, + metadataForKotlinVersion(metadata22, metadata20), "", " // access flags 0x1", " public ()V", @@ -469,7 +490,7 @@ public void kotlinClassWithInlineMethodAndJvmName() throws IOException { "", " // compiled from: A.kt", "", - isKotlin22() ? metadata22 : metadata20, + metadataForKotlinVersion(metadata22, metadata20), "", " // access flags 0x1", " public ()V", @@ -548,7 +569,7 @@ public void kotlinClassWithInlineMethodUsingDefaultParam() throws IOException { "", " // compiled from: A.kt", "", - isKotlin22() ? metadata22 : metadata20, + metadataForKotlinVersion(metadata22, metadata20), "", " // access flags 0x1", " public ()V", @@ -671,7 +692,7 @@ public void kotlinInlineFunCreatingCapturingLambda() throws IOException { "", " OUTERCLASS com/example/buck/AKt test (I)Lkotlin/jvm/functions/Function0;", "", - isKotlin22() ? metadata22 : metadata20, + metadataForKotlinVersion(metadata22, metadata20), " // access flags 0x19", " public final static INNERCLASS com/example/buck/AKt$test$1 null null", "", @@ -788,10 +809,13 @@ public void kotlinInlineFunCreatingSuspendLambda() throws IOException { "", " OUTERCLASS com/example/buck/AKt test ()Lkotlin/jvm/functions/Function2;", "", - " @Lkotlin/coroutines/jvm/internal/DebugMetadata;(f=\"A.kt\", l={}, i={}, s={}, n={}," - + " m=\"invokeSuspend\", c=\"com.example.buck.AKt$test$1\")", + isKotlin23() + ? " @Lkotlin/coroutines/jvm/internal/DebugMetadata;(f=\"A.kt\", l={}, nl={}, i={}," + + " s={}, n={}, m=\"invokeSuspend\", c=\"com.example.buck.AKt$test$1\", v=2)" + : " @Lkotlin/coroutines/jvm/internal/DebugMetadata;(f=\"A.kt\", l={}, i={}, s={}," + + " n={}, m=\"invokeSuspend\", c=\"com.example.buck.AKt$test$1\")", "", - isKotlin22() ? metadata22 : metadata20, + metadataForKotlinVersion(metadata22, metadata20), " // access flags 0x19", " public final static INNERCLASS com/example/buck/AKt$test$1 null null", "", @@ -984,7 +1008,7 @@ public void kotlinInlineFunCreatingNestedLambdas() throws IOException { "", " OUTERCLASS com/example/buck/AKt test (I)Lkotlin/jvm/functions/Function1;", "", - isKotlin22() ? metadata22 : metadata20, + metadataForKotlinVersion(metadata22, metadata20), " // access flags 0x19", " public final static INNERCLASS com/example/buck/AKt$test$1 null null", " // access flags 0x19", @@ -1062,7 +1086,7 @@ public void kotlinInlineFunCreatingNestedLambdas() throws IOException { " // compiled from: A.kt", " OUTERCLASS com/example/buck/AKt$test$1 invoke (I)Lkotlin/jvm/functions/Function1;", "", - isKotlin22() ? metadata22 : metadata20, + metadataForKotlinVersion(metadata22, metadata20), " // access flags 0x19", " public final static INNERCLASS com/example/buck/AKt$test$1 null null", " // access flags 0x19", @@ -1180,7 +1204,7 @@ public void kotlinClassNoInlineMethod() throws IOException { "", " // compiled from: A.kt", "", - isKotlin22() ? metadata22 : metadata20, + metadataForKotlinVersion(metadata22, metadata20), "", " // access flags 0x1", " public ()V", @@ -1250,7 +1274,7 @@ public void kotlinClassWithInlineProperty() throws IOException { "", " // compiled from: A.kt", "", - isKotlin22() ? metadata22 : metadata20, + metadataForKotlinVersion(metadata22, metadata20), "", " // access flags 0x1", " public ()V", @@ -1385,7 +1409,7 @@ public void kotlinClassWithInlineExtensionMethodUsingLambda() throws IOException "", " // compiled from: B.kt", "", - isKotlin22() ? metadata22 : metadata20, + metadataForKotlinVersion(metadata22, metadata20), "", " // access flags 0x19", " public final static getString(Lcom/example/buck/A;)Ljava/lang/String;", @@ -1442,9 +1466,9 @@ public void kotlinClassWithInlineExtensionMethodUsingLambda() throws IOException "", " OUTERCLASS com/example/buck/BKt getString (Lcom/example/buck/A;)Ljava/lang/String;", "", - isKotlin22() - ? " @Lkotlin/Metadata;(mv={2, 2, 0}, k=3, xi=176)" - : " @Lkotlin/Metadata;(mv={2, 0, 0}, k=3, xi=176)", + metadataForKotlinVersion( + " @Lkotlin/Metadata;(mv={2, 2, 0}, k=3, xi=176)", + " @Lkotlin/Metadata;(mv={2, 0, 0}, k=3, xi=176)"), " // access flags 0x19", " public final static INNERCLASS com/example/buck/BKt$getString$1 null null", "", @@ -1573,7 +1597,7 @@ public void kotlinClassWithInlineExtensionMethodUsingLambdaAndDefaultMethod() th "", " // compiled from: A.kt", "", - isKotlin22() ? metadata22 : metadata20, + metadataForKotlinVersion(metadata22, metadata20), "", " // access flags 0x1", " public ()V", @@ -1682,9 +1706,9 @@ public void kotlinClassWithInlineExtensionMethodUsingLambdaAndDefaultMethod() th " OUTERCLASS com/example/buck/A someMethod$default" + " (Lcom/example/buck/A;Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)V", "", - isKotlin22() - ? " @Lkotlin/Metadata;(mv={2, 2, 0}, k=3, xi=176)" - : " @Lkotlin/Metadata;(mv={2, 0, 0}, k=3, xi=176)", + metadataForKotlinVersion( + " @Lkotlin/Metadata;(mv={2, 2, 0}, k=3, xi=176)", + " @Lkotlin/Metadata;(mv={2, 0, 0}, k=3, xi=176)"), " // access flags 0x19", " public final static INNERCLASS com/example/buck/A$someMethod$1 null null", "", @@ -1832,7 +1856,7 @@ public void kotlinClassWithInlineExtensionMethodThatImplementsInterface() throws "", " // compiled from: B.kt", "", - isKotlin22() ? metadata22 : metadata20, + metadataForKotlinVersion(metadata22, metadata20), "", " // access flags 0x19", " public final static useSomeInterface(Lcom/example/buck/A;)V", @@ -1884,7 +1908,7 @@ public void kotlinClassWithInlineExtensionMethodThatImplementsInterface() throws "", " OUTERCLASS com/example/buck/BKt useSomeInterface (Lcom/example/buck/A;)V", "", - isKotlin22() ? innerMetadata21 : innerMetadata20, + metadataForKotlinVersion(innerMetadata21, innerMetadata20), " // access flags 0x609", " public static abstract INNERCLASS com/example/buck/A$SomeInterface" + " com/example/buck/A SomeInterface", @@ -2016,7 +2040,7 @@ public void kotlinClassWithInlineExtensionMethodFromInnerClassThatImplementsInte "", " // compiled from: B.kt", "", - isKotlin22() ? metadata22 : metadata20, + metadataForKotlinVersion(metadata22, metadata20), " // access flags 0x9", " public static INNERCLASS com/example/buck/B$C com/example/buck/B C", "", @@ -2031,7 +2055,7 @@ public void kotlinClassWithInlineExtensionMethodFromInnerClassThatImplementsInte "", " // compiled from: B.kt", "", - isKotlin22() ? innerMetadata21 : innerMetadata20, + metadataForKotlinVersion(innerMetadata21, innerMetadata20), " // access flags 0x9", " public static INNERCLASS com/example/buck/B$C com/example/buck/B C", "", @@ -2088,7 +2112,7 @@ public void kotlinClassWithInlineExtensionMethodFromInnerClassThatImplementsInte "", " OUTERCLASS com/example/buck/B$C useSomeInterface ()V", "", - isKotlin22() ? innerInnerMetadata21 : innerInnerMetadata20, + metadataForKotlinVersion(innerInnerMetadata21, innerInnerMetadata20), " // access flags 0x609", " public static abstract INNERCLASS com/example/buck/A$SomeInterface" + " com/example/buck/A SomeInterface", @@ -2209,7 +2233,7 @@ public void kotlinClassWithInlineExtensionMethodThatUsesRunnable() throws IOExce "", " // compiled from: B.kt", "", - isKotlin22() ? metadata22 : metadata20, + metadataForKotlinVersion(metadata22, metadata20), "", " // access flags 0x19", " // signature" @@ -2277,9 +2301,9 @@ public void kotlinClassWithInlineExtensionMethodThatUsesRunnable() throws IOExce " // compiled from: B.kt", " OUTERCLASS com/example/buck/BKt null", "", - isKotlin22() - ? " @Lkotlin/Metadata;(mv={2, 2, 0}, k=3, xi=176)" - : " @Lkotlin/Metadata;(mv={2, 0, 0}, k=3, xi=176)", + metadataForKotlinVersion( + " @Lkotlin/Metadata;(mv={2, 2, 0}, k=3, xi=176)", + " @Lkotlin/Metadata;(mv={2, 0, 0}, k=3, xi=176)"), " // access flags 0x19", " public final static INNERCLASS com/example/buck/BKt$sam$i$java_lang_Runnable$0 null" + " null", @@ -4308,7 +4332,7 @@ public void kotlinStubsPrivateInnerClasses() throws IOException { "", " // compiled from: A.kt", "", - isKotlin22() ? innerMetadata21 : innerMetadata20, + metadataForKotlinVersion(innerMetadata21, innerMetadata20), " // access flags 0x1A", " private final static INNERCLASS com/example/buck/A$B com/example/buck/A B", "", @@ -4329,7 +4353,7 @@ public void kotlinStubsPrivateInnerClasses() throws IOException { "", " // compiled from: A.kt", "", - isKotlin22() ? metadata22 : metadata20, + metadataForKotlinVersion(metadata22, metadata20), " // access flags 0x1A", " private final static INNERCLASS com/example/buck/A$B com/example/buck/A B", "", @@ -5699,7 +5723,7 @@ public void kotlinIgnoresAnonymousClasses() throws IOException { "", " // compiled from: A.kt", "", - isKotlin22() ? metadata22 : metadata20, + metadataForKotlinVersion(metadata22, metadata20), "", " // access flags 0x1", " public ()V", @@ -5811,7 +5835,7 @@ public void kotlinIgnoresLocalClasses() throws IOException { "", " // compiled from: A.kt", "", - isKotlin22() ? metadata22 : metadata20, + metadataForKotlinVersion(metadata22, metadata20), "", " // access flags 0x1", " public ()V", @@ -7562,7 +7586,7 @@ public void kotlinObjectWithJvmStatic() throws IOException { "", " // compiled from: Obj.kt", "", - isKotlin22() ? metadata22 : metadata20, + metadataForKotlinVersion(metadata22, metadata20), "", " // access flags 0x19", " public final static Lcom/example/buck/Obj; INSTANCE", @@ -7635,7 +7659,7 @@ public void kotlinClassWithDeprecatedMethods() throws IOException { "", " // compiled from: A.kt", "", - isKotlin22() ? metadata22 : metadata20, + metadataForKotlinVersion(metadata22, metadata20), "", " // access flags 0x1", " public ()V", @@ -7828,7 +7852,6 @@ private final class Tester { private final List expectedStubDirectory = new ArrayList<>(); private final List actualStubDirectory = new ArrayList<>(); - private final List actualFullDirectory = new ArrayList<>(); private final Map> expectedFullAbis = new HashMap<>(); private final Map> actualFullAbis = new HashMap<>(); private final Map> expectedStubs = new HashMap<>(); @@ -7860,7 +7883,6 @@ public Tester(Language language) { private void resetActuals() { actualStubDirectory.clear(); - actualFullDirectory.clear(); actualFullAbis.clear(); actualStubs.clear(); stubJarPath = null; @@ -8210,70 +8232,6 @@ public void assertStubJarIsDifferent() throws IOException { assertNotEquals(originalHash, Files.asByteSource(stubJarPath.toFile()).hash(Hashing.sha1())); } - @SuppressWarnings("unused") - public Tester dumpTestCode(boolean includeFullAbi) throws IOException { - if (includeFullAbi) { - compileFullJar(); - dumpFullJarAbi(); - } - createStubJar(); - dumpStubJar(); - - String indent = " "; - StringBuilder result = new StringBuilder(); - result.append("Test lines:\n"); - result.append(" tester\n"); - result.append(" .setSourceFile(\n"); - result.append(indent); - result.append('"'); - result.append(sourceFileName); - for (String sourceLine : sourceFileContents.split("\n")) { - result.append("\",\n"); - result.append(indent); - result.append('"'); - result.append(sourceLine.replace("\"", "\\\"")); - } - result.append("\")\n"); - for (String fileName : actualFullDirectory) { - if (fileName.endsWith("/") || fileName.equals(JarFile.MANIFEST_NAME)) { - continue; - } - if (includeFullAbi) { - result.append(" .addExpectedFullAbi(\n"); - result.append(indent); - result.append('"'); - result.append(fileName, 0, fileName.length() - ".class".length()); - - for (String abiLine : actualFullAbis.get(fileName)) { - result.append("\",\n"); - result.append(indent); - result.append('"'); - result.append(abiLine.replace("\"", "\\\"")); - } - result.append("\")\n"); - } - - if (actualStubs.containsKey(fileName)) { - result.append(" .addExpectedStub(\n"); - result.append(indent); - result.append('"'); - result.append(fileName, 0, fileName.length() - ".class".length()); - - for (String stubLine : actualStubs.get(fileName)) { - result.append("\",\n"); - result.append(indent); - result.append('"'); - result.append(stubLine.replace("\"", "\\\"")); - } - result.append("\")\n"); - } - } - result.append(" .createAndCheckStubJar();\n"); - - fail(result.toString()); - return this; - } - protected void dumpStubJar() throws IOException { try (JarFile file = new JarFile(stubJarPath.toFile())) { Iterable entries = file.stream()::iterator; @@ -8299,7 +8257,6 @@ protected void dumpFullJarAbi() throws IOException { if (JarFile.MANIFEST_NAME.equals(name)) { continue; } - actualFullDirectory.add(name); actualFullAbis.put( name, new JarDumper() diff --git a/prelude/toolchains/android/test/com/facebook/buck/jvm/kotlin/BUCK b/prelude/toolchains/android/test/com/facebook/buck/jvm/kotlin/BUCK index 5db2f35..680245a 100644 --- a/prelude/toolchains/android/test/com/facebook/buck/jvm/kotlin/BUCK +++ b/prelude/toolchains/android/test/com/facebook/buck/jvm/kotlin/BUCK @@ -1,12 +1,17 @@ -load("@prelude//toolchains/android/tools:build_rules.bzl", "standard_java_test") +load("@prelude//toolchains/android/tools:build_rules.bzl", "buck_kotlin_test") load("@prelude//utils:source_listing.bzl", "source_listing") oncall("android_devxx") source_listing() -standard_java_test( +buck_kotlin_test( name = "kotlin", + srcs = glob([ + "*Test.java", + "*Test.kt", + "*Fixture.kt", + ]), deps = [ "prelude//toolchains/android/src/com/facebook/buck/android:helpers", "prelude//toolchains/android/src/com/facebook/buck/cd/resources/proto:javacd-model", @@ -24,8 +29,11 @@ standard_java_test( "prelude//toolchains/android/src/com/facebook/buck/jvm/kotlin/buildtools:buildtools", "prelude//toolchains/android/src/com/facebook/buck/jvm/kotlin/kotlinc:kotlinc", "prelude//toolchains/android/src/com/facebook/buck/jvm/kotlin/kotlinc/incremental:incremental", + "prelude//toolchains/android/src/com/facebook/buck/jvm/kotlin/ksp:ksp", "prelude//toolchains/android/src/com/facebook/buck/jvm/kotlin/ksp/incremental:incremental", "prelude//toolchains/android/src/com/facebook/buck/step:step", + "prelude//toolchains/android/src/com/facebook/buck/step/isolatedsteps:isolatedsteps", + "prelude//toolchains/android/src/com/facebook/buck/step/isolatedsteps/common:common", "prelude//toolchains/android/src/com/facebook/buck/test/result/type:type", "prelude//toolchains/android/src/com/facebook/buck/test/selectors:selectors", "prelude//toolchains/android/src/com/facebook/buck/util:process_executor", @@ -47,8 +55,10 @@ standard_java_test( "prelude//toolchains/android/third-party:jsr305", "prelude//toolchains/android/third-party:junit", "prelude//toolchains/android/third-party:mockito", + "prelude//toolchains/android/third-party:mockito-kotlin", "prelude//toolchains/android/third-party:okhttp", "prelude//toolchains/android/third-party:okio", + "prelude//toolchains/android/third-party:symbol-processing-api", "prelude//toolchains/android/third-party:tools-sdk-common", ], ) diff --git a/prelude/toolchains/android/test/com/facebook/buck/jvm/kotlin/JarsActionMetadataTest.kt b/prelude/toolchains/android/test/com/facebook/buck/jvm/kotlin/JarsActionMetadataTest.kt index 47347ab..3967bf5 100644 --- a/prelude/toolchains/android/test/com/facebook/buck/jvm/kotlin/JarsActionMetadataTest.kt +++ b/prelude/toolchains/android/test/com/facebook/buck/jvm/kotlin/JarsActionMetadataTest.kt @@ -205,14 +205,14 @@ class JarsActionMetadataTest { } @Test - fun `when jar file with uppercase extension then filters correctly`() { + fun `when jar file with uppercase extension then it is not treated as a jar`() { val previousDigest = mutableMapOf(Paths.get("lib1.JAR") to "digest1") val currentDigest = mutableMapOf(Paths.get("lib1.JAR") to "digest1") val actionMetadata = ActionMetadata(Paths.get("metadata.json"), previousDigest, currentDigest) val jarsActionMetadata = JarsActionMetadata(actionMetadata) - assertEquals(1, jarsActionMetadata.previousJarsDigest.size) - assertEquals(1, jarsActionMetadata.currentJarsDigest.size) + assertEquals(0, jarsActionMetadata.previousJarsDigest.size) + assertEquals(0, jarsActionMetadata.currentJarsDigest.size) } } diff --git a/prelude/toolchains/android/test/com/facebook/buck/jvm/kotlin/KotlincStepTest.java b/prelude/toolchains/android/test/com/facebook/buck/jvm/kotlin/KotlincStepTest.java new file mode 100644 index 0000000..8620483 --- /dev/null +++ b/prelude/toolchains/android/test/com/facebook/buck/jvm/kotlin/KotlincStepTest.java @@ -0,0 +1,67 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is dual-licensed under either the MIT license found in the + * LICENSE-MIT file in the root directory of this source tree or the Apache + * License, Version 2.0 found in the LICENSE-APACHE file in the root directory + * of this source tree. You may select, at your option, one of the + * above-listed licenses. + */ + +package com.facebook.buck.jvm.kotlin; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +import com.facebook.buck.core.filesystems.AbsPath; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import java.nio.file.Paths; +import java.util.Optional; +import org.junit.Test; + +public class KotlincStepTest { + private static final AbsPath TEST_ROOT = AbsPath.of(Paths.get(".").toAbsolutePath().normalize()); + + @Test + public void structuredOptionsIncludeContextAndExplicitEmptyClasspath() { + ImmutableList options = + KotlincStep.getKosabiApplicabilityPluginOptions( + "fbcode//example:target", TEST_ROOT, ImmutableList.of(), Optional.of(path("fbcode"))); + + assertEquals( + ImmutableList.of( + "-P", + "plugin:com.facebook.kotlin.compilerplugins.kosabiapplicability:target-label=fbcode//example:target", + "-P", + "plugin:com.facebook.kotlin.compilerplugins.kosabiapplicability:source-root=" + + TEST_ROOT.getPath(), + "-P", + "plugin:com.facebook.kotlin.compilerplugins.kosabiapplicability:source-root-prefix=fbcode", + "-P", + "plugin:com.facebook.kotlin.compilerplugins.kosabiapplicability:source-only-abi-classpath="), + options); + } + + @Test + public void structuredOptionsFailClosedWithoutRequiredBuckContext() { + IllegalStateException missingPlugin = + assertThrows( + IllegalStateException.class, + () -> KotlincStep.getRequiredKosabiApplicabilityPlugin(ImmutableMap.of())); + assertTrue(missingPlugin.getMessage().contains("plugin path is missing")); + + IllegalStateException missingCellRoot = + assertThrows( + IllegalStateException.class, + () -> + KotlincStep.getKosabiApplicabilityPluginOptions( + "fbcode//example:target", TEST_ROOT, ImmutableList.of(), Optional.empty())); + assertTrue(missingCellRoot.getMessage().contains("cell root path is missing")); + } + + private static AbsPath path(String relativePath) { + return TEST_ROOT.resolve(relativePath); + } +} diff --git a/prelude/toolchains/android/test/com/facebook/buck/jvm/kotlin/Ksp2NoOpDetectorTest.kt b/prelude/toolchains/android/test/com/facebook/buck/jvm/kotlin/Ksp2NoOpDetectorTest.kt new file mode 100644 index 0000000..1e7fb7d --- /dev/null +++ b/prelude/toolchains/android/test/com/facebook/buck/jvm/kotlin/Ksp2NoOpDetectorTest.kt @@ -0,0 +1,187 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is dual-licensed under either the MIT license found in the + * LICENSE-MIT file in the root directory of this source tree or the Apache + * License, Version 2.0 found in the LICENSE-APACHE file in the root directory + * of this source tree. You may select, at your option, one of the + * above-listed licenses. + */ + +package com.facebook.buck.jvm.kotlin + +import com.facebook.buck.jvm.kotlin.ksp.Ksp2NoOpDetector +import com.facebook.kotlin.ksp.kspadapter.SymbolProcessorBase +import com.google.devtools.ksp.processing.CodeGenerator +import com.google.devtools.ksp.processing.Resolver +import com.google.devtools.ksp.processing.SymbolProcessor +import com.google.devtools.ksp.processing.SymbolProcessorEnvironment +import com.google.devtools.ksp.processing.SymbolProcessorProvider +import com.google.devtools.ksp.symbol.KSAnnotated +import java.io.File +import org.junit.Assert.assertEquals +import org.junit.Assert.assertSame +import org.junit.Test +import org.mockito.kotlin.mock +import org.mockito.kotlin.whenever + +class Ksp2NoOpDetectorTest { + + /** + * Stands in for `CodeGeneratorImpl`: `generatedFile` is rebuilt on each access and holds only + * files whose output streams are still open, so [closeRound] models KSP emptying it at each round + * boundary. + */ + private class FakeCodeGenerator : CodeGenerator by mock() { + private val open = mutableListOf() + + override val generatedFile: Collection + get() = open.toList() + + fun generate(path: String) { + open.add(File(path)) + } + + fun closeRound() = open.clear() + } + + private val codeGenerator = FakeCodeGenerator() + private val detector = Ksp2NoOpDetector() + private val deferred = listOf(mock()) + + private inner class FakeProcessor( + private val paths: List = emptyList(), + private val finishPaths: List = emptyList(), + ) : SymbolProcessor { + override fun process(resolver: Resolver): List { + paths.forEach(codeGenerator::generate) + return deferred + } + + override fun finish() = finishPaths.forEach(codeGenerator::generate) + } + + /** Subclasses the stand-in declared under the adapter's real package name. */ + private inner class AdapterProcessor : SymbolProcessorBase() { + override fun process(resolver: Resolver): List { + codeGenerator.generate("adapter.kt") + return deferred + } + } + + /** + * Named provider classes rather than lambdas: counts are keyed by provider class name, and all + * lambdas from one call site share a class. Real providers are distinct classes, one per + * processor, which is what these model. + */ + private inner class ProductiveProvider(private val paths: List) : + SymbolProcessorProvider { + override fun create(environment: SymbolProcessorEnvironment) = FakeProcessor(paths) + } + + private inner class IdleProvider : SymbolProcessorProvider { + override fun create(environment: SymbolProcessorEnvironment) = FakeProcessor() + } + + private inner class FinishOnlyProvider : SymbolProcessorProvider { + override fun create(environment: SymbolProcessorEnvironment) = + FakeProcessor(finishPaths = listOf("late.kt")) + } + + private inner class AdapterProvider : SymbolProcessorProvider { + override fun create(environment: SymbolProcessorEnvironment) = AdapterProcessor() + } + + private fun wrap(vararg providers: SymbolProcessorProvider): List { + val environment: SymbolProcessorEnvironment = mock() + whenever(environment.codeGenerator).thenReturn(codeGenerator) + return detector.wrap(providers.toList()).map { it.create(environment) } + } + + private fun countOf(provider: Class<*>) = detector.countsByProcessor.getValue(provider.name) + + /** + * The reason each processor is wrapped rather than the shared generator read once: processors + * share a generator, so one's output would otherwise be credited to another. + */ + @Test + fun `each processor is credited only with the files it generated`() { + val (productive, idle) = wrap(ProductiveProvider(listOf("a.kt", "b.kt")), IdleProvider()) + + productive.process(mock()) + idle.process(mock()) + + assertEquals(2, countOf(ProductiveProvider::class.java)) + assertEquals(0, countOf(IdleProvider::class.java)) + } + + /** KSP closes output streams between rounds, emptying `generatedFile`. */ + @Test + fun `counts accumulate across the round boundary that clears the generator`() { + val processor = wrap(ProductiveProvider(listOf("a.kt", "b.kt"))).single() + + processor.process(mock()) + codeGenerator.closeRound() + processor.process(mock()) + + assertEquals(4, countOf(ProductiveProvider::class.java)) + } + + /** + * KSP calls `finish()` after the last round, so a processor that generates only there has + * generated output and must not read as a no-op. + */ + @Test + fun `files generated in finish are credited to their provider`() { + val processor = wrap(FinishOnlyProvider()).single() + + processor.process(mock()) + processor.finish() + + assertEquals(mapOf(FinishOnlyProvider::class.java.name to 1), detector.countsByProcessor) + } + + /** Dropping the deferred symbols would silently stop KSP reprocessing them. */ + @Test + fun `deferred symbols are returned unchanged`() { + val processor = wrap(ProductiveProvider(listOf("a.kt"))).single() + + assertSame(deferred, processor.process(mock())) + } + + /** A provider whose processor never runs is still attached, so it reports zero, not absent. */ + @Test + fun `provider whose processor never runs counts zero`() { + wrap(ProductiveProvider(listOf("a.kt"))) + + assertEquals(mapOf(ProductiveProvider::class.java.name to 0), detector.countsByProcessor) + } + + /** + * `SymbolProcessorBase` emits a placeholder exactly when a round generated nothing, so an + * adapter-wrapped processor could never be observed as idle. Excluded rather than counted + * wrongly. + */ + @Test + fun `adapter-wrapped processors are not counted at all`() { + val (native, adapter) = wrap(ProductiveProvider(listOf("a.kt")), AdapterProvider()) + + native.process(mock()) + adapter.process(mock()) + + assertEquals(mapOf(ProductiveProvider::class.java.name to 1), detector.countsByProcessor) + } + + /** + * Tripwire half of the adapter-FQN pin: the fixture must keep the real base class's name. The + * other half is `AdapterBaseClassFqnTest` in fbandroid, which pins the real class to this same + * literal; `Ksp2NoOpDetector` matches processors against it by name. + */ + @Test + fun `adapter fixture keeps the real base class name`() { + assertEquals( + "com.facebook.kotlin.ksp.kspadapter.SymbolProcessorBase", + SymbolProcessorBase::class.java.name, + ) + } +} diff --git a/prelude/toolchains/android/test/com/facebook/buck/jvm/kotlin/Ksp2StepTest.kt b/prelude/toolchains/android/test/com/facebook/buck/jvm/kotlin/Ksp2StepTest.kt new file mode 100644 index 0000000..345bce7 --- /dev/null +++ b/prelude/toolchains/android/test/com/facebook/buck/jvm/kotlin/Ksp2StepTest.kt @@ -0,0 +1,57 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is dual-licensed under either the MIT license found in the + * LICENSE-MIT file in the root directory of this source tree or the Apache + * License, Version 2.0 found in the LICENSE-APACHE file in the root directory + * of this source tree. You may select, at your option, one of the + * above-listed licenses. + */ + +package com.facebook.buck.jvm.kotlin + +import com.facebook.buck.core.filesystems.RelPath +import com.facebook.buck.jvm.kotlin.ksp.Ksp2Step +import com.facebook.buck.jvm.kotlin.ksp.incremental.Ksp2Mode +import com.facebook.buck.testutil.TemporaryPaths +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Rule +import org.junit.Test + +class Ksp2StepTest { + + @Rule @JvmField val temporaryPaths = TemporaryPaths() + + private val nonIncremental = Ksp2Mode.NonIncremental(RelPath.get("kspCaches")) + + private fun incremental() = + Ksp2Mode.Incremental( + cachesDir = temporaryPaths.root.resolve("caches"), + incrementalLog = false, + modifiedSources = emptyList(), + removedSources = emptyList(), + changedClasses = emptyList(), + reprocessReason = null, + ) + + @Test + fun `records counts for successful non-incremental runs`() { + assertTrue(Ksp2Step.shouldRecordProcessorCounts(true, true, nonIncremental)) + } + + @Test + fun `skips counts for incremental runs`() { + assertFalse(Ksp2Step.shouldRecordProcessorCounts(true, true, incremental())) + } + + @Test + fun `skips counts on failure`() { + assertFalse(Ksp2Step.shouldRecordProcessorCounts(false, true, nonIncremental)) + } + + @Test + fun `skips counts when empty`() { + assertFalse(Ksp2Step.shouldRecordProcessorCounts(true, false, nonIncremental)) + } +} diff --git a/prelude/toolchains/android/test/com/facebook/buck/jvm/kotlin/KspAdapterBaseFixture.kt b/prelude/toolchains/android/test/com/facebook/buck/jvm/kotlin/KspAdapterBaseFixture.kt new file mode 100644 index 0000000..0ad87cb --- /dev/null +++ b/prelude/toolchains/android/test/com/facebook/buck/jvm/kotlin/KspAdapterBaseFixture.kt @@ -0,0 +1,20 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is dual-licensed under either the MIT license found in the + * LICENSE-MIT file in the root directory of this source tree or the Apache + * License, Version 2.0 found in the LICENSE-APACHE file in the root directory + * of this source tree. You may select, at your option, one of the + * above-listed licenses. + */ + +package com.facebook.kotlin.ksp.kspadapter + +import com.google.devtools.ksp.processing.SymbolProcessor + +/** + * Stands in for the javax adapter's own base class, which lives in fbandroid and is not on this + * target's classpath. `Ksp2NoOpDetector` recognises adapter processors by this fully-qualified + * name, so the package and class name here are what the test actually exercises. + */ +abstract class SymbolProcessorBase : SymbolProcessor diff --git a/prelude/toolchains/android/test/com/facebook/buck/jvm/kotlin/KspStepsBuilderTest.java b/prelude/toolchains/android/test/com/facebook/buck/jvm/kotlin/KspStepsBuilderTest.java new file mode 100644 index 0000000..6a75c9c --- /dev/null +++ b/prelude/toolchains/android/test/com/facebook/buck/jvm/kotlin/KspStepsBuilderTest.java @@ -0,0 +1,114 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is dual-licensed under either the MIT license found in the + * LICENSE-MIT file in the root directory of this source tree or the Apache + * License, Version 2.0 found in the LICENSE-APACHE file in the root directory + * of this source tree. You may select, at your option, one of the + * above-listed licenses. + */ + +package com.facebook.buck.jvm.kotlin; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.facebook.buck.core.filesystems.AbsPath; +import com.facebook.buck.core.filesystems.RelPath; +import com.facebook.buck.jvm.java.ResolvedJavacPluginProperties; +import com.facebook.buck.step.isolatedsteps.IsolatedStep; +import com.facebook.buck.step.isolatedsteps.common.CopyIsolatedStep; +import com.facebook.buck.step.isolatedsteps.common.RmIsolatedStep; +import com.facebook.buck.step.isolatedsteps.common.ZipIsolatedStep; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSortedSet; +import org.junit.Test; + +public class KspStepsBuilderTest { + + private static final String METAGEN_PROCESSOR = + "KSP:com.facebook.metagen.processor.kspmetagen.MetagenKspProcessorProvider"; + + @Test + public void usesFinalRoundPlaceholder_forEverySentinelProducer() { + ImmutableList processorNames = + ImmutableList.of( + "KSP:com.facebook.annotationprocessors.inject.ksp.InjectorKspProcessorProvider", + METAGEN_PROCESSOR, + "KSP:com.facebook.annotationprocessors.gatekeepers.ksp.GatekeeperDeclarationKspProcessor", + "KSP:com.facebook.annotationprocessors.qe.ksp.QEKspProcessorProvider"); + + for (String processorName : processorNames) { + assertTrue( + processorName, + KspStepsBuilder.usesFinalRoundPlaceholder( + ImmutableList.of(processorWithNames(processorName)))); + } + } + + @Test + public void usesFinalRoundPlaceholder_findsLaterProcessorName() { + assertTrue( + KspStepsBuilder.usesFinalRoundPlaceholder( + ImmutableList.of( + processorWithNames("KSP:com.example.UnrelatedProcessor"), + processorWithNames("KSP:com.example.OtherProcessor", METAGEN_PROCESSOR)))); + } + + @Test + public void usesFinalRoundPlaceholder_ignoresUnrelatedProcessor() { + assertFalse( + KspStepsBuilder.usesFinalRoundPlaceholder( + ImmutableList.of( + processorWithNames("KSP:com.example.processor.RealDummyGeneratorProvider")))); + } + + @Test + public void stagingSteps_removePlaceholderAfterCopiesAndBeforeZip() { + ImmutableList steps = stagingStepsFor(METAGEN_PROCESSOR); + + assertEquals(6, steps.size()); + assertTrue(steps.get(0) instanceof CopyIsolatedStep); + assertTrue(steps.get(1) instanceof CopyIsolatedStep); + assertTrue(steps.get(2) instanceof CopyIsolatedStep); + assertTrue(steps.get(3) instanceof RmIsolatedStep); + assertEquals( + RelPath.get("staged/com/facebook/Dummy.java"), ((RmIsolatedStep) steps.get(3)).getPath()); + assertTrue(steps.get(4) instanceof ZipIsolatedStep); + assertTrue(steps.get(5) instanceof CopyIsolatedStep); + } + + @Test + public void stagingSteps_preserveOutputsForUnrelatedProcessor() { + ImmutableList steps = + stagingStepsFor("KSP:com.example.processor.RealDummyGeneratorProvider"); + + assertEquals(5, steps.size()); + assertTrue(steps.get(0) instanceof CopyIsolatedStep); + assertTrue(steps.get(1) instanceof CopyIsolatedStep); + assertTrue(steps.get(2) instanceof CopyIsolatedStep); + assertTrue(steps.get(3) instanceof ZipIsolatedStep); + assertTrue(steps.get(4) instanceof CopyIsolatedStep); + } + + private static ImmutableList stagingStepsFor(String processorName) { + return KspStepsBuilder.createKspOutputStagingSteps( + mock(AbsPath.class), + RelPath.get("kotlin"), + RelPath.get("java"), + RelPath.get("classes"), + RelPath.get("staged"), + RelPath.get("generated.src.zip"), + RelPath.get("annotation-output"), + ImmutableList.of(processorWithNames(processorName))); + } + + private static ResolvedJavacPluginProperties processorWithNames(String... processorNames) { + ResolvedJavacPluginProperties processor = mock(ResolvedJavacPluginProperties.class); + when(processor.getProcessorNames()).thenReturn(ImmutableSortedSet.copyOf(processorNames)); + return processor; + } +} diff --git a/prelude/toolchains/android/test/com/facebook/buck/jvm/kotlin/cd/analytics/logger/KotlinCDLoggerAnalyticsTest.kt b/prelude/toolchains/android/test/com/facebook/buck/jvm/kotlin/cd/analytics/logger/KotlinCDLoggerAnalyticsTest.kt index 6f93db3..2be1b98 100644 --- a/prelude/toolchains/android/test/com/facebook/buck/jvm/kotlin/cd/analytics/logger/KotlinCDLoggerAnalyticsTest.kt +++ b/prelude/toolchains/android/test/com/facebook/buck/jvm/kotlin/cd/analytics/logger/KotlinCDLoggerAnalyticsTest.kt @@ -190,6 +190,26 @@ internal class KotlinCDLoggerAnalyticsTest { verify(kotlinCDLogger, times(1)).log(expectedEntry) } + @Test + fun `when a step duration is set, it is logged`() { + val kotlinCDAnalytics = createFakeKotlinCDAnalytics() + val expectedEntry = createExpectedKotlinCDLogEntry(durationMs = 1234L) + + kotlinCDAnalytics.log(createKotlinCDLoggingContext(durationMs = 1234L)) + + verify(kotlinCDLogger, times(1)).log(expectedEntry) + } + + @Test + fun `when no step duration is set, none is logged`() { + val kotlinCDAnalytics = createFakeKotlinCDAnalytics() + val expectedEntry = createExpectedKotlinCDLogEntry() + + kotlinCDAnalytics.log(createKotlinCDLoggingContext()) + + verify(kotlinCDLogger, times(1)).log(expectedEntry) + } + private fun createKotlinCDLoggingContext( step: StepParam = StepParam.KOTLINC, languageVersion: String = DEFAULT_LANGUAGE_VERSION, @@ -198,11 +218,13 @@ internal class KotlinCDLoggerAnalyticsTest { extras: Map> = mapOf(), numKotlinTokens: Long = 0L, numJavaTokens: Long = 0L, + durationMs: Long? = null, ): KotlinCDLoggingContext { val context = KotlinCDLoggingContext(step, LanguageVersion(languageVersion), kotlincMode) extras.forEach { (key, extras) -> extras.forEach { item -> context.addExtras(key, item) } } context.numKotlinTokens = numKotlinTokens context.numJavaTokens = numJavaTokens + context.durationMs = durationMs return context } @@ -231,6 +253,7 @@ internal class KotlinCDLoggerAnalyticsTest { removedFiles: Set = emptySet(), numKotlinTokens: Long? = null, numJavaTokens: Long? = null, + durationMs: Long? = null, ) = KotlinCDLogEntry( time = Instant.now(clock).epochSecond, eventTime = Instant.now(clock).epochSecond.toDouble(), @@ -250,6 +273,7 @@ internal class KotlinCDLoggerAnalyticsTest { removedFiles = removedFiles, numKotlinTokens = numKotlinTokens, numJavaTokens = numJavaTokens, + durationMs = durationMs, ) companion object TestParams { diff --git a/prelude/toolchains/android/test/com/facebook/buck/jvm/kotlin/testutil/compiler/BUCK b/prelude/toolchains/android/test/com/facebook/buck/jvm/kotlin/testutil/compiler/BUCK index 64c5839..f245d01 100644 --- a/prelude/toolchains/android/test/com/facebook/buck/jvm/kotlin/testutil/compiler/BUCK +++ b/prelude/toolchains/android/test/com/facebook/buck/jvm/kotlin/testutil/compiler/BUCK @@ -17,6 +17,7 @@ buck_java_library( "prelude//toolchains/android/third-party:guava", "prelude//toolchains/android/third-party:junit", "prelude//toolchains/android/third-party:kotlin-compiler-embeddable", + "prelude//toolchains/android/third-party:kotlin-reflect", "prelude//toolchains/android/third-party:kotlin-stdlib", "prelude//toolchains/android/third-party:trove", ], diff --git a/prelude/toolchains/android/test/com/facebook/buck/testrunner/BUCK b/prelude/toolchains/android/test/com/facebook/buck/testrunner/BUCK index 415956e..47bdeaa 100644 --- a/prelude/toolchains/android/test/com/facebook/buck/testrunner/BUCK +++ b/prelude/toolchains/android/test/com/facebook/buck/testrunner/BUCK @@ -54,6 +54,16 @@ standard_java_test( ], ) +buck_kotlin_test( + name = "junit_runner_test", + srcs = ["JUnitRunnerTest.kt"], + deps = [ + "prelude//toolchains/android/src/com/facebook/buck/testrunner:base", + "prelude//toolchains/android/src/com/facebook/buck/testrunner:junit", + "prelude//toolchains/android/third-party:junit", + ], +) + buck_kotlin_test( name = "tpx_timeout_buffer_manager_test", srcs = ["TpxTimeoutBufferManagerTest.kt"], @@ -77,6 +87,9 @@ buck_kotlin_test( buck_kotlin_test( name = "per_test_coverage_listener_test", srcs = ["PerTestJUnitCoverageRunListenerTest.kt"], + friend_paths = [ + "prelude//toolchains/android/src/com/facebook/buck/testrunner:per_test_coverage_base", + ], deps = [ "prelude//toolchains/android/src/com/facebook/buck/testrunner:junit", "prelude//toolchains/android/src/com/facebook/buck/testrunner:per_test_coverage_base", diff --git a/prelude/toolchains/android/test/com/facebook/buck/testrunner/JUnitRunnerTest.kt b/prelude/toolchains/android/test/com/facebook/buck/testrunner/JUnitRunnerTest.kt new file mode 100644 index 0000000..562cc47 --- /dev/null +++ b/prelude/toolchains/android/test/com/facebook/buck/testrunner/JUnitRunnerTest.kt @@ -0,0 +1,80 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is dual-licensed under either the MIT license found in the + * LICENSE-MIT file in the root directory of this source tree or the Apache + * License, Version 2.0 found in the LICENSE-APACHE file in the root directory + * of this source tree. You may select, at your option, one of the + * above-listed licenses. + */ + +package com.facebook.buck.testrunner + +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test +import org.junit.runner.Description +import org.junit.runner.Runner +import org.junit.runner.notification.RunNotifier +import org.junit.runners.ParentRunner + +class JUnitRunnerTest { + + @Test + fun returnsTrueWhenEveryNestedSuiteRunnerMatches() { + val runner = CompositeRunner(TargetRunner(), CompositeRunner(TargetRunner(), TargetRunner())) + + assertTrue( + JUnitRunner.isRunnerOfTypeOrSuiteOfType( + runner, + TargetRunner::class.java, + CompositeRunner::class.java, + ), + ) + } + + @Test + fun returnsFalseWhenAnyNestedSuiteRunnerDoesNotMatch() { + val runner = CompositeRunner(TargetRunner(), CompositeRunner(TargetRunner(), OtherRunner())) + + assertFalse( + JUnitRunner.isRunnerOfTypeOrSuiteOfType( + runner, + TargetRunner::class.java, + CompositeRunner::class.java, + ), + ) + } + + @Test + fun returnsFalseForEmptySuite() { + assertFalse( + JUnitRunner.isRunnerOfTypeOrSuiteOfType( + CompositeRunner(), + TargetRunner::class.java, + CompositeRunner::class.java, + ), + ) + } + + private class CompositeRunner(vararg children: Runner) : + ParentRunner(CompositeRunner::class.java) { + private val childRunners = children.toList() + + override fun getChildren(): List = childRunners + + override fun describeChild(child: Runner): Description = child.description + + override fun runChild(child: Runner, notifier: RunNotifier) { + child.run(notifier) + } + } + + private open class OtherRunner : Runner() { + override fun getDescription(): Description = Description.EMPTY + + override fun run(notifier: RunNotifier) = Unit + } + + private class TargetRunner : OtherRunner() +} diff --git a/prelude/toolchains/android/test/com/facebook/buck/testrunner/PerTestJUnitCoverageRunListenerTest.kt b/prelude/toolchains/android/test/com/facebook/buck/testrunner/PerTestJUnitCoverageRunListenerTest.kt index aa78ef1..1dcf891 100644 --- a/prelude/toolchains/android/test/com/facebook/buck/testrunner/PerTestJUnitCoverageRunListenerTest.kt +++ b/prelude/toolchains/android/test/com/facebook/buck/testrunner/PerTestJUnitCoverageRunListenerTest.kt @@ -11,6 +11,8 @@ package com.facebook.buck.testrunner import java.io.File +import java.util.ServiceConfigurationError +import org.junit.Assert.assertEquals import org.junit.Assert.assertTrue import org.junit.Assert.fail import org.junit.Rule @@ -83,6 +85,104 @@ class PerTestJUnitCoverageRunListenerTest { } } + @Test + fun `coverage extensions follow the per-test listener lifecycle`() { + val outputDir = tempDir.newFolder("coverage") + val events = ArrayList() + val extension = + object : PerTestCoverageExtension { + override fun initialize(outputDir: File) { + events.add("initialize:${outputDir.name}") + } + + override fun testStarted(testName: String) { + events.add("started:$testName") + } + + override fun testFinished(testName: String) { + events.add("finished:$testName") + } + + override fun close() { + events.add("close") + } + } + val listener = PerTestJUnitCoverageRunListener( + outputDir, + FakeCoverageAgent(byteArrayOf(1)), + PerTestCoverageExtensionManager(listOf(extension)), + ) + val test = Description.createTestDescription("Example", "test") + listener.testStarted(test) + listener.testFinished(test) + listener.close() + + assertEquals( + listOf( + "initialize:coverage", + "started:test (Example)", + "finished:test (Example)", + "close", + ), + events, + ) + } + + @Test + fun `coverage extension failures do not fail primary coverage`() { + val outputDir = tempDir.newFolder("coverage") + val extension = + object : PerTestCoverageExtension { + override fun initialize(outputDir: File) { + throw IllegalStateException("optional collector unavailable") + } + + override fun testStarted(testName: String) { + throw IllegalStateException("optional collector unavailable") + } + } + val listener = PerTestJUnitCoverageRunListener( + outputDir, + FakeCoverageAgent(byteArrayOf(1)), + PerTestCoverageExtensionManager(listOf(extension)), + ) + val test = Description.createTestDescription("Example", "test") + + listener.testStarted(test) + listener.testFinished(test) + listener.close() + + assertTrue("Extension failure does not set coverageError", listener.coverageError == null) + assertTrue(File(outputDir, "test_(Example).exec").exists()) + } + + @Test + fun `provider discovery continues after a provider fails`() { + val events = ArrayList() + val extension = + object : PerTestCoverageExtension { + override fun initialize(outputDir: File) { + events.add("initialized") + } + } + var providerIndex = 0 + val providers = + object : Iterator { + override fun hasNext(): Boolean = providerIndex < 2 + + override fun next(): PerTestCoverageExtension { + providerIndex += 1 + if (providerIndex == 1) throw ServiceConfigurationError("broken provider") + return extension + } + } + + val manager = PerTestCoverageExtensionManager.load(providers) + manager.initialize(tempDir.newFolder("coverage")) + + assertEquals(listOf("initialized"), events) + } + /** Fake [BasePerTestCoverageRunListener.CoverageAgent] returning fixed data. */ private class FakeCoverageAgent(private val execData: ByteArray?) : BasePerTestCoverageRunListener.CoverageAgent { diff --git a/prelude/toolchains/android/third-party/BUCK b/prelude/toolchains/android/third-party/BUCK index b73009a..c00c62e 100644 --- a/prelude/toolchains/android/third-party/BUCK +++ b/prelude/toolchains/android/third-party/BUCK @@ -1,4 +1,4 @@ -load("@prelude//toolchains/android/tools:build_rules.bzl", "buck_java_binary", "buck_java_library", "third_party_aar", "third_party_exe", "third_party_jar") +load("@prelude//toolchains/android/tools:build_rules.bzl", "buck_java_binary", "buck_java_library", "third_party_exe", "third_party_jar") load("@prelude//utils:source_listing.bzl", "source_listing") oncall("android_devxx") @@ -829,24 +829,24 @@ third_party_jar( third_party_jar( name = "kotlin-stdlib", # @oss-disable[end= ]: internal_alias = "fbsource//third-party/java/kotlincd:kotlin-stdlib__toolchain", - sha1 = "85f8b81009cda5890e54ba67d64b5e599c645020", - url = "mvn:org.jetbrains.kotlin:kotlin-stdlib:jar:2.1.0", + sha1 = "fdfc65fbc42fda253a26f61dac3c0aca335fae96", + url = "mvn:org.jetbrains.kotlin:kotlin-stdlib:jar:2.2.0", visibility = ["PUBLIC"], ) third_party_jar( name = "classic_kotlin-stdlib", # @oss-disable[end= ]: internal_alias = "fbsource//third-party/java/org/jetbrains/kotlin/kotlin-stdlib:kotlin-stdlib__toolchain", - sha1 = "85f8b81009cda5890e54ba67d64b5e599c645020", - url = "mvn:org.jetbrains.kotlin:kotlin-stdlib:jar:2.1.0", + sha1 = "fdfc65fbc42fda253a26f61dac3c0aca335fae96", + url = "mvn:org.jetbrains.kotlin:kotlin-stdlib:jar:2.2.0", visibility = ["PUBLIC"], ) third_party_jar( name = "jvm-abi-gen", # @oss-disable[end= ]: internal_alias = "fbsource//third-party/java/kotlincd:jvm-abi-gen", - sha1 = "ec44cc7ca0ffc6c21358b0628b5eaafb34a1a20c", - url = "mvn:org.jetbrains.kotlin:jvm-abi-gen:jar:2.1.0", + sha1 = "de3025160a9b2a158fb400a03677167a0ff31144", + url = "mvn:org.jetbrains.kotlin:jvm-abi-gen:jar:2.2.0", visibility = ["PUBLIC"], ) @@ -854,16 +854,16 @@ third_party_jar( third_party_jar( name = "kotlin-compiler-embeddable", # @oss-disable[end= ]: internal_alias = "fbsource//third-party/java/kotlincd:kotlin-compiler-embeddable__toolchain", - sha1 = "988bf980588bbbcfa1ba19c1ffd7aa869b335a31", - url = "mvn:org.jetbrains.kotlin:kotlin-compiler-embeddable:jar:2.1.0", + sha1 = "8cfa2b049a4006d94474296df4abd9b50f288821", + url = "mvn:org.jetbrains.kotlin:kotlin-compiler-embeddable:jar:2.2.0", visibility = ["PUBLIC"], ) third_party_jar( name = "classic_kotlin-compiler-embeddable", # @oss-disable[end= ]: internal_alias = "fbsource//third-party/java/org/jetbrains/kotlin/kotlin-compiler-embeddable:kotlin-compiler-embeddable__toolchain", - sha1 = "988bf980588bbbcfa1ba19c1ffd7aa869b335a31", - url = "mvn:org.jetbrains.kotlin:kotlin-compiler-embeddable:jar:2.1.0", + sha1 = "8cfa2b049a4006d94474296df4abd9b50f288821", + url = "mvn:org.jetbrains.kotlin:kotlin-compiler-embeddable:jar:2.2.0", visibility = ["PUBLIC"], ) @@ -884,56 +884,56 @@ third_party_jar( third_party_jar( name = "kotlin-reflect", # @oss-disable[end= ]: internal_alias = "fbsource//third-party/java/kotlincd:kotlin-reflect__toolchain", - sha1 = "dc3753b59b67d79b7d2f546d51e22a0ec1f082d8", - url = "mvn:org.jetbrains.kotlin:kotlin-reflect:jar:2.1.0", + sha1 = "4258f471b10f4fce1b1c000b59495b7413e1af9a", + url = "mvn:org.jetbrains.kotlin:kotlin-reflect:jar:2.2.0", visibility = ["PUBLIC"], ) third_party_jar( name = "classic_kotlin-reflect", # @oss-disable[end= ]: internal_alias = "fbsource//third-party/java/org/jetbrains/kotlin/kotlin-reflect:kotlin-reflect__toolchain", - sha1 = "dc3753b59b67d79b7d2f546d51e22a0ec1f082d8", - url = "mvn:org.jetbrains.kotlin:kotlin-reflect:jar:2.1.0", + sha1 = "4258f471b10f4fce1b1c000b59495b7413e1af9a", + url = "mvn:org.jetbrains.kotlin:kotlin-reflect:jar:2.2.0", visibility = ["PUBLIC"], ) third_party_jar( name = "kotlin-script-runtime", # @oss-disable[end= ]: internal_alias = "fbsource//third-party/java/kotlincd:kotlin-script-runtime", - sha1 = "5909da8c57b75a117714ab0e348c86101b7a3284", - url = "mvn:org.jetbrains.kotlin:kotlin-script-runtime:jar:2.1.0", + sha1 = "87c92e866fcd68680966a3005a2992e1ab8ec6ad", + url = "mvn:org.jetbrains.kotlin:kotlin-script-runtime:jar:2.2.0", visibility = ["PUBLIC"], ) third_party_jar( name = "classic_kotlin-script-runtime", # @oss-disable[end= ]: internal_alias = "fbsource//third-party/java/org/jetbrains/kotlin/kotlin-script-runtime:kotlin-script-runtime", - sha1 = "5909da8c57b75a117714ab0e348c86101b7a3284", - url = "mvn:org.jetbrains.kotlin:kotlin-script-runtime:jar:2.1.0", + sha1 = "87c92e866fcd68680966a3005a2992e1ab8ec6ad", + url = "mvn:org.jetbrains.kotlin:kotlin-script-runtime:jar:2.2.0", visibility = ["PUBLIC"], ) third_party_jar( name = "kotlin-annotation-processing-embeddable", # @oss-disable[end= ]: internal_alias = "fbsource//third-party/java/kotlincd:kotlin-annotation-processing-embeddable", - sha1 = "5bc00235fae6530545a819fb821ca8b3df1d41d7", - url = "mvn:org.jetbrains.kotlin:kotlin-annotation-processing-embeddable:jar:2.1.0", + sha1 = "944a8b045adf3b81f32c5843229a90d3792dc7d8", + url = "mvn:org.jetbrains.kotlin:kotlin-annotation-processing-embeddable:jar:2.2.0", visibility = ["PUBLIC"], ) third_party_jar( name = "classic_kotlin-annotation-processing-embeddable", # @oss-disable[end= ]: internal_alias = "fbsource//third-party/java/org/jetbrains/kotlin/kotlin-annotation-processing-embeddable:kotlin-annotation-processing-embeddable", - sha1 = "5bc00235fae6530545a819fb821ca8b3df1d41d7", - url = "mvn:org.jetbrains.kotlin:kotlin-annotation-processing-embeddable:jar:2.1.0", + sha1 = "944a8b045adf3b81f32c5843229a90d3792dc7d8", + url = "mvn:org.jetbrains.kotlin:kotlin-annotation-processing-embeddable:jar:2.2.0", visibility = ["PUBLIC"], ) third_party_jar( name = "kotlin-build-tools-api", # @oss-disable[end= ]: internal_alias = "fbsource//third-party/java/kotlincd:kotlin-build-tools-api__toolchain", - sha1 = "e6c176cdaeb5ce3751c120eb1dfd84e5cc64d369", - url = "mvn:org.jetbrains.kotlin:kotlin-build-tools-api:jar:2.1.0", + sha1 = "ceaa386345f22c861908058ffcddbeb23bcde8f4", + url = "mvn:org.jetbrains.kotlin:kotlin-build-tools-api:jar:2.2.0", visibility = [ "prelude//toolchains/android/src/com/facebook/buck/jvm/kotlin/buildtools:buildtools", "prelude//toolchains/android/src/com/facebook/buck/jvm/kotlin/buildtools/snapshot:cp_snapshot_generator_lib", @@ -945,24 +945,24 @@ third_party_jar( third_party_jar( name = "kotlin-build-tools-impl", # @oss-disable[end= ]: internal_alias = "fbsource//third-party/java/kotlincd:kotlin-build-tools-impl__toolchain", - sha1 = "a8f65725498c49ddc384d7107c3fd937c248d28d", - url = "mvn:org.jetbrains.kotlin:kotlin-build-tools-impl:jar:2.1.0", + sha1 = "14270b99947f8344e8ae36e907a05c8e92f57d85", + url = "mvn:org.jetbrains.kotlin:kotlin-build-tools-impl:jar:2.2.0", visibility = ["PUBLIC"], ) third_party_jar( name = "kotlin-daemon-client", # @oss-disable[end= ]: internal_alias = "fbsource//third-party/java/kotlincd:kotlin-daemon-client", - sha1 = "2c219a5026f6c4c1188b40cb3b6bb557f16182b7", - url = "mvn:org.jetbrains.kotlin:kotlin-daemon-client:jar:2.1.0", + sha1 = "77ce7a4cac28c2be4423f2999ff0455ca36ea36f", + url = "mvn:org.jetbrains.kotlin:kotlin-daemon-client:jar:2.2.0", visibility = ["PUBLIC"], ) third_party_jar( name = "kotlin-metadata-jvm", # @oss-disable[end= ]: internal_alias = "fbsource//third-party/java/kotlincd:kotlin-metadata-jvm__toolchain", - sha1 = "950bc99bcacf2f0a2c073e53824a3f2f2086a192", - url = "mvn:org.jetbrains.kotlin:kotlin-metadata-jvm:jar:2.1.0", + sha1 = "5f061327c06ab653e26e022c28e6d431d29205b9", + url = "mvn:org.jetbrains.kotlin:kotlin-metadata-jvm:jar:2.2.0", visibility = ["PUBLIC"], ) @@ -1043,24 +1043,24 @@ third_party_jar( third_party_jar( name = "symbol-processing-aa-embeddable", # @oss-disable[end= ]: internal_alias = "fbsource//third-party/java/kotlincd:symbol-processing-aa-embeddable__toolchain", - sha1 = "60b5adcf312a05e731ee457947c9c1b7719e14a8", - url = "mvn:com.google.devtools.ksp:symbol-processing-aa-embeddable:jar:2.1.0-1.0.29", + sha1 = "12b29cfe8a24588854a083828ac17a040f03b8ff", + url = "mvn:com.google.devtools.ksp:symbol-processing-aa-embeddable:jar:2.2.0-2.0.2", visibility = ["PUBLIC"], ) third_party_jar( name = "symbol-processing-api", # @oss-disable[end= ]: internal_alias = "fbsource//third-party/java/kotlincd:symbol-processing-api", - sha1 = "f958b777b38452f8482c6cfa797600f4f085575d", - url = "mvn:com.google.devtools.ksp:symbol-processing-api:jar:2.1.0-1.0.29", + sha1 = "6a7c41da95b72d99ee1277c80951a9160db1f1a5", + url = "mvn:com.google.devtools.ksp:symbol-processing-api:jar:2.2.0-2.0.2", visibility = ["PUBLIC"], ) third_party_jar( name = "symbol-processing-common-deps", # @oss-disable[end= ]: internal_alias = "fbsource//third-party/java/kotlincd:symbol-processing-common-deps", - sha1 = "981907d8ef3d6d20c54bef09b51b5ae4b33e2f3b", - url = "mvn:com.google.devtools.ksp:symbol-processing-common-deps:jar:2.1.0-1.0.29", + sha1 = "c15826465823bce080cc1693daef699366be0951", + url = "mvn:com.google.devtools.ksp:symbol-processing-common-deps:jar:2.2.0-2.0.2", visibility = ["PUBLIC"], ) # === End: ksp @@ -1149,6 +1149,11 @@ third_party_exe( name = "protoc-linux-x86_64", # @oss-disable[end= ]: internal_alias = "fbsource//third-party/java/com/google/protobuf/protoc:protoc-linux-x86_64", sha1 = "9065c462198f65a0f5ab4945023dac99288cf24f", + target_compatible_with = select({ + "DEFAULT": ["ovr_config//cpu:x86_64"], + "ovr_config//os:macos": [], + "ovr_config//os:windows": [], + }), url = "mvn:com.google.protobuf:protoc:exe:linux-x86_64:3.25.6", visibility = ["PUBLIC"], ) @@ -1210,21 +1215,13 @@ third_party_jar( third_party_jar( name = "zstd-jni", # @oss-disable[end= ]: internal_alias = "fbsource//third-party/java/com/github/luben/zstd-jni:zstd-jni__toolchain", - sha1 = "69cdf67db2077e677113172f5b9cb30a15cffc05", - url = "mvn:com.github.luben:zstd-jni:jar:1.5.6-9", + sha1 = "b4fea046fcd925681fa10c61be99da059f4f745a", + url = "mvn:com.github.luben:zstd-jni:jar:1.5.7-17", visibility = [ "prelude//toolchains/android/src/...", "prelude//toolchains/android/test/...", ], ) - -third_party_aar( - name = "zstd-jni-android", - # @oss-disable[end= ]: internal_alias = "fbsource//third-party/java/com/github/luben/zstd-jni:zstd-jni-android", - sha1 = "dfa697a8cf4f83e0793db1b235058c2ce4fbcf01", - url = "mvn:com.github.luben:zstd-jni:aar:1.5.6-9", - visibility = ["prelude//toolchains/android/src/com/facebook/buck/android/agent:agent"], -) # === End: zstd-jni # === Begin: xz diff --git a/prelude/toolchains/android/tools/protobuf.bzl b/prelude/toolchains/android/tools/protobuf.bzl index 9f9b6da..da2aaa7 100644 --- a/prelude/toolchains/android/tools/protobuf.bzl +++ b/prelude/toolchains/android/tools/protobuf.bzl @@ -36,6 +36,7 @@ def protobuf_src_gen(name, srcs, proto_path = [], deps = [], exported_deps = []) out = name, cmd = " ".join(command), cmd_exe = " ".join(command_exe), + exec_compatible_with = ["ovr_config//cpu:x86_64"], ) zip_rule_name = name + ".src.zip" diff --git a/prelude/toolchains/conan/defs.bzl b/prelude/toolchains/conan/defs.bzl index ef5c2c0..079f214 100644 --- a/prelude/toolchains/conan/defs.bzl +++ b/prelude/toolchains/conan/defs.bzl @@ -690,7 +690,7 @@ def _conan_profile_impl(ctx: AnalysisContext) -> list[Provider]: content.append(cmd_args(ctx.attrs.build_type, format = "build_type={}")) # TODO[AH] Auto-generate the compiler setting based on the toolchain. - # Needs a translation of CxxToolProviderType to compiler setting. + # Needs a translation of the supported C++ compiler types to compiler setting. content.append(cmd_args(ctx.attrs.compiler, format = "compiler={}")) content.append(cmd_args(ctx.attrs.compiler_version, format = "compiler.version={}")) content.append(cmd_args(ctx.attrs.compiler_libcxx, format = "compiler.libcxx={}")) diff --git a/prelude/toolchains/cxx.bzl b/prelude/toolchains/cxx.bzl index 0db4ac5..0748a2d 100644 --- a/prelude/toolchains/cxx.bzl +++ b/prelude/toolchains/cxx.bzl @@ -187,7 +187,6 @@ def _cxx_toolchain_from_cxx_tools_info(ctx: AnalysisContext, cxx_tools_info: Cxx ranlib = RunInfo(args = ["ranlib"]), strip = RunInfo(args = ["strip"]), dwp = None, - bolt_msdk = None, ), cxx_compiler_info = CxxCompilerInfo( compiler = _run_info(cxx_tools_info.cxx_compiler), @@ -244,7 +243,7 @@ system_cxx_toolchain = rule( "archiver": attrs.option(attrs.string(), default = None), "c_flags": attrs.list(attrs.arg(), default = []), "compiler": attrs.option(attrs.string(), default = None), - "compiler_type": attrs.option(attrs.string(), default = None), # one of CxxToolProviderType + "compiler_type": attrs.option(attrs.string(), default = None), # One of the supported C++ compiler types. "cpp_dep_tracking_mode": attrs.string(default = "makefile"), "cvtres_compiler": attrs.option(attrs.string(), default = None), "cvtres_flags": attrs.list(attrs.arg(), default = []), diff --git a/prelude/toolchains/cxx/zig/defs.bzl b/prelude/toolchains/cxx/zig/defs.bzl index 8c3ef19..8cf3925 100644 --- a/prelude/toolchains/cxx/zig/defs.bzl +++ b/prelude/toolchains/cxx/zig/defs.bzl @@ -406,7 +406,6 @@ def _cxx_zig_toolchain_impl(ctx: AnalysisContext) -> list[Provider]: is_pdb_generated = is_pdb_generated(_get_linker_type(dist.os), ctx.attrs.linker_flags), ), binary_utilities_info = BinaryUtilitiesInfo( - bolt_msdk = None, dwp = None, nm = RunInfo(args = ["nm"]), # not included in the zig distribution. objcopy = RunInfo(args = ["objcopy"]), # not included in the zig distribution. diff --git a/prelude/toolchains/demo.bzl b/prelude/toolchains/demo.bzl index d579767..524cf76 100644 --- a/prelude/toolchains/demo.bzl +++ b/prelude/toolchains/demo.bzl @@ -8,7 +8,7 @@ load("@prelude//android/tools:jdk_system_image.bzl", "jdk_system_image") load("@prelude//tests:test_toolchain.bzl", "noop_test_toolchain") -load("@prelude//toolchains:android.bzl", "android_sdk_tools", "system_android_toolchain") +load("@prelude//toolchains:android.bzl", "android_sdk_tools", "system_android_aidl_toolchain", "system_android_toolchain") load("@prelude//toolchains:cxx.bzl", "system_cxx_toolchain") load("@prelude//toolchains:dex.bzl", "system_dex_toolchain", "system_noop_dex_toolchain") load("@prelude//toolchains:erlang.bzl", "system_erlang_toolchain") @@ -25,7 +25,12 @@ load( ) load("@prelude//toolchains:kotlin.bzl", "kotlincd_toolchain", "system_kotlin_bootstrap_toolchain") load("@prelude//toolchains:ocaml.bzl", "system_ocaml_toolchain") -load("@prelude//toolchains:python.bzl", "remote_python_toolchain", "system_python_wheel_toolchain") +load( + "@prelude//toolchains:python.bzl", + "remote_python_toolchain", + "system_python_bootstrap_toolchain", + "system_python_wheel_toolchain", +) load("@prelude//toolchains:remote_test_execution.bzl", "remote_test_execution_toolchain") load("@prelude//toolchains:rust.bzl", "system_rust_toolchain") load("@prelude//toolchains:zip_file.bzl", "zip_file_toolchain") @@ -52,6 +57,12 @@ def system_demo_toolchains(): visibility = ["PUBLIC"], ) + system_android_aidl_toolchain( + name = "android_aidl", + android_sdk_tools_target = ":android_sdk_tools", + visibility = ["PUBLIC"], + ) + jdk_system_image( name = "jdk_system_image", core_for_system_modules_jar = ":android_sdk_tools[core-for-system-modules.jar]", @@ -230,6 +241,12 @@ def system_demo_toolchains(): remote_python_toolchain( name = "python", + bootstrap = False, + visibility = ["PUBLIC"], + ) + + system_python_bootstrap_toolchain( + name = "python_bootstrap", visibility = ["PUBLIC"], ) diff --git a/prelude/toolchains/erlang.bzl b/prelude/toolchains/erlang.bzl index 468708c..9bbc52d 100644 --- a/prelude/toolchains/erlang.bzl +++ b/prelude/toolchains/erlang.bzl @@ -6,6 +6,7 @@ # of this source tree. You may select, at your option, one of the # above-listed licenses. +load("@prelude//erlang:erlang_erts.bzl", "erlang_erts") load("@prelude//erlang:erlang_info.bzl", "ErlangOTPBinariesInfo") load("@prelude//erlang:erlang_toolchain.bzl", "erlang_toolchain") @@ -31,8 +32,15 @@ def system_erlang_toolchain(name, visibility): visibility = visibility, ) + erlang_erts( + name = "{}-erts".format(name), + otp_binaries = ":{}-binaries".format(name), + visibility = visibility, + ) + erlang_toolchain( name = name, + erts_toolchain_info = ":{}-erts".format(name), otp_binaries = ":{}-binaries".format(name), parse_transforms = [], parse_transforms_filters = {}, diff --git a/prelude/toolchains/go/go_toolchain.bzl b/prelude/toolchains/go/go_toolchain.bzl index dd7f657..a7df391 100644 --- a/prelude/toolchains/go/go_toolchain.bzl +++ b/prelude/toolchains/go/go_toolchain.bzl @@ -12,23 +12,27 @@ def _go_toolchain_impl(ctx): # Note: It makes sense to make GoDirstrInfo an attribute of GoToolchainInfo. # That's a breaking change, so we'll need to notify oss users. go_distr = ctx.attrs.go_distr[GoDistrInfo] + + # `buck2 run` executes the `go` subtarget on the invoking machine, so it is + # only offered where the target-configured pair exists; the exec-configured + # pair is built for wherever build actions run. + sub_targets = {} + if ctx.attrs.run_go_distr != None: + run_go_distr = ctx.attrs.run_go_distr[GoDistrInfo] + sub_targets["go"] = [ + RunInfo( + cmd_args( + ctx.attrs.run_go_wrapper[RunInfo], + ["--go", run_go_distr.bin_go], + ["--goroot", run_go_distr.go_root], + ["--default-goarch", ctx.attrs.env_go_arch], + ["--default-goos", ctx.attrs.env_go_os], + "--", + ) + ), + ] return [ - DefaultInfo( - sub_targets = { - "go": [ - RunInfo( - cmd_args( - ctx.attrs.go_wrapper[RunInfo], - ["--go", go_distr.bin_go], - ["--goroot", go_distr.go_root], - ["--default-goarch", ctx.attrs.env_go_arch], - ["--default-goos", ctx.attrs.env_go_os], - "--", - ) - ), - ] - }, - ), + DefaultInfo(sub_targets = sub_targets), GoToolchainInfo( allow_cache_upload = ctx.attrs.allow_cache_upload, assembler = go_distr.tool_asm, @@ -60,7 +64,7 @@ def _go_toolchain_impl(ctx): ), ] -go_toolchain = rule( +_go_toolchain = rule( impl = _go_toolchain_impl, is_toolchain_rule = True, attrs = { @@ -79,14 +83,38 @@ go_toolchain = rule( "fuzz": attrs.bool(default = False), "gen_embedcfg": attrs.exec_dep(providers = [RunInfo], default = "prelude//go/tools:gen_embedcfg"), "go_distr": attrs.exec_dep(providers = [GoDistrInfo]), - "go_wrapper": attrs.exec_dep(providers = [RunInfo], default = "prelude//go/tools:go_wrapper"), + "go_wrapper": attrs.exec_dep(providers = [RunInfo]), "linker_flags": attrs.list(attrs.arg(), default = []), "pkg_analyzer": attrs.exec_dep(providers = [RunInfo], default = "prelude//go/tools:pkg_analyzer"), "race": attrs.bool(default = False), + # Target-configured duals of `go_distr` and `go_wrapper` for the `go` + # subtarget, which `buck2 run` executes on the invoking machine rather + # than on the execution platform. + "run_go_distr": attrs.option(attrs.dep(providers = [GoDistrInfo])), + "run_go_wrapper": attrs.option(attrs.dep(providers = [RunInfo])), "tool_pack": attrs.exec_dep(providers = [RunInfo], default = "prelude//go/tools:tool_pack"), }, ) +def go_toolchain(go_distr, go_wrapper = "prelude//go/tools:go_wrapper", run_go = True, **kwargs): + """ + Go toolchain backed by a `go_distr`. + + Build actions use `go_distr` and `go_wrapper` configured for the execution + platform. The `go` subtarget (`buck2 run [go] -- ...`) runs on + the invoking machine, so it uses them configured for the target platform. + `run_go` (a bool or a `select` of bools) opts target platforms out of the + subtarget, for a `go_distr` that is not compatible with every target + platform the toolchain supports. + """ + _go_toolchain( + go_distr = go_distr, + go_wrapper = go_wrapper, + run_go_distr = select_map(run_go, lambda enabled: go_distr if enabled else None), + run_go_wrapper = select_map(run_go, lambda enabled: go_wrapper if enabled else None), + **kwargs, + ) + def _go_distr_impl(ctx): go_root = ctx.attrs.go_root go_os, go_arch = ctx.attrs.go_os_arch diff --git a/prelude/toolchains/java.bzl b/prelude/toolchains/java.bzl index d7d8ad8..3aaeb3e 100644 --- a/prelude/toolchains/java.bzl +++ b/prelude/toolchains/java.bzl @@ -62,7 +62,8 @@ def _prebuilt_jar_toolchain_rule_impl(ctx): _prebuilt_jar_toolchain_rule = rule( attrs = { - "java": attrs.dep(), + # this just goes into the RunInfo of the result, so it's target, not exec + "java": attrs.dep(providers = [RunInfo]), }, impl = _prebuilt_jar_toolchain_rule_impl, is_toolchain_rule = True, @@ -154,10 +155,16 @@ _java_toolchain = rule( impl = _java_toolchain_impl, is_toolchain_rule = True, attrs = { - "class_abi_generator": attrs.option(attrs.dep(providers = [RunInfo]), default = None), + "class_abi_generator": attrs.option(attrs.exec_dep(providers = [RunInfo]), default = None), "class_loader_bootstrapper": attrs.option(attrs.source(), default = None), - "compile_and_package": attrs.dep(default = "prelude//java/tools:compile_and_package"), - "fat_jar": attrs.dep(default = "prelude//java/tools:fat_jar"), + "compile_and_package": attrs.exec_dep( + default = "prelude//java/tools:compile_and_package", + providers = [RunInfo], + ), + "fat_jar": attrs.exec_dep( + default = "prelude//java/tools:fat_jar", + providers = [RunInfo], + ), "fat_jar_main_class_lib": attrs.option(attrs.source(), default = None), "gen_class_to_source_map": attrs.exec_dep( default = "prelude//java/tools:gen_class_to_source_map", @@ -170,16 +177,16 @@ _java_toolchain = rule( ], ), "is_bootstrap_toolchain": attrs.bool(default = False), - "jar": attrs.option(attrs.dep(providers = [RunInfo]), default = None), + "jar": attrs.option(attrs.exec_dep(providers = [RunInfo]), default = None), "jar_builder": attrs.source(default = "prelude//toolchains/android/src/com/facebook/buck/util/zip:jar_builder"), - "java": attrs.exec_dep(), - "java_for_tests": attrs.option(attrs.dep(providers = [RunInfo]), default = None), - "javac": attrs.option(attrs.one_of(attrs.dep(), attrs.source(), attrs.string()), default = None), + "java": attrs.exec_dep(providers = [RunInfo]), + "java_for_tests": attrs.option(attrs.exec_dep(providers = [RunInfo]), default = None), + "javac": attrs.option(attrs.one_of(attrs.exec_dep(), attrs.source(), attrs.string()), default = None), "javac_protocol": attrs.enum(JavacProtocol.values()), "javacd": attrs.option(attrs.source(), default = None), "javacd_main_class": attrs.option(attrs.string(), default = None), - "jlink": attrs.exec_dep(), - "jmod": attrs.exec_dep(), + "jlink": attrs.exec_dep(providers = [RunInfo]), + "jmod": attrs.exec_dep(providers = [RunInfo]), "jrt_fs_jar": attrs.source(), "merge_class_to_source_maps": attrs.exec_dep( default = "prelude//java/tools:merge_class_to_source_maps", @@ -225,8 +232,8 @@ _java_test_toolchain_rule = rule( attrs.string(), default = [], ), - "list_class_names": attrs.dep(providers = [RunInfo]), - "list_tests": attrs.option(attrs.dep(providers = [RunInfo]), default = None), + "list_class_names": attrs.exec_dep(providers = [RunInfo]), + "list_tests": attrs.option(attrs.exec_dep(providers = [RunInfo]), default = None), "test_runner_library_jar": attrs.source(), "testng_test_runner_main_class_args": attrs.list(attrs.string()), }, diff --git a/prelude/toolchains/kotlin.bzl b/prelude/toolchains/kotlin.bzl index 246977c..75360b9 100644 --- a/prelude/toolchains/kotlin.bzl +++ b/prelude/toolchains/kotlin.bzl @@ -19,7 +19,7 @@ def kotlincd_toolchain(name, java_binary_for_kotlincd = None, visibility = None) dep_files = "none", kapt_base64_encoder = "prelude//kotlin/tools/kapt_base64_encoder:kapt_base64_encoder", kotlin_stdlib = "prelude//toolchains/android/third-party:kotlin-stdlib", - kotlin_version = "2.0.0", + kotlin_version = "2.2.0", kotlin_home_libraries = [ "prelude//toolchains/android/third-party:kotlin-annotations", "prelude//toolchains/android/third-party:kotlin-build-tools-impl", @@ -32,7 +32,7 @@ def kotlincd_toolchain(name, java_binary_for_kotlincd = None, visibility = None) "prelude//toolchains/android/third-party:kotlinx-coroutines-core-jvm", ], kotlinc = "prelude//toolchains/android/third-party:kotlin-compiler-binary", - kotlincd = "prelude//toolchains/android/src/com/facebook/buck/jvm/kotlin/cd/workertool:kotlincd_tool", + kotlincd = "prelude//toolchains/android/src/com/facebook/buck/jvm/kotlin/cd/workertool:kotlincd_tool-main", kotlinc_protocol = "kotlincd", kotlincd_main_class = "com.facebook.buck.jvm.kotlin.cd.workertool.KotlinCDMain", visibility = visibility, diff --git a/prelude/toolchains/ocaml.bzl b/prelude/toolchains/ocaml.bzl index b141a82..1e1eff3 100644 --- a/prelude/toolchains/ocaml.bzl +++ b/prelude/toolchains/ocaml.bzl @@ -11,12 +11,24 @@ load( "OCamlPlatformInfo", "OCamlToolchainInfo", ) +load("@prelude//os_lookup:defs.bzl", "Os", "OsLookup") -def _system_ocaml_toolchain_impl(_ctx): +_EXEC_OS_TYPE = attrs.default_only(attrs.exec_dep(default = "prelude//os_lookup/targets:os_lookup")) + +def _system_ocaml_toolchain_impl(ctx): """ A very simple toolchain that is hardcoded to the current environment. """ + runtime_dep_link_flags = ["-ldl", "-lpthread"] + exec_os = ctx.attrs._exec_os_type[OsLookup].os + if exec_os == Os("macos"): + runtime_dep_link_flags.extend(["-L/opt/homebrew/lib", "-lzstd"]) + elif exec_os == Os("linux"): + # OCaml 5.3's runtime archive uses zstd for marshalled data. Linux + # finds the system library without an explicit -L flag. + runtime_dep_link_flags.append("-lzstd") + return [ DefaultInfo(), OCamlToolchainInfo( @@ -46,10 +58,7 @@ def _system_ocaml_toolchain_impl(_ctx): ocaml_compiler_flags = [], # e.g. "-opaque" ocamlc_flags = [], ocamlopt_flags = [], - # We don't expect /opt/homebrew/lib to exist on Linux but that's not - # a problem. On macOS (aarch64 at least) we expect zstd to live in - # /opt/homebrew/lib. - runtime_dep_link_flags = ["-ldl", "-lpthread", "-L/opt/homebrew/lib", "-lzstd"], + runtime_dep_link_flags = runtime_dep_link_flags, runtime_dep_link_extras = [], ), OCamlPlatformInfo(name = "x86_64"), @@ -57,6 +66,8 @@ def _system_ocaml_toolchain_impl(_ctx): system_ocaml_toolchain = rule( impl = _system_ocaml_toolchain_impl, - attrs = {}, + attrs = { + "_exec_os_type": _EXEC_OS_TYPE, + }, is_toolchain_rule = True, ) diff --git a/prelude/toolchains/python.bzl b/prelude/toolchains/python.bzl index 8d91e34..572b4da 100644 --- a/prelude/toolchains/python.bzl +++ b/prelude/toolchains/python.bzl @@ -7,6 +7,11 @@ # above-listed licenses. load("@prelude//:prelude.bzl", "native") +load( + "@prelude//os_lookup:defs.bzl", + "Os", + "OsLookup", +) load( "@prelude//python:python_wheel_toolchain.bzl", "PythonWheelToolchainInfo", @@ -46,13 +51,25 @@ _INTERPRETER = select({ "config//os:windows": "python.exe", }) -def _python_bootstrap_toolchain_impl(ctx): - interpreter = ctx.attrs.interpreter if isinstance(ctx.attrs.interpreter, str) else ctx.attrs.interpreter[RunInfo].args +_EXEC_OS_TYPE = attrs.default_only(attrs.exec_dep(default = "prelude//os_lookup/targets:os_lookup")) + +def _interpreter_for_os(os: Os) -> str: + return "python.exe" if os == Os("windows") else "python3" + +def _system_python_bootstrap_toolchain_impl(ctx): + # the interpreter name needs to be selected based on the exec platform, not the target + interpreter = ctx.attrs.interpreter or _interpreter_for_os(ctx.attrs._exec_os_type[OsLookup].os) return [ DefaultInfo(), PythonBootstrapToolchainInfo(interpreter = interpreter), ] +def _python_bootstrap_toolchain_impl(ctx): + return [ + DefaultInfo(), + PythonBootstrapToolchainInfo(interpreter = ctx.attrs.interpreter[RunInfo].args), + ] + # Creates a new bootstrap toolchain using Python that is installed on your system. # You may use it in your toolchain cell as follows: # @@ -65,9 +82,10 @@ def _python_bootstrap_toolchain_impl(ctx): # ) # ``` system_python_bootstrap_toolchain = rule( - impl = _python_bootstrap_toolchain_impl, + impl = _system_python_bootstrap_toolchain_impl, attrs = { - "interpreter": attrs.string(default = _INTERPRETER), + "interpreter": attrs.option(attrs.string(), default = None), + "_exec_os_type": _EXEC_OS_TYPE, }, is_toolchain_rule = True, ) @@ -90,12 +108,15 @@ def _system_python_toolchain_impl(ctx): A very simple toolchain that is hardcoded to the current environment. """ + # the "host" (exec platform) interpreter needs to be selected based on the exec platform + host_interpreter = ctx.attrs.host_interpreter or _interpreter_for_os(ctx.attrs._exec_os_type[OsLookup].os) + return [ DefaultInfo(), PythonToolchainInfo( binary_linker_flags = ctx.attrs.binary_linker_flags, linker_flags = ctx.attrs.linker_flags, - host_interpreter = RunInfo(args = [ctx.attrs.interpreter]), + host_interpreter = RunInfo(args = [host_interpreter]), interpreter = RunInfo(args = [ctx.attrs.interpreter]), compile = RunInfo(args = ["echo", "COMPILEINFO"]), package_style = "inplace", @@ -110,6 +131,11 @@ system_python_toolchain = rule( impl = _system_python_toolchain_impl, attrs = { "binary_linker_flags": attrs.default_only(attrs.list(attrs.arg(), default = [])), + "host_interpreter": attrs.option( + attrs.string(), + default = None, + doc = "Interpreter to run actions on the exec platform. Defaults to the usual interpreter name for the exec platform.", + ), "interpreter": attrs.string(default = _INTERPRETER), "linker_flags": attrs.default_only(attrs.list(attrs.arg(), default = [])), "pex_extension": attrs.string(default = ".pex"), @@ -118,6 +144,7 @@ system_python_toolchain = rule( default = None, doc = "See `PythonToolchainInfo.type_checker` for the executable contract.", ), + "_exec_os_type": _EXEC_OS_TYPE, }, is_toolchain_rule = True, ) @@ -127,7 +154,7 @@ def python_toolchain_impl(ctx) -> list[Provider]: DefaultInfo(), PythonToolchainInfo( interpreter = ctx.attrs.interpreter[RunInfo], - host_interpreter = ctx.attrs.interpreter[RunInfo], + host_interpreter = ctx.attrs.host_interpreter[RunInfo], compile = ctx.attrs.compile[DefaultInfo].default_outputs[0], package_style = "inplace", native_link_strategy = "separate", @@ -139,11 +166,19 @@ def python_toolchain_impl(ctx) -> list[Provider]: PythonPlatformInfo(name = "x86_64"), ] -python_toolchain = rule( +_python_toolchain = rule( impl = python_toolchain_impl, attrs = { - "compile": attrs.default_only(attrs.dep(default = "prelude//python/tools:compile.py")), + "compile": attrs.default_only(attrs.exec_dep(default = "prelude//python/tools:compile.py")), "extension_linker_flags": attrs.list(attrs.arg()), + "host_interpreter": attrs.exec_dep( + providers = [RunInfo], + doc = """ + Interpreter used to run build-time actions, such as compiling bytecode. Typically the + same label as `interpreter` (as defaulted by the python_toolchain wrapper macro), but + uses a separate attr to transition to exec configuration. + """, + ), "interpreter": attrs.dep(providers = [RunInfo]), "type_checker": attrs.option( attrs.exec_dep(providers = [RunInfo]), @@ -155,6 +190,14 @@ python_toolchain = rule( doc = "A Python toolchain that can build Python extensions, given an interpreter and the extra linker flags to use with it. See `remote_python_toolchain` for a toolchain that configures the interpreter and linker flags for you.", ) +def python_toolchain(name: str, interpreter, host_interpreter = None, **kwargs) -> None: + """ + A Python toolchain that can build Python extensions, given an interpreter and + the extra linker flags to use with it. See `remote_python_toolchain` for a + toolchain that configures the interpreter and linker flags for you. + """ + _python_toolchain(name = name, interpreter = interpreter, host_interpreter = interpreter if host_interpreter == None else host_interpreter, **kwargs) + # archives for 3.13 # update this by running `prelude//python/tools:discover_python_archives.sh` CPYTHON_ARCHIVE = { diff --git a/prelude/transitions/constraint_override_registry.bzl b/prelude/transitions/constraint_override_registry.bzl new file mode 100644 index 0000000..b9bbd37 --- /dev/null +++ b/prelude/transitions/constraint_override_registry.bzl @@ -0,0 +1,24 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is dual-licensed under either the MIT license found in the +# LICENSE-MIT file in the root directory of this source tree or the Apache +# License, Version 2.0 found in the LICENSE-APACHE file in the root directory +# of this source tree. You may select, at your option, one of the +# above-listed licenses. + +ConstraintOverrideRegistryInfo = provider( + fields = {"refs": provider_field(struct)}, +) + +def constraint_override_refs() -> dict[str, str]: + overrides = read_root_config("buck2", "platforms", "") + "," + read_root_config("buck2", "constraints", "") + return {override.strip(): override.strip() for override in overrides.split(",") if override.strip()} + +def _constraint_override_registry_impl(ctx: AnalysisContext) -> list[Provider]: + return [DefaultInfo(), ConstraintOverrideRegistryInfo(refs = struct(**ctx.attrs.refs))] + +constraint_override_registry = rule( + impl = _constraint_override_registry_impl, + attrs = {"refs": attrs.dict(attrs.string(), attrs.dep())}, + is_configuration_rule = True, +) diff --git a/prelude/transitions/constraint_overrides.bzl b/prelude/transitions/constraint_overrides.bzl index 261c36e..c8ce4fa 100644 --- a/prelude/transitions/constraint_overrides.bzl +++ b/prelude/transitions/constraint_overrides.bzl @@ -25,20 +25,13 @@ load("@prelude//cfg/modifier:name.bzl", "cfg_name") # applying the configuration transition. The use of read_config avoids hard-coding these # repo-specific configuration rules into the prelude. load("@prelude//python:transitions.bzl", "python_transitions") +load("@prelude//transitions:constraint_override_registry.bzl", "ConstraintOverrideRegistryInfo", "constraint_override_refs") _config = struct( - platforms = read_root_config("buck2", "platforms", ""), - constraints = read_root_config("buck2", "constraints", ""), passthrough_constraints = read_root_config("buck2", "passthrough_constraints", ""), split = lambda values: [value.strip() for value in values.split(",") if value.strip()], ) -def _platforms() -> list[str]: - return _config.split(_config.platforms) - -def _constraints() -> list[str]: - return _config.split(_config.constraints) - def _passthrough_constraints() -> list[str]: return _config.split(_config.passthrough_constraints) @@ -185,6 +178,8 @@ def _apply(old_platform: PlatformInfo, *, platform: PlatformInfo | None = None, return new_platform def _impl(platform: PlatformInfo, refs: struct, attrs: struct) -> PlatformInfo: + if hasattr(refs, "_constraint_override_registry"): + refs = refs._constraint_override_registry[ConstraintOverrideRegistryInfo].refs args = _resolve(refs, attrs) if args["platform"] == None and not args["constraints"]: return platform @@ -197,8 +192,11 @@ def _python_impl(platform: PlatformInfo, refs: struct, attrs: struct) -> Platfor # @oss-disable[end= ]: platform = python_transitions.transition_opt_by_default_impl(platform, refs, attrs) return _impl(platform, refs, attrs) -_refs = {override: override for override in _platforms() + _constraints()} -_python_refs = {k: v for k, v in _refs.items()} +_refs = constraint_override_refs() +# Registry dependencies enforce visibility, unlike direct transition refs. +_registry = read_root_config("buck2", "constraint_override_registry", "") +_transition_refs = {"_constraint_override_registry": _registry} if _registry else _refs +_python_refs = {k: v for k, v in _transition_refs.items()} # @oss-disable[end= ]: _python_refs.update(python_transitions.refs()) _attributes = { @@ -211,7 +209,7 @@ _python_attributes = {k: v for k, v in _attributes.items()} _transition = transition( impl = _impl, - refs = _refs, + refs = _transition_refs, attrs = _attributes.keys(), ) diff --git a/prelude/utils/argfile.bzl b/prelude/utils/argfile.bzl index 21ba5e6..a5851d4 100644 --- a/prelude/utils/argfile.bzl +++ b/prelude/utils/argfile.bzl @@ -6,6 +6,8 @@ # of this source tree. You may select, at your option, one of the # above-listed licenses. +load("@prelude//utils:arglike.bzl", "ArgLike") + # Create an argument file. # Return `cmd_args` which is single string containing `@path/to/argfile`. # Returned `cmd_args` contains given files as hidden artifacts. @@ -27,6 +29,29 @@ def at_argfile( args_file = actions.write(name, args, with_inputs = True, has_content_based_path = has_content_based_path) return cmd_args(args_file, format = "@{}", hidden = args) +# An argsfile that renders as its path and also carries the artifacts +# referenced by `args`, the flags written into it. +# +# TODO(jtbraun): if "associated artifacts" ever become inheritable/transitive, +# attaching `args` to `argsfile` would be a better solution. +def argsfile_with_artifacts(argsfile: Artifact, args: ArgLike) -> cmd_args: + return cmd_args(argsfile, hidden = args) + +# Write arguments to a file and return an `argsfile_with_artifacts()` cmd_args +# value pairing the file with everything its contents reference. +def mk_argsfile( + *, + actions: AnalysisActions, + name: str | Artifact, + args: ArgLike, + has_content_based_path: bool = False, +) -> cmd_args: + # Write actions may spill additional file artifacts out to the side for + # certain string macros, which are returned in `macro_files`. These have to be + # carried forward to consumers, just as `args` is. + argsfile, macro_files = actions.write(name, args, allow_args = True, has_content_based_path = has_content_based_path) + return argsfile_with_artifacts(argsfile, [args] + macro_files) + # Write arguments to a file, and return the file path as `cmd_args` # with args attached as hidden artifacts. def argfile( diff --git a/prelude/xplugins/debug_artifacts.bzl b/prelude/xplugins/debug_artifacts.bzl index dc1a480..b022b8c 100644 --- a/prelude/xplugins/debug_artifacts.bzl +++ b/prelude/xplugins/debug_artifacts.bzl @@ -11,6 +11,7 @@ load( "XPluginsDebugArtifactsEntry", "XPluginsDebugArtifactsInfo", "XPluginsDebugArtifactsTSet", + "XPluginsFunctionMappingManifestInfo", "XPluginsManifestInfo", ) @@ -39,42 +40,51 @@ def xplugins_get_debug_artifacts_info(ctx: AnalysisContext, deps: list[Dependenc return XPluginsDebugArtifactsInfo(tset = tset) return None -def xplugins_get_debug_artifacts_subtargets(actions: AnalysisActions, info: XPluginsDebugArtifactsInfo | None) -> list[Provider]: +def xplugins_get_function_mapping_manifest_info( + actions: AnalysisActions, app_target: Label | None, info: XPluginsDebugArtifactsInfo | None +) -> XPluginsFunctionMappingManifestInfo: if not info: info = XPluginsDebugArtifactsInfo(tset = actions.tset(XPluginsDebugArtifactsTSet)) - dir_contents = {} - manifest = {} + function_mappings = [] + function_mapping_artifacts = [] + function_mapping_targets = [] - name_counts = {} for entry in info.tset.traverse(): if entry: - raw_target = str(entry.target.raw_target()) - name = entry.target.name - count = name_counts.get(name, 0) - name_counts[name] = count + 1 - filename = "{}.json".format(name) if count == 0 else "{}{}.json".format(name, count) - dir_contents[filename] = entry.manifest_info.function_mapping - if raw_target not in manifest: - manifest[raw_target] = [] - manifest[raw_target].append({ - "configured_target": entry.target, - "path": filename, + function_mapping_artifacts.append(entry.manifest_info.function_mapping) + function_mapping_targets.append(entry.target) + function_mappings.append({ + "path": entry.manifest_info.function_mapping, + "target": entry.target, }) - manifest_file = actions.write_json("manifest.json", manifest, pretty = True, has_content_based_path = False) - dir_contents["MANIFEST.json"] = manifest_file + if app_target != None and app_target not in function_mapping_targets: + fail("App target {} is not listed in the XPlugins function mappings: {}".format(app_target, function_mapping_targets)) - directory = actions.copied_dir( - "XPluginsFunctionMappings", - dir_contents, + function_mapping_manifest_file = actions.write_json( + "function_mapping_manifest.json", + { + "app_target": app_target, + "mappings": function_mappings, + }, has_content_based_path = False, + pretty = True, + ).with_associated_artifacts(function_mapping_artifacts) + + return XPluginsFunctionMappingManifestInfo( + manifest = function_mapping_manifest_file, ) +def xplugins_get_debug_artifacts_subtargets(info: XPluginsFunctionMappingManifestInfo) -> list[Provider]: return [ DefaultInfo( sub_targets = { - "function_mappings": [DefaultInfo(default_output = directory)], + "function_mapping_manifest": [ + DefaultInfo( + default_output = info.manifest, + ), + ], }, ), ] diff --git a/prelude/xplugins/types.bzl b/prelude/xplugins/types.bzl index ab664ad..fa8b317 100644 --- a/prelude/xplugins/types.bzl +++ b/prelude/xplugins/types.bzl @@ -26,6 +26,12 @@ XPluginsDebugArtifactsInfo = provider( } ) +XPluginsFunctionMappingManifestInfo = provider( + fields = { + "manifest": provider_field(Artifact), + } +) + XPluginsPluginUsageInfo = provider( fields = { "target": provider_field(Label), diff --git a/prelude/xplugins/utils.bzl b/prelude/xplugins/utils.bzl index c35d442..c400ede 100644 --- a/prelude/xplugins/utils.bzl +++ b/prelude/xplugins/utils.bzl @@ -6,8 +6,6 @@ # of this source tree. You may select, at your option, one of the # above-listed licenses. -load("@prelude//cxx:link_groups_types.bzl", "LinkGroupInfo") -load("@prelude//utils:arglike.bzl", "ArgLike") load( ":types.bzl", "XPluginsPluginUsageInfo", @@ -38,103 +36,3 @@ def get_xplugins_usage_info(actions: AnalysisActions, all_deps: list[Dependency] ) else: return None - -def _filter_plugins_artifacts( - actions: AnalysisActions, link_group: str, link_group_info: LinkGroupInfo, out: OutputArtifact, usage_info_tset: XPluginsUsageInfoSet -) -> list[Provider]: - mappings = link_group_info.mappings - usage_infos = [] - for usage_info_list in usage_info_tset.traverse(): - for usage_info in usage_info_list: - # We use empty string to represent not in a link group, which - # generally means part of the main binary. - dep_link_group = mappings.get(usage_info.target, "") - if link_group == dep_link_group: - usage_infos.append(usage_info.usage_info) - - filelist = actions.write( - out, - usage_infos, - with_inputs = True, - ) - - return [DefaultInfo(filelist)] - -_xplugins_manifest_rule = dynamic_actions( - impl = _filter_plugins_artifacts, - attrs = { - "link_group": dynattrs.value(str), - "link_group_info": dynattrs.value(LinkGroupInfo), - "out": dynattrs.output(), - "usage_info_tset": dynattrs.value(XPluginsUsageInfoSet), - }, -) - -_ManifestInfo = record( - argsfile = field(Artifact), - unfiltered_argsfile = field(Artifact), - manifests = field(ArgLike), -) - -def _process_manifest(ctx, kind, info_tset, link_group_info): - """Process a single manifest kind (plugin or socket), returning a _ManifestInfo record.""" - argsfile = ctx.actions.declare_output("xplugins/{}_{}_usage_info.argsfile".format(ctx.label.name, kind), has_content_based_path = False) - manifests = info_tset.project_as_args("artifacts") - - if link_group_info: - ctx.actions.dynamic_output_new( - _xplugins_manifest_rule( - link_group = ctx.attrs.link_group or "", - link_group_info = link_group_info, - out = argsfile.as_output(), - usage_info_tset = info_tset, - ), - ) - unfiltered_argsfile = ctx.actions.write( - "unfiltered_{}_usage_info.argsfile".format(kind), - manifests, - has_content_based_path = False, - ) - else: - unfiltered_argsfile = argsfile - ctx.actions.write( - argsfile.as_output(), - manifests, - ) - - return _ManifestInfo( - argsfile = argsfile, - unfiltered_argsfile = unfiltered_argsfile, - manifests = manifests, - ) - -def _make_manifest_providers(info): - """Build the provider list for a manifest subtarget.""" - return [ - DefaultInfo( - default_output = info.argsfile, - other_outputs = [info.manifests], - sub_targets = { - "unfiltered": [ - DefaultInfo( - default_output = info.unfiltered_argsfile, - other_outputs = [info.manifests], - ) - ], - }, - ), - ] - -def get_xplugins_usage_subtargets( - ctx: AnalysisContext, usage_info: XPluginsUsageInfo | None, link_group_info: LinkGroupInfo | None -) -> dict[str, list[Provider]]: - if not usage_info: - return {} - - plugin = _process_manifest(ctx, "plugin", usage_info.plugin_info_tset, link_group_info) - socket = _process_manifest(ctx, "socket", usage_info.socket_info_tset, link_group_info) - - return { - "xplugins-plugin-manifests": _make_manifest_providers(plugin), - "xplugins-socket-manifests": _make_manifest_providers(socket), - }