diff --git a/Taskfile.yml b/Taskfile.yml index 686c9728f3..c04abc1712 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -47,19 +47,30 @@ tasks: msg: "Couldn't locate decompiler executable in '{{.DECOMP_BIN_RELEASE_DIR}}/decompiler'" cmds: - '{{.DECOMP_BIN_RELEASE_DIR}}/decompiler "./decompiler/config/{{.DECOMP_CONFIG}}" "./iso_data" "./decompiler_out" --version "{{.DECOMP_CONFIG_VERSION}}" --config-override ''{"decompile_code": false, "levels_extract": true, "allowed_objects": []}''' + compile-game-if-needed: + internal: true + cmds: + - | + if ! test -f out/{{.GAME}}/iso/KERNEL.CGO; then + echo "Game is not compiled yet ('out/{{.GAME}}/iso/KERNEL.CGO' is missing) - running (mi) in 1 seconds..." + {{.PYTHON}} -c "import time; time.sleep(1)" + {{.GOALC_BIN_RELEASE_DIR}}/goalc{{.EXE_FILE_EXTENSION}} --user-auto --game {{.GAME}} --cmd "(mi)" + fi boot-game: - desc: "Boots the game, it will fail if it's not already compiled!" + desc: "Boots the game, compiling it first if it's not compiled yet" preconditions: - sh: test -f {{.GK_BIN_RELEASE_DIR}}/gk{{.EXE_FILE_EXTENSION}} msg: "Couldn't locate runtime executable in '{{.GK_BIN_RELEASE_DIR}}/gk'" cmds: + - task: compile-game-if-needed - "{{.GK_BIN_RELEASE_DIR}}/gk -v --game {{.GAME}} -- -boot -fakeiso -debug" boot-game-retail: - desc: "Boots the game without debug mode, it will fail if it's not already compiled!" + desc: "Boots the game without debug mode, compiling it first if it's not compiled yet" preconditions: - sh: test -f {{.GK_BIN_RELEASE_DIR}}/gk{{.EXE_FILE_EXTENSION}} msg: "Couldn't locate runtime executable in '{{.GK_BIN_RELEASE_DIR}}/gk'" cmds: + - task: compile-game-if-needed - "{{.GK_BIN_RELEASE_DIR}}/gk -v --game {{.GAME}} -- -boot -fakeiso" run-game: desc: "Start the game's runtime, to start the game itself the REPL is required" diff --git a/common/util/FileUtil.cpp b/common/util/FileUtil.cpp index 9821c04c72..994a2acf53 100644 --- a/common/util/FileUtil.cpp +++ b/common/util/FileUtil.cpp @@ -278,7 +278,24 @@ fs::path get_iso_dir_for_game(GameVersion game_version) { expected_subdir = "jak3"; } const auto temp_dir = get_jak_project_dir() / "iso_data" / expected_subdir; - if (fs::exists(temp_dir)) { + if (fs::exists(temp_dir / "DGO")) { + g_iso_data_directory = temp_dir; + return g_iso_data_directory; + } + // no DGO folder means no game data was ever extracted to the default iso_data folder - + // fall back to the JAK_ISO_DATA_DIR environment variable if it is set + const auto env_iso_data = get_env("JAK_ISO_DATA_DIR"); + if (!env_iso_data.empty()) { + fs::path env_path = env_iso_data; + // the env var may point at the iso_data root (containing jak1/ etc.) or directly at the + // extracted game data + if (fs::exists(env_path / expected_subdir)) { + env_path /= expected_subdir; + } + lg::info("'{}' has no extracted game data, using JAK_ISO_DATA_DIR instead: {}", + temp_dir.string(), env_path.string()); + g_iso_data_directory = env_path; + } else if (fs::exists(temp_dir)) { g_iso_data_directory = temp_dir; } return g_iso_data_directory; diff --git a/decompiler/extractor/main.cpp b/decompiler/extractor/main.cpp index 70d9e26bef..3cf26d1912 100644 --- a/decompiler/extractor/main.cpp +++ b/decompiler/extractor/main.cpp @@ -375,6 +375,20 @@ int main(int argc, char** argv) { iso_data_path = input_file_path; } else { iso_data_path = file_util::get_jak_project_dir() / "iso_data" / data_subfolder; + // no DGO folder means no game data was ever extracted here + if (!fs::exists(iso_data_path / "DGO")) { + const auto env_iso_data = get_env("JAK_ISO_DATA_DIR"); + // see if we can use JAK_ISO_DATA_DIR + if (!env_iso_data.empty()) { + fs::path env_path = env_iso_data; + if (fs::exists(env_path / data_subfolder)) { + env_path /= data_subfolder; + } + lg::info("'{}' has no extracted game data, using JAK_ISO_DATA_DIR instead: {}", + iso_data_path.string(), env_path.string()); + iso_data_path = env_path; + } + } } } diff --git a/decompiler/level_extractor/extract_level.cpp b/decompiler/level_extractor/extract_level.cpp index af3abb2833..e0376bb45a 100644 --- a/decompiler/level_extractor/extract_level.cpp +++ b/decompiler/level_extractor/extract_level.cpp @@ -448,7 +448,8 @@ void extract_all_levels(const ObjectFileDB& db, game_version_names[config.game_version] / "entities"; file_util::create_dir_if_needed(entities_dir); - int num_workers = dgo_names.size(); + // cap the number of workers - one thread per level holds the whole uncompressed level in memory + int num_workers = std::min((int)dgo_names.size(), (int)std::thread::hardware_concurrency()); if (tex_db.replace_texture_dir) { num_workers = 1; } diff --git a/decompiler/main.cpp b/decompiler/main.cpp index 1ce9d89c5f..941963de67 100644 --- a/decompiler/main.cpp +++ b/decompiler/main.cpp @@ -83,6 +83,19 @@ int main(int argc, char** argv) { } in_folder = in_folder / config.game_name; + // no DGO folder means no game data was ever extracted here - fall back to JAK_ISO_DATA_DIR if it is set + if (!fs::exists(in_folder / "DGO")) { + const auto env_iso_data = get_env("JAK_ISO_DATA_DIR"); + if (!env_iso_data.empty()) { + fs::path env_path = env_iso_data; + if (fs::exists(env_path / config.game_name)) { + env_path /= config.game_name; + } + lg::info("'{}' has no extracted game data, using JAK_ISO_DATA_DIR instead: {}", + in_folder.string(), env_path.string()); + in_folder = env_path; + } + } // Verify the in_folder is correct if (!exists(in_folder)) { lg::error("Aborting - 'in_folder' does not exist '{}'", in_folder.string()); diff --git a/goalc/compiler/compilation/CompilerControl.cpp b/goalc/compiler/compilation/CompilerControl.cpp index 8c77c96b7a..0183574fa8 100644 --- a/goalc/compiler/compilation/CompilerControl.cpp +++ b/goalc/compiler/compilation/CompilerControl.cpp @@ -3,8 +3,10 @@ * Compiler implementation for forms which actually control the compiler. */ +#include #include #include +#include #include "common/repl/repl_wrapper.h" #include "common/util/DgoWriter.h" @@ -12,6 +14,8 @@ #include "common/util/Timer.h" #include "common/util/string_util.h" +#include "decompiler/extractor/extractor_util.h" + #include "goalc/compiler/Compiler.h" #include "goalc/compiler/IR.h" #include "goalc/compiler/docs/DocTypes.h" @@ -834,7 +838,65 @@ Val* Compiler::compile_make(const goos::Object& form, const goos::Object& rest, report = get_true_or_false(form, args.get_named("report")); } - m_make.make(args.unnamed.at(0).as_string()->data, force, verbose, report); + const auto& target = args.unnamed.at(0).as_string()->data; + + // if we're building the iso group but the game was never extracted, automatically run the + // extraction (the equivalent of `task extract`) once before building + if (target == "GROUP:iso") { + const auto game_name = version_to_game_name(m_version); + const auto decomp_out = file_util::get_jak_project_dir() / "decompiler_out" / game_name; + if (!fs::exists(decomp_out / "textures" / "tpage-dir.txt")) { + const auto iso_dir = file_util::get_iso_dir_for_game(m_version); + if (iso_dir.empty() || !fs::exists(iso_dir / "DGO")) { + lg::warn( + "The game has not been extracted yet and no ISO data was found - the build will " + "likely fail. Place the game data in iso_data/{} or set JAK_ISO_DATA_DIR.", + game_name); + } else { + lg::warn("The game has not been extracted yet ('{}' is missing).", decomp_out.string()); + lg::warn("Automatically extracting from '{}' .", iso_dir.string()); + // run the extraction in a separate decompiler process (the same thing `task extract` + // does) - it is memory heavy and can't be allowed to take down the compiler process + try { + const auto version_info = get_version_info_or_default(iso_dir); +#ifdef _WIN32 + const std::string decomp_exe_name = "decompiler.exe"; +#else + const std::string decomp_exe_name = "decompiler"; +#endif + const auto decompiler_path = + fs::path(file_util::get_current_executable_path()).parent_path() / decomp_exe_name; + if (!fs::exists(decompiler_path)) { + lg::error("Could not find the decompiler at '{}', attempting the build anyway...", + decompiler_path.string()); + } else { + const auto config_path = file_util::get_jak_project_dir() / "decompiler" / "config" / + game_name / fmt::format("{}_config.jsonc", game_name); + auto cmd = fmt::format( + "\"{}\" \"{}\" \"{}\" \"{}\" --version \"{}\" --config-override " + "\"{{\\\"decompile_code\\\": false, \\\"levels_extract\\\": true, " + "\\\"allowed_objects\\\": []}}\"", + decompiler_path.string(), config_path.string(), + (file_util::get_jak_project_dir() / "iso_data").string(), + (file_util::get_jak_project_dir() / "decompiler_out").string(), + version_info.decomp_config_version); +#ifdef _WIN32 + // cmd.exe strips the outer quotes of the whole command string, wrap it in an extra + // pair so the quoted executable path survives + cmd = fmt::format("\"{}\"", cmd); +#endif + if (system(cmd.c_str()) != 0) { + lg::error("Extraction failed, attempting the build anyway..."); + } + } + } catch (const std::exception& e) { + lg::error("Extraction failed ({}), attempting the build anyway...", e.what()); + } + } + } + } + + m_make.make(target, force, verbose, report); return get_none(); } diff --git a/goalc/main.cpp b/goalc/main.cpp index 0c50d802a7..36f53f1a0d 100644 --- a/goalc/main.cpp +++ b/goalc/main.cpp @@ -103,7 +103,8 @@ int main(int argc, char** argv) { // if a command is provided on the command line, no REPL just run the compiler on it try { if (!cmd.empty()) { - compiler = std::make_unique(game_version, emitter::InstructionSet::X86); + compiler = std::make_unique(game_version, emitter::InstructionSet::X86, + std::make_optional(repl_config), username); compiler->run_front_end_on_string(cmd); return 0; } diff --git a/goalc/make/MakeSystem.cpp b/goalc/make/MakeSystem.cpp index 551bc45307..338ba1a052 100644 --- a/goalc/make/MakeSystem.cpp +++ b/goalc/make/MakeSystem.cpp @@ -5,6 +5,8 @@ #include "common/util/FileUtil.h" #include "common/util/Timer.h" #include "common/util/string_util.h" +#include "common/util/unicode_util.h" +#include "common/versions/versions.h" #include "goalc/make/CompilerReport.h" #include "goalc/make/Tools.h" @@ -96,8 +98,30 @@ MakeSystem::MakeSystem(const std::optional repl_config, const std: file_util::get_iso_dir_for_game(m_repl_config->game_version).string()); set_constant("*use-iso-data-path*", true); } else { - set_constant("*iso-data*", file_util::get_file_path({"iso_data"})); - set_constant("*use-iso-data-path*", false); + bool use_env_iso_data = false; + if (m_repl_config) { + // no DGO folder means no game data was never extracted + // fall back to the JAK_ISO_DATA_DIR environment variable if it is set + const auto game_name = version_to_game_name(m_repl_config->game_version); + if (!fs::exists(file_util::get_jak_project_dir() / "iso_data" / game_name / "DGO")) { + const auto env_iso_data = get_env("JAK_ISO_DATA_DIR"); + if (!env_iso_data.empty()) { + fs::path env_path = env_iso_data; + if (fs::exists(env_path / game_name)) { + env_path /= game_name; + } + lg::info("iso_data/{} has no extracted game data, using JAK_ISO_DATA_DIR instead: {}", + game_name, env_path.string()); + set_constant("*iso-data*", env_path.string()); + set_constant("*use-iso-data-path*", true); + use_env_iso_data = true; + } + } + } + if (!use_env_iso_data) { + set_constant("*iso-data*", file_util::get_file_path({"iso_data"})); + set_constant("*use-iso-data-path*", false); + } } add_tool(); diff --git a/out/build/Release/bin/decompiler.exe b/out/build/Release/bin/decompiler.exe index 1bdd52b288..4264263116 100644 Binary files a/out/build/Release/bin/decompiler.exe and b/out/build/Release/bin/decompiler.exe differ diff --git a/out/build/Release/bin/extractor.exe b/out/build/Release/bin/extractor.exe index 524e775418..6217fafd55 100644 Binary files a/out/build/Release/bin/extractor.exe and b/out/build/Release/bin/extractor.exe differ diff --git a/out/build/Release/bin/gk.exe b/out/build/Release/bin/gk.exe index 8755a34ee7..4630f0ffb3 100644 Binary files a/out/build/Release/bin/gk.exe and b/out/build/Release/bin/gk.exe differ diff --git a/out/build/Release/bin/goalc.exe b/out/build/Release/bin/goalc.exe index 2f36ddc875..b9dacaeca6 100644 Binary files a/out/build/Release/bin/goalc.exe and b/out/build/Release/bin/goalc.exe differ diff --git a/third-party/curl/lib/curl_config.h.cmake b/third-party/curl/lib/curl_config.h.cmake index 65901a2b1d..899c729c80 100644 --- a/third-party/curl/lib/curl_config.h.cmake +++ b/third-party/curl/lib/curl_config.h.cmake @@ -317,7 +317,12 @@ #cmakedefine HAVE_IOCTLSOCKET_CAMEL_FIONBIO 1 /* Define to 1 if you have a working ioctlsocket FIONBIO function. */ +/* mod-base-change fails with clang + Win11 SDK */ +#ifdef _WIN32 +#define HAVE_IOCTLSOCKET_FIONBIO 1 +#else #cmakedefine HAVE_IOCTLSOCKET_FIONBIO 1 +#endif /* Define to 1 if you have a working ioctl FIONBIO function. */ #cmakedefine HAVE_IOCTL_FIONBIO 1