diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/R-CMD-check.yaml index 2fd552b..b3872e3 100644 --- a/.github/workflows/R-CMD-check.yaml +++ b/.github/workflows/R-CMD-check.yaml @@ -36,25 +36,42 @@ jobs: - uses: r-lib/actions/setup-pandoc@d3c5be51b12e724e68f33216ca3c148b66d5f0b6 # v2 - - uses: r-lib/actions/setup-r@6f6e5bc62fba3a704f74e7ad7ef7676c5c6a2590 # v2 + - uses: r-lib/actions/setup-r@d3c5be51b12e724e68f33216ca3c148b66d5f0b6 # v2 with: r-version: ${{ matrix.config.r }} http-user-agent: ${{ matrix.config.http-user-agent }} use-public-rspm: true - - uses: r-lib/actions/setup-r-dependencies@6f6e5bc62fba3a704f74e7ad7ef7676c5c6a2590 # v2 + # Build stringfish before dependency setup because packages built during + # resolution can load it and fail on a prebuilt oneTBB ABI mismatch. + - name: Source-build stringfish before dependency resolution + run: >- + Rscript -e 'install.packages(c("RcppParallel", "stringfish"), type = "source", + repos = "https://cloud.r-project.org")' + + - uses: r-lib/actions/setup-r-dependencies@d3c5be51b12e724e68f33216ca3c148b66d5f0b6 # v2 with: + # Refresh the dependency cache after the macOS qs2/RcppParallel TBB ABI mismatch. + cache-version: '2' extra-packages: | any::rcmdcheck any::testthat needs: check + # Rebuild every installed package that links to RcppParallel against the + # same local oneTBB ABI before loading kaefa or its test dependencies. + - name: Rebuild RcppParallel-linked packages from source + run: >- + Rscript -e 'r <- "https://cloud.r-project.org"; install.packages("RcppParallel", type = "source", repos = r); + linked <- tools::dependsOnPkgs("RcppParallel", dependencies = "LinkingTo"); + if (length(linked)) install.packages(linked, type = "source", repos = r)' + - name: Run Zh formula regression tests run: | Rscript -e 'install.packages(".", repos = NULL, type = "source")' Rscript -e 'library(kaefa); testthat::test_file("tests/testthat/test-zh-misfit-decision-rule.R")' - - uses: r-lib/actions/check-r-package@6f6e5bc62fba3a704f74e7ad7ef7676c5c6a2590 # v2 + - uses: r-lib/actions/check-r-package@d3c5be51b12e724e68f33216ca3c148b66d5f0b6 # v2 with: error-on: '"error"' upload-snapshots: true diff --git a/.github/workflows/test-fast.yaml b/.github/workflows/test-fast.yaml index ea1cf69..c0c424a 100644 --- a/.github/workflows/test-fast.yaml +++ b/.github/workflows/test-fast.yaml @@ -16,15 +16,30 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: r-lib/actions/setup-r@6f6e5bc62fba3a704f74e7ad7ef7676c5c6a2590 # v2 + - uses: r-lib/actions/setup-r@d3c5be51b12e724e68f33216ca3c148b66d5f0b6 # v2 with: use-public-rspm: true - - uses: r-lib/actions/setup-r-dependencies@6f6e5bc62fba3a704f74e7ad7ef7676c5c6a2590 # v2 + # Build stringfish before dependency setup because packages built during + # resolution can load it and fail on a prebuilt oneTBB ABI mismatch. + - name: Source-build stringfish before dependency resolution + run: >- + Rscript -e 'install.packages(c("RcppParallel", "stringfish"), type = "source", + repos = "https://cloud.r-project.org")' + + - uses: r-lib/actions/setup-r-dependencies@d3c5be51b12e724e68f33216ca3c148b66d5f0b6 # v2 with: extra-packages: any::testthat needs: check + # Rebuild every installed package that links to RcppParallel against the + # same local oneTBB ABI before loading kaefa or its test dependencies. + - name: Rebuild RcppParallel-linked packages from source + run: >- + Rscript -e 'r <- "https://cloud.r-project.org"; install.packages("RcppParallel", type = "source", repos = r); + linked <- tools::dependsOnPkgs("RcppParallel", dependencies = "LinkingTo"); + if (length(linked)) install.packages(linked, type = "source", repos = r)' + - name: Install kaefa package for fast tests run: R CMD INSTALL . @@ -38,4 +53,6 @@ jobs: reporter = reporter) testthat::test_file("tests/testthat/test-core-api-contract.R", reporter = reporter) + testthat::test_file("tests/testthat/test-aefa-parameter-recovery.R", + reporter = reporter) RSCRIPT diff --git a/.github/workflows/test-suite.yaml b/.github/workflows/test-suite.yaml index 439b9b7..0694152 100644 --- a/.github/workflows/test-suite.yaml +++ b/.github/workflows/test-suite.yaml @@ -1,7 +1,8 @@ name: test-suite -# Runs the complete testthat suite (26 files), which the previous CI setup -# skipped: R-CMD-check runs with --no-tests and test-fast runs only 3 files. +# Runs the complete testthat suite, which the previous CI setup skipped: +# R-CMD-check runs with --no-tests and test-fast runs a small productization +# subset. # On push/PR the suite's own CI guards (.skip_expensive_ci_calls, skip_on_ci) # keep expensive AEFA estimations out. The weekly scheduled run and manual # dispatch set RUN_FULL_AEFA_TESTS=1, which unlocks the estimations guarded @@ -32,15 +33,30 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: r-lib/actions/setup-r@6f6e5bc62fba3a704f74e7ad7ef7676c5c6a2590 # v2 + - uses: r-lib/actions/setup-r@d3c5be51b12e724e68f33216ca3c148b66d5f0b6 # v2 with: use-public-rspm: true - - uses: r-lib/actions/setup-r-dependencies@6f6e5bc62fba3a704f74e7ad7ef7676c5c6a2590 # v2 + # Build stringfish before dependency setup because packages built during + # resolution can load it and fail on a prebuilt oneTBB ABI mismatch. + - name: Source-build stringfish before dependency resolution + run: >- + Rscript -e 'install.packages(c("RcppParallel", "stringfish"), type = "source", + repos = "https://cloud.r-project.org")' + + - uses: r-lib/actions/setup-r-dependencies@d3c5be51b12e724e68f33216ca3c148b66d5f0b6 # v2 with: extra-packages: any::testthat needs: check + # Rebuild every installed package that links to RcppParallel against the + # same local oneTBB ABI before loading kaefa or its test dependencies. + - name: Rebuild RcppParallel-linked packages from source + run: >- + Rscript -e 'r <- "https://cloud.r-project.org"; install.packages("RcppParallel", type = "source", repos = r); + linked <- tools::dependsOnPkgs("RcppParallel", dependencies = "LinkingTo"); + if (length(linked)) install.packages(linked, type = "source", repos = r)' + - name: Install kaefa package run: R CMD INSTALL . diff --git a/AGENTS.md b/AGENTS.md index e83e8e3..71aaee8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -11,6 +11,7 @@ Use this file as the local operating guide when modifying `kaefa`. - Core orchestration: `R/kaefa.R` - Estimation engine: `R/newEngine.R` - Shared helpers: `R/utils.R` +- Recovery protocol: `R/recovery.R` - Shiny app: `inst/shiny-app/app.R` - Tests: `tests/testthat/` - Local CI workflow definitions: `.github/workflows/` diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 6ddd063..6dab81f 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -1,6 +1,6 @@ # kaefa Architecture -Last updated: 2026-02-14 +Last updated: 2026-08-17 ## Purpose @@ -32,7 +32,11 @@ explicitly requires vendored source integration. - `R/kaefa.R`: public orchestration entry points and exported runtime behavior. - `R/newEngine.R`: candidate-model estimation engine used by the AEFA loop. -- `R/utils.R`: helper routines and shared utilities. +- `R/utils.R`: helper routines and shared utilities. `.mirt()` and + `.mixedmirt()` treat a missing (`NA`) second-order test as + non-convergence when `leniency` is false. +- `R/recovery.R`: internal true-parameter RMSE helpers used by the AEFA + recovery protocol. Not a public API. - `inst/shiny-app/app.R`: bundled Shiny interface logic. - `inst/shiny-app/README.md`: Shiny usage and minimal UI configuration guide. - `tests/testthat/*.R`: functional, regression, and integration tests. @@ -58,6 +62,10 @@ explicitly requires vendored source integration. - PR merge requires review approval and resolved conversations. - Required checks include R-CMD-check matrix and dependency review. +- Buyer-facing recovery evidence is the five-repeat RMSE protocol in + `tests/testthat/test-aefa-parameter-recovery.R`. The formula, alignment, + and coverage-exclusion contracts run in `test-fast`. Live `aefa()` + five-seed recovery is gated by `RUN_FULL_AEFA_TESTS=1`. - If code scanning is enabled later, alerts can be tracked via GitHub code scanning APIs. diff --git a/CLAUDE.md b/CLAUDE.md index 9747fed..76ab760 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -57,6 +57,8 @@ testthat::test_file("tests/testthat/test-shiny-product-surface.R", reporter = reporter) testthat::test_file("tests/testthat/test-core-api-contract.R", reporter = reporter) +testthat::test_file("tests/testthat/test-aefa-parameter-recovery.R", + reporter = reporter) RSCRIPT # R CMD check as CI runs it (note: tests are skipped here) @@ -124,6 +126,8 @@ psychometrics (see AGENTS.md). - `R/newEngine.R` — `engineAEFA()`: the candidate-model estimation engine used by the aefa loop (MCMC/estimation controls such as NCYCLES, BURNIN, SEMCYCLES, rotation choices, etc.). +- `R/recovery.R` — internal true-parameter RMSE helpers for the AEFA recovery + protocol (not exported). - `R/utils.R` — shared internals (`.mirt`/`.mixedmirt` wrappers around mirt, `.covdataClassifieder`, `.covdataFixedEffectComb`, `.exportParmsEME`) and the theta-prior utilities `fitThetaPrior()`, `testThetaPriorCalibration()`, diff --git a/R/recovery.R b/R/recovery.R new file mode 100644 index 0000000..50bf43e --- /dev/null +++ b/R/recovery.R @@ -0,0 +1,168 @@ +# Internal true-parameter recovery helpers for AEFA / IRT Monte Carlo evidence. +# These functions are not exported. Buyer-facing recovery is the test protocol +# in tests/testthat/test-aefa-parameter-recovery.R and the provenance note in +# docs/traceability/aefa-parameter-recovery.md. + +.parameterRecoveryRMSE <- function(estimated, truth) { + estimated <- as.numeric(estimated) + truth <- as.numeric(truth) + if (length(estimated) != length(truth)) { + stop("Estimated and true vectors must have the same length.", call. = FALSE) + } + if (!length(estimated)) { + stop("Cannot compute RMSE from empty parameter vectors.", call. = FALSE) + } + if (any(!is.finite(estimated)) || any(!is.finite(truth))) { + stop("RMSE requires finite estimated and true values.", call. = FALSE) + } + sqrt(mean((estimated - truth)^2)) +} + +# R data.frames never keep rownames as NULL: `rownames(x) <- NULL` resets them +# to the sequential defaults "1", "2", .... Treat those as unnamed so callers +# cannot silently align on positional indices. +.irtItemNames <- function(x) { + rn <- rownames(x) + if (is.null(rn) || identical(rn, as.character(seq_len(nrow(x))))) { + return(NULL) + } + rn +} + +.alignIrtItemParameters <- function(estimated, truth, columns = c("a", "b")) { + if (!is.data.frame(estimated) && !is.matrix(estimated)) { + stop("estimated must be a matrix or data.frame of item parameters.", call. = FALSE) + } + if (!is.data.frame(truth) && !is.matrix(truth)) { + stop("truth must be a matrix or data.frame of item parameters.", call. = FALSE) + } + estimated <- as.data.frame(estimated, stringsAsFactors = FALSE) + truth <- as.data.frame(truth, stringsAsFactors = FALSE) + est_names <- .irtItemNames(estimated) + true_names <- .irtItemNames(truth) + if (is.null(est_names) || is.null(true_names)) { + stop( + "Estimated and true parameter tables must have item names as row names.", + call. = FALSE + ) + } + missing_estimated <- setdiff(columns, colnames(estimated)) + missing_truth <- setdiff(columns, colnames(truth)) + if (length(missing_estimated) || length(missing_truth)) { + stop( + "Missing recovery columns: ", + paste(unique(c(missing_estimated, missing_truth)), collapse = ", "), + call. = FALSE + ) + } + missing_estimated_items <- setdiff(true_names, est_names) + missing_truth_items <- setdiff(est_names, true_names) + if (length(missing_estimated_items) || length(missing_truth_items)) { + stop( + "Estimated and true parameter tables must contain the same item names.", + call. = FALSE + ) + } + shared <- est_names + list( + estimated = estimated[shared, columns, drop = FALSE], + truth = truth[shared, columns, drop = FALSE], + items = shared + ) +} + +.summariseRecoveryRepeats <- function(rmse_by_repeat) { + if (!is.data.frame(rmse_by_repeat)) { + stop("rmse_by_repeat must be a data.frame.", call. = FALSE) + } + required <- c("seed", "parameter", "rmse") + missing <- setdiff(required, names(rmse_by_repeat)) + if (length(missing)) { + stop( + "rmse_by_repeat must contain columns: ", + paste(required, collapse = ", "), + call. = FALSE + ) + } + seeds <- unique(rmse_by_repeat$seed) + if (length(seeds) != 5L) { + stop("Recovery protocol requires exactly 5 repeats.", call. = FALSE) + } + if (anyNA(rmse_by_repeat$seed) || + anyNA(rmse_by_repeat$parameter) || + anyNA(rmse_by_repeat$rmse)) { + stop("Recovery repeats cannot contain missing values.", call. = FALSE) + } + parameters <- unique(as.character(rmse_by_repeat$parameter)) + repeat_counts <- table( + as.character(rmse_by_repeat$parameter), + rmse_by_repeat$seed + ) + if (!length(repeat_counts) || any(repeat_counts != 1L)) { + stop( + "Each parameter must have exactly one RMSE value for each recovery seed.", + call. = FALSE + ) + } + summary_rows <- lapply(parameters, function(parameter_name) { + values <- rmse_by_repeat$rmse[as.character(rmse_by_repeat$parameter) == parameter_name] + data.frame( + parameter = parameter_name, + n_repeats = length(values), + mean_rmse = mean(values), + sd_rmse = stats::sd(values), + stringsAsFactors = FALSE + ) + }) + list( + n_repeats = 5L, + seeds = seeds, + per_run = rmse_by_repeat[order(rmse_by_repeat$seed, rmse_by_repeat$parameter), ], + summary = do.call(rbind, summary_rows) + ) +} + +.extractAefaIrtItems <- function(fit) { + if (inherits(fit, "aefa") || (is.list(fit) && !is.null(fit$estModelTrials))) { + trials <- fit$estModelTrials + if (!length(trials)) { + stop("aefa history has no estimated model trials.", call. = FALSE) + } + fit <- trials[[length(trials)]] + } + if (!methods::is(fit, "SingleGroupClass") && !methods::is(fit, "MixedClass")) { + stop("Recovery extraction requires an aefa history or a mirt model.", call. = FALSE) + } + items <- mirt::coef(fit, IRTpars = TRUE, simplify = TRUE)$items + if (is.null(items)) { + stop("Could not extract IRT item parameters.", call. = FALSE) + } + as.data.frame(items, stringsAsFactors = FALSE) +} + +.recoveryCoverageExclusions <- function() { + data.frame( + surface = c( + "unidimensional 2PL via .mirt", + "AEFA greedy search on unidimensional 2PL", + "mixedmirt multilevel / random effects", + "multiple-membership crossed random effects", + "time-flow / longitudinal membership" + ), + status = c( + "covered", + "covered when RUN_FULL_AEFA_TESTS=1", + "excluded", + "excluded", + "excluded" + ), + reason = c( + "Known-true 2PL simulation with IRT a/b RMSE.", + "Known-true 2PL simulation through aefa() with five seeds.", + "Engine exposes .mixedmirt, but no true-parameter RMSE protocol yet.", + "random = ~1|G formulas exist, but no recovery design is registered.", + "No time-indexed membership design is implemented in kaefa-core." + ), + stringsAsFactors = FALSE + ) +} diff --git a/R/utils.R b/R/utils.R index 0c9ffdf..95fe14c 100644 --- a/R/utils.R +++ b/R/utils.R @@ -197,11 +197,13 @@ } if(exists('mod')){ - if(mod@OptimInfo$converged){ + if(isTRUE(mod@OptimInfo$converged)){ if(leniency){ mod } else { - if(mod@OptimInfo$secondordertest){ + # mirt can leave secondordertest as NA when the Hessian is unavailable; + # `if (NA)` would abort the caller instead of treating the fit as failed. + if(isTRUE(mod@OptimInfo$secondordertest)){ mod } else { NULL @@ -229,11 +231,11 @@ SEMCYCLES = SEMCYCLES, symmetric = symmetric)) if(exists('mod')){ - if(mod@OptimInfo$converged){ + if(isTRUE(mod@OptimInfo$converged)){ if(leniency){ mod } else { - if(mod@OptimInfo$secondordertest){ + if(isTRUE(mod@OptimInfo$secondordertest)){ mod } else { NULL diff --git a/docs/papers/README.md b/docs/papers/README.md index cc2caa8..718a17e 100644 --- a/docs/papers/README.md +++ b/docs/papers/README.md @@ -119,10 +119,35 @@ cited with its DOI. Open-access / preprint links are noted where available. needed to reconstruct it. Kaefa accepts DIC only when the fitted model supplies a finite DIC value and never relabels AIC as DIC. +## 7. Monte Carlo parameter recovery RMSE + +- **Source:** Harwell, M. R., Stone, C. A., Hsu, T.-C., & Kirisci, L. (1996). + Monte Carlo studies in item response theory. *Applied Psychological + Measurement, 20*(2), 101-125. + DOI: [10.1177/014662169602000201](https://doi.org/10.1177/014662169602000201) +- **Canonical equation.** For recovered parameters \(\hat{\theta}\) and known + true parameters \(\theta\), + + RMSE = sqrt( mean( (hat_theta - theta)^2 ) ) + + Harwell et al. treat RMSE (and related Monte Carlo error summaries) as the + standard way to judge whether an IRT estimator recovers a known generating + model. kaefa uses that definition on IRT `a` and `b` after name alignment. +- **Usage in kaefa:** internal helpers in `R/recovery.R` and the five-repeat + protocol in `docs/traceability/aefa-parameter-recovery.md`. The formula, + alignment, and five-repeat schema are pinned by + `tests/testthat/test-aefa-parameter-recovery.R`. Live `.mirt` and `aefa()` + recovery fits live in `tests/testthat/test-aefa-recovery-fits.R`. +- **Boundary.** Multilevel `mixedmirt`, multiple-membership, and time-flow + designs are recorded as exclusions until a true-parameter design is + registered. The engine remains R/`mirt`; this protocol does not introduce a + Rust or GPU numeric core. + ## Audit note kaefa does **not** re-implement `P(theta)`, the MML-EM E-/M-step, `S-X2`, `infit`, or `outfit`; those are delegated verbatim to `mirt` and remain subject to `mirt`'s validation. Package-local formulas and decision rules are pinned above: -the `Zh` cutoff, the exact Hurvich-Tsai AICc correction, and the explicit -posterior-information boundary that prevents DIC from being fabricated. +the `Zh` cutoff, the exact Hurvich-Tsai AICc correction, the explicit +posterior-information boundary that prevents DIC from being fabricated, and +the Harwell et al. RMSE recovery definition. diff --git a/docs/product/kaefa-core-api-contract.md b/docs/product/kaefa-core-api-contract.md index 5d81b81..ad361f7 100644 --- a/docs/product/kaefa-core-api-contract.md +++ b/docs/product/kaefa-core-api-contract.md @@ -172,6 +172,9 @@ exist and pass in CI: - theta-prior helpers cover fit success, unsupported distribution, and fallback behavior. - dot-prefixed helpers are either documented or covered by deprecation tests. +- true-parameter recovery reports five-repeat RMSE for unidimensional 2PL + `a`/`b` after item-name alignment, and records multilevel / multiple-membership + / time-flow surfaces as exclusions until a registered design exists. ## Split Readiness Rule diff --git a/docs/superpowers/plans/2026-07-02-kaefa-2b-krw-sale-readiness.md b/docs/superpowers/plans/2026-07-02-kaefa-2b-krw-sale-readiness.md index d5192fe..4be08d6 100644 --- a/docs/superpowers/plans/2026-07-02-kaefa-2b-krw-sale-readiness.md +++ b/docs/superpowers/plans/2026-07-02-kaefa-2b-krw-sale-readiness.md @@ -251,11 +251,11 @@ Guardrails: steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - uses: r-lib/actions/setup-r@6f6e5bc62fba3a704f74e7ad7ef7676c5c6a2590 # v2 + - uses: r-lib/actions/setup-r@d3c5be51b12e724e68f33216ca3c148b66d5f0b6 # v2 with: use-public-rspm: true - - uses: r-lib/actions/setup-r-dependencies@6f6e5bc62fba3a704f74e7ad7ef7676c5c6a2590 # v2 + - uses: r-lib/actions/setup-r-dependencies@d3c5be51b12e724e68f33216ca3c148b66d5f0b6 # v2 with: extra-packages: any::testthat needs: check diff --git a/docs/traceability/aefa-parameter-recovery.md b/docs/traceability/aefa-parameter-recovery.md new file mode 100644 index 0000000..236506f --- /dev/null +++ b/docs/traceability/aefa-parameter-recovery.md @@ -0,0 +1,66 @@ +# AEFA true-parameter recovery protocol + +## Decision + +Buyer-facing recovery evidence for `kaefa-core` is a five-repeat RMSE protocol +against known-true item parameters. The protocol is implemented by internal +helpers in `R/recovery.R` and pinned by +`tests/testthat/test-aefa-parameter-recovery.R` (formula, alignment, schema, +coverage exclusions) plus `tests/testthat/test-aefa-recovery-fits.R` (live +fits). + +This is not a public API. External behaviour of `aefa()` and `engineAEFA()` is +unchanged. `.mirt()` / `.mixedmirt()` now treat a missing +(`NA`) second-order test as non-convergence when `leniency` is false, instead +of aborting on `if (NA)`. + +## RMSE definition + +For a recovered parameter vector \(\hat{\theta}\) and a true vector \(\theta\), + + RMSE = sqrt( mean( (hat_theta - theta)^2 ) ) + +Items are aligned by name before the difference is taken. Estimated and true +tables must contain the same item names; a partial intersection is rejected. +The required IRT columns for the unidimensional 2PL case are `a` +(discrimination) and `b` (difficulty) from +`mirt::coef(..., IRTpars = TRUE, simplify = TRUE)$items`. + +Exactly five seeds are required, and each parameter must have one RMSE value +for every seed. The summary schema is: per-run `seed`, `parameter`, `rmse`, +plus `mean_rmse` and `sd_rmse` by parameter. + +## Current coverage + +Covered: + +- Unidimensional 2PL recovery through `kaefa::.mirt` (`N = 1500`, + `SE = TRUE`, same cycle budget as the FIIFM stability fit). +- AEFA greedy search on the same 2PL design when `RUN_FULL_AEFA_TESTS=1`. + +Explicitly excluded until a registered design exists: + +- `mixedmirt` multilevel / random-effect recovery. +- Multiple-membership crossed random effects. +- Time-flow / longitudinal membership. + +The exclusion log is asserted in the recovery contract so a later claim cannot +silently treat those surfaces as covered. + +## Compatibility and rollback + +The helpers are unused by the estimation loop. Removing `R/recovery.R` and the +two test files restores the previous evidence surface. Do not export the +helpers or substitute a different error metric without updating this note and +the formula tests together. + +## References + +Harwell, M. R., Stone, C. A., Hsu, T.-C., & Kirisci, L. (1996). Monte Carlo +studies in item response theory. *Applied Psychological Measurement, 20*(2), +101-125. + + +Chalmers, R. P. (2012). mirt: A multidimensional item response theory package +for the R environment. *Journal of Statistical Software, 48*(6), 1-29. + diff --git a/docs/traceability/r-lib-actions-v2.12.1.md b/docs/traceability/r-lib-actions-v2.12.1.md new file mode 100644 index 0000000..2960000 --- /dev/null +++ b/docs/traceability/r-lib-actions-v2.12.1.md @@ -0,0 +1,29 @@ +# r-lib/actions v2.12.1 공급망 추적 + +## 결정 + +세 개의 R 검증 workflow에서 사용하는 setup-pandoc, setup-r, +setup-r-dependencies, check-r-package를 v2.12.1 release commit +d3c5be51b12e724e68f33216ca3c148b66d5f0b6으로 통일한다. 전체 commit SHA 외의 태그·브랜치·짧은 SHA는 +회귀 계약이 거부한다. + +## 호환성 범위 + +공식 NEWS에 따르면 v2.12는 Node.js 24 전환, public RSPM 기본값 조정, +아키텍처별 cache key와 Pandoc 3.8.3을 포함하고, v2.12.1은 setup-r URL parser +경고와 Quarto action을 갱신한다. 현재 workflow의 R matrix, 권한, testthat 실행, +--no-tests 분리와 scheduled full-suite 계약은 변경하지 않는다. + +## 되돌리기 + +runner 또는 package 호환성 회귀가 확인되면 네 action을 함께 마지막 검증 SHA로 +되돌리고 R-CMD-check, fast/full test suite와 중앙 보안 검사를 같은 헤드에서 다시 +수행한다. 일부 action만 되돌리거나 이동 태그로 우회하지 않는다. + +## 참고문헌 + +R-lib. (2026, June 23). *r-lib/actions v2.12.1* [Software release notes]. +[NEWS.md](https://github.com/r-lib/actions/blob/d3c5be51b12e724e68f33216ca3c148b66d5f0b6/NEWS.md) + +R-lib. (2026). *Update NEWS for v1.12.1* [Source code commit]. +[d3c5be51b12e724e68f33216ca3c148b66d5f0b6](https://github.com/r-lib/actions/commit/d3c5be51b12e724e68f33216ca3c148b66d5f0b6) diff --git a/inst/WORDLIST b/inst/WORDLIST index f03f48c..a0f3812 100644 --- a/inst/WORDLIST +++ b/inst/WORDLIST @@ -88,18 +88,23 @@ Behavioral Behavioural Bentler Bifactor +Carlo DIC DIF DiscreteClass EM FUN +GPU GenRandomPars +Harwell +Hsu IRT Jennrich Jiao Jin Kamata Kang +Kirisci LAN LCA Leiman @@ -114,6 +119,7 @@ MPI Mansolf Mels MixedClass +Monte MultipleGroupClass NCYCLES NROW @@ -126,16 +132,20 @@ Psychometrika Q1 QMC RM +RMSE Rasch +Reckase Reise RemoteClusters Robbin +Rust SEMCYCLES SQUAREM SSH S_X2 Schmid SingleGroupClass +Stone Testlet UIRT X2 @@ -192,6 +202,7 @@ maximising minExtraction mirt mirtModel +mixedmirt modelSelectionCriteria ncat newEngine @@ -236,6 +247,7 @@ tracelines tryLCA turnOffMixedEst ubuntu +unidimensional v048 v1 z0 diff --git a/tests/FAST_TESTS.md b/tests/FAST_TESTS.md index 5629ada..23d0d6e 100644 --- a/tests/FAST_TESTS.md +++ b/tests/FAST_TESTS.md @@ -13,6 +13,8 @@ testthat::test_file("tests/testthat/test-shiny-product-surface.R", reporter = reporter) testthat::test_file("tests/testthat/test-core-api-contract.R", reporter = reporter) +testthat::test_file("tests/testthat/test-aefa-parameter-recovery.R", + reporter = reporter) ``` `R-CMD-check` remains the package installation and multi-OS compatibility gate. diff --git a/tests/testthat/test-aefa-parameter-recovery.R b/tests/testthat/test-aefa-parameter-recovery.R new file mode 100644 index 0000000..6c7edaf --- /dev/null +++ b/tests/testthat/test-aefa-parameter-recovery.R @@ -0,0 +1,118 @@ +test_that("RMSE matches the Monte Carlo recovery definition", { + .ensure_kaefa_namespace() + estimated <- c(0.8, 1.1, 1.4) + truth <- c(1.0, 1.0, 1.0) + expected <- sqrt(mean((estimated - truth)^2)) + testthat::expect_equal(kaefa:::.parameterRecoveryRMSE(estimated, truth), expected) + testthat::expect_equal(expected, sqrt((0.04 + 0.01 + 0.16) / 3)) + testthat::expect_error(kaefa:::.parameterRecoveryRMSE(c(1, 2), 1), "same length") + testthat::expect_error(kaefa:::.parameterRecoveryRMSE(numeric(), numeric()), "empty") + testthat::expect_error(kaefa:::.parameterRecoveryRMSE(c(1, NA), c(1, 2)), "finite") +}) + +test_that("IRT parameter alignment is by item name and required columns", { + .ensure_kaefa_namespace() + estimated <- data.frame( + a = c(1.1, 0.9), + b = c(-0.2, 0.4), + row.names = c("Item2", "Item1") + ) + truth <- data.frame( + a = c(1.0, 1.2), + b = c(0.5, -0.1), + row.names = c("Item1", "Item2") + ) + aligned <- kaefa:::.alignIrtItemParameters(estimated, truth) + testthat::expect_identical(aligned$items, c("Item2", "Item1")) + testthat::expect_equal(aligned$estimated["Item1", "a"], 0.9) + testthat::expect_equal(aligned$truth["Item1", "b"], 0.5) + + unnamed <- estimated + rownames(unnamed) <- NULL + testthat::expect_error(kaefa:::.alignIrtItemParameters(unnamed, truth), "row names") + truth_unnamed <- truth + rownames(truth_unnamed) <- NULL + testthat::expect_error( + kaefa:::.alignIrtItemParameters(unnamed, truth_unnamed), + "row names" + ) + testthat::expect_error( + kaefa:::.alignIrtItemParameters(estimated[, "a", drop = FALSE], truth), + "Missing recovery columns" + ) + other <- truth + rownames(other) <- c("Q1", "Q2") + testthat::expect_error( + kaefa:::.alignIrtItemParameters(estimated, other), + "same item names" + ) + subset_truth <- truth[1, , drop = FALSE] + testthat::expect_error( + kaefa:::.alignIrtItemParameters(estimated, subset_truth), + "same item names" + ) +}) + +test_that("five-repeat recovery summary has a fixed output schema", { + .ensure_kaefa_namespace() + rmse_by_repeat <- data.frame( + seed = rep(c(11L, 22L, 33L, 44L, 55L), each = 2L), + parameter = rep(c("a", "b"), times = 5L), + rmse = c(0.10, 0.20, 0.12, 0.18, 0.11, 0.21, 0.09, 0.19, 0.13, 0.17), + stringsAsFactors = FALSE + ) + summarised <- kaefa:::.summariseRecoveryRepeats(rmse_by_repeat) + testthat::expect_identical(summarised$n_repeats, 5L) + testthat::expect_identical(names(summarised$summary), c("parameter", "n_repeats", "mean_rmse", "sd_rmse")) + testthat::expect_equal(summarised$summary$n_repeats, c(5L, 5L)) + a_rmse <- rmse_by_repeat$rmse[rmse_by_repeat$parameter == "a"] + testthat::expect_equal( + summarised$summary$mean_rmse[summarised$summary$parameter == "a"], + mean(a_rmse) + ) + testthat::expect_equal( + summarised$summary$sd_rmse[summarised$summary$parameter == "a"], + stats::sd(a_rmse) + ) + testthat::expect_error( + kaefa:::.summariseRecoveryRepeats(rmse_by_repeat[1:4, ]), + "exactly 5 repeats" + ) + incomplete_b <- rmse_by_repeat[rmse_by_repeat$parameter == "a" | + rmse_by_repeat$seed != 55L, ] + testthat::expect_error( + kaefa:::.summariseRecoveryRepeats(incomplete_b), + "exactly one RMSE value" + ) + duplicated_a <- rbind( + rmse_by_repeat, + data.frame(seed = 11L, parameter = "a", rmse = 0.99, stringsAsFactors = FALSE) + ) + testthat::expect_error( + kaefa:::.summariseRecoveryRepeats(duplicated_a), + "exactly one RMSE value" + ) + missing_rmse <- rmse_by_repeat + missing_rmse$rmse[1] <- NA_real_ + testthat::expect_error( + kaefa:::.summariseRecoveryRepeats(missing_rmse), + "missing values" + ) +}) + +test_that("recovery coverage log keeps multilevel and time-flow explicit exclusions", { + .ensure_kaefa_namespace() + coverage <- kaefa:::.recoveryCoverageExclusions() + testthat::expect_identical( + names(coverage), + c("surface", "status", "reason") + ) + testthat::expect_true(any(coverage$surface == "unidimensional 2PL via .mirt" & + coverage$status == "covered")) + testthat::expect_true(any(grepl("mixedmirt", coverage$surface) & + coverage$status == "excluded")) + testthat::expect_true(any(grepl("multiple-membership", coverage$surface) & + coverage$status == "excluded")) + testthat::expect_true(any(grepl("time-flow", coverage$surface) & + coverage$status == "excluded")) +}) diff --git a/tests/testthat/test-aefa-recovery-fits.R b/tests/testthat/test-aefa-recovery-fits.R new file mode 100644 index 0000000..e454b67 --- /dev/null +++ b/tests/testthat/test-aefa-recovery-fits.R @@ -0,0 +1,121 @@ +test_that(".mirt recovers known 2PL parameters with bounded RMSE", { + .ensure_kaefa_namespace() + testthat::skip_if_not_installed("mirt") + + set.seed(20260817) + true_a <- matrix(c(0.9, 1.1, 1.3, 1.5, 1.7), ncol = 1) + true_b <- c(-1.2, -0.6, 0, 0.6, 1.2) + true_d <- matrix(-true_a[, 1] * true_b, ncol = 1) + # Same design as test-fiifm-stability.R: N=1500 and SE=TRUE so the Hessian + # exists and leniency=FALSE can keep the fit. SE=FALSE leaves + # secondordertest NA/FALSE and .mirt() returns NULL. + response_data <- as.data.frame(mirt::simdata( + a = true_a, + d = true_d, + itemtype = "2PL", + N = 1500 + )) + names(response_data) <- paste0("Item", seq_len(ncol(response_data))) + truth <- data.frame( + a = true_a[, 1], + b = true_b, + row.names = names(response_data) + ) + + utils::capture.output( + fit <- suppressWarnings(kaefa::.mirt( + data = response_data, + model = 1, + method = "EM", + itemtype = "2PL", + SE = TRUE, + GenRandomPars = FALSE, + calcNull = FALSE, + leniency = FALSE, + NCYCLES = 400, + BURNIN = 100, + SEMCYCLES = 100 + )) + ) + if (!methods::is(fit, "SingleGroupClass")) { + testthat::fail( + "kaefa::.mirt did not return a single-group fit (converged second-order test required)" + ) + return(invisible(NULL)) + } + + estimated <- kaefa:::.extractAefaIrtItems(fit) + aligned <- kaefa:::.alignIrtItemParameters(estimated, truth) + rmse_a <- kaefa:::.parameterRecoveryRMSE(aligned$estimated$a, aligned$truth$a) + rmse_b <- kaefa:::.parameterRecoveryRMSE(aligned$estimated$b, aligned$truth$b) + testthat::expect_lt(rmse_a, 0.35) + testthat::expect_lt(rmse_b, 0.35) + testthat::expect_gt(stats::cor(aligned$estimated$a, aligned$truth$a), 0.90) +}) + +test_that("aefa five-seed 2PL recovery reports mean RMSE", { + .skip_expensive_ci_calls("aefa") + .ensure_kaefa_namespace() + testthat::skip_if_not_installed("mirt") + + true_a <- matrix(c(0.9, 1.1, 1.3, 1.5, 1.7), ncol = 1) + true_b <- c(-1.2, -0.6, 0, 0.6, 1.2) + true_d <- matrix(-true_a[, 1] * true_b, ncol = 1) + seeds <- c(20260817L, 20260818L, 20260819L, 20260820L, 20260821L) + rows <- list() + + for (seed in seeds) { + set.seed(seed) + response_data <- as.data.frame(mirt::simdata( + a = true_a, + d = true_d, + itemtype = "2PL", + N = 250 + )) + names(response_data) <- paste0("Item", seq_len(ncol(response_data))) + truth <- data.frame( + a = true_a[, 1], + b = true_b, + row.names = names(response_data) + ) + result <- try( + aefa( + response_data, + minExtraction = 1, + maxExtraction = 1, + turnOffMixedEst = TRUE, + skipggum = TRUE, + tryLCA = FALSE, + saveModelHistory = TRUE, + printItemFit = FALSE + ), + silent = TRUE + ) + if (inherits(result, "try-error") || is.null(result)) { + testthat::fail(paste("aefa recovery repeat failed for seed", seed)) + return(invisible(NULL)) + } + estimated <- kaefa:::.extractAefaIrtItems(result) + aligned <- kaefa:::.alignIrtItemParameters(estimated, truth) + rows[[length(rows) + 1L]] <- data.frame( + seed = seed, + parameter = c("a", "b"), + rmse = c( + kaefa:::.parameterRecoveryRMSE(aligned$estimated$a, aligned$truth$a), + kaefa:::.parameterRecoveryRMSE(aligned$estimated$b, aligned$truth$b) + ), + stringsAsFactors = FALSE + ) + } + + summarised <- kaefa:::.summariseRecoveryRepeats(do.call(rbind, rows)) + testthat::expect_identical(summarised$n_repeats, 5L) + testthat::expect_lt( + summarised$summary$mean_rmse[summarised$summary$parameter == "a"], + 0.45 + ) + testthat::expect_lt( + summarised$summary$mean_rmse[summarised$summary$parameter == "b"], + 0.45 + ) +}) diff --git a/tests/testthat/test-ci-action-pins.R b/tests/testthat/test-ci-action-pins.R new file mode 100644 index 0000000..4203052 --- /dev/null +++ b/tests/testthat/test-ci-action-pins.R @@ -0,0 +1,142 @@ +testthat::test_that("CI uses exactly the reviewed r-lib action references", { + reviewed_sha <- "d3c5be51b12e724e68f33216ca3c148b66d5f0b6" + workflow_paths <- c( + testthat::test_path("..", "..", ".github", "workflows", "R-CMD-check.yaml"), + testthat::test_path("..", "..", ".github", "workflows", "test-fast.yaml"), + testthat::test_path("..", "..", ".github", "workflows", "test-suite.yaml") + ) + expected_actions <- list( + "R-CMD-check.yaml" = c( + "setup-pandoc", + "setup-r", + "setup-r-dependencies", + "check-r-package" + ), + "test-fast.yaml" = c("setup-r", "setup-r-dependencies"), + "test-suite.yaml" = c("setup-r", "setup-r-dependencies") + ) + action_pattern <- paste0( + "r-lib/actions/", + "(setup-pandoc|setup-r-dependencies|setup-r|check-r-package)", + "@[^[:space:]#]+" + ) + + for (workflow_path in workflow_paths) { + workflow_text <- paste(readLines(workflow_path, warn = FALSE), collapse = "\n") + action_refs <- regmatches( + workflow_text, + gregexpr(action_pattern, workflow_text, perl = TRUE) + )[[1]] + expected_refs <- paste0( + "r-lib/actions/", + expected_actions[[basename(workflow_path)]], + "@", + reviewed_sha + ) + + testthat::expect_identical( + action_refs, + expected_refs, + info = paste("Unexpected r-lib action set in", basename(workflow_path)) + ) + } +}) + +testthat::test_that("R CMD check refreshes the reviewed dependency cache ABI", { + workflow_path <- testthat::test_path( + "..", "..", ".github", "workflows", "R-CMD-check.yaml" + ) + workflow_lines <- readLines(workflow_path, warn = FALSE) + dependency_step <- grep( + "r-lib/actions/setup-r-dependencies@", + workflow_lines, + fixed = TRUE + ) + testthat::expect_length(dependency_step, 1L) + dependency_block <- workflow_lines[ + dependency_step:min(dependency_step + 8L, length(workflow_lines)) + ] + + active_cache_version_pattern <- paste0( + "^[[:space:]]*cache-version:[[:space:]]*", + "['\\\"]2['\\\"][[:space:]]*(#.*)?$" + ) + testthat::expect_false( + grepl(active_cache_version_pattern, "# cache-version: '2'", perl = TRUE), + info = "A commented cache-version example must not satisfy the contract" + ) + testthat::expect_true( + any(grepl(active_cache_version_pattern, dependency_block, perl = TRUE)), + info = "The reviewed macOS TBB ABI cache refresh must remain explicit" + ) +}) + +testthat::test_that("CI rebuilds oneTBB dependents around dependency setup", { + workflow_paths <- c( + testthat::test_path("..", "..", ".github", "workflows", "R-CMD-check.yaml"), + testthat::test_path("..", "..", ".github", "workflows", "test-fast.yaml"), + testthat::test_path("..", "..", ".github", "workflows", "test-suite.yaml") + ) + + for (workflow_path in workflow_paths) { + workflow_lines <- readLines(workflow_path, warn = FALSE) + pre_step <- grep( + "name: Source-build stringfish before dependency resolution", + workflow_lines, + fixed = TRUE + ) + dependency_step <- grep( + "r-lib/actions/setup-r-dependencies@", + workflow_lines, + fixed = TRUE + ) + post_step <- grep( + "name: Rebuild RcppParallel-linked packages from source", + workflow_lines, + fixed = TRUE + ) + + testthat::expect_length(pre_step, 1L) + testthat::expect_length(dependency_step, 1L) + testthat::expect_length(post_step, 1L) + testthat::expect_true(pre_step < dependency_step) + testthat::expect_true(dependency_step < post_step) + + pre_block <- workflow_lines[ + pre_step:min(pre_step + 5L, length(workflow_lines)) + ] + post_block <- workflow_lines[ + post_step:min(post_step + 7L, length(workflow_lines)) + ] + testthat::expect_true(any(grepl( + 'install.packages(c("RcppParallel", "stringfish")', + pre_block, + fixed = TRUE + ))) + testthat::expect_true(any(grepl( + 'repos = "https://cloud.r-project.org"', + pre_block, + fixed = TRUE + ))) + testthat::expect_true(any(grepl( + 'type = "source"', + pre_block, + fixed = TRUE + ))) + testthat::expect_true(any(grepl( + 'dependsOnPkgs("RcppParallel", dependencies = "LinkingTo")', + post_block, + fixed = TRUE + ))) + testthat::expect_true(any(grepl( + 'install.packages("RcppParallel", type = "source"', + post_block, + fixed = TRUE + ))) + testthat::expect_true(any(grepl( + 'install.packages(linked, type = "source"', + post_block, + fixed = TRUE + ))) + } +})