From d55a2b5ca02cb44fa363b6d648c3ca4b6503a6b5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 4 Aug 2026 13:57:40 +0900 Subject: [PATCH 01/24] ci: rebuild stringfish from source on mac/windows (RcppParallel oneTBB ABI skew) --- .github/workflows/R-CMD-check.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/R-CMD-check.yaml index 4705d5a..0f500b4 100644 --- a/.github/workflows/R-CMD-check.yaml +++ b/.github/workflows/R-CMD-check.yaml @@ -49,6 +49,13 @@ jobs: any::testthat needs: check + # ponytail: CRAN/RSPM binary skew — stringfish binaries lag RcppParallel's + # oneTBB update (Symbol not found: tbb::internal::NFS_Allocate). Rebuild from + # source so it links the installed RcppParallel. Drop once upstream binaries sync. + - name: Rebuild stringfish from source (RcppParallel ABI skew) + if: runner.os != 'Linux' + run: Rscript -e 'install.packages("stringfish", type = "source")' + - name: Run Zh formula regression tests run: | Rscript -e 'install.packages(".", repos = NULL, type = "source")' From 31d8d8b24e3b76de645fe73feb407c19b4aa9ff0 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 4 Aug 2026 16:29:58 +0900 Subject: [PATCH 02/24] ci: rebuild stringfish on all platforms (Linux hit too) --- .github/workflows/R-CMD-check.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/R-CMD-check.yaml index 0f500b4..04cb28e 100644 --- a/.github/workflows/R-CMD-check.yaml +++ b/.github/workflows/R-CMD-check.yaml @@ -50,10 +50,9 @@ jobs: needs: check # ponytail: CRAN/RSPM binary skew — stringfish binaries lag RcppParallel's - # oneTBB update (Symbol not found: tbb::internal::NFS_Allocate). Rebuild from + # oneTBB update (undefined tbb::internal symbols, all platforms). Rebuild from # source so it links the installed RcppParallel. Drop once upstream binaries sync. - name: Rebuild stringfish from source (RcppParallel ABI skew) - if: runner.os != 'Linux' run: Rscript -e 'install.packages("stringfish", type = "source")' - name: Run Zh formula regression tests From e7b7f3d6e75cf046df53d0be62df98e01872977c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 4 Aug 2026 16:30:00 +0900 Subject: [PATCH 03/24] ci: rebuild stringfish before install in test-fast --- .github/workflows/test-fast.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/test-fast.yaml b/.github/workflows/test-fast.yaml index fa70edf..c29c2ee 100644 --- a/.github/workflows/test-fast.yaml +++ b/.github/workflows/test-fast.yaml @@ -25,6 +25,11 @@ jobs: extra-packages: any::testthat needs: check + # ponytail: CRAN/RSPM binary skew — stringfish binaries lag RcppParallel's + # oneTBB update. Rebuild from source; drop once upstream binaries sync. + - name: Rebuild stringfish from source (RcppParallel ABI skew) + run: Rscript -e 'install.packages("stringfish", type = "source")' + - name: Install kaefa package for fast tests run: R CMD INSTALL . From 21755e059e7a5ee05aabf18e664aa92d442b67d5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 4 Aug 2026 16:30:02 +0900 Subject: [PATCH 04/24] ci: rebuild stringfish before install in test-suite --- .github/workflows/test-suite.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/test-suite.yaml b/.github/workflows/test-suite.yaml index d447513..8e22232 100644 --- a/.github/workflows/test-suite.yaml +++ b/.github/workflows/test-suite.yaml @@ -41,6 +41,11 @@ jobs: extra-packages: any::testthat needs: check + # ponytail: CRAN/RSPM binary skew — stringfish binaries lag RcppParallel's + # oneTBB update. Rebuild from source; drop once upstream binaries sync. + - name: Rebuild stringfish from source (RcppParallel ABI skew) + run: Rscript -e 'install.packages("stringfish", type = "source")' + - name: Install kaefa package run: R CMD INSTALL . From 0446de9d61ae3fdfa96c88a0715509ba642cb5a1 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 4 Aug 2026 16:35:53 +0900 Subject: [PATCH 05/24] ci: rebuild all RcppParallel-linked packages (qs2 hit the same oneTBB skew) --- .github/workflows/R-CMD-check.yaml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/R-CMD-check.yaml index 04cb28e..4a1f7ed 100644 --- a/.github/workflows/R-CMD-check.yaml +++ b/.github/workflows/R-CMD-check.yaml @@ -49,11 +49,12 @@ jobs: any::testthat needs: check - # ponytail: CRAN/RSPM binary skew — stringfish binaries lag RcppParallel's - # oneTBB update (undefined tbb::internal symbols, all platforms). Rebuild from - # source so it links the installed RcppParallel. Drop once upstream binaries sync. - - name: Rebuild stringfish from source (RcppParallel ABI skew) - run: Rscript -e 'install.packages("stringfish", type = "source")' + # ponytail: CRAN/RSPM binary skew — prebuilt binaries (stringfish, qs2, ...) lag + # RcppParallel's oneTBB update (undefined tbb::internal / tbb::task symbols). + # Rebuild everything LinkingTo RcppParallel from source so it links the installed + # copy. Drop once upstream binaries re-sync. + - name: Rebuild RcppParallel-linked packages from source (oneTBB ABI skew) + run: Rscript -e 'bad <- tools::dependsOnPkgs("RcppParallel", dependencies = "LinkingTo"); if (length(bad)) install.packages(bad, type = "source")' - name: Run Zh formula regression tests run: | From 648e948057a4fb0dc574804c5f24f941f827f1af Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 4 Aug 2026 16:35:55 +0900 Subject: [PATCH 06/24] ci: rebuild all RcppParallel-linked packages (qs2 hit the same oneTBB skew) --- .github/workflows/test-fast.yaml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-fast.yaml b/.github/workflows/test-fast.yaml index c29c2ee..46be129 100644 --- a/.github/workflows/test-fast.yaml +++ b/.github/workflows/test-fast.yaml @@ -25,10 +25,12 @@ jobs: extra-packages: any::testthat needs: check - # ponytail: CRAN/RSPM binary skew — stringfish binaries lag RcppParallel's - # oneTBB update. Rebuild from source; drop once upstream binaries sync. - - name: Rebuild stringfish from source (RcppParallel ABI skew) - run: Rscript -e 'install.packages("stringfish", type = "source")' + # ponytail: CRAN/RSPM binary skew — prebuilt binaries (stringfish, qs2, ...) lag + # RcppParallel's oneTBB update (undefined tbb::internal / tbb::task symbols). + # Rebuild everything LinkingTo RcppParallel from source so it links the installed + # copy. Drop once upstream binaries re-sync. + - name: Rebuild RcppParallel-linked packages from source (oneTBB ABI skew) + run: Rscript -e 'bad <- tools::dependsOnPkgs("RcppParallel", dependencies = "LinkingTo"); if (length(bad)) install.packages(bad, type = "source")' - name: Install kaefa package for fast tests run: R CMD INSTALL . From 34ef00f16b1aa68ece4a58ee88230efb850c0bbd Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 4 Aug 2026 16:35:57 +0900 Subject: [PATCH 07/24] ci: rebuild all RcppParallel-linked packages (qs2 hit the same oneTBB skew) --- .github/workflows/test-suite.yaml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-suite.yaml b/.github/workflows/test-suite.yaml index 8e22232..9ae6174 100644 --- a/.github/workflows/test-suite.yaml +++ b/.github/workflows/test-suite.yaml @@ -41,10 +41,12 @@ jobs: extra-packages: any::testthat needs: check - # ponytail: CRAN/RSPM binary skew — stringfish binaries lag RcppParallel's - # oneTBB update. Rebuild from source; drop once upstream binaries sync. - - name: Rebuild stringfish from source (RcppParallel ABI skew) - run: Rscript -e 'install.packages("stringfish", type = "source")' + # ponytail: CRAN/RSPM binary skew — prebuilt binaries (stringfish, qs2, ...) lag + # RcppParallel's oneTBB update (undefined tbb::internal / tbb::task symbols). + # Rebuild everything LinkingTo RcppParallel from source so it links the installed + # copy. Drop once upstream binaries re-sync. + - name: Rebuild RcppParallel-linked packages from source (oneTBB ABI skew) + run: Rscript -e 'bad <- tools::dependsOnPkgs("RcppParallel", dependencies = "LinkingTo"); if (length(bad)) install.packages(bad, type = "source")' - name: Install kaefa package run: R CMD INSTALL . From b944368140c6e85e0378da52c8c1471b37ad05bf Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 4 Aug 2026 17:02:15 +0900 Subject: [PATCH 08/24] ci: source-build stringfish before dep setup (SimDesign source builds load it); fold long line --- .github/workflows/R-CMD-check.yaml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/R-CMD-check.yaml index 4a1f7ed..0a8a5aa 100644 --- a/.github/workflows/R-CMD-check.yaml +++ b/.github/workflows/R-CMD-check.yaml @@ -42,6 +42,12 @@ jobs: http-user-agent: ${{ matrix.config.http-user-agent }} use-public-rspm: true + # ponytail: source-build stringfish BEFORE dependency setup — packages built + # from source during install (e.g. SimDesign) load it at build time and die + # on the broken prebuilt binary (oneTBB ABI skew). Drop with the step below. + - name: Source-build stringfish (RcppParallel oneTBB ABI skew) + run: Rscript -e 'install.packages("stringfish", type = "source")' + - uses: r-lib/actions/setup-r-dependencies@6f6e5bc62fba3a704f74e7ad7ef7676c5c6a2590 # v2 with: extra-packages: | @@ -54,7 +60,9 @@ jobs: # Rebuild everything LinkingTo RcppParallel from source so it links the installed # copy. Drop once upstream binaries re-sync. - name: Rebuild RcppParallel-linked packages from source (oneTBB ABI skew) - run: Rscript -e 'bad <- tools::dependsOnPkgs("RcppParallel", dependencies = "LinkingTo"); if (length(bad)) install.packages(bad, type = "source")' + run: >- + Rscript -e 'bad <- tools::dependsOnPkgs("RcppParallel", dependencies = "LinkingTo"); + if (length(bad)) install.packages(bad, type = "source")' - name: Run Zh formula regression tests run: | From 46012b7622473a4a626e68e678ec9ebaf081caa9 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 4 Aug 2026 17:02:21 +0900 Subject: [PATCH 09/24] ci: source-build stringfish before dep setup (SimDesign source builds load it); fold long line --- .github/workflows/test-fast.yaml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-fast.yaml b/.github/workflows/test-fast.yaml index 46be129..1d70c5e 100644 --- a/.github/workflows/test-fast.yaml +++ b/.github/workflows/test-fast.yaml @@ -20,6 +20,12 @@ jobs: with: use-public-rspm: true + # ponytail: source-build stringfish BEFORE dependency setup — packages built + # from source during install (e.g. SimDesign) load it at build time and die + # on the broken prebuilt binary (oneTBB ABI skew). Drop with the step below. + - name: Source-build stringfish (RcppParallel oneTBB ABI skew) + run: Rscript -e 'install.packages("stringfish", type = "source")' + - uses: r-lib/actions/setup-r-dependencies@6f6e5bc62fba3a704f74e7ad7ef7676c5c6a2590 # v2 with: extra-packages: any::testthat @@ -30,7 +36,9 @@ jobs: # Rebuild everything LinkingTo RcppParallel from source so it links the installed # copy. Drop once upstream binaries re-sync. - name: Rebuild RcppParallel-linked packages from source (oneTBB ABI skew) - run: Rscript -e 'bad <- tools::dependsOnPkgs("RcppParallel", dependencies = "LinkingTo"); if (length(bad)) install.packages(bad, type = "source")' + run: >- + Rscript -e 'bad <- tools::dependsOnPkgs("RcppParallel", dependencies = "LinkingTo"); + if (length(bad)) install.packages(bad, type = "source")' - name: Install kaefa package for fast tests run: R CMD INSTALL . From 4acfcf08552d26f429a4d49ef92ad6bb49abe73e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 4 Aug 2026 17:02:26 +0900 Subject: [PATCH 10/24] ci: source-build stringfish before dep setup (SimDesign source builds load it); fold long line --- .github/workflows/test-suite.yaml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-suite.yaml b/.github/workflows/test-suite.yaml index 9ae6174..896da6b 100644 --- a/.github/workflows/test-suite.yaml +++ b/.github/workflows/test-suite.yaml @@ -36,6 +36,12 @@ jobs: with: use-public-rspm: true + # ponytail: source-build stringfish BEFORE dependency setup — packages built + # from source during install (e.g. SimDesign) load it at build time and die + # on the broken prebuilt binary (oneTBB ABI skew). Drop with the step below. + - name: Source-build stringfish (RcppParallel oneTBB ABI skew) + run: Rscript -e 'install.packages("stringfish", type = "source")' + - uses: r-lib/actions/setup-r-dependencies@6f6e5bc62fba3a704f74e7ad7ef7676c5c6a2590 # v2 with: extra-packages: any::testthat @@ -46,7 +52,9 @@ jobs: # Rebuild everything LinkingTo RcppParallel from source so it links the installed # copy. Drop once upstream binaries re-sync. - name: Rebuild RcppParallel-linked packages from source (oneTBB ABI skew) - run: Rscript -e 'bad <- tools::dependsOnPkgs("RcppParallel", dependencies = "LinkingTo"); if (length(bad)) install.packages(bad, type = "source")' + run: >- + Rscript -e 'bad <- tools::dependsOnPkgs("RcppParallel", dependencies = "LinkingTo"); + if (length(bad)) install.packages(bad, type = "source")' - name: Install kaefa package run: R CMD INSTALL . From d75e72c037e78886603029797f4ff98654f81980 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 4 Aug 2026 17:24:20 +0900 Subject: [PATCH 11/24] ci: real source rebuilds via CRAN cloud (RSPM serves linux binaries for source URLs); rebuild RcppParallel first --- .github/workflows/R-CMD-check.yaml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/R-CMD-check.yaml index 0a8a5aa..ca02cfe 100644 --- a/.github/workflows/R-CMD-check.yaml +++ b/.github/workflows/R-CMD-check.yaml @@ -45,8 +45,12 @@ jobs: # ponytail: source-build stringfish BEFORE dependency setup — packages built # from source during install (e.g. SimDesign) load it at build time and die # on the broken prebuilt binary (oneTBB ABI skew). Drop with the step below. + # repos= must be a real source repo: RSPM serves prebuilt Linux binaries via + # source-style URLs, so type="source" against RSPM does NOT recompile. - name: Source-build stringfish (RcppParallel oneTBB ABI skew) - run: Rscript -e 'install.packages("stringfish", type = "source")' + run: >- + Rscript -e 'install.packages(c("RcppParallel", "stringfish"), type = "source", + repos = "https://cloud.r-project.org")' - uses: r-lib/actions/setup-r-dependencies@6f6e5bc62fba3a704f74e7ad7ef7676c5c6a2590 # v2 with: @@ -61,8 +65,9 @@ jobs: # copy. Drop once upstream binaries re-sync. - name: Rebuild RcppParallel-linked packages from source (oneTBB ABI skew) run: >- - Rscript -e 'bad <- tools::dependsOnPkgs("RcppParallel", dependencies = "LinkingTo"); - if (length(bad)) install.packages(bad, type = "source")' + Rscript -e 'r <- "https://cloud.r-project.org"; install.packages("RcppParallel", type = "source", repos = r); + bad <- tools::dependsOnPkgs("RcppParallel", dependencies = "LinkingTo"); + if (length(bad)) install.packages(bad, type = "source", repos = r)' - name: Run Zh formula regression tests run: | From 28e844259468d9cd4f462d59072f0fb9d606bedf Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 4 Aug 2026 17:24:25 +0900 Subject: [PATCH 12/24] ci: real source rebuilds via CRAN cloud (RSPM serves linux binaries for source URLs); rebuild RcppParallel first --- .github/workflows/test-fast.yaml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-fast.yaml b/.github/workflows/test-fast.yaml index 1d70c5e..ee046af 100644 --- a/.github/workflows/test-fast.yaml +++ b/.github/workflows/test-fast.yaml @@ -23,8 +23,12 @@ jobs: # ponytail: source-build stringfish BEFORE dependency setup — packages built # from source during install (e.g. SimDesign) load it at build time and die # on the broken prebuilt binary (oneTBB ABI skew). Drop with the step below. + # repos= must be a real source repo: RSPM serves prebuilt Linux binaries via + # source-style URLs, so type="source" against RSPM does NOT recompile. - name: Source-build stringfish (RcppParallel oneTBB ABI skew) - run: Rscript -e 'install.packages("stringfish", type = "source")' + run: >- + Rscript -e 'install.packages(c("RcppParallel", "stringfish"), type = "source", + repos = "https://cloud.r-project.org")' - uses: r-lib/actions/setup-r-dependencies@6f6e5bc62fba3a704f74e7ad7ef7676c5c6a2590 # v2 with: @@ -37,8 +41,9 @@ jobs: # copy. Drop once upstream binaries re-sync. - name: Rebuild RcppParallel-linked packages from source (oneTBB ABI skew) run: >- - Rscript -e 'bad <- tools::dependsOnPkgs("RcppParallel", dependencies = "LinkingTo"); - if (length(bad)) install.packages(bad, type = "source")' + Rscript -e 'r <- "https://cloud.r-project.org"; install.packages("RcppParallel", type = "source", repos = r); + bad <- tools::dependsOnPkgs("RcppParallel", dependencies = "LinkingTo"); + if (length(bad)) install.packages(bad, type = "source", repos = r)' - name: Install kaefa package for fast tests run: R CMD INSTALL . From fbba747eaf1399613c7faf5b9332d3f5a1b557a3 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 4 Aug 2026 17:24:30 +0900 Subject: [PATCH 13/24] ci: real source rebuilds via CRAN cloud (RSPM serves linux binaries for source URLs); rebuild RcppParallel first --- .github/workflows/test-suite.yaml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-suite.yaml b/.github/workflows/test-suite.yaml index 896da6b..407a8fa 100644 --- a/.github/workflows/test-suite.yaml +++ b/.github/workflows/test-suite.yaml @@ -39,8 +39,12 @@ jobs: # ponytail: source-build stringfish BEFORE dependency setup — packages built # from source during install (e.g. SimDesign) load it at build time and die # on the broken prebuilt binary (oneTBB ABI skew). Drop with the step below. + # repos= must be a real source repo: RSPM serves prebuilt Linux binaries via + # source-style URLs, so type="source" against RSPM does NOT recompile. - name: Source-build stringfish (RcppParallel oneTBB ABI skew) - run: Rscript -e 'install.packages("stringfish", type = "source")' + run: >- + Rscript -e 'install.packages(c("RcppParallel", "stringfish"), type = "source", + repos = "https://cloud.r-project.org")' - uses: r-lib/actions/setup-r-dependencies@6f6e5bc62fba3a704f74e7ad7ef7676c5c6a2590 # v2 with: @@ -53,8 +57,9 @@ jobs: # copy. Drop once upstream binaries re-sync. - name: Rebuild RcppParallel-linked packages from source (oneTBB ABI skew) run: >- - Rscript -e 'bad <- tools::dependsOnPkgs("RcppParallel", dependencies = "LinkingTo"); - if (length(bad)) install.packages(bad, type = "source")' + Rscript -e 'r <- "https://cloud.r-project.org"; install.packages("RcppParallel", type = "source", repos = r); + bad <- tools::dependsOnPkgs("RcppParallel", dependencies = "LinkingTo"); + if (length(bad)) install.packages(bad, type = "source", repos = r)' - name: Install kaefa package run: R CMD INSTALL . From 46c76d0d4d1e2882f60703b9630677e830bfffb4 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 14 Aug 2026 23:12:35 +0900 Subject: [PATCH 14/24] chore(ci): consolidate r-lib actions v2.12.1 --- .github/workflows/R-CMD-check.yaml | 6 ++--- .github/workflows/test-fast.yaml | 4 +-- .github/workflows/test-suite.yaml | 4 +-- docs/traceability/r-lib-actions-v2.12.1.md | 29 ++++++++++++++++++++++ tests/testthat/test-ci-action-pins.R | 24 ++++++++++++++++++ 5 files changed, 60 insertions(+), 7 deletions(-) create mode 100644 docs/traceability/r-lib-actions-v2.12.1.md create mode 100644 tests/testthat/test-ci-action-pins.R diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/R-CMD-check.yaml index 2fd552b..4d3e02f 100644 --- a/.github/workflows/R-CMD-check.yaml +++ b/.github/workflows/R-CMD-check.yaml @@ -36,13 +36,13 @@ jobs: - uses: r-lib/actions/setup-pandoc@d3c5be51b12e724e68f33216ca3c148b66d5f0b6 # v2 - - uses: r-lib/actions/setup-r@6f6e5bc62fba3a704f74e7ad7ef7676c5c6a2590 # v2 + - uses: r-lib/actions/setup-r@d3c5be51b12e724e68f33216ca3c148b66d5f0b6 # v2 with: r-version: ${{ matrix.config.r }} http-user-agent: ${{ matrix.config.http-user-agent }} use-public-rspm: true - - uses: r-lib/actions/setup-r-dependencies@6f6e5bc62fba3a704f74e7ad7ef7676c5c6a2590 # v2 + - uses: r-lib/actions/setup-r-dependencies@d3c5be51b12e724e68f33216ca3c148b66d5f0b6 # v2 with: extra-packages: | any::rcmdcheck @@ -54,7 +54,7 @@ jobs: Rscript -e 'install.packages(".", repos = NULL, type = "source")' Rscript -e 'library(kaefa); testthat::test_file("tests/testthat/test-zh-misfit-decision-rule.R")' - - uses: r-lib/actions/check-r-package@6f6e5bc62fba3a704f74e7ad7ef7676c5c6a2590 # v2 + - uses: r-lib/actions/check-r-package@d3c5be51b12e724e68f33216ca3c148b66d5f0b6 # v2 with: error-on: '"error"' upload-snapshots: true diff --git a/.github/workflows/test-fast.yaml b/.github/workflows/test-fast.yaml index ea1cf69..51c82c3 100644 --- a/.github/workflows/test-fast.yaml +++ b/.github/workflows/test-fast.yaml @@ -16,11 +16,11 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: r-lib/actions/setup-r@6f6e5bc62fba3a704f74e7ad7ef7676c5c6a2590 # v2 + - uses: r-lib/actions/setup-r@d3c5be51b12e724e68f33216ca3c148b66d5f0b6 # v2 with: use-public-rspm: true - - uses: r-lib/actions/setup-r-dependencies@6f6e5bc62fba3a704f74e7ad7ef7676c5c6a2590 # v2 + - uses: r-lib/actions/setup-r-dependencies@d3c5be51b12e724e68f33216ca3c148b66d5f0b6 # v2 with: extra-packages: any::testthat needs: check diff --git a/.github/workflows/test-suite.yaml b/.github/workflows/test-suite.yaml index 439b9b7..fc1b872 100644 --- a/.github/workflows/test-suite.yaml +++ b/.github/workflows/test-suite.yaml @@ -32,11 +32,11 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: r-lib/actions/setup-r@6f6e5bc62fba3a704f74e7ad7ef7676c5c6a2590 # v2 + - uses: r-lib/actions/setup-r@d3c5be51b12e724e68f33216ca3c148b66d5f0b6 # v2 with: use-public-rspm: true - - uses: r-lib/actions/setup-r-dependencies@6f6e5bc62fba3a704f74e7ad7ef7676c5c6a2590 # v2 + - uses: r-lib/actions/setup-r-dependencies@d3c5be51b12e724e68f33216ca3c148b66d5f0b6 # v2 with: extra-packages: any::testthat needs: check diff --git a/docs/traceability/r-lib-actions-v2.12.1.md b/docs/traceability/r-lib-actions-v2.12.1.md new file mode 100644 index 0000000..78f6a14 --- /dev/null +++ b/docs/traceability/r-lib-actions-v2.12.1.md @@ -0,0 +1,29 @@ +# r-lib/actions v2.12.1 공급망 추적 + +## 결정 + +세 개의 R 검증 workflow에서 사용하는 setup-pandoc, setup-r, +setup-r-dependencies, check-r-package를 v2.12.1 release commit +d3c5be51b12e724e68f33216ca3c148b66d5f0b6으로 통일한다. 전체 commit SHA 외의 태그·브랜치·짧은 SHA는 +회귀 계약이 거부한다. + +## 호환성 범위 + +공식 NEWS에 따르면 v2.12는 Node.js 24 전환, public RSPM 기본값 조정, +아키텍처별 cache key와 Pandoc 3.8.3을 포함하고, v2.12.1은 setup-r URL parser +경고와 Quarto action을 갱신한다. 현재 workflow의 R matrix, 권한, testthat 실행, +--no-tests 분리와 scheduled full-suite 계약은 변경하지 않는다. + +## 되돌리기 + +runner 또는 package 호환성 회귀가 확인되면 네 action을 함께 마지막 검증 SHA로 +되돌리고 R-CMD-check, fast/full test suite와 중앙 보안 검사를 같은 헤드에서 다시 +수행한다. 일부 action만 되돌리거나 이동 태그로 우회하지 않는다. + +## 참고문헌 + +R-lib. (2026, June 23). *r-lib/actions v2.12.1* [Software release notes]. +https://github.com/r-lib/actions/blob/v2-branch/NEWS.md + +R-lib. (2026). *Update NEWS for v2.12.1* [Source code commit]. +https://github.com/r-lib/actions/commit/d3c5be51b12e724e68f33216ca3c148b66d5f0b6 diff --git a/tests/testthat/test-ci-action-pins.R b/tests/testthat/test-ci-action-pins.R new file mode 100644 index 0000000..1e9a553 --- /dev/null +++ b/tests/testthat/test-ci-action-pins.R @@ -0,0 +1,24 @@ +testthat::test_that("CI r-lib actions use the reviewed v2.12.1 commit", { + workflow_paths <- c( + testthat::test_path("..", "..", ".github", "workflows", "R-CMD-check.yaml"), + testthat::test_path("..", "..", ".github", "workflows", "test-fast.yaml"), + testthat::test_path("..", "..", ".github", "workflows", "test-suite.yaml") + ) + workflow_text <- paste( + unlist(lapply(workflow_paths, readLines, warn = FALSE)), + collapse = "\n" + ) + action_refs <- regmatches( + workflow_text, + gregexpr( + "r-lib/actions/(setup-pandoc|setup-r-dependencies|setup-r|check-r-package)@[^[:space:]#]+", + workflow_text, + perl = TRUE + ) + )[[1]] + + testthat::expect_gt(length(action_refs), 0) + testthat::expect_true(all( + sub("^.*@", "", action_refs) == "d3c5be51b12e724e68f33216ca3c148b66d5f0b6" + )) +}) From bbda0c14cbdc26ec34637e1e791bf5e657d069f8 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 14 Aug 2026 23:30:45 +0900 Subject: [PATCH 15/24] test(ci): reject substituted r-lib actions --- .../2026-07-02-kaefa-2b-krw-sale-readiness.md | 4 +- docs/traceability/r-lib-actions-v2.12.1.md | 2 +- tests/testthat/test-ci-action-pins.R | 47 +++++++++++++------ 3 files changed, 36 insertions(+), 17 deletions(-) diff --git a/docs/superpowers/plans/2026-07-02-kaefa-2b-krw-sale-readiness.md b/docs/superpowers/plans/2026-07-02-kaefa-2b-krw-sale-readiness.md index d5192fe..4be08d6 100644 --- a/docs/superpowers/plans/2026-07-02-kaefa-2b-krw-sale-readiness.md +++ b/docs/superpowers/plans/2026-07-02-kaefa-2b-krw-sale-readiness.md @@ -251,11 +251,11 @@ Guardrails: steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - uses: r-lib/actions/setup-r@6f6e5bc62fba3a704f74e7ad7ef7676c5c6a2590 # v2 + - uses: r-lib/actions/setup-r@d3c5be51b12e724e68f33216ca3c148b66d5f0b6 # v2 with: use-public-rspm: true - - uses: r-lib/actions/setup-r-dependencies@6f6e5bc62fba3a704f74e7ad7ef7676c5c6a2590 # v2 + - uses: r-lib/actions/setup-r-dependencies@d3c5be51b12e724e68f33216ca3c148b66d5f0b6 # v2 with: extra-packages: any::testthat needs: check diff --git a/docs/traceability/r-lib-actions-v2.12.1.md b/docs/traceability/r-lib-actions-v2.12.1.md index 78f6a14..fa52bd6 100644 --- a/docs/traceability/r-lib-actions-v2.12.1.md +++ b/docs/traceability/r-lib-actions-v2.12.1.md @@ -23,7 +23,7 @@ runner 또는 package 호환성 회귀가 확인되면 네 action을 함께 마 ## 참고문헌 R-lib. (2026, June 23). *r-lib/actions v2.12.1* [Software release notes]. -https://github.com/r-lib/actions/blob/v2-branch/NEWS.md +https://github.com/r-lib/actions/blob/d3c5be51b12e724e68f33216ca3c148b66d5f0b6/NEWS.md R-lib. (2026). *Update NEWS for v2.12.1* [Source code commit]. https://github.com/r-lib/actions/commit/d3c5be51b12e724e68f33216ca3c148b66d5f0b6 diff --git a/tests/testthat/test-ci-action-pins.R b/tests/testthat/test-ci-action-pins.R index 1e9a553..36cf087 100644 --- a/tests/testthat/test-ci-action-pins.R +++ b/tests/testthat/test-ci-action-pins.R @@ -1,24 +1,43 @@ -testthat::test_that("CI r-lib actions use the reviewed v2.12.1 commit", { +testthat::test_that("CI uses exactly the reviewed r-lib action references", { + reviewed_sha <- "d3c5be51b12e724e68f33216ca3c148b66d5f0b6" workflow_paths <- c( testthat::test_path("..", "..", ".github", "workflows", "R-CMD-check.yaml"), testthat::test_path("..", "..", ".github", "workflows", "test-fast.yaml"), testthat::test_path("..", "..", ".github", "workflows", "test-suite.yaml") ) - workflow_text <- paste( - unlist(lapply(workflow_paths, readLines, warn = FALSE)), - collapse = "\n" + expected_actions <- list( + "R-CMD-check.yaml" = c( + "setup-pandoc", + "setup-r", + "setup-r-dependencies", + "check-r-package" + ), + "test-fast.yaml" = c("setup-r", "setup-r-dependencies"), + "test-suite.yaml" = c("setup-r", "setup-r-dependencies") ) - action_refs <- regmatches( - workflow_text, - gregexpr( - "r-lib/actions/(setup-pandoc|setup-r-dependencies|setup-r|check-r-package)@[^[:space:]#]+", + action_pattern <- paste0( + "r-lib/actions/", + "(setup-pandoc|setup-r-dependencies|setup-r|check-r-package)", + "@[^[:space:]#]+" + ) + + for (workflow_path in workflow_paths) { + workflow_text <- paste(readLines(workflow_path, warn = FALSE), collapse = "\n") + action_refs <- regmatches( workflow_text, - perl = TRUE + gregexpr(action_pattern, workflow_text, perl = TRUE) + )[[1]] + expected_refs <- paste0( + "r-lib/actions/", + expected_actions[[basename(workflow_path)]], + "@", + reviewed_sha ) - )[[1]] - testthat::expect_gt(length(action_refs), 0) - testthat::expect_true(all( - sub("^.*@", "", action_refs) == "d3c5be51b12e724e68f33216ca3c148b66d5f0b6" - )) + testthat::expect_identical( + action_refs, + expected_refs, + info = paste("Unexpected r-lib action set in", basename(workflow_path)) + ) + } }) From 6abb5565aa80d481d05891590982a136d41fb58b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 15 Aug 2026 01:24:38 +0900 Subject: [PATCH 16/24] fix(ci): refresh macOS dependency ABI cache --- .github/workflows/R-CMD-check.yaml | 2 ++ tests/testthat/test-ci-action-pins.R | 21 +++++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/R-CMD-check.yaml index 4d3e02f..511eac9 100644 --- a/.github/workflows/R-CMD-check.yaml +++ b/.github/workflows/R-CMD-check.yaml @@ -44,6 +44,8 @@ jobs: - uses: r-lib/actions/setup-r-dependencies@d3c5be51b12e724e68f33216ca3c148b66d5f0b6 # v2 with: + # Refresh the dependency cache after the macOS qs2/RcppParallel TBB ABI mismatch. + cache-version: '2' extra-packages: | any::rcmdcheck any::testthat diff --git a/tests/testthat/test-ci-action-pins.R b/tests/testthat/test-ci-action-pins.R index 36cf087..41706ab 100644 --- a/tests/testthat/test-ci-action-pins.R +++ b/tests/testthat/test-ci-action-pins.R @@ -41,3 +41,24 @@ testthat::test_that("CI uses exactly the reviewed r-lib action references", { ) } }) + +testthat::test_that("R CMD check refreshes the reviewed dependency cache ABI", { + workflow_path <- testthat::test_path( + "..", "..", ".github", "workflows", "R-CMD-check.yaml" + ) + workflow_lines <- readLines(workflow_path, warn = FALSE) + dependency_step <- grep( + "r-lib/actions/setup-r-dependencies@", + workflow_lines, + fixed = TRUE + ) + testthat::expect_length(dependency_step, 1L) + dependency_block <- workflow_lines[ + dependency_step:min(dependency_step + 8L, length(workflow_lines)) + ] + + testthat::expect_true( + any(grepl("cache-version: '2'", dependency_block, fixed = TRUE)), + info = "The reviewed macOS TBB ABI cache refresh must remain explicit" + ) +}) From 5a2a54126c742996aca663cae1bc7eea0b9db2bd Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 15 Aug 2026 01:34:55 +0900 Subject: [PATCH 17/24] test(ci): require active cache version configuration --- tests/testthat/test-ci-action-pins.R | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/testthat/test-ci-action-pins.R b/tests/testthat/test-ci-action-pins.R index 41706ab..fae6a3e 100644 --- a/tests/testthat/test-ci-action-pins.R +++ b/tests/testthat/test-ci-action-pins.R @@ -57,8 +57,16 @@ testthat::test_that("R CMD check refreshes the reviewed dependency cache ABI", { dependency_step:min(dependency_step + 8L, length(workflow_lines)) ] + active_cache_version_pattern <- paste0( + "^[[:space:]]*cache-version:[[:space:]]*", + "['\\\"]2['\\\"][[:space:]]*(#.*)?$" + ) + testthat::expect_false( + grepl(active_cache_version_pattern, "# cache-version: '2'", perl = TRUE), + info = "A commented cache-version example must not satisfy the contract" + ) testthat::expect_true( - any(grepl("cache-version: '2'", dependency_block, fixed = TRUE)), + any(grepl(active_cache_version_pattern, dependency_block, perl = TRUE)), info = "The reviewed macOS TBB ABI cache refresh must remain explicit" ) }) From ce826e9e7188efe0652f3e368eef726a9ac65166 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 15 Aug 2026 02:49:23 +0900 Subject: [PATCH 18/24] ci: rebuild macOS TBB dependents from source --- .github/workflows/R-CMD-check.yaml | 4 ++++ tests/testthat/test-ci-action-pins.R | 33 ++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/R-CMD-check.yaml index 511eac9..dbc4de2 100644 --- a/.github/workflows/R-CMD-check.yaml +++ b/.github/workflows/R-CMD-check.yaml @@ -51,6 +51,10 @@ jobs: any::testthat needs: check + - name: Rebuild macOS TBB dependents from source + if: runner.os == 'macOS' + run: Rscript -e 'install.packages(c("RcppParallel", "qs2"), repos = "https://cloud.r-project.org", type = "source")' + - name: Run Zh formula regression tests run: | Rscript -e 'install.packages(".", repos = NULL, type = "source")' diff --git a/tests/testthat/test-ci-action-pins.R b/tests/testthat/test-ci-action-pins.R index fae6a3e..415fa6f 100644 --- a/tests/testthat/test-ci-action-pins.R +++ b/tests/testthat/test-ci-action-pins.R @@ -70,3 +70,36 @@ testthat::test_that("R CMD check refreshes the reviewed dependency cache ABI", { info = "The reviewed macOS TBB ABI cache refresh must remain explicit" ) }) + +testthat::test_that("macOS rebuilds TBB-linked packages from source", { + workflow_path <- testthat::test_path( + "..", "..", ".github", "workflows", "R-CMD-check.yaml" + ) + workflow_lines <- readLines(workflow_path, warn = FALSE) + rebuild_step <- grep( + "name: Rebuild macOS TBB dependents from source", + workflow_lines, + fixed = TRUE + ) + testthat::expect_length(rebuild_step, 1L) + rebuild_block <- workflow_lines[ + rebuild_step:min(rebuild_step + 3L, length(workflow_lines)) + ] + + testthat::expect_true( + any(trimws(rebuild_block) == "if: runner.os == 'macOS'"), + info = "Native package rebuild must remain scoped to macOS" + ) + testthat::expect_true( + any(grepl( + 'install.packages(c("RcppParallel", "qs2")', + rebuild_block, + fixed = TRUE + )), + info = "Both sides of the observed qs2/RcppParallel ABI boundary must rebuild" + ) + testthat::expect_true( + any(grepl('type = "source"', rebuild_block, fixed = TRUE)), + info = "macOS native packages must compile against the same local TBB ABI" + ) +}) From 5092a70c9737221d6367e74643d06980609fe0b1 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 15 Aug 2026 03:11:57 +0900 Subject: [PATCH 19/24] test(ci): pin every oneTBB source rebuild argument --- tests/testthat/test-ci-action-pins.R | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/testthat/test-ci-action-pins.R b/tests/testthat/test-ci-action-pins.R index 5ab2737..4203052 100644 --- a/tests/testthat/test-ci-action-pins.R +++ b/tests/testthat/test-ci-action-pins.R @@ -118,13 +118,23 @@ testthat::test_that("CI rebuilds oneTBB dependents around dependency setup", { pre_block, fixed = TRUE ))) + testthat::expect_true(any(grepl( + 'type = "source"', + pre_block, + fixed = TRUE + ))) testthat::expect_true(any(grepl( 'dependsOnPkgs("RcppParallel", dependencies = "LinkingTo")', post_block, fixed = TRUE ))) testthat::expect_true(any(grepl( - 'type = "source"', + 'install.packages("RcppParallel", type = "source"', + post_block, + fixed = TRUE + ))) + testthat::expect_true(any(grepl( + 'install.packages(linked, type = "source"', post_block, fixed = TRUE ))) From 3792f42265b3c4494f87f398e3645ef29529642f Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 17 Aug 2026 16:41:24 +0000 Subject: [PATCH 20/24] feat(recovery): add five-repeat AEFA RMSE protocol Introduce internal true-parameter recovery helpers and pin the Harwell et al. RMSE definition, item-name alignment, and five-repeat summary schema. Live .mirt recovery runs in the full suite; aefa() five-seed recovery stays behind RUN_FULL_AEFA_TESTS. Multilevel, multiple-membership, and time-flow surfaces are recorded as explicit exclusions. Co-authored-by: Seongho Bae --- .github/workflows/test-fast.yaml | 2 + .github/workflows/test-suite.yaml | 5 +- AGENTS.md | 1 + ARCHITECTURE.md | 8 +- CLAUDE.md | 4 + R/recovery.R | 135 ++++++++++++++++++ docs/papers/README.md | 29 +++- docs/product/kaefa-core-api-contract.md | 3 + docs/traceability/aefa-parameter-recovery.md | 60 ++++++++ inst/WORDLIST | 12 ++ tests/FAST_TESTS.md | 2 + tests/testthat/test-aefa-parameter-recovery.R | 84 +++++++++++ tests/testthat/test-aefa-recovery-fits.R | 116 +++++++++++++++ 13 files changed, 456 insertions(+), 5 deletions(-) create mode 100644 R/recovery.R create mode 100644 docs/traceability/aefa-parameter-recovery.md create mode 100644 tests/testthat/test-aefa-parameter-recovery.R create mode 100644 tests/testthat/test-aefa-recovery-fits.R diff --git a/.github/workflows/test-fast.yaml b/.github/workflows/test-fast.yaml index 4032abe..c0c424a 100644 --- a/.github/workflows/test-fast.yaml +++ b/.github/workflows/test-fast.yaml @@ -53,4 +53,6 @@ jobs: reporter = reporter) testthat::test_file("tests/testthat/test-core-api-contract.R", reporter = reporter) + testthat::test_file("tests/testthat/test-aefa-parameter-recovery.R", + reporter = reporter) RSCRIPT diff --git a/.github/workflows/test-suite.yaml b/.github/workflows/test-suite.yaml index 886de1b..0694152 100644 --- a/.github/workflows/test-suite.yaml +++ b/.github/workflows/test-suite.yaml @@ -1,7 +1,8 @@ name: test-suite -# Runs the complete testthat suite (26 files), which the previous CI setup -# skipped: R-CMD-check runs with --no-tests and test-fast runs only 3 files. +# Runs the complete testthat suite, which the previous CI setup skipped: +# R-CMD-check runs with --no-tests and test-fast runs a small productization +# subset. # On push/PR the suite's own CI guards (.skip_expensive_ci_calls, skip_on_ci) # keep expensive AEFA estimations out. The weekly scheduled run and manual # dispatch set RUN_FULL_AEFA_TESTS=1, which unlocks the estimations guarded diff --git a/AGENTS.md b/AGENTS.md index e83e8e3..71aaee8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -11,6 +11,7 @@ Use this file as the local operating guide when modifying `kaefa`. - Core orchestration: `R/kaefa.R` - Estimation engine: `R/newEngine.R` - Shared helpers: `R/utils.R` +- Recovery protocol: `R/recovery.R` - Shiny app: `inst/shiny-app/app.R` - Tests: `tests/testthat/` - Local CI workflow definitions: `.github/workflows/` diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 6ddd063..d944ebb 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -1,6 +1,6 @@ # kaefa Architecture -Last updated: 2026-02-14 +Last updated: 2026-08-17 ## Purpose @@ -33,6 +33,8 @@ explicitly requires vendored source integration. - `R/kaefa.R`: public orchestration entry points and exported runtime behavior. - `R/newEngine.R`: candidate-model estimation engine used by the AEFA loop. - `R/utils.R`: helper routines and shared utilities. +- `R/recovery.R`: internal true-parameter RMSE helpers used by the AEFA + recovery protocol. Not a public API. - `inst/shiny-app/app.R`: bundled Shiny interface logic. - `inst/shiny-app/README.md`: Shiny usage and minimal UI configuration guide. - `tests/testthat/*.R`: functional, regression, and integration tests. @@ -58,6 +60,10 @@ explicitly requires vendored source integration. - PR merge requires review approval and resolved conversations. - Required checks include R-CMD-check matrix and dependency review. +- Buyer-facing recovery evidence is the five-repeat RMSE protocol in + `tests/testthat/test-aefa-parameter-recovery.R`. The formula, alignment, + and coverage-exclusion contracts run in `test-fast`. Live `aefa()` + five-seed recovery is gated by `RUN_FULL_AEFA_TESTS=1`. - If code scanning is enabled later, alerts can be tracked via GitHub code scanning APIs. diff --git a/CLAUDE.md b/CLAUDE.md index 9747fed..76ab760 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -57,6 +57,8 @@ testthat::test_file("tests/testthat/test-shiny-product-surface.R", reporter = reporter) testthat::test_file("tests/testthat/test-core-api-contract.R", reporter = reporter) +testthat::test_file("tests/testthat/test-aefa-parameter-recovery.R", + reporter = reporter) RSCRIPT # R CMD check as CI runs it (note: tests are skipped here) @@ -124,6 +126,8 @@ psychometrics (see AGENTS.md). - `R/newEngine.R` — `engineAEFA()`: the candidate-model estimation engine used by the aefa loop (MCMC/estimation controls such as NCYCLES, BURNIN, SEMCYCLES, rotation choices, etc.). +- `R/recovery.R` — internal true-parameter RMSE helpers for the AEFA recovery + protocol (not exported). - `R/utils.R` — shared internals (`.mirt`/`.mixedmirt` wrappers around mirt, `.covdataClassifieder`, `.covdataFixedEffectComb`, `.exportParmsEME`) and the theta-prior utilities `fitThetaPrior()`, `testThetaPriorCalibration()`, diff --git a/R/recovery.R b/R/recovery.R new file mode 100644 index 0000000..8e7d99c --- /dev/null +++ b/R/recovery.R @@ -0,0 +1,135 @@ +# Internal true-parameter recovery helpers for AEFA / IRT Monte Carlo evidence. +# These functions are not exported. Buyer-facing recovery is the test protocol +# in tests/testthat/test-aefa-parameter-recovery.R and the provenance note in +# docs/traceability/aefa-parameter-recovery.md. + +.parameterRecoveryRMSE <- function(estimated, truth) { + estimated <- as.numeric(estimated) + truth <- as.numeric(truth) + if (length(estimated) != length(truth)) { + stop("Estimated and true vectors must have the same length.", call. = FALSE) + } + if (!length(estimated)) { + stop("Cannot compute RMSE from empty parameter vectors.", call. = FALSE) + } + if (any(!is.finite(estimated)) || any(!is.finite(truth))) { + stop("RMSE requires finite estimated and true values.", call. = FALSE) + } + sqrt(mean((estimated - truth)^2)) +} + +.alignIrtItemParameters <- function(estimated, truth, columns = c("a", "b")) { + if (!is.data.frame(estimated) && !is.matrix(estimated)) { + stop("estimated must be a matrix or data.frame of item parameters.", call. = FALSE) + } + if (!is.data.frame(truth) && !is.matrix(truth)) { + stop("truth must be a matrix or data.frame of item parameters.", call. = FALSE) + } + estimated <- as.data.frame(estimated, stringsAsFactors = FALSE) + truth <- as.data.frame(truth, stringsAsFactors = FALSE) + if (is.null(rownames(estimated)) || is.null(rownames(truth))) { + stop( + "Estimated and true parameter tables must have item names as row names.", + call. = FALSE + ) + } + missing_estimated <- setdiff(columns, colnames(estimated)) + missing_truth <- setdiff(columns, colnames(truth)) + if (length(missing_estimated) || length(missing_truth)) { + stop( + "Missing recovery columns: ", + paste(unique(c(missing_estimated, missing_truth)), collapse = ", "), + call. = FALSE + ) + } + shared <- intersect(rownames(estimated), rownames(truth)) + if (!length(shared)) { + stop("No shared item names between estimated and true parameters.", call. = FALSE) + } + list( + estimated = estimated[shared, columns, drop = FALSE], + truth = truth[shared, columns, drop = FALSE], + items = shared + ) +} + +.summariseRecoveryRepeats <- function(rmse_by_repeat) { + if (!is.data.frame(rmse_by_repeat)) { + stop("rmse_by_repeat must be a data.frame.", call. = FALSE) + } + required <- c("seed", "parameter", "rmse") + missing <- setdiff(required, names(rmse_by_repeat)) + if (length(missing)) { + stop( + "rmse_by_repeat must contain columns: ", + paste(required, collapse = ", "), + call. = FALSE + ) + } + seeds <- unique(rmse_by_repeat$seed) + if (length(seeds) != 5L) { + stop("Recovery protocol requires exactly 5 repeats.", call. = FALSE) + } + parameters <- unique(as.character(rmse_by_repeat$parameter)) + summary_rows <- lapply(parameters, function(parameter_name) { + values <- rmse_by_repeat$rmse[as.character(rmse_by_repeat$parameter) == parameter_name] + data.frame( + parameter = parameter_name, + n_repeats = length(values), + mean_rmse = mean(values), + sd_rmse = stats::sd(values), + stringsAsFactors = FALSE + ) + }) + list( + n_repeats = 5L, + seeds = seeds, + per_run = rmse_by_repeat[order(rmse_by_repeat$seed, rmse_by_repeat$parameter), ], + summary = do.call(rbind, summary_rows) + ) +} + +.extractAefaIrtItems <- function(fit) { + if (inherits(fit, "aefa") || (is.list(fit) && !is.null(fit$estModelTrials))) { + trials <- fit$estModelTrials + if (!length(trials)) { + stop("aefa history has no estimated model trials.", call. = FALSE) + } + fit <- trials[[length(trials)]] + } + if (!methods::is(fit, "SingleGroupClass") && !methods::is(fit, "MixedClass")) { + stop("Recovery extraction requires an aefa history or a mirt model.", call. = FALSE) + } + items <- mirt::coef(fit, IRTpars = TRUE, simplify = TRUE)$items + if (is.null(items)) { + stop("Could not extract IRT item parameters.", call. = FALSE) + } + as.data.frame(items, stringsAsFactors = FALSE) +} + +.recoveryCoverageExclusions <- function() { + data.frame( + surface = c( + "unidimensional 2PL via .mirt", + "AEFA greedy search on unidimensional 2PL", + "mixedmirt multilevel / random effects", + "multiple-membership crossed random effects", + "time-flow / longitudinal membership" + ), + status = c( + "covered", + "covered when RUN_FULL_AEFA_TESTS=1", + "excluded", + "excluded", + "excluded" + ), + reason = c( + "Known-true 2PL simulation with IRT a/b RMSE.", + "Known-true 2PL simulation through aefa() with five seeds.", + "Engine exposes .mixedmirt, but no true-parameter RMSE protocol yet.", + "random = ~1|G formulas exist, but no recovery design is registered.", + "No time-indexed membership design is implemented in kaefa-core." + ), + stringsAsFactors = FALSE + ) +} diff --git a/docs/papers/README.md b/docs/papers/README.md index cc2caa8..718a17e 100644 --- a/docs/papers/README.md +++ b/docs/papers/README.md @@ -119,10 +119,35 @@ cited with its DOI. Open-access / preprint links are noted where available. needed to reconstruct it. Kaefa accepts DIC only when the fitted model supplies a finite DIC value and never relabels AIC as DIC. +## 7. Monte Carlo parameter recovery RMSE + +- **Source:** Harwell, M. R., Stone, C. A., Hsu, T.-C., & Kirisci, L. (1996). + Monte Carlo studies in item response theory. *Applied Psychological + Measurement, 20*(2), 101-125. + DOI: [10.1177/014662169602000201](https://doi.org/10.1177/014662169602000201) +- **Canonical equation.** For recovered parameters \(\hat{\theta}\) and known + true parameters \(\theta\), + + RMSE = sqrt( mean( (hat_theta - theta)^2 ) ) + + Harwell et al. treat RMSE (and related Monte Carlo error summaries) as the + standard way to judge whether an IRT estimator recovers a known generating + model. kaefa uses that definition on IRT `a` and `b` after name alignment. +- **Usage in kaefa:** internal helpers in `R/recovery.R` and the five-repeat + protocol in `docs/traceability/aefa-parameter-recovery.md`. The formula, + alignment, and five-repeat schema are pinned by + `tests/testthat/test-aefa-parameter-recovery.R`. Live `.mirt` and `aefa()` + recovery fits live in `tests/testthat/test-aefa-recovery-fits.R`. +- **Boundary.** Multilevel `mixedmirt`, multiple-membership, and time-flow + designs are recorded as exclusions until a true-parameter design is + registered. The engine remains R/`mirt`; this protocol does not introduce a + Rust or GPU numeric core. + ## Audit note kaefa does **not** re-implement `P(theta)`, the MML-EM E-/M-step, `S-X2`, `infit`, or `outfit`; those are delegated verbatim to `mirt` and remain subject to `mirt`'s validation. Package-local formulas and decision rules are pinned above: -the `Zh` cutoff, the exact Hurvich-Tsai AICc correction, and the explicit -posterior-information boundary that prevents DIC from being fabricated. +the `Zh` cutoff, the exact Hurvich-Tsai AICc correction, the explicit +posterior-information boundary that prevents DIC from being fabricated, and +the Harwell et al. RMSE recovery definition. diff --git a/docs/product/kaefa-core-api-contract.md b/docs/product/kaefa-core-api-contract.md index 5d81b81..ad361f7 100644 --- a/docs/product/kaefa-core-api-contract.md +++ b/docs/product/kaefa-core-api-contract.md @@ -172,6 +172,9 @@ exist and pass in CI: - theta-prior helpers cover fit success, unsupported distribution, and fallback behavior. - dot-prefixed helpers are either documented or covered by deprecation tests. +- true-parameter recovery reports five-repeat RMSE for unidimensional 2PL + `a`/`b` after item-name alignment, and records multilevel / multiple-membership + / time-flow surfaces as exclusions until a registered design exists. ## Split Readiness Rule diff --git a/docs/traceability/aefa-parameter-recovery.md b/docs/traceability/aefa-parameter-recovery.md new file mode 100644 index 0000000..a3b418e --- /dev/null +++ b/docs/traceability/aefa-parameter-recovery.md @@ -0,0 +1,60 @@ +# AEFA true-parameter recovery protocol + +## Decision + +Buyer-facing recovery evidence for `kaefa-core` is a five-repeat RMSE protocol +against known-true item parameters. The protocol is implemented by internal +helpers in `R/recovery.R` and pinned by +`tests/testthat/test-aefa-parameter-recovery.R` (formula, alignment, schema, +coverage exclusions) plus `tests/testthat/test-aefa-recovery-fits.R` (live +fits). + +This is not a public API. External behaviour of `aefa()`, `engineAEFA()`, and +`.mirt()` is unchanged. + +## RMSE definition + +For a recovered parameter vector \(\hat{\theta}\) and a true vector \(\theta\), + + RMSE = sqrt( mean( (hat_theta - theta)^2 ) ) + +Items are aligned by name before the difference is taken. The required IRT +columns for the unidimensional 2PL case are `a` (discrimination) and `b` +(difficulty) from `mirt::coef(..., IRTpars = TRUE, simplify = TRUE)$items`. + +Exactly five seeds are required. The summary schema is: per-run `seed`, +`parameter`, `rmse`, plus `mean_rmse` and `sd_rmse` by parameter. + +## Current coverage + +Covered: + +- Unidimensional 2PL recovery through `kaefa::.mirt`. +- AEFA greedy search on the same 2PL design when `RUN_FULL_AEFA_TESTS=1`. + +Explicitly excluded until a registered design exists: + +- `mixedmirt` multilevel / random-effect recovery. +- Multiple-membership crossed random effects. +- Time-flow / longitudinal membership. + +The exclusion log is asserted in the recovery contract so a later claim cannot +silently treat those surfaces as covered. + +## Compatibility and rollback + +The helpers are unused by the estimation loop. Removing `R/recovery.R` and the +two test files restores the previous evidence surface. Do not export the +helpers or substitute a different error metric without updating this note and +the formula tests together. + +## References + +Harwell, M. R., Stone, C. A., Hsu, T.-C., & Kirisci, L. (1996). Monte Carlo +studies in item response theory. *Applied Psychological Measurement, 20*(2), +101-125. + + +Chalmers, R. P. (2012). mirt: A multidimensional item response theory package +for the R environment. *Journal of Statistical Software, 48*(6), 1-29. + diff --git a/inst/WORDLIST b/inst/WORDLIST index f03f48c..a0f3812 100644 --- a/inst/WORDLIST +++ b/inst/WORDLIST @@ -88,18 +88,23 @@ Behavioral Behavioural Bentler Bifactor +Carlo DIC DIF DiscreteClass EM FUN +GPU GenRandomPars +Harwell +Hsu IRT Jennrich Jiao Jin Kamata Kang +Kirisci LAN LCA Leiman @@ -114,6 +119,7 @@ MPI Mansolf Mels MixedClass +Monte MultipleGroupClass NCYCLES NROW @@ -126,16 +132,20 @@ Psychometrika Q1 QMC RM +RMSE Rasch +Reckase Reise RemoteClusters Robbin +Rust SEMCYCLES SQUAREM SSH S_X2 Schmid SingleGroupClass +Stone Testlet UIRT X2 @@ -192,6 +202,7 @@ maximising minExtraction mirt mirtModel +mixedmirt modelSelectionCriteria ncat newEngine @@ -236,6 +247,7 @@ tracelines tryLCA turnOffMixedEst ubuntu +unidimensional v048 v1 z0 diff --git a/tests/FAST_TESTS.md b/tests/FAST_TESTS.md index 5629ada..23d0d6e 100644 --- a/tests/FAST_TESTS.md +++ b/tests/FAST_TESTS.md @@ -13,6 +13,8 @@ testthat::test_file("tests/testthat/test-shiny-product-surface.R", reporter = reporter) testthat::test_file("tests/testthat/test-core-api-contract.R", reporter = reporter) +testthat::test_file("tests/testthat/test-aefa-parameter-recovery.R", + reporter = reporter) ``` `R-CMD-check` remains the package installation and multi-OS compatibility gate. diff --git a/tests/testthat/test-aefa-parameter-recovery.R b/tests/testthat/test-aefa-parameter-recovery.R new file mode 100644 index 0000000..932fa5b --- /dev/null +++ b/tests/testthat/test-aefa-parameter-recovery.R @@ -0,0 +1,84 @@ +test_that("RMSE matches the Monte Carlo recovery definition", { + .ensure_kaefa_namespace() + estimated <- c(0.8, 1.1, 1.4) + truth <- c(1.0, 1.0, 1.0) + expected <- sqrt(mean((estimated - truth)^2)) + testthat::expect_equal(kaefa:::.parameterRecoveryRMSE(estimated, truth), expected) + testthat::expect_equal(expected, sqrt((0.04 + 0.01 + 0.16) / 3)) + testthat::expect_error(kaefa:::.parameterRecoveryRMSE(c(1, 2), 1), "same length") + testthat::expect_error(kaefa:::.parameterRecoveryRMSE(numeric(), numeric()), "empty") + testthat::expect_error(kaefa:::.parameterRecoveryRMSE(c(1, NA), c(1, 2)), "finite") +}) + +test_that("IRT parameter alignment is by item name and required columns", { + .ensure_kaefa_namespace() + estimated <- data.frame( + a = c(1.1, 0.9), + b = c(-0.2, 0.4), + row.names = c("Item2", "Item1") + ) + truth <- data.frame( + a = c(1.0, 1.2), + b = c(0.5, -0.1), + row.names = c("Item1", "Item2") + ) + aligned <- kaefa:::.alignIrtItemParameters(estimated, truth) + testthat::expect_identical(aligned$items, c("Item2", "Item1")) + testthat::expect_equal(aligned$estimated["Item1", "a"], 0.9) + testthat::expect_equal(aligned$truth["Item1", "b"], 0.5) + + unnamed <- estimated + rownames(unnamed) <- NULL + testthat::expect_error(kaefa:::.alignIrtItemParameters(unnamed, truth), "row names") + testthat::expect_error( + kaefa:::.alignIrtItemParameters(estimated[, "a", drop = FALSE], truth), + "Missing recovery columns" + ) + other <- truth + rownames(other) <- c("Q1", "Q2") + testthat::expect_error(kaefa:::.alignIrtItemParameters(estimated, other), "shared item names") +}) + +test_that("five-repeat recovery summary has a fixed output schema", { + .ensure_kaefa_namespace() + rmse_by_repeat <- data.frame( + seed = rep(c(11L, 22L, 33L, 44L, 55L), each = 2L), + parameter = rep(c("a", "b"), times = 5L), + rmse = c(0.10, 0.20, 0.12, 0.18, 0.11, 0.21, 0.09, 0.19, 0.13, 0.17), + stringsAsFactors = FALSE + ) + summarised <- kaefa:::.summariseRecoveryRepeats(rmse_by_repeat) + testthat::expect_identical(summarised$n_repeats, 5L) + testthat::expect_identical(names(summarised$summary), c("parameter", "n_repeats", "mean_rmse", "sd_rmse")) + testthat::expect_equal(summarised$summary$n_repeats, c(5L, 5L)) + a_rmse <- rmse_by_repeat$rmse[rmse_by_repeat$parameter == "a"] + testthat::expect_equal( + summarised$summary$mean_rmse[summarised$summary$parameter == "a"], + mean(a_rmse) + ) + testthat::expect_equal( + summarised$summary$sd_rmse[summarised$summary$parameter == "a"], + stats::sd(a_rmse) + ) + testthat::expect_error( + kaefa:::.summariseRecoveryRepeats(rmse_by_repeat[1:4, ]), + "exactly 5 repeats" + ) +}) + +test_that("recovery coverage log keeps multilevel and time-flow explicit exclusions", { + .ensure_kaefa_namespace() + coverage <- kaefa:::.recoveryCoverageExclusions() + testthat::expect_identical( + names(coverage), + c("surface", "status", "reason") + ) + testthat::expect_true(any(coverage$surface == "unidimensional 2PL via .mirt" & + coverage$status == "covered")) + testthat::expect_true(any(grepl("mixedmirt", coverage$surface) & + coverage$status == "excluded")) + testthat::expect_true(any(grepl("multiple-membership", coverage$surface) & + coverage$status == "excluded")) + testthat::expect_true(any(grepl("time-flow", coverage$surface) & + coverage$status == "excluded")) +}) diff --git a/tests/testthat/test-aefa-recovery-fits.R b/tests/testthat/test-aefa-recovery-fits.R new file mode 100644 index 0000000..4e91f6f --- /dev/null +++ b/tests/testthat/test-aefa-recovery-fits.R @@ -0,0 +1,116 @@ +test_that(".mirt recovers known 2PL parameters with bounded RMSE", { + .ensure_kaefa_namespace() + testthat::skip_if_not_installed("mirt") + + set.seed(20260817) + true_a <- matrix(c(0.9, 1.1, 1.3, 1.5, 1.7), ncol = 1) + true_b <- c(-1.2, -0.6, 0, 0.6, 1.2) + true_d <- matrix(-true_a[, 1] * true_b, ncol = 1) + response_data <- as.data.frame(mirt::simdata( + a = true_a, + d = true_d, + itemtype = "2PL", + N = 400 + )) + names(response_data) <- paste0("Item", seq_len(ncol(response_data))) + truth <- data.frame( + a = true_a[, 1], + b = true_b, + row.names = names(response_data) + ) + + utils::capture.output( + fit <- suppressWarnings(kaefa::.mirt( + data = response_data, + model = 1, + method = "EM", + itemtype = "2PL", + SE = FALSE, + GenRandomPars = FALSE, + calcNull = FALSE, + leniency = FALSE, + NCYCLES = 200, + BURNIN = 50, + SEMCYCLES = 50 + )) + ) + if (!methods::is(fit, "SingleGroupClass")) { + testthat::fail("kaefa::.mirt did not return a single-group fit") + return(invisible(NULL)) + } + + estimated <- kaefa:::.extractAefaIrtItems(fit) + aligned <- kaefa:::.alignIrtItemParameters(estimated, truth) + rmse_a <- kaefa:::.parameterRecoveryRMSE(aligned$estimated$a, aligned$truth$a) + rmse_b <- kaefa:::.parameterRecoveryRMSE(aligned$estimated$b, aligned$truth$b) + testthat::expect_lt(rmse_a, 0.35) + testthat::expect_lt(rmse_b, 0.35) + testthat::expect_gt(stats::cor(aligned$estimated$a, aligned$truth$a), 0.90) +}) + +test_that("aefa five-seed 2PL recovery reports mean RMSE", { + .skip_expensive_ci_calls("aefa") + .ensure_kaefa_namespace() + testthat::skip_if_not_installed("mirt") + + true_a <- matrix(c(0.9, 1.1, 1.3, 1.5, 1.7), ncol = 1) + true_b <- c(-1.2, -0.6, 0, 0.6, 1.2) + true_d <- matrix(-true_a[, 1] * true_b, ncol = 1) + seeds <- c(20260817L, 20260818L, 20260819L, 20260820L, 20260821L) + rows <- list() + + for (seed in seeds) { + set.seed(seed) + response_data <- as.data.frame(mirt::simdata( + a = true_a, + d = true_d, + itemtype = "2PL", + N = 250 + )) + names(response_data) <- paste0("Item", seq_len(ncol(response_data))) + truth <- data.frame( + a = true_a[, 1], + b = true_b, + row.names = names(response_data) + ) + result <- try( + aefa( + response_data, + minExtraction = 1, + maxExtraction = 1, + turnOffMixedEst = TRUE, + skipggum = TRUE, + tryLCA = FALSE, + saveModelHistory = TRUE, + printItemFit = FALSE + ), + silent = TRUE + ) + if (inherits(result, "try-error") || is.null(result)) { + testthat::fail(paste("aefa recovery repeat failed for seed", seed)) + return(invisible(NULL)) + } + estimated <- kaefa:::.extractAefaIrtItems(result) + aligned <- kaefa:::.alignIrtItemParameters(estimated, truth) + rows[[length(rows) + 1L]] <- data.frame( + seed = seed, + parameter = c("a", "b"), + rmse = c( + kaefa:::.parameterRecoveryRMSE(aligned$estimated$a, aligned$truth$a), + kaefa:::.parameterRecoveryRMSE(aligned$estimated$b, aligned$truth$b) + ), + stringsAsFactors = FALSE + ) + } + + summarised <- kaefa:::.summariseRecoveryRepeats(do.call(rbind, rows)) + testthat::expect_identical(summarised$n_repeats, 5L) + testthat::expect_lt( + summarised$summary$mean_rmse[summarised$summary$parameter == "a"], + 0.45 + ) + testthat::expect_lt( + summarised$summary$mean_rmse[summarised$summary$parameter == "b"], + 0.45 + ) +}) From 4cd21a80103ce56ac1351c5e9ed57a06be6de517 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 17 Aug 2026 16:59:25 +0000 Subject: [PATCH 21/24] fix(recovery): treat default rownames and NA Hessian as failures R data.frames reset rownames(NULL) to "1","2",... so alignment now rejects those sequential defaults instead of claiming no shared names. .mirt/.mixedmirt use isTRUE() so a missing second-order test returns NULL instead of aborting the caller. Co-authored-by: Seongho Bae --- ARCHITECTURE.md | 4 +++- R/recovery.R | 17 +++++++++++++-- R/utils.R | 10 +++++---- docs/traceability/aefa-parameter-recovery.md | 6 ++++-- tests/testthat/test-aefa-parameter-recovery.R | 6 ++++++ tests/testthat/test-aefa-recovery-fits.R | 21 ++++++++++++++++++- 6 files changed, 54 insertions(+), 10 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index d944ebb..6dab81f 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -32,7 +32,9 @@ explicitly requires vendored source integration. - `R/kaefa.R`: public orchestration entry points and exported runtime behavior. - `R/newEngine.R`: candidate-model estimation engine used by the AEFA loop. -- `R/utils.R`: helper routines and shared utilities. +- `R/utils.R`: helper routines and shared utilities. `.mirt()` and + `.mixedmirt()` treat a missing (`NA`) second-order test as + non-convergence when `leniency` is false. - `R/recovery.R`: internal true-parameter RMSE helpers used by the AEFA recovery protocol. Not a public API. - `inst/shiny-app/app.R`: bundled Shiny interface logic. diff --git a/R/recovery.R b/R/recovery.R index 8e7d99c..0530039 100644 --- a/R/recovery.R +++ b/R/recovery.R @@ -18,6 +18,17 @@ sqrt(mean((estimated - truth)^2)) } +# R data.frames never keep rownames as NULL: `rownames(x) <- NULL` resets them +# to the sequential defaults "1", "2", .... Treat those as unnamed so callers +# cannot silently align on positional indices. +.irtItemNames <- function(x) { + rn <- rownames(x) + if (is.null(rn) || identical(rn, as.character(seq_len(nrow(x))))) { + return(NULL) + } + rn +} + .alignIrtItemParameters <- function(estimated, truth, columns = c("a", "b")) { if (!is.data.frame(estimated) && !is.matrix(estimated)) { stop("estimated must be a matrix or data.frame of item parameters.", call. = FALSE) @@ -27,7 +38,9 @@ } estimated <- as.data.frame(estimated, stringsAsFactors = FALSE) truth <- as.data.frame(truth, stringsAsFactors = FALSE) - if (is.null(rownames(estimated)) || is.null(rownames(truth))) { + est_names <- .irtItemNames(estimated) + true_names <- .irtItemNames(truth) + if (is.null(est_names) || is.null(true_names)) { stop( "Estimated and true parameter tables must have item names as row names.", call. = FALSE @@ -42,7 +55,7 @@ call. = FALSE ) } - shared <- intersect(rownames(estimated), rownames(truth)) + shared <- intersect(est_names, true_names) if (!length(shared)) { stop("No shared item names between estimated and true parameters.", call. = FALSE) } diff --git a/R/utils.R b/R/utils.R index 0c9ffdf..95fe14c 100644 --- a/R/utils.R +++ b/R/utils.R @@ -197,11 +197,13 @@ } if(exists('mod')){ - if(mod@OptimInfo$converged){ + if(isTRUE(mod@OptimInfo$converged)){ if(leniency){ mod } else { - if(mod@OptimInfo$secondordertest){ + # mirt can leave secondordertest as NA when the Hessian is unavailable; + # `if (NA)` would abort the caller instead of treating the fit as failed. + if(isTRUE(mod@OptimInfo$secondordertest)){ mod } else { NULL @@ -229,11 +231,11 @@ SEMCYCLES = SEMCYCLES, symmetric = symmetric)) if(exists('mod')){ - if(mod@OptimInfo$converged){ + if(isTRUE(mod@OptimInfo$converged)){ if(leniency){ mod } else { - if(mod@OptimInfo$secondordertest){ + if(isTRUE(mod@OptimInfo$secondordertest)){ mod } else { NULL diff --git a/docs/traceability/aefa-parameter-recovery.md b/docs/traceability/aefa-parameter-recovery.md index a3b418e..4e5e676 100644 --- a/docs/traceability/aefa-parameter-recovery.md +++ b/docs/traceability/aefa-parameter-recovery.md @@ -9,8 +9,10 @@ helpers in `R/recovery.R` and pinned by coverage exclusions) plus `tests/testthat/test-aefa-recovery-fits.R` (live fits). -This is not a public API. External behaviour of `aefa()`, `engineAEFA()`, and -`.mirt()` is unchanged. +This is not a public API. External behaviour of `aefa()` and `engineAEFA()` is +unchanged. `.mirt()` / `.mixedmirt()` now treat a missing +(`NA`) second-order test as non-convergence when `leniency` is false, instead +of aborting on `if (NA)`. ## RMSE definition diff --git a/tests/testthat/test-aefa-parameter-recovery.R b/tests/testthat/test-aefa-parameter-recovery.R index 932fa5b..6651c63 100644 --- a/tests/testthat/test-aefa-parameter-recovery.R +++ b/tests/testthat/test-aefa-parameter-recovery.R @@ -30,6 +30,12 @@ test_that("IRT parameter alignment is by item name and required columns", { unnamed <- estimated rownames(unnamed) <- NULL testthat::expect_error(kaefa:::.alignIrtItemParameters(unnamed, truth), "row names") + truth_unnamed <- truth + rownames(truth_unnamed) <- NULL + testthat::expect_error( + kaefa:::.alignIrtItemParameters(unnamed, truth_unnamed), + "row names" + ) testthat::expect_error( kaefa:::.alignIrtItemParameters(estimated[, "a", drop = FALSE], truth), "Missing recovery columns" diff --git a/tests/testthat/test-aefa-recovery-fits.R b/tests/testthat/test-aefa-recovery-fits.R index 4e91f6f..7e509ad 100644 --- a/tests/testthat/test-aefa-recovery-fits.R +++ b/tests/testthat/test-aefa-recovery-fits.R @@ -29,11 +29,30 @@ test_that(".mirt recovers known 2PL parameters with bounded RMSE", { GenRandomPars = FALSE, calcNull = FALSE, leniency = FALSE, - NCYCLES = 200, + NCYCLES = 800, BURNIN = 50, SEMCYCLES = 50 )) ) + if (is.null(fit)) { + # A missing Hessian (secondordertest = NA) is not a recovery failure. + # Retry with more cycles and allow a missing second-order test. + utils::capture.output( + fit <- suppressWarnings(kaefa::.mirt( + data = response_data, + model = 1, + method = "EM", + itemtype = "2PL", + SE = FALSE, + GenRandomPars = FALSE, + calcNull = FALSE, + leniency = TRUE, + NCYCLES = 1500, + BURNIN = 50, + SEMCYCLES = 50 + )) + ) + } if (!methods::is(fit, "SingleGroupClass")) { testthat::fail("kaefa::.mirt did not return a single-group fit") return(invisible(NULL)) From bd7a530360cb1eff77bf04b3598086244abd3d2a Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 17 Aug 2026 17:18:05 +0000 Subject: [PATCH 22/24] test(recovery): use FIIFM-sized N for live 2PL RMSE N=400 left the Hessian missing and made five-item a-parameter RMSE and correlation too noisy. Match the passing FIIFM stability design (N=1500, 400 cycles) and drop the lenient retry that accepted a poorly recovered fit. Co-authored-by: Seongho Bae --- docs/traceability/aefa-parameter-recovery.md | 3 +- tests/testthat/test-aefa-recovery-fits.R | 29 ++++---------------- 2 files changed, 8 insertions(+), 24 deletions(-) diff --git a/docs/traceability/aefa-parameter-recovery.md b/docs/traceability/aefa-parameter-recovery.md index 4e5e676..aaa6e2e 100644 --- a/docs/traceability/aefa-parameter-recovery.md +++ b/docs/traceability/aefa-parameter-recovery.md @@ -31,7 +31,8 @@ Exactly five seeds are required. The summary schema is: per-run `seed`, Covered: -- Unidimensional 2PL recovery through `kaefa::.mirt`. +- Unidimensional 2PL recovery through `kaefa::.mirt` (`N = 1500`, same + cycle budget as the FIIFM stability fit). - AEFA greedy search on the same 2PL design when `RUN_FULL_AEFA_TESTS=1`. Explicitly excluded until a registered design exists: diff --git a/tests/testthat/test-aefa-recovery-fits.R b/tests/testthat/test-aefa-recovery-fits.R index 7e509ad..ce004ff 100644 --- a/tests/testthat/test-aefa-recovery-fits.R +++ b/tests/testthat/test-aefa-recovery-fits.R @@ -6,11 +6,13 @@ test_that(".mirt recovers known 2PL parameters with bounded RMSE", { true_a <- matrix(c(0.9, 1.1, 1.3, 1.5, 1.7), ncol = 1) true_b <- c(-1.2, -0.6, 0, 0.6, 1.2) true_d <- matrix(-true_a[, 1] * true_b, ncol = 1) + # Same N and cycle budget as test-fiifm-stability.R. N=400 left the Hessian + # missing and a-parameter RMSE / 5-item correlation too noisy for a gate. response_data <- as.data.frame(mirt::simdata( a = true_a, d = true_d, itemtype = "2PL", - N = 400 + N = 1500 )) names(response_data) <- paste0("Item", seq_len(ncol(response_data))) truth <- data.frame( @@ -29,30 +31,11 @@ test_that(".mirt recovers known 2PL parameters with bounded RMSE", { GenRandomPars = FALSE, calcNull = FALSE, leniency = FALSE, - NCYCLES = 800, - BURNIN = 50, - SEMCYCLES = 50 + NCYCLES = 400, + BURNIN = 100, + SEMCYCLES = 100 )) ) - if (is.null(fit)) { - # A missing Hessian (secondordertest = NA) is not a recovery failure. - # Retry with more cycles and allow a missing second-order test. - utils::capture.output( - fit <- suppressWarnings(kaefa::.mirt( - data = response_data, - model = 1, - method = "EM", - itemtype = "2PL", - SE = FALSE, - GenRandomPars = FALSE, - calcNull = FALSE, - leniency = TRUE, - NCYCLES = 1500, - BURNIN = 50, - SEMCYCLES = 50 - )) - ) - } if (!methods::is(fit, "SingleGroupClass")) { testthat::fail("kaefa::.mirt did not return a single-group fit") return(invisible(NULL)) From a18f1ce2f73c8fb12d50d26186b9bb75d6a16aba Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 17 Aug 2026 17:35:44 +0000 Subject: [PATCH 23/24] test(recovery): require SE=TRUE for the live 2PL RMSE fit SE=FALSE left secondordertest NA/FALSE, so .mirt() returned NULL and the suite failed in milliseconds. Use the passing FIIFM settings (N=1500, SE=TRUE, 400 cycles) and keep the RMSE bounds. Co-authored-by: Seongho Bae --- docs/traceability/aefa-parameter-recovery.md | 4 ++-- tests/testthat/test-aefa-recovery-fits.R | 11 +++++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/docs/traceability/aefa-parameter-recovery.md b/docs/traceability/aefa-parameter-recovery.md index aaa6e2e..c950f9b 100644 --- a/docs/traceability/aefa-parameter-recovery.md +++ b/docs/traceability/aefa-parameter-recovery.md @@ -31,8 +31,8 @@ Exactly five seeds are required. The summary schema is: per-run `seed`, Covered: -- Unidimensional 2PL recovery through `kaefa::.mirt` (`N = 1500`, same - cycle budget as the FIIFM stability fit). +- Unidimensional 2PL recovery through `kaefa::.mirt` (`N = 1500`, + `SE = TRUE`, same cycle budget as the FIIFM stability fit). - AEFA greedy search on the same 2PL design when `RUN_FULL_AEFA_TESTS=1`. Explicitly excluded until a registered design exists: diff --git a/tests/testthat/test-aefa-recovery-fits.R b/tests/testthat/test-aefa-recovery-fits.R index ce004ff..e454b67 100644 --- a/tests/testthat/test-aefa-recovery-fits.R +++ b/tests/testthat/test-aefa-recovery-fits.R @@ -6,8 +6,9 @@ test_that(".mirt recovers known 2PL parameters with bounded RMSE", { true_a <- matrix(c(0.9, 1.1, 1.3, 1.5, 1.7), ncol = 1) true_b <- c(-1.2, -0.6, 0, 0.6, 1.2) true_d <- matrix(-true_a[, 1] * true_b, ncol = 1) - # Same N and cycle budget as test-fiifm-stability.R. N=400 left the Hessian - # missing and a-parameter RMSE / 5-item correlation too noisy for a gate. + # Same design as test-fiifm-stability.R: N=1500 and SE=TRUE so the Hessian + # exists and leniency=FALSE can keep the fit. SE=FALSE leaves + # secondordertest NA/FALSE and .mirt() returns NULL. response_data <- as.data.frame(mirt::simdata( a = true_a, d = true_d, @@ -27,7 +28,7 @@ test_that(".mirt recovers known 2PL parameters with bounded RMSE", { model = 1, method = "EM", itemtype = "2PL", - SE = FALSE, + SE = TRUE, GenRandomPars = FALSE, calcNull = FALSE, leniency = FALSE, @@ -37,7 +38,9 @@ test_that(".mirt recovers known 2PL parameters with bounded RMSE", { )) ) if (!methods::is(fit, "SingleGroupClass")) { - testthat::fail("kaefa::.mirt did not return a single-group fit") + testthat::fail( + "kaefa::.mirt did not return a single-group fit (converged second-order test required)" + ) return(invisible(NULL)) } From 1c5d9f0491fc178be3f7f307dac521fbcbba6978 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 17 Aug 2026 18:34:58 +0000 Subject: [PATCH 24/24] fix(recovery): require identical item sets and complete seeds Reject partial name intersections so RMSE cannot drop missing items. Require one RMSE per parameter-seed pair and reject NA repeats. Wrap r-lib reference URLs and use the real v1.12.1 NEWS commit title. Co-authored-by: Seongho Bae --- R/recovery.R | 26 ++++++++++++++-- docs/traceability/aefa-parameter-recovery.md | 15 ++++++---- docs/traceability/r-lib-actions-v2.12.1.md | 6 ++-- tests/testthat/test-aefa-parameter-recovery.R | 30 ++++++++++++++++++- 4 files changed, 64 insertions(+), 13 deletions(-) diff --git a/R/recovery.R b/R/recovery.R index 0530039..50bf43e 100644 --- a/R/recovery.R +++ b/R/recovery.R @@ -55,10 +55,15 @@ call. = FALSE ) } - shared <- intersect(est_names, true_names) - if (!length(shared)) { - stop("No shared item names between estimated and true parameters.", call. = FALSE) + missing_estimated_items <- setdiff(true_names, est_names) + missing_truth_items <- setdiff(est_names, true_names) + if (length(missing_estimated_items) || length(missing_truth_items)) { + stop( + "Estimated and true parameter tables must contain the same item names.", + call. = FALSE + ) } + shared <- est_names list( estimated = estimated[shared, columns, drop = FALSE], truth = truth[shared, columns, drop = FALSE], @@ -83,7 +88,22 @@ if (length(seeds) != 5L) { stop("Recovery protocol requires exactly 5 repeats.", call. = FALSE) } + if (anyNA(rmse_by_repeat$seed) || + anyNA(rmse_by_repeat$parameter) || + anyNA(rmse_by_repeat$rmse)) { + stop("Recovery repeats cannot contain missing values.", call. = FALSE) + } parameters <- unique(as.character(rmse_by_repeat$parameter)) + repeat_counts <- table( + as.character(rmse_by_repeat$parameter), + rmse_by_repeat$seed + ) + if (!length(repeat_counts) || any(repeat_counts != 1L)) { + stop( + "Each parameter must have exactly one RMSE value for each recovery seed.", + call. = FALSE + ) + } summary_rows <- lapply(parameters, function(parameter_name) { values <- rmse_by_repeat$rmse[as.character(rmse_by_repeat$parameter) == parameter_name] data.frame( diff --git a/docs/traceability/aefa-parameter-recovery.md b/docs/traceability/aefa-parameter-recovery.md index c950f9b..236506f 100644 --- a/docs/traceability/aefa-parameter-recovery.md +++ b/docs/traceability/aefa-parameter-recovery.md @@ -20,12 +20,15 @@ For a recovered parameter vector \(\hat{\theta}\) and a true vector \(\theta\), RMSE = sqrt( mean( (hat_theta - theta)^2 ) ) -Items are aligned by name before the difference is taken. The required IRT -columns for the unidimensional 2PL case are `a` (discrimination) and `b` -(difficulty) from `mirt::coef(..., IRTpars = TRUE, simplify = TRUE)$items`. - -Exactly five seeds are required. The summary schema is: per-run `seed`, -`parameter`, `rmse`, plus `mean_rmse` and `sd_rmse` by parameter. +Items are aligned by name before the difference is taken. Estimated and true +tables must contain the same item names; a partial intersection is rejected. +The required IRT columns for the unidimensional 2PL case are `a` +(discrimination) and `b` (difficulty) from +`mirt::coef(..., IRTpars = TRUE, simplify = TRUE)$items`. + +Exactly five seeds are required, and each parameter must have one RMSE value +for every seed. The summary schema is: per-run `seed`, `parameter`, `rmse`, +plus `mean_rmse` and `sd_rmse` by parameter. ## Current coverage diff --git a/docs/traceability/r-lib-actions-v2.12.1.md b/docs/traceability/r-lib-actions-v2.12.1.md index fa52bd6..2960000 100644 --- a/docs/traceability/r-lib-actions-v2.12.1.md +++ b/docs/traceability/r-lib-actions-v2.12.1.md @@ -23,7 +23,7 @@ runner 또는 package 호환성 회귀가 확인되면 네 action을 함께 마 ## 참고문헌 R-lib. (2026, June 23). *r-lib/actions v2.12.1* [Software release notes]. -https://github.com/r-lib/actions/blob/d3c5be51b12e724e68f33216ca3c148b66d5f0b6/NEWS.md +[NEWS.md](https://github.com/r-lib/actions/blob/d3c5be51b12e724e68f33216ca3c148b66d5f0b6/NEWS.md) -R-lib. (2026). *Update NEWS for v2.12.1* [Source code commit]. -https://github.com/r-lib/actions/commit/d3c5be51b12e724e68f33216ca3c148b66d5f0b6 +R-lib. (2026). *Update NEWS for v1.12.1* [Source code commit]. +[d3c5be51b12e724e68f33216ca3c148b66d5f0b6](https://github.com/r-lib/actions/commit/d3c5be51b12e724e68f33216ca3c148b66d5f0b6) diff --git a/tests/testthat/test-aefa-parameter-recovery.R b/tests/testthat/test-aefa-parameter-recovery.R index 6651c63..6c7edaf 100644 --- a/tests/testthat/test-aefa-parameter-recovery.R +++ b/tests/testthat/test-aefa-parameter-recovery.R @@ -42,7 +42,15 @@ test_that("IRT parameter alignment is by item name and required columns", { ) other <- truth rownames(other) <- c("Q1", "Q2") - testthat::expect_error(kaefa:::.alignIrtItemParameters(estimated, other), "shared item names") + testthat::expect_error( + kaefa:::.alignIrtItemParameters(estimated, other), + "same item names" + ) + subset_truth <- truth[1, , drop = FALSE] + testthat::expect_error( + kaefa:::.alignIrtItemParameters(estimated, subset_truth), + "same item names" + ) }) test_that("five-repeat recovery summary has a fixed output schema", { @@ -70,6 +78,26 @@ test_that("five-repeat recovery summary has a fixed output schema", { kaefa:::.summariseRecoveryRepeats(rmse_by_repeat[1:4, ]), "exactly 5 repeats" ) + incomplete_b <- rmse_by_repeat[rmse_by_repeat$parameter == "a" | + rmse_by_repeat$seed != 55L, ] + testthat::expect_error( + kaefa:::.summariseRecoveryRepeats(incomplete_b), + "exactly one RMSE value" + ) + duplicated_a <- rbind( + rmse_by_repeat, + data.frame(seed = 11L, parameter = "a", rmse = 0.99, stringsAsFactors = FALSE) + ) + testthat::expect_error( + kaefa:::.summariseRecoveryRepeats(duplicated_a), + "exactly one RMSE value" + ) + missing_rmse <- rmse_by_repeat + missing_rmse$rmse[1] <- NA_real_ + testthat::expect_error( + kaefa:::.summariseRecoveryRepeats(missing_rmse), + "missing values" + ) }) test_that("recovery coverage log keeps multilevel and time-flow explicit exclusions", {