diff --git a/.github/actions/compile-ruleset/action.yml b/.github/actions/compile-ruleset/action.yml new file mode 100644 index 0000000000..4d0f6cecb6 --- /dev/null +++ b/.github/actions/compile-ruleset/action.yml @@ -0,0 +1,43 @@ +name: Compile ruleset +description: Compiles a XULE ruleset, verifies it, and uploads it as a workflow artifact. + +inputs: + source_path: + description: Path to the XULE source directory to compile + required: true + zip_name: + description: Output ruleset zip filename + required: true + +runs: + using: composite + steps: + - name: Compile XULE ruleset + shell: bash + run: | + python -m arelle.CntlrCmdLine \ + --plugins xule \ + --xule-compile ${{ inputs.source_path }} \ + --xule-rule-set ./dqc_us_rules/${{ inputs.zip_name }} \ + --xule-crash + + - name: Verify output + shell: bash + run: | + ls -lh ./dqc_us_rules/${{ inputs.zip_name }} + python3 -c " + import zipfile + with zipfile.ZipFile('./dqc_us_rules/${{ inputs.zip_name }}') as z: + print(f'Ruleset contains {len(z.namelist())} files') + for name in sorted(z.namelist())[:10]: + print(f' {name}') + if len(z.namelist()) > 10: + print(f' ... and {len(z.namelist()) - 10} more') + " + + - name: Upload ruleset artifact + uses: actions/upload-artifact@v4 + with: + name: "${{ inputs.zip_name }}" + path: "./dqc_us_rules/${{ inputs.zip_name }}" + retention-days: 90 diff --git a/.github/workflows/buildcontainer.yml b/.github/workflows/buildcontainer.yml index 11a16b8ac7..4864f81697 100644 --- a/.github/workflows/buildcontainer.yml +++ b/.github/workflows/buildcontainer.yml @@ -2,13 +2,33 @@ name: Build Validation Container on: workflow_dispatch: + inputs: + arelle_version: + description: 'Arelle-release pip version ("latest" for the newest release or other supported version)' + required: true + default: '2.44.0' + type: choice + options: + - latest + - '2.44.0' + - '2.37.58' + xule_version: + description: 'Xule plugin branch/tag (avoid "/" — it is used verbatim in the published image tag)' + required: true + default: '30052.1' + type: string + edgar_version: + description: 'EDGAR transform plugin branch/tag (from Arelle/EDGAR repo)' + required: true + default: '26.1.3' + type: string env: REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }}/validation + XULE_REPO: xbrlus permissions: - contents: read + contents: write packages: write jobs: @@ -18,30 +38,81 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v6 + + - name: Set lowercase image name + run: echo "IMAGE_NAME=$(echo '${{ github.repository }}/validation' | tr '[:upper:]' '[:lower:]')" >> "$GITHUB_ENV" + + - name: Detect Python version from Dockerfile + id: pyver + run: | + PY_VERSION=$(grep -oP '(?<=FROM python:)[^\s-]+' Dockerfile) + echo "python_version=$PY_VERSION" >> "$GITHUB_OUTPUT" + + - name: Print summary + env: + PY_VERSION: ${{ steps.pyver.outputs.python_version }} + run: | + { + echo "### Build Validation Container" + echo "" + echo "| Component | Version |" + echo "|-----------|---------|" + echo "| **Python** | \`${PY_VERSION}\` |" + echo "| **Arelle** | \`${{ inputs.arelle_version }}\` |" + echo "| **Xule** | \`${{ inputs.xule_version }}\` |" + echo "| **EDGAR** | \`${{ inputs.edgar_version }}\` |" + echo "" + echo "**Image:** \`${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:arelle-${{ inputs.arelle_version }}-xule-${{ inputs.xule_version }}-edgar-${{ inputs.edgar_version }}\`" + echo "" + echo "**Repository:** \`${{ github.repository }}\` | **Triggered by:** \`${{ github.actor }}\`" + } >> "$GITHUB_STEP_SUMMARY" - name: Log in to GitHub Container Registry - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4 - name: Build and push - uses: docker/build-push-action@v6 + uses: docker/build-push-action@v7 with: context: . push: true tags: | ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.sha }} + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:arelle-${{ inputs.arelle_version }}-xule-${{ inputs.xule_version }}-edgar-${{ inputs.edgar_version }} cache-from: type=gha cache-to: type=gha,mode=max build-args: | - XULE_VERSION=30050 - XULE_REPO=xbrlus - TRANSFORM_VERSION=25.1 - ARELLE_VERSION=2.37.58 + XULE_VERSION=${{ inputs.xule_version }} + XULE_REPO=${{ env.XULE_REPO }} + TRANSFORM_VERSION=${{ inputs.edgar_version }} + ARELLE_VERSION=${{ inputs.arelle_version }} + + - name: Update Dockerfile defaults and commit + env: + PY_VERSION: ${{ steps.pyver.outputs.python_version }} + run: | + sed -i "s/^ARG ARELLE_VERSION=.*/ARG ARELLE_VERSION=${{ inputs.arelle_version }}/" Dockerfile + sed -i "s/^ARG XULE_VERSION=.*/ARG XULE_VERSION=${{ inputs.xule_version }}/" Dockerfile + sed -i "s/^ARG TRANSFORM_VERSION=.*/ARG TRANSFORM_VERSION=${{ inputs.edgar_version }}/" Dockerfile + if git diff --quiet Dockerfile; then + echo "Dockerfile already up to date — no commit needed." + exit 0 + fi + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add Dockerfile + git commit -m "$(cat <> "$GITHUB_STEP_SUMMARY" + + # ────────────────────────────────────────────── + # Resolve effective environment choice + # Auto-disables prebuilt when versions differ from the defaults baked into the container image. + # ────────────────────────────────────────────── + resolve-env: + name: Resolve environment + runs-on: ubuntu-latest + outputs: + use_prebuilt: ${{ steps.resolve.outputs.use_prebuilt }} + image: ${{ steps.resolve.outputs.image }} + steps: + - name: Resolve effective use_prebuilt flag + id: resolve + run: | + IMAGE_NAME="$(echo '${{ github.repository }}/validation' | tr '[:upper:]' '[:lower:]')" + echo "image=ghcr.io/${IMAGE_NAME}:arelle-${{ inputs.arelle_version }}-xule-${{ inputs.xule_version }}-edgar-${{ inputs.edgar_version }}" >> "$GITHUB_OUTPUT" + DEFAULT_ARELLE="2.44.0" + DEFAULT_XULE="30052.1" + DEFAULT_EDGAR="26.1.3" + USE_PREBUILT="${{ inputs.use_prebuilt_env }}" + if [ "$USE_PREBUILT" = "true" ]; then + if [ "${{ inputs.arelle_version }}" != "$DEFAULT_ARELLE" ] || [ "${{ inputs.xule_version }}" != "$DEFAULT_XULE" ] || [ "${{ inputs.edgar_version }}" != "$DEFAULT_EDGAR" ]; then + echo "::warning::Prebuilt container uses Arelle $DEFAULT_ARELLE / Xule $DEFAULT_XULE / EDGAR $DEFAULT_EDGAR — custom versions requested, switching to fresh build." + USE_PREBUILT="false" + fi + fi + echo "use_prebuilt=$USE_PREBUILT" >> "$GITHUB_OUTPUT" + + # ────────────────────────────────────────────── + # Detect changed .xule files (test_mode=changed only) + # ────────────────────────────────────────────── + detect-changes: + name: Detect changed .xule files + if: inputs.test_mode == 'changed' + runs-on: ubuntu-latest + outputs: + changed_dirs: ${{ steps.analyze.outputs.changed_dirs }} + changed_rules: ${{ steps.analyze.outputs.changed_rules }} + is_full_rebuild: ${{ steps.analyze.outputs.is_full_rebuild }} + steps: + - name: Checkout repository (with history) + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Determine base commit and analyze changes + id: analyze + env: + BASE_COMMIT: ${{ inputs.base_commit }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + BRANCH: ${{ github.ref_name }} + run: | + BASE="$BASE_COMMIT" + + if [ -z "$BASE" ]; then + echo "No base_commit provided — finding last successful run..." + LAST_SUCCESS_SHA=$( + gh api \ + "repos/$REPO/actions/workflows/compileandvalidatedqc.yml/runs?status=success&per_page=1&branch=$BRANCH" \ + --jq '.workflow_runs[0].head_sha // empty' 2>/dev/null || true + ) + if [ -n "$LAST_SUCCESS_SHA" ]; then + echo "Last successful run commit: $LAST_SUCCESS_SHA" + BASE="$LAST_SUCCESS_SHA" + else + echo "No previous successful run found — falling back to HEAD~1" + BASE="HEAD~1" + fi + fi + + echo "Diffing $BASE..HEAD for .xule file changes..." + CHANGED_XULE=$(git diff --name-only "$BASE"..HEAD -- '*.xule' || true) + + if [ -z "$CHANGED_XULE" ]; then + echo "No .xule files changed." + echo "changed_dirs=" >> "$GITHUB_OUTPUT" + echo "changed_rules=" >> "$GITHUB_OUTPUT" + echo "is_full_rebuild=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + + echo "Changed .xule files:" + echo "$CHANGED_XULE" + + export CHANGED_XULE + python3 << 'PYEOF' + import re, os, json + + raw = os.environ.get('CHANGED_XULE', '') + files = [f for f in raw.strip().split('\n') if f.strip()] + rule_ids = set() + changed_dirs = set() + is_full_rebuild = False + + for f in files: + basename = f.rsplit('/', 1)[-1] + + parts = f.split('/') + if 'source' in parts: + idx = parts.index('source') + if idx + 2 < len(parts): + taxonomy = parts[idx + 1] + year_dir = parts[idx + 2] + if taxonomy != 'lib': + changed_dirs.add(f'{taxonomy}/{year_dir}') + + m = re.match(r'DQC_IFRS_(\d+)', basename) + if m: + rule_ids.add(f'DQC.IFRS.{m.group(1)}') + continue + m = re.match(r'DQC_(\d+)', basename) + if m: + rule_ids.add(f'DQC.US.{m.group(1)}') + continue + name_lower = basename.lower().replace('.xule', '') + if any(keyword in name_lower for keyword in ('constants', 'functions', 'namespace', 'base-taxonomy')): + rule_ids.add('__ALL__') + is_full_rebuild = True + + if is_full_rebuild: + source_root = 'dqc_us_rules/source' + for taxonomy in sorted(os.listdir(source_root)): + tax_path = os.path.join(source_root, taxonomy) + if not os.path.isdir(tax_path) or taxonomy == 'lib': + continue + for year_dir in sorted(os.listdir(tax_path)): + if os.path.isdir(os.path.join(tax_path, year_dir)): + changed_dirs.add(f'{taxonomy}/{year_dir}') + + with open(os.environ['GITHUB_OUTPUT'], 'a') as out: + dirs_json = json.dumps(sorted(changed_dirs)) + out.write(f'changed_dirs={dirs_json}\n') + out.write(f'is_full_rebuild={str(is_full_rebuild).lower()}\n') + rules_str = '\n'.join(sorted(rule_ids)) + out.write(f'changed_rules<> "$GITHUB_PATH" + + - name: Install Arelle, Xule, and EDGAR plugins + run: | + pip install --no-cache-dir -r requirements.txt + if [ "${{ inputs.arelle_version }}" = "latest" ]; then + pip install --no-cache-dir --upgrade Arelle-release + else + pip install --no-cache-dir Arelle-release==${{ inputs.arelle_version }} + fi + SITE_PACKAGES=$(python -c "import sysconfig; print(sysconfig.get_paths()['purelib'])") + git clone --depth=1 --branch ${{ inputs.xule_version }} --single-branch \ + https://github.com/${{ env.XULE_REPO }}/xule.git /tmp/xule + cp -r /tmp/xule/plugin/xule "$SITE_PACKAGES/arelle/plugin/" + cp /tmp/xule/plugin/semanticHash.py "$SITE_PACKAGES/arelle/plugin/" + cp /tmp/xule/plugin/validate/DQC.py "$SITE_PACKAGES/arelle/plugin/validate/" + rm -rf /tmp/xule + git clone --depth=1 --branch ${{ inputs.edgar_version }} --single-branch \ + https://github.com/Arelle/EDGAR.git /tmp/EDGAR + cp -r /tmp/EDGAR "$SITE_PACKAGES/arelle/plugin/" + rm -rf /tmp/EDGAR + + - name: Archive virtual environment + run: tar -C "${{ env.VENV_PATH }}" -czf /tmp/dqc-venv.tar.gz . + + - name: Upload environment artifact + uses: actions/upload-artifact@v6 + with: + name: dqc-venv + path: /tmp/dqc-venv.tar.gz + retention-days: 1 + + # ────────────────────────────────────────────── + # Discover source directories to compile + # ────────────────────────────────────────────── + build-compile-matrix: + name: Discover source directories + needs: detect-changes + if: always() && inputs.skip_compilation == false + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.discover.outputs.matrix }} + has_sources: ${{ steps.discover.outputs.has_sources }} + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Discover compilable sources + id: discover + env: + TEST_MODE: ${{ inputs.test_mode }} + CHANGED_DIRS: ${{ needs.detect-changes.outputs.changed_dirs }} + TAXONOMY: ${{ inputs.taxonomy }} + YEAR: ${{ inputs.year }} + VERSION_SUFFIX: ${{ inputs.version_suffix }} + BRANCH_NAME: ${{ github.ref_name }} + run: | + VERSION_TAG="$(echo "$BRANCH_NAME" | sed 's/^v/V/')${VERSION_SUFFIX}" + export RESOLVED_VERSION_TAG="$VERSION_TAG" + + python3 << 'PYEOF' + import json, os, sys + + source_root = 'dqc_us_rules/source' + version_tag = os.environ['RESOLVED_VERSION_TAG'] + taxonomy_input = os.environ.get('TAXONOMY', 'all') + year_input = os.environ.get('YEAR', 'all') + test_mode = os.environ.get('TEST_MODE', 'all') + changed_dirs_raw = os.environ.get('CHANGED_DIRS', '') + + changed_dirs = set() + if test_mode == 'changed' and changed_dirs_raw: + changed_dirs = set(json.loads(changed_dirs_raw)) + print(f'Filtering compilation to changed directories: {sorted(changed_dirs)}') + + taxonomy_filters = [t.strip() for t in taxonomy_input.split(',') if t.strip()] + year_filters = [y.strip() for y in year_input.split(',') if y.strip()] + includes = [] + + for taxonomy in sorted(os.listdir(source_root)): + tax_path = os.path.join(source_root, taxonomy) + if not os.path.isdir(tax_path) or taxonomy == 'lib': + continue + if 'all' not in taxonomy_filters and taxonomy not in taxonomy_filters: + continue + for year_dir in sorted(os.listdir(tax_path)): + dir_path = os.path.join(tax_path, year_dir) + if not os.path.isdir(dir_path): + continue + if 'all' not in year_filters and not any(year_dir == yf or year_dir.endswith(yf) for yf in year_filters): + continue + dir_key = f'{taxonomy}/{year_dir}' + if test_mode == 'changed' and changed_dirs and dir_key not in changed_dirs: + continue + zip_name = f'dqc-{year_dir if taxonomy == "esef" else taxonomy + "-" + year_dir}-{version_tag}-ruleset.zip' + includes.append({ + 'taxonomy': taxonomy, + 'year_dir': year_dir, + 'source_path': dir_path, + 'zip_name': zip_name, + 'label': f'{taxonomy}/{year_dir}', + }) + + has_sources = 'true' if includes else 'false' + if not includes and test_mode == 'changed': + print('No source directories matched the changed .xule files.') + elif not includes: + print('No matching source directories found', file=sys.stderr) + sys.exit(1) + + matrix = {'include': includes} + with open(os.environ['GITHUB_OUTPUT'], 'a') as out: + out.write(f'matrix={json.dumps(matrix)}\n') + out.write(f'has_sources={has_sources}\n') + + print(f'Sources to compile ({len(includes)}):') + for item in includes: + print(f" {item['label']} -> {item['zip_name']}") + PYEOF + + # ────────────────────────────────────────────── + # Build test matrix (parallel with compilation) + # ────────────────────────────────────────────── + build-test-matrix: + name: Build test matrix + needs: detect-changes + if: always() && inputs.test_mode != 'none' + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + has_tests: ${{ steps.set-matrix.outputs.has_tests }} + changed_rules: ${{ steps.set-matrix.outputs.changed_rules }} + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Build matrix from test config + id: set-matrix + env: + TEST_MODE: ${{ inputs.test_mode }} + CHANGED_RULES: ${{ needs.detect-changes.outputs.changed_rules }} + TAXONOMY: ${{ inputs.taxonomy }} + YEAR: ${{ inputs.year }} + run: | + python3 << 'PYEOF' + import json, os, re, sys + + test_mode = os.environ.get('TEST_MODE', 'all') + changed_rules_raw = os.environ.get('CHANGED_RULES', '') + taxonomy_input = os.environ.get('TAXONOMY', 'all').strip().lower() + year_input = os.environ.get('YEAR', 'all').strip() + + with open('tests/secvalidationtests.json') as f: + tests = json.load(f) + + changed_rules = set() + if test_mode == 'changed' and changed_rules_raw: + changed_rules = {r.strip() for r in changed_rules_raw.strip().split('\n') if r.strip()} + print(f'Test mode: {test_mode}') + print(f'Changed rules detected: {sorted(changed_rules)}') + + taxonomy_filters = {t.strip() for t in taxonomy_input.split(',') if t.strip()} + year_filters = {y.strip() for y in year_input.split(',') if y.strip()} + filter_by_scope = 'all' not in taxonomy_filters or 'all' not in year_filters + + def matches_scope(name): + if not filter_by_scope: + return True + name_lower = name.lower() + if name_lower.startswith('us gaap') or 'roll to' in name_lower: + test_tax = 'us' + elif name_lower.startswith('ifrs'): + test_tax = 'ifrs' + elif name_lower.startswith('v'): + return True + else: + return True + if 'all' not in taxonomy_filters and test_tax not in taxonomy_filters: + if not (test_tax == 'ifrs' and 'esef' in taxonomy_filters): + return False + if 'all' not in year_filters: + years_in_name = set(re.findall(r'20\d{2}', name)) + if years_in_name and not years_in_name & year_filters: + return False + return True + + if filter_by_scope: + print(f'Filtering tests to taxonomy={taxonomy_input}, year={year_input}') + + is_full = '__ALL__' in changed_rules + includes = [] + + for i, t in enumerate(tests): + if not matches_scope(t['name']): + continue + if test_mode == 'all' or is_full: + includes.append({'index': i, 'name': t['name']}) + elif test_mode == 'changed' and changed_rules: + matched = False + for filing in t['infiles']: + run_only = filing.get('xule_run_only', '') + for rule_id in changed_rules: + if rule_id in run_only: + matched = True + break + if matched: + break + if matched: + includes.append({'index': i, 'name': t['name']}) + + has_tests = 'true' if includes else 'false' + matrix = {'include': includes} if includes else {'include': []} + + rules_json = json.dumps(sorted(changed_rules)) if changed_rules else '[]' + with open(os.environ['GITHUB_OUTPUT'], 'a') as out: + out.write(f'matrix={json.dumps(matrix)}\n') + out.write(f'has_tests={has_tests}\n') + out.write(f'changed_rules={rules_json}\n') + + if includes: + print(f'Test sets to validate ({len(includes)}):') + for item in includes: + print(f" {item['name']}") + else: + print('No test sets matched the filters.') + PYEOF + + # ────────────────────────────────────────────── + # Compile rulesets (prebuilt container path) + # ────────────────────────────────────────────── + compile-prebuilt: + name: "${{ matrix.label }} (prebuilt env)" + needs: [resolve-env, build-compile-matrix] + if: >- + always() + && needs.build-compile-matrix.result == 'success' + && needs.resolve-env.outputs.use_prebuilt == 'true' + && needs.build-compile-matrix.outputs.has_sources == 'true' + runs-on: ubuntu-latest + container: + image: ${{ needs.resolve-env.outputs.image }} + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.build-compile-matrix.outputs.matrix) }} + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Compile ruleset + uses: ./.github/actions/compile-ruleset + with: + source_path: ${{ matrix.source_path }} + zip_name: ${{ matrix.zip_name }} + + # ────────────────────────────────────────────── + # Compile rulesets (fresh environment path) + # ────────────────────────────────────────────── + compile-fresh: + name: "${{ matrix.label }} (fresh env)" + needs: [resolve-env, build-env, build-compile-matrix] + if: >- + always() + && needs.build-compile-matrix.result == 'success' + && (needs.build-env.result == 'success') + && needs.resolve-env.outputs.use_prebuilt == 'false' + && needs.build-compile-matrix.outputs.has_sources == 'true' + runs-on: ubuntu-latest + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.build-compile-matrix.outputs.matrix) }} + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: '3.13' + + - name: Download environment artifact + uses: actions/download-artifact@v6 + with: + name: dqc-venv + path: /tmp/dqc-venv + + - name: Restore virtual environment + run: | + mkdir -p "${{ env.VENV_PATH }}" + tar -C "${{ env.VENV_PATH }}" -xzf /tmp/dqc-venv/dqc-venv.tar.gz + + - name: Add virtual environment to PATH + run: echo "${{ env.VENV_PATH }}/bin" >> "$GITHUB_PATH" + + - name: Compile ruleset + uses: ./.github/actions/compile-ruleset + with: + source_path: ${{ matrix.source_path }} + zip_name: ${{ matrix.zip_name }} + + # ────────────────────────────────────────────── + # Validate compiled rulesets (prebuilt container) + # ────────────────────────────────────────────── + validate-prebuilt: + name: "${{ matrix.name }} (prebuilt env)" + needs: [resolve-env, compile-prebuilt, compile-fresh, build-test-matrix, build-compile-matrix] + if: >- + always() + && needs.resolve-env.outputs.use_prebuilt == 'true' + && inputs.test_mode != 'none' + && needs.build-test-matrix.outputs.has_tests == 'true' + && (inputs.skip_compilation || needs.compile-prebuilt.result == 'success') + runs-on: ubuntu-latest + container: + image: ${{ needs.resolve-env.outputs.image }} + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + timeout-minutes: 120 + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.build-test-matrix.outputs.matrix) }} + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Download all compiled rulesets + if: inputs.skip_compilation == false + uses: actions/download-artifact@v6 + with: + path: ./artifacts + pattern: 'dqc-*-ruleset.zip' + + - name: Install compiled rulesets + if: inputs.skip_compilation == false + run: | + echo "Downloaded artifacts:" + find ./artifacts -name '*.zip' -type f + find ./artifacts -name '*.zip' -type f -exec cp {} ./dqc_us_rules/ \; + + - name: Configure ruleset map to use local zips + run: | + echo "Rulesets in dqc_us_rules/:" + ls -lh ./dqc_us_rules/*.zip + + python3 << 'PYEOF' + import json, os + + with open('rulesetMap.json') as f: + rmap = json.load(f) + + updated = {} + for uri, url in rmap.items(): + filename = url.rsplit('/', 1)[-1].split('?')[0] + local_path = os.path.join(os.getcwd(), 'dqc_us_rules', filename) + if os.path.exists(local_path): + updated[uri] = local_path + print(f' {uri} -> {local_path}') + else: + updated[uri] = url + print(f' {uri} -> {url} (remote, no local zip)') + + with open('rulesetMap.json', 'w') as f: + json.dump(updated, f, indent=2) + PYEOF + + - name: Extract test set and write filing list + id: testdata + env: + TEST_MODE: ${{ inputs.test_mode }} + CHANGED_RULES_JSON: ${{ needs.build-test-matrix.outputs.changed_rules }} + MATRIX_INDEX: ${{ matrix.index }} + run: | + python3 << 'PYEOF' + import json, os + + test_mode = os.environ.get('TEST_MODE', 'all') + changed_rules_json = os.environ.get('CHANGED_RULES_JSON', '[]') + index = int(os.environ['MATRIX_INDEX']) + + with open('tests/secvalidationtests.json') as f: + tests = json.load(f) + test = tests[index] + + infiles = test['infiles'] + exfiles_list = [e.strip() for e in test['exfiles'].split(',') if e.strip()] + + changed_rules = set() + if test_mode == 'changed' and changed_rules_json: + changed_rules = set(json.loads(changed_rules_json)) + + if test_mode == 'changed' and changed_rules and '__ALL__' not in changed_rules: + filtered_in = [] + filtered_ex = [] + for i, filing in enumerate(infiles): + run_only = filing.get('xule_run_only', '') + if any(rule_id in run_only for rule_id in changed_rules): + filtered_in.append(filing) + if i < len(exfiles_list): + filtered_ex.append(exfiles_list[i]) + print(f'Filtered {len(infiles)} filings down to {len(filtered_in)} matching changed rules') + infiles = filtered_in + exfiles_list = filtered_ex + + with open('infiles.json', 'w') as f: + json.dump(infiles, f) + exfiles_str = ','.join(exfiles_list) + with open('exfiles.txt', 'w') as f: + f.write(exfiles_str) + + print('Filing list:') + for filing in infiles: + print(f" {filing['xule_run_only']}: {filing['file']}") + + with open(os.environ['GITHUB_OUTPUT'], 'a') as out: + out.write(f'exfiles={exfiles_str}\n') + out.write(f'filing_count={len(infiles)}\n') + PYEOF + + - name: Run Arelle DQC validation + id: validation + if: steps.testdata.outputs.filing_count != '0' + run: | + set +e + python -m arelle.CntlrCmdLine \ + --httpUserAgent "DQC-Arelle (xbrl.us; dqc@xbrl.us)" \ + --plugin "validate/DQC|EDGAR/transform|inlineXbrlDocumentSet" \ + -v \ + --dqc-replace-rule-set-map ./rulesetMap.json \ + --xule-filing-list infiles.json \ + --logFile ./log.xml \ + --xule-debug + ARELLE_EXIT=$? + set -e + echo "arelle_exit=$ARELLE_EXIT" >> "$GITHUB_OUTPUT" + if [ $ARELLE_EXIT -ne 0 ]; then + echo "::warning::Arelle exited with code $ARELLE_EXIT (non-zero is expected when validation errors are found in test filings)" + fi + + - name: Compare results against expected + id: compare + if: always() && steps.testdata.outputs.filing_count != '0' && steps.validation.outcome != 'cancelled' + run: | + python compare.py \ + --compare-file ./report.txt \ + --test-files ./log.xml \ + --expected-results "${{ steps.testdata.outputs.exfiles }}" \ + --html-file ./report.html + continue-on-error: true + + - name: Print log.xml + if: always() + run: head -3000 log.xml || true + + - name: Print comparison report + if: always() + run: | + echo "===== COMPARISON REPORT (TEXT) =====" + cat report.txt 2>/dev/null || echo "No text report generated." + echo "===== END COMPARISON REPORT =====" + + - name: Upload reports as artifacts + if: always() + uses: actions/upload-artifact@v6 + with: + name: "reports-${{ matrix.name }}" + path: | + log.xml + report.txt + report.html + retention-days: 30 + + - name: Upload HTML report + if: always() && steps.compare.outcome == 'failure' + uses: actions/upload-artifact@v6 + with: + name: "discrepancies-${{ matrix.name }}.html" + path: report.html + retention-days: 30 + + - name: Report outcome + if: always() + run: | + if [ "${{ steps.testdata.outputs.filing_count }}" = "0" ]; then + echo "No filings matched changed rules for ${{ matrix.name }} — skipped." + elif [ "${{ steps.compare.outcome }}" = "failure" ]; then + echo "### Discrepancies: ${{ matrix.name }}" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + echo '```' >> "$GITHUB_STEP_SUMMARY" + cat report.txt >> "$GITHUB_STEP_SUMMARY" 2>/dev/null || true + echo '```' >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "::error::Discrepancies found in ${{ matrix.name }}. See uploaded artifacts." + exit 1 + else + echo "All validation results match expected output for ${{ matrix.name }}." + fi + + # ────────────────────────────────────────────── + # Validate compiled rulesets (fresh environment) + # ────────────────────────────────────────────── + validate-fresh: + name: "${{ matrix.name }} (fresh env)" + needs: [resolve-env, compile-prebuilt, compile-fresh, build-test-matrix, build-compile-matrix, build-env] + if: >- + always() + && needs.resolve-env.outputs.use_prebuilt == 'false' + && inputs.test_mode != 'none' + && needs.build-test-matrix.outputs.has_tests == 'true' + && needs.build-env.result == 'success' + && (inputs.skip_compilation || needs.compile-fresh.result == 'success') + runs-on: ubuntu-latest + timeout-minutes: 120 + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.build-test-matrix.outputs.matrix) }} + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: '3.13' + + - name: Download environment artifact + uses: actions/download-artifact@v6 + with: + name: dqc-venv + path: /tmp/dqc-venv + + - name: Restore virtual environment + run: | + mkdir -p "${{ env.VENV_PATH }}" + tar -C "${{ env.VENV_PATH }}" -xzf /tmp/dqc-venv/dqc-venv.tar.gz + + - name: Add virtual environment to PATH + run: echo "${{ env.VENV_PATH }}/bin" >> "$GITHUB_PATH" + + - name: Download all compiled rulesets + if: inputs.skip_compilation == false + uses: actions/download-artifact@v6 + with: + path: ./artifacts + pattern: 'dqc-*-ruleset.zip' + + - name: Install compiled rulesets + if: inputs.skip_compilation == false + run: | + echo "Downloaded artifacts:" + find ./artifacts -name '*.zip' -type f + find ./artifacts -name '*.zip' -type f -exec cp {} ./dqc_us_rules/ \; + + - name: Configure ruleset map to use local zips + run: | + echo "Rulesets in dqc_us_rules/:" + ls -lh ./dqc_us_rules/*.zip + + python3 << 'PYEOF' + import json, os + + with open('rulesetMap.json') as f: + rmap = json.load(f) + + updated = {} + for uri, url in rmap.items(): + filename = url.rsplit('/', 1)[-1].split('?')[0] + local_path = os.path.join(os.getcwd(), 'dqc_us_rules', filename) + if os.path.exists(local_path): + updated[uri] = local_path + print(f' {uri} -> {local_path}') + else: + updated[uri] = url + print(f' {uri} -> {url} (remote, no local zip)') + + with open('rulesetMap.json', 'w') as f: + json.dump(updated, f, indent=2) + PYEOF + + - name: Extract test set and write filing list + id: testdata + env: + TEST_MODE: ${{ inputs.test_mode }} + CHANGED_RULES_JSON: ${{ needs.build-test-matrix.outputs.changed_rules }} + MATRIX_INDEX: ${{ matrix.index }} + run: | + python3 << 'PYEOF' + import json, os + + test_mode = os.environ.get('TEST_MODE', 'all') + changed_rules_json = os.environ.get('CHANGED_RULES_JSON', '[]') + index = int(os.environ['MATRIX_INDEX']) + + with open('tests/secvalidationtests.json') as f: + tests = json.load(f) + test = tests[index] + + infiles = test['infiles'] + exfiles_list = [e.strip() for e in test['exfiles'].split(',') if e.strip()] + + changed_rules = set() + if test_mode == 'changed' and changed_rules_json: + changed_rules = set(json.loads(changed_rules_json)) + + if test_mode == 'changed' and changed_rules and '__ALL__' not in changed_rules: + filtered_in = [] + filtered_ex = [] + for i, filing in enumerate(infiles): + run_only = filing.get('xule_run_only', '') + if any(rule_id in run_only for rule_id in changed_rules): + filtered_in.append(filing) + if i < len(exfiles_list): + filtered_ex.append(exfiles_list[i]) + print(f'Filtered {len(infiles)} filings down to {len(filtered_in)} matching changed rules') + infiles = filtered_in + exfiles_list = filtered_ex + + with open('infiles.json', 'w') as f: + json.dump(infiles, f) + exfiles_str = ','.join(exfiles_list) + with open('exfiles.txt', 'w') as f: + f.write(exfiles_str) + + print('Filing list:') + for filing in infiles: + print(f" {filing['xule_run_only']}: {filing['file']}") + + with open(os.environ['GITHUB_OUTPUT'], 'a') as out: + out.write(f'exfiles={exfiles_str}\n') + out.write(f'filing_count={len(infiles)}\n') + PYEOF + + - name: Run Arelle DQC validation + id: validation + if: steps.testdata.outputs.filing_count != '0' + run: | + set +e + python -m arelle.CntlrCmdLine \ + --httpUserAgent "DQC-Arelle (xbrl.us; dqc@xbrl.us)" \ + --plugin "validate/DQC|EDGAR/transform|inlineXbrlDocumentSet" \ + -v \ + --dqc-replace-rule-set-map ./rulesetMap.json \ + --xule-filing-list infiles.json \ + --logFile ./log.xml \ + --xule-debug + ARELLE_EXIT=$? + set -e + echo "arelle_exit=$ARELLE_EXIT" >> "$GITHUB_OUTPUT" + if [ $ARELLE_EXIT -ne 0 ]; then + echo "::warning::Arelle exited with code $ARELLE_EXIT (non-zero is expected when validation errors are found in test filings)" + fi + + - name: Compare results against expected + id: compare + if: always() && steps.testdata.outputs.filing_count != '0' && steps.validation.outcome != 'cancelled' + run: | + python compare.py \ + --compare-file ./report.txt \ + --test-files ./log.xml \ + --expected-results "${{ steps.testdata.outputs.exfiles }}" \ + --html-file ./report.html + continue-on-error: true + + - name: Print log.xml + if: always() + run: head -3000 log.xml || true + + - name: Print comparison report + if: always() + run: | + echo "===== COMPARISON REPORT (TEXT) =====" + cat report.txt 2>/dev/null || echo "No text report generated." + echo "===== END COMPARISON REPORT =====" + + - name: Upload reports as artifacts + if: always() + uses: actions/upload-artifact@v6 + with: + name: "reports-${{ matrix.name }}" + path: | + log.xml + report.txt + report.html + retention-days: 30 + + - name: Upload HTML report + if: always() && steps.compare.outcome == 'failure' + uses: actions/upload-artifact@v6 + with: + name: "discrepancies-${{ matrix.name }}.html" + path: report.html + retention-days: 30 + + - name: Report outcome + if: always() + run: | + if [ "${{ steps.testdata.outputs.filing_count }}" = "0" ]; then + echo "No filings matched changed rules for ${{ matrix.name }} — skipped." + elif [ "${{ steps.compare.outcome }}" = "failure" ]; then + echo "### Discrepancies: ${{ matrix.name }}" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + echo '```' >> "$GITHUB_STEP_SUMMARY" + cat report.txt >> "$GITHUB_STEP_SUMMARY" 2>/dev/null || true + echo '```' >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "::error::Discrepancies found in ${{ matrix.name }}. See uploaded artifacts." + exit 1 + else + echo "All validation results match expected output for ${{ matrix.name }}." + fi + + # ────────────────────────────────────────────── + # Commit compiled rulesets + # ────────────────────────────────────────────── + commit: + name: Commit all rulesets + needs: [compile-prebuilt, compile-fresh, validate-prebuilt, validate-fresh] + if: >- + always() + && inputs.skip_compilation == false + && (needs.compile-prebuilt.result == 'success' || needs.compile-fresh.result == 'success') + && (needs.validate-prebuilt.result == 'success' || needs.validate-prebuilt.result == 'skipped') + && (needs.validate-fresh.result == 'success' || needs.validate-fresh.result == 'skipped') + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Download all compiled rulesets + uses: actions/download-artifact@v6 + with: + path: ./artifacts + pattern: 'dqc-*-ruleset.zip' + + - name: Copy rulesets into repo + run: | + echo "Downloaded artifacts:" + find ./artifacts -name '*.zip' -type f + find ./artifacts -name '*.zip' -type f -exec cp {} ./dqc_us_rules/ \; + echo "" + echo "Rulesets in dqc_us_rules/:" + ls -lh ./dqc_us_rules/*.zip + + - name: Update rulesetMap.json version references + env: + BRANCH_NAME: ${{ github.ref_name }} + RULESET_VERSION: ${{ inputs.ruleset_version }} + run: | + MAP_VERSION="${RULESET_VERSION}" + if [ -z "$MAP_VERSION" ]; then + MAP_VERSION="$BRANCH_NAME" + fi + sed -i -E "s|/dqc_us_rules/raw/[^/]+/dqc_us_rules/|/dqc_us_rules/raw/${MAP_VERSION}/dqc_us_rules/|g" rulesetMap.json + echo "Updated rulesetMap.json version to: ${MAP_VERSION}" + cat rulesetMap.json + + - name: Commit and push + env: + VERSION_SUFFIX: ${{ inputs.version_suffix }} + BRANCH_NAME: ${{ github.ref_name }} + RULESET_VERSION: ${{ inputs.ruleset_version }} + ARELLE_VERSION: ${{ inputs.arelle_version }} + XULE_VERSION: ${{ inputs.xule_version }} + EDGAR_VERSION: ${{ inputs.edgar_version }} + TEST_MODE: ${{ inputs.test_mode }} + SKIP_COMPILATION: ${{ inputs.skip_compilation }} + run: | + VERSION_TAG="$(echo "$BRANCH_NAME" | sed 's/^v/V/')${VERSION_SUFFIX}" + COMMIT_VERSION="${RULESET_VERSION}" + if [ -z "$COMMIT_VERSION" ]; then + COMMIT_VERSION="$VERSION_TAG" + fi + if [ "$SKIP_COMPILATION" = "true" ]; then + ACTION="Test" + elif [ "$TEST_MODE" = "none" ]; then + ACTION="Recompile" + else + ACTION="Recompile and test" + fi + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add ./dqc_us_rules/*.zip rulesetMap.json + git diff --cached --quiet && echo "No changes to commit" && exit 0 + git commit -m "$(cat <> "$GITHUB_PATH" - - name: Compile XULE ruleset - run: | - python -m arelle.CntlrCmdLine \ - --plugins xule \ - --xule-compile ${{ matrix.source_path }} \ - --xule-rule-set ./dqc_us_rules/${{ matrix.zip_name }} \ - --xule-crash + - name: Compile ruleset + uses: ./.github/actions/compile-ruleset + with: + source_path: ${{ matrix.source_path }} + zip_name: ${{ matrix.zip_name }} + + commit: + name: Commit all rulesets + needs: [compile-prebuilt, compile-fresh] + if: always() && (needs.compile-prebuilt.result == 'success' || needs.compile-fresh.result == 'success') + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Checkout repository + uses: actions/checkout@v4 - - name: Verify output + - name: Download all compiled rulesets + uses: actions/download-artifact@v4 + with: + path: ./artifacts + pattern: 'dqc-*-ruleset.zip' + + - name: Copy rulesets into repo run: | - ls -lh ./dqc_us_rules/${{ matrix.zip_name }} - python3 -c " - import zipfile - with zipfile.ZipFile('./dqc_us_rules/${{ matrix.zip_name }}') as z: - print(f'Ruleset contains {len(z.namelist())} files') - for name in sorted(z.namelist())[:10]: - print(f' {name}') - if len(z.namelist()) > 10: - print(f' ... and {len(z.namelist()) - 10} more') - " + echo "Downloaded artifacts:" + find ./artifacts -name '*.zip' -type f + find ./artifacts -name '*.zip' -type f -exec cp {} ./dqc_us_rules/ \; + echo "" + echo "Rulesets in dqc_us_rules/:" + ls -lh ./dqc_us_rules/*.zip - - name: Commit compiled ruleset + - name: Commit and push run: | VERSION_TAG="$(echo '${{ github.ref_name }}' | sed 's/^v/V/')${{ inputs.version_suffix }}" - PYTHON_VERSION=$(python --version | awk '{print $2}') + COMMIT_VERSION="${{ inputs.ruleset_version }}" + if [ -z "$COMMIT_VERSION" ]; then + COMMIT_VERSION="$VERSION_TAG" + fi git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add ./dqc_us_rules/${{ matrix.zip_name }} + git add ./dqc_us_rules/*.zip + git diff --cached --quiet && echo "No changes to commit" && exit 0 git commit -m "$(cat <