diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..6773798 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,101 @@ +# CI guards for rillmd/rill (PUBLIC repo). +# +# Three jobs, all claude-free: +# lint -- bash -n syntax check + pinned shellcheck (severity=warning) +# test -- the pure-shell test/cli/ suites on ubuntu + macos +# guard -- CJK allowlist scan + email/phone/secrets regex scan, each run +# twice: over the full tree and over the PR's commit messages +# +# The private-vocabulary PII scan is deliberately NOT here: that vocabulary +# cannot live in a public repo. It runs locally via +# bin/hooks/pre-push-pii-mapping-check.sh (terms read from an out-of-repo +# file). CI covers only vocabulary-independent checks. + +name: CI + +on: + pull_request: + branches: [main] + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: bash -n (syntax) over bin/, plugins/, test/ shell files + run: | + set -euo pipefail + mapfile -t files < <(git ls-files \ + 'bin/rill' 'bin/rill-inbox-process' 'bin/hooks/*.sh' \ + 'plugins/*.sh' 'plugins/*/*.sh' 'plugins/*/lib/*.sh' \ + 'test/*.sh' 'test/*/*.sh') + echo "bash -n over ${#files[@]} files" + [ "${#files[@]}" -gt 0 ] + for f in "${files[@]}"; do bash -n "$f"; done + + - name: shellcheck v0.11.0 (pinned) at severity=warning + run: | + set -euo pipefail + url="https://github.com/koalaman/shellcheck/releases/download/v0.11.0/shellcheck-v0.11.0.linux.x86_64.tar.xz" + sha256="8c3be12b05d5c177a04c29e3c78ce89ac86f1595681cab149b65b97c4e227198" + curl -fsSL "$url" -o /tmp/shellcheck.tar.xz + echo "$sha256 /tmp/shellcheck.tar.xz" | sha256sum -c - + tar -xf /tmp/shellcheck.tar.xz -C /tmp + sc=/tmp/shellcheck-v0.11.0/shellcheck + "$sc" --version + mapfile -t files < <(git ls-files \ + 'bin/rill' 'bin/rill-inbox-process' 'bin/hooks/*.sh' \ + 'plugins/_lib.sh' 'plugins/*/*.sh' 'plugins/*/lib/*.sh') + echo "shellcheck over ${#files[@]} files" + [ "${#files[@]}" -gt 0 ] + "$sc" -x --severity=warning "${files[@]}" + + test: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + + - name: pure-shell suites (test/cli/, no claude CLI) + run: | + set -euo pipefail + fail=0 + for t in test/cli/test-*.sh; do + echo "== $t" + if ! bash "$t"; then + echo "== $t: FAILED" + fail=1 + fi + done + exit "$fail" + + guard: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # commit-message scans need base..head history + + - name: CJK guard -- full tree (allowlisted) + run: python3 test/cli/cjk-guard.py + + - name: CJK guard -- PR commit messages (no allowlist) + run: | + set -euo pipefail + git log --format=%B \ + "${{ github.event.pull_request.base.sha }}..${{ github.event.pull_request.head.sha }}" \ + | python3 test/cli/cjk-guard.py --stdin + + - name: PII regex guard -- full tree (allowlisted) + run: bash test/cli/pii-regex-guard.sh + + - name: PII regex guard -- PR commit messages (no allowlist) + run: | + set -euo pipefail + git log --format=%B \ + "${{ github.event.pull_request.base.sha }}..${{ github.event.pull_request.head.sha }}" \ + | bash test/cli/pii-regex-guard.sh --stdin diff --git a/.shellcheckrc b/.shellcheckrc new file mode 100644 index 0000000..227bad2 --- /dev/null +++ b/.shellcheckrc @@ -0,0 +1,21 @@ +# shellcheck policy for this repo (initial adoption, task rill-ci-guards). +# +# Gate tiering: +# - CI (and the documented local command) runs `shellcheck --severity=warning`: +# warning-and-above findings BLOCK, note/style findings do not. +# `severity` is a command-line-only option (not honored in this rc file, +# verified against shellcheck v0.11.0), so the threshold lives in +# .github/workflows/ci.yml, not here. +# - A bare local `shellcheck ` intentionally still shows note/style +# findings (SC2001/SC2005/SC2012/SC2295/SC1091 class): visible for cleanup, +# but not enforced. Fix them opportunistically; do not scatter silent +# per-line disables to hide them. +# +# Per-line disables are allowed only with a reason comment at the call site +# (current inventory: SC2088 in plugins/twitter/requires.sh and +# plugins/voice-memo/requires.sh -- require_dir expands ~ itself). +# +# Local parity command (pin v0.11.0, same as CI): +# git ls-files 'bin/rill' 'bin/rill-inbox-process' 'bin/hooks/*.sh' \ +# 'plugins/_lib.sh' 'plugins/*/*.sh' 'plugins/*/lib/*.sh' \ +# | xargs shellcheck -x --severity=warning diff --git a/bin/hooks/pre-push-pii-mapping-check.sh b/bin/hooks/pre-push-pii-mapping-check.sh new file mode 100644 index 0000000..32ba841 --- /dev/null +++ b/bin/hooks/pre-push-pii-mapping-check.sh @@ -0,0 +1,84 @@ +#!/bin/bash +# pre-push-pii-mapping-check.sh -- block pushes whose commits contain private +# vocabulary (real client/product names), read from an OUT-OF-REPO terms file. +# +# Role split (two hooks, two repos, two audiences): +# - bin/hooks/pre-commit-pii-check.sh = vault-content guard. Installed into +# end-user VAULTS via `rill crypt init` / `rill crypt hook`; scans vault +# files for emails/phones with the vault-local .rill/pii-allowlist.txt. +# - THIS file = rillmd/rill SOURCE-REPO guard. Manually self-installed by a +# contributor into this repo's .git/hooks; never distributed to vaults. +# It scans pushed commits (added diff lines + commit messages) for the +# contributor's private term list, which must never appear in this PUBLIC +# repo -- so the list itself lives OUTSIDE the repo: +# $RILL_DEV_PII_TERMS_FILE, or ~/.config/rill-dev/pii-terms.txt +# (one term per line, # comments and blank lines ignored, matched +# case-insensitively as fixed strings). +# No terms file -> the hook is a no-op: the repo ships the MECHANISM only. +# +# Install (from the repo root): +# ln -s ../../bin/hooks/pre-push-pii-mapping-check.sh .git/hooks/pre-push +# (or cp; for worktrees, use `git rev-parse --git-path hooks`) +# +# stdin (per git pre-push contract): +# Exit: 0 allow push, 1 block push, 2 internal error. + +set -euo pipefail + +ZERO40="0000000000000000000000000000000000000000" + +terms_file="${RILL_DEV_PII_TERMS_FILE:-$HOME/.config/rill-dev/pii-terms.txt}" +if [ ! -f "$terms_file" ]; then + echo "pre-push-pii-mapping-check: no terms file at $terms_file -- skipping (mechanism-only mode)" >&2 + exit 0 +fi + +# Strip comments and blank lines; an empty pattern line would match everything. +terms="$(grep -vE '^[[:space:]]*(#|$)' "$terms_file" || true)" +if [ -z "$terms" ]; then + echo "pre-push-pii-mapping-check: terms file is empty -- skipping" >&2 + exit 0 +fi + +blocked=0 + +scan_commit() { + local sha="$1" + local msg added + msg="$(git log -1 --format=%B "$sha")" + # Added diff lines only (strip the +++ file header); merges diff against + # the first parent, which is what lands on the remote branch. + added="$(git show --first-parent --format= "$sha" | grep '^+' | grep -v '^+++' || true)" + if printf '%s\n%s\n' "$msg" "$added" | grep -iFq -- "$terms" 2>/dev/null; then + echo "pre-push-pii-mapping-check: BLOCKED -- private term found in commit $sha ($(git log -1 --format=%s "$sha"))" >&2 + blocked=1 + fi +} + +while read -r _local_ref local_sha _remote_ref remote_sha; do + # Branch deletion: nothing new is pushed. + [ "$local_sha" = "$ZERO40" ] && continue + + if [ "$remote_sha" = "$ZERO40" ]; then + # New remote branch: no remote tip to diff against. Prefer the merge-base + # with origin/main; fall back to "commits not already on any origin ref". + base="$(git merge-base "$local_sha" origin/main 2>/dev/null || true)" + if [ -n "$base" ]; then + revs="$(git rev-list "$base..$local_sha")" + else + revs="$(git rev-list "$local_sha" --not --remotes=origin)" + fi + else + revs="$(git rev-list "$remote_sha..$local_sha")" + fi + + for sha in $revs; do + scan_commit "$sha" + done +done + +if [ "$blocked" -ne 0 ]; then + echo "pre-push-pii-mapping-check: push rejected -- remove the private term(s) (rewrite the commit/message) and retry" >&2 + exit 1 +fi +exit 0 diff --git a/bin/rill b/bin/rill index 28bd351..05bf617 100755 --- a/bin/rill +++ b/bin/rill @@ -107,7 +107,6 @@ JOURNAL_DIR="$RILL_HOME/inbox/journal" WEBCLIPS_DIR="$RILL_HOME/inbox/web-clips" TWEETS_DIR="$RILL_HOME/inbox/tweets" MEETINGS_DIR="$RILL_HOME/inbox/meetings" -SOURCES_DIR="$RILL_HOME/inbox/sources" PLUGINS_DIR="$RILL_HOME/plugins" COMMANDS_DIR="$RILL_HOME/.claude/commands" EDITOR="${EDITOR:-vim}" @@ -147,7 +146,7 @@ generate_filename() { # Collision at second level: append counter local counter=1 while [ -f "$JOURNAL_DIR/${base}-${counter}.md" ]; do - ((counter++)) + counter=$((counter + 1)) done filepath="$JOURNAL_DIR/${base}-${counter}.md" fi @@ -376,7 +375,7 @@ cmd_interactive() { # Skip empty entries [ -z "$text" ] && continue - ((count++)) + count=$((count + 1)) local created created="$(_create_entry "$text")" echo " -> $created (#$count)" @@ -464,7 +463,7 @@ _clip_tweet() { if [ -f "$filepath" ]; then local counter=1 while [ -f "$TWEETS_DIR/${timestamp}-${slug}-${counter}.md" ]; do - ((counter++)) + counter=$((counter + 1)) done filename="${timestamp}-${slug}-${counter}.md" filepath="$TWEETS_DIR/$filename" @@ -545,7 +544,7 @@ _clip_web() { if [ -f "$filepath" ]; then local counter=1 while [ -f "$WEBCLIPS_DIR/${timestamp}-${slug}-${counter}.md" ]; do - ((counter++)) + counter=$((counter + 1)) done filename="${timestamp}-${slug}-${counter}.md" filepath="$WEBCLIPS_DIR/$filename" @@ -1030,7 +1029,8 @@ _plugin_status() { [ -f "$cmd" ] || continue [[ "$(basename "$cmd")" == _* ]] && continue cmd_total=$((cmd_total + 1)) - local target="$COMMANDS_DIR/$(basename "$cmd")" + local target + target="$COMMANDS_DIR/$(basename "$cmd")" if [ -L "$target" ]; then cmd_linked=$((cmd_linked + 1)) echo " ✓ $(basename "$cmd")" @@ -1432,7 +1432,7 @@ _index_rebuild_tweets() { if [ -n "$tid" ]; then echo -e "${tid}\t${fname}" >> "$index_file" - ((count++)) + count=$((count + 1)) fi done @@ -1456,7 +1456,7 @@ _index_rebuild_meetings() { if [ -n "$did" ]; then echo -e "${did}\t${fname}" >> "$index_file" - ((count++)) + count=$((count + 1)) fi done @@ -2156,7 +2156,7 @@ _crypt_doctor() { echo "✓ git-crypt initialized" else echo "✗ git-crypt not initialized (run: rill crypt init)" - ((issues++)) + issues=$((issues + 1)) fi # 2. Encrypted file count @@ -2180,7 +2180,7 @@ _crypt_doctor() { if [ "$found" = true ]; then if [ "$locked" = true ]; then echo "✗ git-crypt is LOCKED (run: git-crypt unlock )" - ((issues++)) + issues=$((issues + 1)) else echo "✓ git-crypt unlocked (files readable)" fi @@ -2190,7 +2190,7 @@ _crypt_doctor() { # 4. Key backup residual check if [ -f "$HOME/rill-git-crypt.key" ]; then echo "⚠️ Key file found at ~/rill-git-crypt.key — move to secure storage and delete" - ((issues++)) + issues=$((issues + 1)) else echo "✓ No residual key file on disk" fi @@ -2204,7 +2204,7 @@ _crypt_doctor() { echo "⚠️ Pre-commit hook exists but is not a symlink to bin/hooks/" else echo "✗ Pre-commit hook not installed (run: rill crypt hook)" - ((issues++)) + issues=$((issues + 1)) fi # 6. PII scan in non-encrypted files @@ -3788,14 +3788,14 @@ EOF # Filter by --vault if specified if [ -n "$target_vault" ] && [ "$vname" != "$target_vault" ]; then - ((i++)) + i=$((i + 1)) continue fi if [ ! -d "$vpath/.rill" ]; then echo " ⚠ Skipping '$vname': $vpath/.rill not found" >&2 - ((skipped++)) - ((i++)) + skipped=$((skipped + 1)) + i=$((i + 1)) continue fi @@ -3821,7 +3821,7 @@ EOF mkdir -p "$vpath/$(dirname "$rel")" cp "$src" "$vpath/$rel" echo "$rel" >> "$managed_file" - ((file_count++)) + file_count=$((file_count + 1)) done # Skills: .claude/commands/ (*.md and *.sh) @@ -3880,7 +3880,7 @@ EOF _install_codex_container_guidance "$vpath" "$rill_source" \ "$subdir_claude" "$managed_file" fi - ((file_count++)) + file_count=$((file_count + 1)) fi done @@ -3894,7 +3894,7 @@ EOF mkdir -p "$vpath/eval" cp "$rill_source/eval/concept.md" "$vpath/eval/concept.md" echo "eval/concept.md" >> "$managed_file" - ((file_count++)) + file_count=$((file_count + 1)) fi if [ ! -f "$vpath/eval/queries.yaml" ] && [ -f "$rill_source/eval/queries.yaml" ]; then mkdir -p "$vpath/eval" @@ -3915,7 +3915,7 @@ EOF mkdir -p "$vpath/$(dirname "$rel")" cp "$src" "$vpath/$rel" echo "$rel" >> "$managed_file" - ((file_count++)) + file_count=$((file_count + 1)) done # Plugin directories (code only — state lives in plugins/.state/) @@ -4037,8 +4037,8 @@ EOF _ensure_codex_command_skills "$vpath" echo " ✓ $vname: $file_count files updated" - ((updated++)) - ((i++)) + updated=$((updated + 1)) + i=$((i + 1)) done if [ -n "$target_vault" ] && [ "$updated" -eq 0 ] && [ "$skipped" -eq 0 ]; then @@ -4435,7 +4435,7 @@ cmd_mkfile() { local base="${filename%.md}" local counter=1 while [ -f "$abs_dir/${base}-${counter}.md" ]; do - ((counter++)) + counter=$((counter + 1)) done filename="${base}-${counter}.md" filepath="$abs_dir/$filename" @@ -4755,7 +4755,7 @@ cmd_strip_entity_tags() { for filepath in "${files[@]+"${files[@]}"}"; do [ -f "$filepath" ] || continue - ((checked++)) + checked=$((checked + 1)) # Extract tags line from frontmatter (between first --- and second ---) local in_frontmatter=false @@ -4767,7 +4767,7 @@ cmd_strip_entity_tags() { local fm_ended=false while IFS= read -r line; do - ((linenum++)) + linenum=$((linenum + 1)) if [ "$fm_ended" = true ]; then break fi @@ -4900,7 +4900,7 @@ ${new_mentions_str} " "$filepath" fi - ((fixed++)) + fixed=$((fixed + 1)) echo " fixed: $relpath — moved [$(IFS=', '; echo "${moved_entities[*]}")] to mentions (qualified)" done @@ -5380,7 +5380,7 @@ EOF # ADR-080 D80-4: claude-code-ecosystem migrates to knowledge/self/watches.md manually if [ "$slug" = "claude-code-ecosystem" ]; then echo " ⏭ skip (hard-coded): $slug — handled manually (ADR-080 D80-4)" - ((skipped_hardcoded++)) + skipped_hardcoded=$((skipped_hardcoded + 1)) continue fi @@ -5389,13 +5389,13 @@ EOF if [ -f "$dst_file" ] && [ "$force" = false ]; then echo " ⏭ skip (exists): $slug → already at projects/$slug/_project.md" - ((skipped_existing++)) + skipped_existing=$((skipped_existing + 1)) continue fi if [ "$dry_run" = true ]; then echo " ✓ would migrate: $slug → projects/$slug/_project.md" - ((migrated++)) + migrated=$((migrated + 1)) continue fi @@ -5469,7 +5469,7 @@ EOF } ' "$f" > "$dst_file"; then echo " ✗ error: failed to transform $slug" >&2 - ((errors++)) + errors=$((errors + 1)) continue fi @@ -5529,7 +5529,7 @@ PY echo " ✗ error: $slug → no status field after transform; reverting" >&2 rm -f "$dst_file" rmdir "$dst_dir" 2>/dev/null || true - ((errors++)) + errors=$((errors + 1)) continue fi @@ -5537,7 +5537,7 @@ PY rm -f "$f" echo " ✓ migrated: $slug" - ((migrated++)) + migrated=$((migrated + 1)) done echo "" diff --git a/plugins/google-meet/adapter.sh b/plugins/google-meet/adapter.sh index 2794d6d..cfba96a 100755 --- a/plugins/google-meet/adapter.sh +++ b/plugins/google-meet/adapter.sh @@ -99,13 +99,13 @@ while IFS=$'\t' read -r doc_id doc_name doc_modified; do # Defense-in-depth: check meetings index first if grep -q "^${doc_id} " "$MEETINGS_DIR/.index" 2>/dev/null; then - ((skipped++)) + skipped=$((skipped + 1)) continue fi # Primary check: plugin sync state if is_already_synced "$doc_id"; then - ((skipped++)) + skipped=$((skipped + 1)) continue fi @@ -165,7 +165,7 @@ print(f'{local_date}\t{created_ts}\t{slug_text}') if [ -f "$MEETINGS_DIR/$filename" ]; then counter=2 while [ -f "$MEETINGS_DIR/${local_date}-${slug}-${counter}.md" ]; do - ((counter++)) + counter=$((counter + 1)) done filename="${local_date}-${slug}-${counter}.md" fi @@ -177,7 +177,7 @@ google-doc-id: \"$doc_id\"" if create_source_file "$filename" "meeting" "$created_ts" "$extra_fm" "$doc_text"; then mark_synced "$doc_id" "$filename" echo -e "${doc_id}\t${filename}" >> "$MEETINGS_DIR/.index" - ((count++)) + count=$((count + 1)) fi done < <(printf '%s\n' "$docs_tsv") diff --git a/plugins/google-meet/requires.sh b/plugins/google-meet/requires.sh index 51c3d5f..6badb03 100755 --- a/plugins/google-meet/requires.sh +++ b/plugins/google-meet/requires.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# shellcheck source=plugins/_lib.sh source "$(cd "$(dirname "$0")/.." && pwd)/_lib.sh" require_command gog "brew install steipete/tap/gogcli" diff --git a/plugins/google-workspace/requires.sh b/plugins/google-workspace/requires.sh index 79f0f65..734bef2 100755 --- a/plugins/google-workspace/requires.sh +++ b/plugins/google-workspace/requires.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# shellcheck source=plugins/_lib.sh source "$(cd "$(dirname "$0")/.." && pwd)/_lib.sh" require_command gog "brew install steipete/tap/gogcli" diff --git a/plugins/meeting-materials/requires.sh b/plugins/meeting-materials/requires.sh index 6c323b1..37d221d 100755 --- a/plugins/meeting-materials/requires.sh +++ b/plugins/meeting-materials/requires.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# shellcheck source=plugins/_lib.sh source "$(cd "$(dirname "$0")/.." && pwd)/_lib.sh" require_command gog "brew install steipete/tap/gogcli — needed for Google Drive upload" diff --git a/plugins/twitter/adapter.sh b/plugins/twitter/adapter.sh index bf91daa..28f17de 100755 --- a/plugins/twitter/adapter.sh +++ b/plugins/twitter/adapter.sh @@ -38,7 +38,7 @@ for file in "$ICLOUD_DIR"/*.txt; do # Check if already synced if is_already_synced "$filename"; then - ((skipped++)) + skipped=$((skipped + 1)) continue fi @@ -55,10 +55,10 @@ for file in "$ICLOUD_DIR"/*.txt; do echo "Processing: $filename → $url" if rill clip "$url"; then mark_synced "$filename" "$url" - ((count++)) + count=$((count + 1)) else echo "ERROR: Failed to clip $url from $filename" - ((errors++)) + errors=$((errors + 1)) fi done diff --git a/plugins/twitter/requires.sh b/plugins/twitter/requires.sh index 8a7ea10..6b00249 100755 --- a/plugins/twitter/requires.sh +++ b/plugins/twitter/requires.sh @@ -1,6 +1,8 @@ #!/usr/bin/env bash +# shellcheck source=plugins/_lib.sh source "$(cd "$(dirname "$0")/.." && pwd)/_lib.sh" +# shellcheck disable=SC2088 # intentional: require_dir expands ~ itself (see plugins/_lib.sh) require_dir "~/Library/Mobile Documents/com~apple~CloudDocs/Rill/tweet-urls" \ "mkdir -p ~/Library/Mobile\\ Documents/com~apple~CloudDocs/Rill/tweet-urls" requires_check diff --git a/plugins/voice-memo/adapter.sh b/plugins/voice-memo/adapter.sh index b0f0f87..4683ed1 100755 --- a/plugins/voice-memo/adapter.sh +++ b/plugins/voice-memo/adapter.sh @@ -38,7 +38,7 @@ for file in "$ICLOUD_DIR"/*.md "$ICLOUD_DIR"/*.txt; do # Check if already synced if is_already_synced "$filename"; then - ((skipped++)) + skipped=$((skipped + 1)) continue fi @@ -49,7 +49,7 @@ for file in "$ICLOUD_DIR"/*.md "$ICLOUD_DIR"/*.txt; do if [ -f "$JOURNAL_DIR/$local_filename" ]; then echo "SKIP: $local_filename already exists in journal" mark_synced "$filename" "skipped:exists:$local_filename" - ((skipped++)) + skipped=$((skipped + 1)) continue fi @@ -57,7 +57,7 @@ for file in "$ICLOUD_DIR"/*.md "$ICLOUD_DIR"/*.txt; do echo "Created: inbox/journal/$local_filename" mark_synced "$filename" "$local_filename" - ((count++)) + count=$((count + 1)) done echo "" diff --git a/plugins/voice-memo/requires.sh b/plugins/voice-memo/requires.sh index 36d1afa..35ba2d9 100755 --- a/plugins/voice-memo/requires.sh +++ b/plugins/voice-memo/requires.sh @@ -1,6 +1,8 @@ #!/usr/bin/env bash +# shellcheck source=plugins/_lib.sh source "$(cd "$(dirname "$0")/.." && pwd)/_lib.sh" +# shellcheck disable=SC2088 # intentional: require_dir expands ~ itself (see plugins/_lib.sh) require_dir "~/Library/Mobile Documents/com~apple~CloudDocs/Rill/voice-memos" \ "mkdir -p ~/Library/Mobile\\ Documents/com~apple~CloudDocs/Rill/voice-memos" requires_check diff --git a/test/cjk-allowlist.txt b/test/cjk-allowlist.txt index 21e6549..2a66835 100644 --- a/test/cjk-allowlist.txt +++ b/test/cjk-allowlist.txt @@ -9,7 +9,7 @@ # git ls-files -z | xargs -0 rg -n --no-messages '[\p{Hiragana}\p{Katakana}\p{Han}]' # ... then drop hits whose (path, line) match an entry here; remaining hits fail. # -# Format: +# Format: # Entries are line-scoped, not file-scoped: new CJK in these files outside the # matching lines still fails the gate. bin/rill ^- (本文|技術用語|ファイル名・ディレクトリ名|frontmatter のキー|コミットメッセージ): `rill init --lang ja` seed template for personal-language.md (deliberate Japanese for ja-locale vaults) diff --git a/test/cli/cjk-guard.py b/test/cli/cjk-guard.py new file mode 100644 index 0000000..02f0abb --- /dev/null +++ b/test/cli/cjk-guard.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 +"""CJK guard -- block CJK (Hiragana / Katakana / Han) outside the allowlist. + +This repo is English-only (SPEC): CJK characters are allowed only where +test/cjk-allowlist.txt explicitly permits them, line by line. + +Modes: + (default) Tree mode. Scan every git-tracked text file; a line containing + CJK is a violation unless an allowlist entry has (1) a path equal + to the file AND (2) a regex matching the line. + --raw Tree mode without the allowlist: print every CJK hit as + path:lineno:line and exit 0. Used for the parity check against + the reference scan documented in test/cjk-allowlist.txt: + git ls-files -z | xargs -0 rg -n --no-messages \ + '[\\p{Hiragana}\\p{Katakana}\\p{Han}]' + --stdin Read raw text from stdin (e.g. commit messages). ANY CJK line is + a violation -- no allowlist applies; commit messages must be + CJK-free, period. + +Exit codes: 0 clean, 1 violations found, 2 usage / internal error. +Stdlib only -- no third-party dependencies (CI runs it on a bare runner). +""" + +import os +import re +import subprocess +import sys + +ALLOWLIST_PATH = "test/cjk-allowlist.txt" + + +def _chdir_repo_root(): + """Paths (allowlist, git ls-files) are repo-root-relative; run from anywhere.""" + top = subprocess.run(["git", "rev-parse", "--show-toplevel"], check=True, + capture_output=True).stdout.decode().strip() + os.chdir(top) + +# Unicode Script ranges for Hiragana, Katakana and Han, mirrored from UCD +# Scripts.txt so the hit-set matches rg's [\p{Hiragana}\p{Katakana}\p{Han}]. +# (Python's re has no \p{Script=...}; these ranges are the stdlib equivalent.) +_CJK_RANGES = [ + # Script=Hiragana + (0x3041, 0x3096), (0x309D, 0x309F), (0x1B001, 0x1B11F), (0x1B132, 0x1B132), + (0x1B150, 0x1B152), (0x1F200, 0x1F200), + # Script=Katakana + (0x30A1, 0x30FA), (0x30FD, 0x30FF), (0x31F0, 0x31FF), (0x32D0, 0x32FE), + (0x3300, 0x3357), (0xFF66, 0xFF9D), (0x1AFF0, 0x1AFF3), (0x1AFF5, 0x1AFFB), + (0x1AFFD, 0x1AFFE), (0x1B000, 0x1B000), (0x1B120, 0x1B122), + (0x1B155, 0x1B155), (0x1B164, 0x1B167), + # Script=Han + (0x2E80, 0x2E99), (0x2E9B, 0x2EF3), (0x2F00, 0x2FD5), (0x3005, 0x3005), + (0x3007, 0x3007), (0x3021, 0x3029), (0x3038, 0x303B), (0x3400, 0x4DBF), + (0x4E00, 0x9FFF), (0xF900, 0xFA6D), (0xFA70, 0xFAD9), (0x20000, 0x2A6DF), + (0x2A700, 0x2B739), (0x2B740, 0x2B81D), (0x2B820, 0x2CEA1), + (0x2CEB0, 0x2EBE0), (0x2EBF0, 0x2EE5D), (0x2F800, 0x2FA1D), + (0x30000, 0x3134A), (0x31350, 0x323AF), +] + +CJK_RE = re.compile( + "[" + "".join(f"{chr(lo)}-{chr(hi)}" for lo, hi in _CJK_RANGES) + "]" +) + + +def load_allowlist(): + """Return [(path, compiled_regex, reason)] from test/cjk-allowlist.txt.""" + entries = [] + try: + with open(ALLOWLIST_PATH, encoding="utf-8") as f: + for n, raw in enumerate(f, 1): + line = raw.rstrip("\n") + if not line or line.lstrip().startswith("#"): + continue + parts = line.split("\t") + if len(parts) < 3: + print(f"{ALLOWLIST_PATH}:{n}: malformed entry " + f"(need \\t\\t)", file=sys.stderr) + sys.exit(2) + try: + entries.append((parts[0], re.compile(parts[1]), parts[2])) + except re.error as e: + print(f"{ALLOWLIST_PATH}:{n}: bad regex: {e}", file=sys.stderr) + sys.exit(2) + except FileNotFoundError: + pass # no allowlist -> every hit is a violation + return entries + + +def tracked_files(): + out = subprocess.run(["git", "ls-files", "-z"], check=True, + capture_output=True).stdout + return [p.decode("utf-8", "surrogateescape") for p in out.split(b"\0") if p] + + +def iter_tree_hits(): + """Yield (path, lineno, line_text) for every CJK hit in tracked text files.""" + for path in tracked_files(): + try: + with open(path, "rb") as f: + data = f.read() + except (OSError, IsADirectoryError): + continue + if b"\0" in data[:8192]: + continue # binary, same heuristic rg uses + for lineno, raw in enumerate(data.split(b"\n"), 1): + text = raw.decode("utf-8", "replace").rstrip("\r") + if CJK_RE.search(text): + yield path, lineno, text + + +def main(argv): + mode = argv[1] if len(argv) > 1 else "tree" + if mode not in ("tree", "--raw", "--stdin"): + print(__doc__, file=sys.stderr) + return 2 + + if mode != "--stdin": + _chdir_repo_root() + + if mode == "--stdin": + violations = [ + f"(stdin):{n}:{line}" + for n, line in enumerate(sys.stdin.read().splitlines(), 1) + if CJK_RE.search(line) + ] + for v in violations: + print(v) + if violations: + print(f"cjk-guard: {len(violations)} CJK line(s) in stdin " + f"(no allowlist applies here)", file=sys.stderr) + return 1 + return 0 + + if mode == "--raw": + for path, lineno, text in iter_tree_hits(): + print(f"{path}:{lineno}:{text}") + return 0 + + allowlist = load_allowlist() + used = set() + violations = [] + for path, lineno, text in iter_tree_hits(): + allowed = False + for i, (apath, aregex, _reason) in enumerate(allowlist): + if path == apath and aregex.search(text): + allowed = True + used.add(i) + break + if not allowed: + violations.append(f"{path}:{lineno}:{text}") + for v in violations: + print(v) + for i, (apath, aregex, reason) in enumerate(allowlist): + if i not in used: + print(f"cjk-guard: note: unused allowlist entry " + f"{apath} / {aregex.pattern!r} ({reason})", file=sys.stderr) + if violations: + print(f"cjk-guard: {len(violations)} unallowlisted CJK line(s); " + f"add an entry to {ALLOWLIST_PATH} only if the CJK is deliberate", + file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/test/cli/pii-regex-guard.sh b/test/cli/pii-regex-guard.sh new file mode 100644 index 0000000..f3d9c9a --- /dev/null +++ b/test/cli/pii-regex-guard.sh @@ -0,0 +1,116 @@ +#!/bin/bash +# pii-regex-guard.sh -- block emails / phone numbers / secret-shaped strings +# outside the allowlist. +# +# Generic patterns ONLY -- this guard carries no private vocabulary; the +# private-terms scan lives in bin/hooks/pre-push-pii-mapping-check.sh, which +# reads its terms from an out-of-repo file (see that file's header). +# +# Modes: +# (default) Tree mode. Scan every git-tracked text file; a hit is a +# violation unless test/pii-regex-allowlist.txt has an entry with +# (1) a path equal to the file AND (2) an ERE matching the line. +# --stdin Read raw text from stdin (e.g. commit messages). ANY hit is a +# violation -- no allowlist applies. +# +# Allowlist format (same contract as test/cjk-allowlist.txt): +# +# +# Exit codes: 0 clean, 1 violations, 2 usage error. + +set -euo pipefail + +# Paths (allowlist, git ls-files) are repo-root-relative; run from anywhere. +cd "$(git rev-parse --show-toplevel)" + +ALLOWLIST="test/pii-regex-allowlist.txt" + +EMAIL_RE='[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}' +# Phone formats mirror bin/hooks/pre-commit-pii-check.sh (international with +# -/space/dot separators, JP 0-prefixed, US/CA parenthesized) plus the generic +# hyphenated shape used by the repo-wide push-guard scans. +PHONE_RE='\b[0-9]{2,4}-[0-9]{2,4}-[0-9]{4}\b|\+[0-9]{1,3}([- .][0-9]{1,4}){1,3}[- .][0-9]{3,4}|\b0[0-9]{1,4}-[0-9]{1,4}-[0-9]{3,4}\b|\([0-9]{2,4}\) ?[0-9]{3,4}[- .][0-9]{3,4}' +# sk- covers hyphenated OpenAI key families (sk-proj-..., sk-svcacct-...). +SECRET_RE='AKIA[0-9A-Z]{16}|\bgh[pousr]_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,}|-----BEGIN [A-Z ]*PRIVATE KEY-----|xox[baprs]-[A-Za-z0-9-]{10,}|\bsk-[A-Za-z0-9_-]{20,}' +PATTERN="$EMAIL_RE|$PHONE_RE|$SECRET_RE" + +mode="${1:-tree}" + +scan_stdin() { + local hits + # grep exits 1 on no match; that is the clean case, not an error. + hits="$(grep -nE "$PATTERN" || true)" + if [ -n "$hits" ]; then + printf '%s\n' "$hits" | sed 's/^/(stdin):/' + echo "pii-regex-guard: email/phone/secret pattern in stdin (no allowlist applies here)" >&2 + return 1 + fi + return 0 +} + +scan_tree() { + local raw allowed apath aregex areason hit_path hit_line violations=0 + # -I skips binary files; git ls-files scopes to tracked files only. + # -e/-- keep a pattern or filename starting with '-' from being parsed as + # an option; grep errors go to stderr (visible in CI) instead of /dev/null. + raw="$(git ls-files -z | xargs -0 grep -nHEI -e "$PATTERN" -- || true)" + [ -z "$raw" ] && return 0 + + # Load allowlist entries into parallel arrays (skip comments / blanks). + local -a paths=() regexes=() reasons=() used=() + if [ -f "$ALLOWLIST" ]; then + while IFS=$'\t' read -r apath aregex areason; do + case "$apath" in ''|\#*) continue ;; esac + if [ -z "$aregex" ] || [ -z "$areason" ]; then + echo "pii-regex-guard: malformed allowlist entry for '$apath' (need )" >&2 + return 2 + fi + paths+=("$apath"); regexes+=("$aregex"); reasons+=("$areason"); used+=(0) + done < "$ALLOWLIST" + fi + + while IFS= read -r line; do + hit_path="${line%%:*}" + hit_line="${line#*:}"; hit_line="${hit_line#*:}" # strip path: and lineno: + # Per-value allowlisting: strip every allowlisted match from the line, + # then re-scan the remainder. A real secret sharing a line with an + # allowlisted placeholder therefore still fails. + allowed=0 + local i remainder stripped_any=0 + remainder="$hit_line" + for i in "${!paths[@]}"; do + [ "$hit_path" = "${paths[$i]}" ] || continue + while [[ $remainder =~ ${regexes[$i]} ]]; do + [ -z "${BASH_REMATCH[0]}" ] && break # zero-length match: nothing to strip + remainder="${remainder/"${BASH_REMATCH[0]}"/}" + used[i]=1; stripped_any=1 + done + done + if [ "$stripped_any" -eq 1 ] && ! grep -qE "$PATTERN" <<< "$remainder"; then + allowed=1 + fi + if [ "$allowed" -eq 0 ]; then + printf '%s\n' "$line" + violations=$((violations + 1)) + fi + done <<< "$raw" + + local i + for i in "${!paths[@]}"; do + if [ "${used[$i]}" -eq 0 ]; then + echo "pii-regex-guard: note: unused allowlist entry ${paths[$i]} / ${regexes[$i]} (${reasons[$i]})" >&2 + fi + done + + if [ "$violations" -gt 0 ]; then + echo "pii-regex-guard: $violations unallowlisted hit(s); add to $ALLOWLIST only if the value is a deliberate placeholder" >&2 + return 1 + fi + return 0 +} + +case "$mode" in + tree) scan_tree ;; + --stdin) scan_stdin ;; + *) echo "usage: $0 [--stdin]" >&2; exit 2 ;; +esac diff --git a/test/pii-regex-allowlist.txt b/test/pii-regex-allowlist.txt new file mode 100644 index 0000000..76a2791 --- /dev/null +++ b/test/pii-regex-allowlist.txt @@ -0,0 +1,21 @@ +# PII regex allowlist -- the only places where email/phone/secret-shaped strings are deliberate. +# +# Contract (same as test/cjk-allowlist.txt): a tracked text line matching the +# generic email/phone/secret patterns in test/cli/pii-regex-guard.sh is allowed +# iff some entry below has (1) a path equal to the file AND (2) an ERE matching +# the line. Any other hit is a violation. Entries are line-scoped, not file-scoped. +# +# Format: +bin/hooks/pre-commit-pii-check.sh (ann|joann)@example\.com comment example explaining the -Fx allowlist matching semantics +knowledge/orgs/CLAUDE.md info@example\.com entity template placeholder address +knowledge/people/CLAUDE.md alex\.chen@example\.com entity template placeholder address +plugins/google-meet/plugin.md your-email@example\.com setup doc placeholder address +test/cli/test-cli-smoke.sh smoke@example\.com test fixture git identity +test/cli/test-pii-hook.sh @(example\.com|client-corp\.co\.jp|service\.io|partner-firm\.com|real-company\.net|my-own-domain\.jp|corp-x\.jp|real-client\.org) PII-hook suite synthetic email fixtures the tests assert on +test/cli/test-pii-hook.sh (090-1234-5678|080-9999-8888|\+81[- ]90-1234-5678|\+81 80 4232 1097|\+1\.2\.345) PII-hook suite synthetic phone/version fixtures the tests assert on +test/cli/test-track-managed-gitignore.sh owner@example-own\.jp test fixture allowlist content +.claude/rules/rill-workspace.md [0-9]{4}-[0-9]{2}-[0-9]{2}-[0-9]{4}\.md journal timestamp filename (YYYY-MM-DD-HHMM.md), not a phone number +SPEC.md [0-9]{4}-[0-9]{2}-[0-9]{2}-[0-9]{4}\.md journal timestamp filename examples, not phone numbers +skills/distill/SKILL.md [0-9]{4}-[0-9]{2}-[0-9]{2}-[0-9]{4}\.md journal timestamp filename example, not a phone number +skills/focus/SKILL.md [0-9]{4}-[0-9]{2}-[0-9]{2}-[0-9]{4}\.md journal timestamp filename example, not a phone number +test/pii-regex-allowlist.txt (090-1234-5678|080-9999-8888|90-1234-5678|\+81[- ]90-1234-5678|\+81 80 4232 1097) self-reference: only the synthetic phone fixtures quoted above may appear here unescaped; any other email/phone/secret in this file still fails the scan diff --git a/test/run-all.sh b/test/run-all.sh index 2c30f26..15ebbc8 100755 --- a/test/run-all.sh +++ b/test/run-all.sh @@ -41,6 +41,18 @@ run_test "context-map + processed" "$SCRIPT_DIR/cli/test-context-map-processed.s run_test "Skill preamble" "$SCRIPT_DIR/cli/test-skill-preamble.sh" run_test "book build" "$SCRIPT_DIR/cli/test-book-build.sh" +# Repo guards (tree mode): CJK + email/phone/secrets, allowlist-filtered. +# Same checks CI runs per-PR (.github/workflows/ci.yml guard job). +echo "--- Running: CJK guard (tree) ---" +if python3 "$SCRIPT_DIR/cli/cjk-guard.py"; then + echo " -> CJK guard: PASSED" +else + echo " -> CJK guard: FAILED" + TOTAL_FAIL=$((TOTAL_FAIL + 1)) +fi +echo "" +run_test "PII regex guard" "$SCRIPT_DIR/cli/pii-regex-guard.sh" + run_test "/distill" "$SCRIPT_DIR/skills/test-distill.sh" run_test "/briefing" "$SCRIPT_DIR/skills/test-briefing.sh" run_test "/clip-tweet" "$SCRIPT_DIR/skills/test-clip-tweet.sh"