From 6ed2530a9066bdbe07f84aa7a3bba37894da68b3 Mon Sep 17 00:00:00 2001 From: Parth Rohit Date: Wed, 5 Aug 2026 12:59:21 +0100 Subject: [PATCH] docs: update learn page statistics and add CI/CD workflow Signed-off-by: Parth Rohit --- .github/scripts/update_learn_page.py | 299 ++++++++++++++++++++++++ .github/workflows/update-learn-page.yml | 47 ++++ README.md | 8 +- docs/learn/index.html | 37 +-- 4 files changed, 369 insertions(+), 22 deletions(-) create mode 100644 .github/scripts/update_learn_page.py create mode 100644 .github/workflows/update-learn-page.yml diff --git a/.github/scripts/update_learn_page.py b/.github/scripts/update_learn_page.py new file mode 100644 index 0000000..03f4f20 --- /dev/null +++ b/.github/scripts/update_learn_page.py @@ -0,0 +1,299 @@ +#!/usr/bin/env python3 +"""Refresh the rule/playbook/severity statistics in the Learn page and README. + +docs/learn/index.html hardcodes rule, playbook, severity and category counts +in five places: the headline metric tiles, the hero terminal line, the +pipeline step, the rules-section title/intro, the severity-box grid, and the +"coverage by category" chart. README.md hardcodes the same rule and playbook +counts in its feature table and in two nodes of its Mermaid architecture +diagram. This script recomputes every count from scanner/rules/ and +playbooks/cli/ and rewrites both files in place, so neither can drift as +rules are added. + +Every substitution is tracked. If a pattern matches zero times — because the +surrounding wording changed — the script fails loudly instead of silently +leaving the file unchanged and exiting 0. + +Running the script against already-current files produces no changes, which +lets CI commit only on a real diff. +""" + +import re +import sys +from pathlib import Path +from typing import Dict, List, Tuple + +REPO_ROOT = Path(__file__).resolve().parents[2] +RULES_DIR = REPO_ROOT / "scanner" / "rules" +PLAYBOOKS_DIR = REPO_ROOT / "playbooks" / "cli" +LEARN_PAGE = REPO_ROOT / "docs" / "learn" / "index.html" +README_PATH = REPO_ROOT / "README.md" + +# Rule modules are named az__.py. Matching on that prefix is +# the same convention .github/workflows/ci.yml uses to discover rules, and it +# correctly excludes __init__.py and the shared _*_common.py helpers. +RULE_GLOB = "az_*.py" + +SEVERITY_PATTERN = re.compile(r"^SEVERITY\s*=\s*[\"']([^\"']+)[\"']", re.MULTILINE) +CATEGORY_PATTERN = re.compile(r"^CATEGORY\s*=\s*[\"']([^\"']+)[\"']", re.MULTILINE) +RULE_ID_PATTERN = re.compile(r"^RULE_ID\s*=\s*[\"']([^\"']+)[\"']", re.MULTILINE) + +# Counts that are not derived from the filesystem. +COMPLIANCE_FRAMEWORK_COUNT = 4 # CIS, NIST, ISO 27001, SOC 2 +AI_SECURITY_SKILL_COUNT = 8 + + +def count_rules() -> int: + """Return the number of scanner rule modules that declare a RULE_ID.""" + if not RULES_DIR.is_dir(): + return 0 + return sum(1 for path in RULES_DIR.glob(RULE_GLOB) if RULE_ID_PATTERN.search(path.read_text(encoding="utf-8"))) + + +def count_playbooks() -> int: + """Return the number of CLI remediation playbooks.""" + if not PLAYBOOKS_DIR.is_dir(): + return 0 + return len(list(PLAYBOOKS_DIR.glob("*.sh"))) + + +def collect_rule_stats() -> Tuple[Dict[str, int], Dict[str, int], List[str], List[str]]: + """Parse every rule file once for its SEVERITY and CATEGORY. + + Returns (severity_counts, category_counts, files_missing_severity, + files_missing_category). Severities and categories outside the values + seen so far are still counted, keyed by whatever string the rule + declares, so a new value is never silently dropped. + """ + severities: Dict[str, int] = {"CRITICAL": 0, "HIGH": 0, "MEDIUM": 0, "LOW": 0, "INFO": 0} + categories: Dict[str, int] = {} + missing_severity: List[str] = [] + missing_category: List[str] = [] + + if not RULES_DIR.is_dir(): + return severities, categories, missing_severity, missing_category + + for path in sorted(RULES_DIR.glob(RULE_GLOB)): + content = path.read_text(encoding="utf-8") + + sev_match = SEVERITY_PATTERN.search(content) + if sev_match: + severity = sev_match.group(1).strip().upper() + severities[severity] = severities.get(severity, 0) + 1 + else: + missing_severity.append(path.name) + + cat_match = CATEGORY_PATTERN.search(content) + if cat_match: + category = cat_match.group(1).strip() + categories[category] = categories.get(category, 0) + 1 + else: + missing_category.append(path.name) + + return severities, categories, missing_severity, missing_category + + +def render_category_rows(categories: Dict[str, int]) -> str: + """Build the '
...' lines for the category chart. + + Rows are sorted by descending count (alphabetical tiebreak) so a newly + added category appears automatically instead of requiring a code change. + Bar width is each category's count as a percentage of the largest + category, matching the original hand-authored chart's convention. + """ + if not categories: + return "" + + max_count = max(categories.values()) + rows = [] + for name, count in sorted(categories.items(), key=lambda item: (-item[1], item[0])): + width = round(count / max_count * 100) + rows.append( + f'
{name}' + f'
' + f"{count}
" + ) + return "\n".join(rows) + + +def _metric(label: str) -> str: + """Build the pattern for one headline metric tile on the Learn page.""" + return rf'(
)\d+({re.escape(label)}
)' + + +def apply_replacements(content: str, replacements: Tuple[Tuple[str, str, int], ...]) -> Tuple[str, List[str]]: + """Apply each (name, pattern, value) substitution, tracking zero-match failures. + + A pattern that matches zero times means the surrounding wording no longer + matches what this script expects — that is reported as a failure rather + than silently leaving the file unchanged. + """ + failures: List[str] = [] + for name, pattern, value in replacements: + content, count = re.subn(pattern, rf"\g<1>{value}\g<2>", content) + if count == 0: + failures.append(name) + return content, failures + + +def render( + content: str, + rule_count: int, + playbook_count: int, + high_count: int, + medium_count: int, + low_count: int, + category_rows: str, +) -> Tuple[str, List[str]]: + """Return (updated_content, failed_pattern_names) for docs/learn/index.html.""" + intro = ( + r'(

\s*OpenShield currently has )\d+' + r"( dynamic rules\. The strongest contributor work improves rule " + r"accuracy, reduces false positives,)" + ) + pipeline = ( + r'(

Rule Evaluation)' + r"\d+( dynamic checks
)" + ) + section_title = r'(

)\d+( Azure security rules

)' + hero_terminal = r'(loading rules: )\d+( dynamic checks

)' + severity_high = r'(
)\d+(HIGH
)' + severity_medium = r'(
)\d+(MEDIUM
)' + severity_low = r'(
)\d+(LOW
)' + + replacements: Tuple[Tuple[str, str, int], ...] = ( + ("headline metric: Azure scan rules", _metric("Azure scan rules"), rule_count), + ("headline metric: CLI remediation playbooks", _metric("CLI remediation playbooks"), playbook_count), + ("headline metric: Compliance frameworks", _metric("Compliance frameworks"), COMPLIANCE_FRAMEWORK_COUNT), + ("headline metric: AI security skills", _metric("AI security skills"), AI_SECURITY_SKILL_COUNT), + ("headline metric: High-severity checks", _metric("High-severity checks"), high_count), + ("pipeline step: Rule Evaluation", pipeline, rule_count), + ("rules section title", section_title, rule_count), + ("rules section intro paragraph", intro, rule_count), + ("hero terminal: dynamic checks line", hero_terminal, rule_count), + ("severity box: HIGH", severity_high, high_count), + ("severity box: MEDIUM", severity_medium, medium_count), + ("severity box: LOW", severity_low, low_count), + ) + + content, failures = apply_replacements(content, replacements) + + category_block = r'(
\n)(.*?)(\n {10}
)' + content, count = re.subn( + category_block, + lambda m: m.group(1) + category_rows + m.group(3), + content, + flags=re.DOTALL, + ) + if count == 0: + failures.append("coverage-by-category chart") + + return content, failures + + +def render_readme(content: str, rule_count: int, playbook_count: int) -> Tuple[str, List[str]]: + """Return (updated_content, failed_pattern_names) for README.md.""" + feature_row = ( + r"(\| \*\*Misconfiguration Scanner\*\* \| Runs )\d+" + r"( Azure security rules across storage, network, identity, database, " + r"compute, Key Vault, AKS, supply chain, and post-quantum cryptography \|)" + ) + playbook_row = ( + r"(\| \*\*Remediation Playbooks\*\* \| Every rule ships with a matching " + r"Azure CLI remediation script \()\d+( playbooks\) \|)" + ) + mermaid_scanner = r'(C\["Scanner Engine\\n)\d+( Python rules"\])' + mermaid_playbooks = r'(G\["Azure CLI Playbooks\\n)\d+( remediation scripts"\])' + + replacements: Tuple[Tuple[str, str, int], ...] = ( + ("feature table: Misconfiguration Scanner row", feature_row, rule_count), + ("feature table: Remediation Playbooks row", playbook_row, playbook_count), + ("Mermaid diagram: Scanner Engine node", mermaid_scanner, rule_count), + ("Mermaid diagram: Azure CLI Playbooks node", mermaid_playbooks, playbook_count), + ) + + return apply_replacements(content, replacements) + + +def main() -> int: + """Rewrite the Learn page and README statistics; return a process exit code.""" + for path in (LEARN_PAGE, README_PATH): + if not path.is_file(): + print(f"Error: {path} not found", file=sys.stderr) + return 1 + + rule_count = count_rules() + playbook_count = count_playbooks() + + if rule_count == 0 or playbook_count == 0: + print( + "Error: found no rules or no playbooks; refusing to write zeroes into the docs.", + file=sys.stderr, + ) + return 1 + + severities, categories, missing_severity, missing_category = collect_rule_stats() + + if missing_severity: + print( + f"Warning: {len(missing_severity)} rule file(s) have no parseable SEVERITY " + f"and are excluded from the severity counts: {', '.join(missing_severity)}", + file=sys.stderr, + ) + if missing_category: + print( + f"Warning: {len(missing_category)} rule file(s) have no parseable CATEGORY " + f"and are excluded from the coverage-by-category chart: {', '.join(missing_category)}", + file=sys.stderr, + ) + + category_rows = render_category_rows(categories) + + learn_original = LEARN_PAGE.read_text(encoding="utf-8") + learn_updated, learn_failures = render( + learn_original, + rule_count, + playbook_count, + severities["HIGH"], + severities["MEDIUM"], + severities["LOW"], + category_rows, + ) + + readme_original = README_PATH.read_text(encoding="utf-8") + readme_updated, readme_failures = render_readme(readme_original, rule_count, playbook_count) + + failures = [f"docs/learn/index.html -> {name}" for name in learn_failures] + failures += [f"README.md -> {name}" for name in readme_failures] + + if failures: + print( + "Error: the following patterns matched zero times. The surrounding wording " + "has likely changed and this script needs updating to match:", + file=sys.stderr, + ) + for name in failures: + print(f" - {name}", file=sys.stderr) + return 1 + + changed: List[str] = [] + if learn_updated != learn_original: + LEARN_PAGE.write_text(learn_updated, encoding="utf-8") + changed.append(str(LEARN_PAGE.relative_to(REPO_ROOT))) + if readme_updated != readme_original: + README_PATH.write_text(readme_updated, encoding="utf-8") + changed.append(str(README_PATH.relative_to(REPO_ROOT))) + + if not changed: + print("Learn page and README statistics already current; nothing to do.") + return 0 + + print( + f"Updated {', '.join(changed)} - rules: {rule_count}, playbooks: {playbook_count}, " + f"severity HIGH: {severities['HIGH']}, MEDIUM: {severities['MEDIUM']}, LOW: {severities['LOW']}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/workflows/update-learn-page.yml b/.github/workflows/update-learn-page.yml new file mode 100644 index 0000000..4855ae6 --- /dev/null +++ b/.github/workflows/update-learn-page.yml @@ -0,0 +1,47 @@ +name: Update Learn Page and README Stats + +on: + push: + branches: [dev] + +# Only the final commit step writes; nothing here needs any other scope. +permissions: + contents: write + +concurrency: + group: update-learn-page-${{ github.ref }} + cancel-in-progress: true + +jobs: + update-learn-page: + name: Refresh Learn page and README statistics + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + + - name: Set up Python 3.11 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.11" + + - name: Refresh statistics + run: python .github/scripts/update_learn_page.py + + - name: Detect changes + id: diff + run: | + if git diff --quiet -- docs/learn/index.html README.md; then + echo "changed=false" >> "$GITHUB_OUTPUT" + else + echo "changed=true" >> "$GITHUB_OUTPUT" + fi + + - name: Commit and push + if: steps.diff.outputs.changed == 'true' + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add docs/learn/index.html README.md + git commit -s -m "docs: refresh learn page and README statistics [skip ci]" + git push diff --git a/README.md b/README.md index 09ecd71..1e32998 100644 --- a/README.md +++ b/README.md @@ -51,10 +51,10 @@ Findings map to NIST FIPS 203 (ML-KEM), FIPS 204 (ML-DSA), and FIPS 205 (SLH-DSA | Feature | Description | |---|---| -| **Misconfiguration Scanner** | Runs 51 Azure security rules across storage, network, identity, database, compute, Key Vault, AKS, and post-quantum cryptography | +| **Misconfiguration Scanner** | Runs 65 Azure security rules across storage, network, identity, database, compute, Key Vault, AKS, supply chain, and post-quantum cryptography | | **Compliance Mapper** | Maps findings to CIS Benchmarks, NIST CSF, ISO 27001, and SOC 2 framework JSON files | | **Scan History API** | Stores scans and findings in PostgreSQL and exposes findings, score, scan history, compliance posture, drift, and resource inventory over REST | -| **Remediation Playbooks** | Every rule ships with a matching Azure CLI remediation script (51 playbooks) | +| **Remediation Playbooks** | Every rule ships with a matching Azure CLI remediation script (65 playbooks) | | **Security Dashboard** | Full React dashboard deployed on Vercel - live monitoring, findings, compliance, drift, prioritization, and AI-layer views | | **Project Website** | Documentation and reference site at [openshield-website.vercel.app](https://openshield-website.vercel.app) - blog, rules gallery, docs, roadmap, releases, and interactive playground | | **Sentinel Integration** | Normalises findings and pushes them into Microsoft Sentinel via a Log Analytics custom table and KQL analytics rules | @@ -96,11 +96,11 @@ Project policies and assurance evidence: flowchart TD A["React Dashboard\nVercel · Live"] B["Flask REST API\nJWT · CORS · Blueprints"] - C["Scanner Engine\n51 Python rules"] + C["Scanner Engine\n65 Python rules"] D["Azure Subscription\nScanned via Azure SDK + Graph"] E["Compliance Framework JSON\nCIS · NIST · ISO 27001 · SOC 2"] F["PostgreSQL Database\nFindings · Scans"] - G["Azure CLI Playbooks\n51 remediation scripts"] + G["Azure CLI Playbooks\n65 remediation scripts"] H["sentinel/ingest.py\nNormalise + HMAC upload"] I["Microsoft Sentinel\nOpenShieldFindings_CL · KQL rules"] diff --git a/docs/learn/index.html b/docs/learn/index.html index a9aaa4f..f26f64c 100644 --- a/docs/learn/index.html +++ b/docs/learn/index.html @@ -752,7 +752,7 @@

Learn Azure security posture with OpenShield.

openshield scan --subscription Azure

-

loading rules: 39 dynamic checks

+

loading rules: 65 dynamic checks

enrichment: NVD / CVE intelligence

storage: PostgreSQL scan history

api: Flask + JWT + CORS

@@ -764,11 +764,11 @@

Learn Azure security posture with OpenShield.
-
39Azure scan rules
-
39CLI remediation playbooks
+
65Azure scan rules
+
65CLI remediation playbooks
4Compliance frameworks
8AI security skills
-
22High-severity checks
+
38High-severity checks
@@ -821,7 +821,7 @@

Production-shaped, MVP-friendly architecture

Azure SubscriptionResources and configuration
Scanner EnginePython rule execution
-
Rule Evaluation39 dynamic checks
+
Rule Evaluation65 dynamic checks
CVE EnrichmentNVD risk context
PostgreSQLFindings and scan history
Flask APIJWT-protected REST routes
@@ -841,9 +841,9 @@

Production-shaped, MVP-friendly architecture

Rule coverage

-

51 Azure security rules

+

65 Azure security rules

- OpenShield currently has 39 dynamic rules. The strongest contributor work improves rule accuracy, reduces false positives, + OpenShield currently has 65 dynamic rules. The strongest contributor work improves rule accuracy, reduces false positives, strengthens validation, or improves remediation quality.

@@ -851,13 +851,15 @@

51 Azure security rules

Coverage by category

-
Network
14
-
Storage
5
-
Key Vault
5
-
Compute
4
-
Database
4
-
Identity
4
-
PostQuantum
3
+
Identity
15
+
Network
15
+
Supply Chain
8
+
Kubernetes
6
+
KeyVault
5
+
Storage
5
+
Compute
4
+
Database
4
+
PostQuantum
3
@@ -865,11 +867,10 @@

Coverage by category

Severity distribution

Most checks are high severity. That makes validation important: high-severity false positives damage trust quickly.

-
22HIGH
-
13MEDIUM
+
38HIGH
+
22MEDIUM
4LOW
-

Known cleanup item: keep category names consistent, especially KeyVault vs Key Vault.

@@ -931,7 +932,7 @@

Current cleanup items

Documentation drift

    -
  • Rule coverage and documentation counts are checked and updated with each release.
    • +
  • Rule, playbook, and severity statistics on this page are generated from scanner/rules/ and playbooks/cli/ by .github/scripts/update_learn_page.py on every push to dev. The equivalent counts in README.md are not automated and are still updated by hand.
  • Some startup commands assume python, but local environments may only expose python3.
  • API docs and implementation should stay aligned, especially score response shape.