diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml index b350331..a535e6f 100644 --- a/.pre-commit-hooks.yaml +++ b/.pre-commit-hooks.yaml @@ -97,9 +97,10 @@ - id: terraform_tflint name: TFLint (Terraform linter) - entry: tflint - language: system + entry: hooks/terraform_tflint.sh + language: script files: \.tf$ + pass_filenames: false require_serial: false - id: cloudformation_validate @@ -135,6 +136,11 @@ language: python types: [text] require_serial: false + # Example usage with exclusions: + # args: ['--exclude-patterns', 'localhost,127.0.0.1'] + # args: ['--exclude-files', 'README.md,CHANGELOG.md,LICENSE'] + # args: ['--safe-domains', 'cms.gov,amazonaws.com,github.com,yourcompany.com'] + # args: ['--safe-protocols', 'https,http,jdbc,postgresql,mysql,mongodb'] - id: hardcoded_credentials name: Detect hardcoded credentials diff --git a/CONFIGURATION_GUIDE.md b/CONFIGURATION_GUIDE.md index 01e99d5..2f7dcb5 100644 --- a/CONFIGURATION_GUIDE.md +++ b/CONFIGURATION_GUIDE.md @@ -127,7 +127,7 @@ Exclude entire files or directories: ```yaml repos: - repo: https://github.com/TriaFed/pre-commit-library - rev: v1.1.7 + rev: v1.1.8 hooks: - id: genai-security-check exclude: '^(tests/|spec/|__tests__/|\.test\.|\.spec\.)' @@ -210,7 +210,7 @@ console.error('Login failed:', user.password); // Don't do this! ```yaml repos: - repo: https://github.com/TriaFed/pre-commit-library - rev: v1.1.7 + rev: v1.1.8 hooks: # Security with baselines and exclusions - id: detect-secrets diff --git a/README.md b/README.md index c9258fc..c6e4f24 100644 --- a/README.md +++ b/README.md @@ -89,7 +89,7 @@ Create a `.pre-commit-config.yaml` file in your project root: ```yaml repos: - repo: https://github.com/TriaFed/pre-commit-library - rev: v1.1.7 + rev: v1.1.8 hooks: # Security hooks (recommended for all projects) - id: detect_secrets @@ -196,7 +196,7 @@ For projects using GenAI tools, start with these essential security hooks: ```yaml repos: - repo: https://github.com/TriaFed/pre-commit-library - rev: v1.1.7 + rev: v1.1.8 hooks: - id: detect_secrets - id: hardcoded_credentials @@ -213,7 +213,7 @@ repos: ```yaml repos: - repo: https://github.com/TriaFed/pre-commit-library - rev: v1.1.7 + rev: v1.1.8 hooks: # Security - id: detect_secrets @@ -243,7 +243,7 @@ repos: ```yaml repos: - repo: https://github.com/TriaFed/pre-commit-library - rev: v1.1.7 + rev: v1.1.8 hooks: # Security - id: detect_secrets @@ -271,7 +271,7 @@ repos: ```yaml repos: - repo: https://github.com/TriaFed/pre-commit-library - rev: v1.1.7 + rev: v1.1.8 hooks: # Security - id: detect_secrets @@ -300,7 +300,7 @@ repos: ```yaml repos: - repo: https://github.com/TriaFed/pre-commit-library - rev: v1.1.7 + rev: v1.1.8 hooks: # Security - id: detect_secrets @@ -327,7 +327,7 @@ repos: ```yaml repos: - repo: https://github.com/TriaFed/pre-commit-library - rev: v1.1.7 + rev: v1.1.8 hooks: # Security - id: detect_secrets @@ -354,7 +354,7 @@ repos: ```yaml repos: - repo: https://github.com/TriaFed/pre-commit-library - rev: v1.1.7 + rev: v1.1.8 hooks: # Security - id: detect_secrets @@ -380,7 +380,7 @@ repos: ```yaml repos: - repo: https://github.com/TriaFed/pre-commit-library - rev: v1.1.7 + rev: v1.1.8 hooks: # Security - id: detect_secrets @@ -408,7 +408,7 @@ repos: ```yaml repos: - repo: https://github.com/TriaFed/pre-commit-library - rev: v1.1.7 + rev: v1.1.8 hooks: # Security (essential for GenAI projects) - id: detect_secrets @@ -724,7 +724,7 @@ For large repositories: ```yaml repos: - repo: https://github.com/TriaFed/pre-commit-library - rev: v1.1.7 + rev: v1.1.8 hooks: - id: detect_secrets exclude: ^(docs/|tests/fixtures/) diff --git a/examples/ansible-project.yaml b/examples/ansible-project.yaml index d66176c..7a011c1 100644 --- a/examples/ansible-project.yaml +++ b/examples/ansible-project.yaml @@ -3,7 +3,7 @@ repos: - repo: https://github.com/TriaFed/pre-commit-library - rev: v1.1.7 # Use the latest version + rev: v1.1.8 # Use the latest version hooks: # Security hooks (essential for GenAI projects) - id: detect_secrets diff --git a/examples/dotnet-project.yaml b/examples/dotnet-project.yaml index 1067e83..34a6c15 100644 --- a/examples/dotnet-project.yaml +++ b/examples/dotnet-project.yaml @@ -3,7 +3,7 @@ repos: - repo: https://github.com/TriaFed/pre-commit-library - rev: v1.1.7 # Use the latest version + rev: v1.1.8 # Use the latest version hooks: # Security hooks (essential for GenAI projects) - id: detect_secrets diff --git a/examples/full-stack.yaml b/examples/full-stack.yaml index 7ed49c8..21b745b 100644 --- a/examples/full-stack.yaml +++ b/examples/full-stack.yaml @@ -3,7 +3,7 @@ repos: - repo: https://github.com/TriaFed/pre-commit-library - rev: v1.1.7 # Use the latest version + rev: v1.1.8 # Use the latest version hooks: # Security hooks (essential for ALL GenAI projects) - id: detect_secrets diff --git a/examples/go-project.yaml b/examples/go-project.yaml index b929c61..8bc65f2 100644 --- a/examples/go-project.yaml +++ b/examples/go-project.yaml @@ -3,7 +3,7 @@ repos: - repo: https://github.com/TriaFed/pre-commit-library - rev: v1.1.7 # Use the latest version + rev: v1.1.8 # Use the latest version hooks: # Security hooks (essential for GenAI projects) - id: detect_secrets diff --git a/examples/infrastructure.yaml b/examples/infrastructure.yaml index d04638c..9fbd0ba 100644 --- a/examples/infrastructure.yaml +++ b/examples/infrastructure.yaml @@ -3,7 +3,7 @@ repos: - repo: https://github.com/TriaFed/pre-commit-library - rev: v1.1.7 # Use the latest version + rev: v1.1.8 # Use the latest version hooks: # Security hooks (essential for GenAI projects) - id: detect_secrets diff --git a/examples/java-project.yaml b/examples/java-project.yaml index 31cac5e..f5e57cb 100644 --- a/examples/java-project.yaml +++ b/examples/java-project.yaml @@ -3,7 +3,7 @@ repos: - repo: https://github.com/TriaFed/pre-commit-library - rev: v1.1.7 # Use the latest version + rev: v1.1.8 # Use the latest version hooks: # Security hooks (essential for GenAI projects) - id: detect_secrets diff --git a/examples/minimal-security.yaml b/examples/minimal-security.yaml index 1a2d7f6..e5bc986 100644 --- a/examples/minimal-security.yaml +++ b/examples/minimal-security.yaml @@ -3,7 +3,7 @@ repos: - repo: https://github.com/TriaFed/pre-commit-library - rev: v1.1.7 # Use the latest version + rev: v1.1.8 # Use the latest version hooks: # Critical security hooks for GenAI-generated code - id: detect_secrets diff --git a/examples/nodejs-react.yaml b/examples/nodejs-react.yaml index 6405709..faf5e5d 100644 --- a/examples/nodejs-react.yaml +++ b/examples/nodejs-react.yaml @@ -3,7 +3,7 @@ repos: - repo: https://github.com/TriaFed/pre-commit-library - rev: v1.1.7 # Use the latest version + rev: v1.1.8 # Use the latest version hooks: # Security hooks (essential for GenAI projects) - id: detect_secrets diff --git a/examples/practical-security.yaml b/examples/practical-security.yaml index 857b7bd..f6fc4eb 100644 --- a/examples/practical-security.yaml +++ b/examples/practical-security.yaml @@ -3,7 +3,7 @@ repos: - repo: https://github.com/TriaFed/pre-commit-library - rev: v1.1.7 # Use the latest version + rev: v1.1.8 # Use the latest version hooks: # Essential security hooks with practical settings - id: detect-secrets diff --git a/examples/python-project.yaml b/examples/python-project.yaml index ba42567..833566b 100644 --- a/examples/python-project.yaml +++ b/examples/python-project.yaml @@ -3,7 +3,7 @@ repos: - repo: https://github.com/TriaFed/pre-commit-library - rev: v1.1.7 # Use the latest version + rev: v1.1.8 # Use the latest version hooks: # Security hooks (essential for GenAI projects) - id: detect_secrets diff --git a/hooks/detect_hardcoded_urls.py b/hooks/detect_hardcoded_urls.py index 615cdd4..5e26735 100755 --- a/hooks/detect_hardcoded_urls.py +++ b/hooks/detect_hardcoded_urls.py @@ -6,23 +6,93 @@ import re import sys +import os import argparse -from typing import List, Tuple, Set - -# Common patterns that indicate hardcoded URLs -URL_PATTERNS = [ - # HTTP/HTTPS URLs - r'https?://[^\s\'">\]]+', - # FTP URLs - r'ftp://[^\s\'">\]]+', - # Database connection strings with URLs - r'(?:jdbc|mongodb|mysql|postgresql)://[^\s\'">\]]+', - # API endpoints patterns - r'(?:api\.|www\.)[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}(?:/[^\s\'">\]]*)?', -] +from typing import List, Tuple, Set, Optional, Pattern + +# Protocol definitions - defined early to avoid circular dependencies +_WEB_PROTOCOLS = ['https', 'http'] # Standard web protocols + +# Database protocols that can be used directly (not via JDBC) +_DIRECT_DB_PROTOCOLS = ['postgresql', 'mysql', 'mongodb'] + +# JDBC subprotocols that are allowed when using jdbc: prefix +_JDBC_SUBPROTOCOLS = ['postgresql', 'mysql', 'mariadb', 'h2', 'sqlite', 'oracle', 'sqlserver'] + +# All database protocols combined (direct + JDBC) +_DATABASE_PROTOCOLS = ['jdbc'] + _DIRECT_DB_PROTOCOLS + +# All protocols combined - includes database protocols because: +# 1. Government/AWS systems often use managed database services (RDS, etc.) +# 2. Connection strings may legitimately reference these domains +# 3. Infrastructure-as-code often includes database configurations +_ALL_PROTOCOLS = _WEB_PROTOCOLS + _DATABASE_PROTOCOLS + +# Regex pattern components for URL matching +# RFC-compliant subdomain pattern: 1-63 chars, alphanumeric + hyphens (not at start/end) +# This prevents domain spoofing like badsite.com.legitimate.gov matching legitimate.gov +_SUBDOMAIN_PATTERN = r'[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?' + +# Optional subdomain prefix: up to 5 levels of subdomains +# Each subdomain followed by a dot, entire group can appear 1-5 times, whole thing is optional +_OPTIONAL_SUBDOMAIN_PREFIX = f'(?:(?:{_SUBDOMAIN_PATTERN}\\.)+)?' + +# URL component patterns +_PORT_PATTERN = r'(?::\d+)?' # Optional port number (:80, :443, :8080, etc.) +_PATH_PATTERN = r'(?:/.*)?' # Optional path and query string (/path?query=value) + +def generate_domain_patterns(domains: List[str], protocols: List[str]) -> List[str]: + """Generate URL patterns for given domains and protocols. + + Args: + domains: List of domain names (e.g., ['cms.gov', 'github.com']) + protocols: List of plain protocol names (e.g., ['https', 'http', 'jdbc', 'postgresql']) + Protocol names are automatically escaped for regex safety, except 'jdbc' + which is treated specially to match 'jdbc:subprotocol' patterns. + + Returns: + List of regex patterns that match URLs with the specified protocols and domains. + """ + patterns = [] + + # Strip protocols once to avoid redundant operations in the loop + stripped_protocols = [protocol.strip() for protocol in protocols] + + # Convert plain protocol names to regex patterns + regex_protocols = [] + for protocol in stripped_protocols: + if protocol == 'jdbc': + # JDBC URLs have format jdbc:subprotocol://... + # Use the centrally defined JDBC subprotocols for consistency + # This prevents inadvertent whitelisting of malicious URLs + jdbc_patterns = [f'jdbc:{re.escape(subprotocol)}' for subprotocol in _JDBC_SUBPROTOCOLS] + regex_protocols.extend(jdbc_patterns) + else: + # For other protocols, escape any special regex characters to treat them as literals + escaped_protocol = re.escape(protocol) + regex_protocols.append(escaped_protocol) + + protocol_group = f'(?:{"|".join(regex_protocols)})' + + for domain in domains: + # Escape dots in domain names for regex + escaped_domain = domain.replace('.', r'\.') + + # Build URL pattern with clear components: + # 1. Protocol group (https, http, jdbc:mysql, etc.) + # 2. Optional subdomain prefix (api., secure.login., etc.) - prevents domain spoofing + # 3. Base domain (cms.gov, github.com, etc.) + # 4. Optional port number (:80, :443, :8080, etc.) + # 5. Optional path and query string (/path?query=value, etc.) + url_pattern = f"{protocol_group}://{_OPTIONAL_SUBDOMAIN_PREFIX}{escaped_domain}{_PORT_PATTERN}{_PATH_PATTERN}" + patterns.append(url_pattern) + + return patterns + # URLs that are typically safe to ignore -SAFE_URL_PATTERNS = [ +# Static patterns for common localhost and development URLs +_STATIC_SAFE_PATTERNS = [ r'https?://localhost', r'https?://127\.0\.0\.1', r'https?://0\.0\.0\.0', @@ -33,8 +103,17 @@ r'https?://.*\.test', r'https?://.*\.local', r'https?://.*\.localhost', - # Common documentation URLs + # Git URLs (common version control systems) r'https?://github\.com/.*', + r'https?://gitlab\.com/.*', + r'https?://bitbucket\.org/.*', + r'https?://.*\.github\.io/.*', + r'git://.*', + r'ssh://git@.*', + # Schema and specification URLs (HTTP only) + r'https?://adaptivecards\.io/.*', + r'https?://.*\.adaptivecards\.io/.*', + # Common documentation URLs (HTTP only) r'https?://docs\..*', r'https?://www\.w3\.org/.*', r'https?://tools\.ietf\.org/.*', @@ -45,11 +124,62 @@ r'https?://central\.maven\.org/.*', ] +# Domain definitions +_GOVERNMENT_DOMAINS = ['cms.gov', 'cmscloud.local'] +_AWS_DOMAINS = ['amazonaws.com'] + +# Function to generate URL detection patterns based on our protocol definitions +def _generate_url_detection_patterns() -> List[str]: + """Generate URL detection patterns using our centralized protocol definitions.""" + patterns = [ + # HTTP/HTTPS URLs (exclude whitespace, quotes, >, backslash, closing ]) + # In raw string r'...\\\]': the \\\] sequence = 3 backslashes + bracket in string + # Regex interprets this as: \\ (escaped backslash) + \] (escaped bracket) + # Both \ and ] are excluded independently. Do NOT change to \\] (would break regex). + r'https?://[^\s\'">\\\]]+', + # FTP URLs (same exclusions) + r'ftp://[^\s\'">\\\]]+', + # API endpoints (same exclusions after optional /) + r'(?:api\.|www\.)[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}(?:/[^\s\'">\\\]]*)?', + ] + + # Add direct database protocol patterns (raw f-string for clarity of escapes) + for protocol in _DIRECT_DB_PROTOCOLS: + patterns.append(fr'{re.escape(protocol)}://[^\s\'">\\\]]+') # direct DB protocols + + # Add JDBC subprotocol patterns (raw f-string) + for subprotocol in _JDBC_SUBPROTOCOLS: + patterns.append(fr'jdbc:{re.escape(subprotocol)}://[^\s\'">\\\]]+') # JDBC subprotocols + + return patterns + +# URL patterns that indicate hardcoded URLs (pre-compiled for performance) +def _compile_patterns_safely(patterns: List[str], pattern_type: str = "URL detection") -> List[Pattern[str]]: + """Compile regex patterns with error handling and user warnings.""" + compiled = [] + for pattern in patterns: + try: + compiled.append(re.compile(pattern, re.IGNORECASE)) + except re.error as err: + # Log a warning to stderr when skipping malformed regex patterns + print(f"Warning: Malformed {pattern_type} regex pattern ignored: '{pattern}' ({err})", file=sys.stderr) + return compiled + +URL_PATTERNS = _compile_patterns_safely(_generate_url_detection_patterns(), "URL detection") + +# Build the final SAFE_URL_PATTERNS by combining static and dynamic patterns +SAFE_URL_PATTERNS = _STATIC_SAFE_PATTERNS.copy() +SAFE_URL_PATTERNS.extend(generate_domain_patterns(_GOVERNMENT_DOMAINS, _ALL_PROTOCOLS)) +SAFE_URL_PATTERNS.extend(generate_domain_patterns(_AWS_DOMAINS, _ALL_PROTOCOLS)) + +# Pre-compile safe patterns for performance and security +_COMPILED_SAFE_PATTERNS = _compile_patterns_safely(SAFE_URL_PATTERNS, "safe URL whitelist") + # File extensions to skip SKIP_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.gif', '.ico', '.svg', '.pdf', '.zip', '.tar', '.gz'} # Patterns that suggest this might be in a comment or documentation -COMMENT_PATTERNS = [ +_COMMENT_PATTERN_STRINGS = [ r'^\s*#', # Python, shell comments r'^\s*//', # JavaScript, Java, C++ comments r'^\s*/\*', # Multi-line comment start @@ -57,69 +187,142 @@ r'^\s*