# Generate CA and certificates
export JOBLET_SERVER_ADDRESS='192.168.1.100'
sudo /usr/local/bin/certs_gen_embedded.sh

# 1. Create CA
openssl genrsa -out ca-key.pem 4096
openssl req -new -x509 -key ca-key.pem -out ca-cert.pem -days 3650 \
  -subj "/CN=Joblet CA"

# 2. Server certificate (CN must be "joblet")
openssl genrsa -out server-key.pem 4096
openssl req -new -key server-key.pem -out server.csr \
  -subj "/CN=joblet"
openssl x509 -req -in server.csr -CA ca-cert.pem -CAkey ca-key.pem \
  -out server-cert.pem -days 365 -CAcreateserial \
  -extensions v3_req -extfile <(echo "[v3_req]
subjectAltName = DNS:localhost,DNS:joblet,IP:127.0.0.1,IP:${SERVER_IP}")

# 3. Client certificates (one per role; the OU field carries the role)
for role in admin maintainer developer reader; do
  openssl genrsa -out ${role}-key.pem 4096
  openssl req -new -key ${role}-key.pem -out ${role}.csr \
    -subj "/CN=${role}-client/OU=${role}"
  openssl x509 -req -in ${role}.csr -CA ca-cert.pem -CAkey ca-key.pem \
    -out ${role}-cert.pem -days 365 -CAcreateserial
done

# 1. Generate new certificates
sudo /usr/local/bin/certs_gen_embedded.sh

# 2. Update server configuration
sudo systemctl restart joblet

# 3. Distribute new client certificates
scp /opt/joblet/config/rnx-config.yml client:~/.rnx/

# 4. Verify new certificates
rnx job list  # Should work with new certs

# Every operation, including removal of shared infrastructure
rnx runtime remove openjdk-21
rnx network remove old-net
rnx volume remove old-vol

# Plus everything maintainer, developer, and reader can do

# Infrastructure provisioning (allowed)
rnx runtime build ./examples/java-21/runtime.yaml
rnx runtime validate ./examples/java-21/runtime.yaml
rnx network create ci-net --cidr=10.1.0.0/24
rnx volume create ci-vol --size=1GB

# Plus everything developer can do

# Removal of shared infrastructure (denied)
rnx runtime remove openjdk-21    # ERROR: Permission denied
rnx network remove ci-net        # ERROR: Permission denied
rnx volume remove ci-vol         # ERROR: Permission denied

# Job execution on existing infrastructure (allowed)
rnx job run echo "Developers can run jobs"
rnx job stop <job-id>
rnx job delete <job-id>
rnx runtime test openjdk-21

# Plus all read access

# Infrastructure changes (denied)
rnx runtime build ./runtime.yaml # ERROR: Permission denied
rnx network create dev-net       # ERROR: Permission denied
rnx volume create dev-vol        # ERROR: Permission denied

# Read-only operations (allowed)
rnx job list
rnx job status <job-id>
rnx job log <job-id>
rnx runtime list
rnx network list
rnx volume list
rnx monitor status

# Create, execute, and delete operations (denied)
rnx job run echo "test"          # ERROR: Permission denied
rnx job stop <job-id>            # ERROR: Permission denied
rnx volume create test           # ERROR: Permission denied

# Generate certificates for different users
# DevOps team (admin access)
openssl req -new -key devops-key.pem -out devops.csr \
  -subj "/CN=devops-team/OU=admin"

# CI/CD service account (maintainer access)
openssl req -new -key cicd-key.pem -out cicd.csr \
  -subj "/CN=cicd-pipeline/OU=maintainer"

# Developers (developer access)
openssl req -new -key dev-key.pem -out dev.csr \
  -subj "/CN=developer/OU=developer"

# Reporting/observability (reader access)
openssl req -new -key report-key.pem -out report.csr \
  -subj "/CN=reporting/OU=reader"

# Sign all certificates with CA
for cert in devops cicd dev report; do
  openssl x509 -req -in ${cert}.csr -CA ca-cert.pem -CAkey ca-key.pem \
    -out ${cert}-cert.pem -days 365 -CAcreateserial
done

# Production Jobs (JobService API) - Minimal Chroot
rnx job run echo "Hello World"           # Uses minimal chroot isolation
rnx job run --runtime=java:21 java App  # Secure runtime mounting

# Runtime Build Jobs (RuntimeService API) - Builder Chroot
rnx runtime build ./examples/java-21/runtime.yaml  # Uses builder chroot with host OS access

# Minimal filesystem access
rnx job run ls /                    # Limited directories
rnx job run which apt              # Command not found
rnx job run ls /opt/joblet         # No access to joblet internals

# Controlled host OS access (ONLY during runtime building)
# - Full host filesystem via OverlayFS (host root is a read-only lower layer)
# - Writes are captured in an upper layer; the host filesystem is never modified
# - Package managers available (runtime builds share host networking)
# - Automatic cleanup creates isolated runtime structure

# PID namespace - process isolation
rnx job run ps aux  # Only sees job processes

# Mount namespace - filesystem isolation
rnx job run mount  # Shows only job-specific mounts

# Network namespace - network isolation
rnx job run ip addr show  # Shows only job network interface

# IPC namespace - inter-process communication isolation
rnx job run ipcs  # No shared memory/semaphores from host

# UTS namespace - hostname isolation
rnx job run hostname  # Job-specific hostname

# Cgroup namespace - resource telematics
rnx job run cat /proc/cgroups  # Limited cgroup view

# BEFORE cleanup (INSECURE): Runtime mounts point to host OS paths
# runtime.yml contained:
# - source: "usr/lib/jvm/java-21-openjdk-amd64"  # HOST PATH!

# AFTER cleanup (SECURE): Runtime mounts point to isolated copies  
# runtime.yml contains:
# - source: "isolated/usr/lib/jvm/java-21-openjdk-amd64"  # ISOLATED COPY

# Production jobs using runtimes are completely isolated
rnx job run --runtime=java:21 find /usr -type f | head -5
# Only shows isolated runtime files, not host OS files

# Jobs run as unprivileged user
rnx job run id
# Output: uid=65534(nobody) gid=65534(nogroup)

# No sudo/setuid capabilities
rnx job run sudo echo "test"  # Command not found

# Limited filesystem access
rnx job run ls /root  # Permission denied
rnx job run ls /etc/shadow  # Permission denied

# Prevent fork bombs
rnx job run --max-cpu=100 :(){ :|:& };:  # Limited by CPU quota

# Prevent memory exhaustion
rnx job run --max-memory=512 bash -c 'a=(); while true; do a+=($a); done'  # Killed by OOM

# Prevent I/O attacks
rnx job run --max-iobps=1048576 dd if=/dev/zero of=/work/attack bs=1M  # Limited bandwidth

# Create isolated networks for different security zones
rnx network create dmz --cidr=10.1.0.0/24           # Public-facing
rnx network create internal --cidr=10.2.0.0/24      # Internal services
rnx network create secure --cidr=10.3.0.0/24        # Sensitive data

# Jobs in different networks cannot communicate
rnx job run --network=dmz ping 10.2.0.1        # Will fail
rnx job run --network=internal ping 10.3.0.1   # Will fail

# No network access for sensitive processing
rnx job run --network=none --volume=sensitive-data \
  python process_classified.py

# Limited network access
rnx job run --network=internal --volume=app-data \
  python internal_service.py

# Full internet access (carefully controlled)
rnx job run --network=bridge \
  curl https://api.trusted-service.com

# Limit bandwidth to prevent data exfiltration
rnx job run --max-iobps=1048576 --network=bridge \
  curl https://malicious-site.com  # Limited to 1MB/s

# Monitor network usage
rnx job run --network=monitored iftop

# 1. Create encrypted volume
rnx volume create encrypted-data --size=10GB

# 2. Encrypt data before storage
rnx job run --volume=encrypted-data bash -c '
  echo "sensitive information" | \
  openssl enc -aes-256-cbc -k "$ENCRYPTION_KEY" \
  > /volumes/encrypted-data/secret.enc
'

# 3. Decrypt only when needed
rnx job run --volume=encrypted-data --env=ENCRYPTION_KEY=xxx bash -c '
  openssl enc -aes-256-cbc -d -k "$ENCRYPTION_KEY" \
  < /volumes/encrypted-data/secret.enc
'

# Avoid embedding secrets in commands (BAD)
rnx job run curl -H "Authorization: Bearer secret123" api.com

# Use environment variables (BETTER)
rnx job run --env=API_TOKEN=secret123 \
  curl -H "Authorization: Bearer \$API_TOKEN" api.com

# Use volume-based secrets (BEST)
echo "secret123" | rnx job run --volume=secrets bash -c '
  cat > /volumes/secrets/api-token
  chmod 600 /volumes/secrets/api-token
'

rnx job run --volume=secrets bash -c '
  API_TOKEN=$(cat /volumes/secrets/api-token)
  curl -H "Authorization: Bearer $API_TOKEN" api.com
'

# Public data - no restrictions
rnx job run --network=bridge --volume=public-data \
  wget https://public-dataset.com/data.csv

# Internal data - network restrictions
rnx job run --network=internal --volume=internal-data \
  python process_internal.py

# Confidential data - maximum isolation
rnx job run --network=none --volume=confidential-data \
  python process_confidential.py

# Secret data - encrypted storage
rnx job run --network=none --volume=encrypted-secrets \
  --env=DECRYPT_KEY=xxx \
  python process_secrets.py

# 1. Minimal server installation
sudo apt install --no-install-recommends joblet

# 2. Disable unnecessary services
sudo systemctl disable apache2 nginx mysql

# 3. Configure firewall
sudo ufw allow 50051/tcp  # Joblet port only
sudo ufw enable

# 4. Regular updates
sudo apt update && sudo apt upgrade

# 5. Secure SSH
# /etc/ssh/sshd_config
PermitRootLogin no
PasswordAuthentication no
ChallengeResponseAuthentication no

# Secure configuration files
sudo chmod 600 /opt/joblet/config/joblet-config.yml
sudo chown root:root /opt/joblet/config/joblet-config.yml

# Secure certificates
sudo chmod 600 /opt/joblet/certs/*.pem
sudo chown root:root /opt/joblet/certs/*.pem

# Secure log files
sudo chmod 640 /var/log/joblet/*.log
sudo chown root:joblet /var/log/joblet/*.log

# Monitor failed authentication attempts
sudo tail -f /var/log/joblet/audit.log | grep "auth_failed"

# Monitor admin operations
sudo tail -f /var/log/joblet/audit.log | grep "admin_operation"

# Monitor unusual job patterns
sudo tail -f /var/log/joblet/audit.log | jq 'select(.job_count > 100)'

# Monitor resource usage spikes
rnx monitor --json | jq 'select(.cpu_usage > 90 or .memory_usage > 90)'

# Create monitoring script
cat > security_monitor.sh << 'EOF'
#!/bin/bash
# Monitor for security events

# Check for multiple failed auth attempts
FAILED_AUTH=$(grep -c "auth_failed" /var/log/joblet/audit.log | tail -100)
if [ $FAILED_AUTH -gt 10 ]; then
  echo "ALERT: Multiple authentication failures detected"
fi

# Check for unusual job patterns
RUNNING_JOBS=$(rnx job list --json | jq '[.[] | select(.status == "RUNNING")] | length')
if [ $RUNNING_JOBS -gt 50 ]; then
  echo "ALERT: Unusual number of running jobs: $RUNNING_JOBS"
fi

# Check for resource exhaustion
CPU_USAGE=$(rnx monitor status --json | jq .cpu_usage)
if [ $(echo "$CPU_USAGE > 95" | bc) -eq 1 ]; then
  echo "ALERT: High CPU usage: $CPU_USAGE%"
fi
EOF

# Schedule monitoring
echo "*/5 * * * * /opt/joblet/scripts/security_monitor.sh" | sudo crontab -

# HIPAA-compliant setup
# 1. Encrypted storage
rnx volume create phi-data --size=100GB --type=filesystem

# 2. Encrypted transit (already provided by mTLS)

# 3. Access logging
# (Already provided by audit logging)

# 4. Data minimization
rnx job run --network=none --volume=phi-data \
  python anonymize_phi.py

# 5. Secure disposal
rnx volume remove phi-data  # Secure deletion

# PCI DSS network segmentation
rnx network create pci-zone --cidr=10.100.0.0/24

# Restricted processing environment
rnx job run \
  --network=pci-zone \
  --volume=pci-secure \
  --max-memory=2048 \
  --env=PCI_MODE=true \
  python process_payments.py

# Automated threat detection
cat > threat_detection.sh << 'EOF'
#!/bin/bash

# Detect privilege escalation attempts
if grep -q "setuid\|sudo\|su " /var/log/joblet/audit.log; then
  echo "THREAT: Privilege escalation attempt detected"
fi

# Detect network scanning
if rnx job list --json | jq '.[] | select(.command | contains("nmap"))' | grep -q .; then
  echo "THREAT: Network scanning detected"
fi

# Detect data exfiltration patterns
if rnx job list --json | jq '.[] | select(.command | contains("curl") and .max_iobps == 0)' | grep -q .; then
  echo "THREAT: Potential data exfiltration (unlimited bandwidth)"
fi
EOF

# 1. Immediate response
# Stop suspicious jobs
rnx job list --json | jq -r '.[] | select(.status == "RUNNING" and (.command | contains("suspicious"))) | .id' | xargs rnx job stop

# 2. Isolate affected networks
rnx network delete compromised-network

# 3. Preserve evidence
sudo cp -r /var/log/joblet/ /var/incident-evidence/$(date +%Y%m%d-%H%M%S)

# 4. Reset certificates
sudo /usr/local/bin/certs_gen_embedded.sh
sudo systemctl restart joblet

# 5. Audit all access
sudo grep "auth_success" /var/log/joblet/audit.log | tail -1000