Container Forensics and Evidence Preservation Guide

# Create immutable forensics bucketgsutil mb gs://forensics-evidence-prod # Enable object immutability (cannot delete for 1 year)gsutil retention set 365d gs://forensics-evidence-prod # Enable audit logging (who accessed forensics data)gcloud logging sinks create forensics-audit \  gs://forensics-evidence-prod \  --log-filter='resource.type="gcs_bucket" AND bucket_name="forensics-evidence-prod"'

# Build forensics container with analysis toolscat > Dockerfile.forensics <<EOFFROM registry.cleanstart.com/cleanstart/ubuntu:22.04RUN apt-get update && apt-get install -y \    strace \    ltrace \    strings \    objdump \    hexdump \    tcpdump \    wireshark-commonEOF docker build -f Dockerfile.forensics -t registry.company.com/forensics-toolkit:latest .docker push registry.company.com/forensics-toolkit:latest

apiVersion: v1kind: Podmetadata:  name: forensics-toolkit  namespace: securityspec:  containers:  - name: toolkit    image: registry.company.com/forensics-toolkit:latest    volumeMounts:    - name: suspect-container-mnt      mountPath: /mnt/suspect  # Will mount suspect container's filesystem    - name: evidence-storage      mountPath: /evidence  # For captured evidence    securityContext:      privileged: true  # Required for deep forensics  serviceAccountName: forensics  volumes:  - name: evidence-storage    gcePersistentDisk:      pdName: forensics-evidence-disk      readOnly: false

#!/bin/bashset -e # 1. Alert incident response teamecho "INCIDENT DECLARED: Suspected container compromise"# (Send to PagerDuty, Slack #security, etc.) # 2. Identify suspect containerSUSPECT_POD="auth-service-7f4d5c6"SUSPECT_NS="production"SUSPECT_CONTAINER="auth-app" # 3. Document pod details (before killing)kubectl get pod $SUSPECT_POD -n $SUSPECT_NS -o yaml > pod-metadata.yaml # 4. Capture running processeskubectl exec $SUSPECT_POD -n $SUSPECT_NS -c $SUSPECT_CONTAINER \  -- ps aux > evidence-001-processes.txt # 5. Capture open fileskubectl exec $SUSPECT_POD -n $SUSPECT_NS -c $SUSPECT_CONTAINER \  -- lsof > evidence-002-open-files.txt # 6. Capture network connectionskubectl exec $SUSPECT_POD -n $SUSPECT_NS -c $SUSPECT_CONTAINER \  -- netstat -tupan > evidence-003-network-connections.txt # 7. Capture environment variableskubectl exec $SUSPECT_POD -n $SUSPECT_NS -c $SUSPECT_CONTAINER \  -- env > evidence-004-environment.txt # 8. Capture system logs from containerkubectl logs $SUSPECT_POD -n $SUSPECT_NS -c $SUSPECT_CONTAINER \  --timestamps=true > evidence-005-container-logs.txt # 9. Timestamp all evidencefor file in evidence-*.txt; do  echo "Captured at: $(date -u +%Y-%m-%dT%H:%M:%SZ)" >> $filedone # 10. Calculate checksums (for chain of custody)sha256sum evidence-*.txt > MANIFEST.sha256 # 11. Archive evidencetar czf suspect-pod-evidence-$(date +%s).tar.gz evidence-*.txt MANIFEST.sha256 # 12. Upload to immutable forensics storagegsutil cp suspect-pod-evidence-*.tar.gz gs://forensics-evidence-prod/$(date +%Y/%m/%d)/

evidence-001-processes.txt: All processes in container (may show injected malware)evidence-002-open-files.txt: Files/sockets open (may show C2 connections)evidence-003-network-connections.txt: Network state (which hosts was container talking to)evidence-004-environment.txt: Environment variables (may contain secrets, command injection)evidence-005-container-logs.txt: Container stdout/stderr logs (what did app log?)

#!/bin/bash SUSPECT_POD="auth-service-7f4d5c6"SUSPECT_NS="production"SUSPECT_CONTAINER="auth-app" # Option 1: Copy filesystem from running container# (requires container to be interactive)kubectl debug $SUSPECT_POD -n $SUSPECT_NS -c $SUSPECT_CONTAINER \  --image=registry.cleanstart.com/cleanstart/ubuntu:22.04 \  --target=$SUSPECT_CONTAINER \  -- tar czf /tmp/container-fs-dump.tar.gz / # This spawns ephemeral debugging pod with access to suspect container's namespace# Then tars entire filesystem # Option 2: Docker-level snapshot (if on single node)CONTAINER_ID=$(kubectl describe pod $SUSPECT_POD -n $SUSPECT_NS | grep "Container ID" | head -1 | cut -d'/' -f3) docker export $CONTAINER_ID | gzip > container-filesystem-$(date +%s).tar.gz # Option 3: Overlay filesystem diff (only changed files)# (for running container with overlay2 storage driver)CONTAINER_ID=$(docker inspect $SUSPECT_POD | jq -r '.[0].Id')docker diff $CONTAINER_ID > filesystem-changes.txt

#!/bin/bash # Get container PID on hostCONTAINER_ID=$(kubectl get pod auth-service-7f4d5c6 -n production -o jsonpath='{.status.containerStatuses[0].containerID}' | cut -d'/' -f3) # Find main process PIDPID=$(docker inspect $CONTAINER_ID | jq -r '.[0].State.Pid') # Dump memory (requires root)gcore -o memory-dump $PID # This creates memory-dump.[PID] file# Analyze with:# - volatility: detect injected code, rootkits# - radare2: analyze binary code# - strings: search for suspicious strings

#!/bin/bash # 1. List image layersIMAGE="registry.cleanstart.com/cleanstart/python:3.11"docker image inspect $IMAGE --format='{{json .RootFS.Layers}}' | jq # 2. Extract each layermkdir -p layer-analysis docker save $IMAGE | tar -xf - -O | while read line; do  LAYER=$(echo $line | grep -o '"sha256:[^"]*"' | head -1 | tr -d '"')   # Extract layer contents  docker run --rm \    -v /var/lib/docker/overlay2:/overlay2:ro \    alpine:latest \    tar -xzf /overlay2/$LAYER/diffdone # 3. Diff layers against baselinediff -r layer-analysis/python-3.11-base layer-analysis/python-3.11-deployed

#!/bin/bash SUSPECT_IMAGE="registry.cleanstart.com/cleanstart/python:3.11" # 1. Verify image signaturecosign verify --key https://keys.cleanstart.com/prod.pub $SUSPECT_IMAGE # Output should show: Signature valid# If fails: Image was modified after build (forensic evidence of tampering) # 2. Download SBOMcosign download sbom $SUSPECT_IMAGE > evidence-sbom.json # 3. Download provenancecosign download attestation $SUSPECT_IMAGE > evidence-provenance.json # 4. Verify provenance signaturecosign verify-blob \  --key https://keys.cleanstart.com/prod.pub \  --signature evidence-provenance.json.sig \  evidence-provenance.json # If all verifications pass: Image came from CleanStart build system# If any fail: Supply chain compromise or post-build tampering

#!/bin/bash # 1. Download embedded SBOM from imagecosign download sbom registry.cleanstart.com/cleanstart/python:3.11 | jq > image-sbom.json # 2. For each package in SBOM, check if it matches deployed statejq '.components[] | {name, version}' image-sbom.json > expected-packages.txt # 3. Extract actual packages from container filesystemdocker run --rm registry.cleanstart.com/cleanstart/python:3.11 \  sh -c 'pip freeze' > actual-packages.txt # 4. Diffdiff expected-packages.txt actual-packages.txt # If differences found:# - New packages: Container runtime installed additional software (compromise indicator)# - Missing packages: Image layer was modified (tampering)

#!/bin/bash # Extract timestamps from multiple sources echo "=== TIMELINE RECONSTRUCTION ===" > incident-timeline.txt # 1. Image build time (from SBOM)echo "Image Build:" >> incident-timeline.txtcosign download sbom registry.cleanstart.com/cleanstart/python:3.11 | \  jq '.metadata.component.buildTime' >> incident-timeline.txt # 2. Pod creation timeecho "Pod Creation:" >> incident-timeline.txtkubectl get pod auth-service-7f4d5c6 -n production -o jsonpath='{.metadata.creationTimestamp}' >> incident-timeline.txt # 3. Image pull timeecho "Image Pull:" >> incident-timeline.txtkubectl get event -n production --field-selector involvedObject.name=auth-service-7f4d5c6 | grep "Image" >> incident-timeline.txt # 4. Alert timeecho "Falco Alert:" >> incident-timeline.txtgrep "Shell in Distroless" /var/log/falco/alerts.log | head -1 >> incident-timeline.txt # 5. Container stderr/stdoutecho "Container Logs:" >> incident-timeline.txtkubectl logs auth-service-7f4d5c6 -n production --timestamps=true >> incident-timeline.txt

=== TIMELINE RECONSTRUCTION ===Image Build: 2026-03-20T08:30:00Z (2 days ago)Pod Creation: 2026-03-22T10:00:00Z (4 hours ago)Image Pull: 2026-03-22T10:00:15ZFalco Alert: 2026-03-22T14:32:45Z (CRITICAL: Shell in distroless container)Container Logs: [app startup] -> [API calls] -> [unexpected network activity at 14:32:20]

1. Is image signature valid?   Yes -> Likely runtime compromise (go to 4.2a)   No  -> SUPPLY CHAIN COMPROMISE (go to 4.2b) 4.2a: Runtime Compromise  - Check Falco logs: Is there evidence of exploitation?  - Check network connections: Unexpected outbound?  - Check filesystem modifications: Malware written to disk?  - Containment: Kill container, patch vulnerability in app 4.2b: Supply Chain Compromise  - Image tampered with after build  - OR CleanStart signing key was compromised  - Containment: Assume all deployments compromised  - Immediate: Revoke image, rebuild, audit all other images  - Escalate: CISO, Chief Legal Officer

Evidence ID: EV-2026-0322-001Description: Memory dump from auth-service podCollected By: john.smith@company.com (IR Team Lead)Collection Time: 2026-03-22T14:35:00ZStorage Location: gs://forensics-evidence-prod/2026/03/22/memory-dump.binStorage Hash (SHA256): abc123def456...Accessed By: [log all access]Final Status: [sealed for legal proceedings / destroyed after 90 days]

# Enable immutable logging of all evidence accessgcloud logging create-sink forensics-access-log \  logging.googleapis.com/log-sink \  --log-filter='resource.name=~"gs://forensics-evidence-prod/.*"' # Every read/write is logged with actor, time, and intent

#!/bin/bash # Set immutability on evidence files (GCS)gsutil retention set 3650d gs://forensics-evidence-prod/ # Or use WORM (Write-Once-Read-Many) at container levelgsutil set-object-hold gs://forensics-evidence-prod/evidence-001.tar.gz # Verify immutabilitygsutil object-hold get gs://forensics-evidence-prod/evidence-001.tar.gz# Output: Object Hold: ENABLED (cannot be deleted or overwritten)

#!/bin/bash # Parse evidence and send to Splunk/DataDog/ELK # 1. Convert forensics evidence to structured logcat evidence-001-processes.txt | while read line; do  jq -n \    --arg timestamp "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \    --arg source "container-forensics" \    --arg process_line "$line" \    '{timestamp: $timestamp, source: $source, process: $process_line}'done > forensics-structured.jsonl # 2. Send to SIEMsplunk_hec_token="12345-67890-ABCDEF"splunk_endpoint="https://splunk.company.com:8088/services/collector" curl -k $splunk_endpoint \  -H "Authorization: Splunk $splunk_hec_token" \  -d @forensics-structured.jsonl

# Incident Root Cause Analysis: Container Compromise ## Summary- **Incident ID**: INC-2026-0322-001- **Severity**: P1 (Critical)- **Duration**: 4 hours 32 minutes (detected to containment)- **Root Cause**: [CVE-2026-XXXXX in application dependency]- **Impact**: [1 container compromised; no data exfiltration detected] ## Timeline- **2026-03-22 14:00**: Pod created from CleanStart/python:3.11- **2026-03-22 14:32**: Shell spawned in container (exploitation)- **2026-03-22 14:33**: Falco alert fired- **2026-03-22 14:35**: Container killed- **2026-03-22 14:45**: Forensics completed; image verified clean- **2026-03-22 15:30**: Application patch deployed ## Forensic Findings- **Image Signature**: VALID (image came from CleanStart)- **SBOM Verification**: PASSED (no unexpected packages)- **Filesystem**: Modified to include [malware name]- **Compromise Vector**: Exploited [CVE] in application code ## Evidence Preserved- Container memory dump: 2.3 GB (gs://forensics-evidence-prod/.../memory.bin)- Filesystem snapshot: 500 MB (gs://forensics-evidence-prod/.../fs.tar.gz)- Network capture: 150 MB (gs://forensics-evidence-prod/.../network.pcap) ## Remediation1. [Patch application dependency to X.Y.Z]2. [Rebuild image with fixed dependency]3. [Redeploy to all environments]4. [Rotate secrets exposed during incident]5. [Enable Falco detection for similar exploits] ## Prevention- Implement static SBOM scanning in CI/CD (catch vulnerable dependencies)- Enable Falco runtime detection on all production pods- Increase frequency of image rebuilds (weekly -> daily)- Add integration test for known CVE-2026-XXXXX