Regression Testing Strategy for CleanStart Image Upgrades

# Get image digestsOLD_DIGEST=$(docker pull registry.cleanstart.com/py:3.12.1 2>&1 | grep "Digest:" | awk '{print $2}')NEW_DIGEST=$(docker pull registry.cleanstart.com/py:3.12.2 2>&1 | grep "Digest:" | awk '{print $2}') echo "Old: $OLD_DIGEST"echo "New: $NEW_DIGEST"

# Extract SBOM from image (Cosign + SPDX format)cosign blob-url registry.cleanstart.com/py:3.12.1 sbom > old-sbom.jsoncosign blob-url registry.cleanstart.com/py:3.12.2 sbom > new-sbom.json # Compare package listsjq '.components[] | .name + ":" + .version' old-sbom.json | sort > old-packages.txtjq '.components[] | .name + ":" + .version' new-sbom.json | sort > new-packages.txt diff old-packages.txt new-packages.txt

# If you use Grype or Trivygrype registry.cleanstart.com/py:3.12.1 --format json > old-vulns.jsongrype registry.cleanstart.com/py:3.12.2 --format json > new-vulns.json # Compare critical/high findingsjq '.matches[] | select(.vulnerability.severity == "Critical" or .vulnerability.severity == "High")' old-vulns.json | wc -ljq '.matches[] | select(.vulnerability.severity == "Critical" or .vulnerability.severity == "High")' new-vulns.json | wc -l

#!/bin/bashset -e IMAGE=$1CONTAINER_NAME="smoke-test-$(date +%s)" echo "🧪 Starting smoke test for $IMAGE..." # Start containerdocker run -d \  --name "$CONTAINER_NAME" \  --health-cmd='curl -f http://localhost:8080/health || exit 1' \  --health-interval=5s \  --health-timeout=3s \  --health-retries=3 \  -p 8080:8080 \  "$IMAGE" & CONTAINER_ID=$!sleep 2 # Check startup logsif docker logs "$CONTAINER_NAME" | grep -i "error\|segfault\|panic"; then  echo "❌ FAIL: Errors in startup logs"  docker logs "$CONTAINER_NAME"  docker rm -f "$CONTAINER_NAME"  exit 1fi # Wait for health checkfor i in {1..30}; do  if docker inspect "$CONTAINER_NAME" | grep '"Status": "healthy"' > /dev/null; then    echo "✅ Health check passed"    break  fi  if [ $i -eq 30 ]; then    echo "❌ FAIL: Health check timeout"    docker logs "$CONTAINER_NAME"    docker rm -f "$CONTAINER_NAME"    exit 1  fi  sleep 1done # Test basic requestRESPONSE=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:8080/health)if [ "$RESPONSE" != "200" ]; then  echo "❌ FAIL: Health endpoint returned $RESPONSE"  docker logs "$CONTAINER_NAME"  docker rm -f "$CONTAINER_NAME"  exit 1fi echo "✅ Smoke test PASSED"docker rm -f "$CONTAINER_NAME"exit 0

#!/bin/bashset -e NEW_IMAGE=$1TEST_CONTAINER="test-$(date +%s)" # Start container with test-friendly environmentdocker run -d \  --name "$TEST_CONTAINER" \  -e "NODE_ENV=test" \  -e "LOG_LEVEL=debug" \  -p 8080:8080 \  --network test-network \  "$NEW_IMAGE" & sleep 5  # Wait for startup # Run test suitedocker exec "$TEST_CONTAINER" \  /bin/sh -c "cd /app && npm test -- --coverage --reporter=json > /tmp/test-results.json" # Copy resultsdocker cp "$TEST_CONTAINER":/tmp/test-results.json ./test-results-new.json # Parse resultsPASS=$(jq '.numPassedTests' test-results-new.json)FAIL=$(jq '.numFailedTests' test-results-new.json)SKIPPED=$(jq '.numPendingTests' test-results-new.json) echo "Test Results: $PASS passed, $FAIL failed, $SKIPPED skipped" # Cleanupdocker rm -f "$TEST_CONTAINER" if [ "$FAIL" -gt 0 ]; then  echo "❌ FAIL: $FAIL test(s) failed"  jq '.testResults[]' test-results-new.json  exit 1fi echo "✅ Functional regression PASSED"exit 0

#!/bin/bash OLD_IMAGE=$1NEW_IMAGE=$2 echo "📊 Performance baseline comparison: $OLD_IMAGE → $NEW_IMAGE" # Helper function to run benchmarkrun_benchmark() {  local IMAGE=$1  local LABEL=$2  local CONTAINER="perf-test-$(date +%s)"   echo "Starting $LABEL..."  docker run -d \    --name "$CONTAINER" \    -p 8080:8080 \    --cpus=2 \    --memory=1g \    "$IMAGE" &   sleep 10  # Warmup   # Measure startup time  START=$(date +%s%N)  curl -s http://localhost:8080/health > /dev/null  END=$(date +%s%N)  STARTUP_TIME=$(( (END - START) / 1000000 ))  # ms   # Run load test (30 sec @ 100 RPS)  wrk -t4 -c100 -d30s -R100 \    --script /tmp/latency.lua \    http://localhost:8080/ \    > /tmp/perf-${LABEL}.txt   # Collect memory stats  PEAK_MEMORY=$(docker stats --no-stream "$CONTAINER" | tail -1 | awk '{print $4}')   # Cleanup  docker stop "$CONTAINER"  docker rm "$CONTAINER"   echo "$LABEL: startup=${STARTUP_TIME}ms, memory=$PEAK_MEMORY"  return 0} # Run benchmarksrun_benchmark "$OLD_IMAGE" "OLD"OLD_STARTUP=$(grep "startup=" /tmp/perf-OLD.txt | awk -F= '{print $2}' | awk '{print $1}')OLD_MEMORY=$(grep "memory=" /tmp/perf-OLD.txt | awk -F= '{print $2}' | awk '{print $1}') run_benchmark "$NEW_IMAGE" "NEW"NEW_STARTUP=$(grep "startup=" /tmp/perf-NEW.txt | awk -F= '{print $2}' | awk '{print $1}')NEW_MEMORY=$(grep "memory=" /tmp/perf-NEW.txt | awk -F= '{print $2}' | awk '{print $1}') # CompareSTARTUP_DELTA=$(( (NEW_STARTUP - OLD_STARTUP) * 100 / OLD_STARTUP ))MEMORY_DELTA=$(( (NEW_MEMORY - OLD_MEMORY) * 100 / OLD_MEMORY )) echo "📈 Results:"echo "   Startup: ${OLD_STARTUP}ms → ${NEW_STARTUP}ms (${STARTUP_DELTA:+${STARTUP_DELTA}>0?+:}-${STARTUP_DELTA#-}%)"echo "   Memory:  ${OLD_MEMORY} → ${NEW_MEMORY} (${MEMORY_DELTA:+${MEMORY_DELTA}>0?+:}-${MEMORY_DELTA#-}%)" if [ "$STARTUP_DELTA" -gt 10 ] || [ "$MEMORY_DELTA" -gt 10 ]; then  echo "⚠️  WARN: Metrics degraded >10%"  exit 1fi echo "✅ Performance regression PASSED"exit 0

#!/bin/bash IMAGE=$1CONTAINER="security-test-$(date +%s)" echo "🔒 Security hardening verification for $IMAGE" # Start containerdocker run -d \  --name "$CONTAINER" \  "$IMAGE" & sleep 3 # 1. Verify non-root userecho -n "Checking non-root user... "if docker exec "$CONTAINER" id | grep -q "uid=65532"; then  echo "✅"else  echo "❌ FAIL: Not running as UID 65532"  docker rm -f "$CONTAINER"  exit 1fi # 2. Verify no shellecho -n "Checking shell-less mode... "if docker exec "$CONTAINER" test ! -f /bin/sh > /dev/null 2>&1; then  echo "✅"else  echo "⚠️  WARN: Shell found (not required for all images)"fi # 3. Verify read-only root filesystemecho -n "Checking read-only root FS... "if docker run --read-only "$IMAGE" /app/health-check > /dev/null 2>&1; then  echo "✅"else  echo "❌ FAIL: Application requires writable root FS (unexpected)"  docker rm -f "$CONTAINER"  exit 1fi # 4. Verify signatureecho -n "Verifying Cosign signature... "if cosign verify --certificate-identity-regexp='^https://github.com/cleanstart' \   --certificate-oidc-issuer=https://token.actions.githubusercontent.com \   "$IMAGE" > /dev/null 2>&1; then  echo "✅"else  echo "⚠️  WARN: Signature verification failed (check certificate)"fi # 5. Verify SBOM presentecho -n "Verifying SBOM present... "if cosign blob-url "$IMAGE" sbom | jq .components > /dev/null 2>&1; then  echo "✅"else  echo "❌ FAIL: SBOM not found"  docker rm -f "$CONTAINER"  exit 1fi # Cleanupdocker rm -f "$CONTAINER" echo "✅ Security hardening PASSED"exit 0

#!/bin/bash IMAGE=$1DURATION_HOURS=${2:-24}INTERVAL_SECONDS=60 CONTAINER="soak-test-$(date +%s)" echo "🔄 Starting ${DURATION_HOURS}-hour soak test for $IMAGE" # Start containerdocker run -d \  --name "$CONTAINER" \  -e "SOAK_TEST=true" \  --cpus=2 \  --memory=2g \  -p 8080:8080 \  "$IMAGE" & sleep 10 # Monitor loopELAPSED=0MAX_SECONDS=$(( DURATION_HOURS * 3600 ))ITERATION=0 while [ $ELAPSED -lt $MAX_SECONDS ]; do  ITERATION=$(( ITERATION + 1 ))  ELAPSED=$(( ITERATION * INTERVAL_SECONDS ))  HOURS_ELAPSED=$(( ELAPSED / 3600 ))   # Collect metrics  STATS=$(docker stats --no-stream "$CONTAINER" | tail -1)  MEMORY=$(echo "$STATS" | awk '{print $4}')  CPU=$(echo "$STATS" | awk '{print $3}')   # Check application health  HEALTH=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:8080/health)   # Check for memory growth (should stabilize after warmup)  if [ "$ITERATION" -eq 1 ]; then    BASELINE_MEMORY="$MEMORY"    echo "Baseline memory: $BASELINE_MEMORY"  else    # Extract numeric value (e.g., "256MiB" → "256")    BASELINE_MB=$(echo "$BASELINE_MEMORY" | sed 's/[^0-9]*//g')    CURRENT_MB=$(echo "$MEMORY" | sed 's/[^0-9]*//g')    GROWTH=$(( (CURRENT_MB - BASELINE_MB) * 100 / BASELINE_MB ))     # Alert if memory grows >30% (likely leak)    if [ "$GROWTH" -gt 30 ]; then      echo "⚠️  Memory leak suspected: grew ${GROWTH}% (${BASELINE_MB}MB → ${CURRENT_MB}MB)"    fi  fi   # Log progress  printf "[%2dh:%02dm] Health: $HEALTH | Memory: $MEMORY | CPU: $CPU\n" \    $(( HOURS_ELAPSED )) $(( (ELAPSED % 3600) / 60 ))   # Check health endpoint  if [ "$HEALTH" != "200" ]; then    echo "❌ FAIL: Health check returned $HEALTH"    docker logs "$CONTAINER" | tail -20    docker rm -f "$CONTAINER"    exit 1  fi   sleep "$INTERVAL_SECONDS"done # Final diagnosticsecho ""echo "📊 Soak test complete. Final diagnostics:"docker logs "$CONTAINER" | grep -i "error\|warning" | tail -20 # Cleanupdocker rm -f "$CONTAINER" echo "✅ Soak test PASSED"exit 0

name: Image Regression Test on:  workflow_dispatch:    inputs:      old_image:        description: 'Previous image tag (for comparison)'        required: true        default: 'registry.cleanstart.com/py:3.12.1'      new_image:        description: 'New image tag to test'        required: true        default: 'registry.cleanstart.com/py:3.12.2'      soak_hours:        description: 'Soak test duration (0 = skip)'        required: false        default: '0' jobs:  regression-test:    runs-on: ubuntu-latest    timeout-minutes: 120    steps:      - name: Checkout tests        uses: actions/checkout@v4       - name: Layer 1 - Diff Analysis        run: |          ./scripts/layer1-diff-analysis.sh "${{ inputs.new_image }}"       - name: Layer 2 - Smoke Tests        run: |          ./scripts/layer2-smoke-tests.sh "${{ inputs.new_image }}"       - name: Layer 3 - Functional Regression        run: |          npm test -- --coverage        env:          TEST_IMAGE: "${{ inputs.new_image }}"       - name: Layer 4 - Performance Baseline        run: |          ./scripts/layer4-perf-baseline.sh "${{ inputs.old_image }}" "${{ inputs.new_image }}"       - name: Layer 5 - Security Hardening        run: |          ./scripts/layer5-security-check.sh "${{ inputs.new_image }}"       - name: Layer 6 - Soak Test (if requested)        if: ${{ inputs.soak_hours != '0' }}        run: |          ./scripts/layer6-soak-test.sh "${{ inputs.new_image }}" "${{ inputs.soak_hours }}"        timeout-minutes: ${{ inputs.soak_hours * 60 + 30 }}       - name: Upload results        if: always()        uses: actions/upload-artifact@v4        with:          name: regression-results          path: ./test-results/       - name: Comment on PR        if: always()        uses: actions/github-script@v7        with:          script: |            const fs = require('fs');            const results = JSON.parse(fs.readFileSync('./test-results/summary.json'));            github.rest.issues.createComment({              issue_number: context.issue.number,              owner: context.repo.owner,              repo: context.repo.repo,              body: `## Image Regression Test Results\n\n${results.summary}`            });

image: docker:latest stages:  - diff-analysis  - smoke-test  - functional-test  - performance-test  - security-test  - soak-test variables:  OLD_IMAGE: ${OLD_IMAGE:-registry.cleanstart.com/py:3.12.1}  NEW_IMAGE: ${NEW_IMAGE:-registry.cleanstart.com/py:3.12.2}  SOAK_HOURS: ${SOAK_HOURS:-0} layer1-diff:  stage: diff-analysis  script:    - ./scripts/layer1-diff-analysis.sh $NEW_IMAGE  artifacts:    reports:      junit: results/diff-report.xml layer2-smoke:  stage: smoke-test  services:    - docker:dind  script:    - ./scripts/layer2-smoke-tests.sh $NEW_IMAGE  artifacts:    reports:      junit: results/smoke-report.xml layer3-functional:  stage: functional-test  services:    - docker:dind  script:    - docker run $NEW_IMAGE npm test -- --reporter=junit --outputFile=results/functional.xml  artifacts:    reports:      junit: results/functional.xml layer4-perf:  stage: performance-test  services:    - docker:dind  script:    - ./scripts/layer4-perf-baseline.sh $OLD_IMAGE $NEW_IMAGE  artifacts:    paths:      - results/perf-*.json layer5-security:  stage: security-test  script:    - ./scripts/layer5-security-check.sh $NEW_IMAGE  artifacts:    reports:      junit: results/security-report.xml layer6-soak:  stage: soak-test  services:    - docker:dind  script:    - ./scripts/layer6-soak-test.sh $NEW_IMAGE $SOAK_HOURS  artifacts:    paths:      - results/soak-*.json  only:    - schedules  timeout: 72h

# 1. Identify running containers with new imagedocker ps -a | grep "new-image-tag" # 2. Redeploy previous image versionkubectl set image deployment/myapp \  app=registry.cleanstart.com/py:3.12.1@sha256:abc123... # 3. Monitor rolloutkubectl rollout status deployment/myapp --timeout=5m # 4. Verify traffic is routing correctlycurl https://api.example.com/health # 5. Capture logs for diagnosticskubectl logs -l app=myapp --tail=1000 > rollback-logs.txt

# Add custom regression test for this specific issuecat > tests/regression-py312-issue.test.js << 'EOF'describe('Python 3.12 specific regression', () => {  it('should handle X without error', async () => {    // Test that specifically covers the failed scenario  });});EOF # Ensure test runs as part of Layer 3 functional suitegit add tests/regression-py312-issue.test.jsgit commit -m "Add regression test for Python 3.12 issue"