Secret Management in Shell-Less Containers

# Create secret from literal valueskubectl create secret generic app-secrets \  --from-literal=db-password=MyP@ssw0rd123 \  --from-literal=api-key=sk-1234567890abcdef \  --namespace=production # Create secret from fileskubectl create secret generic tls-certs \  --from-file=tls.crt=./certs/server.crt \  --from-file=tls.key=./certs/server.key \  --namespace=production # Create Docker registry secret for image pullskubectl create secret docker-registry gcr-secret \  --docker-server=gcr.io \  --docker-username=_json_key \  --docker-password="$(cat ~/gcr-key.json)" \  --namespace=production

apiVersion: v1kind: Podmetadata:  name: app-with-env-secrets  namespace: productionspec:  containers:  - name: app    image: gcr.io/cleanstart/app:1.0.0    env:    # Simple environment variable    - name: DB_PASSWORD      valueFrom:        secretKeyRef:          name: app-secrets          key: db-password    # Prefixed set of secrets    - name: API_KEY      valueFrom:        secretKeyRef:          name: app-secrets          key: api-key    # Read entire secret as JSON    - name: DATABASE_CONFIG      valueFrom:        secretKeyRef:          name: db-config-json          key: config

apiVersion: v1kind: Podmetadata:  name: app-with-file-secrets  namespace: productionspec:  containers:  - name: app    image: gcr.io/cleanstart/app:1.0.0    volumeMounts:    - name: db-secret      mountPath: /run/secrets/db      readOnly: true    - name: tls-secret      mountPath: /run/secrets/tls      readOnly: true  volumes:  - name: db-secret    secret:      secretName: app-secrets      defaultMode: 0400  # Read-only for owner      items:      - key: db-password        path: password.txt      - key: db-username        path: username.txt  - name: tls-secret    secret:      secretName: tls-certs      defaultMode: 0400      items:      - key: tls.crt        path: server.crt      - key: tls.key        path: server.key

# Add Vault Helm repohelm repo add hashicorp https://helm.releases.hashicorp.comhelm repo update # Install Vault Agent Injectorhelm install vault-injector hashicorp/vault \  --namespace vault \  --create-namespace \  --values - <<EOFinjector:  enabled: trueserver:  enabled: falseglobal:  tlsDisable: falseEOF

apiVersion: apps/v1kind: Deploymentmetadata:  name: app-with-vault  namespace: productionspec:  replicas: 3  selector:    matchLabels:      app: app  template:    metadata:      labels:        app: app      annotations:        vault.hashicorp.com/agent-inject: "true"        vault.hashicorp.com/agent-inject-token: "true"        vault.hashicorp.com/agent-cache-enable: "true"        # Inject database secret        vault.hashicorp.com/agent-inject-secret-db: "secret/data/app/database"        vault.hashicorp.com/agent-inject-template-db: |          {{- with secret "secret/data/app/database" -}}          username={{ .Data.data.username }}          password={{ .Data.data.password }}          host={{ .Data.data.host }}          {{- end }}        vault.hashicorp.com/agent-inject-file-db: "db.conf"        # Inject TLS certificate        vault.hashicorp.com/agent-inject-secret-tls: "secret/data/app/tls"        vault.hashicorp.com/agent-inject-template-tls-cert: |          {{- with secret "secret/data/app/tls" -}}          {{- .Data.data.certificate -}}          {{- end }}        vault.hashicorp.com/agent-inject-file-tls-cert: "server.crt"        vault.hashicorp.com/agent-inject-template-tls-key: |          {{- with secret "secret/data/app/tls" -}}          {{- .Data.data.private_key -}}          {{- end }}        vault.hashicorp.com/agent-inject-file-tls-key: "server.key"        vault.hashicorp.com/role: "app-role"    spec:      serviceAccountName: app-sa      containers:      - name: app        image: gcr.io/cleanstart/app:1.0.0        volumeMounts:        - name: vault-tokens          mountPath: /vault/secrets          readOnly: true        env:        - name: VAULT_ADDR          value: "https://vault.vault.svc.cluster.local:8200"      volumes:      - name: vault-tokens        emptyDir:          medium: Memory

# Enable AppRole auth in Vaultvault auth enable approle # Create policyvault policy write app - <<EOFpath "secret/data/app/*" {  capabilities = ["read"]}path "auth/token/renew-self" {  capabilities = ["update"]}EOF # Create AppRolevault write auth/approle/role/app-role \  token_ttl=1h \  token_max_ttl=24h \  policies="app" # Get role-id and secret-idvault read auth/approle/role/app-role/role-idvault write -f auth/approle/role/app-role/secret-id

helm repo add external-secrets https://charts.external-secrets.iohelm repo update helm install external-secrets \  external-secrets/external-secrets \  --namespace external-secrets-system \  --create-namespace

apiVersion: external-secrets.io/v1beta1kind: SecretStoremetadata:  name: aws-secret-store  namespace: productionspec:  provider:    aws:      service: SecretsManager      region: us-west-2      auth:        jwt:          serviceAccountRef:            name: external-secrets-sa ---apiVersion: external-secrets.io/v1beta1kind: ExternalSecretmetadata:  name: app-secrets  namespace: productionspec:  refreshInterval: 1h  secretStoreRef:    name: aws-secret-store    kind: SecretStore  target:    name: app-secrets  # Creates Kubernetes Secret    creationPolicy: Owner  data:  - secretKey: db-password    remoteRef:      key: production/app/db-password  - secretKey: api-key    remoteRef:      key: production/app/api-key  - secretKey: tls.crt    remoteRef:      key: production/app/tls-cert

# Install Sealed Secrets controllerkubectl apply -f https://github.com/bitnami-labs/sealed-secrets/releases/download/v0.24.0/controller.yaml # Install kubeseal CLIbrew install kubeseal  # or download from releases

# Create a secretkubectl create secret generic db-secret \  --from-literal=password=MyP@ss123 \  --dry-run=client \  --output=yaml > db-secret.yaml # Seal it (encrypt for your cluster)kubeseal --format yaml < db-secret.yaml > db-secret-sealed.yaml # Commit sealed version to Gitgit add db-secret-sealed.yamlgit commit -m "Add sealed database secret" # Apply to cluster (controller decrypts it)kubectl apply -f db-secret-sealed.yaml

#!/bin/bash# rotate-db-password.sh — zero-downtime password rotation set -e DB_USER="app_user"OLD_PASSWORD=$(kubectl get secret db-creds -o jsonpath='{.data.password}' | base64 -d)NEW_PASSWORD=$(openssl rand -base64 32) # 1. Create new secret versionkubectl create secret generic db-creds-new \  --from-literal=password=$NEW_PASSWORD \  --from-literal=username=$DB_USER \  --dry-run=client \  --output=yaml | kubectl apply -f - # 2. Update PostgreSQL password (in-cluster)PGPASSWORD=$OLD_PASSWORD psql \  -h postgres.default.svc.cluster.local \  -U $DB_USER \  -c "ALTER USER $DB_USER WITH PASSWORD '$NEW_PASSWORD';" # 3. Patch Deployment to use new secretkubectl set env deployment/app \  DB_PASSWORD=$NEW_PASSWORD # 4. Monitor rolloutkubectl rollout status deployment/app # 5. Clean up old secretkubectl delete secret db-creds-old echo "Password rotation complete"

#!/bin/bash# rotate-tls-cert.sh set -e # 1. Generate new certificate and keyopenssl req -x509 -newkey rsa:4096 -nodes \  -out server.crt -keyout server.key -days 365 \  -subj "/CN=app.example.com" # 2. Create new TLS secretkubectl create secret tls app-tls-new \  --cert=server.crt \  --key=server.key \  --dry-run=client \  --output=yaml | kubectl apply -f - # 3. Patch Deployment volumekubectl patch deployment app --type='json' -p='[  {    "op": "replace",    "path": "/spec/template/spec/volumes/0/secret/secretName",    "value": "app-tls-new"  }]' # 4. Rolloutkubectl rollout status deployment/app # 5. Update Ingress if neededkubectl patch ingress app-ingress --type merge -p '{  "spec": {    "tls": [{      "hosts": ["app.example.com"],      "secretName": "app-tls-new"    }]  }}' echo "TLS certificate rotated"

apiVersion: v1kind: Secretmetadata:  name: postgres-creds  namespace: productiontype: OpaquestringData:  username: app_user  password: SecureP@ss2024  host: postgres.production.svc.cluster.local  port: "5432"  dbname: app_db ---apiVersion: v1kind: Secretmetadata:  name: postgres-tls  namespace: productiontype: kubernetes.io/tlsdata:  tls.crt: <base64-encoded-cert>  tls.key: <base64-encoded-key>  ca.crt: <base64-encoded-ca> ---apiVersion: v1kind: Podmetadata:  name: app-with-postgresspec:  containers:  - name: app    image: gcr.io/cleanstart/app:1.0.0    volumeMounts:    - name: pg-creds      mountPath: /run/secrets/postgres      readOnly: true    - name: pg-tls      mountPath: /run/secrets/postgres/tls      readOnly: true    env:    - name: DATABASE_URL      value: "postgresql://app_user:$(DB_PASSWORD)@postgres.production.svc.cluster.local:5432/app_db?sslmode=require"    - name: DB_PASSWORD      valueFrom:        secretKeyRef:          name: postgres-creds          key: password  volumes:  - name: pg-creds    secret:      secretName: postgres-creds      defaultMode: 0400  - name: pg-tls    secret:      secretName: postgres-tls      defaultMode: 0400

apiVersion: v1kind: Secretmetadata:  name: redis-auth  namespace: productionstringData:  password: RedisSecure@2024  username: default ---apiVersion: v1kind: Podmetadata:  name: app-with-redisspec:  containers:  - name: app    image: gcr.io/cleanstart/app:1.0.0    env:    - name: REDIS_URL      value: "redis://default:$(REDIS_PASSWORD)@redis.production.svc.cluster.local:6379"    - name: REDIS_PASSWORD      valueFrom:        secretKeyRef:          name: redis-auth          key: password

apiVersion: v1kind: Secretmetadata:  name: kafka-sasl  namespace: productionstringData:  username: app_user  password: KafkaSecure@2024  jaas.conf: |    org.apache.kafka.common.security.scram.ScramLoginModule required \      username="app_user" \      password="KafkaSecure@2024"; ---apiVersion: v1kind: Podmetadata:  name: app-with-kafkaspec:  containers:  - name: app    image: gcr.io/cleanstart/app:1.0.0    volumeMounts:    - name: kafka-sasl      mountPath: /run/secrets/kafka      readOnly: true    env:    - name: KAFKA_BOOTSTRAP_SERVERS      value: "kafka-0.kafka.production.svc.cluster.local:9092"    - name: KAFKA_SECURITY_PROTOCOL      value: "SASL_SSL"    - name: KAFKA_SASL_MECHANISM      value: "SCRAM-SHA-512"  volumes:  - name: kafka-sasl    secret:      secretName: kafka-sasl      defaultMode: 0400      items:      - key: jaas.conf        path: jaas.conf

apiVersion: v1kind: Secretmetadata:  name: nginx-tls  namespace: productiontype: kubernetes.io/tlsdata:  tls.crt: <base64-encoded-cert>  tls.key: <base64-encoded-key> ---apiVersion: networking.k8s.io/v1kind: Ingressmetadata:  name: app-ingress  namespace: productionspec:  tls:  - hosts:    - app.example.com    secretName: nginx-tls  rules:  - host: app.example.com    http:      paths:      - path: /        pathType: Prefix        backend:          service:            name: app            port:              number: 8080

apiVersion: v1kind: Podmetadata:  name: app-init-examplespec:  containers:  - name: app    image: gcr.io/cleanstart/app:1.0.0    volumeMounts:    - name: secrets      mountPath: /run/secrets      readOnly: true    # cleanimg-init reads /run/secrets files automatically    # and passes them to the application process  volumes:  - name: secrets    secret:      secretName: app-secrets      defaultMode: 0400