HashiCorp Vault — Project: Production Vault Cluster

Build a production-grade 3-node Vault cluster with Raft storage, auto-unseal, TLS, monitoring, and automated backups.

---

Architecture

                    ┌─────────────┐
                    │Load Balancer│
                    │  (HAProxy)  │
                    └──────┬──────┘
                           │
          ┌────────────────┼────────────────┐
          │                │                │
   ┌──────▼──────┐  ┌─────▼───────┐  ┌─────▼───────┐
   │  vault-1    │  │  vault-2    │  │  vault-3    │
   │  (leader)   │◄─│  (standby)  │◄─│  (standby)  │
   │  Raft       │──│  Raft       │──│  Raft       │
   └─────────────┘  └─────────────┘  └─────────────┘
          │
   ┌──────▼──────┐
   │  AWS KMS    │  (auto-unseal)
   └─────────────┘

---

Step 1 — TLS Certificates

# Generate CA (or use Vault's own PKI)
openssl genrsa -out ca-key.pem 4096
openssl req -new -x509 -key ca-key.pem -sha256 \
    -subj "/CN=Vault CA" -days 3650 -out ca.pem

# Generate server certificate for each node
for NODE in vault-1 vault-2 vault-3; do
  openssl genrsa -out ${NODE}-key.pem 2048

  cat > ${NODE}.cnf <<EOF
[req]
default_bits = 2048
prompt = no
distinguished_name = dn
req_extensions = v3_req

[dn]
CN = ${NODE}.vault.internal

[v3_req]
subjectAltName = @alt_names

[alt_names]
DNS.1 = ${NODE}.vault.internal
DNS.2 = vault.example.com
DNS.3 = localhost
IP.1 = 127.0.0.1
EOF

  openssl req -new -key ${NODE}-key.pem \
      -config ${NODE}.cnf -out ${NODE}.csr

  openssl x509 -req -in ${NODE}.csr \
      -CA ca.pem -CAkey ca-key.pem -CAcreateserial \
      -days 365 -sha256 \
      -extensions v3_req -extfile ${NODE}.cnf \
      -out ${NODE}.pem
done

---

Step 2 — Vault Configuration

# /opt/vault/config/vault.hcl (vault-1)
cluster_name = "vault-prod"
ui           = true

listener "tcp" {
  address            = "0.0.0.0:8200"
  cluster_address    = "0.0.0.0:8201"
  tls_cert_file      = "/opt/vault/tls/vault-1.pem"
  tls_key_file       = "/opt/vault/tls/vault-1-key.pem"
  tls_client_ca_file = "/opt/vault/tls/ca.pem"
  tls_min_version    = "tls12"
}

storage "raft" {
  path    = "/opt/vault/data"
  node_id = "vault-1"

  retry_join {
    leader_api_addr         = "https://vault-2.vault.internal:8200"
    leader_ca_cert_file     = "/opt/vault/tls/ca.pem"
    leader_client_cert_file = "/opt/vault/tls/vault-1.pem"
    leader_client_key_file  = "/opt/vault/tls/vault-1-key.pem"
  }
  retry_join {
    leader_api_addr         = "https://vault-3.vault.internal:8200"
    leader_ca_cert_file     = "/opt/vault/tls/ca.pem"
    leader_client_cert_file = "/opt/vault/tls/vault-1.pem"
    leader_client_key_file  = "/opt/vault/tls/vault-1-key.pem"
  }
}

seal "awskms" {
  region     = "us-east-1"
  kms_key_id = "arn:aws:kms:us-east-1:123456789:key/abc-def-ghi"
}

api_addr     = "https://vault-1.vault.internal:8200"
cluster_addr = "https://vault-1.vault.internal:8201"

telemetry {
  prometheus_retention_time = "24h"
  disable_hostname          = true
}

disable_mlock   = false
default_lease_ttl = "1h"
max_lease_ttl     = "24h"

---

Step 3 — Systemd Service

# /etc/systemd/system/vault.service
[Unit]
Description=HashiCorp Vault
Documentation=https://developer.hashicorp.com/vault
Requires=network-online.target
After=network-online.target
ConditionFileNotEmpty=/opt/vault/config/vault.hcl

[Service]
User=vault
Group=vault
ExecStart=/usr/local/bin/vault server -config=/opt/vault/config/vault.hcl
ExecReload=/bin/kill --signal HUP $MAINPID
KillMode=process
KillSignal=SIGINT
Restart=on-failure
RestartSec=5
LimitNOFILE=65536
LimitMEMLOCK=infinity

[Install]
WantedBy=multi-user.target

# Start on all three nodes
sudo systemctl enable vault
sudo systemctl start vault

---

Step 4 — Initialize and Bootstrap

# Initialize (auto-unseal with KMS — no manual unseal needed)
vault operator init -recovery-shares=5 -recovery-threshold=3

# Nodes auto-unseal using AWS KMS
# Check status
vault status

# Verify Raft peers
vault operator raft list-peers

---

Step 5 — Load Balancer

# haproxy.cfg
frontend vault_frontend
    bind *:8200 ssl crt /etc/haproxy/tls/vault-lb.pem
    default_backend vault_backend

backend vault_backend
    option httpchk GET /v1/sys/health?standbyok=true
    http-check expect status 200,429

    server vault-1 vault-1.vault.internal:8200 check ssl verify required ca-file /etc/haproxy/tls/ca.pem
    server vault-2 vault-2.vault.internal:8200 check ssl verify required ca-file /etc/haproxy/tls/ca.pem
    server vault-3 vault-3.vault.internal:8200 check ssl verify required ca-file /etc/haproxy/tls/ca.pem

---

Step 6 — Monitoring Setup

# prometheus.yml
scrape_configs:
  - job_name: vault
    metrics_path: /v1/sys/metrics
    params:
      format: [prometheus]
    scheme: https
    tls_config:
      ca_file: /etc/prometheus/vault-ca.pem
    bearer_token_file: /etc/prometheus/vault-token
    static_configs:
      - targets:
          - vault-1.vault.internal:8200
          - vault-2.vault.internal:8200
          - vault-3.vault.internal:8200

---

Step 7 — Automated Backups

#!/bin/bash
# /opt/vault/scripts/backup.sh
VAULT_ADDR="https://vault.example.com:8200"
VAULT_TOKEN=$(cat /opt/vault/.backup-token)
BACKUP_DIR="/opt/vault/backups"
S3_BUCKET="s3://company-vault-backups"
TIMESTAMP=$(date +%Y%m%d-%H%M%S)

export VAULT_ADDR VAULT_TOKEN

# Take snapshot
vault operator raft snapshot save \
    "${BACKUP_DIR}/vault-${TIMESTAMP}.snap"

# Upload to S3
aws s3 cp "${BACKUP_DIR}/vault-${TIMESTAMP}.snap" \
    "${S3_BUCKET}/daily/"

# Cleanup local (keep 7 days)
find "$BACKUP_DIR" -name "*.snap" -mtime +7 -delete

---

Post-Deployment Checklist

#	Task	Status
1	TLS on all nodes	☐
2	Auto-unseal with KMS	☐
3	3-node Raft cluster	☐
4	Load balancer configured	☐
5	Root token revoked	☐
6	Audit logging (2 devices)	☐
7	Prometheus scraping	☐
8	Automated backups	☐
9	Alerting rules	☐
10	Recovery keys distributed	☐

---

Next: Practice Projects Exercises