AI Agent NATS Client Certificate Rotation

opts := []nats.Option{
  nats.MaxReconnects(-1),
  nats.ReconnectWait(2 * time.Second),
}

if strings.HasPrefix(url, "tls://") {
  tlsConfig, err := natsTLSConfigFromEnv()
  if err != nil {
    return nil, fmt.Errorf("nats tls config: %w", err)
  }
  if tlsConfig != nil {
    opts = append(opts, nats.Secure(tlsConfig))
  }
}

nc, err := nats.Connect(url, opts...)

func natsTLSConfigFromEnv() (*tls.Config, error) {
  certPath := strings.TrimSpace(os.Getenv("NATS_TLS_CERT"))
  keyPath := strings.TrimSpace(os.Getenv("NATS_TLS_KEY"))

  cfg := &tls.Config{MinVersion: tls.VersionTLS12}

  if certPath != "" || keyPath != "" {
    if certPath == "" || keyPath == "" {
      return nil, fmt.Errorf("nats tls cert/key must be set together")
    }
    cert, err := tls.LoadX509KeyPair(certPath, keyPath)
    if err != nil {
      return nil, fmt.Errorf("nats tls keypair: %w", err)
    }
    cfg.Certificates = []tls.Certificate{cert}
  }

  return cfg, nil
}

# Rotation budget inputs
RECONNECT_WAIT_SEC=2
TLS_HANDSHAKE_TIMEOUT_SEC=2
PODS=24
MAX_UNAVAILABLE=3

# Worst-case reconnect cycle per pod (simple model)
CYCLE_SEC=$((RECONNECT_WAIT_SEC + TLS_HANDSHAKE_TIMEOUT_SEC))
# Estimated per-wave exposure window
WAVE_WINDOW_SEC=$((CYCLE_SEC * 2))

echo "cycle=${CYCLE_SEC}s wave_window=${WAVE_WINDOW_SEC}s"

# Rollout in small waves
kubectl -n cordum rollout restart deploy/cordum-scheduler
kubectl -n cordum rollout status deploy/cordum-scheduler --timeout=10m

# Validate reconnect and auth errors between waves
kubectl -n cordum logs deploy/cordum-scheduler | rg "disconnected from nats|reconnected to nats|tls|certificate|auth"

// Optional design for no-restart cert rotation.
// Keep current static approach unless you can validate this behavior in staging.

type TLSMaterial interface {
  Cert() (tls.Certificate, error)
  RootCAs() (*x509.CertPool, error)
}

func NatsOptionsFromMaterial(m TLSMaterial) []nats.Option {
  return []nats.Option{
    nats.ClientTLSConfig(
      func() (tls.Certificate, error) { return m.Cert() },
      func() (*x509.CertPool, error) { return m.RootCAs() },
    ),
  }
}

// Rotation note:
// replace m's backing cert/CA atomically,
// then force a reconnect test to verify the new material is used.