fix(certificatee): persist validated haproxy certificates

README.md

@@ -16,7 +16,7 @@ A daemon that synchronizes certificates from Vault to HAProxy using the HAProxy
 - The certificate is expiring within the configured threshold (default: 30 days)
 - The certificate serial number differs from the one stored in Vault
 
-`certificatee` requires HAProxy Data Plane API v3. It reads expiry and serial metadata from HAProxy's live runtime certificate state and uses Vault only as the source for replacement certificate payloads.
+`certificatee` requires HAProxy Data Plane API v3. It reads expiry and serial metadata from HAProxy's live runtime certificate state and uses Vault only as the source for replacement certificate payloads. Replacement certificates are written to HAProxy storage with `skip_reload=true`, then written to HAProxy runtime so updates both survive future reloads and take effect immediately.
 
 ## Configuration
 
@@ -61,7 +61,7 @@ Certificatee uses the HAProxy Data Plane API to update certificates at runtime w
 - **Basic authentication**: Authenticate using username/password credentials
 - **Automatic retries**: Connections are retried with exponential backoff (default: 3 retries, 1-30s delays)
 - **Graceful degradation**: If one HAProxy instance is unreachable, the tool continues updating reachable instances
-- **REST API**: Certificates are managed via the HAProxy Data Plane API v3 `/v3/services/haproxy/runtime/ssl_certs` endpoints
+- **REST API**: Certificates are persisted via the HAProxy Data Plane API v3 `/v3/services/haproxy/storage/ssl_certificates` endpoints and activated via `/v3/services/haproxy/runtime/ssl_certs`
 - **Live certificate state**: Expiry checks use HAProxy's currently loaded certificate metadata, not Vault certificate expiry
 
 ### HAProxy Data Plane API Configuration

cmd/certificatee/main.go

@@ -150,21 +150,22 @@ func processHAProxyEndpoint(logger *logrus.Logger, cfg config.Config, vaultClien
 		// Use HAProxy's live certificate metadata for expiry. Vault provides the
 		// replacement payload and serial used to detect a newer issued certificate.
 		shouldUpdate, reason, isExpiring, err := shouldUpdateCertificate(domain, vaultClient, haproxyCert, cfg.Certificatee.RenewBeforeDays)
+
+		// Track expiring certificates even when Vault replacement material is invalid.
+		if isExpiring {
+			expiringCount++
+		}
+
 		if err != nil {
 			errs = append(errs, err)
 			logger.Errorf("[%s] %v", endpoint, err)
 			continue
 		}
 
-		// Track expiring certificates
-		if isExpiring {
-			expiringCount++
-		}
-
 		if shouldUpdate {
 			logger.Infof("[%s] Certificate %s needs update: %s", endpoint, displayName, reason)
 
-			if err := updateCertificate(apiName, domain, vaultClient, haproxyClient); err != nil {
+			if err := updateCertificate(apiName, domain, vaultClient, haproxyClient, haproxyCert); err != nil {
 				errs = append(errs, err)
 				logger.Errorf("[%s] %v", endpoint, err)
 				certmetrics.CertificatesUpdateFailures.WithLabelValues(endpoint, domain).Inc()
@@ -195,17 +196,21 @@ func setDataPlaneAPIVersion(endpoint, version string) {
 
 func shouldUpdateCertificate(domain string, vaultClient *vault.VaultClient, haproxyCert *haproxy.CertificateDetail, renewBeforeDays int) (shouldUpdate bool, reason string, isExpiring bool, err error) {
 	shouldUpdate, reason, isExpiring, err = shouldUpdateForLiveExpiry(domain, haproxyCert, renewBeforeDays)
-	if err != nil || shouldUpdate {
+	if err != nil {
 		return shouldUpdate, reason, isExpiring, err
 	}
 
 	vaultCert, err := certificate.GetCertificate(domain, vaultClient)
 	if err != nil {
-		return false, "", false, fmt.Errorf("failed to get certificate %s from vault: %w", domain, err)
+		return false, "", isExpiring, fmt.Errorf("failed to get certificate %s from vault: %w", domain, err)
 	}
 
-	if vaultCert == nil {
-		return false, "", false, fmt.Errorf("certificate for %s does not exist in vault", domain)
+	if err := validateVaultCertificateForUpdate(domain, vaultCert, haproxyCert); err != nil {
+		return false, "", isExpiring, err
+	}
+
+	if shouldUpdate {
+		return true, reason, true, nil
 	}
 
 	shouldUpdate, reason = shouldUpdateForSerialMismatch(vaultCert, haproxyCert)
@@ -240,22 +245,67 @@ func shouldUpdateForSerialMismatch(vaultCert *x509.Certificate, haproxyCert *hap
 	return false, ""
 }
 
-func updateCertificate(certPath, domain string, vaultClient *vault.VaultClient, haproxyClient *haproxy.Client) error {
+func validateVaultCertificateForUpdate(domain string, vaultCert *x509.Certificate, haproxyCert *haproxy.CertificateDetail) error {
+	return validateVaultCertificateForUpdateAt(domain, vaultCert, haproxyCert, time.Now())
+}
+
+func validateVaultCertificateForUpdateAt(domain string, vaultCert *x509.Certificate, haproxyCert *haproxy.CertificateDetail, now time.Time) error {
+	if vaultCert == nil {
+		return fmt.Errorf("certificate for %s does not exist in vault", domain)
+	}
+
+	if vaultCert.IsCA {
+		return fmt.Errorf("refusing to update %s: Vault certificate bundle starts with a CA certificate", domain)
+	}
+
+	if certificate.IsExpired(vaultCert, now) {
+		return fmt.Errorf("refusing to update %s: Vault certificate expired on %s", domain, vaultCert.NotAfter.Format(time.RFC3339))
+	}
+
+	if haproxyCert != nil && !haproxyCert.NotAfter.IsZero() && vaultCert.NotAfter.Before(haproxyCert.NotAfter) {
+		return fmt.Errorf("refusing to update %s: Vault certificate expires on %s before live HAProxy certificate expires on %s", domain, vaultCert.NotAfter.Format(time.RFC3339), haproxyCert.NotAfter.Format(time.RFC3339))
+	}
+
+	return nil
+}
+
+func parseVaultLeafCertificate(secrets map[string]any) (*x509.Certificate, error) {
+	certPEM, ok := secrets["certificate"].(string)
+	if !ok || certPEM == "" {
+		return nil, fmt.Errorf("certificate not found in vault secrets")
+	}
+
+	return certificate.ParsePEMCertificate(certPEM)
+}
+
+func updateCertificate(certPath, domain string, vaultClient *vault.VaultClient, haproxyClient *haproxy.Client, haproxyCert *haproxy.CertificateDetail) error {
 	// Read certificate data from Vault
 	certificateSecrets, err := vaultClient.KVRead(certificate.VaultCertLocation(domain))
 	if err != nil {
 		return fmt.Errorf("failed to read certificate data from vault for %s: %w", domain, err)
 	}
 
+	vaultCert, err := parseVaultLeafCertificate(certificateSecrets)
+	if err != nil {
+		return fmt.Errorf("failed to parse Vault certificate for %s: %w", domain, err)
+	}
+	if err := validateVaultCertificateForUpdate(domain, vaultCert, haproxyCert); err != nil {
+		return err
+	}
+
 	// Build PEM bundle (certificate + private key)
 	pemData, err := buildPEMBundle(certificateSecrets)
 	if err != nil {
 		return fmt.Errorf("failed to build PEM bundle for %s: %w", domain, err)
 	}
 
-	// Update certificate in HAProxy
+	storageCertName := haproxy.StorageCertificateName(certPath)
+	if err := haproxyClient.EnsureStorageCertificate(storageCertName, pemData); err != nil {
+		return fmt.Errorf("failed to persist certificate %s in HAProxy storage: %w", storageCertName, err)
+	}
+
 	if err := haproxyClient.UpdateCertificate(certPath, pemData); err != nil {
-		return fmt.Errorf("failed to update certificate %s in HAProxy: %w", certPath, err)
+		return fmt.Errorf("failed to update certificate %s in HAProxy runtime: %w", certPath, err)
 	}
 
 	return nil

cmd/certificatee/main_test.go

@@ -63,6 +63,55 @@ func TestShouldUpdateForLiveExpiry(t *testing.T) {
 	})
 }
 
+func TestValidateVaultCertificateForUpdate(t *testing.T) {
+	now := time.Date(2026, time.July, 3, 12, 0, 0, 0, time.UTC)
+
+	t.Run("rejects missing vault certificate", func(t *testing.T) {
+		err := validateVaultCertificateForUpdateAt("example.com", nil, &haproxy.CertificateDetail{}, now)
+		if err == nil || !strings.Contains(err.Error(), "does not exist in vault") {
+			t.Fatalf("expected missing vault certificate error, got %v", err)
+		}
+	})
+
+	t.Run("rejects expired vault certificate", func(t *testing.T) {
+		vaultCert := &x509.Certificate{
+			NotAfter: now.Add(-time.Minute),
+		}
+
+		err := validateVaultCertificateForUpdateAt("example.com", vaultCert, &haproxy.CertificateDetail{}, now)
+		if err == nil || !strings.Contains(err.Error(), "Vault certificate expired") {
+			t.Fatalf("expected expired vault certificate error, got %v", err)
+		}
+	})
+
+	t.Run("rejects certificate expiry downgrade", func(t *testing.T) {
+		vaultCert := &x509.Certificate{
+			NotAfter: now.AddDate(0, 0, 30),
+		}
+		liveCert := &haproxy.CertificateDetail{
+			NotAfter: now.AddDate(0, 0, 60),
+		}
+
+		err := validateVaultCertificateForUpdateAt("example.com", vaultCert, liveCert, now)
+		if err == nil || !strings.Contains(err.Error(), "before live HAProxy certificate") {
+			t.Fatalf("expected downgrade rejection, got %v", err)
+		}
+	})
+
+	t.Run("accepts valid non-downgrade vault certificate", func(t *testing.T) {
+		vaultCert := &x509.Certificate{
+			NotAfter: now.AddDate(0, 0, 90),
+		}
+		liveCert := &haproxy.CertificateDetail{
+			NotAfter: now.AddDate(0, 0, 60),
+		}
+
+		if err := validateVaultCertificateForUpdateAt("example.com", vaultCert, liveCert, now); err != nil {
+			t.Fatalf("validateVaultCertificateForUpdateAt() error = %v, want nil", err)
+		}
+	})
+}
+
 func TestShouldUpdateForSerialMismatch(t *testing.T) {
 	vaultCert := &x509.Certificate{SerialNumber: big.NewInt(0x1f5202e0)}
 

cmd/certificator/main.go

@@ -2,9 +2,14 @@ package main
 
 import (
 	"context"
+	"crypto/x509"
+	"errors"
+	"fmt"
 	"strings"
+	"time"
 
 	legoLog "github.com/go-acme/lego/v4/log"
+	"github.com/sirupsen/logrus"
 	"github.com/sourcegraph/conc/pool"
 	"github.com/vinted/certificator/pkg/acme"
 	"github.com/vinted/certificator/pkg/certificate"
@@ -78,6 +83,9 @@ func main() {
 				cfg.DNSAddress, cfg.Acme.DNSChallengeProvider, cfg.Acme.DNSPropagationRequirement); err != nil {
 				certmetrics.CertificatesRenewalFailures.WithLabelValues(mainDomain).Inc()
 				certmetrics.CertificatesChecked.WithLabelValues(mainDomain, "failure").Inc()
+				if cleanupErr := deleteExpiredVaultCertificateAfterRenewalFailure(mainDomain, cert, vaultClient, logger); cleanupErr != nil {
+					return errors.Join(err, cleanupErr)
+				}
 				return err
 			}
 			certmetrics.CertificatesRenewed.WithLabelValues(mainDomain).Inc()
@@ -92,3 +100,16 @@ func main() {
 		logger.Fatal(err)
 	}
 }
+
+func deleteExpiredVaultCertificateAfterRenewalFailure(mainDomain string, cert *x509.Certificate, vaultClient *vault.VaultClient, logger *logrus.Logger) error {
+	if !certificate.IsExpired(cert, time.Now()) {
+		return nil
+	}
+
+	logger.Warnf("renewal failed for %s and existing Vault certificate expired on %s; deleting expired Vault certificate", mainDomain, cert.NotAfter.Format(time.RFC3339))
+	if err := certificate.DeleteCertificate(mainDomain, vaultClient); err != nil {
+		return fmt.Errorf("failed deleting expired Vault certificate for %s: %w", mainDomain, err)
+	}
+
+	return nil
+}

pkg/certificate/certificate.go

@@ -38,12 +38,12 @@ func ObtainCertificate(client *lego.Client, vault *vault.VaultClient, domains []
 		Domains: domains,
 		Bundle:  true,
 	}
-	certificate, err := client.Certificate.Obtain(request)
+	certResource, err := client.Certificate.Obtain(request)
 	if err != nil {
 		return err
 	}
 
-	return storeCertificateInVault(domains[0], certificate, vault)
+	return storeCertificateInVault(domains[0], certResource, vault)
 }
 
 // GetCertificate reads certificate from Vault KV store and parses it
@@ -53,16 +53,35 @@ func GetCertificate(domain string, vault *vault.VaultClient) (*x509.Certificate,
 		return nil, err
 	}
 	if cert, ok := secrets["certificate"].(string); ok {
-		parsedCert, err := certcrypto.ParsePEMBundle([]byte(cert))
-		if err != nil {
-			return nil, err
-		}
-		return parsedCert[0], nil
+		return ParsePEMCertificate(cert)
 	}
 
 	return nil, nil
 }
 
+// ParsePEMCertificate parses a PEM bundle and returns the first leaf certificate.
+func ParsePEMCertificate(certPEM string) (*x509.Certificate, error) {
+	parsedCerts, err := certcrypto.ParsePEMBundle([]byte(certPEM))
+	if err != nil {
+		return nil, err
+	}
+	if len(parsedCerts) == 0 {
+		return nil, fmt.Errorf("certificate bundle is empty")
+	}
+
+	return parsedCerts[0], nil
+}
+
+// DeleteCertificate removes certificate data from Vault KV storage.
+func DeleteCertificate(domain string, vault *vault.VaultClient) error {
+	return vault.KVDelete(VaultCertLocation(domain))
+}
+
+// IsExpired reports whether a certificate is already expired at now.
+func IsExpired(certificate *x509.Certificate, now time.Time) bool {
+	return certificate != nil && !certificate.NotAfter.After(now)
+}
+
 // NeedsReissuing checks if certificate domains and required domains match
 // and if certificate expiration date is earlier than configured in config.Cfg.RenewBeforeDays
 func NeedsReissuing(certificate *x509.Certificate, domains []string, days int, logger *logrus.Logger) (bool, error) {