From 7d41463cc4ff88643d110cc06a63a6cd08abe9d6 Mon Sep 17 00:00:00 2001 From: Jona Neef Date: Fri, 31 Jul 2026 14:53:59 +0200 Subject: [PATCH 1/2] feat(installer): derive a data center's vault from the primary's MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A multi-data-center installation shares one PostgreSQL server, so the postgres roles, the token signing keys and everything else that encrypts rows in the shared database has to be identical in every data center's vault. Everything tied to a single cluster — the ingress CA, the cephadm SSH key, the kubeconfig, the Ceph credentials the installer writes back — must not be. DeriveDataCenterVault copies the primary data center's vault and drops exactly the second group, so running EnsureSecrets over the result regenerates those and leaves the shared ones untouched. The Ceph cluster credentials are matched by prefix, because the closed-source installer owns those names. ClearDataCenterScopedConfig resets the config fields that are written alongside a dropped secret, since the Ensure* functions are gated on the vault entry and would otherwise leave a stale value paired with a freshly generated key. No caller yet; this is the pure, GCP-independent part of multi-DC support. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Jona Neef --- internal/installer/secrets/datacenter.go | 87 ++++++++ internal/installer/secrets/datacenter_test.go | 197 ++++++++++++++++++ 2 files changed, 284 insertions(+) create mode 100644 internal/installer/secrets/datacenter.go create mode 100644 internal/installer/secrets/datacenter_test.go diff --git a/internal/installer/secrets/datacenter.go b/internal/installer/secrets/datacenter.go new file mode 100644 index 00000000..133e148e --- /dev/null +++ b/internal/installer/secrets/datacenter.go @@ -0,0 +1,87 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +package secrets + +import ( + "strings" + + "github.com/codesphere-cloud/oms/internal/installer/files" +) + +// DataCenterScopedSecretNames are the secrets that belong to exactly one data center and must +// therefore be regenerated for every additional data center of a multi-DC installation. +// +// Everything not listed here (and not matched by DataCenterScopedSecretPrefixes) is shared: +// the postgres roles because both data centers talk to the same server, and the auth and +// encryption keys because tokens minted and rows written in one data center are consumed in +// the other. +var DataCenterScopedSecretNames = []string{ + // Cluster ingress CA — paired with cluster.certificates.ca.certPem. + files.SecretSelfSignedCaKeyPem, + // cephadm SSH key — paired with ceph.cephAdmSshKey.publicKey. + files.SecretCephSshPrivateKey, + // Written by the installer's kubernetes step; points at one cluster's API server. + files.SecretKubeConfig, + // ACME external account binding — paired with codesphere.certIssuer.acme.eabKeyId. + files.SecretAcmeEabMacKey, + // Only present in recovered vaults; keyed to a single cluster's nix cache. + files.SecretPrivNixSigningKey, + files.SecretPubNixSigningKey, +} + +// DataCenterScopedSecretPrefixes cover the Ceph cluster credentials that the installer's ceph +// step writes back into the vault (cephFsId, cephfsAdmin, csiRbdNode, rgwAdminAccessKey, ...). +// The installer owns those names, so they are matched by prefix rather than enumerated. +var DataCenterScopedSecretPrefixes = []string{"ceph", "csi", "rgw"} + +// IsDataCenterScopedSecret reports whether a vault entry belongs to a single data center. +func IsDataCenterScopedSecret(name string) bool { + for _, scoped := range DataCenterScopedSecretNames { + if name == scoped { + return true + } + } + for _, prefix := range DataCenterScopedSecretPrefixes { + if strings.HasPrefix(name, prefix) { + return true + } + } + return false +} + +// DeriveDataCenterVault returns a copy of the primary data center's vault with all +// data-center-scoped secrets removed. Running EnsureSecrets over the result regenerates exactly +// those, while the shared secrets stay byte-identical across data centers. +func DeriveDataCenterVault(primary *files.InstallVault) *files.InstallVault { + derived := primary.Clone() + if derived == nil { + return nil + } + + kept := make([]files.SecretEntry, 0, len(derived.Secrets)) + for _, entry := range derived.Secrets { + if IsDataCenterScopedSecret(entry.Name) { + continue + } + kept = append(kept, entry) + } + derived.Secrets = kept + + return derived +} + +// ClearDataCenterScopedConfig resets the config fields that are written by the Ensure* functions +// alongside a data-center-scoped vault secret. Those pairs must always be mutated together: the +// Ensure* functions are gated on the vault entry, so a config field left in place would keep a +// stale value paired with a freshly generated key. +func ClearDataCenterScopedConfig(config *files.RootConfig) { + // Paired with selfSignedCaKeyPem, written by EnsureIngressCA. + config.Cluster.Certificates.CA.CertPem = "" + // Paired with cephSshPrivateKey, written by EnsureCephSSHKeys. + config.Ceph.CephAdmSSHKey.PublicKey = "" + // Paired with acmeEabMacKey, obtained per data center from the ACME CA. + if config.Codesphere.CertIssuer.Acme != nil { + config.Codesphere.CertIssuer.Acme.EABKeyID = "" + } +} diff --git a/internal/installer/secrets/datacenter_test.go b/internal/installer/secrets/datacenter_test.go new file mode 100644 index 00000000..179b6f2a --- /dev/null +++ b/internal/installer/secrets/datacenter_test.go @@ -0,0 +1,197 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +package secrets_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/codesphere-cloud/oms/internal/installer/files" + "github.com/codesphere-cloud/oms/internal/installer/secrets" +) + +// primaryConfig returns a config shaped like the one the GCP bootstrapper generates for the +// primary data center: postgres installed on a dedicated node, so EnsureSecrets produces the +// full postgres secret set. +func primaryConfig() *files.RootConfig { + return &files.RootConfig{ + Postgres: files.PostgresConfig{ + Mode: "install", + Primary: &files.PostgresPrimaryConfig{ + IP: "10.10.0.5", + Hostname: "postgres", + }, + }, + Codesphere: files.CodesphereConfig{ + CertIssuer: files.CertIssuerConfig{ + Type: "acme", + Acme: &files.ACMEConfig{Enabled: true, EABKeyID: "primary-eab-key"}, + }, + }, + } +} + +// secondaryConfig returns a config shaped like the one generated for a secondary data center: +// the shared postgres server is external, so no postgres secrets are regenerated. +func secondaryConfig(caCertPem string) *files.RootConfig { + return &files.RootConfig{ + Postgres: files.PostgresConfig{ + Mode: "external", + ServerAddress: "10.10.0.5", + CACertPem: caCertPem, + }, + Codesphere: files.CodesphereConfig{ + CertIssuer: files.CertIssuerConfig{ + Type: "acme", + Acme: &files.ACMEConfig{Enabled: true, EABKeyID: "primary-eab-key"}, + }, + }, + } +} + +var _ = Describe("IsDataCenterScopedSecret", func() { + DescribeTable("classifies vault secrets", + func(name string, expected bool) { + Expect(secrets.IsDataCenterScopedSecret(name)).To(Equal(expected)) + }, + Entry("cluster ingress CA key", files.SecretSelfSignedCaKeyPem, true), + Entry("cephadm ssh key", files.SecretCephSshPrivateKey, true), + Entry("kubeconfig", files.SecretKubeConfig, true), + Entry("acme EAB mac key", files.SecretAcmeEabMacKey, true), + Entry("nix signing key", files.SecretPrivNixSigningKey, true), + Entry("ceph fs id", "cephFsId", true), + Entry("cephfs admin", "cephfsAdminCodesphere", true), + Entry("csi provisioner", "csiRbdProvisioner", true), + Entry("rgw admin key", "rgwAdminAccessKey", true), + + Entry("postgres admin password", files.SecretPostgresPassword, false), + Entry("postgres CA key", files.SecretPostgresCaKeyPem, false), + Entry("postgres primary server key", files.SecretPostgresPrimaryServerKeyPem, false), + Entry("per-service postgres user", "postgresUserAuth", false), + Entry("per-service postgres password", "postgresPasswordAuth", false), + Entry("token signing key", files.SecretTokenPrivateKey, false), + Entry("domain auth key", files.SecretDomainAuthPrivateKey, false), + Entry("mounter hmac", files.SecretMounterHmacSecret, false), + Entry("managed service password encryption key", files.SecretMongoDbPasswordEncryptionKey, false), + Entry("registry password", files.SecretRegistryPassword, false), + Entry("ssh workspace proxy host key", files.SecretSshWorkspaceProxyHostKey, false), + ) +}) + +var _ = Describe("DeriveDataCenterVault", func() { + var primary *files.InstallVault + + BeforeEach(func() { + primary = newVault() + Expect(secrets.EnsureSecrets(primary, primaryConfig())).To(Succeed()) + // Simulate the ceph and kubernetes install steps writing their credentials back. + primary.SetSecret(files.SecretEntry{Name: "cephFsId", Fields: &files.SecretFields{Password: "fsid-1"}}) + primary.SetSecret(files.SecretEntry{Name: "csiRbdNode", Fields: &files.SecretFields{Password: "csi-1"}}) + primary.SetSecret(files.SecretEntry{Name: "rgwAdminSecretKey", Fields: &files.SecretFields{Password: "rgw-1"}}) + primary.SetSecret(files.SecretEntry{Name: files.SecretKubeConfig, File: &files.SecretFile{Name: "kubeConfig", Content: "dc1-kubeconfig"}}) + }) + + It("keeps the secrets that both data centers must share", func() { + derived := secrets.DeriveDataCenterVault(primary) + + shared := []string{ + files.SecretPostgresPassword, + files.SecretPostgresReplicaPassword, + files.SecretPostgresCaKeyPem, + files.SecretPostgresPrimaryServerKeyPem, + files.SecretPostgresReplicaServerKeyPem, + files.SecretTokenPrivateKey, + files.SecretTokenPublicKey, + files.SecretDomainAuthPrivateKey, + files.SecretDomainAuthPublicKey, + files.SecretMounterHmacSecret, + files.SecretMongoDbPasswordEncryptionKey, + files.SecretSshWorkspaceProxyHostKey, + } + for _, name := range shared { + original := primary.GetSecret(name) + Expect(original).ToNot(BeNil(), "primary vault should contain %s", name) + Expect(derived.GetSecret(name)).To(Equal(original), "%s should be shared", name) + } + }) + + It("keeps every per-service postgres role, because the roles live on the shared server", func() { + derived := secrets.DeriveDataCenterVault(primary) + + for _, svc := range []string{"Auth", "Deployment", "Ide", "Marketplace", "Payment", "PublicApi", "Team", "Workspace"} { + for _, prefix := range []string{"postgresUser", "postgresPassword"} { + name := prefix + svc + if primary.GetSecret(name) == nil { + continue + } + Expect(derived.GetSecret(name)).To(Equal(primary.GetSecret(name)), "%s should be shared", name) + } + } + }) + + It("drops the data-center-scoped secrets", func() { + derived := secrets.DeriveDataCenterVault(primary) + + for _, name := range []string{ + files.SecretSelfSignedCaKeyPem, + files.SecretCephSshPrivateKey, + files.SecretKubeConfig, + "cephFsId", + "csiRbdNode", + "rgwAdminSecretKey", + } { + Expect(derived.GetSecret(name)).To(BeNil(), "%s should be dropped", name) + } + }) + + It("does not alias the primary vault", func() { + derived := secrets.DeriveDataCenterVault(primary) + + derived.GetSecret(files.SecretPostgresPassword).Fields.Password = "mutated" + Expect(primary.GetSecret(files.SecretPostgresPassword).Fields.Password).ToNot(Equal("mutated")) + }) + + It("returns nil for a nil vault", func() { + Expect(secrets.DeriveDataCenterVault(nil)).To(BeNil()) + }) +}) + +var _ = Describe("secondary data center secret generation", func() { + It("shares the database and auth secrets and regenerates the per-cluster ones", func() { + primaryVault := newVault() + primaryCfg := primaryConfig() + Expect(secrets.EnsureSecrets(primaryVault, primaryCfg)).To(Succeed()) + + secondaryVault := secrets.DeriveDataCenterVault(primaryVault) + secondaryCfg := secondaryConfig(primaryCfg.Postgres.CACertPem) + secrets.ClearDataCenterScopedConfig(secondaryCfg) + Expect(secrets.EnsureSecrets(secondaryVault, secondaryCfg)).To(Succeed()) + + By("keeping the shared postgres and auth credentials byte-identical") + Expect(secondaryVault.GetSecret(files.SecretPostgresPassword).Fields.Password). + To(Equal(primaryVault.GetSecret(files.SecretPostgresPassword).Fields.Password)) + Expect(secondaryVault.GetSecret(files.SecretTokenPrivateKey).File.Content). + To(Equal(primaryVault.GetSecret(files.SecretTokenPrivateKey).File.Content)) + Expect(secondaryVault.GetSecret("postgresPasswordAuth").Fields.Password). + To(Equal(primaryVault.GetSecret("postgresPasswordAuth").Fields.Password)) + + By("regenerating the per-cluster ingress CA and cephadm key") + Expect(secondaryVault.GetSecret(files.SecretSelfSignedCaKeyPem)).ToNot(BeNil()) + Expect(secondaryVault.GetSecret(files.SecretSelfSignedCaKeyPem).File.Content). + ToNot(Equal(primaryVault.GetSecret(files.SecretSelfSignedCaKeyPem).File.Content)) + Expect(secondaryVault.GetSecret(files.SecretCephSshPrivateKey)).ToNot(BeNil()) + Expect(secondaryVault.GetSecret(files.SecretCephSshPrivateKey).File.Content). + ToNot(Equal(primaryVault.GetSecret(files.SecretCephSshPrivateKey).File.Content)) + + By("rewriting the config fields paired with the regenerated secrets") + Expect(secondaryCfg.Cluster.Certificates.CA.CertPem).ToNot(BeEmpty()) + Expect(secondaryCfg.Cluster.Certificates.CA.CertPem).ToNot(Equal(primaryCfg.Cluster.Certificates.CA.CertPem)) + Expect(secondaryCfg.Ceph.CephAdmSSHKey.PublicKey).ToNot(BeEmpty()) + Expect(secondaryCfg.Ceph.CephAdmSSHKey.PublicKey).ToNot(Equal(primaryCfg.Ceph.CephAdmSSHKey.PublicKey)) + Expect(secondaryCfg.Codesphere.CertIssuer.Acme.EABKeyID).To(BeEmpty()) + + By("not regenerating the shared postgres CA, since the server is external") + Expect(secondaryCfg.Postgres.CACertPem).To(Equal(primaryCfg.Postgres.CACertPem)) + }) +}) From 7b7d1c1190d453d3716e3c2207e1dc2a43efcc04 Mon Sep 17 00:00:00 2001 From: NJona <25478046+NJona@users.noreply.github.com> Date: Fri, 31 Jul 2026 13:40:18 +0000 Subject: [PATCH 2/2] chore(docs): Auto-update docs and licenses Signed-off-by: NJona <25478046+NJona@users.noreply.github.com> --- NOTICE | 8 ++++---- internal/tmpl/NOTICE | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/NOTICE b/NOTICE index 6d865aef..7cfa0a89 100644 --- a/NOTICE +++ b/NOTICE @@ -1231,25 +1231,25 @@ License URL: https://github.com/protocolbuffers/protobuf-go/blob/f2248ac996af/LI Module: gopkg.in/evanphx/json-patch.v4 Version: v4.13.0 License: BSD-3-Clause -License URL: https://github.com/evanphx/json-patch/blob/v4.13.0/LICENSE +License URL: Unknown ---------- Module: gopkg.in/inf.v0 Version: v0.9.1 License: BSD-3-Clause -License URL: https://github.com/go-inf/inf/blob/v0.9.1/LICENSE +License URL: Unknown ---------- Module: gopkg.in/validator.v2 Version: v2.0.1 License: Apache-2.0 -License URL: https://github.com/go-validator/validator/blob/v2.0.1/LICENSE +License URL: Unknown ---------- Module: gopkg.in/yaml.v3 Version: v3.0.1 License: MIT -License URL: https://github.com/go-yaml/yaml/blob/v3.0.1/LICENSE +License URL: Unknown ---------- Module: helm.sh/helm/v4 diff --git a/internal/tmpl/NOTICE b/internal/tmpl/NOTICE index 6d865aef..7cfa0a89 100644 --- a/internal/tmpl/NOTICE +++ b/internal/tmpl/NOTICE @@ -1231,25 +1231,25 @@ License URL: https://github.com/protocolbuffers/protobuf-go/blob/f2248ac996af/LI Module: gopkg.in/evanphx/json-patch.v4 Version: v4.13.0 License: BSD-3-Clause -License URL: https://github.com/evanphx/json-patch/blob/v4.13.0/LICENSE +License URL: Unknown ---------- Module: gopkg.in/inf.v0 Version: v0.9.1 License: BSD-3-Clause -License URL: https://github.com/go-inf/inf/blob/v0.9.1/LICENSE +License URL: Unknown ---------- Module: gopkg.in/validator.v2 Version: v2.0.1 License: Apache-2.0 -License URL: https://github.com/go-validator/validator/blob/v2.0.1/LICENSE +License URL: Unknown ---------- Module: gopkg.in/yaml.v3 Version: v3.0.1 License: MIT -License URL: https://github.com/go-yaml/yaml/blob/v3.0.1/LICENSE +License URL: Unknown ---------- Module: helm.sh/helm/v4