diff --git a/NOTICE b/NOTICE index 6d865aef..7cfa0a89 100644 --- a/NOTICE +++ b/NOTICE @@ -1231,25 +1231,25 @@ License URL: https://github.com/protocolbuffers/protobuf-go/blob/f2248ac996af/LI Module: gopkg.in/evanphx/json-patch.v4 Version: v4.13.0 License: BSD-3-Clause -License URL: https://github.com/evanphx/json-patch/blob/v4.13.0/LICENSE +License URL: Unknown ---------- Module: gopkg.in/inf.v0 Version: v0.9.1 License: BSD-3-Clause -License URL: https://github.com/go-inf/inf/blob/v0.9.1/LICENSE +License URL: Unknown ---------- Module: gopkg.in/validator.v2 Version: v2.0.1 License: Apache-2.0 -License URL: https://github.com/go-validator/validator/blob/v2.0.1/LICENSE +License URL: Unknown ---------- Module: gopkg.in/yaml.v3 Version: v3.0.1 License: MIT -License URL: https://github.com/go-yaml/yaml/blob/v3.0.1/LICENSE +License URL: Unknown ---------- Module: helm.sh/helm/v4 diff --git a/internal/installer/secrets/datacenter.go b/internal/installer/secrets/datacenter.go new file mode 100644 index 00000000..133e148e --- /dev/null +++ b/internal/installer/secrets/datacenter.go @@ -0,0 +1,87 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +package secrets + +import ( + "strings" + + "github.com/codesphere-cloud/oms/internal/installer/files" +) + +// DataCenterScopedSecretNames are the secrets that belong to exactly one data center and must +// therefore be regenerated for every additional data center of a multi-DC installation. +// +// Everything not listed here (and not matched by DataCenterScopedSecretPrefixes) is shared: +// the postgres roles because both data centers talk to the same server, and the auth and +// encryption keys because tokens minted and rows written in one data center are consumed in +// the other. +var DataCenterScopedSecretNames = []string{ + // Cluster ingress CA — paired with cluster.certificates.ca.certPem. + files.SecretSelfSignedCaKeyPem, + // cephadm SSH key — paired with ceph.cephAdmSshKey.publicKey. + files.SecretCephSshPrivateKey, + // Written by the installer's kubernetes step; points at one cluster's API server. + files.SecretKubeConfig, + // ACME external account binding — paired with codesphere.certIssuer.acme.eabKeyId. + files.SecretAcmeEabMacKey, + // Only present in recovered vaults; keyed to a single cluster's nix cache. + files.SecretPrivNixSigningKey, + files.SecretPubNixSigningKey, +} + +// DataCenterScopedSecretPrefixes cover the Ceph cluster credentials that the installer's ceph +// step writes back into the vault (cephFsId, cephfsAdmin, csiRbdNode, rgwAdminAccessKey, ...). +// The installer owns those names, so they are matched by prefix rather than enumerated. +var DataCenterScopedSecretPrefixes = []string{"ceph", "csi", "rgw"} + +// IsDataCenterScopedSecret reports whether a vault entry belongs to a single data center. +func IsDataCenterScopedSecret(name string) bool { + for _, scoped := range DataCenterScopedSecretNames { + if name == scoped { + return true + } + } + for _, prefix := range DataCenterScopedSecretPrefixes { + if strings.HasPrefix(name, prefix) { + return true + } + } + return false +} + +// DeriveDataCenterVault returns a copy of the primary data center's vault with all +// data-center-scoped secrets removed. Running EnsureSecrets over the result regenerates exactly +// those, while the shared secrets stay byte-identical across data centers. +func DeriveDataCenterVault(primary *files.InstallVault) *files.InstallVault { + derived := primary.Clone() + if derived == nil { + return nil + } + + kept := make([]files.SecretEntry, 0, len(derived.Secrets)) + for _, entry := range derived.Secrets { + if IsDataCenterScopedSecret(entry.Name) { + continue + } + kept = append(kept, entry) + } + derived.Secrets = kept + + return derived +} + +// ClearDataCenterScopedConfig resets the config fields that are written by the Ensure* functions +// alongside a data-center-scoped vault secret. Those pairs must always be mutated together: the +// Ensure* functions are gated on the vault entry, so a config field left in place would keep a +// stale value paired with a freshly generated key. +func ClearDataCenterScopedConfig(config *files.RootConfig) { + // Paired with selfSignedCaKeyPem, written by EnsureIngressCA. + config.Cluster.Certificates.CA.CertPem = "" + // Paired with cephSshPrivateKey, written by EnsureCephSSHKeys. + config.Ceph.CephAdmSSHKey.PublicKey = "" + // Paired with acmeEabMacKey, obtained per data center from the ACME CA. + if config.Codesphere.CertIssuer.Acme != nil { + config.Codesphere.CertIssuer.Acme.EABKeyID = "" + } +} diff --git a/internal/installer/secrets/datacenter_test.go b/internal/installer/secrets/datacenter_test.go new file mode 100644 index 00000000..179b6f2a --- /dev/null +++ b/internal/installer/secrets/datacenter_test.go @@ -0,0 +1,197 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +package secrets_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/codesphere-cloud/oms/internal/installer/files" + "github.com/codesphere-cloud/oms/internal/installer/secrets" +) + +// primaryConfig returns a config shaped like the one the GCP bootstrapper generates for the +// primary data center: postgres installed on a dedicated node, so EnsureSecrets produces the +// full postgres secret set. +func primaryConfig() *files.RootConfig { + return &files.RootConfig{ + Postgres: files.PostgresConfig{ + Mode: "install", + Primary: &files.PostgresPrimaryConfig{ + IP: "10.10.0.5", + Hostname: "postgres", + }, + }, + Codesphere: files.CodesphereConfig{ + CertIssuer: files.CertIssuerConfig{ + Type: "acme", + Acme: &files.ACMEConfig{Enabled: true, EABKeyID: "primary-eab-key"}, + }, + }, + } +} + +// secondaryConfig returns a config shaped like the one generated for a secondary data center: +// the shared postgres server is external, so no postgres secrets are regenerated. +func secondaryConfig(caCertPem string) *files.RootConfig { + return &files.RootConfig{ + Postgres: files.PostgresConfig{ + Mode: "external", + ServerAddress: "10.10.0.5", + CACertPem: caCertPem, + }, + Codesphere: files.CodesphereConfig{ + CertIssuer: files.CertIssuerConfig{ + Type: "acme", + Acme: &files.ACMEConfig{Enabled: true, EABKeyID: "primary-eab-key"}, + }, + }, + } +} + +var _ = Describe("IsDataCenterScopedSecret", func() { + DescribeTable("classifies vault secrets", + func(name string, expected bool) { + Expect(secrets.IsDataCenterScopedSecret(name)).To(Equal(expected)) + }, + Entry("cluster ingress CA key", files.SecretSelfSignedCaKeyPem, true), + Entry("cephadm ssh key", files.SecretCephSshPrivateKey, true), + Entry("kubeconfig", files.SecretKubeConfig, true), + Entry("acme EAB mac key", files.SecretAcmeEabMacKey, true), + Entry("nix signing key", files.SecretPrivNixSigningKey, true), + Entry("ceph fs id", "cephFsId", true), + Entry("cephfs admin", "cephfsAdminCodesphere", true), + Entry("csi provisioner", "csiRbdProvisioner", true), + Entry("rgw admin key", "rgwAdminAccessKey", true), + + Entry("postgres admin password", files.SecretPostgresPassword, false), + Entry("postgres CA key", files.SecretPostgresCaKeyPem, false), + Entry("postgres primary server key", files.SecretPostgresPrimaryServerKeyPem, false), + Entry("per-service postgres user", "postgresUserAuth", false), + Entry("per-service postgres password", "postgresPasswordAuth", false), + Entry("token signing key", files.SecretTokenPrivateKey, false), + Entry("domain auth key", files.SecretDomainAuthPrivateKey, false), + Entry("mounter hmac", files.SecretMounterHmacSecret, false), + Entry("managed service password encryption key", files.SecretMongoDbPasswordEncryptionKey, false), + Entry("registry password", files.SecretRegistryPassword, false), + Entry("ssh workspace proxy host key", files.SecretSshWorkspaceProxyHostKey, false), + ) +}) + +var _ = Describe("DeriveDataCenterVault", func() { + var primary *files.InstallVault + + BeforeEach(func() { + primary = newVault() + Expect(secrets.EnsureSecrets(primary, primaryConfig())).To(Succeed()) + // Simulate the ceph and kubernetes install steps writing their credentials back. + primary.SetSecret(files.SecretEntry{Name: "cephFsId", Fields: &files.SecretFields{Password: "fsid-1"}}) + primary.SetSecret(files.SecretEntry{Name: "csiRbdNode", Fields: &files.SecretFields{Password: "csi-1"}}) + primary.SetSecret(files.SecretEntry{Name: "rgwAdminSecretKey", Fields: &files.SecretFields{Password: "rgw-1"}}) + primary.SetSecret(files.SecretEntry{Name: files.SecretKubeConfig, File: &files.SecretFile{Name: "kubeConfig", Content: "dc1-kubeconfig"}}) + }) + + It("keeps the secrets that both data centers must share", func() { + derived := secrets.DeriveDataCenterVault(primary) + + shared := []string{ + files.SecretPostgresPassword, + files.SecretPostgresReplicaPassword, + files.SecretPostgresCaKeyPem, + files.SecretPostgresPrimaryServerKeyPem, + files.SecretPostgresReplicaServerKeyPem, + files.SecretTokenPrivateKey, + files.SecretTokenPublicKey, + files.SecretDomainAuthPrivateKey, + files.SecretDomainAuthPublicKey, + files.SecretMounterHmacSecret, + files.SecretMongoDbPasswordEncryptionKey, + files.SecretSshWorkspaceProxyHostKey, + } + for _, name := range shared { + original := primary.GetSecret(name) + Expect(original).ToNot(BeNil(), "primary vault should contain %s", name) + Expect(derived.GetSecret(name)).To(Equal(original), "%s should be shared", name) + } + }) + + It("keeps every per-service postgres role, because the roles live on the shared server", func() { + derived := secrets.DeriveDataCenterVault(primary) + + for _, svc := range []string{"Auth", "Deployment", "Ide", "Marketplace", "Payment", "PublicApi", "Team", "Workspace"} { + for _, prefix := range []string{"postgresUser", "postgresPassword"} { + name := prefix + svc + if primary.GetSecret(name) == nil { + continue + } + Expect(derived.GetSecret(name)).To(Equal(primary.GetSecret(name)), "%s should be shared", name) + } + } + }) + + It("drops the data-center-scoped secrets", func() { + derived := secrets.DeriveDataCenterVault(primary) + + for _, name := range []string{ + files.SecretSelfSignedCaKeyPem, + files.SecretCephSshPrivateKey, + files.SecretKubeConfig, + "cephFsId", + "csiRbdNode", + "rgwAdminSecretKey", + } { + Expect(derived.GetSecret(name)).To(BeNil(), "%s should be dropped", name) + } + }) + + It("does not alias the primary vault", func() { + derived := secrets.DeriveDataCenterVault(primary) + + derived.GetSecret(files.SecretPostgresPassword).Fields.Password = "mutated" + Expect(primary.GetSecret(files.SecretPostgresPassword).Fields.Password).ToNot(Equal("mutated")) + }) + + It("returns nil for a nil vault", func() { + Expect(secrets.DeriveDataCenterVault(nil)).To(BeNil()) + }) +}) + +var _ = Describe("secondary data center secret generation", func() { + It("shares the database and auth secrets and regenerates the per-cluster ones", func() { + primaryVault := newVault() + primaryCfg := primaryConfig() + Expect(secrets.EnsureSecrets(primaryVault, primaryCfg)).To(Succeed()) + + secondaryVault := secrets.DeriveDataCenterVault(primaryVault) + secondaryCfg := secondaryConfig(primaryCfg.Postgres.CACertPem) + secrets.ClearDataCenterScopedConfig(secondaryCfg) + Expect(secrets.EnsureSecrets(secondaryVault, secondaryCfg)).To(Succeed()) + + By("keeping the shared postgres and auth credentials byte-identical") + Expect(secondaryVault.GetSecret(files.SecretPostgresPassword).Fields.Password). + To(Equal(primaryVault.GetSecret(files.SecretPostgresPassword).Fields.Password)) + Expect(secondaryVault.GetSecret(files.SecretTokenPrivateKey).File.Content). + To(Equal(primaryVault.GetSecret(files.SecretTokenPrivateKey).File.Content)) + Expect(secondaryVault.GetSecret("postgresPasswordAuth").Fields.Password). + To(Equal(primaryVault.GetSecret("postgresPasswordAuth").Fields.Password)) + + By("regenerating the per-cluster ingress CA and cephadm key") + Expect(secondaryVault.GetSecret(files.SecretSelfSignedCaKeyPem)).ToNot(BeNil()) + Expect(secondaryVault.GetSecret(files.SecretSelfSignedCaKeyPem).File.Content). + ToNot(Equal(primaryVault.GetSecret(files.SecretSelfSignedCaKeyPem).File.Content)) + Expect(secondaryVault.GetSecret(files.SecretCephSshPrivateKey)).ToNot(BeNil()) + Expect(secondaryVault.GetSecret(files.SecretCephSshPrivateKey).File.Content). + ToNot(Equal(primaryVault.GetSecret(files.SecretCephSshPrivateKey).File.Content)) + + By("rewriting the config fields paired with the regenerated secrets") + Expect(secondaryCfg.Cluster.Certificates.CA.CertPem).ToNot(BeEmpty()) + Expect(secondaryCfg.Cluster.Certificates.CA.CertPem).ToNot(Equal(primaryCfg.Cluster.Certificates.CA.CertPem)) + Expect(secondaryCfg.Ceph.CephAdmSSHKey.PublicKey).ToNot(BeEmpty()) + Expect(secondaryCfg.Ceph.CephAdmSSHKey.PublicKey).ToNot(Equal(primaryCfg.Ceph.CephAdmSSHKey.PublicKey)) + Expect(secondaryCfg.Codesphere.CertIssuer.Acme.EABKeyID).To(BeEmpty()) + + By("not regenerating the shared postgres CA, since the server is external") + Expect(secondaryCfg.Postgres.CACertPem).To(Equal(primaryCfg.Postgres.CACertPem)) + }) +}) diff --git a/internal/tmpl/NOTICE b/internal/tmpl/NOTICE index 6d865aef..7cfa0a89 100644 --- a/internal/tmpl/NOTICE +++ b/internal/tmpl/NOTICE @@ -1231,25 +1231,25 @@ License URL: https://github.com/protocolbuffers/protobuf-go/blob/f2248ac996af/LI Module: gopkg.in/evanphx/json-patch.v4 Version: v4.13.0 License: BSD-3-Clause -License URL: https://github.com/evanphx/json-patch/blob/v4.13.0/LICENSE +License URL: Unknown ---------- Module: gopkg.in/inf.v0 Version: v0.9.1 License: BSD-3-Clause -License URL: https://github.com/go-inf/inf/blob/v0.9.1/LICENSE +License URL: Unknown ---------- Module: gopkg.in/validator.v2 Version: v2.0.1 License: Apache-2.0 -License URL: https://github.com/go-validator/validator/blob/v2.0.1/LICENSE +License URL: Unknown ---------- Module: gopkg.in/yaml.v3 Version: v3.0.1 License: MIT -License URL: https://github.com/go-yaml/yaml/blob/v3.0.1/LICENSE +License URL: Unknown ---------- Module: helm.sh/helm/v4