feat(gcp): bootstrap two data centers with --multi-dc

.gitignore

@@ -48,6 +48,10 @@ internal/util/testdata/
 config.yaml
 prod.vault.yaml
 configure-k0s.sh
+# ... and their per-data-center variants written by a --multi-dc bootstrap
+config-dc*.yaml
+prod-dc*.vault.yaml
+configure-k0s-dc*.sh
 
 # Debugger files
 __debug*

NOTICE

@@ -1231,25 +1231,25 @@ License URL: https://github.com/protocolbuffers/protobuf-go/blob/f2248ac996af/LI
 Module: gopkg.in/evanphx/json-patch.v4
 Version: v4.13.0
 License: BSD-3-Clause
-License URL: https://github.com/evanphx/json-patch/blob/v4.13.0/LICENSE
+License URL: Unknown
 
 ----------
 Module: gopkg.in/inf.v0
 Version: v0.9.1
 License: BSD-3-Clause
-License URL: https://github.com/go-inf/inf/blob/v0.9.1/LICENSE
+License URL: Unknown
 
 ----------
 Module: gopkg.in/validator.v2
 Version: v2.0.1
 License: Apache-2.0
-License URL: https://github.com/go-validator/validator/blob/v2.0.1/LICENSE
+License URL: Unknown
 
 ----------
 Module: gopkg.in/yaml.v3
 Version: v3.0.1
 License: MIT
-License URL: https://github.com/go-yaml/yaml/blob/v3.0.1/LICENSE
+License URL: Unknown
 
 ----------
 Module: helm.sh/helm/v4

cli/cmd/bootstrap_gcp.go

@@ -89,6 +89,7 @@ func AddBootstrapGcpCmd(parent *cobra.Command, opts *util.GlobalOptions) {
 	flags.BoolVar(&bootstrapGcpCmd.CodesphereEnv.SpotVMs, "spot-vms", false, "Use Spot VMs for Codesphere infrastructure. Falls back to standard VMs if spot capacity unavailable. Mutually exclusive with --preemptible (default: false)")
 	flags.IntVar(&bootstrapGcpCmd.CodesphereEnv.DatacenterID, "datacenter-id", 1, "Datacenter ID (default: 1)")
 	flags.StringVar(&bootstrapGcpCmd.CodesphereEnv.DatacenterName, "datacenter-name", "dev", "Datacenter name (default: dev)")
+	flags.BoolVar(&bootstrapGcpCmd.CodesphereEnv.MultiDC, "multi-dc", false, "Bootstrap two data centers that share one PostgreSQL server but run separate Kubernetes and Ceph clusters. Doubles the Ceph and k0s nodes to 14 VMs (~100 vCPUs) and reserves 6 static IPs, so the region's CPU quota may need raising. Cannot be combined with --datacenter-id. (default: false)")
 	flags.StringVar(&bootstrapGcpCmd.CodesphereEnv.CustomPgIP, "custom-pg-ip", "", "Custom PostgreSQL IP (optional)")
 	flags.StringVar(&bootstrapGcpCmd.CodesphereEnv.Region, "region", "europe-west4", "GCP Region (default: europe-west4)")
 	flags.StringVar(&bootstrapGcpCmd.CodesphereEnv.Zone, "zone", "europe-west4-a", "GCP Zone (default: europe-west4-a)")
@@ -174,6 +175,8 @@ func (c *BootstrapGcpCmd) BootstrapGcp() error {
 
 	c.CodesphereEnv.RegistryType = gcp.RegistryType(c.InputRegistryType)
 	c.CodesphereEnv.OmsWorkdir = c.Env.GetOmsWorkdir()
+	// The value alone cannot distinguish the default 1 from an explicit --datacenter-id=1.
+	c.CodesphereEnv.DatacenterIDExplicit = c.cmd.Flags().Changed("datacenter-id")
 	if c.CodesphereEnv.GitHubPAT != "" {
 		c.CodesphereEnv.RegistryType = gcp.RegistryTypeGitHub
 		if c.CodesphereEnv.RegistryUser == "" {
@@ -208,18 +211,27 @@ func (c *BootstrapGcpCmd) BootstrapGcp() error {
 
 	if bs.Env.InstallVersion != "" {
 		log.Printf("Access Codesphere in your web browser at https://cs.%s", bs.Env.BaseDomain)
+		for _, dc := range bs.Env.DataCenters {
+			log.Printf("Data center %d hosts workspaces under %s", dc.ID, dc.WorkspaceHostingBaseDomain)
+		}
 
 		return nil
 	}
 
 	packageName := "<package-name>-installer"
-	installCmd := "oms install codesphere -c /etc/codesphere/config.yaml -k /etc/codesphere/secrets/age_key.txt --vault /etc/codesphere/secrets/prod.vault.yaml"
 	if gcp.RegistryType(bs.Env.RegistryType) == gcp.RegistryTypeGitHub {
 		log.Printf("You set a GitHub PAT for direct image access. Make sure to use a lite package, as VM root disk sizes are reduced.")
-		installCmd += " -s load-container-images"
 		packageName += "-lite"
 	}
-	log.Printf("example install command (run from jumpbox):\n%s -p %s.tar.gz", installCmd, packageName)
+	packageFile := packageName + ".tar.gz"
+	if len(bs.Env.DataCenters) > 1 {
+		log.Printf("example install commands (run from jumpbox). Run the data center 1 command to completion first — the other data centers share its database:")
+	} else {
+		log.Printf("example install command (run from jumpbox):")
+	}
+	for _, dc := range bs.Env.DataCenters {
+		log.Printf("# data center %d\n%s", dc.ID, bs.InstallCommand(dc, packageFile))
+	}
 
 	return nil
 }

docs/oms_beta_bootstrap-gcp.md

@@ -57,6 +57,7 @@ oms beta bootstrap-gcp [flags]
       --install-version string                    Codesphere version to install (default: none)
       --internal-flags stringArray                Internal flags to enable in Codesphere installation (optional) (default [headless-services,vcluster,custom-service-image,ms-in-ls])
       --local-trace-endpoint string               Endpoint for exporting traces to an in-cluster storage (optional)
+      --multi-dc                                  Bootstrap two data centers that share one PostgreSQL server but run separate Kubernetes and Ceph clusters. Doubles the Ceph and k0s nodes to 14 VMs (~100 vCPUs) and reserves 6 static IPs, so the region's CPU quota may need raising. Cannot be combined with --datacenter-id. (default: false)
       --oidc-client-id string                     OIDC OAuth provider Client ID (optional)
       --oidc-client-secret string                 OIDC OAuth provider Client Secret (optional)
       --oidc-issuer-url string                    OIDC OAuth provider issuer URL (optional)

internal/bootstrap/gcp/gcp.go

@@ -150,6 +150,11 @@ func (b *GCPBootstrapper) primaryDC() *DataCenter {
 	return b.Env.DataCenters[0]
 }
 
+// secondaryDCs returns every data center apart from the primary one.
+func (b *GCPBootstrapper) secondaryDCs() []*DataCenter {
+	return b.Env.DataCenters[1:]
+}
+
 // allNodes returns every node of the project: the jumpbox, the shared postgres node and all
 // data centers' Ceph and k0s nodes.
 func (b *GCPBootstrapper) allNodes() []*node.Node {
@@ -416,19 +421,41 @@ func (b *GCPBootstrapper) Bootstrap() error {
 	}
 
 	if b.Env.WriteConfig {
-		err = b.stlog.Step("Update install config", b.UpdateInstallConfig)
+		err = b.writeDataCenterConfig(b.primaryDC())
 		if err != nil {
-			return fmt.Errorf("failed to update install config: %w", err)
+			return err
+		}
+	}
+
+	// Secondary data centers fall back to deriving their config and vault from the primary one,
+	// so this has to run after the primary's secrets were generated above.
+	for _, dc := range b.secondaryDCs() {
+		err = b.stlog.Step(dc.StepName("Ensure install config"), func() error {
+			return b.ensureInstallConfig(dc)
+		})
+		if err != nil {
+			return fmt.Errorf("failed to ensure install config of data center %d: %w", dc.ID, err)
 		}
 
-		err = b.stlog.Step("Ensure age key", b.EnsureAgeKey)
+		err = b.stlog.Step(dc.StepName("Ensure secrets"), func() error {
+			return b.ensureSecrets(dc)
+		})
 		if err != nil {
-			return fmt.Errorf("failed to ensure age key: %w", err)
+			return fmt.Errorf("failed to ensure secrets of data center %d: %w", dc.ID, err)
 		}
 
-		err = b.stlog.Step("Encrypt vault", b.EncryptVault)
+		err = b.stlog.Step(dc.StepName("Derive config and vault"), func() error {
+			return b.seedSecondaryDataCenter(b.primaryDC(), dc)
+		})
 		if err != nil {
-			return fmt.Errorf("failed to encrypt vault: %w", err)
+			return fmt.Errorf("failed to derive data center %d: %w", dc.ID, err)
+		}
+
+		if b.Env.WriteConfig {
+			err = b.writeDataCenterConfig(dc)
+			if err != nil {
+				return err
+			}
 		}
 	}
 
@@ -554,9 +581,50 @@ func (b *GCPBootstrapper) ValidateInput() error {
 		return err
 	}
 
+	err = b.validateMultiDC()
+	if err != nil {
+		return err
+	}
+
 	return b.validateTelemetryExportParams()
 }
 
+// validateMultiDC rejects flag combinations a multi-data-center bootstrap cannot satisfy.
+func (b *GCPBootstrapper) validateMultiDC() error {
+	if !b.Env.MultiDC {
+		return nil
+	}
+
+	// A secondary data center's config and vault are derived from the primary's, which only
+	// happens when configs are written.
+	if !b.Env.WriteConfig {
+		return fmt.Errorf("multi-dc requires write-config to be enabled")
+	}
+
+	// The data center IDs are derived (1 and 2) and drive the workspace hosting domains.
+	if b.Env.DatacenterIDExplicit {
+		return fmt.Errorf("datacenter-id cannot be combined with multi-dc, the IDs are derived")
+	}
+
+	// The k0s cluster is named codesphere-<datacenter name>, so both names must be set and
+	// distinct. BuildDataCenters derives the second one by suffixing the first.
+	if b.Env.DatacenterName == "" {
+		return fmt.Errorf("datacenter-name is required with multi-dc")
+	}
+
+	// Every data center gets its own local config and vault, derived by suffixing these paths.
+	for name, path := range map[string]string{
+		"install-config": b.Env.InstallConfigPath,
+		"secrets-file":   b.Env.SecretsFilePath,
+	} {
+		if path == "" {
+			return fmt.Errorf("cannot derive a per-data-center path: %s is empty", name)
+		}
+	}
+
+	return nil
+}
+
 func (b *GCPBootstrapper) validateClusterAdminEmail() error {
 	if b.Env.ClusterAdminEmail == "" {
 		return nil