feat(gcp): install k0s with k0sctl before Codesphere

cli/cmd/k0s/install_k0s.go

@@ -66,7 +66,7 @@ func AddInstallCmd(install *cobra.Command, opts *util.GlobalOptions) {
 			- Deploy k0s to all nodes defined in the install-config using k0sctl`),
 			Example: util.FormatExamples("install k0s", []packageio.Example{
 				{Cmd: "--install-config <path>", Desc: "Path to Codesphere install-config file to generate k0s config from"},
-				{Cmd: "--version <version>", Desc: "Version of k0s to install (e.g., v1.30.0+k0s.0)"},
+				{Cmd: "--version <version>", Desc: "Version of k0s to install (e.g., v1.31.14+k0s.0)"},
 				{Cmd: "--k0sctl-version <version>", Desc: "Version of k0sctl to use (e.g., v0.17.4)"},
 				{Cmd: "--package <file>", Desc: "Package file (e.g. codesphere-v1.2.3-installer-lite.tar.gz) to load k0s from"},
 				{Cmd: "--ssh-key-path <path>", Desc: "SSH private key path for remote installation"},
@@ -78,7 +78,7 @@ func AddInstallCmd(install *cobra.Command, opts *util.GlobalOptions) {
 		Env:        env.NewEnv(),
 		FileWriter: intutil.NewFilesystemWriter(),
 	}
-	k0s.cmd.Flags().StringVarP(&k0s.Opts.Version, "version", "v", "", "Version of k0s to install")
+	k0s.cmd.Flags().StringVarP(&k0s.Opts.Version, "version", "v", installer.DefaultK0sVersion, "Version of k0s to install")
 	k0s.cmd.Flags().StringVar(&k0s.Opts.K0sctlVersion, "k0sctl-version", "", "Version of k0sctl to use")
 	k0s.cmd.Flags().StringVarP(&k0s.Opts.Package, "package", "p", "", "Package file (e.g. codesphere-v1.2.3-installer-lite.tar.gz) to load k0s from")
 	k0s.cmd.Flags().StringVar(&k0s.Opts.InstallConfig, "install-config", "", "Path to Codesphere install-config file (required)")

docs/oms_install_k0s.md

@@ -22,7 +22,7 @@ oms install k0s [flags]
 # Path to Codesphere install-config file to generate k0s config from
 $ oms install k0s --install-config <path>
 
-# Version of k0s to install (e.g., v1.30.0+k0s.0)
+# Version of k0s to install (e.g., v1.31.14+k0s.0)
 $ oms install k0s --version <version>
 
 # Version of k0sctl to use (e.g., v0.17.4)
@@ -54,7 +54,7 @@ $ oms install k0s --no-download
       --ssh-key-path string     SSH private key path for remote installation
       --vault string            Path to prod.vault.yaml to save the kubeconfig into (optional)
       --vault-priv-key string   Path to the age private key to decrypt the vault (optional, for SOPS-encrypted vaults)
-  -v, --version string          Version of k0s to install
+  -v, --version string          Version of k0s to install (default "v1.31.14+k0s.0")
 ```
 
 ### SEE ALSO

internal/bootstrap/gcp/gcp.go

@@ -351,6 +351,21 @@ func (b *GCPBootstrapper) Bootstrap() error {
 	}
 
 	if b.Env.InstallVersion != "" || b.Env.InstallLocal != "" {
+		err = b.stlog.Step("Install k0s", b.InstallK0s)
+		if err != nil {
+			return fmt.Errorf("failed to install k0s: %w", err)
+		}
+
+		err = b.stlog.Step("Wait for k0s nodes", b.WaitForK0sNodes)
+		if err != nil {
+			return fmt.Errorf("failed waiting for k0s nodes: %w", err)
+		}
+
+		err = b.stlog.Step("Ensure Codesphere prerequisites", b.EnsureCodespherePrerequisites)
+		if err != nil {
+			return fmt.Errorf("failed to ensure Codesphere prerequisites: %w", err)
+		}
+
 		err = b.stlog.Step("Install Codesphere", b.InstallCodesphere)
 		if err != nil {
 			return fmt.Errorf("failed to install Codesphere: %w", err)
@@ -1058,52 +1073,103 @@ func (b *GCPBootstrapper) EnsureDNSRecords() error {
 }
 
 func (b *GCPBootstrapper) InstallCodesphere() error {
-	fullPackageFilename, err := b.ensureCodespherePackageOnJumpbox()
-	if err != nil {
+	if err := b.ensureCodespherePackageOnJumpbox(); err != nil {
 		return fmt.Errorf("failed to ensure Codesphere package on jumpbox: %w", err)
 	}
 
-	err = b.runInstallCommand(fullPackageFilename)
-	if err != nil {
+	if err := b.runInstallCommand(b.codespherePackageFilename()); err != nil {
 		return fmt.Errorf("failed to install Codesphere from jumpbox: %w", err)
 	}
 
 	return nil
 }
 
-func (b *GCPBootstrapper) ensureCodespherePackageOnJumpbox() (string, error) {
-	packageFilename := "installer.tar.gz"
+// InstallK0s deploys k0s with the native OMS installer and stores its
+// kubeconfig in the encrypted install vault for the remaining installer steps.
+func (b *GCPBootstrapper) InstallK0s() error {
+	// Reuse matching cached binaries and let k0sctl reconcile normally. Without
+	// --force, an unchanged cluster remains untouched on bootstrap retries.
+	installCmd := fmt.Sprintf("oms install k0s --version %s --install-config /etc/codesphere/config.yaml --vault %s --vault-priv-key %s/age_key.txt",
+		installer.DefaultK0sVersion, filepath.Join(b.Env.SecretsDir, "prod.vault.yaml"), b.Env.SecretsDir)
+	if err := b.Env.Jumpbox.RunSSHCommand("root", installCmd); err != nil {
+		return fmt.Errorf("failed to install k0s from jumpbox: %w", err)
+	}
+
+	return nil
+}
+
+// WaitForK0sNodes restores the readiness barrier from the TypeScript
+// Kubernetes setup. k0sctl apply completing is not sufficient for the
+// Codesphere charts: all schedulable nodes must be Ready before gateway
+// controllers and their admission webhooks are installed.
+func (b *GCPBootstrapper) WaitForK0sNodes() error {
+	const command = "k0s kubectl wait --for=condition=Ready nodes --all --timeout=30m"
+	if err := b.Env.ControlPlaneNodes[0].RunSSHCommand("root", command); err != nil {
+		return fmt.Errorf("k0s nodes did not become ready: %w", err)
+	}
+	return nil
+}
+
+// EnsureCodespherePrerequisites recreates the resources normally installed by
+// the skipped TypeScript Kubernetes step. The dummy error-page-server Service
+// must exist before ingress-nginx starts; otherwise both gateway controllers
+// exit while resolving their configured default backend. The TypeScript
+// platform step removes this unmanaged Service before Helm installs the real
+// one.
+func (b *GCPBootstrapper) EnsureCodespherePrerequisites() error {
+	const namespaceCommand = "k0s kubectl create namespace codesphere --dry-run=client -o yaml | k0s kubectl apply -f -"
+	if err := b.Env.ControlPlaneNodes[0].RunSSHCommand("root", namespaceCommand); err != nil {
+		return fmt.Errorf("failed to create Codesphere namespace: %w", err)
+	}
+
+	const serviceCommand = "k0s kubectl -n codesphere create service clusterip error-page-server --tcp=8080:8080 --dry-run=client -o yaml | k0s kubectl apply -f -"
+	if err := b.Env.ControlPlaneNodes[0].RunSSHCommand("root", serviceCommand); err != nil {
+		return fmt.Errorf("failed to create dummy error-page-server service: %w", err)
+	}
+	return nil
+}
+
+func (b *GCPBootstrapper) codespherePackageFilename() string {
+	packageFilename := b.codespherePackageArchiveName()
+	if b.Env.InstallLocal != "" {
+		return "local-" + packageFilename
+	}
+	return portal.BuildPackageFilenameFromParts(b.Env.InstallVersion, b.Env.InstallHash, packageFilename)
+}
+
+func (b *GCPBootstrapper) codespherePackageArchiveName() string {
 	if b.Env.RegistryType == RegistryTypeGitHub {
-		packageFilename = "installer-lite.tar.gz"
+		return "installer-lite.tar.gz"
 	}
+	return "installer.tar.gz"
+}
 
+func (b *GCPBootstrapper) ensureCodespherePackageOnJumpbox() error {
 	if b.Env.InstallLocal != "" {
 		b.stlog.Logf("Copying local package %s to jumpbox...", b.Env.InstallLocal)
-		fullPackageFilename := fmt.Sprintf("local-%s", packageFilename)
-		err := b.Env.Jumpbox.NodeClient.CopyFile(b.Env.Jumpbox, b.Env.InstallLocal, "/root/"+fullPackageFilename)
+		err := b.Env.Jumpbox.NodeClient.CopyFile(b.Env.Jumpbox, b.Env.InstallLocal, "/root/"+b.codespherePackageFilename())
 		if err != nil {
-			return "", fmt.Errorf("failed to copy local install package to jumpbox: %w", err)
+			return fmt.Errorf("failed to copy local install package to jumpbox: %w", err)
 		}
-		return fullPackageFilename, nil
+		return nil
 	}
 
 	if b.Env.InstallVersion == "" {
-		return "", errors.New("either install version or a local package must be specified to install Codesphere")
+		return errors.New("either install version or a local package must be specified to install Codesphere")
 	}
 
-	fullPackageFilename := portal.BuildPackageFilenameFromParts(b.Env.InstallVersion, b.Env.InstallHash, packageFilename)
 	if b.Env.InstallHash == "" {
-		return "", fmt.Errorf("install hash must be set when install version is set")
+		return fmt.Errorf("install hash must be set when install version is set")
 	}
 	b.stlog.Logf("Downloading Codesphere package...")
 	downloadCmd := fmt.Sprintf("oms download package -f %s -H %s %s",
-		packageFilename, b.Env.InstallHash, b.Env.InstallVersion)
+		b.codespherePackageArchiveName(), b.Env.InstallHash, b.Env.InstallVersion)
 	err := b.Env.Jumpbox.RunSSHCommand("root", downloadCmd)
 	if err != nil {
-		return "", fmt.Errorf("failed to download Codesphere package from jumpbox: %w", err)
+		return fmt.Errorf("failed to download Codesphere package from jumpbox: %w", err)
 	}
 
-	return fullPackageFilename, nil
+	return nil
 }
 
 func (b *GCPBootstrapper) runInstallCommand(packageFilename string) error {
@@ -1114,17 +1180,19 @@ func (b *GCPBootstrapper) runInstallCommand(packageFilename string) error {
 }
 
 func (b *GCPBootstrapper) generateSkipStepsArg() string {
-	skipSteps := b.Env.InstallSkipSteps
+	// k0s is installed by OMS before the TypeScript installer runs, so the
+	// TypeScript Kubernetes step must never run during GCP bootstrapping.
+	skipSteps := util.AppendUnique(nil, b.Env.InstallSkipSteps...)
+	skipSteps = util.AppendUnique(skipSteps, "kubernetes")
 	if b.Env.RegistryType == RegistryTypeGitHub {
-		skipSteps = append(skipSteps, "load-container-images")
+		skipSteps = util.AppendUnique(skipSteps, "load-container-images")
 	}
 	if len(skipSteps) == 0 {
 		return ""
 	}
 
 	return " -s " + strings.Join(skipSteps, ",")
 }
-
 func (b *GCPBootstrapper) GenerateK0sConfigScript() error {
 	script := `#!/bin/bash
 

internal/bootstrap/gcp/gcp_test.go

@@ -1395,7 +1395,7 @@ var _ = Describe("GCP Bootstrapper", func() {
 
 					// Expect install codesphere
 					nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root",
-						"oms install codesphere -c /etc/codesphere/config.yaml -k /etc/codesphere/secrets/age_key.txt --vault /etc/codesphere/secrets/prod.vault.yaml -p v1.2.3-abc1234567890-installer-lite.tar.gz -s load-container-images").Return(nil)
+						"oms install codesphere -c /etc/codesphere/config.yaml -k /etc/codesphere/secrets/age_key.txt --vault /etc/codesphere/secrets/prod.vault.yaml -p v1.2.3-abc1234567890-installer-lite.tar.gz -s kubernetes,load-container-images").Return(nil)
 
 					err := bs.InstallCodesphere()
 					Expect(err).NotTo(HaveOccurred())
@@ -1412,7 +1412,7 @@ var _ = Describe("GCP Bootstrapper", func() {
 					nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root", "oms download package -f installer.tar.gz -H def9876543210 v1.2.3").Return(nil)
 
 					// Expect install codesphere
-					nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root", "oms install codesphere -c /etc/codesphere/config.yaml -k /etc/codesphere/secrets/age_key.txt --vault /etc/codesphere/secrets/prod.vault.yaml -p v1.2.3-def9876543210-installer.tar.gz").Return(nil)
+					nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root", "oms install codesphere -c /etc/codesphere/config.yaml -k /etc/codesphere/secrets/age_key.txt --vault /etc/codesphere/secrets/prod.vault.yaml -p v1.2.3-def9876543210-installer.tar.gz -s kubernetes").Return(nil)
 
 					err := bs.InstallCodesphere()
 					Expect(err).NotTo(HaveOccurred())
@@ -1421,7 +1421,16 @@ var _ = Describe("GCP Bootstrapper", func() {
 
 			It("downloads and installs codesphere with hash", func() {
 				nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root", "oms download package -f installer.tar.gz -H abc1234567890 v1.2.3").Return(nil)
-				nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root", "oms install codesphere -c /etc/codesphere/config.yaml -k /etc/codesphere/secrets/age_key.txt --vault /etc/codesphere/secrets/prod.vault.yaml -p v1.2.3-abc1234567890-installer.tar.gz").Return(nil)
+				nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root", "oms install codesphere -c /etc/codesphere/config.yaml -k /etc/codesphere/secrets/age_key.txt --vault /etc/codesphere/secrets/prod.vault.yaml -p v1.2.3-abc1234567890-installer.tar.gz -s kubernetes").Return(nil)
+
+				err := bs.InstallCodesphere()
+				Expect(err).NotTo(HaveOccurred())
+			})
+
+			It("preserves requested skip steps without duplicating kubernetes", func() {
+				csEnv.InstallSkipSteps = []string{"postgres", "kubernetes"}
+				nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root", "oms download package -f installer.tar.gz -H abc1234567890 v1.2.3").Return(nil)
+				nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root", "oms install codesphere -c /etc/codesphere/config.yaml -k /etc/codesphere/secrets/age_key.txt --vault /etc/codesphere/secrets/prod.vault.yaml -p v1.2.3-abc1234567890-installer.tar.gz -s postgres,kubernetes").Return(nil)
 
 				err := bs.InstallCodesphere()
 				Expect(err).NotTo(HaveOccurred())
@@ -1440,7 +1449,7 @@ var _ = Describe("GCP Bootstrapper", func() {
 					It("installs codesphere from local package", func() {
 						nodeClient.EXPECT().CopyFile(mock.Anything, csEnv.InstallLocal, "/root/local-installer-lite.tar.gz").Return(nil)
 						nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root",
-							"oms install codesphere -c /etc/codesphere/config.yaml -k /etc/codesphere/secrets/age_key.txt --vault /etc/codesphere/secrets/prod.vault.yaml -p local-installer-lite.tar.gz -s load-container-images").Return(nil)
+							"oms install codesphere -c /etc/codesphere/config.yaml -k /etc/codesphere/secrets/age_key.txt --vault /etc/codesphere/secrets/prod.vault.yaml -p local-installer-lite.tar.gz -s kubernetes,load-container-images").Return(nil)
 
 						err := bs.InstallCodesphere()
 						Expect(err).NotTo(HaveOccurred())
@@ -1454,7 +1463,7 @@ var _ = Describe("GCP Bootstrapper", func() {
 					It("installs codesphere from local package", func() {
 						nodeClient.EXPECT().CopyFile(mock.Anything, csEnv.InstallLocal, "/root/local-installer.tar.gz").Return(nil)
 						nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root",
-							"oms install codesphere -c /etc/codesphere/config.yaml -k /etc/codesphere/secrets/age_key.txt --vault /etc/codesphere/secrets/prod.vault.yaml -p local-installer.tar.gz").Return(nil)
+							"oms install codesphere -c /etc/codesphere/config.yaml -k /etc/codesphere/secrets/age_key.txt --vault /etc/codesphere/secrets/prod.vault.yaml -p local-installer.tar.gz -s kubernetes").Return(nil)
 
 						err := bs.InstallCodesphere()
 						Expect(err).NotTo(HaveOccurred())
@@ -1498,7 +1507,7 @@ var _ = Describe("GCP Bootstrapper", func() {
 
 			It("fails when install codesphere fails", func() {
 				nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root", "oms download package -f installer.tar.gz -H abc1234567890 v1.2.3").Return(nil).Once()
-				nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root", "oms install codesphere -c /etc/codesphere/config.yaml -k /etc/codesphere/secrets/age_key.txt --vault /etc/codesphere/secrets/prod.vault.yaml -p v1.2.3-abc1234567890-installer.tar.gz").Return(fmt.Errorf("install error")).Once()
+				nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root", "oms install codesphere -c /etc/codesphere/config.yaml -k /etc/codesphere/secrets/age_key.txt --vault /etc/codesphere/secrets/prod.vault.yaml -p v1.2.3-abc1234567890-installer.tar.gz -s kubernetes").Return(fmt.Errorf("install error")).Once()
 
 				err := bs.InstallCodesphere()
 				Expect(err).To(HaveOccurred())
@@ -1507,6 +1516,71 @@ var _ = Describe("GCP Bootstrapper", func() {
 		})
 	})
 
+	Describe("InstallK0s", func() {
+		BeforeEach(func() {
+			csEnv.InstallVersion = "v1.2.3"
+			csEnv.InstallHash = "abc1234567890"
+		})
+
+		It("downloads k0s and lets k0sctl distribute it independently of the Codesphere package", func() {
+			nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root",
+				"oms install k0s --version v1.31.14+k0s.0 --install-config /etc/codesphere/config.yaml --vault /etc/codesphere/secrets/prod.vault.yaml --vault-priv-key /etc/codesphere/secrets/age_key.txt").Return(nil)
+
+			err := bs.InstallK0s()
+			Expect(err).NotTo(HaveOccurred())
+		})
+
+		It("reports a native k0s installation failure", func() {
+			nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root", mock.MatchedBy(func(command string) bool {
+				return strings.HasPrefix(command, "oms install k0s ")
+			})).Return(fmt.Errorf("k0s error"))
+
+			err := bs.InstallK0s()
+			Expect(err).To(MatchError(ContainSubstring("failed to install k0s from jumpbox")))
+		})
+	})
+
+	Describe("EnsureCodespherePrerequisites", func() {
+		const namespaceCommand = "k0s kubectl create namespace codesphere --dry-run=client -o yaml | k0s kubectl apply -f -"
+		const serviceCommand = "k0s kubectl -n codesphere create service clusterip error-page-server --tcp=8080:8080 --dry-run=client -o yaml | k0s kubectl apply -f -"
+
+		It("idempotently creates the namespace and dummy default-backend service", func() {
+			nodeClient.EXPECT().RunCommand(bs.Env.ControlPlaneNodes[0], "root", namespaceCommand).Return(nil)
+			nodeClient.EXPECT().RunCommand(bs.Env.ControlPlaneNodes[0], "root", serviceCommand).Return(nil)
+
+			Expect(bs.EnsureCodespherePrerequisites()).To(Succeed())
+		})
+
+		It("reports namespace creation failures", func() {
+			nodeClient.EXPECT().RunCommand(bs.Env.ControlPlaneNodes[0], "root", namespaceCommand).Return(fmt.Errorf("kubectl error"))
+
+			Expect(bs.EnsureCodespherePrerequisites()).To(MatchError(ContainSubstring("failed to create Codesphere namespace")))
+		})
+
+		It("reports dummy service creation failures", func() {
+			nodeClient.EXPECT().RunCommand(bs.Env.ControlPlaneNodes[0], "root", namespaceCommand).Return(nil)
+			nodeClient.EXPECT().RunCommand(bs.Env.ControlPlaneNodes[0], "root", serviceCommand).Return(fmt.Errorf("kubectl error"))
+
+			Expect(bs.EnsureCodespherePrerequisites()).To(MatchError(ContainSubstring("failed to create dummy error-page-server service")))
+		})
+	})
+
+	Describe("WaitForK0sNodes", func() {
+		const command = "k0s kubectl wait --for=condition=Ready nodes --all --timeout=30m"
+
+		It("waits for every node before installing cluster components", func() {
+			nodeClient.EXPECT().RunCommand(bs.Env.ControlPlaneNodes[0], "root", command).Return(nil)
+
+			Expect(bs.WaitForK0sNodes()).To(Succeed())
+		})
+
+		It("reports nodes that fail to become ready", func() {
+			nodeClient.EXPECT().RunCommand(bs.Env.ControlPlaneNodes[0], "root", command).Return(fmt.Errorf("timeout"))
+
+			Expect(bs.WaitForK0sNodes()).To(MatchError(ContainSubstring("k0s nodes did not become ready")))
+		})
+	})
+
 	Describe("GenerateK0sConfigScript", func() {
 		Describe("Valid GenerateK0sConfigScript", func() {
 			It("generates script", func() {

internal/bootstrap/gcp/install_config.go

@@ -378,12 +378,14 @@ func (b *GCPBootstrapper) UpdateInstallConfig() error {
 	b.applyExternalLokiConfig()
 	b.applyPrometheusRemoteWriteConfig()
 
-	if !b.Env.ExistingConfigUsed {
-		err := b.icg.GenerateSecrets()
-		if err != nil {
-			return fmt.Errorf("failed to generate secrets: %w", err)
-		}
-	} else {
+	// Secret generation is idempotent and also backfills secrets introduced
+	// after an existing vault was created (for example the auth keys required by
+	// the ArgoCD pre-step).
+	if err := b.icg.GenerateSecrets(); err != nil {
+		return fmt.Errorf("failed to generate secrets: %w", err)
+	}
+
+	if b.Env.ExistingConfigUsed {
 		if err := b.regeneratePostgresCerts(previousPrimaryIP, previousPrimaryHostname); err != nil {
 			return err
 		}

internal/bootstrap/gcp/install_config_test.go

@@ -1072,6 +1072,7 @@ var _ = Describe("Installconfig & Secrets", func() {
 		Describe("ExistingConfigUsed", func() {
 			BeforeEach(func() {
 				csEnv.ExistingConfigUsed = true
+				icg.EXPECT().GenerateSecrets().Return(nil)
 			})
 
 			Context("with unchanged IP and existing key", func() {
@@ -1101,7 +1102,6 @@ var _ = Describe("Installconfig & Secrets", func() {
 					err := bs.UpdateInstallConfig()
 					Expect(err).NotTo(HaveOccurred())
 
-					icg.AssertNotCalled(GinkgoT(), "GenerateSecrets")
 					Expect(vault.GetSecret(files.SecretPostgresPrimaryServerKeyPem).File.Content).To(Equal(origKey))
 					Expect(bs.Env.InstallConfig.Postgres.Primary.SSLConfig.ServerCertPem).To(Equal(origCert))
 				})