diff --git a/cli/cmd/k0s/install_k0s.go b/cli/cmd/k0s/install_k0s.go index 09b95f15..56a28fde 100644 --- a/cli/cmd/k0s/install_k0s.go +++ b/cli/cmd/k0s/install_k0s.go @@ -66,7 +66,7 @@ func AddInstallCmd(install *cobra.Command, opts *util.GlobalOptions) { - Deploy k0s to all nodes defined in the install-config using k0sctl`), Example: util.FormatExamples("install k0s", []packageio.Example{ {Cmd: "--install-config ", Desc: "Path to Codesphere install-config file to generate k0s config from"}, - {Cmd: "--version ", Desc: "Version of k0s to install (e.g., v1.30.0+k0s.0)"}, + {Cmd: "--version ", Desc: "Version of k0s to install (e.g., v1.31.14+k0s.0)"}, {Cmd: "--k0sctl-version ", Desc: "Version of k0sctl to use (e.g., v0.17.4)"}, {Cmd: "--package ", Desc: "Package file (e.g. codesphere-v1.2.3-installer-lite.tar.gz) to load k0s from"}, {Cmd: "--ssh-key-path ", Desc: "SSH private key path for remote installation"}, @@ -78,7 +78,7 @@ func AddInstallCmd(install *cobra.Command, opts *util.GlobalOptions) { Env: env.NewEnv(), FileWriter: intutil.NewFilesystemWriter(), } - k0s.cmd.Flags().StringVarP(&k0s.Opts.Version, "version", "v", "", "Version of k0s to install") + k0s.cmd.Flags().StringVarP(&k0s.Opts.Version, "version", "v", installer.DefaultK0sVersion, "Version of k0s to install") k0s.cmd.Flags().StringVar(&k0s.Opts.K0sctlVersion, "k0sctl-version", "", "Version of k0sctl to use") k0s.cmd.Flags().StringVarP(&k0s.Opts.Package, "package", "p", "", "Package file (e.g. codesphere-v1.2.3-installer-lite.tar.gz) to load k0s from") k0s.cmd.Flags().StringVar(&k0s.Opts.InstallConfig, "install-config", "", "Path to Codesphere install-config file (required)") diff --git a/docs/oms_install_k0s.md b/docs/oms_install_k0s.md index 89e22328..77777349 100644 --- a/docs/oms_install_k0s.md +++ b/docs/oms_install_k0s.md @@ -22,7 +22,7 @@ oms install k0s [flags] # Path to Codesphere install-config file to generate k0s config from $ oms install k0s --install-config -# Version of k0s to install (e.g., v1.30.0+k0s.0) +# Version of k0s to install (e.g., v1.31.14+k0s.0) $ oms install k0s --version # Version of k0sctl to use (e.g., v0.17.4) @@ -54,7 +54,7 @@ $ oms install k0s --no-download --ssh-key-path string SSH private key path for remote installation --vault string Path to prod.vault.yaml to save the kubeconfig into (optional) --vault-priv-key string Path to the age private key to decrypt the vault (optional, for SOPS-encrypted vaults) - -v, --version string Version of k0s to install + -v, --version string Version of k0s to install (default "v1.31.14+k0s.0") ``` ### SEE ALSO diff --git a/internal/bootstrap/gcp/gcp.go b/internal/bootstrap/gcp/gcp.go index 19eb5c30..43ccf2f1 100644 --- a/internal/bootstrap/gcp/gcp.go +++ b/internal/bootstrap/gcp/gcp.go @@ -351,6 +351,21 @@ func (b *GCPBootstrapper) Bootstrap() error { } if b.Env.InstallVersion != "" || b.Env.InstallLocal != "" { + err = b.stlog.Step("Install k0s", b.InstallK0s) + if err != nil { + return fmt.Errorf("failed to install k0s: %w", err) + } + + err = b.stlog.Step("Wait for k0s nodes", b.WaitForK0sNodes) + if err != nil { + return fmt.Errorf("failed waiting for k0s nodes: %w", err) + } + + err = b.stlog.Step("Ensure Codesphere prerequisites", b.EnsureCodespherePrerequisites) + if err != nil { + return fmt.Errorf("failed to ensure Codesphere prerequisites: %w", err) + } + err = b.stlog.Step("Install Codesphere", b.InstallCodesphere) if err != nil { return fmt.Errorf("failed to install Codesphere: %w", err) @@ -1058,52 +1073,103 @@ func (b *GCPBootstrapper) EnsureDNSRecords() error { } func (b *GCPBootstrapper) InstallCodesphere() error { - fullPackageFilename, err := b.ensureCodespherePackageOnJumpbox() - if err != nil { + if err := b.ensureCodespherePackageOnJumpbox(); err != nil { return fmt.Errorf("failed to ensure Codesphere package on jumpbox: %w", err) } - err = b.runInstallCommand(fullPackageFilename) - if err != nil { + if err := b.runInstallCommand(b.codespherePackageFilename()); err != nil { return fmt.Errorf("failed to install Codesphere from jumpbox: %w", err) } return nil } -func (b *GCPBootstrapper) ensureCodespherePackageOnJumpbox() (string, error) { - packageFilename := "installer.tar.gz" +// InstallK0s deploys k0s with the native OMS installer and stores its +// kubeconfig in the encrypted install vault for the remaining installer steps. +func (b *GCPBootstrapper) InstallK0s() error { + // Reuse matching cached binaries and let k0sctl reconcile normally. Without + // --force, an unchanged cluster remains untouched on bootstrap retries. + installCmd := fmt.Sprintf("oms install k0s --version %s --install-config /etc/codesphere/config.yaml --vault %s --vault-priv-key %s/age_key.txt", + installer.DefaultK0sVersion, filepath.Join(b.Env.SecretsDir, "prod.vault.yaml"), b.Env.SecretsDir) + if err := b.Env.Jumpbox.RunSSHCommand("root", installCmd); err != nil { + return fmt.Errorf("failed to install k0s from jumpbox: %w", err) + } + + return nil +} + +// WaitForK0sNodes restores the readiness barrier from the TypeScript +// Kubernetes setup. k0sctl apply completing is not sufficient for the +// Codesphere charts: all schedulable nodes must be Ready before gateway +// controllers and their admission webhooks are installed. +func (b *GCPBootstrapper) WaitForK0sNodes() error { + const command = "k0s kubectl wait --for=condition=Ready nodes --all --timeout=30m" + if err := b.Env.ControlPlaneNodes[0].RunSSHCommand("root", command); err != nil { + return fmt.Errorf("k0s nodes did not become ready: %w", err) + } + return nil +} + +// EnsureCodespherePrerequisites recreates the resources normally installed by +// the skipped TypeScript Kubernetes step. The dummy error-page-server Service +// must exist before ingress-nginx starts; otherwise both gateway controllers +// exit while resolving their configured default backend. The TypeScript +// platform step removes this unmanaged Service before Helm installs the real +// one. +func (b *GCPBootstrapper) EnsureCodespherePrerequisites() error { + const namespaceCommand = "k0s kubectl create namespace codesphere --dry-run=client -o yaml | k0s kubectl apply -f -" + if err := b.Env.ControlPlaneNodes[0].RunSSHCommand("root", namespaceCommand); err != nil { + return fmt.Errorf("failed to create Codesphere namespace: %w", err) + } + + const serviceCommand = "k0s kubectl -n codesphere create service clusterip error-page-server --tcp=8080:8080 --dry-run=client -o yaml | k0s kubectl apply -f -" + if err := b.Env.ControlPlaneNodes[0].RunSSHCommand("root", serviceCommand); err != nil { + return fmt.Errorf("failed to create dummy error-page-server service: %w", err) + } + return nil +} + +func (b *GCPBootstrapper) codespherePackageFilename() string { + packageFilename := b.codespherePackageArchiveName() + if b.Env.InstallLocal != "" { + return "local-" + packageFilename + } + return portal.BuildPackageFilenameFromParts(b.Env.InstallVersion, b.Env.InstallHash, packageFilename) +} + +func (b *GCPBootstrapper) codespherePackageArchiveName() string { if b.Env.RegistryType == RegistryTypeGitHub { - packageFilename = "installer-lite.tar.gz" + return "installer-lite.tar.gz" } + return "installer.tar.gz" +} +func (b *GCPBootstrapper) ensureCodespherePackageOnJumpbox() error { if b.Env.InstallLocal != "" { b.stlog.Logf("Copying local package %s to jumpbox...", b.Env.InstallLocal) - fullPackageFilename := fmt.Sprintf("local-%s", packageFilename) - err := b.Env.Jumpbox.NodeClient.CopyFile(b.Env.Jumpbox, b.Env.InstallLocal, "/root/"+fullPackageFilename) + err := b.Env.Jumpbox.NodeClient.CopyFile(b.Env.Jumpbox, b.Env.InstallLocal, "/root/"+b.codespherePackageFilename()) if err != nil { - return "", fmt.Errorf("failed to copy local install package to jumpbox: %w", err) + return fmt.Errorf("failed to copy local install package to jumpbox: %w", err) } - return fullPackageFilename, nil + return nil } if b.Env.InstallVersion == "" { - return "", errors.New("either install version or a local package must be specified to install Codesphere") + return errors.New("either install version or a local package must be specified to install Codesphere") } - fullPackageFilename := portal.BuildPackageFilenameFromParts(b.Env.InstallVersion, b.Env.InstallHash, packageFilename) if b.Env.InstallHash == "" { - return "", fmt.Errorf("install hash must be set when install version is set") + return fmt.Errorf("install hash must be set when install version is set") } b.stlog.Logf("Downloading Codesphere package...") downloadCmd := fmt.Sprintf("oms download package -f %s -H %s %s", - packageFilename, b.Env.InstallHash, b.Env.InstallVersion) + b.codespherePackageArchiveName(), b.Env.InstallHash, b.Env.InstallVersion) err := b.Env.Jumpbox.RunSSHCommand("root", downloadCmd) if err != nil { - return "", fmt.Errorf("failed to download Codesphere package from jumpbox: %w", err) + return fmt.Errorf("failed to download Codesphere package from jumpbox: %w", err) } - return fullPackageFilename, nil + return nil } func (b *GCPBootstrapper) runInstallCommand(packageFilename string) error { @@ -1114,9 +1180,12 @@ func (b *GCPBootstrapper) runInstallCommand(packageFilename string) error { } func (b *GCPBootstrapper) generateSkipStepsArg() string { - skipSteps := b.Env.InstallSkipSteps + // k0s is installed by OMS before the TypeScript installer runs, so the + // TypeScript Kubernetes step must never run during GCP bootstrapping. + skipSteps := util.AppendUnique(nil, b.Env.InstallSkipSteps...) + skipSteps = util.AppendUnique(skipSteps, "kubernetes") if b.Env.RegistryType == RegistryTypeGitHub { - skipSteps = append(skipSteps, "load-container-images") + skipSteps = util.AppendUnique(skipSteps, "load-container-images") } if len(skipSteps) == 0 { return "" @@ -1124,7 +1193,6 @@ func (b *GCPBootstrapper) generateSkipStepsArg() string { return " -s " + strings.Join(skipSteps, ",") } - func (b *GCPBootstrapper) GenerateK0sConfigScript() error { script := `#!/bin/bash diff --git a/internal/bootstrap/gcp/gcp_test.go b/internal/bootstrap/gcp/gcp_test.go index d0d6ff15..e120e629 100644 --- a/internal/bootstrap/gcp/gcp_test.go +++ b/internal/bootstrap/gcp/gcp_test.go @@ -1395,7 +1395,7 @@ var _ = Describe("GCP Bootstrapper", func() { // Expect install codesphere nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root", - "oms install codesphere -c /etc/codesphere/config.yaml -k /etc/codesphere/secrets/age_key.txt --vault /etc/codesphere/secrets/prod.vault.yaml -p v1.2.3-abc1234567890-installer-lite.tar.gz -s load-container-images").Return(nil) + "oms install codesphere -c /etc/codesphere/config.yaml -k /etc/codesphere/secrets/age_key.txt --vault /etc/codesphere/secrets/prod.vault.yaml -p v1.2.3-abc1234567890-installer-lite.tar.gz -s kubernetes,load-container-images").Return(nil) err := bs.InstallCodesphere() Expect(err).NotTo(HaveOccurred()) @@ -1412,7 +1412,7 @@ var _ = Describe("GCP Bootstrapper", func() { nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root", "oms download package -f installer.tar.gz -H def9876543210 v1.2.3").Return(nil) // Expect install codesphere - nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root", "oms install codesphere -c /etc/codesphere/config.yaml -k /etc/codesphere/secrets/age_key.txt --vault /etc/codesphere/secrets/prod.vault.yaml -p v1.2.3-def9876543210-installer.tar.gz").Return(nil) + nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root", "oms install codesphere -c /etc/codesphere/config.yaml -k /etc/codesphere/secrets/age_key.txt --vault /etc/codesphere/secrets/prod.vault.yaml -p v1.2.3-def9876543210-installer.tar.gz -s kubernetes").Return(nil) err := bs.InstallCodesphere() Expect(err).NotTo(HaveOccurred()) @@ -1421,7 +1421,16 @@ var _ = Describe("GCP Bootstrapper", func() { It("downloads and installs codesphere with hash", func() { nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root", "oms download package -f installer.tar.gz -H abc1234567890 v1.2.3").Return(nil) - nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root", "oms install codesphere -c /etc/codesphere/config.yaml -k /etc/codesphere/secrets/age_key.txt --vault /etc/codesphere/secrets/prod.vault.yaml -p v1.2.3-abc1234567890-installer.tar.gz").Return(nil) + nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root", "oms install codesphere -c /etc/codesphere/config.yaml -k /etc/codesphere/secrets/age_key.txt --vault /etc/codesphere/secrets/prod.vault.yaml -p v1.2.3-abc1234567890-installer.tar.gz -s kubernetes").Return(nil) + + err := bs.InstallCodesphere() + Expect(err).NotTo(HaveOccurred()) + }) + + It("preserves requested skip steps without duplicating kubernetes", func() { + csEnv.InstallSkipSteps = []string{"postgres", "kubernetes"} + nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root", "oms download package -f installer.tar.gz -H abc1234567890 v1.2.3").Return(nil) + nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root", "oms install codesphere -c /etc/codesphere/config.yaml -k /etc/codesphere/secrets/age_key.txt --vault /etc/codesphere/secrets/prod.vault.yaml -p v1.2.3-abc1234567890-installer.tar.gz -s postgres,kubernetes").Return(nil) err := bs.InstallCodesphere() Expect(err).NotTo(HaveOccurred()) @@ -1440,7 +1449,7 @@ var _ = Describe("GCP Bootstrapper", func() { It("installs codesphere from local package", func() { nodeClient.EXPECT().CopyFile(mock.Anything, csEnv.InstallLocal, "/root/local-installer-lite.tar.gz").Return(nil) nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root", - "oms install codesphere -c /etc/codesphere/config.yaml -k /etc/codesphere/secrets/age_key.txt --vault /etc/codesphere/secrets/prod.vault.yaml -p local-installer-lite.tar.gz -s load-container-images").Return(nil) + "oms install codesphere -c /etc/codesphere/config.yaml -k /etc/codesphere/secrets/age_key.txt --vault /etc/codesphere/secrets/prod.vault.yaml -p local-installer-lite.tar.gz -s kubernetes,load-container-images").Return(nil) err := bs.InstallCodesphere() Expect(err).NotTo(HaveOccurred()) @@ -1454,7 +1463,7 @@ var _ = Describe("GCP Bootstrapper", func() { It("installs codesphere from local package", func() { nodeClient.EXPECT().CopyFile(mock.Anything, csEnv.InstallLocal, "/root/local-installer.tar.gz").Return(nil) nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root", - "oms install codesphere -c /etc/codesphere/config.yaml -k /etc/codesphere/secrets/age_key.txt --vault /etc/codesphere/secrets/prod.vault.yaml -p local-installer.tar.gz").Return(nil) + "oms install codesphere -c /etc/codesphere/config.yaml -k /etc/codesphere/secrets/age_key.txt --vault /etc/codesphere/secrets/prod.vault.yaml -p local-installer.tar.gz -s kubernetes").Return(nil) err := bs.InstallCodesphere() Expect(err).NotTo(HaveOccurred()) @@ -1498,7 +1507,7 @@ var _ = Describe("GCP Bootstrapper", func() { It("fails when install codesphere fails", func() { nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root", "oms download package -f installer.tar.gz -H abc1234567890 v1.2.3").Return(nil).Once() - nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root", "oms install codesphere -c /etc/codesphere/config.yaml -k /etc/codesphere/secrets/age_key.txt --vault /etc/codesphere/secrets/prod.vault.yaml -p v1.2.3-abc1234567890-installer.tar.gz").Return(fmt.Errorf("install error")).Once() + nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root", "oms install codesphere -c /etc/codesphere/config.yaml -k /etc/codesphere/secrets/age_key.txt --vault /etc/codesphere/secrets/prod.vault.yaml -p v1.2.3-abc1234567890-installer.tar.gz -s kubernetes").Return(fmt.Errorf("install error")).Once() err := bs.InstallCodesphere() Expect(err).To(HaveOccurred()) @@ -1507,6 +1516,71 @@ var _ = Describe("GCP Bootstrapper", func() { }) }) + Describe("InstallK0s", func() { + BeforeEach(func() { + csEnv.InstallVersion = "v1.2.3" + csEnv.InstallHash = "abc1234567890" + }) + + It("downloads k0s and lets k0sctl distribute it independently of the Codesphere package", func() { + nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root", + "oms install k0s --version v1.31.14+k0s.0 --install-config /etc/codesphere/config.yaml --vault /etc/codesphere/secrets/prod.vault.yaml --vault-priv-key /etc/codesphere/secrets/age_key.txt").Return(nil) + + err := bs.InstallK0s() + Expect(err).NotTo(HaveOccurred()) + }) + + It("reports a native k0s installation failure", func() { + nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root", mock.MatchedBy(func(command string) bool { + return strings.HasPrefix(command, "oms install k0s ") + })).Return(fmt.Errorf("k0s error")) + + err := bs.InstallK0s() + Expect(err).To(MatchError(ContainSubstring("failed to install k0s from jumpbox"))) + }) + }) + + Describe("EnsureCodespherePrerequisites", func() { + const namespaceCommand = "k0s kubectl create namespace codesphere --dry-run=client -o yaml | k0s kubectl apply -f -" + const serviceCommand = "k0s kubectl -n codesphere create service clusterip error-page-server --tcp=8080:8080 --dry-run=client -o yaml | k0s kubectl apply -f -" + + It("idempotently creates the namespace and dummy default-backend service", func() { + nodeClient.EXPECT().RunCommand(bs.Env.ControlPlaneNodes[0], "root", namespaceCommand).Return(nil) + nodeClient.EXPECT().RunCommand(bs.Env.ControlPlaneNodes[0], "root", serviceCommand).Return(nil) + + Expect(bs.EnsureCodespherePrerequisites()).To(Succeed()) + }) + + It("reports namespace creation failures", func() { + nodeClient.EXPECT().RunCommand(bs.Env.ControlPlaneNodes[0], "root", namespaceCommand).Return(fmt.Errorf("kubectl error")) + + Expect(bs.EnsureCodespherePrerequisites()).To(MatchError(ContainSubstring("failed to create Codesphere namespace"))) + }) + + It("reports dummy service creation failures", func() { + nodeClient.EXPECT().RunCommand(bs.Env.ControlPlaneNodes[0], "root", namespaceCommand).Return(nil) + nodeClient.EXPECT().RunCommand(bs.Env.ControlPlaneNodes[0], "root", serviceCommand).Return(fmt.Errorf("kubectl error")) + + Expect(bs.EnsureCodespherePrerequisites()).To(MatchError(ContainSubstring("failed to create dummy error-page-server service"))) + }) + }) + + Describe("WaitForK0sNodes", func() { + const command = "k0s kubectl wait --for=condition=Ready nodes --all --timeout=30m" + + It("waits for every node before installing cluster components", func() { + nodeClient.EXPECT().RunCommand(bs.Env.ControlPlaneNodes[0], "root", command).Return(nil) + + Expect(bs.WaitForK0sNodes()).To(Succeed()) + }) + + It("reports nodes that fail to become ready", func() { + nodeClient.EXPECT().RunCommand(bs.Env.ControlPlaneNodes[0], "root", command).Return(fmt.Errorf("timeout")) + + Expect(bs.WaitForK0sNodes()).To(MatchError(ContainSubstring("k0s nodes did not become ready"))) + }) + }) + Describe("GenerateK0sConfigScript", func() { Describe("Valid GenerateK0sConfigScript", func() { It("generates script", func() { diff --git a/internal/bootstrap/gcp/install_config.go b/internal/bootstrap/gcp/install_config.go index 83ebdf0a..14f4ea63 100644 --- a/internal/bootstrap/gcp/install_config.go +++ b/internal/bootstrap/gcp/install_config.go @@ -378,12 +378,14 @@ func (b *GCPBootstrapper) UpdateInstallConfig() error { b.applyExternalLokiConfig() b.applyPrometheusRemoteWriteConfig() - if !b.Env.ExistingConfigUsed { - err := b.icg.GenerateSecrets() - if err != nil { - return fmt.Errorf("failed to generate secrets: %w", err) - } - } else { + // Secret generation is idempotent and also backfills secrets introduced + // after an existing vault was created (for example the auth keys required by + // the ArgoCD pre-step). + if err := b.icg.GenerateSecrets(); err != nil { + return fmt.Errorf("failed to generate secrets: %w", err) + } + + if b.Env.ExistingConfigUsed { if err := b.regeneratePostgresCerts(previousPrimaryIP, previousPrimaryHostname); err != nil { return err } diff --git a/internal/bootstrap/gcp/install_config_test.go b/internal/bootstrap/gcp/install_config_test.go index 6ccd50d0..9a359e41 100644 --- a/internal/bootstrap/gcp/install_config_test.go +++ b/internal/bootstrap/gcp/install_config_test.go @@ -1072,6 +1072,7 @@ var _ = Describe("Installconfig & Secrets", func() { Describe("ExistingConfigUsed", func() { BeforeEach(func() { csEnv.ExistingConfigUsed = true + icg.EXPECT().GenerateSecrets().Return(nil) }) Context("with unchanged IP and existing key", func() { @@ -1101,7 +1102,6 @@ var _ = Describe("Installconfig & Secrets", func() { err := bs.UpdateInstallConfig() Expect(err).NotTo(HaveOccurred()) - icg.AssertNotCalled(GinkgoT(), "GenerateSecrets") Expect(vault.GetSecret(files.SecretPostgresPrimaryServerKeyPem).File.Content).To(Equal(origKey)) Expect(bs.Env.InstallConfig.Postgres.Primary.SSLConfig.ServerCertPem).To(Equal(origCert)) }) diff --git a/internal/installer/download.go b/internal/installer/download.go index 7a72373e..e1163ce6 100644 --- a/internal/installer/download.go +++ b/internal/installer/download.go @@ -5,24 +5,13 @@ package installer import ( "fmt" - "path/filepath" + "strings" "github.com/codesphere-cloud/oms/internal/portal" "github.com/codesphere-cloud/oms/internal/util" ) -// downloadBinary downloads a binary from downloadURL into workdir/binaryName. -// It handles workdir creation, existing binary checks, file creation, download, and chmod. -func downloadBinary(fw util.FileIO, http portal.Http, workdir, binaryName, downloadURL string, force bool, quiet bool) (string, error) { - if err := fw.MkdirAll(workdir, 0755); err != nil { - return "", fmt.Errorf("failed to create workdir: %w", err) - } - - binaryPath := filepath.Join(workdir, binaryName) - if fw.Exists(binaryPath) && !force { - return "", fmt.Errorf("%s binary already exists at %s. Use --force to overwrite", binaryName, binaryPath) - } - +func downloadBinaryToPath(fw util.FileIO, http portal.Http, binaryPath, binaryName, downloadURL string, quiet bool) (string, error) { dstFile, err := fw.Create(binaryPath) if err != nil { return "", fmt.Errorf("failed to create %s binary file: %w", binaryName, err) @@ -39,3 +28,22 @@ func downloadBinary(fw util.FileIO, http portal.Http, workdir, binaryName, downl return binaryPath, nil } + +func localBinaryVersion(binaryPath string) (string, error) { + output, err := util.RunCommandWithOutput(binaryPath, []string{"version"}, "") + if err != nil { + return "", err + } + + for _, line := range strings.Split(output, "\n") { + line = strings.TrimSpace(line) + if version, found := strings.CutPrefix(line, "version:"); found { + return strings.TrimSpace(version), nil + } + if line != "" { + return line, nil + } + } + + return "", fmt.Errorf("version output is empty") +} diff --git a/internal/installer/k0s.go b/internal/installer/k0s.go index 6ba3bf97..d70a888d 100644 --- a/internal/installer/k0s.go +++ b/internal/installer/k0s.go @@ -6,6 +6,7 @@ package installer import ( "fmt" "log" + "path/filepath" "runtime" "strings" @@ -14,6 +15,8 @@ import ( "github.com/codesphere-cloud/oms/internal/util" ) +const DefaultK0sVersion = "v1.31.14+k0s.0" + //mockery:generate: true type K0sManager interface { GetLatestVersion() (string, error) @@ -64,9 +67,31 @@ func (k *K0s) Download(version string, force bool, quiet bool) (string, error) { if err != nil { return "", fmt.Errorf("failed to determine cache directory: %w", err) } + if err := k.FileWriter.MkdirAll(cacheDir, 0755); err != nil { + return "", fmt.Errorf("failed to create workdir: %w", err) + } + + cachePath := filepath.Join(cacheDir, "k0s") + if k.FileWriter.Exists(cachePath) && !force { + cachedVersion, versionErr := localBinaryVersion(cachePath) + if versionErr == nil && cachedVersion == version { + if !quiet { + log.Printf("Using cached k0s %s at %s", version, cachePath) + } + return cachePath, nil + } + + if !quiet { + if versionErr != nil { + log.Printf("Cached k0s version could not be determined; replacing it: %v", versionErr) + } else { + log.Printf("Cached k0s version %s does not match requested version %s; replacing it", cachedVersion, version) + } + } + } downloadURL := fmt.Sprintf("https://github.com/k0sproject/k0s/releases/download/%s/k0s-%s-%s", version, version, k.Goarch) - path, err := downloadBinary(k.FileWriter, k.Http, cacheDir, "k0s", downloadURL, force, quiet) + path, err := downloadBinaryToPath(k.FileWriter, k.Http, cachePath, "k0s", downloadURL, quiet) if err != nil { return "", err } diff --git a/internal/installer/k0s_test.go b/internal/installer/k0s_test.go index 48a5a0a1..0d2e5e60 100644 --- a/internal/installer/k0s_test.go +++ b/internal/installer/k0s_test.go @@ -168,13 +168,36 @@ var _ = Describe("K0s", func() { mockFileWriter.EXPECT().MkdirAll(workDir, os.FileMode(0755)).Return(nil) }) - It("should fail when k0s binary exists and force is false", func() { + It("should reuse a cached k0s binary with the requested version", func() { + err := os.MkdirAll(workDir, 0755) + Expect(err).ToNot(HaveOccurred()) + err = os.WriteFile(k0sPath, []byte("#!/bin/sh\nprintf 'v1.29.1+k0s.0\\n'\n"), 0755) + Expect(err).ToNot(HaveOccurred()) mockFileWriter.EXPECT().Exists(k0sPath).Return(true) - _, err := k0s.Download("v1.29.1+k0s.0", false, false) - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("k0s binary already exists")) - Expect(err.Error()).To(ContainSubstring("Use --force to overwrite")) + path, err := k0s.Download("v1.29.1+k0s.0", false, false) + Expect(err).ToNot(HaveOccurred()) + Expect(path).To(Equal(k0sPath)) + }) + + It("should replace a cached k0s binary with a different version", func() { + err := os.MkdirAll(workDir, 0755) + Expect(err).ToNot(HaveOccurred()) + err = os.WriteFile(k0sPath, []byte("#!/bin/sh\nprintf 'v1.28.0+k0s.0\\n'\n"), 0755) + Expect(err).ToNot(HaveOccurred()) + mockFileWriter.EXPECT().Exists(k0sPath).Return(true) + + realFile, err := os.Create(k0sPath) + Expect(err).ToNot(HaveOccurred()) + defer util.CloseFileIgnoreError(realFile) + + mockFileWriter.EXPECT().Create(k0sPath).Return(realFile, nil) + mockHttp.EXPECT().Download("https://github.com/k0sproject/k0s/releases/download/v1.29.1+k0s.0/k0s-v1.29.1+k0s.0-amd64", realFile, false).Return(nil) + mockFileWriter.EXPECT().Chmod(k0sPath, os.FileMode(0755)).Return(nil) + + path, err := k0s.Download("v1.29.1+k0s.0", false, false) + Expect(err).ToNot(HaveOccurred()) + Expect(path).To(Equal(k0sPath)) }) It("should proceed when k0s binary exists and force is true", func() { diff --git a/internal/installer/k0sctl.go b/internal/installer/k0sctl.go index 93f17e6c..4919dd4d 100644 --- a/internal/installer/k0sctl.go +++ b/internal/installer/k0sctl.go @@ -7,6 +7,7 @@ import ( "encoding/json" "fmt" "log" + "path/filepath" "runtime" "strings" @@ -66,6 +67,14 @@ func (k *K0sctl) GetLatestVersion() (string, error) { } func (k *K0sctl) Download(version string, force bool, quiet bool) (string, error) { + cacheDir, err := k.Env.GetOmsCacheDir() + if err != nil { + return "", fmt.Errorf("failed to determine cache directory: %w", err) + } + if err := k.FileWriter.MkdirAll(cacheDir, 0755); err != nil { + return "", fmt.Errorf("failed to create workdir: %w", err) + } + if version == "" { var err error version, err = k.GetLatestVersion() @@ -77,11 +86,29 @@ func (k *K0sctl) Download(version string, force bool, quiet bool) (string, error } } - // Ensure version has v prefix for GitHub URL if !strings.HasPrefix(version, "v") { version = "v" + version } + cachePath := filepath.Join(cacheDir, "k0sctl") + if k.FileWriter.Exists(cachePath) && !force { + cachedVersion, versionErr := localBinaryVersion(cachePath) + if versionErr == nil && cachedVersion == version { + if !quiet { + log.Printf("Using cached k0sctl %s at %s", version, cachePath) + } + return cachePath, nil + } + + if !quiet { + if versionErr != nil { + log.Printf("Cached k0sctl version could not be determined; replacing it: %v", versionErr) + } else { + log.Printf("Cached k0sctl version %s does not match requested version %s; replacing it", cachedVersion, version) + } + } + } + binaryName := fmt.Sprintf("k0sctl-%s-%s", k.Goos, k.Goarch) downloadURL := fmt.Sprintf("https://github.com/k0sproject/k0sctl/releases/download/%s/%s", version, binaryName) @@ -89,12 +116,7 @@ func (k *K0sctl) Download(version string, force bool, quiet bool) (string, error log.Printf("Downloading k0sctl %s from %s", version, downloadURL) } - cacheDir, err := k.Env.GetOmsCacheDir() - if err != nil { - return "", fmt.Errorf("failed to determine cache directory: %w", err) - } - - path, err := downloadBinary(k.FileWriter, k.Http, cacheDir, "k0sctl", downloadURL, force, quiet) + path, err := downloadBinaryToPath(k.FileWriter, k.Http, cachePath, "k0sctl", downloadURL, quiet) if err != nil { return "", err } diff --git a/internal/installer/k0sctl_config.go b/internal/installer/k0sctl_config.go index b7a6dc88..29c46d24 100644 --- a/internal/installer/k0sctl_config.go +++ b/internal/installer/k0sctl_config.go @@ -123,12 +123,26 @@ func GenerateK0sctlConfig(installConfig *files.RootConfig, k0sVersion string, ss }, } - // Track added IPs to avoid duplicates + // A node may intentionally be listed as both a control plane and a worker. + // The TypeScript installer preserves that topology by installing k0s on the + // controller with --enable-worker and --no-taints. Keep the same behaviour + // here instead of silently turning overlapping nodes into controller-only + // nodes when deduplicating the host list. + workerIPs := make(map[string]bool, len(installConfig.Kubernetes.Workers)) + for _, worker := range installConfig.Kubernetes.Workers { + workerIPs[worker.IPAddress] = true + } + + // Track added IPs to avoid emitting the same host twice. addedIPs := make(map[string]bool) - // Add controller-only nodes from control planes + // Add control-plane nodes, enabling their worker role when configured. for _, cp := range installConfig.Kubernetes.ControlPlanes { - host := createK0sctlHost(cp, "controller", nil, sshKeyPath, k0sBinaryPath) + var installFlags []string + if workerIPs[cp.IPAddress] { + installFlags = []string{"--enable-worker", "--no-taints=true"} + } + host := createK0sctlHost(cp, "controller", installFlags, sshKeyPath, k0sBinaryPath) k0sctlConfig.Spec.Hosts = append(k0sctlConfig.Spec.Hosts, host) addedIPs[cp.IPAddress] = true } diff --git a/internal/installer/k0sctl_config_test.go b/internal/installer/k0sctl_config_test.go index 72654d4f..55ca856a 100644 --- a/internal/installer/k0sctl_config_test.go +++ b/internal/installer/k0sctl_config_test.go @@ -69,7 +69,7 @@ var _ = Describe("K0sctlConfig", func() { Expect(k0sctlConfig.Spec.Hosts[2].Role).To(Equal("worker")) }) - It("should skip duplicate IPs between control planes and workers", func() { + It("should enable the worker role on control planes also listed as workers", func() { installConfig := newTestConfig("test-dc", true, "10.0.1.10") installConfig.Kubernetes.Workers = []files.K8sNode{ {IPAddress: "10.0.1.10"}, // Duplicate @@ -79,10 +79,12 @@ var _ = Describe("K0sctlConfig", func() { k0sctlConfig, err := installer.GenerateK0sctlConfig(installConfig, "v1.30.0+k0s.0", "/path/to/key", "") Expect(err).ToNot(HaveOccurred()) - // Should only have 2 hosts: 1 control plane + 1 unique worker + // The overlapping node is emitted once, but retains both roles in + // the same way as the TypeScript installer. Expect(k0sctlConfig.Spec.Hosts).To(HaveLen(2)) Expect(k0sctlConfig.Spec.Hosts[0].SSH.Address).To(Equal("10.0.1.10")) Expect(k0sctlConfig.Spec.Hosts[0].Role).To(Equal("controller")) + Expect(k0sctlConfig.Spec.Hosts[0].InstallFlags).To(Equal([]string{"--enable-worker", "--no-taints=true"})) Expect(k0sctlConfig.Spec.Hosts[1].SSH.Address).To(Equal("10.0.2.10")) Expect(k0sctlConfig.Spec.Hosts[1].Role).To(Equal("worker")) }) diff --git a/internal/installer/k0sctl_test.go b/internal/installer/k0sctl_test.go new file mode 100644 index 00000000..0aec57ad --- /dev/null +++ b/internal/installer/k0sctl_test.go @@ -0,0 +1,90 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +package installer_test + +import ( + "os" + "path/filepath" + + "github.com/codesphere-cloud/oms/internal/env" + "github.com/codesphere-cloud/oms/internal/installer" + "github.com/codesphere-cloud/oms/internal/portal" + "github.com/codesphere-cloud/oms/internal/util" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("K0sctl", func() { + var ( + mockEnv *env.MockEnv + mockHTTP *portal.MockHttp + mockFileWriter *util.MockFileIO + cacheDir string + cachedPath string + k0sctl *installer.K0sctl + ) + + BeforeEach(func() { + mockEnv = env.NewMockEnv(GinkgoT()) + mockHTTP = portal.NewMockHttp(GinkgoT()) + mockFileWriter = util.NewMockFileIO(GinkgoT()) + cacheDir = GinkgoT().TempDir() + cachedPath = filepath.Join(cacheDir, "k0sctl") + + mockEnv.EXPECT().GetOmsCacheDir().Return(cacheDir, nil) + mockFileWriter.EXPECT().MkdirAll(cacheDir, os.FileMode(0755)).Return(nil) + + k0sctl = installer.NewK0sctl(mockHTTP, mockEnv, mockFileWriter) + k0sctl.Goos = "linux" + k0sctl.Goarch = "amd64" + }) + + writeCachedVersion := func(version string) { + script := "#!/bin/sh\nprintf 'version: " + version + "\\ncommit: test\\n'\n" + Expect(os.WriteFile(cachedPath, []byte(script), 0755)).To(Succeed()) + } + + expectDownload := func(version string) { + url := "https://github.com/k0sproject/k0sctl/releases/download/" + version + "/k0sctl-linux-amd64" + downloadFile, err := os.CreateTemp(cacheDir, "k0sctl-download") + Expect(err).NotTo(HaveOccurred()) + mockFileWriter.EXPECT().Create(cachedPath).Return(downloadFile, nil) + mockHTTP.EXPECT().Download(url, downloadFile, false).Return(nil) + mockFileWriter.EXPECT().Chmod(cachedPath, os.FileMode(0755)).Return(nil) + } + + It("reuses a cached binary with the requested version", func() { + writeCachedVersion("v0.32.1") + mockFileWriter.EXPECT().Exists(cachedPath).Return(true) + + path, err := k0sctl.Download("v0.32.1", false, false) + + Expect(err).NotTo(HaveOccurred()) + Expect(path).To(Equal(cachedPath)) + }) + + It("replaces a cached binary with a different version without force", func() { + writeCachedVersion("v0.31.0") + mockFileWriter.EXPECT().Exists(cachedPath).Return(true) + expectDownload("v0.32.1") + + path, err := k0sctl.Download("v0.32.1", false, false) + + Expect(err).NotTo(HaveOccurred()) + Expect(path).To(Equal(cachedPath)) + }) + + It("resolves an unpinned version and reuses the matching binary", func() { + writeCachedVersion("v0.32.1") + mockHTTP.EXPECT().Get("https://api.github.com/repos/k0sproject/k0sctl/releases/latest"). + Return([]byte(`{"tag_name":"v0.32.1"}`), nil) + mockFileWriter.EXPECT().Exists(cachedPath).Return(true) + + path, err := k0sctl.Download("", false, false) + + Expect(err).NotTo(HaveOccurred()) + Expect(path).To(Equal(cachedPath)) + }) + +}) diff --git a/internal/installer/vault/vault_encryption_test.go b/internal/installer/vault/vault_encryption_test.go index ba392b56..7045a68f 100644 --- a/internal/installer/vault/vault_encryption_test.go +++ b/internal/installer/vault/vault_encryption_test.go @@ -226,7 +226,7 @@ var _ = Describe("VaultEncryption", func() { plainYAML := "secrets:\n - name: test-secret\n fields:\n password: hunter2\n" Expect(os.WriteFile(vaultPath, []byte(plainYAML), 0644)).To(Succeed()) - vault, err := vault.LoadVaultData(vaultPath, "") + vault, err := vault.LoadUnencryptedVaultData(vaultPath) Expect(err).ToNot(HaveOccurred()) Expect(vault.Secrets).To(HaveLen(1)) Expect(vault.Secrets[0].Name).To(Equal("test-secret")) @@ -238,7 +238,7 @@ var _ = Describe("VaultEncryption", func() { wrappedYAML := "data: |\n secrets:\n - name: test-secret\n fields:\n password: hunter2\n" Expect(os.WriteFile(vaultPath, []byte(wrappedYAML), 0644)).To(Succeed()) - vault, err := vault.LoadVaultData(vaultPath, "") + vault, err := vault.LoadUnencryptedVaultData(vaultPath) Expect(err).ToNot(HaveOccurred()) Expect(vault.Secrets).To(HaveLen(1)) Expect(vault.Secrets[0].Name).To(Equal("test-secret")) diff --git a/internal/installer/vault/vault_secret_creator.go b/internal/installer/vault/vault_secret_creator.go index 67951a5a..20938e84 100644 --- a/internal/installer/vault/vault_secret_creator.go +++ b/internal/installer/vault/vault_secret_creator.go @@ -65,6 +65,10 @@ func (v *VaultSecretCreator) CreateSecretFromVault(ctx context.Context, vault *f return err } + if err := v.ensureNamespace(ctx, namespace); err != nil { + return err + } + secret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: secretName, @@ -85,6 +89,16 @@ func (v *VaultSecretCreator) CreateSecretFromVault(ctx context.Context, vault *f return nil } +func (v *VaultSecretCreator) ensureNamespace(ctx context.Context, namespace string) error { + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: namespace}, + } + if _, err := controllerutil.CreateOrUpdate(ctx, v.client, ns, func() error { return nil }); err != nil { + return fmt.Errorf("failed to ensure namespace %q: %w", namespace, err) + } + return nil +} + // vaultToSecretData converts the entries of an InstallVault into Kubernetes secret data. // File entries produce a single key equal to the entry name containing the file content. // Field entries produce "entryName.password" and, when a username is present, "entryName.username". diff --git a/internal/installer/vault/vault_secret_creator_test.go b/internal/installer/vault/vault_secret_creator_test.go index a2dc9974..891ad1be 100644 --- a/internal/installer/vault/vault_secret_creator_test.go +++ b/internal/installer/vault/vault_secret_creator_test.go @@ -4,11 +4,66 @@ package vault import ( + "context" + "github.com/codesphere-cloud/oms/internal/installer/files" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" ) +var _ = Describe("VaultSecretCreator", func() { + var ( + ctx context.Context + kubeClient ctrlclient.Client + creator *VaultSecretCreator + ) + + BeforeEach(func() { + ctx = context.Background() + scheme := runtime.NewScheme() + Expect(corev1.AddToScheme(scheme)).To(Succeed()) + kubeClient = fake.NewClientBuilder().WithScheme(scheme).Build() + creator = NewVaultSecretCreator(kubeClient) + }) + + It("creates the target namespace before syncing the vault secret", func() { + installVault := &files.InstallVault{Secrets: []files.SecretEntry{ + {Name: "registry", Fields: &files.SecretFields{Password: "first"}}, + }} + + Expect(creator.CreateSecretFromVault(ctx, installVault, VaultSecretNamespace, VaultSecretName)).To(Succeed()) + + namespace := &corev1.Namespace{} + Expect(kubeClient.Get(ctx, ctrlclient.ObjectKey{Name: VaultSecretNamespace}, namespace)).To(Succeed()) + + secret := &corev1.Secret{} + Expect(kubeClient.Get(ctx, ctrlclient.ObjectKey{Namespace: VaultSecretNamespace, Name: VaultSecretName}, secret)).To(Succeed()) + Expect(secret.Data).To(HaveKeyWithValue("registry.password", []byte("first"))) + }) + + It("keeps namespace creation idempotent when updating the vault secret", func() { + namespace := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: VaultSecretNamespace}} + Expect(kubeClient.Create(ctx, namespace)).To(Succeed()) + + installVault := &files.InstallVault{Secrets: []files.SecretEntry{ + {Name: "registry", Fields: &files.SecretFields{Password: "first"}}, + }} + Expect(creator.CreateSecretFromVault(ctx, installVault, VaultSecretNamespace, VaultSecretName)).To(Succeed()) + + installVault.SetSecret(files.SecretEntry{Name: "registry", Fields: &files.SecretFields{Password: "updated"}}) + Expect(creator.CreateSecretFromVault(ctx, installVault, VaultSecretNamespace, VaultSecretName)).To(Succeed()) + + secret := &corev1.Secret{} + Expect(kubeClient.Get(ctx, ctrlclient.ObjectKey{Namespace: VaultSecretNamespace, Name: VaultSecretName}, secret)).To(Succeed()) + Expect(secret.Data).To(HaveKeyWithValue("registry.password", []byte("updated"))) + }) +}) + var _ = Describe("vaultToSecretData", func() { It("stores file entry content under the entry name", func() { vault := &files.InstallVault{ diff --git a/internal/installer/vault/vault_suite_test.go b/internal/installer/vault/vault_suite_test.go new file mode 100644 index 00000000..7ad67417 --- /dev/null +++ b/internal/installer/vault/vault_suite_test.go @@ -0,0 +1,16 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +package vault + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestVault(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Vault Suite") +} diff --git a/internal/util/slice.go b/internal/util/slice.go new file mode 100644 index 00000000..49f011e5 --- /dev/null +++ b/internal/util/slice.go @@ -0,0 +1,22 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +package util + +// AppendUnique appends values that are not already present, preserving the +// order of the original slice and the additions. +func AppendUnique[T comparable](values []T, additions ...T) []T { + for _, addition := range additions { + found := false + for _, value := range values { + if value == addition { + found = true + break + } + } + if !found { + values = append(values, addition) + } + } + return values +} diff --git a/internal/util/slice_test.go b/internal/util/slice_test.go new file mode 100644 index 00000000..19d3a1af --- /dev/null +++ b/internal/util/slice_test.go @@ -0,0 +1,26 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +package util_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/codesphere-cloud/oms/internal/util" +) + +var _ = Describe("AppendUnique", func() { + It("appends only values that are not already present", func() { + Expect(util.AppendUnique([]string{"one", "two"}, "two", "three", "three")). + To(Equal([]string{"one", "two", "three"})) + }) + + It("preserves a nil slice when there are no additions", func() { + Expect(util.AppendUnique[string](nil)).To(BeNil()) + }) + + It("supports comparable non-string values", func() { + Expect(util.AppendUnique([]int{1}, 2, 1)).To(Equal([]int{1, 2})) + }) +})