From 9bfd18f7a62eac83dccfd391d08fef93d0ee8374 Mon Sep 17 00:00:00 2001 From: Brandon Palm Date: Wed, 15 Jul 2026 11:36:53 -0500 Subject: [PATCH] CM-716: Add HTTP01 Challenge Proxy binary and CI Dockerfile Adds the HTTP01 Challenge Proxy binary source code and CI build infrastructure. The proxy runs as a DaemonSet on baremetal control-plane nodes, redirecting HTTP-01 ACME challenge traffic from the API VIP to the ingress router via NFTables rules and a Go reverse proxy. This is split out from the full HTTP01Proxy controller PR to allow the image build pipeline to start early while the controller is reviewed. New files: - cmd/http01-proxy/ - proxy binary source (~200 LOC) - images/ci/http01proxy.Dockerfile - FIPS-compliant CI image build - Makefile targets: build-http01-proxy, image-build-http01-proxy --- .gitignore | 1 + Makefile | 9 + cmd/http01-proxy/main.go | 75 ++++++++ cmd/http01-proxy/pkg/utils.go | 242 ++++++++++++++++++++++++ cmd/http01-proxy/templates/templates.go | 78 ++++++++ images/ci/http01proxy.Dockerfile | 15 ++ 6 files changed, 420 insertions(+) create mode 100644 cmd/http01-proxy/main.go create mode 100644 cmd/http01-proxy/pkg/utils.go create mode 100644 cmd/http01-proxy/templates/templates.go create mode 100644 images/ci/http01proxy.Dockerfile diff --git a/.gitignore b/.gitignore index d6495a05c..7c9713113 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ *.out /cert-manager-operator +/http01-proxy # Log output from telepresence telepresence.log diff --git a/Makefile b/Makefile index 0caec65d0..07e3e7a5e 100644 --- a/Makefile +++ b/Makefile @@ -60,6 +60,7 @@ endif CERT_MANAGER_VERSION ?= v1.19.4 ISTIO_CSR_VERSION ?= v0.16.0 TRUST_MANAGER_VERSION ?= v0.20.3 +HTTP01PROXY_VERSION ?= v0.1.0 # --- Test Versions --- @@ -345,6 +346,10 @@ build: generate fmt vet build-operator ## Build operator binary with all checks build-operator: ## Build operator binary only (no checks or code generation). @GOFLAGS="-mod=vendor" source hack/go-fips.sh && $(GO) build $(GOBUILD_VERSION_ARGS) -o $(BIN) +.PHONY: build-http01-proxy +build-http01-proxy: ## Build HTTP01 proxy binary. + @GOFLAGS="-mod=vendor" source hack/go-fips.sh && $(GO) build $(GOBUILD_VERSION_ARGS) -o $(PROJECT_ROOT)/http01-proxy ./cmd/http01-proxy + .PHONY: run run: manifests generate fmt vet ## Run the operator from your host (for development). go run $(PACKAGE) @@ -357,6 +362,10 @@ image-build: ## Build container image with the operator. image-push: ## Push container image with the operator. $(CONTAINER_ENGINE) push $(IMG) $(CONTAINER_PUSH_ARGS) +.PHONY: image-build-http01-proxy +image-build-http01-proxy: ## Build HTTP01 proxy container image. + $(CONTAINER_ENGINE) build -t cert-manager-http01-proxy:$(HTTP01PROXY_VERSION) -f images/ci/http01proxy.Dockerfile . + # ============================================================================ # Deployment # ============================================================================ diff --git a/cmd/http01-proxy/main.go b/cmd/http01-proxy/main.go new file mode 100644 index 000000000..d7a6d628a --- /dev/null +++ b/cmd/http01-proxy/main.go @@ -0,0 +1,75 @@ +package main + +import ( + "context" + "log" + "net/http" + "os" + "strconv" + "strings" + "time" + + utils "github.com/openshift/cert-manager-operator/cmd/http01-proxy/pkg" +) + +const defaultPort = "8888" + +func main() { + ctx := context.Background() + + env, err := utils.GetOCPEnvDetails(ctx) + if err != nil { + log.Fatalf("Error getting OCP environment details: %v", err) + } + log.Printf("OCP API: %s, API VIP: %s, APPS VIP: %s, Platform: %s, Version: %s", + env.APIHostname, env.APIVIP, env.AppsVIP, env.PlatformType, env.ClusterVersion) + + if err := utils.SupportedOCPVersion(env.ClusterVersion); err != nil { + log.Fatalf("Detected non-supported version: %v", err) + } + + if env.AppsVIP == env.APIVIP { + log.Printf("API VIP and APPS VIP are equal, no proxy needed") + os.Exit(0) + } + + port := os.Getenv("PROXY_PORT") + if port == "" { + port = defaultPort + } + if p, err := strconv.Atoi(port); err != nil || p < 1 || p > 65535 { + log.Fatalf("Invalid PROXY_PORT %q: must be a number between 1 and 65535", port) + } + + if err := utils.CreateNFTablesRuleMachineConfig(ctx, env.Client, env.APIVIP, port); err != nil { + log.Fatalf("Error creating nft rules machineconfig: %v", err) + } + log.Println("NFTables Rules MachineConfig created/updated") + + backendServer := "http://" + env.AppsVIP + ":80" + proxy, err := utils.NewReverseProxy(backendServer) + if err != nil { + log.Fatalf("Error creating reverse proxy: %v", err) + } + + http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + if strings.HasPrefix(r.URL.Path, "/.well-known/acme-challenge/") { + log.Printf("Forwarding request to APPS VIP: %s", r.URL.Path) + proxy.ServeHTTP(w, r) + } else { + http.Error(w, "Forbidden: Only /.well-known/acme-challenge/* is allowed", http.StatusForbidden) + } + }) + + server := &http.Server{ + Addr: ":" + port, + ReadHeaderTimeout: 10 * time.Second, + ReadTimeout: 15 * time.Second, + WriteTimeout: 15 * time.Second, + IdleTimeout: 60 * time.Second, + } + log.Printf("Reverse proxy listening on :%s, forwarding http01 challenges for %s to %s", port, env.APIHostname, backendServer) + if err := server.ListenAndServe(); err != nil { + log.Fatalf("Error starting proxy: %v", err) + } +} diff --git a/cmd/http01-proxy/pkg/utils.go b/cmd/http01-proxy/pkg/utils.go new file mode 100644 index 000000000..89727722c --- /dev/null +++ b/cmd/http01-proxy/pkg/utils.go @@ -0,0 +1,242 @@ +package utils + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "fmt" + "log" + "net" + "net/http" + "net/http/httputil" + "net/url" + "strconv" + "strings" + "text/template" + + "github.com/openshift/cert-manager-operator/cmd/http01-proxy/templates" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + yamlserializer "k8s.io/apimachinery/pkg/runtime/serializer/yaml" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/rest" +) + +var ( + infrastructureGVR = schema.GroupVersionResource{Group: "config.openshift.io", Version: "v1", Resource: "infrastructures"} + ingressGVR = schema.GroupVersionResource{Group: "config.openshift.io", Version: "v1", Resource: "ingresses"} + clusterVersionGVR = schema.GroupVersionResource{Group: "config.openshift.io", Version: "v1", Resource: "clusterversions"} + machineConfigGVR = schema.GroupVersionResource{Group: "machineconfiguration.openshift.io", Version: "v1", Resource: "machineconfigs"} + machineConfigurationGVR = schema.GroupVersionResource{Group: "operator.openshift.io", Version: "v1", Resource: "machineconfigurations"} +) + +// OCPEnvironment holds the discovered OpenShift cluster details needed by the proxy. +type OCPEnvironment struct { + APIHostname string + AppsVIP string + APIVIP string + PlatformType string + ClusterVersion string + Client *dynamic.DynamicClient +} + +func NewReverseProxy(targetURL string) (*httputil.ReverseProxy, error) { + target, err := url.Parse(targetURL) + if err != nil { + return nil, fmt.Errorf("failed to parse target URL %q: %w", targetURL, err) + } + + proxy := httputil.NewSingleHostReverseProxy(target) + proxy.Director = func(req *http.Request) { + req.URL.Scheme = target.Scheme + req.URL.Host = target.Host + req.Header.Set("X-Proxy-Server", "cert-manager-http01-proxy") + } + proxy.ErrorHandler = func(w http.ResponseWriter, req *http.Request, err error) { + log.Printf("Error handling request: %v", err) + http.Error(w, "Backend unavailable", http.StatusBadGateway) + } + + return proxy, nil +} + +func newKubeClient() (*dynamic.DynamicClient, error) { + config, err := rest.InClusterConfig() + if err != nil { + return nil, fmt.Errorf("failed to get in-cluster config: %w", err) + } + + client, err := dynamic.NewForConfig(config) + if err != nil { + return nil, fmt.Errorf("failed to create dynamic client: %w", err) + } + + return client, nil +} + +func GetOCPEnvDetails(ctx context.Context) (*OCPEnvironment, error) { + client, err := newKubeClient() + if err != nil { + return nil, err + } + + infrastructureData, err := client.Resource(infrastructureGVR).Get(ctx, "cluster", metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to get infrastructure: %w", err) + } + platformType, found, err := unstructured.NestedString(infrastructureData.Object, "status", "platform") + if err != nil { + return nil, fmt.Errorf("failed to read platform type: %w", err) + } + if !found { + return nil, fmt.Errorf("platform type not found in infrastructure status") + } + + ingressData, err := client.Resource(ingressGVR).Get(ctx, "cluster", metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to get ingress: %w", err) + } + ingressDomain, found, err := unstructured.NestedString(ingressData.Object, "spec", "domain") + if err != nil { + return nil, fmt.Errorf("failed to read ingress domain: %w", err) + } + if !found { + return nil, fmt.Errorf("ingress domain not found in ingress spec") + } + + clusterVersionData, err := client.Resource(clusterVersionGVR).Get(ctx, "version", metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to get clusterversion: %w", err) + } + clusterVersion, found, err := unstructured.NestedString(clusterVersionData.Object, "status", "desired", "version") + if err != nil { + return nil, fmt.Errorf("failed to read cluster version: %w", err) + } + if !found { + return nil, fmt.Errorf("desired version not found in clusterversion status") + } + + suffix, ok := strings.CutPrefix(ingressDomain, "apps.") + if !ok { + return nil, fmt.Errorf("ingress domain %q does not start with \"apps.\"", ingressDomain) + } + apiHostname := "api." + suffix + + // Resolve a subdomain under the wildcard *.apps record to get the Apps VIP + appsIPs, err := resolveDNSRecord(ctx, "test."+ingressDomain) + if err != nil { + return nil, fmt.Errorf("failed to resolve apps domain: %w", err) + } + + apiIPs, err := resolveDNSRecord(ctx, apiHostname) + if err != nil { + return nil, fmt.Errorf("failed to resolve api hostname: %w", err) + } + + return &OCPEnvironment{ + APIHostname: apiHostname, + AppsVIP: appsIPs[0].String(), + APIVIP: apiIPs[0].String(), + PlatformType: platformType, + ClusterVersion: clusterVersion, + Client: client, + }, nil +} + +func CreateNFTablesRuleMachineConfig(ctx context.Context, client *dynamic.DynamicClient, apiVIP, port string) error { + data := templates.TemplateData{ + APIVIP: apiVIP, + ProxyPort: port, + } + + tmpl, err := template.New("nftables").Parse(templates.NFTRuleTemplate) + if err != nil { + return fmt.Errorf("failed to parse nftables template: %w", err) + } + var nftablesResult bytes.Buffer + if err := tmpl.Execute(&nftablesResult, data); err != nil { + return fmt.Errorf("failed to execute nftables template: %w", err) + } + data.NFTRules = base64.StdEncoding.EncodeToString(nftablesResult.Bytes()) + + tmpl, err = template.New("machineconfig").Parse(templates.MachineConfigTemplate) + if err != nil { + return fmt.Errorf("failed to parse machineconfig template: %w", err) + } + var machineConfigResult bytes.Buffer + if err := tmpl.Execute(&machineConfigResult, data); err != nil { + return fmt.Errorf("failed to execute machineconfig template: %w", err) + } + + fieldManager := "cert-manager-http01-proxy" + + if err := applyManifest(ctx, client, []byte(templates.MachineConfigurationManifest), machineConfigurationGVR, fieldManager); err != nil { + return err + } + + if err := applyManifest(ctx, client, machineConfigResult.Bytes(), machineConfigGVR, fieldManager); err != nil { + return err + } + + return nil +} + +func applyManifest(ctx context.Context, client *dynamic.DynamicClient, manifest []byte, gvr schema.GroupVersionResource, fieldManager string) error { + decoder := yamlserializer.NewDecodingSerializer(unstructured.UnstructuredJSONScheme) + obj := &unstructured.Unstructured{} + if _, _, err := decoder.Decode(manifest, nil, obj); err != nil { + return fmt.Errorf("failed to decode %s: %w", gvr.Resource, err) + } + + objData, err := json.Marshal(obj) + if err != nil { + return fmt.Errorf("failed to marshal %s: %w", gvr.Resource, err) + } + + if _, err := client.Resource(gvr).Patch( + ctx, + obj.GetName(), + types.ApplyPatchType, + objData, + metav1.PatchOptions{FieldManager: fieldManager}, + ); err != nil { + return fmt.Errorf("failed to apply %s: %w", gvr.Resource, err) + } + + return nil +} + +func SupportedOCPVersion(runningVersion string) error { + version := strings.Split(runningVersion, ".") + if len(version) < 3 { + return fmt.Errorf("invalid OCP version %q (expecting X.Y.Z format)", runningVersion) + } + major, err := strconv.Atoi(version[0]) + if err != nil { + return fmt.Errorf("invalid OCP major version in %q: %w", runningVersion, err) + } + minor, err := strconv.Atoi(version[1]) + if err != nil { + return fmt.Errorf("invalid OCP minor version in %q: %w", runningVersion, err) + } + if major < 4 || (major == 4 && minor < 17) { + return fmt.Errorf("unsupported OCP version %q (minimum supported is 4.17+)", runningVersion) + } + return nil +} + +func resolveDNSRecord(ctx context.Context, hostname string) ([]net.IP, error) { + var r net.Resolver + ips, err := r.LookupIP(ctx, "ip4", hostname) + if err != nil { + return nil, fmt.Errorf("failed to resolve %q: %w", hostname, err) + } + if len(ips) == 0 { + return nil, fmt.Errorf("no IPs found for %q", hostname) + } + return ips, nil +} diff --git a/cmd/http01-proxy/templates/templates.go b/cmd/http01-proxy/templates/templates.go new file mode 100644 index 000000000..686bf203c --- /dev/null +++ b/cmd/http01-proxy/templates/templates.go @@ -0,0 +1,78 @@ +package templates + +type TemplateData struct { + APIVIP string + ProxyPort string + NFTRules string +} + +const NFTRuleTemplate = ` +table inet crtmgr_proxy_table +delete table inet crtmgr_proxy_table +table inet crtmgr_proxy_table { + chain crtmgr_proxy_PREROUTING { + type nat hook prerouting priority 0; + # Redirect to proxy port + ip daddr {{ .APIVIP }} tcp dport 80 redirect to {{ .ProxyPort }} + } +}` + +const MachineConfigurationManifest = ` +apiVersion: operator.openshift.io/v1 +kind: MachineConfiguration +metadata: + name: cluster +spec: + nodeDisruptionPolicy: + files: + - actions: + - restart: + serviceName: nftables.service + type: Restart + path: /etc/sysconfig/nftables.conf + units: + - actions: + - type: DaemonReload + - type: Reload + reload: + serviceName: nftables.service + name: nftables.service` + +const MachineConfigTemplate = ` +apiVersion: machineconfiguration.openshift.io/v1 +kind: MachineConfig +metadata: + labels: + machineconfiguration.openshift.io/role: master + name: 98-nftables-crtmgr-proxy +spec: + config: + ignition: + version: 3.4.0 + storage: + files: + - contents: + source: data:text/plain;charset=utf-8;base64,{{ .NFTRules }} + mode: 384 + overwrite: true + path: /etc/sysconfig/nftables.conf + systemd: + units: + - contents: | + [Unit] + Description=Netfilter Tables + Documentation=man:nft(8) + Wants=network-pre.target + Before=network-pre.target + [Service] + Type=oneshot + ProtectSystem=full + ProtectHome=true + ExecStart=/sbin/nft -f /etc/sysconfig/nftables.conf + ExecReload=/sbin/nft -f /etc/sysconfig/nftables.conf + ExecStop=/sbin/nft 'add table inet crtmgr_proxy_table; delete table inet crtmgr_proxy_table' + RemainAfterExit=yes + [Install] + WantedBy=multi-user.target + enabled: true + name: nftables.service` diff --git a/images/ci/http01proxy.Dockerfile b/images/ci/http01proxy.Dockerfile new file mode 100644 index 000000000..fbb692694 --- /dev/null +++ b/images/ci/http01proxy.Dockerfile @@ -0,0 +1,15 @@ +FROM registry.ci.openshift.org/ocp/builder:rhel-9-golang-1.26-openshift-4.23 AS builder +WORKDIR /go/src/github.com/openshift/cert-manager-operator + +ARG GO_BUILD_TAGS=strictfipsruntime,openssl +ENV GOEXPERIMENT=strictfipsruntime +ENV CGO_ENABLED=1 + +COPY . . +RUN go build -mod=vendor -tags $GO_BUILD_TAGS -ldflags '-w -s' \ + -o /app/cert-manager-http01-proxy ./cmd/http01-proxy + +FROM registry.access.redhat.com/ubi9-minimal@sha256:062c52ff973065752b0965787649db2bcf551a6c727a00e95a3eb42cebadbdab +COPY --from=builder /app/cert-manager-http01-proxy /usr/local/bin/cert-manager-http01-proxy +USER 65532:65532 +ENTRYPOINT ["/usr/local/bin/cert-manager-http01-proxy"]