feat(stacks): karpenter node isolation per organization/stack

api/formance.com/v1beta1/settings_types.go

@@ -96,6 +96,43 @@ type SettingsSpec struct {
 // Refer to the documentation of each module and resource to discover available Settings.
 //
 // ##### Global settings
+// ###### Karpenter node isolation
+//
+// The operator can provision dedicated Karpenter node pools per organization (or per
+// organization+stack) so that EC2 instances carry customer/organization/stack tags for
+// cost attribution, and so a stack's workloads run only on its dedicated nodes. The
+// organization and customer identity are read from the Stack labels
+// `formance.com/organization` and `formance.com/customer` (customer defaults to the
+// organization when absent).
+//
+// Available keys:
+//   - `karpenter.enabled` (bool): master switch. Requires the Karpenter CRDs to be installed.
+//   - `karpenter.isolation` (`organization` | `stack`, default `organization`): chooses the
+//     pool naming — a shared `<organization>` pool, or a dedicated `<organization>-<stack>` pool.
+//   - `karpenter.ec2-node-class.reference` (string, required when enabled): name of the
+//     cluster-scoped reference EC2NodeClass to clone.
+//   - `karpenter.node-pool.reference` (string, required when enabled): name of the
+//     cluster-scoped reference NodePool to clone.
+//   - `karpenter.api.ec2-node-class.group-version` (string, default `karpenter.k8s.aws/v1`)
+//     and `karpenter.api.node-pool.group-version` (string, default `karpenter.sh/v1`):
+//     override the Karpenter API group/version.
+//
+// ```yaml
+// apiVersion: formance.com/v1beta1
+// kind: Settings
+// metadata:
+//
+//	name: karpenter-enabled
+//
+// spec:
+//
+//	key: karpenter.enabled
+//	stacks:
+//	- '*'
+//	value: "true"
+//
+// ```
+//
 // ###### AWS account
 //
 // A stack can use an AWS account for authentication.

api/formance.com/v1beta1/shared.go

@@ -392,4 +392,9 @@ const (
 	StackLabel          = "formance.com/stack"
 	SkipLabel           = "formance.com/skip"
 	CreatedByAgentLabel = "formance.com/created-by-agent"
+	// OrganizationLabel and CustomerLabel are set externally (by the control plane)
+	// on the Stack CR metadata. They are not guaranteed to be present and are used to
+	// tag/label/taint dedicated Karpenter node pools for cost attribution.
+	OrganizationLabel = "formance.com/organization"
+	CustomerLabel     = "formance.com/customer"
 )

config/crd/bases/formance.com_settings.yaml

@@ -48,17 +48,34 @@ spec:
           all the stacks\n\tvalue: postgresql://postgresql.formance.svc.cluster.local:5432\n\n```\n\nSome
           settings are really global, while some are used by specific module.\n\nRefer
           to the documentation of each module and resource to discover available Settings.\n\n#####
-          Global settings\n###### AWS account\n\nA stack can use an AWS account for
-          authentication.\n\nIt can be used to connect to any AWS service we could
-          use.\n\nIt includes RDS, OpenSearch and MSK. To do so, you can create the
-          following setting:\n```yaml\napiVersion: formance.com/v1beta1\nkind: Settings\nmetadata:\n\n\tname:
-          aws-service-account\n\nspec:\n\n\tkey: aws.service-account\n\tstacks:\n\t-
-          '*'\n\tvalue: aws-access\n\n```\nThis setting instruct the operator than
-          there is somewhere on the cluster a service account named `aws-access`.\n\nSo,
-          each time a service has the capability to use AWS, the operator will use
-          this service account.\n\nThe service account could look like that :\n```yaml\napiVersion:
-          v1\nkind: ServiceAccount\nmetadata:\n\n\tannotations:\n\t  eks.amazonaws.com/role-arn:
-          arn:aws:iam::************:role/staging-eu-west-1-hosting-stack-access\n\tlabels:\n\t
+          Global settings\n###### Karpenter node isolation\n\nThe operator can provision
+          dedicated Karpenter node pools per organization (or per\norganization+stack)
+          so that EC2 instances carry customer/organization/stack tags for\ncost attribution,
+          and so a stack's workloads run only on its dedicated nodes. The\norganization
+          and customer identity are read from the Stack labels\n`formance.com/organization`
+          and `formance.com/customer` (customer defaults to the\norganization when
+          absent).\n\nAvailable keys:\n  - `karpenter.enabled` (bool): master switch.
+          Requires the Karpenter CRDs to be installed.\n  - `karpenter.isolation`
+          (`organization` | `stack`, default `organization`): chooses the\n    pool
+          naming — a shared `<organization>` pool, or a dedicated `<organization>-<stack>`
+          pool.\n  - `karpenter.ec2-node-class.reference` (string, required when enabled):
+          name of the\n    cluster-scoped reference EC2NodeClass to clone.\n  - `karpenter.node-pool.reference`
+          (string, required when enabled): name of the\n    cluster-scoped reference
+          NodePool to clone.\n  - `karpenter.api.ec2-node-class.group-version` (string,
+          default `karpenter.k8s.aws/v1`)\n    and `karpenter.api.node-pool.group-version`
+          (string, default `karpenter.sh/v1`):\n    override the Karpenter API group/version.\n\n```yaml\napiVersion:
+          formance.com/v1beta1\nkind: Settings\nmetadata:\n\n\tname: karpenter-enabled\n\nspec:\n\n\tkey:
+          karpenter.enabled\n\tstacks:\n\t- '*'\n\tvalue: \"true\"\n\n```\n\n######
+          AWS account\n\nA stack can use an AWS account for authentication.\n\nIt
+          can be used to connect to any AWS service we could use.\n\nIt includes RDS,
+          OpenSearch and MSK. To do so, you can create the following setting:\n```yaml\napiVersion:
+          formance.com/v1beta1\nkind: Settings\nmetadata:\n\n\tname: aws-service-account\n\nspec:\n\n\tkey:
+          aws.service-account\n\tstacks:\n\t- '*'\n\tvalue: aws-access\n\n```\nThis
+          setting instruct the operator than there is somewhere on the cluster a service
+          account named `aws-access`.\n\nSo, each time a service has the capability
+          to use AWS, the operator will use this service account.\n\nThe service account
+          could look like that :\n```yaml\napiVersion: v1\nkind: ServiceAccount\nmetadata:\n\n\tannotations:\n\t
+          \ eks.amazonaws.com/role-arn: arn:aws:iam::************:role/staging-eu-west-1-hosting-stack-access\n\tlabels:\n\t
           \ formance.com/stack: any\n\tname: aws-access\n\n```\nYou can note two things
           :\n 1. We have an annotation indicating the role arn used to connect to
           AWS. Refer to the AWS documentation to create this role\n 2. We have a label

config/rbac/role.yaml

@@ -29,6 +29,7 @@ rules:
   verbs:
   - get
   - list
+  - watch
 - apiGroups:
   - apps
   resources:
@@ -189,6 +190,30 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - karpenter.k8s.aws
+  resources:
+  - ec2nodeclasses
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - karpenter.sh
+  resources:
+  - nodepools
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - networking.k8s.io
   resources:

helm/operator/templates/gen/rbac.authorization.k8s.io_v1_clusterrole_formance-manager-role.yaml

@@ -28,6 +28,7 @@ rules:
   verbs:
   - get
   - list
+  - watch
 - apiGroups:
   - apps
   resources:
@@ -188,6 +189,30 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - karpenter.k8s.aws
+  resources:
+  - ec2nodeclasses
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - karpenter.sh
+  resources:
+  - nodepools
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - networking.k8s.io
   resources:

internal/resources/applications/application.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"maps"
+	"slices"
 	"strconv"
 
 	"github.com/stoewer/go-strcase"
@@ -20,6 +21,7 @@ import (
 	"github.com/formancehq/operator/v3/api/formance.com/v1beta1"
 	"github.com/formancehq/operator/v3/internal/core"
 	"github.com/formancehq/operator/v3/internal/resources/licence"
+	"github.com/formancehq/operator/v3/internal/resources/nodeisolation"
 	"github.com/formancehq/operator/v3/internal/resources/settings"
 )
 
@@ -465,6 +467,47 @@ func (a Application) withNodeIP(_ core.Context) core.ObjectMutator[*appsv1.Deplo
 	}
 }
 
+// withNodeIsolation pins the deployment's pods onto the stack's dedicated Karpenter nodes
+// by injecting a nodeSelector (enforces placement) and tolerations (permit the node
+// taints). The operator-managed scheduling keys are stripped before re-applying so the
+// result is idempotent (tolerations don't accumulate) and disable / mode-change / relabel
+// removes stale rules. It also no-ops when the Karpenter CRDs are unavailable, to avoid
+// pinning pods onto nodes that will never be provisioned.
+func (a Application) withNodeIsolation(ctx core.Context) core.ObjectMutator[*appsv1.Deployment] {
+	return func(deployment *appsv1.Deployment) error {
+		stack := &v1beta1.Stack{}
+		if err := ctx.GetClient().Get(ctx, types.NamespacedName{Name: a.owner.GetStack()}, stack); err != nil {
+			return err
+		}
+
+		cfg, err := nodeisolation.Resolve(ctx, stack)
+		if err != nil {
+			return err
+		}
+
+		// Reconcile (not accumulate): drop previously-injected operator-managed keys first.
+		spec := &deployment.Spec.Template.Spec
+		for _, key := range nodeisolation.ManagedSchedulingKeys {
+			delete(spec.NodeSelector, key)
+		}
+		spec.Tolerations = slices.DeleteFunc(spec.Tolerations, func(t corev1.Toleration) bool {
+			return slices.Contains(nodeisolation.ManagedSchedulingKeys, t.Key)
+		})
+
+		if !cfg.Enabled || !nodeisolation.IsAvailable() {
+			return nil
+		}
+
+		if spec.NodeSelector == nil {
+			spec.NodeSelector = map[string]string{}
+		}
+		maps.Copy(spec.NodeSelector, cfg.NodeSelector)
+		spec.Tolerations = append(spec.Tolerations, cfg.Tolerations...)
+
+		return nil
+	}
+}
+
 func (a Application) withTerminationGracePeriod(ctx core.Context) core.ObjectMutator[*appsv1.Deployment] {
 	return func(deployment *appsv1.Deployment) error {
 		terminationGracePeriod, err := settings.GetInt64(ctx, a.owner.GetStack(), "deployments", deployment.Name, "spec", "template", "spec", "termination-grace-period-seconds")
@@ -504,6 +547,7 @@ func (a Application) handleDeployment(ctx core.Context, deploymentLabels map[str
 		a.withSemconvMetricsNames(ctx),
 		a.withNodeIP(ctx),
 		a.withTerminationGracePeriod(ctx),
+		a.withNodeIsolation(ctx),
 		core.WithController[*appsv1.Deployment](ctx.GetScheme(), a.owner),
 	)
 

internal/resources/applications/application_test.go

@@ -16,9 +16,64 @@ import (
 
 	"github.com/formancehq/operator/v3/api/formance.com/v1beta1"
 	"github.com/formancehq/operator/v3/internal/core"
+	"github.com/formancehq/operator/v3/internal/resources/nodeisolation"
 	"github.com/formancehq/operator/v3/internal/resources/settings"
 )
 
+func TestWithNodeIsolationIsIdempotent(t *testing.T) {
+	scheme := runtime.NewScheme()
+	require.NoError(t, v1beta1.AddToScheme(scheme))
+	require.NoError(t, appsv1.AddToScheme(scheme))
+	require.NoError(t, v1.AddToScheme(scheme))
+
+	nodeisolation.SetAvailable(true)
+	t.Cleanup(func() { nodeisolation.SetAvailable(false) })
+
+	stackName := "stack0"
+	enabled := &v1beta1.Settings{ObjectMeta: metav1.ObjectMeta{Name: "enabled"},
+		Spec: v1beta1.SettingsSpec{Stacks: []string{"*"}, Key: "karpenter.enabled", Value: "true"}}
+	ec2 := &v1beta1.Settings{ObjectMeta: metav1.ObjectMeta{Name: "ec2"},
+		Spec: v1beta1.SettingsSpec{Stacks: []string{"*"}, Key: "karpenter.ec2-node-class.reference", Value: "default"}}
+	np := &v1beta1.Settings{ObjectMeta: metav1.ObjectMeta{Name: "np"},
+		Spec: v1beta1.SettingsSpec{Stacks: []string{"*"}, Key: "karpenter.node-pool.reference", Value: "default"}}
+	stack := &v1beta1.Stack{ObjectMeta: metav1.ObjectMeta{
+		Name:   stackName,
+		Labels: map[string]string{v1beta1.OrganizationLabel: "acme"},
+	}}
+
+	fakeClient := fake.NewClientBuilder().
+		WithScheme(scheme).
+		WithObjects(enabled, ec2, np, stack).
+		WithIndex(&v1beta1.Settings{}, "stack", func(obj client.Object) []string {
+			return obj.(*v1beta1.Settings).GetStacks()
+		}).
+		WithIndex(&v1beta1.Settings{}, "keylen", func(obj client.Object) []string {
+			return []string{fmt.Sprint(len(settings.SplitKeywordWithDot(obj.(*v1beta1.Settings).Spec.Key)))}
+		}).
+		Build()
+
+	ctx := &mockContext{Context: context.Background(), client: fakeClient, scheme: scheme}
+	app := Application{owner: &v1beta1.Ledger{
+		ObjectMeta: metav1.ObjectMeta{Name: "ledger"},
+		Spec:       v1beta1.LedgerSpec{StackDependency: v1beta1.StackDependency{Stack: stackName}},
+	}}
+	deployment := &appsv1.Deployment{}
+
+	// Applying the mutator repeatedly must not accumulate tolerations or selectors.
+	for i := 0; i < 3; i++ {
+		require.NoError(t, app.withNodeIsolation(ctx)(deployment))
+	}
+	require.Equal(t, "acme", deployment.Spec.Template.Spec.NodeSelector[v1beta1.OrganizationLabel])
+	require.Len(t, deployment.Spec.Template.Spec.Tolerations, 1)
+
+	// Disabling must strip the operator-managed selector keys and tolerations.
+	enabled.Spec.Value = "false"
+	require.NoError(t, ctx.GetClient().Update(ctx, enabled))
+	require.NoError(t, app.withNodeIsolation(ctx)(deployment))
+	require.NotContains(t, deployment.Spec.Template.Spec.NodeSelector, v1beta1.OrganizationLabel)
+	require.Empty(t, deployment.Spec.Template.Spec.Tolerations)
+}
+
 func TestWithAnnotations(t *testing.T) {
 	t.Parallel()
 	type testCase struct {

internal/resources/nodeisolation/available.go

@@ -0,0 +1,52 @@
+package nodeisolation
+
+import (
+	"sync/atomic"
+
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/types"
+
+	"github.com/formancehq/operator/v3/internal/core"
+)
+
+// CRD names Karpenter installs for the two kinds this feature manages.
+const (
+	ec2NodeClassCRDName = "ec2nodeclasses.karpenter.k8s.aws"
+	nodePoolCRDName     = "nodepools.karpenter.sh"
+)
+
+// available reports whether both Karpenter CRDs are installed. It is read on every
+// reconcile and refreshed by DetectCRDs at controller setup and whenever a Karpenter CRD
+// appears/disappears (see the CustomResourceDefinition watch in the stacks reconciler), so
+// installing Karpenter after the operator starts does not require a restart.
+var available atomic.Bool
+
+// IsAvailable reports whether the Karpenter CRDs are currently installed.
+func IsAvailable() bool { return available.Load() }
+
+// SetAvailable overrides the availability flag. Intended for tests.
+func SetAvailable(v bool) { available.Store(v) }
+
+// IsKarpenterCRD reports whether the named CustomResourceDefinition is one of the
+// Karpenter CRDs this feature depends on.
+func IsKarpenterCRD(name string) bool {
+	return name == ec2NodeClassCRDName || name == nodePoolCRDName
+}
+
+// DetectCRDs performs precise discovery (a targeted Get per CRD) and updates the
+// availability flag. It sets available=true only if both Karpenter CRDs are present.
+func DetectCRDs(ctx core.Context) error {
+	for _, name := range []string{ec2NodeClassCRDName, nodePoolCRDName} {
+		crd := &apiextensionsv1.CustomResourceDefinition{}
+		if err := ctx.GetAPIReader().Get(ctx, types.NamespacedName{Name: name}, crd); err != nil {
+			if apierrors.IsNotFound(err) {
+				available.Store(false)
+				return nil
+			}
+			return err
+		}
+	}
+	available.Store(true)
+	return nil
+}