diff --git a/CHANGELOG.md b/CHANGELOG.md index 85362f20..3d21347a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] - Fix: triggering a scheduled task job on a scope that is not deployed now fails with a clear "deploy the scope first" message instead of an opaque error +- Add "Kill instance" action to scheduled task scopes to terminate an individual running job instance ## [1.13.0] - 2026-07-10 - Add support to get AWS credentials via assume role diff --git a/azure-aro/specs/actions/kill-instance.json.tpl b/azure-aro/specs/actions/kill-instance.json.tpl new file mode 100644 index 00000000..edebec02 --- /dev/null +++ b/azure-aro/specs/actions/kill-instance.json.tpl @@ -0,0 +1,36 @@ +{ + "name": "Kill instance", + "type": "custom", + "icon": "material-symbols:delete-outline", + "results": { + "schema": { "type": "object", "required": [], "properties": {} }, + "values": {} + }, + "service_specification_id": "{{ env.Getenv "SERVICE_SPECIFICATION_ID" }}", + "parameters": { + "schema": { + "type": "object", + "required": [ + "instance_id", + "scope_id", + "deployment_id" + ], + "properties": { + "scope_id": { + "type": "string" + }, + "instance_id": { + "type": "string" + }, + "deployment_id": { + "type": "string" + } + } + }, + "values": {} + }, + "annotations": { + "show_on": ["performance"], + "runs_over": "scope" + } +} \ No newline at end of file diff --git a/azure-aro/specs/actions/kill-instances.json.tpl b/azure-aro/specs/actions/kill-instances.json.tpl deleted file mode 100644 index c1351f94..00000000 --- a/azure-aro/specs/actions/kill-instances.json.tpl +++ /dev/null @@ -1,18 +0,0 @@ -{ - "name": "Kill instances", - "type": "custom", - "icon": "material-symbols:delete-outline", - "results": { - "schema": { "type": "object", "required": [], "properties": {} }, - "values": {} - }, - "service_specification_id": "{{ env.Getenv "SERVICE_SPECIFICATION_ID" }}", - "parameters": { - "schema": { "type": "object", "required": [], "properties": {} }, - "values": {} - }, - "annotations": { - "show_on": ["performance"], - "runs_over": "scope" - } -} \ No newline at end of file diff --git a/azure/specs/actions/kill-instance.json.tpl b/azure/specs/actions/kill-instance.json.tpl new file mode 100644 index 00000000..edebec02 --- /dev/null +++ b/azure/specs/actions/kill-instance.json.tpl @@ -0,0 +1,36 @@ +{ + "name": "Kill instance", + "type": "custom", + "icon": "material-symbols:delete-outline", + "results": { + "schema": { "type": "object", "required": [], "properties": {} }, + "values": {} + }, + "service_specification_id": "{{ env.Getenv "SERVICE_SPECIFICATION_ID" }}", + "parameters": { + "schema": { + "type": "object", + "required": [ + "instance_id", + "scope_id", + "deployment_id" + ], + "properties": { + "scope_id": { + "type": "string" + }, + "instance_id": { + "type": "string" + }, + "deployment_id": { + "type": "string" + } + } + }, + "values": {} + }, + "annotations": { + "show_on": ["performance"], + "runs_over": "scope" + } +} \ No newline at end of file diff --git a/azure/specs/actions/kill-instances.json.tpl b/azure/specs/actions/kill-instances.json.tpl deleted file mode 100644 index c1351f94..00000000 --- a/azure/specs/actions/kill-instances.json.tpl +++ /dev/null @@ -1,18 +0,0 @@ -{ - "name": "Kill instances", - "type": "custom", - "icon": "material-symbols:delete-outline", - "results": { - "schema": { "type": "object", "required": [], "properties": {} }, - "values": {} - }, - "service_specification_id": "{{ env.Getenv "SERVICE_SPECIFICATION_ID" }}", - "parameters": { - "schema": { "type": "object", "required": [], "properties": {} }, - "values": {} - }, - "annotations": { - "show_on": ["performance"], - "runs_over": "scope" - } -} \ No newline at end of file diff --git a/k8s/deployment/kill_instances b/k8s/deployment/kill_instance similarity index 69% rename from k8s/deployment/kill_instances rename to k8s/deployment/kill_instance index a11b774c..d03e6a4f 100755 --- a/k8s/deployment/kill_instances +++ b/k8s/deployment/kill_instance @@ -6,14 +6,14 @@ set -euo pipefail log debug "🔍 Starting instance kill operation..." DEPLOYMENT_ID=$(echo "$CONTEXT" | jq -r '.parameters.deployment_id // .notification.parameters.deployment_id // empty') -INSTANCE_NAME=$(echo "$CONTEXT" | jq -r '.parameters.instance_name // .notification.parameters.instance_name // empty') +INSTANCE_ID=$(echo "$CONTEXT" | jq -r '.parameters.instance_id // .notification.parameters.instance_id // empty') if [[ -z "$DEPLOYMENT_ID" ]] && [[ -n "${NP_ACTION_CONTEXT:-}" ]]; then DEPLOYMENT_ID=$(echo "$NP_ACTION_CONTEXT" | jq -r '.notification.parameters.deployment_id // empty') fi -if [[ -z "$INSTANCE_NAME" ]] && [[ -n "${NP_ACTION_CONTEXT:-}" ]]; then - INSTANCE_NAME=$(echo "$NP_ACTION_CONTEXT" | jq -r '.notification.parameters.instance_name // empty') +if [[ -z "$INSTANCE_ID" ]] && [[ -n "${NP_ACTION_CONTEXT:-}" ]]; then + INSTANCE_ID=$(echo "$NP_ACTION_CONTEXT" | jq -r '.notification.parameters.instance_id // empty') fi if [[ -z "$DEPLOYMENT_ID" ]]; then @@ -26,18 +26,18 @@ if [[ -z "$DEPLOYMENT_ID" ]]; then exit 1 fi -if [[ -z "$INSTANCE_NAME" ]]; then - log error "❌ instance_name parameter not found" +if [[ -z "$INSTANCE_ID" ]]; then + log error "❌ instance_id parameter not found" log error "💡 Possible causes:" log error " - Parameter not provided in action request" log error " - Context structure is different than expected" log error "🔧 How to fix:" - log error " - Ensure instance_name is passed in the action parameters" + log error " - Ensure instance_id is passed in the action parameters" exit 1 fi log debug "📋 Deployment ID: $DEPLOYMENT_ID" -log debug "📋 Instance name: $INSTANCE_NAME" +log debug "📋 Instance name: $INSTANCE_ID" SCOPE_ID=$(echo "$CONTEXT" | jq -r '.tags.scope_id // .scope.id // .notification.tags.scope_id // empty') @@ -63,8 +63,8 @@ log debug "📋 Scope ID: $SCOPE_ID" log debug "📋 Namespace: $K8S_NAMESPACE" log debug "🔍 Verifying pod exists..." -if ! kubectl get pod "$INSTANCE_NAME" -n "$K8S_NAMESPACE" >/dev/null 2>&1; then - log error "❌ Pod $INSTANCE_NAME not found in namespace $K8S_NAMESPACE" +if ! kubectl get pod "$INSTANCE_ID" -n "$K8S_NAMESPACE" >/dev/null 2>&1; then + log error "❌ Pod $INSTANCE_ID not found in namespace $K8S_NAMESPACE" log error "💡 Possible causes:" log error " - Pod was already terminated" log error " - Pod name is incorrect" @@ -75,15 +75,15 @@ if ! kubectl get pod "$INSTANCE_NAME" -n "$K8S_NAMESPACE" >/dev/null 2>&1; then fi log debug "📋 Fetching pod details..." -POD_STATUS=$(kubectl get pod "$INSTANCE_NAME" -n "$K8S_NAMESPACE" -o jsonpath='{.status.phase}') -POD_NODE=$(kubectl get pod "$INSTANCE_NAME" -n "$K8S_NAMESPACE" -o jsonpath='{.spec.nodeName}') -POD_START_TIME=$(kubectl get pod "$INSTANCE_NAME" -n "$K8S_NAMESPACE" -o jsonpath='{.status.startTime}') +POD_STATUS=$(kubectl get pod "$INSTANCE_ID" -n "$K8S_NAMESPACE" -o jsonpath='{.status.phase}') +POD_NODE=$(kubectl get pod "$INSTANCE_ID" -n "$K8S_NAMESPACE" -o jsonpath='{.spec.nodeName}') +POD_START_TIME=$(kubectl get pod "$INSTANCE_ID" -n "$K8S_NAMESPACE" -o jsonpath='{.status.startTime}') -log debug "📋 Pod: $INSTANCE_NAME | Status: $POD_STATUS | Node: $POD_NODE | Started: $POD_START_TIME" +log debug "📋 Pod: $INSTANCE_ID | Status: $POD_STATUS | Node: $POD_NODE | Started: $POD_START_TIME" DEPLOYMENT_NAME="d-$SCOPE_ID-$DEPLOYMENT_ID" -POD_DEPLOYMENT=$(kubectl get pod "$INSTANCE_NAME" -n "$K8S_NAMESPACE" -o jsonpath='{.metadata.ownerReferences[0].name}' 2>/dev/null || echo "") +POD_DEPLOYMENT=$(kubectl get pod "$INSTANCE_ID" -n "$K8S_NAMESPACE" -o jsonpath='{.metadata.ownerReferences[0].name}' 2>/dev/null || echo "") if [[ -n "$POD_DEPLOYMENT" ]]; then REPLICASET_DEPLOYMENT=$(kubectl get replicaset "$POD_DEPLOYMENT" -n "$K8S_NAMESPACE" -o jsonpath='{.metadata.ownerReferences[0].name}' 2>/dev/null || echo "") log debug "📋 Pod ownership: ReplicaSet=$POD_DEPLOYMENT -> Deployment=$REPLICASET_DEPLOYMENT" @@ -95,14 +95,14 @@ else log warn "⚠️ Could not verify pod ownership" fi -log debug "📝 Deleting pod $INSTANCE_NAME with 30s grace period..." -kubectl delete pod "$INSTANCE_NAME" -n "$K8S_NAMESPACE" --grace-period=30 +log debug "📝 Deleting pod $INSTANCE_ID with 30s grace period..." +kubectl delete pod "$INSTANCE_ID" -n "$K8S_NAMESPACE" --grace-period=30 log debug "📝 Waiting for pod termination..." -kubectl wait --for=delete pod/"$INSTANCE_NAME" -n "$K8S_NAMESPACE" --timeout=60s || log warn "⚠️ Pod deletion timeout reached" +kubectl wait --for=delete pod/"$INSTANCE_ID" -n "$K8S_NAMESPACE" --timeout=60s || log warn "⚠️ Pod deletion timeout reached" -if kubectl get pod "$INSTANCE_NAME" -n "$K8S_NAMESPACE" >/dev/null 2>&1; then - POD_STATUS_AFTER=$(kubectl get pod "$INSTANCE_NAME" -n "$K8S_NAMESPACE" -o jsonpath='{.status.phase}') +if kubectl get pod "$INSTANCE_ID" -n "$K8S_NAMESPACE" >/dev/null 2>&1; then + POD_STATUS_AFTER=$(kubectl get pod "$INSTANCE_ID" -n "$K8S_NAMESPACE" -o jsonpath='{.status.phase}') log warn "⚠️ Pod still exists after deletion attempt (status: $POD_STATUS_AFTER)" else log info "✅ Pod successfully terminated and removed" @@ -123,4 +123,4 @@ else log warn "⚠️ Deployment $DEPLOYMENT_NAME not found" fi -log info "✨ Instance kill operation completed for $INSTANCE_NAME" +log info "✨ Instance kill operation completed for $INSTANCE_ID" diff --git a/k8s/deployment/tests/kill_instances.bats b/k8s/deployment/tests/kill_instance.bats similarity index 85% rename from k8s/deployment/tests/kill_instances.bats rename to k8s/deployment/tests/kill_instance.bats index a3f25079..d0699f42 100644 --- a/k8s/deployment/tests/kill_instances.bats +++ b/k8s/deployment/tests/kill_instance.bats @@ -1,6 +1,6 @@ #!/usr/bin/env bats # ============================================================================= -# Unit tests for deployment/kill_instances - pod termination +# Unit tests for deployment/kill_instance - pod termination # ============================================================================= setup() { @@ -15,7 +15,7 @@ setup() { export CONTEXT='{ "parameters": { "deployment_id": "deploy-456", - "instance_name": "my-pod-abc123" + "instance_id": "my-pod-abc123" }, "tags": { "scope_id": "scope-123" @@ -84,8 +84,8 @@ teardown() { # ============================================================================= # Success Case # ============================================================================= -@test "kill_instances: successfully kills pod with correct logging" { - run bash "$BATS_TEST_DIRNAME/../kill_instances" +@test "kill_instance: successfully kills pod with correct logging" { + run bash "$BATS_TEST_DIRNAME/../kill_instance" [ "$status" -eq 0 ] # Start message @@ -110,14 +110,14 @@ teardown() { # ============================================================================= # Error Cases # ============================================================================= -@test "kill_instances: fails with troubleshooting when deployment_id missing" { +@test "kill_instance: fails with troubleshooting when deployment_id missing" { export CONTEXT='{ "parameters": { - "instance_name": "my-pod-abc123" + "instance_id": "my-pod-abc123" } }' - run bash "$BATS_TEST_DIRNAME/../kill_instances" + run bash "$BATS_TEST_DIRNAME/../kill_instance" [ "$status" -eq 1 ] assert_contains "$output" "❌ deployment_id parameter not found" @@ -127,32 +127,32 @@ teardown() { assert_contains "$output" "Ensure deployment_id is passed in the action parameters" } -@test "kill_instances: fails with troubleshooting when instance_name missing" { +@test "kill_instance: fails with troubleshooting when instance_id missing" { export CONTEXT='{ "parameters": { "deployment_id": "deploy-456" } }' - run bash "$BATS_TEST_DIRNAME/../kill_instances" + run bash "$BATS_TEST_DIRNAME/../kill_instance" [ "$status" -eq 1 ] - assert_contains "$output" "❌ instance_name parameter not found" + assert_contains "$output" "❌ instance_id parameter not found" assert_contains "$output" "💡 Possible causes:" assert_contains "$output" "Parameter not provided in action request" assert_contains "$output" "🔧 How to fix:" - assert_contains "$output" "Ensure instance_name is passed in the action parameters" + assert_contains "$output" "Ensure instance_id is passed in the action parameters" } -@test "kill_instances: fails with troubleshooting when scope_id missing" { +@test "kill_instance: fails with troubleshooting when scope_id missing" { export CONTEXT='{ "parameters": { "deployment_id": "deploy-456", - "instance_name": "my-pod-abc123" + "instance_id": "my-pod-abc123" } }' - run bash "$BATS_TEST_DIRNAME/../kill_instances" + run bash "$BATS_TEST_DIRNAME/../kill_instance" [ "$status" -eq 1 ] assert_contains "$output" "❌ scope_id not found in context" @@ -162,7 +162,7 @@ teardown() { assert_contains "$output" "Verify the action is invoked with proper scope context" } -@test "kill_instances: fails with troubleshooting when pod not found" { +@test "kill_instance: fails with troubleshooting when pod not found" { kubectl() { case "$1" in get) @@ -175,7 +175,7 @@ teardown() { } export -f kubectl - run bash "$BATS_TEST_DIRNAME/../kill_instances" + run bash "$BATS_TEST_DIRNAME/../kill_instance" [ "$status" -eq 1 ] assert_contains "$output" "❌ Pod my-pod-abc123 not found in namespace test-namespace" @@ -188,7 +188,7 @@ teardown() { # ============================================================================= # Warning Cases # ============================================================================= -@test "kill_instances: warns when pod belongs to different deployment" { +@test "kill_instance: warns when pod belongs to different deployment" { kubectl() { case "$1" in get) @@ -230,13 +230,13 @@ teardown() { } export -f kubectl - run bash "$BATS_TEST_DIRNAME/../kill_instances" + run bash "$BATS_TEST_DIRNAME/../kill_instance" [ "$status" -eq 0 ] assert_contains "$output" "⚠️ Pod does not belong to expected deployment d-scope-123-deploy-456" } -@test "kill_instances: warns when pod still exists after deletion" { +@test "kill_instance: warns when pod still exists after deletion" { local delete_called=0 kubectl() { case "$1" in @@ -279,7 +279,7 @@ teardown() { } export -f kubectl - run bash "$BATS_TEST_DIRNAME/../kill_instances" + run bash "$BATS_TEST_DIRNAME/../kill_instance" [ "$status" -eq 0 ] assert_contains "$output" "⚠️ Pod deletion timeout reached" diff --git a/k8s/deployment/workflows/kill_instances.yaml b/k8s/deployment/workflows/kill_instance.yaml similarity index 77% rename from k8s/deployment/workflows/kill_instances.yaml rename to k8s/deployment/workflows/kill_instance.yaml index aa162316..74f8427c 100644 --- a/k8s/deployment/workflows/kill_instances.yaml +++ b/k8s/deployment/workflows/kill_instance.yaml @@ -10,6 +10,6 @@ steps: parameters: level: string message: string - - name: kill instances + - name: kill instance type: script - file: "$SERVICE_PATH/deployment/kill_instances" \ No newline at end of file + file: "$SERVICE_PATH/deployment/kill_instance" \ No newline at end of file diff --git a/k8s/specs/actions/kill-instance.json.tpl b/k8s/specs/actions/kill-instance.json.tpl new file mode 100644 index 00000000..edebec02 --- /dev/null +++ b/k8s/specs/actions/kill-instance.json.tpl @@ -0,0 +1,36 @@ +{ + "name": "Kill instance", + "type": "custom", + "icon": "material-symbols:delete-outline", + "results": { + "schema": { "type": "object", "required": [], "properties": {} }, + "values": {} + }, + "service_specification_id": "{{ env.Getenv "SERVICE_SPECIFICATION_ID" }}", + "parameters": { + "schema": { + "type": "object", + "required": [ + "instance_id", + "scope_id", + "deployment_id" + ], + "properties": { + "scope_id": { + "type": "string" + }, + "instance_id": { + "type": "string" + }, + "deployment_id": { + "type": "string" + } + } + }, + "values": {} + }, + "annotations": { + "show_on": ["performance"], + "runs_over": "scope" + } +} \ No newline at end of file diff --git a/k8s/specs/actions/kill-instances.json.tpl b/k8s/specs/actions/kill-instances.json.tpl deleted file mode 100644 index c1351f94..00000000 --- a/k8s/specs/actions/kill-instances.json.tpl +++ /dev/null @@ -1,18 +0,0 @@ -{ - "name": "Kill instances", - "type": "custom", - "icon": "material-symbols:delete-outline", - "results": { - "schema": { "type": "object", "required": [], "properties": {} }, - "values": {} - }, - "service_specification_id": "{{ env.Getenv "SERVICE_SPECIFICATION_ID" }}", - "parameters": { - "schema": { "type": "object", "required": [], "properties": {} }, - "values": {} - }, - "annotations": { - "show_on": ["performance"], - "runs_over": "scope" - } -} \ No newline at end of file diff --git a/scheduled_task/deployment/kill_instance b/scheduled_task/deployment/kill_instance new file mode 100755 index 00000000..1ec9f2f5 --- /dev/null +++ b/scheduled_task/deployment/kill_instance @@ -0,0 +1,126 @@ +#!/bin/bash + +set -euo pipefail + + +log debug "🔍 Starting instance kill operation..." + +DEPLOYMENT_ID=$(echo "$CONTEXT" | jq -r '.parameters.deployment_id // .notification.parameters.deployment_id // empty') +INSTANCE_ID=$(echo "$CONTEXT" | jq -r '.parameters.instance_id // .notification.parameters.instance_id // empty') + +if [[ -z "$DEPLOYMENT_ID" ]] && [[ -n "${NP_ACTION_CONTEXT:-}" ]]; then + DEPLOYMENT_ID=$(echo "$NP_ACTION_CONTEXT" | jq -r '.notification.parameters.deployment_id // empty') +fi + +if [[ -z "$INSTANCE_ID" ]] && [[ -n "${NP_ACTION_CONTEXT:-}" ]]; then + INSTANCE_ID=$(echo "$NP_ACTION_CONTEXT" | jq -r '.notification.parameters.instance_id // empty') +fi + +if [[ -z "$DEPLOYMENT_ID" ]]; then + log error "❌ deployment_id parameter not found" + log error "💡 Possible causes:" + log error " - Parameter not provided in action request" + log error " - Context structure is different than expected" + log error "🔧 How to fix:" + log error " - Ensure deployment_id is passed in the action parameters" + exit 1 +fi + +if [[ -z "$INSTANCE_ID" ]]; then + log error "❌ instance_id parameter not found" + log error "💡 Possible causes:" + log error " - Parameter not provided in action request" + log error " - Context structure is different than expected" + log error "🔧 How to fix:" + log error " - Ensure instance_id is passed in the action parameters" + exit 1 +fi + +log debug "📋 Deployment ID: $DEPLOYMENT_ID" +log debug "📋 Instance name: $INSTANCE_ID" + +SCOPE_ID=$(echo "$CONTEXT" | jq -r '.tags.scope_id // .scope.id // .notification.tags.scope_id // empty') + +if [[ -z "$SCOPE_ID" ]] && [[ -n "${NP_ACTION_CONTEXT:-}" ]]; then + SCOPE_ID=$(echo "$NP_ACTION_CONTEXT" | jq -r '.notification.tags.scope_id // empty') +fi + +K8S_NAMESPACE=$(echo "$CONTEXT" | jq -r --arg default "$K8S_NAMESPACE" ' + .providers["container-orchestration"].cluster.namespace // $default +' 2>/dev/null || echo "nullplatform") + +if [[ -z "$SCOPE_ID" ]]; then + log error "❌ scope_id not found in context" + log error "💡 Possible causes:" + log error " - Context missing scope information" + log error " - Action invoked outside of scope context" + log error "🔧 How to fix:" + log error " - Verify the action is invoked with proper scope context" + exit 1 +fi + +log debug "📋 Scope ID: $SCOPE_ID" +log debug "📋 Namespace: $K8S_NAMESPACE" + +log debug "🔍 Verifying pod exists..." +if ! kubectl get pod "$INSTANCE_ID" -n "$K8S_NAMESPACE" >/dev/null 2>&1; then + log error "❌ Pod $INSTANCE_ID not found in namespace $K8S_NAMESPACE" + log error "💡 Possible causes:" + log error " - Pod was already terminated" + log error " - Pod name is incorrect" + log error " - Pod exists in a different namespace" + log error "🔧 How to fix:" + log error " - List pods: kubectl get pods -n $K8S_NAMESPACE -l scope_id=$SCOPE_ID" + exit 1 +fi + +log debug "📋 Fetching pod details..." +POD_STATUS=$(kubectl get pod "$INSTANCE_ID" -n "$K8S_NAMESPACE" -o jsonpath='{.status.phase}') +POD_NODE=$(kubectl get pod "$INSTANCE_ID" -n "$K8S_NAMESPACE" -o jsonpath='{.spec.nodeName}') +POD_START_TIME=$(kubectl get pod "$INSTANCE_ID" -n "$K8S_NAMESPACE" -o jsonpath='{.status.startTime}') + +log debug "📋 Pod: $INSTANCE_ID | Status: $POD_STATUS | Node: $POD_NODE | Started: $POD_START_TIME" + +CRONJOB_NAME="job-$SCOPE_ID-$DEPLOYMENT_ID" + +POD_JOB=$(kubectl get pod "$INSTANCE_ID" -n "$K8S_NAMESPACE" -o jsonpath='{.metadata.ownerReferences[0].name}' 2>/dev/null || echo "") +if [[ -n "$POD_JOB" ]]; then + JOB_CRONJOB=$(kubectl get job "$POD_JOB" -n "$K8S_NAMESPACE" -o jsonpath='{.metadata.ownerReferences[0].name}' 2>/dev/null || echo "") + log debug "📋 Pod ownership: Job=$POD_JOB -> CronJob=$JOB_CRONJOB" + + if [[ "$JOB_CRONJOB" != "$CRONJOB_NAME" ]]; then + log warn "⚠️ Pod does not belong to expected scheduled task $CRONJOB_NAME (continuing anyway)" + fi +else + log warn "⚠️ Could not verify pod ownership" +fi + +log debug "📝 Deleting pod $INSTANCE_ID with 30s grace period..." +kubectl delete pod "$INSTANCE_ID" -n "$K8S_NAMESPACE" --grace-period=30 + +log debug "📝 Waiting for pod termination..." +kubectl wait --for=delete pod/"$INSTANCE_ID" -n "$K8S_NAMESPACE" --timeout=60s || log warn "⚠️ Pod deletion timeout reached" + +if kubectl get pod "$INSTANCE_ID" -n "$K8S_NAMESPACE" >/dev/null 2>&1; then + POD_STATUS_AFTER=$(kubectl get pod "$INSTANCE_ID" -n "$K8S_NAMESPACE" -o jsonpath='{.status.phase}') + log warn "⚠️ Pod still exists after deletion attempt (status: $POD_STATUS_AFTER)" +else + log info "✅ Pod successfully terminated and removed" +fi + +log debug "📋 Checking job status after pod deletion..." +if [[ -n "$POD_JOB" ]] && kubectl get job "$POD_JOB" -n "$K8S_NAMESPACE" >/dev/null 2>&1; then + JOB_ACTIVE=$(kubectl get job "$POD_JOB" -n "$K8S_NAMESPACE" -o jsonpath='{.status.active}') + JOB_SUCCEEDED=$(kubectl get job "$POD_JOB" -n "$K8S_NAMESPACE" -o jsonpath='{.status.succeeded}') + JOB_FAILED=$(kubectl get job "$POD_JOB" -n "$K8S_NAMESPACE" -o jsonpath='{.status.failed}') + + log debug "📋 Job $POD_JOB: active=${JOB_ACTIVE:-0}, succeeded=${JOB_SUCCEEDED:-0}, failed=${JOB_FAILED:-0}" + + if [[ "${JOB_ACTIVE:-0}" -gt 0 ]]; then + log debug "📋 Job is still active; Kubernetes may start a replacement pod (backoffLimit permitting)" + fi +else + log warn "⚠️ Job for pod not found (it may have already completed)" +fi + +log info "✨ Instance kill operation completed for $INSTANCE_ID" diff --git a/scheduled_task/deployment/tests/kill_instance.bats b/scheduled_task/deployment/tests/kill_instance.bats new file mode 100644 index 00000000..95694a27 --- /dev/null +++ b/scheduled_task/deployment/tests/kill_instance.bats @@ -0,0 +1,412 @@ +#!/usr/bin/env bats +# ============================================================================= +# Unit tests for scheduled_task deployment/kill_instance - job pod termination +# +# scheduled_task pods are owned by Job -> CronJob (job--), +# unlike the base k8s scope where pods are owned by ReplicaSet -> Deployment. +# ============================================================================= + +setup() { + export PROJECT_ROOT="$(cd "$BATS_TEST_DIRNAME/../../.." && pwd)" + source "$PROJECT_ROOT/testing/assertions.sh" + log() { if [ "$1" = "error" ]; then echo "$2" >&2; else echo "$2"; fi; } + export -f log + + export K8S_NAMESPACE="test-namespace" + export SCOPE_ID="scope-123" + export KILL_STATE="$BATS_TEST_TMPDIR/deleted" + rm -f "$KILL_STATE" + + export CONTEXT='{ + "parameters": { + "deployment_id": "deploy-456", + "instance_id": "my-pod-abc123" + }, + "tags": { + "scope_id": "scope-123" + }, + "providers": { + "container-orchestration": { + "cluster": { + "namespace": "test-namespace" + } + } + } + }' + + # Default mock: pod owned by a Job that belongs to the expected CronJob. + # A state file flips the plain (no -o) pod existence check to "gone" once + # `kubectl delete` has run, exercising the happy termination path. + kubectl() { + case "$1" in + get) + case "$2" in + pod) + if [[ "$*" == *"-o jsonpath"* ]]; then + if [[ "$*" == *"phase"* ]]; then + echo "Running" + elif [[ "$*" == *"nodeName"* ]]; then + echo "node-1" + elif [[ "$*" == *"startTime"* ]]; then + echo "2024-01-01T00:00:00Z" + elif [[ "$*" == *"ownerReferences"* ]]; then + echo "job-scope-123-deploy-456-abc" + fi + return 0 + fi + # existence check (no -o): gone once deleted + [[ -f "$KILL_STATE" ]] && return 1 + return 0 + ;; + job) + if [[ "$*" == *"-o jsonpath"* ]]; then + if [[ "$*" == *"ownerReferences"* ]]; then + echo "job-scope-123-deploy-456" + elif [[ "$*" == *"active"* ]]; then + echo "0" + elif [[ "$*" == *"succeeded"* ]]; then + echo "1" + elif [[ "$*" == *"failed"* ]]; then + echo "0" + fi + return 0 + fi + return 0 + ;; + esac + ;; + delete) + touch "$KILL_STATE" + return 0 + ;; + wait) + return 0 + ;; + esac + return 0 + } + export -f kubectl +} + +teardown() { + unset CONTEXT + unset -f kubectl +} + +# ============================================================================= +# Success Case +# ============================================================================= +@test "kill_instance: successfully kills job pod with correct logging" { + run bash "$BATS_TEST_DIRNAME/../kill_instance" + + [ "$status" -eq 0 ] + # Start message + assert_contains "$output" "🔍 Starting instance kill operation..." + # Parameter display + assert_contains "$output" "📋 Deployment ID: deploy-456" + assert_contains "$output" "📋 Instance name: my-pod-abc123" + assert_contains "$output" "📋 Scope ID: scope-123" + assert_contains "$output" "📋 Namespace: test-namespace" + # Pod verification + assert_contains "$output" "🔍 Verifying pod exists..." + assert_contains "$output" "📋 Fetching pod details..." + assert_contains "$output" "📋 Pod: my-pod-abc123 | Status: Running | Node: node-1 | Started: 2024-01-01T00:00:00Z" + # Ownership (Job -> CronJob) + assert_contains "$output" "📋 Pod ownership: Job=job-scope-123-deploy-456-abc -> CronJob=job-scope-123-deploy-456" + # Delete operation + assert_contains "$output" "📝 Deleting pod my-pod-abc123 with 30s grace period..." + assert_contains "$output" "📝 Waiting for pod termination..." + assert_contains "$output" "✅ Pod successfully terminated and removed" + # Job status + assert_contains "$output" "📋 Checking job status after pod deletion..." + assert_contains "$output" "📋 Job job-scope-123-deploy-456-abc: active=0, succeeded=1, failed=0" + # Completion + assert_contains "$output" "✨ Instance kill operation completed for my-pod-abc123" +} + +# ============================================================================= +# Error Cases +# ============================================================================= +@test "kill_instance: fails with troubleshooting when deployment_id missing" { + export CONTEXT='{ + "parameters": { + "instance_id": "my-pod-abc123" + } + }' + + run bash "$BATS_TEST_DIRNAME/../kill_instance" + + [ "$status" -eq 1 ] + assert_contains "$output" "❌ deployment_id parameter not found" + assert_contains "$output" "💡 Possible causes:" + assert_contains "$output" "Parameter not provided in action request" + assert_contains "$output" "🔧 How to fix:" + assert_contains "$output" "Ensure deployment_id is passed in the action parameters" +} + +@test "kill_instance: fails with troubleshooting when instance_id missing" { + export CONTEXT='{ + "parameters": { + "deployment_id": "deploy-456" + } + }' + + run bash "$BATS_TEST_DIRNAME/../kill_instance" + + [ "$status" -eq 1 ] + assert_contains "$output" "❌ instance_id parameter not found" + assert_contains "$output" "💡 Possible causes:" + assert_contains "$output" "Parameter not provided in action request" + assert_contains "$output" "🔧 How to fix:" + assert_contains "$output" "Ensure instance_id is passed in the action parameters" +} + +@test "kill_instance: fails with troubleshooting when scope_id missing" { + export CONTEXT='{ + "parameters": { + "deployment_id": "deploy-456", + "instance_id": "my-pod-abc123" + } + }' + + run bash "$BATS_TEST_DIRNAME/../kill_instance" + + [ "$status" -eq 1 ] + assert_contains "$output" "❌ scope_id not found in context" + assert_contains "$output" "💡 Possible causes:" + assert_contains "$output" "Context missing scope information" + assert_contains "$output" "🔧 How to fix:" + assert_contains "$output" "Verify the action is invoked with proper scope context" +} + +@test "kill_instance: fails with troubleshooting when pod not found" { + kubectl() { + case "$1" in + get) + if [[ "$2" == "pod" ]] && [[ "$*" != *"-o"* ]]; then + return 1 + fi + ;; + esac + return 0 + } + export -f kubectl + + run bash "$BATS_TEST_DIRNAME/../kill_instance" + + [ "$status" -eq 1 ] + assert_contains "$output" "❌ Pod my-pod-abc123 not found in namespace test-namespace" + assert_contains "$output" "💡 Possible causes:" + assert_contains "$output" "Pod was already terminated" + assert_contains "$output" "🔧 How to fix:" + assert_contains "$output" "kubectl get pods" +} + +# ============================================================================= +# Warning Cases +# ============================================================================= +@test "kill_instance: warns when pod belongs to a different scheduled task" { + kubectl() { + case "$1" in + get) + case "$2" in + pod) + if [[ "$*" == *"-o jsonpath"* ]]; then + if [[ "$*" == *"phase"* ]]; then + echo "Running" + elif [[ "$*" == *"nodeName"* ]]; then + echo "node-1" + elif [[ "$*" == *"startTime"* ]]; then + echo "2024-01-01T00:00:00Z" + elif [[ "$*" == *"ownerReferences"* ]]; then + echo "job-scope-123-deploy-456-abc" + fi + return 0 + fi + [[ -f "$KILL_STATE" ]] && return 1 + return 0 + ;; + job) + if [[ "$*" == *"-o jsonpath"* ]]; then + if [[ "$*" == *"ownerReferences"* ]]; then + echo "job-scope-123-different-deploy" # Different CronJob + elif [[ "$*" == *"active"* ]]; then + echo "0" + fi + return 0 + fi + return 0 + ;; + esac + ;; + delete) + touch "$KILL_STATE" + return 0 + ;; + wait) + return 0 + ;; + esac + return 0 + } + export -f kubectl + + run bash "$BATS_TEST_DIRNAME/../kill_instance" + + [ "$status" -eq 0 ] + assert_contains "$output" "⚠️ Pod does not belong to expected scheduled task job-scope-123-deploy-456 (continuing anyway)" +} + +@test "kill_instance: warns when pod ownership cannot be verified" { + kubectl() { + case "$1" in + get) + case "$2" in + pod) + if [[ "$*" == *"-o jsonpath"* ]]; then + if [[ "$*" == *"phase"* ]]; then + echo "Running" + elif [[ "$*" == *"nodeName"* ]]; then + echo "node-1" + elif [[ "$*" == *"startTime"* ]]; then + echo "2024-01-01T00:00:00Z" + elif [[ "$*" == *"ownerReferences"* ]]; then + echo "" # Bare pod, no owner + fi + return 0 + fi + [[ -f "$KILL_STATE" ]] && return 1 + return 0 + ;; + job) + [[ -f "$KILL_STATE" ]] && return 1 + return 0 + ;; + esac + ;; + delete) + touch "$KILL_STATE" + return 0 + ;; + wait) + return 0 + ;; + esac + return 0 + } + export -f kubectl + + run bash "$BATS_TEST_DIRNAME/../kill_instance" + + [ "$status" -eq 0 ] + assert_contains "$output" "⚠️ Could not verify pod ownership" + # With no owner Job, the post-deletion job lookup cannot resolve one either + assert_contains "$output" "⚠️ Job for pod not found (it may have already completed)" +} + +@test "kill_instance: warns when pod still exists after deletion" { + kubectl() { + case "$1" in + get) + case "$2" in + pod) + if [[ "$*" == *"-o jsonpath"* ]]; then + if [[ "$*" == *"phase"* ]]; then + echo "Terminating" + elif [[ "$*" == *"nodeName"* ]]; then + echo "node-1" + elif [[ "$*" == *"startTime"* ]]; then + echo "2024-01-01T00:00:00Z" + elif [[ "$*" == *"ownerReferences"* ]]; then + echo "job-scope-123-deploy-456-abc" + fi + return 0 + fi + return 0 # Pod still exists + ;; + job) + if [[ "$*" == *"-o jsonpath"* ]]; then + if [[ "$*" == *"ownerReferences"* ]]; then + echo "job-scope-123-deploy-456" + elif [[ "$*" == *"active"* ]]; then + echo "1" + fi + return 0 + fi + return 0 + ;; + esac + ;; + delete) + return 0 + ;; + wait) + return 1 # Timeout + ;; + esac + return 0 + } + export -f kubectl + + run bash "$BATS_TEST_DIRNAME/../kill_instance" + + [ "$status" -eq 0 ] + assert_contains "$output" "⚠️ Pod deletion timeout reached" + assert_contains "$output" "⚠️ Pod still exists after deletion attempt (status: Terminating)" + # Job still active -> replacement pod may be started + assert_contains "$output" "📋 Checking job status after pod deletion..." + assert_contains "$output" "📋 Job job-scope-123-deploy-456-abc: active=1, succeeded=0, failed=0" + assert_contains "$output" "📋 Job is still active; Kubernetes may start a replacement pod (backoffLimit permitting)" +} + +@test "kill_instance: warns when job already completed after pod deletion" { + kubectl() { + case "$1" in + get) + case "$2" in + pod) + if [[ "$*" == *"-o jsonpath"* ]]; then + if [[ "$*" == *"phase"* ]]; then + echo "Running" + elif [[ "$*" == *"nodeName"* ]]; then + echo "node-1" + elif [[ "$*" == *"startTime"* ]]; then + echo "2024-01-01T00:00:00Z" + elif [[ "$*" == *"ownerReferences"* ]]; then + echo "job-scope-123-deploy-456-abc" + fi + return 0 + fi + [[ -f "$KILL_STATE" ]] && return 1 + return 0 + ;; + job) + if [[ "$*" == *"-o jsonpath"* ]]; then + if [[ "$*" == *"ownerReferences"* ]]; then + echo "job-scope-123-deploy-456" + fi + return 0 + fi + # Job existence check: gone once the pod was deleted + [[ -f "$KILL_STATE" ]] && return 1 + return 0 + ;; + esac + ;; + delete) + touch "$KILL_STATE" + return 0 + ;; + wait) + return 0 + ;; + esac + return 0 + } + export -f kubectl + + run bash "$BATS_TEST_DIRNAME/../kill_instance" + + [ "$status" -eq 0 ] + assert_contains "$output" "⚠️ Job for pod not found (it may have already completed)" +} diff --git a/scheduled_task/deployment/tests/workflow_overrides.bats b/scheduled_task/deployment/tests/workflow_overrides.bats index 591d00db..52e2a838 100644 --- a/scheduled_task/deployment/tests/workflow_overrides.bats +++ b/scheduled_task/deployment/tests/workflow_overrides.bats @@ -34,3 +34,43 @@ setup() { assert_equal "$status" "0" assert_contains "$output" "action: skip" } + +# ============================================================================= +# kill instance +# +# scheduled_task job pods are owned by Job -> CronJob, so the base k8s kill +# script (which reasons about Deployment/ReplicaSet ownership) is replaced by a +# job-aware script. The base must keep the step name so the overlay keeps +# matching it after upstream changes. +# ============================================================================= +@test "base k8s deployment kill_instance workflow declares 'kill instance' step" { + run grep -A 2 "name: kill instance" "$PROJECT_ROOT/k8s/deployment/workflows/kill_instance.yaml" + + assert_equal "$status" "0" + assert_contains "$output" "type: script" + assert_contains "$output" "deployment/kill_instance" +} + +@test "scheduled_task deployment kill_instance overlay replaces 'kill instance' with its own script" { + run grep -A 3 "name: kill instance" "$PROJECT_ROOT/scheduled_task/deployment/workflows/kill_instance.yaml" + + assert_equal "$status" "0" + assert_contains "$output" "action: replace" + assert_contains "$output" "\$OVERRIDES_PATH/deployment/kill_instance" +} + +@test "base k8s deployment kill_instance workflow declares 'load logging' step" { + run grep -A 2 "name: load logging" "$PROJECT_ROOT/k8s/deployment/workflows/kill_instance.yaml" + + assert_equal "$status" "0" + assert_contains "$output" "type: script" + assert_contains "$output" "logging" +} + +@test "scheduled_task deployment kill_instance overlay loads its own logging function" { + run grep -A 3 "name: load logging" "$PROJECT_ROOT/scheduled_task/deployment/workflows/kill_instance.yaml" + + assert_equal "$status" "0" + assert_contains "$output" "action: replace" + assert_contains "$output" "\$OVERRIDES_PATH/logging" +} diff --git a/scheduled_task/deployment/workflows/kill_instance.yaml b/scheduled_task/deployment/workflows/kill_instance.yaml new file mode 100644 index 00000000..ca3e705b --- /dev/null +++ b/scheduled_task/deployment/workflows/kill_instance.yaml @@ -0,0 +1,17 @@ +include: + - "$SERVICE_PATH/values.yaml" +steps: + - name: load logging + type: script + action: replace + file: "$OVERRIDES_PATH/logging" + output: + - name: log + type: function + parameters: + level: string + message: string + - name: kill instance + type: script + action: replace + file: "$OVERRIDES_PATH/deployment/kill_instance" diff --git a/scheduled_task/specs/actions/kill-instance.json.tpl b/scheduled_task/specs/actions/kill-instance.json.tpl new file mode 100644 index 00000000..edebec02 --- /dev/null +++ b/scheduled_task/specs/actions/kill-instance.json.tpl @@ -0,0 +1,36 @@ +{ + "name": "Kill instance", + "type": "custom", + "icon": "material-symbols:delete-outline", + "results": { + "schema": { "type": "object", "required": [], "properties": {} }, + "values": {} + }, + "service_specification_id": "{{ env.Getenv "SERVICE_SPECIFICATION_ID" }}", + "parameters": { + "schema": { + "type": "object", + "required": [ + "instance_id", + "scope_id", + "deployment_id" + ], + "properties": { + "scope_id": { + "type": "string" + }, + "instance_id": { + "type": "string" + }, + "deployment_id": { + "type": "string" + } + } + }, + "values": {} + }, + "annotations": { + "show_on": ["performance"], + "runs_over": "scope" + } +} \ No newline at end of file diff --git a/service/deployment/entrypoint b/service/deployment/entrypoint index 9a1ee9b1..b8df68c6 100755 --- a/service/deployment/entrypoint +++ b/service/deployment/entrypoint @@ -26,8 +26,8 @@ case "$SERVICE_ACTION" in "diagnose-deployment") ACTION_TO_EXECUTE="diagnose" ;; - "kill-instances") - ACTION_TO_EXECUTE="kill_instances" + "kill-instance") + ACTION_TO_EXECUTE="kill_instance" ;; *) echo "Unknown action: $SERVICE_ACTION" @@ -39,7 +39,7 @@ WORKFLOW_PATH="$SERVICE_PATH/deployment/workflows/$ACTION_TO_EXECUTE.yaml" NEEDS_PARAMS=true case "$SERVICE_ACTION" in - "switch-traffic"|"kill-instances"|"diagnose-deployment") + "switch-traffic"|"kill-instance"|"diagnose-deployment") NEEDS_PARAMS=false ;; esac diff --git a/service/deployment/tests/entrypoint.bats b/service/deployment/tests/entrypoint.bats index fac10a8f..4281ef76 100644 --- a/service/deployment/tests/entrypoint.bats +++ b/service/deployment/tests/entrypoint.bats @@ -20,7 +20,7 @@ setup() { touch "$SERVICE_PATH/deployment/workflows/finalize.yaml" touch "$SERVICE_PATH/deployment/workflows/delete.yaml" touch "$SERVICE_PATH/deployment/workflows/diagnose.yaml" - touch "$SERVICE_PATH/deployment/workflows/kill_instances.yaml" + touch "$SERVICE_PATH/deployment/workflows/kill_instance.yaml" export NP_EXECUTED_CMD="" export NP_HELP_SUPPORTS_NO_PARAMS="true" @@ -59,9 +59,9 @@ mock_np() { assert_contains "$output" "--no-params" } -@test "deployment entrypoint: kill-instances includes --no-params when CLI supports it" { +@test "deployment entrypoint: kill-instance includes --no-params when CLI supports it" { mock_np - export SERVICE_ACTION="kill-instances" + export SERVICE_ACTION="kill-instance" run bash "$BATS_TEST_DIRNAME/../entrypoint" @@ -148,10 +148,10 @@ mock_np() { [[ "$output" != *"--no-params"* ]] } -@test "deployment entrypoint: kill-instances omits --no-params when CLI does not support it" { +@test "deployment entrypoint: kill-instance omits --no-params when CLI does not support it" { export NP_HELP_SUPPORTS_NO_PARAMS="false" mock_np - export SERVICE_ACTION="kill-instances" + export SERVICE_ACTION="kill-instance" run bash "$BATS_TEST_DIRNAME/../entrypoint"