Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

## [Unreleased]
- Fix: triggering a scheduled task job on a scope that is not deployed now fails with a clear "deploy the scope first" message instead of an opaque error
- Add "Kill instance" action to scheduled task scopes to terminate an individual running job instance

## [1.13.0] - 2026-07-10
- Add support to get AWS credentials via assume role
Expand Down
36 changes: 36 additions & 0 deletions azure-aro/specs/actions/kill-instance.json.tpl
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
{
"name": "Kill instance",
"type": "custom",
"icon": "material-symbols:delete-outline",
"results": {
"schema": { "type": "object", "required": [], "properties": {} },
"values": {}
},
"service_specification_id": "{{ env.Getenv "SERVICE_SPECIFICATION_ID" }}",
"parameters": {
"schema": {
"type": "object",
"required": [
"instance_id",
"scope_id",
"deployment_id"
],
"properties": {
"scope_id": {
"type": "string"
},
"instance_id": {
"type": "string"
},
"deployment_id": {
"type": "string"
}
}
},
"values": {}
},
"annotations": {
"show_on": ["performance"],
"runs_over": "scope"
}
}
18 changes: 0 additions & 18 deletions azure-aro/specs/actions/kill-instances.json.tpl

This file was deleted.

36 changes: 36 additions & 0 deletions azure/specs/actions/kill-instance.json.tpl
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
{
"name": "Kill instance",
"type": "custom",
"icon": "material-symbols:delete-outline",
"results": {
"schema": { "type": "object", "required": [], "properties": {} },
"values": {}
},
"service_specification_id": "{{ env.Getenv "SERVICE_SPECIFICATION_ID" }}",
"parameters": {
"schema": {
"type": "object",
"required": [
"instance_id",
"scope_id",
"deployment_id"
],
"properties": {
"scope_id": {
"type": "string"
},
"instance_id": {
"type": "string"
},
"deployment_id": {
"type": "string"
}
}
},
"values": {}
},
"annotations": {
"show_on": ["performance"],
"runs_over": "scope"
}
}
18 changes: 0 additions & 18 deletions azure/specs/actions/kill-instances.json.tpl

This file was deleted.

40 changes: 20 additions & 20 deletions k8s/deployment/kill_instances → k8s/deployment/kill_instance
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,14 @@ set -euo pipefail
log debug "🔍 Starting instance kill operation..."

DEPLOYMENT_ID=$(echo "$CONTEXT" | jq -r '.parameters.deployment_id // .notification.parameters.deployment_id // empty')
INSTANCE_NAME=$(echo "$CONTEXT" | jq -r '.parameters.instance_name // .notification.parameters.instance_name // empty')
INSTANCE_ID=$(echo "$CONTEXT" | jq -r '.parameters.instance_id // .notification.parameters.instance_id // empty')

if [[ -z "$DEPLOYMENT_ID" ]] && [[ -n "${NP_ACTION_CONTEXT:-}" ]]; then
DEPLOYMENT_ID=$(echo "$NP_ACTION_CONTEXT" | jq -r '.notification.parameters.deployment_id // empty')
fi

if [[ -z "$INSTANCE_NAME" ]] && [[ -n "${NP_ACTION_CONTEXT:-}" ]]; then
INSTANCE_NAME=$(echo "$NP_ACTION_CONTEXT" | jq -r '.notification.parameters.instance_name // empty')
if [[ -z "$INSTANCE_ID" ]] && [[ -n "${NP_ACTION_CONTEXT:-}" ]]; then
INSTANCE_ID=$(echo "$NP_ACTION_CONTEXT" | jq -r '.notification.parameters.instance_id // empty')
fi

if [[ -z "$DEPLOYMENT_ID" ]]; then
Expand All @@ -26,18 +26,18 @@ if [[ -z "$DEPLOYMENT_ID" ]]; then
exit 1
fi

if [[ -z "$INSTANCE_NAME" ]]; then
log error "❌ instance_name parameter not found"
if [[ -z "$INSTANCE_ID" ]]; then
log error "❌ instance_id parameter not found"
log error "💡 Possible causes:"
log error " - Parameter not provided in action request"
log error " - Context structure is different than expected"
log error "🔧 How to fix:"
log error " - Ensure instance_name is passed in the action parameters"
log error " - Ensure instance_id is passed in the action parameters"
exit 1
fi

log debug "📋 Deployment ID: $DEPLOYMENT_ID"
log debug "📋 Instance name: $INSTANCE_NAME"
log debug "📋 Instance name: $INSTANCE_ID"

SCOPE_ID=$(echo "$CONTEXT" | jq -r '.tags.scope_id // .scope.id // .notification.tags.scope_id // empty')

Expand All @@ -63,8 +63,8 @@ log debug "📋 Scope ID: $SCOPE_ID"
log debug "📋 Namespace: $K8S_NAMESPACE"

log debug "🔍 Verifying pod exists..."
if ! kubectl get pod "$INSTANCE_NAME" -n "$K8S_NAMESPACE" >/dev/null 2>&1; then
log error "❌ Pod $INSTANCE_NAME not found in namespace $K8S_NAMESPACE"
if ! kubectl get pod "$INSTANCE_ID" -n "$K8S_NAMESPACE" >/dev/null 2>&1; then
log error "❌ Pod $INSTANCE_ID not found in namespace $K8S_NAMESPACE"
log error "💡 Possible causes:"
log error " - Pod was already terminated"
log error " - Pod name is incorrect"
Expand All @@ -75,15 +75,15 @@ if ! kubectl get pod "$INSTANCE_NAME" -n "$K8S_NAMESPACE" >/dev/null 2>&1; then
fi

log debug "📋 Fetching pod details..."
POD_STATUS=$(kubectl get pod "$INSTANCE_NAME" -n "$K8S_NAMESPACE" -o jsonpath='{.status.phase}')
POD_NODE=$(kubectl get pod "$INSTANCE_NAME" -n "$K8S_NAMESPACE" -o jsonpath='{.spec.nodeName}')
POD_START_TIME=$(kubectl get pod "$INSTANCE_NAME" -n "$K8S_NAMESPACE" -o jsonpath='{.status.startTime}')
POD_STATUS=$(kubectl get pod "$INSTANCE_ID" -n "$K8S_NAMESPACE" -o jsonpath='{.status.phase}')
POD_NODE=$(kubectl get pod "$INSTANCE_ID" -n "$K8S_NAMESPACE" -o jsonpath='{.spec.nodeName}')
POD_START_TIME=$(kubectl get pod "$INSTANCE_ID" -n "$K8S_NAMESPACE" -o jsonpath='{.status.startTime}')

log debug "📋 Pod: $INSTANCE_NAME | Status: $POD_STATUS | Node: $POD_NODE | Started: $POD_START_TIME"
log debug "📋 Pod: $INSTANCE_ID | Status: $POD_STATUS | Node: $POD_NODE | Started: $POD_START_TIME"

DEPLOYMENT_NAME="d-$SCOPE_ID-$DEPLOYMENT_ID"

POD_DEPLOYMENT=$(kubectl get pod "$INSTANCE_NAME" -n "$K8S_NAMESPACE" -o jsonpath='{.metadata.ownerReferences[0].name}' 2>/dev/null || echo "")
POD_DEPLOYMENT=$(kubectl get pod "$INSTANCE_ID" -n "$K8S_NAMESPACE" -o jsonpath='{.metadata.ownerReferences[0].name}' 2>/dev/null || echo "")
if [[ -n "$POD_DEPLOYMENT" ]]; then
REPLICASET_DEPLOYMENT=$(kubectl get replicaset "$POD_DEPLOYMENT" -n "$K8S_NAMESPACE" -o jsonpath='{.metadata.ownerReferences[0].name}' 2>/dev/null || echo "")
log debug "📋 Pod ownership: ReplicaSet=$POD_DEPLOYMENT -> Deployment=$REPLICASET_DEPLOYMENT"
Expand All @@ -95,14 +95,14 @@ else
log warn "⚠️ Could not verify pod ownership"
fi

log debug "📝 Deleting pod $INSTANCE_NAME with 30s grace period..."
kubectl delete pod "$INSTANCE_NAME" -n "$K8S_NAMESPACE" --grace-period=30
log debug "📝 Deleting pod $INSTANCE_ID with 30s grace period..."
kubectl delete pod "$INSTANCE_ID" -n "$K8S_NAMESPACE" --grace-period=30

log debug "📝 Waiting for pod termination..."
kubectl wait --for=delete pod/"$INSTANCE_NAME" -n "$K8S_NAMESPACE" --timeout=60s || log warn "⚠️ Pod deletion timeout reached"
kubectl wait --for=delete pod/"$INSTANCE_ID" -n "$K8S_NAMESPACE" --timeout=60s || log warn "⚠️ Pod deletion timeout reached"

if kubectl get pod "$INSTANCE_NAME" -n "$K8S_NAMESPACE" >/dev/null 2>&1; then
POD_STATUS_AFTER=$(kubectl get pod "$INSTANCE_NAME" -n "$K8S_NAMESPACE" -o jsonpath='{.status.phase}')
if kubectl get pod "$INSTANCE_ID" -n "$K8S_NAMESPACE" >/dev/null 2>&1; then
POD_STATUS_AFTER=$(kubectl get pod "$INSTANCE_ID" -n "$K8S_NAMESPACE" -o jsonpath='{.status.phase}')
log warn "⚠️ Pod still exists after deletion attempt (status: $POD_STATUS_AFTER)"
else
log info "✅ Pod successfully terminated and removed"
Expand All @@ -123,4 +123,4 @@ else
log warn "⚠️ Deployment $DEPLOYMENT_NAME not found"
fi

log info "✨ Instance kill operation completed for $INSTANCE_NAME"
log info "✨ Instance kill operation completed for $INSTANCE_ID"
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#!/usr/bin/env bats
# =============================================================================
# Unit tests for deployment/kill_instances - pod termination
# Unit tests for deployment/kill_instance - pod termination
# =============================================================================

setup() {
Expand All @@ -15,7 +15,7 @@ setup() {
export CONTEXT='{
"parameters": {
"deployment_id": "deploy-456",
"instance_name": "my-pod-abc123"
"instance_id": "my-pod-abc123"
},
"tags": {
"scope_id": "scope-123"
Expand Down Expand Up @@ -84,8 +84,8 @@ teardown() {
# =============================================================================
# Success Case
# =============================================================================
@test "kill_instances: successfully kills pod with correct logging" {
run bash "$BATS_TEST_DIRNAME/../kill_instances"
@test "kill_instance: successfully kills pod with correct logging" {
run bash "$BATS_TEST_DIRNAME/../kill_instance"

[ "$status" -eq 0 ]
# Start message
Expand All @@ -110,14 +110,14 @@ teardown() {
# =============================================================================
# Error Cases
# =============================================================================
@test "kill_instances: fails with troubleshooting when deployment_id missing" {
@test "kill_instance: fails with troubleshooting when deployment_id missing" {
export CONTEXT='{
"parameters": {
"instance_name": "my-pod-abc123"
"instance_id": "my-pod-abc123"
}
}'

run bash "$BATS_TEST_DIRNAME/../kill_instances"
run bash "$BATS_TEST_DIRNAME/../kill_instance"

[ "$status" -eq 1 ]
assert_contains "$output" "❌ deployment_id parameter not found"
Expand All @@ -127,32 +127,32 @@ teardown() {
assert_contains "$output" "Ensure deployment_id is passed in the action parameters"
}

@test "kill_instances: fails with troubleshooting when instance_name missing" {
@test "kill_instance: fails with troubleshooting when instance_id missing" {
export CONTEXT='{
"parameters": {
"deployment_id": "deploy-456"
}
}'

run bash "$BATS_TEST_DIRNAME/../kill_instances"
run bash "$BATS_TEST_DIRNAME/../kill_instance"

[ "$status" -eq 1 ]
assert_contains "$output" "❌ instance_name parameter not found"
assert_contains "$output" "❌ instance_id parameter not found"
assert_contains "$output" "💡 Possible causes:"
assert_contains "$output" "Parameter not provided in action request"
assert_contains "$output" "🔧 How to fix:"
assert_contains "$output" "Ensure instance_name is passed in the action parameters"
assert_contains "$output" "Ensure instance_id is passed in the action parameters"
}

@test "kill_instances: fails with troubleshooting when scope_id missing" {
@test "kill_instance: fails with troubleshooting when scope_id missing" {
export CONTEXT='{
"parameters": {
"deployment_id": "deploy-456",
"instance_name": "my-pod-abc123"
"instance_id": "my-pod-abc123"
}
}'

run bash "$BATS_TEST_DIRNAME/../kill_instances"
run bash "$BATS_TEST_DIRNAME/../kill_instance"

[ "$status" -eq 1 ]
assert_contains "$output" "❌ scope_id not found in context"
Expand All @@ -162,7 +162,7 @@ teardown() {
assert_contains "$output" "Verify the action is invoked with proper scope context"
}

@test "kill_instances: fails with troubleshooting when pod not found" {
@test "kill_instance: fails with troubleshooting when pod not found" {
kubectl() {
case "$1" in
get)
Expand All @@ -175,7 +175,7 @@ teardown() {
}
export -f kubectl

run bash "$BATS_TEST_DIRNAME/../kill_instances"
run bash "$BATS_TEST_DIRNAME/../kill_instance"

[ "$status" -eq 1 ]
assert_contains "$output" "❌ Pod my-pod-abc123 not found in namespace test-namespace"
Expand All @@ -188,7 +188,7 @@ teardown() {
# =============================================================================
# Warning Cases
# =============================================================================
@test "kill_instances: warns when pod belongs to different deployment" {
@test "kill_instance: warns when pod belongs to different deployment" {
kubectl() {
case "$1" in
get)
Expand Down Expand Up @@ -230,13 +230,13 @@ teardown() {
}
export -f kubectl

run bash "$BATS_TEST_DIRNAME/../kill_instances"
run bash "$BATS_TEST_DIRNAME/../kill_instance"

[ "$status" -eq 0 ]
assert_contains "$output" "⚠️ Pod does not belong to expected deployment d-scope-123-deploy-456"
}

@test "kill_instances: warns when pod still exists after deletion" {
@test "kill_instance: warns when pod still exists after deletion" {
local delete_called=0
kubectl() {
case "$1" in
Expand Down Expand Up @@ -279,7 +279,7 @@ teardown() {
}
export -f kubectl

run bash "$BATS_TEST_DIRNAME/../kill_instances"
run bash "$BATS_TEST_DIRNAME/../kill_instance"

[ "$status" -eq 0 ]
assert_contains "$output" "⚠️ Pod deletion timeout reached"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,6 @@ steps:
parameters:
level: string
message: string
- name: kill instances
- name: kill instance
type: script
file: "$SERVICE_PATH/deployment/kill_instances"
file: "$SERVICE_PATH/deployment/kill_instance"
Loading
Loading