diff --git a/CHANGELOG.md b/CHANGELOG.md index 1aa174b9..1e6ca99b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] +- Fix: triggering a scheduled task job on a scope that is not deployed now fails with a clear "deploy the scope first" message instead of an opaque error +- Add "Kill job execution" action to scheduled task scopes to terminate an individual running job execution + ## [1.13.0] - 2026-07-10 - Add support to get AWS credentials via assume role - Add support to auto-create ALBs on scope create diff --git a/scheduled_task/deployment/kill_job_execution b/scheduled_task/deployment/kill_job_execution new file mode 100755 index 00000000..6314c168 --- /dev/null +++ b/scheduled_task/deployment/kill_job_execution @@ -0,0 +1,126 @@ +#!/bin/bash + +set -euo pipefail + + +log debug "🔍 Starting job execution kill operation..." + +DEPLOYMENT_ID=$(echo "$CONTEXT" | jq -r '.parameters.deployment_id // .notification.parameters.deployment_id // empty') +INSTANCE_NAME=$(echo "$CONTEXT" | jq -r '.parameters.instance_name // .notification.parameters.instance_name // empty') + +if [[ -z "$DEPLOYMENT_ID" ]] && [[ -n "${NP_ACTION_CONTEXT:-}" ]]; then + DEPLOYMENT_ID=$(echo "$NP_ACTION_CONTEXT" | jq -r '.notification.parameters.deployment_id // empty') +fi + +if [[ -z "$INSTANCE_NAME" ]] && [[ -n "${NP_ACTION_CONTEXT:-}" ]]; then + INSTANCE_NAME=$(echo "$NP_ACTION_CONTEXT" | jq -r '.notification.parameters.instance_name // empty') +fi + +if [[ -z "$DEPLOYMENT_ID" ]]; then + log error "❌ deployment_id parameter not found" + log error "💡 Possible causes:" + log error " - Parameter not provided in action request" + log error " - Context structure is different than expected" + log error "🔧 How to fix:" + log error " - Ensure deployment_id is passed in the action parameters" + exit 1 +fi + +if [[ -z "$INSTANCE_NAME" ]]; then + log error "❌ instance_name parameter not found" + log error "💡 Possible causes:" + log error " - Parameter not provided in action request" + log error " - Context structure is different than expected" + log error "🔧 How to fix:" + log error " - Ensure instance_name is passed in the action parameters" + exit 1 +fi + +log debug "📋 Deployment ID: $DEPLOYMENT_ID" +log debug "📋 Job execution: $INSTANCE_NAME" + +SCOPE_ID=$(echo "$CONTEXT" | jq -r '.tags.scope_id // .scope.id // .notification.tags.scope_id // empty') + +if [[ -z "$SCOPE_ID" ]] && [[ -n "${NP_ACTION_CONTEXT:-}" ]]; then + SCOPE_ID=$(echo "$NP_ACTION_CONTEXT" | jq -r '.notification.tags.scope_id // empty') +fi + +K8S_NAMESPACE=$(echo "$CONTEXT" | jq -r --arg default "$K8S_NAMESPACE" ' + .providers["container-orchestration"].cluster.namespace // $default +' 2>/dev/null || echo "nullplatform") + +if [[ -z "$SCOPE_ID" ]]; then + log error "❌ scope_id not found in context" + log error "💡 Possible causes:" + log error " - Context missing scope information" + log error " - Action invoked outside of scope context" + log error "🔧 How to fix:" + log error " - Verify the action is invoked with proper scope context" + exit 1 +fi + +log debug "📋 Scope ID: $SCOPE_ID" +log debug "📋 Namespace: $K8S_NAMESPACE" + +log debug "🔍 Verifying job execution exists..." +if ! kubectl get pod "$INSTANCE_NAME" -n "$K8S_NAMESPACE" >/dev/null 2>&1; then + log error "❌ Job execution $INSTANCE_NAME not found in namespace $K8S_NAMESPACE" + log error "💡 Possible causes:" + log error " - The job execution was already terminated" + log error " - The job execution name is incorrect" + log error " - The job execution runs in a different namespace" + log error "🔧 How to fix:" + log error " - List job executions: kubectl get pods -n $K8S_NAMESPACE -l scope_id=$SCOPE_ID" + exit 1 +fi + +log debug "📋 Fetching job execution details..." +POD_STATUS=$(kubectl get pod "$INSTANCE_NAME" -n "$K8S_NAMESPACE" -o jsonpath='{.status.phase}') +POD_NODE=$(kubectl get pod "$INSTANCE_NAME" -n "$K8S_NAMESPACE" -o jsonpath='{.spec.nodeName}') +POD_START_TIME=$(kubectl get pod "$INSTANCE_NAME" -n "$K8S_NAMESPACE" -o jsonpath='{.status.startTime}') + +log debug "📋 Status: $POD_STATUS | Node: $POD_NODE | Started: $POD_START_TIME" + +CRONJOB_NAME="job-$SCOPE_ID-$DEPLOYMENT_ID" + +POD_JOB=$(kubectl get pod "$INSTANCE_NAME" -n "$K8S_NAMESPACE" -o jsonpath='{.metadata.ownerReferences[0].name}' 2>/dev/null || echo "") +if [[ -n "$POD_JOB" ]]; then + JOB_CRONJOB=$(kubectl get job "$POD_JOB" -n "$K8S_NAMESPACE" -o jsonpath='{.metadata.ownerReferences[0].name}' 2>/dev/null || echo "") + log debug "📋 Ownership: Job=$POD_JOB -> CronJob=$JOB_CRONJOB" + + if [[ "$JOB_CRONJOB" != "$CRONJOB_NAME" ]]; then + log warn "⚠️ Job execution does not belong to expected scheduled task $CRONJOB_NAME (continuing anyway)" + fi +else + log warn "⚠️ Could not verify job execution ownership" +fi + +log debug "📝 Deleting job execution $INSTANCE_NAME with 30s grace period..." +kubectl delete pod "$INSTANCE_NAME" -n "$K8S_NAMESPACE" --grace-period=30 + +log debug "📝 Waiting for job execution termination..." +kubectl wait --for=delete pod/"$INSTANCE_NAME" -n "$K8S_NAMESPACE" --timeout=60s || log warn "⚠️ Job execution deletion timeout reached" + +if kubectl get pod "$INSTANCE_NAME" -n "$K8S_NAMESPACE" >/dev/null 2>&1; then + POD_STATUS_AFTER=$(kubectl get pod "$INSTANCE_NAME" -n "$K8S_NAMESPACE" -o jsonpath='{.status.phase}') + log warn "⚠️ Job execution still exists after deletion attempt (status: $POD_STATUS_AFTER)" +else + log info "✅ Job execution successfully terminated and removed" +fi + +log debug "📋 Checking parent job status after execution deletion..." +if [[ -n "$POD_JOB" ]] && kubectl get job "$POD_JOB" -n "$K8S_NAMESPACE" >/dev/null 2>&1; then + JOB_ACTIVE=$(kubectl get job "$POD_JOB" -n "$K8S_NAMESPACE" -o jsonpath='{.status.active}') + JOB_SUCCEEDED=$(kubectl get job "$POD_JOB" -n "$K8S_NAMESPACE" -o jsonpath='{.status.succeeded}') + JOB_FAILED=$(kubectl get job "$POD_JOB" -n "$K8S_NAMESPACE" -o jsonpath='{.status.failed}') + + log debug "📋 Job $POD_JOB: active=${JOB_ACTIVE:-0}, succeeded=${JOB_SUCCEEDED:-0}, failed=${JOB_FAILED:-0}" + + if [[ "${JOB_ACTIVE:-0}" -gt 0 ]]; then + log debug "📋 Job is still active; Kubernetes may start a replacement execution (backoffLimit permitting)" + fi +else + log warn "⚠️ Parent job not found (it may have already completed)" +fi + +log info "✨ Job execution kill operation completed for $INSTANCE_NAME" diff --git a/scheduled_task/deployment/tests/kill_job_execution.bats b/scheduled_task/deployment/tests/kill_job_execution.bats new file mode 100644 index 00000000..e0a44034 --- /dev/null +++ b/scheduled_task/deployment/tests/kill_job_execution.bats @@ -0,0 +1,412 @@ +#!/usr/bin/env bats +# ============================================================================= +# Unit tests for scheduled_task deployment/kill_job_execution - job pod termination +# +# scheduled_task pods are owned by Job -> CronJob (job--), +# unlike the base k8s scope where pods are owned by ReplicaSet -> Deployment. +# ============================================================================= + +setup() { + export PROJECT_ROOT="$(cd "$BATS_TEST_DIRNAME/../../.." && pwd)" + source "$PROJECT_ROOT/testing/assertions.sh" + log() { if [ "$1" = "error" ]; then echo "$2" >&2; else echo "$2"; fi; } + export -f log + + export K8S_NAMESPACE="test-namespace" + export SCOPE_ID="scope-123" + export KILL_STATE="$BATS_TEST_TMPDIR/deleted" + rm -f "$KILL_STATE" + + export CONTEXT='{ + "parameters": { + "deployment_id": "deploy-456", + "instance_name": "my-pod-abc123" + }, + "tags": { + "scope_id": "scope-123" + }, + "providers": { + "container-orchestration": { + "cluster": { + "namespace": "test-namespace" + } + } + } + }' + + # Default mock: pod owned by a Job that belongs to the expected CronJob. + # A state file flips the plain (no -o) pod existence check to "gone" once + # `kubectl delete` has run, exercising the happy termination path. + kubectl() { + case "$1" in + get) + case "$2" in + pod) + if [[ "$*" == *"-o jsonpath"* ]]; then + if [[ "$*" == *"phase"* ]]; then + echo "Running" + elif [[ "$*" == *"nodeName"* ]]; then + echo "node-1" + elif [[ "$*" == *"startTime"* ]]; then + echo "2024-01-01T00:00:00Z" + elif [[ "$*" == *"ownerReferences"* ]]; then + echo "job-scope-123-deploy-456-abc" + fi + return 0 + fi + # existence check (no -o): gone once deleted + [[ -f "$KILL_STATE" ]] && return 1 + return 0 + ;; + job) + if [[ "$*" == *"-o jsonpath"* ]]; then + if [[ "$*" == *"ownerReferences"* ]]; then + echo "job-scope-123-deploy-456" + elif [[ "$*" == *"active"* ]]; then + echo "0" + elif [[ "$*" == *"succeeded"* ]]; then + echo "1" + elif [[ "$*" == *"failed"* ]]; then + echo "0" + fi + return 0 + fi + return 0 + ;; + esac + ;; + delete) + touch "$KILL_STATE" + return 0 + ;; + wait) + return 0 + ;; + esac + return 0 + } + export -f kubectl +} + +teardown() { + unset CONTEXT + unset -f kubectl +} + +# ============================================================================= +# Success Case +# ============================================================================= +@test "kill_job_execution: successfully kills job pod with correct logging" { + run bash "$BATS_TEST_DIRNAME/../kill_job_execution" + + [ "$status" -eq 0 ] + # Start message + assert_contains "$output" "🔍 Starting job execution kill operation..." + # Parameter display + assert_contains "$output" "📋 Deployment ID: deploy-456" + assert_contains "$output" "📋 Job execution: my-pod-abc123" + assert_contains "$output" "📋 Scope ID: scope-123" + assert_contains "$output" "📋 Namespace: test-namespace" + # Job execution verification + assert_contains "$output" "🔍 Verifying job execution exists..." + assert_contains "$output" "📋 Fetching job execution details..." + assert_contains "$output" "📋 Status: Running | Node: node-1 | Started: 2024-01-01T00:00:00Z" + # Ownership (Job -> CronJob) + assert_contains "$output" "📋 Ownership: Job=job-scope-123-deploy-456-abc -> CronJob=job-scope-123-deploy-456" + # Delete operation + assert_contains "$output" "📝 Deleting job execution my-pod-abc123 with 30s grace period..." + assert_contains "$output" "📝 Waiting for job execution termination..." + assert_contains "$output" "✅ Job execution successfully terminated and removed" + # Parent job status + assert_contains "$output" "📋 Checking parent job status after execution deletion..." + assert_contains "$output" "📋 Job job-scope-123-deploy-456-abc: active=0, succeeded=1, failed=0" + # Completion + assert_contains "$output" "✨ Job execution kill operation completed for my-pod-abc123" +} + +# ============================================================================= +# Error Cases +# ============================================================================= +@test "kill_job_execution: fails with troubleshooting when deployment_id missing" { + export CONTEXT='{ + "parameters": { + "instance_name": "my-pod-abc123" + } + }' + + run bash "$BATS_TEST_DIRNAME/../kill_job_execution" + + [ "$status" -eq 1 ] + assert_contains "$output" "❌ deployment_id parameter not found" + assert_contains "$output" "💡 Possible causes:" + assert_contains "$output" "Parameter not provided in action request" + assert_contains "$output" "🔧 How to fix:" + assert_contains "$output" "Ensure deployment_id is passed in the action parameters" +} + +@test "kill_job_execution: fails with troubleshooting when instance_name missing" { + export CONTEXT='{ + "parameters": { + "deployment_id": "deploy-456" + } + }' + + run bash "$BATS_TEST_DIRNAME/../kill_job_execution" + + [ "$status" -eq 1 ] + assert_contains "$output" "❌ instance_name parameter not found" + assert_contains "$output" "💡 Possible causes:" + assert_contains "$output" "Parameter not provided in action request" + assert_contains "$output" "🔧 How to fix:" + assert_contains "$output" "Ensure instance_name is passed in the action parameters" +} + +@test "kill_job_execution: fails with troubleshooting when scope_id missing" { + export CONTEXT='{ + "parameters": { + "deployment_id": "deploy-456", + "instance_name": "my-pod-abc123" + } + }' + + run bash "$BATS_TEST_DIRNAME/../kill_job_execution" + + [ "$status" -eq 1 ] + assert_contains "$output" "❌ scope_id not found in context" + assert_contains "$output" "💡 Possible causes:" + assert_contains "$output" "Context missing scope information" + assert_contains "$output" "🔧 How to fix:" + assert_contains "$output" "Verify the action is invoked with proper scope context" +} + +@test "kill_job_execution: fails with troubleshooting when pod not found" { + kubectl() { + case "$1" in + get) + if [[ "$2" == "pod" ]] && [[ "$*" != *"-o"* ]]; then + return 1 + fi + ;; + esac + return 0 + } + export -f kubectl + + run bash "$BATS_TEST_DIRNAME/../kill_job_execution" + + [ "$status" -eq 1 ] + assert_contains "$output" "❌ Job execution my-pod-abc123 not found in namespace test-namespace" + assert_contains "$output" "💡 Possible causes:" + assert_contains "$output" "The job execution was already terminated" + assert_contains "$output" "🔧 How to fix:" + assert_contains "$output" "kubectl get pods" +} + +# ============================================================================= +# Warning Cases +# ============================================================================= +@test "kill_job_execution: warns when pod belongs to a different scheduled task" { + kubectl() { + case "$1" in + get) + case "$2" in + pod) + if [[ "$*" == *"-o jsonpath"* ]]; then + if [[ "$*" == *"phase"* ]]; then + echo "Running" + elif [[ "$*" == *"nodeName"* ]]; then + echo "node-1" + elif [[ "$*" == *"startTime"* ]]; then + echo "2024-01-01T00:00:00Z" + elif [[ "$*" == *"ownerReferences"* ]]; then + echo "job-scope-123-deploy-456-abc" + fi + return 0 + fi + [[ -f "$KILL_STATE" ]] && return 1 + return 0 + ;; + job) + if [[ "$*" == *"-o jsonpath"* ]]; then + if [[ "$*" == *"ownerReferences"* ]]; then + echo "job-scope-123-different-deploy" # Different CronJob + elif [[ "$*" == *"active"* ]]; then + echo "0" + fi + return 0 + fi + return 0 + ;; + esac + ;; + delete) + touch "$KILL_STATE" + return 0 + ;; + wait) + return 0 + ;; + esac + return 0 + } + export -f kubectl + + run bash "$BATS_TEST_DIRNAME/../kill_job_execution" + + [ "$status" -eq 0 ] + assert_contains "$output" "⚠️ Job execution does not belong to expected scheduled task job-scope-123-deploy-456 (continuing anyway)" +} + +@test "kill_job_execution: warns when pod ownership cannot be verified" { + kubectl() { + case "$1" in + get) + case "$2" in + pod) + if [[ "$*" == *"-o jsonpath"* ]]; then + if [[ "$*" == *"phase"* ]]; then + echo "Running" + elif [[ "$*" == *"nodeName"* ]]; then + echo "node-1" + elif [[ "$*" == *"startTime"* ]]; then + echo "2024-01-01T00:00:00Z" + elif [[ "$*" == *"ownerReferences"* ]]; then + echo "" # Bare pod, no owner + fi + return 0 + fi + [[ -f "$KILL_STATE" ]] && return 1 + return 0 + ;; + job) + [[ -f "$KILL_STATE" ]] && return 1 + return 0 + ;; + esac + ;; + delete) + touch "$KILL_STATE" + return 0 + ;; + wait) + return 0 + ;; + esac + return 0 + } + export -f kubectl + + run bash "$BATS_TEST_DIRNAME/../kill_job_execution" + + [ "$status" -eq 0 ] + assert_contains "$output" "⚠️ Could not verify job execution ownership" + # With no owner Job, the post-deletion job lookup cannot resolve one either + assert_contains "$output" "⚠️ Parent job not found (it may have already completed)" +} + +@test "kill_job_execution: warns when pod still exists after deletion" { + kubectl() { + case "$1" in + get) + case "$2" in + pod) + if [[ "$*" == *"-o jsonpath"* ]]; then + if [[ "$*" == *"phase"* ]]; then + echo "Terminating" + elif [[ "$*" == *"nodeName"* ]]; then + echo "node-1" + elif [[ "$*" == *"startTime"* ]]; then + echo "2024-01-01T00:00:00Z" + elif [[ "$*" == *"ownerReferences"* ]]; then + echo "job-scope-123-deploy-456-abc" + fi + return 0 + fi + return 0 # Pod still exists + ;; + job) + if [[ "$*" == *"-o jsonpath"* ]]; then + if [[ "$*" == *"ownerReferences"* ]]; then + echo "job-scope-123-deploy-456" + elif [[ "$*" == *"active"* ]]; then + echo "1" + fi + return 0 + fi + return 0 + ;; + esac + ;; + delete) + return 0 + ;; + wait) + return 1 # Timeout + ;; + esac + return 0 + } + export -f kubectl + + run bash "$BATS_TEST_DIRNAME/../kill_job_execution" + + [ "$status" -eq 0 ] + assert_contains "$output" "⚠️ Job execution deletion timeout reached" + assert_contains "$output" "⚠️ Job execution still exists after deletion attempt (status: Terminating)" + # Job still active -> replacement execution may be started + assert_contains "$output" "📋 Checking parent job status after execution deletion..." + assert_contains "$output" "📋 Job job-scope-123-deploy-456-abc: active=1, succeeded=0, failed=0" + assert_contains "$output" "📋 Job is still active; Kubernetes may start a replacement execution (backoffLimit permitting)" +} + +@test "kill_job_execution: warns when job already completed after pod deletion" { + kubectl() { + case "$1" in + get) + case "$2" in + pod) + if [[ "$*" == *"-o jsonpath"* ]]; then + if [[ "$*" == *"phase"* ]]; then + echo "Running" + elif [[ "$*" == *"nodeName"* ]]; then + echo "node-1" + elif [[ "$*" == *"startTime"* ]]; then + echo "2024-01-01T00:00:00Z" + elif [[ "$*" == *"ownerReferences"* ]]; then + echo "job-scope-123-deploy-456-abc" + fi + return 0 + fi + [[ -f "$KILL_STATE" ]] && return 1 + return 0 + ;; + job) + if [[ "$*" == *"-o jsonpath"* ]]; then + if [[ "$*" == *"ownerReferences"* ]]; then + echo "job-scope-123-deploy-456" + fi + return 0 + fi + # Job existence check: gone once the pod was deleted + [[ -f "$KILL_STATE" ]] && return 1 + return 0 + ;; + esac + ;; + delete) + touch "$KILL_STATE" + return 0 + ;; + wait) + return 0 + ;; + esac + return 0 + } + export -f kubectl + + run bash "$BATS_TEST_DIRNAME/../kill_job_execution" + + [ "$status" -eq 0 ] + assert_contains "$output" "⚠️ Parent job not found (it may have already completed)" +} diff --git a/scheduled_task/deployment/tests/workflow_overrides.bats b/scheduled_task/deployment/tests/workflow_overrides.bats index 591d00db..69aa520f 100644 --- a/scheduled_task/deployment/tests/workflow_overrides.bats +++ b/scheduled_task/deployment/tests/workflow_overrides.bats @@ -34,3 +34,39 @@ setup() { assert_equal "$status" "0" assert_contains "$output" "action: skip" } + +# ============================================================================= +# kill job execution +# +# scheduled_task pods are owned by Job -> CronJob, so this action ships a +# STANDALONE deployment workflow (not an override of a k8s base) with its own +# steps. The deployment entrypoint falls back to the override-provided workflow +# because k8s has no kill_job_execution counterpart. +# ============================================================================= +@test "kill_job_execution has no base k8s workflow (it is standalone)" { + run test -f "$PROJECT_ROOT/k8s/deployment/workflows/kill_job_execution.yaml" + + assert_equal "$status" "1" +} + +@test "scheduled_task deployment kill_job_execution workflow loads its own logging function" { + run grep -A 3 "name: load logging" "$PROJECT_ROOT/scheduled_task/deployment/workflows/kill_job_execution.yaml" + + assert_equal "$status" "0" + assert_contains "$output" "type: script" + assert_contains "$output" "\$OVERRIDES_PATH/logging" +} + +@test "scheduled_task deployment kill_job_execution workflow runs its own kill script" { + run grep -A 2 "name: kill job execution" "$PROJECT_ROOT/scheduled_task/deployment/workflows/kill_job_execution.yaml" + + assert_equal "$status" "0" + assert_contains "$output" "type: script" + assert_contains "$output" "\$OVERRIDES_PATH/deployment/kill_job_execution" +} + +@test "scheduled_task deployment kill_job_execution workflow is not an override (no action: replace)" { + run grep -c "action: replace" "$PROJECT_ROOT/scheduled_task/deployment/workflows/kill_job_execution.yaml" + + assert_contains "$output" "0" +} diff --git a/scheduled_task/deployment/workflows/kill_job_execution.yaml b/scheduled_task/deployment/workflows/kill_job_execution.yaml new file mode 100644 index 00000000..ef8de4bd --- /dev/null +++ b/scheduled_task/deployment/workflows/kill_job_execution.yaml @@ -0,0 +1,15 @@ +include: + - "$SERVICE_PATH/values.yaml" +steps: + - name: load logging + type: script + file: "$OVERRIDES_PATH/logging" + output: + - name: log + type: function + parameters: + level: string + message: string + - name: kill job execution + type: script + file: "$OVERRIDES_PATH/deployment/kill_job_execution" diff --git a/scheduled_task/logging b/scheduled_task/logging new file mode 100644 index 00000000..d0df55d7 --- /dev/null +++ b/scheduled_task/logging @@ -0,0 +1,41 @@ +#!/bin/bash + +# Logging utility — log4j-style level filtering +# Usage: log "level" "message" +# Levels: debug < info < warn < error +# Control: LOG_LEVEL env var (default: info) +# +# Example: +# LOG_LEVEL=info +# log debug "verbose details" # suppressed +# log info "deployment done" # shown +log() { + local level="${1:-info}" + local message="${2:-}" + + local -i msg_num threshold + + case "${level,,}" in + debug) msg_num=0 ;; + info) msg_num=1 ;; + warn) msg_num=2 ;; + error) msg_num=3 ;; + *) msg_num=1 ;; + esac + + case "${LOG_LEVEL:-info}" in + debug) threshold=0 ;; + info) threshold=1 ;; + warn) threshold=2 ;; + error) threshold=3 ;; + *) threshold=1 ;; + esac + + if [ "$msg_num" -ge "$threshold" ]; then + if [ "$msg_num" -ge 3 ]; then + echo "$message" >&2 + else + echo "$message" + fi + fi +} diff --git a/scheduled_task/scope/tests/trigger.bats b/scheduled_task/scope/tests/trigger.bats new file mode 100644 index 00000000..18ff2732 --- /dev/null +++ b/scheduled_task/scope/tests/trigger.bats @@ -0,0 +1,236 @@ +#!/usr/bin/env bats +# ============================================================================= +# Unit tests for scheduled_task scope/trigger - manually trigger the CronJob +# +# Contract: +# - A scope with no active deployment has no CronJob, so triggering must fail +# early with a clear "deploy first" message instead of an opaque kubectl +# error from `kubectl create job --from=cronjob/`. +# ============================================================================= + +setup() { + export PROJECT_ROOT="$(cd "$BATS_TEST_DIRNAME/../../.." && pwd)" + source "$PROJECT_ROOT/testing/assertions.sh" + + # The workflow loads the `log` function via a `load logging` step, so the + # script assumes it exists. Mock it here (errors go to stderr, like the real one). + log() { if [ "$1" = "error" ]; then echo "$2" >&2; else echo "$2"; fi; } + export -f log + + export K8S_NAMESPACE="default-namespace" + + # Base CONTEXT: deployed scope with an active deployment + export CONTEXT='{ + "scope": { + "id": "scope-123", + "current_active_deployment": "deploy-456" + }, + "providers": { + "container-orchestration": { + "cluster": { + "namespace": "provider-namespace" + } + } + } + }' + + # Mock kubectl: cronjob exists, job creation succeeds + kubectl() { + case "$*" in + "get cronjob -n provider-namespace -l scope_id=scope-123 -o jsonpath={.items[0].metadata.name}") + echo "my-cronjob" + return 0 + ;; + "create job --from=cronjob/my-cronjob"*) + return 0 + ;; + *) + return 0 + ;; + esac + } + export -f kubectl + + # Deterministic timestamp for job name suffix + date() { echo "1700000000"; } + export -f date +} + +teardown() { + unset -f kubectl date log +} + +# ============================================================================= +# Success Flow +# ============================================================================= +@test "trigger: success flow - finds cronjob and triggers a job" { + run bash "$BATS_TEST_DIRNAME/../trigger" + + [ "$status" -eq 0 ] + assert_contains "$output" "📝 Triggering job my-cronjob" + assert_contains "$output" "✅ The job my-cronjob was triggered, you can follow the execution from the logs screen" +} + +# ============================================================================= +# Error: scope not deployed (no active deployment) +# ============================================================================= +@test "trigger: fails with a clear message when the scope has no active deployment" { + export CONTEXT=$(echo "$CONTEXT" | jq 'del(.scope.current_active_deployment)') + + run bash "$BATS_TEST_DIRNAME/../trigger" + + [ "$status" -eq 1 ] + assert_contains "$output" "❌ The scope has no active deployment, so there is no scheduled job to trigger" + assert_contains "$output" "💡 Possible causes:" + assert_contains "$output" "The scope has not been deployed yet" + assert_contains "$output" "🔧 How to fix:" + assert_contains "$output" "• Deploy the scope first, then trigger the job" +} + +@test "trigger: fails when current_active_deployment is null" { + export CONTEXT=$(echo "$CONTEXT" | jq '.scope.current_active_deployment = null') + + run bash "$BATS_TEST_DIRNAME/../trigger" + + [ "$status" -eq 1 ] + assert_contains "$output" "❌ The scope has no active deployment, so there is no scheduled job to trigger" + assert_contains "$output" "• Deploy the scope first, then trigger the job" +} + +# ============================================================================= +# Error: deployed but no CronJob found +# ============================================================================= +@test "trigger: fails with a clear message when no CronJob exists for the scope" { + kubectl() { + case "$*" in + "get cronjob -n provider-namespace -l scope_id=scope-123 -o jsonpath={.items[0].metadata.name}") + echo "" + return 0 + ;; + *) + return 0 + ;; + esac + } + export -f kubectl + + run bash "$BATS_TEST_DIRNAME/../trigger" + + [ "$status" -eq 1 ] + assert_contains "$output" "❌ No CronJob found for scope 'scope-123' in namespace 'provider-namespace'" + assert_contains "$output" "💡 Possible causes:" + assert_contains "$output" "🔧 How to fix:" +} + +# ============================================================================= +# Namespace resolution +# ============================================================================= +@test "trigger: uses the namespace from the provider for lookup and job creation" { + # kubectl only succeeds when called against the provider namespace; any other + # namespace makes it fail, so this asserts the resolved namespace is threaded + # through both the cronjob lookup and the job creation. + kubectl() { + case "$*" in + "get cronjob -n provider-namespace"*) + echo "my-cronjob" + return 0 + ;; + "create job --from=cronjob/my-cronjob"*"-n provider-namespace") + return 0 + ;; + *) + echo "unexpected namespace: $*" >&2 + return 1 + ;; + esac + } + export -f kubectl + + run bash "$BATS_TEST_DIRNAME/../trigger" + + [ "$status" -eq 0 ] + assert_contains "$output" "✅ The job my-cronjob was triggered, you can follow the execution from the logs screen" +} + +@test "trigger: falls back to the default namespace when the provider namespace is not set" { + export CONTEXT=$(echo "$CONTEXT" | jq 'del(.providers["container-orchestration"].cluster.namespace)') + + kubectl() { + case "$*" in + "get cronjob -n default-namespace"*) + echo "my-cronjob" + return 0 + ;; + "create job --from=cronjob/my-cronjob"*"-n default-namespace") + return 0 + ;; + *) + echo "unexpected namespace: $*" >&2 + return 1 + ;; + esac + } + export -f kubectl + + run bash "$BATS_TEST_DIRNAME/../trigger" + + [ "$status" -eq 0 ] + assert_contains "$output" "✅ The job my-cronjob was triggered, you can follow the execution from the logs screen" +} + +# The workflow includes values.yaml to set K8S_NAMESPACE, but the script also +# guards the expansion so that a missing include can never abort it under +# `set -u`. These reproduce that unguarded environment (K8S_NAMESPACE unset). +@test "trigger: does not abort under set -u when K8S_NAMESPACE is unset (uses provider namespace)" { + unset K8S_NAMESPACE + + run bash "$BATS_TEST_DIRNAME/../trigger" + + [ "$status" -eq 0 ] + assert_contains "$output" "✅ The job my-cronjob was triggered, you can follow the execution from the logs screen" +} + +@test "trigger: falls back to the nullplatform default when K8S_NAMESPACE is unset and the provider namespace is absent" { + unset K8S_NAMESPACE + export CONTEXT=$(echo "$CONTEXT" | jq 'del(.providers["container-orchestration"].cluster.namespace)') + + kubectl() { + case "$*" in + "get cronjob -n nullplatform"*) + echo "my-cronjob" + return 0 + ;; + "create job --from=cronjob/my-cronjob"*"-n nullplatform") + return 0 + ;; + *) + echo "unexpected namespace: $*" >&2 + return 1 + ;; + esac + } + export -f kubectl + + run bash "$BATS_TEST_DIRNAME/../trigger" + + [ "$status" -eq 0 ] + assert_contains "$output" "✅ The job my-cronjob was triggered, you can follow the execution from the logs screen" +} + +# ============================================================================= +# Workflow wiring (trigger-job.yaml) — the layer the script tests can't see +# ============================================================================= +@test "trigger-job workflow includes values.yaml so K8S_NAMESPACE is provided" { + run grep -A2 "^include:" "$BATS_TEST_DIRNAME/../workflows/trigger-job.yaml" + + assert_equal "$status" "0" + assert_contains "$output" "values.yaml" +} + +@test "trigger-job workflow loads the log function before the trigger step" { + run cat "$BATS_TEST_DIRNAME/../workflows/trigger-job.yaml" + + assert_equal "$status" "0" + assert_contains "$output" "name: load logging" + assert_contains "$output" "\$OVERRIDES_PATH/logging" +} diff --git a/scheduled_task/scope/trigger b/scheduled_task/scope/trigger index 1ebf6cce..0aeefc5a 100644 --- a/scheduled_task/scope/trigger +++ b/scheduled_task/scope/trigger @@ -1,6 +1,22 @@ #!/bin/bash -K8S_NAMESPACE=$(echo "$CONTEXT" | jq -r --arg default "$K8S_NAMESPACE" ' +set -euo pipefail + +DEPLOYMENT_ID=$(echo "$CONTEXT" | jq -r '.scope.current_active_deployment // empty') + +if [[ -z "$DEPLOYMENT_ID" ]]; then + log error "❌ The scope has no active deployment, so there is no scheduled job to trigger" + log error "" + log error "💡 Possible causes:" + log error " The scope has not been deployed yet" + log error "" + log error "🔧 How to fix:" + log error " • Deploy the scope first, then trigger the job" + log error "" + exit 1 +fi + +K8S_NAMESPACE=$(echo "$CONTEXT" | jq -r --arg default "${K8S_NAMESPACE:-nullplatform}" ' .providers["container-orchestration"].cluster.namespace // $default ') @@ -8,8 +24,21 @@ SCOPE_ID=$(echo "$CONTEXT" | jq -r '.scope.id') JOB=$(kubectl get cronjob -n "$K8S_NAMESPACE" -l "scope_id=$SCOPE_ID" -o jsonpath="{.items[0].metadata.name}" 2>/dev/null || echo "") -echo "Triggering job $JOB" +if [[ -z "$JOB" ]]; then + log error "❌ No CronJob found for scope '$SCOPE_ID' in namespace '$K8S_NAMESPACE'" + log error "" + log error "💡 Possible causes:" + log error " The scope's scheduled job may not have been created yet, or the deployment is still in progress" + log error "" + log error "🔧 How to fix:" + log error " • Verify the CronJob exists: kubectl get cronjob -n $K8S_NAMESPACE -l scope_id=$SCOPE_ID" + log error " • Redeploy the scope if the CronJob is missing" + log error "" + exit 1 +fi + +log info "📝 Triggering job $JOB" kubectl create job --from="cronjob/$JOB" "$JOB-$(date +%s)" -n "$K8S_NAMESPACE" -echo "The job $JOB was triggered, you can follow the execution from the logs screen" \ No newline at end of file +log info "✅ The job $JOB was triggered, you can follow the execution from the logs screen" diff --git a/scheduled_task/scope/workflows/trigger-job.yaml b/scheduled_task/scope/workflows/trigger-job.yaml index 6b07c1ce..02df28f0 100644 --- a/scheduled_task/scope/workflows/trigger-job.yaml +++ b/scheduled_task/scope/workflows/trigger-job.yaml @@ -1,7 +1,18 @@ +include: + - "$SERVICE_PATH/values.yaml" provider_categories: - container-orchestration - cloud-providers steps: + - name: load logging + type: script + file: "$OVERRIDES_PATH/logging" + output: + - name: log + type: function + parameters: + level: string + message: string - name: trigger type: script - file: "$OVERRIDES_PATH/scope/trigger" \ No newline at end of file + file: "$OVERRIDES_PATH/scope/trigger" diff --git a/scheduled_task/specs/actions/kill-job-execution.json.tpl b/scheduled_task/specs/actions/kill-job-execution.json.tpl new file mode 100644 index 00000000..405a6f85 --- /dev/null +++ b/scheduled_task/specs/actions/kill-job-execution.json.tpl @@ -0,0 +1,18 @@ +{ + "name": "Kill job execution", + "type": "custom", + "icon": "material-symbols:delete-outline", + "results": { + "schema": { "type": "object", "required": [], "properties": {} }, + "values": {} + }, + "service_specification_id": "{{ env.Getenv "SERVICE_SPECIFICATION_ID" }}", + "parameters": { + "schema": { "type": "object", "required": [], "properties": {} }, + "values": {} + }, + "annotations": { + "show_on": ["performance"], + "runs_over": "scope" + } +} \ No newline at end of file diff --git a/service/deployment/entrypoint b/service/deployment/entrypoint index 9a1ee9b1..58339602 100755 --- a/service/deployment/entrypoint +++ b/service/deployment/entrypoint @@ -29,6 +29,9 @@ case "$SERVICE_ACTION" in "kill-instances") ACTION_TO_EXECUTE="kill_instances" ;; + "kill-job-execution") + ACTION_TO_EXECUTE="kill_job_execution" + ;; *) echo "Unknown action: $SERVICE_ACTION" exit 1 @@ -37,9 +40,24 @@ esac WORKFLOW_PATH="$SERVICE_PATH/deployment/workflows/$ACTION_TO_EXECUTE.yaml" +# Fall back to an override-provided workflow when the base scope does not define +# one. This lets an override scope (e.g. scheduled_task) ship a standalone +# deployment workflow that has no base counterpart. +if [[ ! -f "$WORKFLOW_PATH" ]]; then + IFS=',' read -ra OVERRIDE_PATHS <<< "$OVERRIDES_PATH" + for path in "${OVERRIDE_PATHS[@]}"; do + path=$(echo "$path" | xargs) + CANDIDATE_WORKFLOW_PATH="$path/deployment/workflows/$ACTION_TO_EXECUTE.yaml" + if [[ -f "$CANDIDATE_WORKFLOW_PATH" ]]; then + WORKFLOW_PATH="$CANDIDATE_WORKFLOW_PATH" + break + fi + done +fi + NEEDS_PARAMS=true case "$SERVICE_ACTION" in - "switch-traffic"|"kill-instances"|"diagnose-deployment") + "switch-traffic"|"kill-instances"|"kill-job-execution"|"diagnose-deployment") NEEDS_PARAMS=false ;; esac diff --git a/service/deployment/tests/entrypoint.bats b/service/deployment/tests/entrypoint.bats index fac10a8f..1ea13eac 100644 --- a/service/deployment/tests/entrypoint.bats +++ b/service/deployment/tests/entrypoint.bats @@ -79,6 +79,32 @@ mock_np() { assert_contains "$output" "--no-params" } +@test "deployment entrypoint: kill-job-execution includes --no-params when CLI supports it" { + mock_np + export SERVICE_ACTION="kill-job-execution" + mkdir -p "$BATS_TEST_TMPDIR/override/deployment/workflows" + touch "$BATS_TEST_TMPDIR/override/deployment/workflows/kill_job_execution.yaml" + export OVERRIDES_PATH="$BATS_TEST_TMPDIR/override" + + run bash "$BATS_TEST_DIRNAME/../entrypoint" + + [ "$status" -eq 0 ] + assert_contains "$output" "--no-params" +} + +@test "deployment entrypoint: kill-job-execution falls back to override workflow when base is missing" { + mock_np + export SERVICE_ACTION="kill-job-execution" + mkdir -p "$BATS_TEST_TMPDIR/override/deployment/workflows" + touch "$BATS_TEST_TMPDIR/override/deployment/workflows/kill_job_execution.yaml" + export OVERRIDES_PATH="$BATS_TEST_TMPDIR/override" + + run bash "$BATS_TEST_DIRNAME/../entrypoint" + + [ "$status" -eq 0 ] + assert_contains "$output" "--workflow $BATS_TEST_TMPDIR/override/deployment/workflows/kill_job_execution.yaml" +} + # ============================================================================= # Actions that SHOULD NOT include --no-params # =============================================================================