Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).

## [Unreleased]
- k8s scope deployments now report launched and healthy instance counts, so the deployment page shows live "X/Y launched" and "X/Y healthy" progress
- Fix: triggering a scheduled task job on a scope that is not deployed now fails with a clear "deploy the scope first" message instead of an opaque error
- Add "Kill instance" action to scheduled task scopes to terminate an individual running job instance

Expand Down
154 changes: 154 additions & 0 deletions k8s/deployment/tests/wait_deployment_active.bats
Original file line number Diff line number Diff line change
Expand Up @@ -703,3 +703,157 @@ teardown() {
return 1
fi
}

# =============================================================================
# Instance Count Reporting Tests
# =============================================================================
@test "wait_deployment_active: reports the instance counters as a partial strategy_data patch" {
export PATCH_LOG="$BATS_TEST_TMPDIR/patches.log"

kubectl() {
case "$*" in
"get deployment"*"-o json"*)
echo '{
"spec": {"replicas": 3},
"status": {
"replicas": 3,
"availableReplicas": 3,
"updatedReplicas": 3,
"readyReplicas": 3
}
}'
;;
"get pods"*)
echo ""
;;
"get events"*)
echo '{"items":[]}'
;;
esac
}
export -f kubectl

np() {
case "$*" in
*"deployment patch"*)
echo "$*" >> "$PATCH_LOG"
;;
*)
echo "running"
;;
esac
}
export -f np

run bash "$BATS_TEST_DIRNAME/../wait_deployment_active"

[ "$status" -eq 0 ]
assert_contains "$output" "📡 Reported instance counts — launched: 3/3, healthy: 3/3"
[ "$(wc -l < "$PATCH_LOG")" -eq 1 ]
assert_contains "$(cat "$PATCH_LOG")" '"amount_instances_to_wait": 3'
assert_contains "$(cat "$PATCH_LOG")" '"launched_instances": 3'
assert_contains "$(cat "$PATCH_LOG")" '"healthy_instances": 3'
# ONLY the counters travel — the server merges them over the current
# strategy_data, so nothing else may ride (or wipe) the body.
if grep -q "switched_traffic" "$PATCH_LOG"; then
echo "Expected the patch body to carry only the counters"
cat "$PATCH_LOG"
return 1
fi
}

@test "wait_deployment_active: re-reports only when the counts change" {
export PATCH_LOG="$BATS_TEST_TMPDIR/patches.log"
export KUBECTL_CALLS="$BATS_TEST_TMPDIR/kubectl.calls"

run bash -c "
sleep() { :; }
export -f sleep

kubectl() {
case \"\$*\" in
\"get deployment\"*\"-o json\"*)
echo x >> \"\$KUBECTL_CALLS\"
if [ \"\$(wc -l < \"\$KUBECTL_CALLS\")\" -lt 3 ]; then
echo '{\"spec\": {\"replicas\": 3}, \"status\": {\"replicas\": 3, \"availableReplicas\": 1, \"updatedReplicas\": 3, \"readyReplicas\": 1}}'
else
echo '{\"spec\": {\"replicas\": 3}, \"status\": {\"replicas\": 3, \"availableReplicas\": 3, \"updatedReplicas\": 3, \"readyReplicas\": 3}}'
fi
;;
\"get pods\"*)
echo ''
;;
\"get events\"*)
echo '{\"items\":[]}'
;;
esac
}
export -f kubectl

np() {
case \"\$*\" in
*'deployment patch'*)
echo \"\$*\" >> \"\$PATCH_LOG\"
;;
*)
echo 'running'
;;
esac
}
export -f np

export SERVICE_PATH='$SERVICE_PATH' K8S_NAMESPACE='$K8S_NAMESPACE'
export SCOPE_ID='$SCOPE_ID' DEPLOYMENT_ID='$DEPLOYMENT_ID'
export TIMEOUT=30 NP_API_KEY='$NP_API_KEY' SKIP_DEPLOYMENT_STATUS_CHECK='false'
bash '$BATS_TEST_DIRNAME/../wait_deployment_active'
"

[ "$status" -eq 0 ]
# Three iterations (1/3, 1/3, 3/3) but only two distinct counts — two patches.
[ "$(wc -l < "$PATCH_LOG")" -eq 2 ]
assert_contains "$(sed -n 1p "$PATCH_LOG")" '"healthy_instances": 1'
assert_contains "$(sed -n 2p "$PATCH_LOG")" '"healthy_instances": 3'
}

@test "wait_deployment_active: a failed count report never fails the deployment" {
kubectl() {
case "$*" in
"get deployment"*"-o json"*)
echo '{
"spec": {"replicas": 3},
"status": {
"replicas": 3,
"availableReplicas": 3,
"updatedReplicas": 3,
"readyReplicas": 3
}
}'
;;
"get pods"*)
echo ""
;;
"get events"*)
echo '{"items":[]}'
;;
esac
}
export -f kubectl

np() {
case "$*" in
*"deployment patch"*)
return 1
;;
*)
echo "running"
;;
esac
}
export -f np

run bash "$BATS_TEST_DIRNAME/../wait_deployment_active"

[ "$status" -eq 0 ]
assert_contains "$output" "Could not report instance counts"
assert_contains "$output" "✅ All pods in deployment 'd-scope-123-deploy-456' are available and ready!"
}
35 changes: 35 additions & 0 deletions k8s/deployment/wait_deployment_active
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,38 @@ K8S_DEPLOYMENT_NAME="d-$SCOPE_ID-$DEPLOYMENT_ID"
iteration=0
LATEST_TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
SKIP_DEPLOYMENT_STATUS_CHECK="${SKIP_DEPLOYMENT_STATUS_CHECK:=false}"
LAST_REPORTED_COUNTS=""

# Report the instance counters onto the deployment's strategy_data
# (amount_instances_to_wait / launched_instances / healthy_instances) so the
# platform can show launched/healthy progress. A partial strategy_data body
# MERGES server-side (the deployment strategies fold the update over the
# current data), so only the counters travel — switched_traffic and friends are
# untouched. Same contract the scope-workflow-manager uses for its scopes.
# Best-effort telemetry: a failed report must never fail the deployment.
report_instance_counts() {
local desired="$1" launched="$2" healthy="$3"

if ! [[ "$desired" =~ ^[0-9]+$ ]] || [ "$desired" -eq 0 ]; then
return 0
fi
[[ "$launched" =~ ^[0-9]+$ ]] || launched=0
[[ "$healthy" =~ ^[0-9]+$ ]] || healthy=0

local counts="$desired/$launched/$healthy"
if [ "$counts" = "$LAST_REPORTED_COUNTS" ]; then
return 0
fi

local body="{\"strategy_data\": {\"amount_instances_to_wait\": $desired, \"launched_instances\": $launched, \"healthy_instances\": $healthy}}"
if np deployment patch --id "$DEPLOYMENT_ID" --api-key "$NP_API_KEY" --no-output --body "$body" 2>/dev/null; then
LAST_REPORTED_COUNTS="$counts"
log debug "📡 Reported instance counts — launched: $launched/$desired, healthy: $healthy/$desired"
else
log debug "Could not report instance counts (will retry on next change)"
fi
return 0
}

HEARTBEAT_INTERVAL=$(( MAX_ITERATIONS / 10 ))
[ "$HEARTBEAT_INTERVAL" -lt 1 ] && HEARTBEAT_INTERVAL=1
Expand Down Expand Up @@ -104,8 +136,11 @@ while true; do
current=$(echo "$deployment_status" | jq '.status.availableReplicas // 0')
updated=$(echo "$deployment_status" | jq '.status.updatedReplicas // 0')
ready=$(echo "$deployment_status" | jq '.status.readyReplicas // 0')
launched=$(echo "$deployment_status" | jq '.status.replicas // 0')
log debug "🔍 $(date): Iteration $iteration - Deployment status - Available: $current/$desired, Updated: $updated/$desired, Ready: $ready/$desired"

report_instance_counts "$desired" "$launched" "$ready"

if [ "$desired" = "$current" ] && [ "$desired" = "$updated" ] && [ "$desired" = "$ready" ] && [ "$desired" -gt 0 ]; then
log debug ""
log info "✅ All pods in deployment '$K8S_DEPLOYMENT_NAME' are available and ready!"
Expand Down
Loading