From d06facd756994cbd0c5dfffc35050b486cccc86d Mon Sep 17 00:00:00 2001 From: AlinsRan Date: Fri, 24 Jul 2026 17:29:34 +0800 Subject: [PATCH] fix(control): reconcile /v1/plugins/reload like the admin reload path #13714 made the admin plugins reload converge missed workers by bumping a shared plugins_conf_version and reconciling against it in a periodic timer, but the control API /v1/plugins/reload was left out: it only broadcasts the event. The resty.events broker gives no delivery guarantee while a worker is reconnecting, so a worker can miss the broadcast and stay on stale plugin state even though the API returned 200 -- the same failure #13537 described, via the control path. Bump the same shared version from the control path so the existing reconciliation timer converges missed workers too. --- apisix/control/v1.lua | 18 +++++++ t/control/plugins-reload-reconcile.t | 77 ++++++++++++++++++++++++++++ 2 files changed, 95 insertions(+) create mode 100644 t/control/plugins-reload-reconcile.t diff --git a/apisix/control/v1.lua b/apisix/control/v1.lua index 496b8b57d5fe..f3e2fa071802 100644 --- a/apisix/control/v1.lua +++ b/apisix/control/v1.lua @@ -30,6 +30,11 @@ local str_format = string.format local ngx = ngx local ngx_var = ngx.var local events = require("apisix.events") +-- Shared with apisix/admin/init.lua: the admin reload path bumps this version and +-- runs a periodic reconciliation timer that reloads any worker whose applied +-- version is behind. Keep the dict name and key in sync with admin/init.lua. +local plugins_conf_ver_dict = ngx.shared["internal-status"] +local PLUGINS_CONF_VERSION_KEY = "plugins_conf_version" local _M = {} @@ -409,6 +414,19 @@ function _M.dump_plugin_metadata() end function _M.post_reload_plugins() + -- Bump the shared version before broadcasting so that a worker which misses the + -- event (the resty.events broker gives no delivery guarantee while a worker is + -- reconnecting) still converges through the admin reconciliation timer. This is + -- the same guard the admin reload path added in #13714; the control path was + -- left out. When the admin is disabled the timer is absent and this is a no-op. + if plugins_conf_ver_dict then + local _, incr_err = plugins_conf_ver_dict:incr(PLUGINS_CONF_VERSION_KEY, 1, 0) + if incr_err then + core.log.error("failed to increase plugins conf version: ", incr_err) + core.response.exit(503, {error_msg = "failed to record plugins reload"}) + end + end + local success, err = events:post(_M.RELOAD_EVENT, ngx.req.get_method(), ngx.time()) if not success then core.response.exit(503, err) diff --git a/t/control/plugins-reload-reconcile.t b/t/control/plugins-reload-reconcile.t new file mode 100644 index 000000000000..2ce0dba69313 --- /dev/null +++ b/t/control/plugins-reload-reconcile.t @@ -0,0 +1,77 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +use t::APISIX 'no_plan'; + +log_level('info'); +repeat_each(1); +no_long_string(); +no_root_location(); +no_shuffle(); + +add_block_preprocessor(sub { + my ($block) = @_; + if (!defined $block->request) { + $block->set_value("request", "GET /t"); + } +}); + +run_tests(); + +__DATA__ + +=== TEST 1: a control-API reload request bumps the shared plugins conf version +--- config + location /t { + content_by_lua_block { + local dict = ngx.shared["internal-status"] + local before = dict:get("plugins_conf_version") or 0 + local res = ngx.location.capture("/v1/plugins/reload", + { method = ngx.HTTP_PUT }) + local after = dict:get("plugins_conf_version") or 0 + ngx.say(res.status, " bumped=", tostring(after > before)) + } + } +--- response_body +200 bumped=true +--- no_error_log +failed to increase plugins conf version + + + +=== TEST 2: a worker that missed the control reload broadcast converges via reconciliation +--- config + location /t { + content_by_lua_block { + -- A reload whose broadcast never arrives (the resty.events broker has + -- no delivery guarantee while a worker is reconnecting) leaves the + -- shared version ahead of what this worker applied. Reproduce exactly + -- that state without going through the events layer, and let the admin + -- reconciliation timer (once a second) catch it up. + local dict = ngx.shared["internal-status"] + local newver, incr_err = dict:incr("plugins_conf_version", 1, 0) + if not newver then + ngx.say("failed to bump version: ", incr_err) + return + end + ngx.sleep(2.5) + ngx.say("done") + } + } +--- response_body +done +--- error_log +start to hot reload plugins