feat: extend secret references to all plugins with central resolution (#13312)

Author: nic-6443

apisix-master-0.rockspec

@@ -52,7 +52,7 @@ dependencies = {
     "lua-resty-openidc = 1.8.0-1",
     "luafilesystem = 1.8.0-1",
     "nginx-lua-prometheus-api7 = 0.20250302-1",
-    "jsonschema = 0.9.9-0",
+    "jsonschema = 0.9.13-0",
     "lua-resty-ipmatcher = 0.6.1-0",
     "lua-resty-kafka = 0.23-0",
     "lua-resty-logger-socket = 2.0.3-0",

apisix/core/schema.lua

@@ -24,6 +24,7 @@ local lrucache = require("apisix.core.lrucache")
 local schema_def = require("apisix.schema_def")
 local cached_validator = lrucache.new({count = 1000, ttl = 0})
 local pcall = pcall
+local require = require
 local error = error
 
 local _M = {
@@ -39,7 +40,10 @@ local function create_validator(schema)
     -- local file2=io.output("/tmp/2.txt")
     -- file2:write(code)
     -- file2:close()
-    local ok, res = pcall(jsonschema.generate_validator, schema)
+    local secret = require("apisix.secret")
+    local ok, res = pcall(jsonschema.generate_validator, schema, {
+        skip_validation = secret.is_secret_ref,
+    })
     if ok then
         return res
     end
@@ -58,6 +62,7 @@ local function get_validator(schema)
     return validator, nil
 end
 
+
 function _M.check(schema, json)
     local validator, err = get_validator(schema)
 

apisix/plugin.lua

@@ -21,6 +21,7 @@ local enable_debug  = require("apisix.debug").enable_debug
 local wasm          = require("apisix.wasm")
 local expr          = require("resty.expr.v1")
 local apisix_ssl    = require("apisix.ssl")
+local secret        = require("apisix.secret")
 
 local ngx           = ngx
 local ngx_ok        = ngx.OK
@@ -65,6 +66,56 @@ local merge_global_rule_lrucache = core.lrucache.new({
     ttl = 300, count = 512
 })
 
+-- Cache for resolved plugin confs: original_conf -> {resolved, secret_vals}
+-- Weak keys ensure entries are GC'd when original conf is replaced (config reload)
+-- Weak-keyed cache: original_conf -> {resolved, secret_vals}.
+-- Avoids deepcopy on every request when secret values haven't changed,
+-- which preserves plugins' internal caches that use conf table identity
+-- as cache key (e.g. ai-rate-limiting's limit_conf_cache).
+local _resolved_cache = setmetatable({}, {__mode = "k"})
+local _no_secret_ref = setmetatable({}, {__mode = "k"})
+
+local function vals_equal(a, b)
+    if a == b then
+        return true
+    end
+    if not a or not b then
+        return false
+    end
+    for k, v in pairs(a) do
+        if b[k] ~= v then
+            return false
+        end
+    end
+    for k in pairs(b) do
+        if a[k] == nil then
+            return false
+        end
+    end
+    return true
+end
+
+local function resolve_plugin_conf(conf)
+    if _no_secret_ref[conf] then
+        return conf
+    end
+    if not secret.has_secret_ref(conf) then
+        _no_secret_ref[conf] = true
+        return conf
+    end
+
+    local current_vals = secret.collect_secret_values(conf, true)
+
+    local cached = _resolved_cache[conf]
+    if cached and vals_equal(cached.secret_vals, current_vals) then
+        return cached.resolved
+    end
+
+    local resolved = secret.fetch_secrets(conf, true)
+    _resolved_cache[conf] = {resolved = resolved, secret_vals = current_vals}
+    return resolved
+end
+
 local local_conf
 local check_plugin_metadata
 
@@ -575,6 +626,14 @@ function _M.filter(ctx, conf, plugins, route_conf, phase)
         core.tablepool.release("tmp_plugin_confs", tmp_plugin_confs)
     end
 
+    -- resolve $secret:// and $env:// references in plugin confs
+    for i = 2, #plugins, 2 do
+        local resolved = resolve_plugin_conf(plugins[i])
+        if resolved ~= plugins[i] then
+            plugins[i] = resolved
+        end
+    end
+
     return plugins
 end
 
@@ -600,6 +659,14 @@ function _M.stream_filter(user_route, plugins)
 
     trace_plugins_info_for_debug(nil, plugins)
 
+    -- resolve $secret:// and $env:// references in stream plugin confs
+    for i = 2, #plugins, 2 do
+        local resolved = resolve_plugin_conf(plugins[i])
+        if resolved ~= plugins[i] then
+            plugins[i] = resolved
+        end
+    end
+
     return plugins
 end
 
@@ -902,6 +969,7 @@ function _M.conf_version(conf)
 end
 
 
+
 local function check_single_plugin_schema(name, plugin_conf, schema_type, skip_disabled_plugin)
     if type(plugin_conf) ~= "table" then
         return false, "invalid plugin conf " ..

apisix/plugins/ai-aws-content-moderation.lua

@@ -21,7 +21,6 @@ local aws = require("resty.aws")
 local aws_instance
 
 local http = require("resty.http")
-local fetch_secrets = require("apisix.secret").fetch_secrets
 
 local pairs = pairs
 local unpack = unpack
@@ -88,11 +87,6 @@ end
 
 
 function _M.rewrite(conf, ctx)
-    conf = fetch_secrets(conf, true)
-    if not conf then
-        return HTTP_INTERNAL_SERVER_ERROR, "failed to retrieve secrets from conf"
-    end
-
     local body, err = core.request.get_body()
     if not body then
         return HTTP_BAD_REQUEST, err

apisix/plugins/ai-proxy-multi.lua

@@ -16,6 +16,7 @@
 --
 
 local core = require("apisix.core")
+local secret = require("apisix.secret")
 local schema = require("apisix.plugins.ai-proxy.schema")
 local base   = require("apisix.plugins.ai-proxy.base")
 local plugin = require("apisix.plugin")
@@ -110,7 +111,7 @@ function _M.check_schema(conf)
             return false, "ai provider: " .. instance.provider .. " is not supported."
         end
         local sa_json = core.table.try_read_attr(instance, "auth", "gcp", "service_account_json")
-        if sa_json then
+        if sa_json and not secret.is_secret_ref(sa_json) then
             local _, err = core.json.decode(sa_json)
             if err then
                 return false, "invalid gcp service_account_json: " .. err

apisix/plugins/ai-proxy.lua

@@ -15,6 +15,7 @@
 -- limitations under the License.
 --
 local core = require("apisix.core")
+local secret = require("apisix.secret")
 local schema = require("apisix.plugins.ai-proxy.schema")
 local base = require("apisix.plugins.ai-proxy.base")
 local exporter = require("apisix.plugins.prometheus.exporter")
@@ -42,7 +43,7 @@ function _M.check_schema(conf)
         return false, "ai provider: " .. conf.provider .. " is not supported."
     end
     local sa_json = core.table.try_read_attr(conf, "auth", "gcp", "service_account_json")
-    if sa_json then
+    if sa_json and not secret.is_secret_ref(sa_json) then
         local _, err = core.json.decode(sa_json)
         if err then
             return false, "invalid gcp service_account_json: " .. err

apisix/plugins/authz-keycloak.lua

@@ -20,7 +20,6 @@ local sub_str   = string.sub
 local type      = type
 local ngx       = ngx
 local plugin_name = "authz-keycloak"
-local fetch_secrets    = require("apisix.secret").fetch_secrets
 
 local log = core.log
 local pairs = pairs
@@ -763,8 +762,6 @@ local function generate_token_using_password_grant(conf,ctx)
 end
 
 function _M.access(conf, ctx)
-    -- resolve secrets
-    conf = fetch_secrets(conf, true)
     local headers = core.request.headers(ctx)
     local need_grant_token = conf.password_grant_token_generation_incoming_uri and
         ctx.var.request_uri == conf.password_grant_token_generation_incoming_uri and

apisix/plugins/clickhouse-logger.lua

@@ -15,7 +15,6 @@
 -- limitations under the License.
 --
 
-local fetch_secrets   = require("apisix.secret").fetch_secrets
 local bp_manager_mod  = require("apisix.utils.batch-processor-manager")
 local log_util        = require("apisix.utils.log-util")
 local plugin          = require("apisix.plugin")
@@ -107,8 +106,6 @@ end
 
 
 local function send_http_data(conf, log_message)
-    conf = fetch_secrets(conf, true)
-
     local err_msg
     local res = true
     local selected_endpoint_addr

apisix/plugins/jwt-auth.lua

@@ -19,6 +19,7 @@ local consumer_mod = require("apisix.consumer")
 local new_tab = require ("table.new")
 local auth_utils = require("apisix.utils.auth")
 
+local secret   = require("apisix.secret")
 local ngx_decode_base64 = ngx.decode_base64
 local ngx      = ngx
 local sub_str  = string.sub
@@ -182,7 +183,7 @@ function _M.check_schema(conf, schema_type)
         return false, "property \"secret\" is required when using HS based algorithms"
     end
 
-    if conf.base64_secret then
+    if conf.base64_secret and not secret.is_secret_ref(conf.secret) then
         if ngx_decode_base64(conf.secret) == nil then
             return false, "base64_secret required but the secret is not in base64 format"
         end

apisix/plugins/limit-count.lua

@@ -14,7 +14,6 @@
 -- See the License for the specific language governing permissions and
 -- limitations under the License.
 --
-local fetch_secrets = require("apisix.secret").fetch_secrets
 local limit_count = require("apisix.plugins.limit-count.init")
 local workflow = require("apisix.plugins.workflow")
 
@@ -34,7 +33,6 @@ end
 
 
 function _M.access(conf, ctx)
-    conf = fetch_secrets(conf, true)
     return limit_count.rate_limit(conf, ctx, plugin_name, 1)
 end