fix(stream): support upstream client certificate (mTLS) in L4 proxy

apisix/cli/ngx_tpl.lua

@@ -255,6 +255,14 @@ stream {
         set $upstream_sni "apisix_backend";
         proxy_ssl_server_name on;
         proxy_ssl_name $upstream_sni;
+
+        # vars are set in the preread phase to support upstream client
+        # certificate (mTLS) when proxying to a TLS upstream. When empty,
+        # nginx skips loading the certificate, so no mTLS is performed.
+        set $upstream_mtls_cert "";
+        set $upstream_mtls_key "";
+        proxy_ssl_certificate $upstream_mtls_cert;
+        proxy_ssl_certificate_key $upstream_mtls_key;
         {% end %}
 
         log_by_lua_block {

apisix/init.lua

@@ -493,6 +493,31 @@ local function common_phase(phase_name)
 end
 
 
+-- Resolve the upstream client certificate referenced by `tls.client_cert_id`
+-- into `api_ctx.upstream_ssl`. Shared by the http and stream subsystems.
+-- Returns false on error (invalid/missing referenced ssl object).
+local function resolve_upstream_client_cert(api_ctx)
+    if not (api_ctx.matched_upstream and api_ctx.matched_upstream.tls and
+            api_ctx.matched_upstream.tls.client_cert_id) then
+        return true
+    end
+
+    local cert_id = api_ctx.matched_upstream.tls.client_cert_id
+    local upstream_ssl = router.router_ssl.get_by_id(cert_id)
+    if not upstream_ssl or upstream_ssl.type ~= "client" then
+        local err = upstream_ssl and
+            "ssl type should be 'client'" or
+            "ssl id [" .. cert_id .. "] not exits"
+        core.log.error("failed to get ssl cert: ", err)
+        return false
+    end
+
+    core.log.info("matched ssl: ", core.json.delay_encode(upstream_ssl, true))
+    api_ctx.upstream_ssl = upstream_ssl
+    return true
+end
+
+
 function _M.handle_upstream(api_ctx, route, enable_websocket)
     -- some plugins(ai-proxy...) request upstream by http client directly
     if api_ctx.bypass_nginx_upstream then
@@ -537,27 +562,12 @@ function _M.handle_upstream(api_ctx, route, enable_websocket)
         api_ctx.matched_upstream = route_val.upstream
     end
 
-    if api_ctx.matched_upstream and api_ctx.matched_upstream.tls and
-        api_ctx.matched_upstream.tls.client_cert_id then
-
-        local cert_id = api_ctx.matched_upstream.tls.client_cert_id
-        local upstream_ssl = router.router_ssl.get_by_id(cert_id)
-        if not upstream_ssl or upstream_ssl.type ~= "client" then
-            local err  = upstream_ssl and
-                "ssl type should be 'client'" or
-                "ssl id [" .. cert_id .. "] not exits"
-            core.log.error("failed to get ssl cert: ", err)
-
-            if is_http then
-                return core.response.exit(502)
-            end
-
-            return ngx_exit(1)
+    local ok = resolve_upstream_client_cert(api_ctx)
+    if not ok then
+        if is_http then
+            return core.response.exit(502)
         end
-
-        core.log.info("matched ssl: ",
-                  core.json.delay_encode(upstream_ssl, true))
-        api_ctx.upstream_ssl = upstream_ssl
+        return ngx_exit(1)
     end
 
     if enable_websocket then
@@ -1385,6 +1395,11 @@ function _M.stream_preread_phase()
         return
     end
 
+    local ok = resolve_upstream_client_cert(api_ctx)
+    if not ok then
+        return ngx_exit(1)
+    end
+
     local code, err = set_upstream(matched_route, api_ctx)
     if code then
         core.log.error("failed to set upstream: ", err)

apisix/upstream.lua

@@ -160,6 +160,51 @@ local function fill_node_info(up_conf, scheme, is_stream)
 end
 
 
+-- Set upstream client certificate (mTLS) for the stream (L4) subsystem.
+-- Unlike the http subsystem, the stream proxy has no per-request C API to
+-- inject the client cert into the SSL connection, so we rely on the native
+-- nginx `proxy_ssl_certificate`/`proxy_ssl_certificate_key` directives, which
+-- accept inline PEM via the `data:` scheme and support variables. The vars are
+-- declared empty in the stream server block and filled here in the preread
+-- phase; an empty value means no client certificate is presented.
+local function set_stream_upstream_client_cert(api_ctx, up_conf)
+    local tls = up_conf.tls
+    if not (tls and (tls.client_cert or tls.client_cert_id)) then
+        return true
+    end
+
+    local client_cert, client_key
+    if tls.client_cert_id then
+        if not api_ctx.upstream_ssl then
+            return nil, "failed to find upstream ssl object for client_cert_id"
+        end
+        client_cert = api_ctx.upstream_ssl.cert
+        client_key = api_ctx.upstream_ssl.key
+    else
+        client_cert = tls.client_cert
+        client_key = tls.client_key
+    end
+
+    if not (client_cert and client_key) then
+        return nil, "missing client certificate or key for upstream mTLS"
+    end
+
+    -- The private key is stored AES-encrypted at rest (see encrypt_conf and the
+    -- ssl object), so decrypt it back to PEM before handing it to nginx. The
+    -- certificate is always stored as plaintext PEM.
+    local key, err = apisix_ssl.aes_decrypt_pkey(client_key)
+    if not key then
+        return nil, err
+    end
+
+    -- `data:` lets nginx read the PEM from the variable value directly,
+    -- avoiding any temporary file on disk.
+    ngx_var.upstream_mtls_cert = "data:" .. client_cert
+    ngx_var.upstream_mtls_key = "data:" .. key
+    return true
+end
+
+
 function _M.set_by_route(route, api_ctx)
     if api_ctx.upstream_conf then
         -- upstream_conf has been set by traffic-split plugin
@@ -246,6 +291,11 @@ function _M.set_by_route(route, api_ctx)
             if sni then
                 ngx_var.upstream_sni = sni
             end
+
+            local ok, err = set_stream_upstream_client_cert(api_ctx, up_conf)
+            if not ok then
+                return 503, err
+            end
         end
         local node_ver = resource.get_nodes_ver(up_conf.resource_key)
         local resource_version = upstream_util.version(up_conf.resource_version,

docs/en/latest/mtls.md

@@ -208,3 +208,8 @@ curl http://127.0.0.1:9180/apisix/admin/upstreams/1 \
     }
 }'
 ```
+
+This also works in the stream (L4) subsystem: when an upstream uses the `tls`
+scheme and configures `tls.client_cert`/`tls.client_key` (or
+`tls.client_cert_id`), APISIX presents the client certificate while
+establishing the TLS connection to the upstream.

t/APISIX.pm

@@ -563,6 +563,10 @@ _EOC_
     proxy_ssl_server_name on;
     proxy_ssl_name \$upstream_sni;
     set \$upstream_sni "apisix_backend";
+    set \$upstream_mtls_cert "";
+    set \$upstream_mtls_key "";
+    proxy_ssl_certificate \$upstream_mtls_cert;
+    proxy_ssl_certificate_key \$upstream_mtls_key;
 _EOC_
     }
 

t/cli/test_stream_proxy_protocol.sh

@@ -214,15 +214,15 @@ apisix:
 ' > conf/config.yaml
 make init
 
-if ! block_with_listen 9101 | grep -E "ssl_certificate " > /dev/null; then
+if ! block_with_listen 9101 | grep -E "[[:space:]]ssl_certificate " > /dev/null; then
     echo "failed: tls port in the to-upstream block should render ssl_certificate"
     exit 1
 fi
 if ! block_with_listen 9101 | grep -E "proxy_protocol on;" > /dev/null; then
     echo "failed: 9101 should send PROXY protocol upstream"
     exit 1
 fi
-if block_with_listen 9100 | grep -E "ssl_certificate " > /dev/null; then
+if block_with_listen 9100 | grep -E "[[:space:]]ssl_certificate " > /dev/null; then
     echo "failed: the plain block must not render ssl_certificate"
     exit 1
 fi