spiffe validator: support multi-tenancy via filter state (#43886)

Commit Message: Add a condition to the trust domain selector using a
per-connection filter state object. This is useful on multi-tenant
set-ups, where a single validation context can be used between multiple
tenants using the same proxy, as it happens to be on k8s (see the two
trust domain labels in
https://kube-agentic-networking.sigs.k8s.io/guides/quickstart/agent-identity-demo/#1-define-the-volume).
This is only implemented on the xDS version of SPIFFE bundle since the
upstream specification for the trust bundle does not cover this
multi-tenant use yet.
Risk Level: low (opt-in field)
Testing: unit and integration
Docs Changes: yes
Release Notes: yes

---------

Signed-off-by: Kuat Yessenov <kuat@google.com>

Author: kyessenov

api/envoy/extensions/transport_sockets/tls/v3/tls_spiffe_validator_config.proto

@@ -45,6 +45,10 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE;
 // - :ref:`allow_expired_certificate <envoy_v3_api_field_extensions.transport_sockets.tls.v3.CertificateValidationContext.allow_expired_certificate>` to allow expired certificates.
 // - :ref:`match_typed_subject_alt_names <envoy_v3_api_field_extensions.transport_sockets.tls.v3.CertificateValidationContext.match_typed_subject_alt_names>` to match **URI** SAN of certificates. Unlike the default validator, SPIFFE validator only matches **URI** SAN (which equals to SVID in SPIFFE terminology) and ignore other SAN types.
 //
+// To support multi-tenant use cases, a filter state object ``envoy.tls.cert_validator.spiffe.workload_trust_domain``
+// should be used to define the per-connection workload trust domain. When matching a peer trust domain, both the
+// workload and the peer trust domains are used in selecting the validation certificate. The filter state object
+// should be shared with the upstream to be used in the upstream TLS context SPIFFE validation context.
 message SPIFFECertValidatorConfig {
   message TrustDomain {
     // Name of the trust domain, ``example.com``, ``foo.bar.gov`` for example.
@@ -53,6 +57,11 @@ message SPIFFECertValidatorConfig {
 
     // Specify a data source holding x.509 trust bundle used for validating incoming SVID(s) in this trust domain.
     config.core.v3.DataSource trust_bundle = 2;
+
+    // Optional workload trust domain selection condition. The filter object
+    // ``envoy.tls.cert_validator.spiffe.workload_trust_domain`` must match exactly the value of this field.
+    // If not specified, the filter state object must be absent or be empty to match this trust domain.
+    string workload_trust_domain = 3;
   }
 
   // This field specifies trust domains used for validating incoming X.509-SVID(s).

changelogs/current.yaml

@@ -267,6 +267,10 @@ new_features:
 - area: filters
   change: |
     Added filters to update the filter state in :ref:`a listener filter <config_listener_filters_set_filter_state>`.
+- area: tls
+  change: |
+    Added a per-connection filter state object to select a workload trust domain in the SPIFFE validator in
+    the multi-tenant deployments.
 - area: tls
   change: |
     Extended TLS certificate compression (RFC 8879): added brotli to QUIC (which already supported zlib),

docs/root/configuration/advanced/well_known_filter_state.rst

@@ -117,6 +117,10 @@ The following lists the filter state object keys used by the Envoy extensions to
   Allows overriding the certificate to use per-connection using the :ref:`filter state certificate mapper
   <envoy_v3_api_msg_extensions.transport_sockets.tls.cert_mappers.filter_state_override.v3.Config>`.
 
+``envoy.tls.cert_validator.spiffe.workload_trust_domain``
+  Specifies per-connection workload trust domain to be used in the :ref:`SPIFFE certificate validator
+  <envoy_v3_api_msg_extensions.transport_sockets.tls.v3.SPIFFECertValidatorConfig>`.
+
 Filter state object factories
 -----------------------------
 

source/extensions/transport_sockets/tls/cert_validator/spiffe/BUILD

@@ -18,6 +18,7 @@ envoy_cc_extension(
     ],
     external_deps = ["ssl"],
     deps = [
+        "//envoy/router:string_accessor_interface",
         "//envoy/ssl:context_config_interface",
         "//envoy/ssl:ssl_socket_extended_info_interface",
         "//source/common/common:assert_lib",

source/extensions/transport_sockets/tls/cert_validator/spiffe/spiffe_validator.cc

@@ -11,6 +11,7 @@
 #include "envoy/extensions/transport_sockets/tls/v3/tls_spiffe_validator_config.pb.h"
 #include "envoy/network/transport_socket.h"
 #include "envoy/registry/registry.h"
+#include "envoy/router/string_accessor.h"
 #include "envoy/ssl/context_config.h"
 #include "envoy/ssl/ssl_socket_extended_info.h"
 
@@ -66,7 +67,7 @@ parseTrustBundles(absl::string_view trust_bundle_mapping_str) {
             // TODO: Duplicates are currently ignored and only the last value is used.
             // This is because our json parser auto de-dupes keys in the dict and
             // only include the last one in this iteration function.
-            spiffe_data->trust_bundle_stores_[domain_name] = X509StorePtr(X509_STORE_new());
+            spiffe_data->trust_bundle_stores_[domain_name][""] = X509StorePtr(X509_STORE_new());
 
             ENVOY_LOG_TO_LOGGER(Logger::Registry::getLog(Logger::Id::secret), info,
                                 "Loading domain '{}' from SPIFFE bundle map", domain_name);
@@ -113,7 +114,7 @@ parseTrustBundles(absl::string_view trust_bundle_mapping_str) {
                       fmt::format("Invalid x509 object in certs for domain '{}'", domain_name));
                   return false;
                 }
-                if (X509_STORE_add_cert(spiffe_data->trust_bundle_stores_[domain_name].get(),
+                if (X509_STORE_add_cert(spiffe_data->trust_bundle_stores_[domain_name][""].get(),
                                         x509.get()) != 1) {
                   parsing_status = absl::InternalError(
                       fmt::format("Failed to add x509 object while loading certs for domain '{}'",
@@ -186,11 +187,15 @@ SPIFFEValidator::SPIFFEValidator(const Envoy::Ssl::CertificateValidationContextC
   spiffe_data_ = std::make_shared<SpiffeData>();
   spiffe_data_->trust_bundle_stores_.reserve(message.trust_domains().size());
   for (auto& domain : message.trust_domains()) {
-    if (spiffe_data_->trust_bundle_stores_.find(domain.name()) !=
-        spiffe_data_->trust_bundle_stores_.end()) {
-      creation_status = absl::InvalidArgumentError(absl::StrCat(
-          "Multiple trust bundles are given for one trust domain for ", domain.name()));
-      return;
+    if (auto it = spiffe_data_->trust_bundle_stores_.find(domain.name());
+        it != spiffe_data_->trust_bundle_stores_.end()) {
+      if (auto local_it = it->second.find(domain.workload_trust_domain());
+          local_it != it->second.end()) {
+        creation_status = absl::InvalidArgumentError(
+            absl::StrCat("Multiple trust bundles are given for one trust domain for ",
+                         domain.name(), ", workload: ", domain.workload_trust_domain()));
+        return;
+      }
     }
 
     auto cert = Config::DataSource::read(domain.trust_bundle(), true, config->api());
@@ -233,7 +238,8 @@ SPIFFEValidator::SPIFFEValidator(const Envoy::Ssl::CertificateValidationContextC
     if (has_crl) {
       X509_STORE_set_flags(store.get(), X509_V_FLAG_CRL_CHECK | X509_V_FLAG_CRL_CHECK_ALL);
     }
-    spiffe_data_->trust_bundle_stores_[domain.name()] = std::move(store);
+    spiffe_data_->trust_bundle_stores_[domain.name()][domain.workload_trust_domain()] =
+        std::move(store);
   }
 
   initializeCertExpirationStats(scope, config->caCertName());
@@ -286,14 +292,15 @@ absl::StatusOr<int> SPIFFEValidator::initializeSslContexts(std::vector<SSL_CTX*>
 bool SPIFFEValidator::verifyCertChainUsingTrustBundleStore(X509& leaf_cert,
                                                            STACK_OF(X509)* cert_chain,
                                                            X509_VERIFY_PARAM* verify_param,
+                                                           absl::string_view workload_trust_domain,
                                                            std::string& error_details) {
   if (!SPIFFEValidator::certificatePrecheck(&leaf_cert)) {
     error_details = "verify cert failed: cert precheck";
     stats_.fail_verify_error_.inc();
     return false;
   }
 
-  auto trust_bundle = getTrustBundleStore(&leaf_cert);
+  auto trust_bundle = getTrustBundleStore(&leaf_cert, workload_trust_domain);
   if (!trust_bundle) {
     error_details = "verify cert failed: no trust bundle store";
     stats_.fail_verify_error_.inc();
@@ -328,21 +335,41 @@ bool SPIFFEValidator::verifyCertChainUsingTrustBundleStore(X509& leaf_cert,
   return san_match;
 }
 
+constexpr absl::string_view WorkloadTrustDomainKey =
+    "envoy.tls.cert_validator.spiffe.workload_trust_domain";
+
 ValidationResults SPIFFEValidator::doVerifyCertChain(
     STACK_OF(X509)& cert_chain, Ssl::ValidateResultCallbackPtr /*callback*/,
-    const Network::TransportSocketOptionsConstSharedPtr& /*transport_socket_options*/,
-    SSL_CTX& ssl_ctx, const CertValidator::ExtraValidationContext& /*validation_context*/,
-    bool /*is_server*/, absl::string_view /*host_name*/) {
+    const Network::TransportSocketOptionsConstSharedPtr& transport_socket_options, SSL_CTX& ssl_ctx,
+    const CertValidator::ExtraValidationContext& validation_context, bool is_server,
+    absl::string_view /*host_name*/) {
   if (sk_X509_num(&cert_chain) == 0) {
     stats_.fail_verify_error_.inc();
     return {ValidationResults::ValidationStatus::Failed,
             Envoy::Ssl::ClientValidationStatus::NotValidated, absl::nullopt,
             "verify cert failed: empty cert chain"};
   }
   X509* leaf_cert = sk_X509_value(&cert_chain, 0);
+  const Router::StringAccessor* obj = nullptr;
+  if (is_server) {
+    if (auto* cb = validation_context.callbacks; cb) {
+      const StreamInfo::StreamInfo& info = cb->connection().streamInfo();
+      obj = info.filterState().getDataReadOnly<Router::StringAccessor>(WorkloadTrustDomainKey);
+    }
+  } else {
+    if (transport_socket_options) {
+      for (const auto& obj_meta : transport_socket_options->downstreamSharedFilterStateObjects()) {
+        if (obj_meta.name_ == WorkloadTrustDomainKey) {
+          obj = dynamic_cast<const Router::StringAccessor*>(obj_meta.data_.get());
+          break;
+        }
+      }
+    }
+  }
+  absl::string_view workload_trust_domain = obj ? obj->asString() : "";
   std::string error_details;
-  bool verified = verifyCertChainUsingTrustBundleStore(*leaf_cert, &cert_chain,
-                                                       SSL_CTX_get0_param(&ssl_ctx), error_details);
+  bool verified = verifyCertChainUsingTrustBundleStore(
+      *leaf_cert, &cert_chain, SSL_CTX_get0_param(&ssl_ctx), workload_trust_domain, error_details);
   return verified ? ValidationResults{ValidationResults::ValidationStatus::Successful,
                                       Envoy::Ssl::ClientValidationStatus::Validated, absl::nullopt,
                                       absl::nullopt}
@@ -351,7 +378,8 @@ ValidationResults SPIFFEValidator::doVerifyCertChain(
                                       error_details};
 }
 
-X509_STORE* SPIFFEValidator::getTrustBundleStore(X509* leaf_cert) {
+X509_STORE* SPIFFEValidator::getTrustBundleStore(X509* leaf_cert,
+                                                 absl::string_view workload_trust_domain) {
   bssl::UniquePtr<GENERAL_NAMES> san_names(static_cast<GENERAL_NAMES*>(
       X509_get_ext_d2i(leaf_cert, NID_subject_alt_name, nullptr, nullptr)));
   if (!san_names) {
@@ -376,8 +404,14 @@ X509_STORE* SPIFFEValidator::getTrustBundleStore(X509* leaf_cert) {
 
   auto spiffe_data = getSpiffeData();
   auto target_store = spiffe_data->trust_bundle_stores_.find(trust_domain);
-  return target_store != spiffe_data->trust_bundle_stores_.end() ? target_store->second.get()
-                                                                 : nullptr;
+  if (target_store == spiffe_data->trust_bundle_stores_.end()) {
+    return nullptr;
+  }
+  auto it = target_store->second.find(workload_trust_domain);
+  if (it == target_store->second.end()) {
+    return nullptr;
+  }
+  return it->second.get();
 }
 
 bool SPIFFEValidator::certificatePrecheck(X509* leaf_cert) {

source/extensions/transport_sockets/tls/cert_validator/spiffe/spiffe_validator.h

@@ -34,7 +34,10 @@ namespace Tls {
 using X509StorePtr = CSmartPtr<X509_STORE, X509_STORE_free>;
 
 struct SpiffeData {
-  absl::flat_hash_map<std::string, CSmartPtr<X509_STORE, X509_STORE_free>> trust_bundle_stores_;
+  // Mapping for "peer trust domain" -> "local trust domain" -> certificate.
+  absl::flat_hash_map<std::string,
+                      absl::flat_hash_map<std::string, CSmartPtr<X509_STORE, X509_STORE_free>>>
+      trust_bundle_stores_;
   std::vector<bssl::UniquePtr<X509>> ca_certs_;
 };
 
@@ -70,7 +73,7 @@ class SPIFFEValidator : public CertValidator, Logger::Loggable<Logger::Id::secre
   Envoy::Ssl::CertificateDetailsPtr getCaCertInformation() const override;
 
   // Utility functions
-  X509_STORE* getTrustBundleStore(X509* leaf_cert);
+  X509_STORE* getTrustBundleStore(X509* leaf_cert, absl::string_view workload_trust_domain);
   static std::string extractTrustDomain(const std::string& san);
   static bool certificatePrecheck(X509* leaf_cert);
   OptRef<SpiffeData> getSpiffeData() const {
@@ -84,6 +87,7 @@ class SPIFFEValidator : public CertValidator, Logger::Loggable<Logger::Id::secre
 private:
   bool verifyCertChainUsingTrustBundleStore(X509& leaf_cert, STACK_OF(X509)* cert_chain,
                                             X509_VERIFY_PARAM* verify_param,
+                                            absl::string_view workload_trust_domain,
                                             std::string& error_details);
 
   void initializeCertExpirationStats(Stats::Scope& scope, const std::string& cert_name);

test/config/integration/certs/cacert.csr

@@ -0,0 +1,19 @@
+-----BEGIN CERTIFICATE REQUEST-----
+MIIDCTCCAfECAQAwdjELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNhbGlmb3JuaWEx
+FjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xDTALBgNVBAoMBEx5ZnQxGTAXBgNVBAsM
+EEx5ZnQgRW5naW5lZXJpbmcxEDAOBgNVBAMMB1Rlc3QgQ0EwggEiMA0GCSqGSIb3
+DQEBAQUAA4IBDwAwggEKAoIBAQDbOPJ7bf3WVPm7IuCvwtAGmssZe57ztyjOg1o8
+JVGnmxhrk4T+8Jtq0yg/Qma4Aipy0hlYIgsk5NtBD6YhVlAAXqh0MBt7ppKl2+Qe
+AEEJ9qT+59S+YdHfJjcwGKGXYsgzvlkGjc81s1cRyA8kUxtxgEeDh7RFf211ipBe
+lhaBMRb9T+x2p/roXvK5l0iUVd7CSZ9WZlqwMj/JHxtyKxHKPaZu/OBo0ieKnNUj
+E4vbJ6SehY1MPZl33MxaCNVgUXSstIDOawwZp/Wkzni/fHh8CDucuYiLHpmR6VFF
+D70OYiLs1qZ27DvNSyE3IjB3WP4hcCzEHyKe4Y3qRsEepad1AgMBAAGgTjBMBgkq
+hkiG9w0BCQ4xPzA9MAwGA1UdEwQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMB0GA1Ud
+DgQWBBSdYEZKo5ykKAgjW/MmjXsdvillLzANBgkqhkiG9w0BAQsFAAOCAQEAyehS
+aCwwc+rICPVzRI3VYJcls81qTBfQoHxJn3CLhwFeTYYcbGHhh9+JZh/NY/qkDvW0
+9ghnvkga5MUemuqpqXc38Vm6AIESdvA1OGDtViuTdRrcD44Bak6QzTvMgPGC1dhr
++c6ywTOQ7Q4ZYs63Tvc2MJkw9Y/zpcisgmYT4VD5ocyg2ENDg6DXmIwCm9IO0FUK
+/Thve+Ro2wI2JRZ4FUfN7DghT6azsx0VW+7hd1yBvV5/s1smEF47uQmoBM+RnXTR
+94sCc4ROHfD1Mqwnub9pjsLHN7xijStb9kGc8G0ys0qPv66YrjzNHFzvK7os8m+q
+yJ0GSc+qanlQuKM8gQ==
+-----END CERTIFICATE REQUEST-----