diff --git a/.gitignore b/.gitignore index af70c452..1234c631 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,8 @@ # env .env* +trusted-server.toml +fastly.local.toml # backup **/*.rs.bk diff --git a/crates/integration-tests/README.md b/crates/integration-tests/README.md index c9af0e78..b9d76016 100644 --- a/crates/integration-tests/README.md +++ b/crates/integration-tests/README.md @@ -168,8 +168,8 @@ fixtures/ 1. A Docker container starts for the frontend framework, mapped to a fixed origin port (default 8888) -2. The WASM binary is pre-built with `TRUSTED_SERVER__PUBLISHER__ORIGIN_URL` - pointing to `http://127.0.0.1:8888` so the proxy knows where to forward +2. A rendered Viceroy config projects the integration-test application config + into the local config store under the fixed `ts-config` key 3. Viceroy spawns with the WASM binary on a random port 4. **HTTP tests**: reqwest sends requests to Viceroy and asserts on responses 5. **Browser tests**: Playwright opens Chromium pointing at Viceroy and verifies @@ -177,8 +177,8 @@ fixtures/ ### Why `--test-threads=1` / `workers: 1` -All tests share the same fixed origin port (8888). The trusted server config is -baked into the WASM binary at compile time with this port, so only one Docker +All tests share the same fixed origin port (8888). The integration-test app +config fixture points Trusted Server at this port, so only one Docker container can be bound to it at a time. ## CI diff --git a/crates/integration-tests/browser/global-setup.ts b/crates/integration-tests/browser/global-setup.ts index 04a72929..be34c04e 100644 --- a/crates/integration-tests/browser/global-setup.ts +++ b/crates/integration-tests/browser/global-setup.ts @@ -1,5 +1,7 @@ import { writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; import { resolve } from "node:path"; +import { execFileSync } from "node:child_process"; import { startContainer, startViceroy, @@ -16,14 +18,24 @@ const WASM_PATH = "../../../target/wasm32-wasip1/release/trusted-server-adapter-fastly.wasm", ); -const VICEROY_CONFIG = - process.env.VICEROY_CONFIG_PATH || - resolve(__dirname, "../fixtures/configs/viceroy-template.toml"); +const VICEROY_TEMPLATE = resolve( + __dirname, + "../fixtures/configs/viceroy-template.toml", +); +const APP_CONFIG = resolve( + __dirname, + "../fixtures/configs/trusted-server.integration.toml", +); +const RENDER_SCRIPT = resolve( + __dirname, + "../../../scripts/render-fastly-local-config.py", +); /** Persist current state so global-teardown can always clean up. */ function writeState(state: { baseUrl?: string; containerId?: string; + renderedConfigPath?: string; viceroyPid?: number; framework: string; }): void { @@ -33,6 +45,7 @@ function writeState(state: { async function globalSetup(): Promise { const framework = process.env.TEST_FRAMEWORK || "nextjs"; let containerId: string | undefined; + let renderedConfig: string | undefined; let viceroyPid: number | undefined; try { @@ -43,8 +56,22 @@ async function globalSetup(): Promise { // even if Viceroy startup fails below. writeState({ containerId, framework }); + renderedConfig = resolve( + tmpdir(), + `trusted-server-browser-${Date.now()}.toml`, + ); + execFileSync("python3", [ + RENDER_SCRIPT, + "--app-config", + APP_CONFIG, + "--template", + VICEROY_TEMPLATE, + "--output", + renderedConfig, + ]); + console.log(`[global-setup] Starting Viceroy (WASM: ${WASM_PATH})...`); - const viceroy = await startViceroy(WASM_PATH, VICEROY_CONFIG); + const viceroy = await startViceroy(WASM_PATH, renderedConfig); viceroyPid = viceroy.process.pid; console.log(`[global-setup] Viceroy ready at ${viceroy.baseUrl}`); @@ -53,6 +80,7 @@ async function globalSetup(): Promise { writeState({ baseUrl: viceroy.baseUrl, containerId, + renderedConfigPath: renderedConfig, viceroyPid, framework, }); @@ -61,6 +89,12 @@ async function globalSetup(): Promise { console.error("[global-setup] Setup failed, cleaning up..."); if (viceroyPid) await stopViceroy(viceroyPid); if (containerId) stopContainer(containerId); + try { + const { unlinkSync } = await import("node:fs"); + if (renderedConfig) unlinkSync(renderedConfig); + } catch { + // Rendered config may not exist + } // Remove partial state file since we cleaned up manually try { diff --git a/crates/integration-tests/browser/global-teardown.ts b/crates/integration-tests/browser/global-teardown.ts index 97bdf282..1eb2b42f 100644 --- a/crates/integration-tests/browser/global-teardown.ts +++ b/crates/integration-tests/browser/global-teardown.ts @@ -5,7 +5,11 @@ import { stopContainer, stopViceroy } from "./helpers/infra.js"; const STATE_FILE = resolve(__dirname, ".browser-test-state.json"); async function globalTeardown(): Promise { - let state: { containerId?: string; viceroyPid?: number }; + let state: { + containerId?: string; + renderedConfigPath?: string; + viceroyPid?: number; + }; try { state = JSON.parse(readFileSync(STATE_FILE, "utf-8")); } catch { @@ -23,6 +27,14 @@ async function globalTeardown(): Promise { stopContainer(state.containerId); } + if (state.renderedConfigPath) { + try { + unlinkSync(state.renderedConfigPath); + } catch { + // Already removed + } + } + try { unlinkSync(STATE_FILE); } catch { diff --git a/crates/integration-tests/fixtures/configs/trusted-server.integration.toml b/crates/integration-tests/fixtures/configs/trusted-server.integration.toml new file mode 100644 index 00000000..9bab25ea --- /dev/null +++ b/crates/integration-tests/fixtures/configs/trusted-server.integration.toml @@ -0,0 +1,101 @@ +[[handlers]] +path = "^/secure" +username = "user" +password = "pass" + +[[handlers]] +path = "^/admin" +username = "admin" +password = "changeme" + +[publisher] +domain = "test-publisher.com" +cookie_domain = ".test-publisher.com" +origin_url = "http://127.0.0.1:8888" +proxy_secret = "change-me-proxy-secret" + +[edge_cookie] +secret_key = "trusted-server" + +[request_signing] +enabled = false +config_store_id = "" +secret_store_id = "" + +[integrations.prebid] +enabled = true +server_url = "http://68.183.113.79:8000" +timeout_ms = 1000 +bidders = ["kargo", "appnexus", "openx"] +debug = false +client_side_bidders = ["rubicon"] + +[integrations.nextjs] +enabled = false +rewrite_attributes = ["href", "link", "siteBaseUrl", "siteProductionDomain", "url"] +max_combined_payload_bytes = 10485760 + +[integrations.testlight] +endpoint = "https://testlight.example/openrtb2/auction" +timeout_ms = 1200 +rewrite_scripts = true + +[integrations.didomi] +enabled = false +sdk_origin = "https://sdk.privacy-center.org" +api_origin = "https://api.privacy-center.org" + +[integrations.permutive] +enabled = false +organization_id = "" +workspace_id = "" +project_id = "" +api_endpoint = "https://api.permutive.com" +secure_signals_endpoint = "https://secure-signals.permutive.app" + +[integrations.lockr] +enabled = false +app_id = "" +api_endpoint = "https://identity.loc.kr" +sdk_url = "https://aim.loc.kr/identity-lockr-trust-server.js" +cache_ttl_seconds = 3600 +rewrite_sdk = true + +[integrations.datadome] +enabled = false +sdk_origin = "https://js.datadome.co" +api_origin = "https://api-js.datadome.co" +cache_ttl_seconds = 3600 +rewrite_sdk = true + +[integrations.gpt] +enabled = false +script_url = "https://securepubads.g.doubleclick.net/tag/js/gpt.js" +cache_ttl_seconds = 3600 +rewrite_script = true + +[proxy] + +[auction] +enabled = true +providers = ["prebid"] +timeout_ms = 2000 +allowed_context_keys = ["permutive_segments"] + +[integrations.aps] +enabled = false +pub_id = "your-aps-publisher-id" +endpoint = "https://origin-mocktioneer.cdintel.com/e/dtb/bid" +timeout_ms = 1000 + +[integrations.google_tag_manager] +enabled = false +container_id = "GTM-XXXXXX" + +[integrations.adserver_mock] +enabled = false +endpoint = "https://origin-mocktioneer.cdintel.com/adserver/mediate" +timeout_ms = 1000 + +[integrations.adserver_mock.context_query_params] +permutive_segments = "permutive" diff --git a/crates/integration-tests/tests/common/runtime.rs b/crates/integration-tests/tests/common/runtime.rs index bf8b795a..0f4ba211 100644 --- a/crates/integration-tests/tests/common/runtime.rs +++ b/crates/integration-tests/tests/common/runtime.rs @@ -71,10 +71,10 @@ pub trait RuntimeProcessHandle: Send + Sync {} /// Trait defining how to run the trusted-server on different platforms. /// -/// The application configuration (origin URL, integrations, etc.) is baked -/// into the WASM binary at build time via `build.rs`. The runtime environment -/// only needs the WASM binary path and its own platform-specific config -/// (e.g. Viceroy's `fastly.toml` for KV stores and secret stores). +/// The application configuration is loaded at runtime from the platform config +/// store. Test environments render a local Viceroy/Fastly config that projects +/// a canonical TOML payload into the fixed `ts-config` key before spawning the +/// runtime. pub trait RuntimeEnvironment: Send + Sync { /// Platform identifier (e.g., "fastly", "cloudflare") fn id(&self) -> &'static str; @@ -112,8 +112,8 @@ pub fn wasm_binary_path() -> PathBuf { /// Get the fixed origin port used for Docker container port mapping. /// -/// This must match the port baked into the WASM binary via -/// `TRUSTED_SERVER__PUBLISHER__ORIGIN_URL` at build time. +/// This must match the origin URL stored in the integration-test application +/// config fixture that is projected into the local config store. pub fn origin_port() -> u16 { match std::env::var("INTEGRATION_ORIGIN_PORT") { Ok(value) => value diff --git a/crates/integration-tests/tests/environments/fastly.rs b/crates/integration-tests/tests/environments/fastly.rs index ec758432..e0a666f0 100644 --- a/crates/integration-tests/tests/environments/fastly.rs +++ b/crates/integration-tests/tests/environments/fastly.rs @@ -3,15 +3,15 @@ use crate::common::runtime::{ }; use error_stack::ResultExt as _; use std::io::{BufRead as _, BufReader}; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::process::{Child, Command, Stdio}; +use std::time::{SystemTime, UNIX_EPOCH}; /// Fastly Compute runtime using Viceroy local simulator. /// -/// Spawns a `viceroy` child process with the WASM binary and the -/// Viceroy-specific `fastly.toml` config (KV stores, secrets). -/// The application config (origin URL, integrations) is baked into -/// the WASM binary at build time. +/// Spawns a `viceroy` child process with the WASM binary and a rendered +/// Viceroy-specific config (KV stores, secrets, and runtime app config store +/// contents). pub struct FastlyViceroy; impl RuntimeEnvironment for FastlyViceroy { @@ -22,7 +22,7 @@ impl RuntimeEnvironment for FastlyViceroy { fn spawn(&self, wasm_path: &Path) -> TestResult { let port = super::find_available_port()?; - let viceroy_config = self.viceroy_config_path(); + let viceroy_config = self.render_viceroy_config()?; let mut child = Command::new("viceroy") .arg(wasm_path) @@ -48,7 +48,10 @@ impl RuntimeEnvironment for FastlyViceroy { } // Wrap immediately so Drop::drop kills the process if readiness check fails - let handle = ViceroyHandle { child }; + let handle = ViceroyHandle { + child, + rendered_config_path: viceroy_config, + }; let base_url = format!("http://127.0.0.1:{port}"); // Fastly exposes a dedicated `/health` route, so root fallback only @@ -63,13 +66,45 @@ impl RuntimeEnvironment for FastlyViceroy { } impl FastlyViceroy { - /// Path to the Viceroy-specific `fastly.toml` template. - /// - /// This contains `[local_server]` configuration (backends, KV stores, - /// secret stores) that Viceroy needs, separate from the application config. - fn viceroy_config_path(&self) -> std::path::PathBuf { - std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("fixtures/configs/viceroy-template.toml") + /// Render a Viceroy config with the application config projected into the + /// runtime config store. + fn render_viceroy_config(&self) -> TestResult { + let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + let template = manifest_dir.join("fixtures/configs/viceroy-template.toml"); + let app_config = manifest_dir.join("fixtures/configs/trusted-server.integration.toml"); + let unique_suffix = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("should compute monotonic temp suffix") + .as_nanos(); + let output = std::env::temp_dir().join(format!( + "trusted-server-viceroy-{unique_suffix}.toml" + )); + let render_script = manifest_dir + .parent() + .and_then(Path::parent) + .expect("should find repository root") + .join("scripts/render-fastly-local-config.py"); + + let status = Command::new("python3") + .arg(render_script) + .arg("--app-config") + .arg(app_config) + .arg("--template") + .arg(template) + .arg("--output") + .arg(&output) + .stdout(Stdio::null()) + .stderr(Stdio::inherit()) + .status() + .change_context(TestError::RuntimeSpawn) + .attach("Failed to render Viceroy config")?; + + if !status.success() { + return Err(error_stack::Report::new(TestError::RuntimeSpawn) + .attach("render-fastly-local-config.py exited unsuccessfully")); + } + + Ok(output) } } @@ -79,6 +114,7 @@ impl FastlyViceroy { /// preventing orphaned Viceroy processes. struct ViceroyHandle { child: Child, + rendered_config_path: PathBuf, } impl RuntimeProcessHandle for ViceroyHandle {} @@ -87,5 +123,6 @@ impl Drop for ViceroyHandle { fn drop(&mut self) { let _ = self.child.kill(); let _ = self.child.wait(); + let _ = std::fs::remove_file(&self.rendered_config_path); } } diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index d2b905d3..f566531e 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -1,4 +1,4 @@ -use error_stack::Report; +use error_stack::{Report, ResultExt}; use fastly::http::{header, Method}; use fastly::{Request, Response}; @@ -13,7 +13,7 @@ use trusted_server_core::constants::{ use trusted_server_core::error::TrustedServerError; use trusted_server_core::geo::GeoInfo; use trusted_server_core::integrations::IntegrationRegistry; -use trusted_server_core::platform::RuntimeServices; +use trusted_server_core::platform::{RuntimeServices, StoreName}; use trusted_server_core::proxy::{ handle_first_party_click, handle_first_party_proxy, handle_first_party_proxy_rebuild, handle_first_party_proxy_sign, @@ -25,8 +25,10 @@ use trusted_server_core::request_signing::{ handle_deactivate_key, handle_rotate_key, handle_trusted_server_discovery, handle_verify_signature, }; -use trusted_server_core::settings::Settings; -use trusted_server_core::settings_data::get_settings; +use trusted_server_core::runtime_config::{ + load_runtime_config, APPLICATION_CONFIG_KEY, APPLICATION_CONFIG_STORE_NAME, +}; +use trusted_server_core::settings::{EdgeCookie, Publisher, Settings}; mod error; mod logging; @@ -51,25 +53,35 @@ fn main() { let req = Request::from_client(); - // Keep the health probe independent from settings loading and routing so - // readiness checks still get a cheap liveness response during startup. - if req.get_method() == Method::GET && req.get_path() == "/health" { - Response::from_status(200) - .with_body_text_plain("ok") - .send_to_client(); - return; - } + // Start with an unavailable KV slot. Consent-dependent routes lazily + // replace it with the configured store at dispatch time so unrelated + // routes stay available when consent persistence is misconfigured. + let kv_store = std::sync::Arc::new(UnavailableKvStore) + as std::sync::Arc; + let runtime_services = build_runtime_services(&req, kv_store); - let settings = match get_settings() { - Ok(s) => s, + let loaded_config = match load_settings_from_config_store(&runtime_services) { + Ok(config) => config, Err(e) => { log::error!("Failed to load settings: {:?}", e); to_error_response(&e).send_to_client(); return; } }; + let settings = loaded_config.settings; + log::debug!("Loaded runtime config hash={}", loaded_config.config_hash); log::debug!("Settings {settings:?}"); + // `/health` intentionally depends on successful runtime config loading. + // A missing or invalid `ts-config` payload means the service is not ready + // to serve application traffic, so health checks should fail as well. + if req.get_method() == Method::GET && req.get_path() == "/health" { + Response::from_status(200) + .with_body_text_plain("ok") + .send_to_client(); + return; + } + // Short-circuit the ja4 debug probe before finalize_response so that // Cache-Control: no-store, private cannot be replaced by operator [response_headers]. if req.get_method() == Method::GET && req.get_path() == "/_ts/debug/ja4" { @@ -100,13 +112,6 @@ fn main() { } }; - // Start with an unavailable KV slot. Consent-dependent routes lazily - // replace it with the configured store at dispatch time so unrelated - // routes stay available when consent persistence is misconfigured. - let kv_store = std::sync::Arc::new(UnavailableKvStore) - as std::sync::Arc; - let runtime_services = build_runtime_services(&req, kv_store); - // route_request may send the response directly (streaming path) or // return it for us to send (buffered path). if let Some(response) = futures::executor::block_on(route_request( @@ -162,6 +167,45 @@ fn build_ja4_debug_response(req: &Request) -> Response { .with_body(body) } +fn load_settings_from_config_store( + runtime_services: &RuntimeServices, +) -> Result> { + let store_name = StoreName::from(APPLICATION_CONFIG_STORE_NAME); + let payload = runtime_services + .config_store() + .get(&store_name, APPLICATION_CONFIG_KEY) + .change_context(TrustedServerError::Configuration { + message: format!( + "Failed to read application config from store `{}` key `{}`", + APPLICATION_CONFIG_STORE_NAME, APPLICATION_CONFIG_KEY + ), + })?; + + let loaded = load_runtime_config(&payload)?; + + if !loaded.settings.proxy.certificate_check { + log::warn!( + "INSECURE: proxy.certificate_check is disabled — TLS certificates will NOT be verified" + ); + } + + if EdgeCookie::is_placeholder_secret_key(loaded.settings.edge_cookie.secret_key.expose()) { + log::warn!( + "INSECURE: edge_cookie.secret_key is set to a default placeholder — \ + HMAC-SHA256 signatures can be forged" + ); + } + + if Publisher::is_placeholder_proxy_secret(loaded.settings.publisher.proxy_secret.expose()) { + log::warn!( + "INSECURE: publisher.proxy_secret is set to a default placeholder — \ + XChaCha20-Poly1305 encrypted URLs can be decrypted by anyone" + ); + } + + Ok(loaded) +} + async fn route_request( settings: &Settings, orchestrator: &AuctionOrchestrator, @@ -184,7 +228,7 @@ async fn route_request( None }); - // `get_settings()` should already have rejected invalid handler regexes. + // Runtime config loading should already have rejected invalid handler regexes. // Keep this fallback so manually-constructed or otherwise unprepared // settings still become an error response instead of panicking. let auth_req = compat::from_fastly_headers_ref(&req); diff --git a/crates/trusted-server-core/build.rs b/crates/trusted-server-core/build.rs index 6f56e85a..f328e4d9 100644 --- a/crates/trusted-server-core/build.rs +++ b/crates/trusted-server-core/build.rs @@ -1,54 +1 @@ -// Build script includes source modules (`error`, `auction_config_types`, etc.) -// for compile-time config validation. Not all items from those modules are used -// in the build context, so `dead_code` is expected. -#![allow(clippy::unwrap_used, clippy::panic, dead_code)] - -#[path = "src/error.rs"] -mod error; - -#[path = "src/auction_config_types.rs"] -mod auction_config_types; - -#[path = "src/redacted.rs"] -mod redacted; - -#[path = "src/consent_config.rs"] -mod consent_config; - -#[path = "src/settings.rs"] -mod settings; - -use std::fs; -use std::path::Path; - -const TRUSTED_SERVER_INIT_CONFIG_PATH: &str = "../../trusted-server.toml"; -const TRUSTED_SERVER_OUTPUT_CONFIG_PATH: &str = "../../target/trusted-server-out.toml"; - -fn main() { - // Always rerun build.rs: integration settings are stored in a flat - // HashMap, so we cannot enumerate all possible env - // var keys ahead of time. Emitting rerun-if-changed for a nonexistent - // file forces cargo to always rerun the build script. - println!("cargo:rerun-if-changed=_always_rebuild_sentinel_"); - - // Read init config - let init_config_path = Path::new(TRUSTED_SERVER_INIT_CONFIG_PATH); - let toml_content = fs::read_to_string(init_config_path) - .unwrap_or_else(|_| panic!("Failed to read {init_config_path:?}")); - - // Merge base TOML with environment variable overrides and write output. - // Panics if admin endpoints are not covered by a handler. - let settings = settings::Settings::from_toml_and_env(&toml_content) - .expect("Failed to parse settings at build time"); - - let merged_toml = - toml::to_string_pretty(&settings).expect("Failed to serialize settings to TOML"); - - // Only write when content changes to avoid unnecessary recompilation. - let dest_path = Path::new(TRUSTED_SERVER_OUTPUT_CONFIG_PATH); - let current = fs::read_to_string(dest_path).unwrap_or_default(); - if current != merged_toml { - fs::write(dest_path, merged_toml) - .unwrap_or_else(|_| panic!("Failed to write {dest_path:?}")); - } -} +fn main() {} diff --git a/crates/trusted-server-core/src/auction_config_types.rs b/crates/trusted-server-core/src/auction_config_types.rs index bc486ded..88751358 100644 --- a/crates/trusted-server-core/src/auction_config_types.rs +++ b/crates/trusted-server-core/src/auction_config_types.rs @@ -5,6 +5,7 @@ use std::collections::HashSet; /// Auction orchestration configuration. #[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] pub struct AuctionConfig { /// Enable the auction orchestrator #[serde(default)] diff --git a/crates/trusted-server-core/src/bin/ts-config-canonicalize.rs b/crates/trusted-server-core/src/bin/ts-config-canonicalize.rs new file mode 100644 index 00000000..5f978f69 --- /dev/null +++ b/crates/trusted-server-core/src/bin/ts-config-canonicalize.rs @@ -0,0 +1,36 @@ +use std::env; +use std::fs; +use std::io::{self, Write as _}; +use std::process::ExitCode; + +use trusted_server_core::runtime_config::load_runtime_config; + +fn main() -> ExitCode { + let Some(path) = env::args().nth(1) else { + let _ = writeln!( + io::stderr(), + "usage: ts-config-canonicalize " + ); + return ExitCode::from(2); + }; + + let toml = match fs::read_to_string(&path) { + Ok(contents) => contents, + Err(error) => { + let _ = writeln!(io::stderr(), "failed to read {path}: {error}"); + return ExitCode::from(1); + } + }; + + let loaded = match load_runtime_config(&toml) { + Ok(config) => config, + Err(error) => { + let _ = writeln!(io::stderr(), "failed to canonicalize config: {error:?}"); + return ExitCode::from(1); + } + }; + + let _ = writeln!(io::stderr(), "config hash: {}", loaded.config_hash); + let _ = write!(io::stdout(), "{}", loaded.canonical_toml); + ExitCode::SUCCESS +} diff --git a/crates/trusted-server-core/src/consent_config.rs b/crates/trusted-server-core/src/consent_config.rs index e2074c48..7051f3ef 100644 --- a/crates/trusted-server-core/src/consent_config.rs +++ b/crates/trusted-server-core/src/consent_config.rs @@ -35,6 +35,7 @@ fn str_vec(codes: &[&str]) -> Vec { /// Top-level consent configuration (`[consent]` in TOML). #[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] pub struct ConsentConfig { /// Operating mode for consent handling. /// @@ -178,6 +179,7 @@ impl ConsentForwardingMode { /// this list, the system logs that GDPR applies, enabling publishers to /// monitor compliance coverage. #[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] pub struct GdprConfig { /// ISO 3166-1 alpha-2 country codes where GDPR applies. #[serde(default = "default_gdpr_countries")] @@ -200,6 +202,7 @@ impl Default for GdprConfig { /// /// Config-driven to avoid recompilation when new state laws take effect. #[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] pub struct UsStatesConfig { /// US state codes with active comprehensive privacy laws. #[serde(default = "default_us_privacy_states")] @@ -224,6 +227,7 @@ impl Default for UsStatesConfig { /// These reflect the publisher's actual compliance posture — they are /// **publisher policy**, not protocol requirements. #[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] pub struct UsPrivacyDefaultsConfig { /// Whether the publisher has actually shown a CCPA notice to the user. #[serde(default = "default_true")] @@ -257,6 +261,7 @@ impl Default for UsPrivacyDefaultsConfig { /// How to resolve disagreements between GPP and TC String when both are /// present (`[consent.conflict_resolution]`). #[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] pub struct ConflictResolutionConfig { /// Resolution strategy. #[serde(default = "default_conflict_mode")] diff --git a/crates/trusted-server-core/src/lib.rs b/crates/trusted-server-core/src/lib.rs index e1af33b7..23bdaa6f 100644 --- a/crates/trusted-server-core/src/lib.rs +++ b/crates/trusted-server-core/src/lib.rs @@ -57,8 +57,8 @@ pub mod publisher; pub mod redacted; pub mod request_signing; pub mod rsc_flight; +pub mod runtime_config; pub mod settings; -pub mod settings_data; pub mod storage; pub mod streaming_processor; pub mod streaming_replacer; diff --git a/crates/trusted-server-core/src/runtime_config.rs b/crates/trusted-server-core/src/runtime_config.rs new file mode 100644 index 00000000..04e10a5d --- /dev/null +++ b/crates/trusted-server-core/src/runtime_config.rs @@ -0,0 +1,252 @@ +//! Runtime application configuration loading and canonicalization. +//! +//! This module defines the runtime configuration contract for Trusted Server: +//! application config is loaded as TOML, parsed strictly, validated +//! semantically, canonicalized deterministically, and hashed from canonical +//! bytes. + +use std::collections::BTreeSet; + +use error_stack::{Report, ResultExt}; +use sha2::{Digest as _, Sha256}; +use toml::Value as TomlValue; +use validator::Validate; + +use crate::error::TrustedServerError; +use crate::settings::{parse_toml_document, Settings, TOP_LEVEL_APPLICATION_CONFIG_KEYS}; + +/// Hardcoded runtime config store name. +pub const APPLICATION_CONFIG_STORE_NAME: &str = "ts_config_store"; + +/// Hardcoded runtime config payload key. +pub const APPLICATION_CONFIG_KEY: &str = "ts-config"; + +/// Fully processed runtime config. +#[derive(Debug, Clone)] +pub struct LoadedRuntimeConfig { + /// Validated immutable settings snapshot used for a single request. + pub settings: Settings, + /// Deterministic canonical TOML payload. + pub canonical_toml: String, + /// Lowercase hex SHA-256 of [`Self::canonical_toml`]. + pub config_hash: String, +} + +/// Parse, validate, canonicalize, and hash runtime configuration. +/// +/// # Errors +/// +/// Returns [`TrustedServerError::Configuration`] when the TOML is malformed, +/// contains unknown fields, fails semantic validation, or cannot be +/// canonicalized. +pub fn load_runtime_config( + toml_str: &str, +) -> Result> { + let parsed_document = parse_toml_document(toml_str)?; + let settings = Settings::from_toml(toml_str)?; + + settings + .validate() + .change_context(TrustedServerError::Configuration { + message: "Failed to validate configuration".to_string(), + })?; + + let normalized_value = + TomlValue::try_from(&settings).change_context(TrustedServerError::Configuration { + message: "Failed to serialize validated configuration".to_string(), + })?; + + let canonical_value = retain_declared_fields( + &TomlValue::Table(parsed_document), + &normalized_value, + "root", + )?; + + let canonical_toml = + toml::to_string(&canonical_value).change_context(TrustedServerError::Configuration { + message: "Failed to serialize canonical TOML".to_string(), + })?; + + let config_hash = hex::encode(Sha256::digest(canonical_toml.as_bytes())); + + Ok(LoadedRuntimeConfig { + settings, + canonical_toml, + config_hash, + }) +} + +/// Returns the known top-level application-config keys. +#[must_use] +pub fn known_top_level_keys() -> &'static [&'static str] { + TOP_LEVEL_APPLICATION_CONFIG_KEYS +} + +fn retain_declared_fields( + raw: &TomlValue, + normalized: &TomlValue, + path: &str, +) -> Result> { + match (raw, normalized) { + (TomlValue::Table(raw_table), TomlValue::Table(normalized_table)) => { + let mut output = toml::map::Map::new(); + let mut keys: BTreeSet<&String> = BTreeSet::new(); + for key in raw_table.keys() { + keys.insert(key); + } + + for key in keys { + let child_path = format!("{path}.{key}"); + let raw_value = raw_table.get(key).expect("should contain raw key"); + let normalized_value = normalized_table.get(key).ok_or_else(|| { + Report::new(TrustedServerError::Configuration { + message: format!( + "Canonicalization failed because `{child_path}` was not preserved" + ), + }) + })?; + output.insert( + key.clone(), + retain_declared_fields(raw_value, normalized_value, &child_path)?, + ); + } + + Ok(TomlValue::Table(output)) + } + (TomlValue::Array(raw_items), TomlValue::Array(normalized_items)) => { + if raw_items.len() != normalized_items.len() { + return Err(Report::new(TrustedServerError::Configuration { + message: format!( + "Canonicalization failed because array length changed at `{path}`" + ), + })); + } + + let mut items = Vec::with_capacity(raw_items.len()); + for (index, (raw_item, normalized_item)) in + raw_items.iter().zip(normalized_items).enumerate() + { + items.push(retain_declared_fields( + raw_item, + normalized_item, + &format!("{path}[{index}]"), + )?); + } + Ok(TomlValue::Array(items)) + } + (_, normalized_scalar) => Ok(normalized_scalar.clone()), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn base_config() -> &'static str { + r#" +[[handlers]] +path = "^/admin" +username = "admin" +password = "secret" + +[publisher] +domain = "publisher.com" +cookie_domain = ".publisher.com" +origin_url = "https://origin.publisher.com" +proxy_secret = "proxy-secret" + +[edge_cookie] +secret_key = "secret-key" +"# + } + + #[test] + fn canonicalization_preserves_only_declared_fields() { + let loaded = load_runtime_config(base_config()).expect("should load valid config"); + assert!( + !loaded.canonical_toml.contains("[proxy]"), + "should not serialize omitted default sections" + ); + assert!( + !loaded.canonical_toml.contains("certificate_check"), + "should not serialize omitted default fields" + ); + } + + #[test] + fn canonicalization_normalizes_declared_values() { + let config = r#" +[[handlers]] +path = "^/admin" +username = "admin" +password = "secret" + +[publisher] +domain = "publisher.com" +cookie_domain = ".publisher.com" +origin_url = "https://origin.publisher.com" +proxy_secret = "proxy-secret" + +[edge_cookie] +secret_key = "secret-key" + +[proxy] +allowed_domains = [" Example.COM ", "*.DoubleClick.Net"] +"#; + + let loaded = load_runtime_config(config).expect("should load valid config"); + assert!( + loaded + .canonical_toml + .contains("allowed_domains = [\"example.com\", \"*.doubleclick.net\"]"), + "should canonicalize normalized proxy allowed_domains" + ); + } + + #[test] + fn config_hash_is_stable_for_reordered_input() { + let a = r#" +[[handlers]] +path = "^/admin" +username = "admin" +password = "secret" + +[publisher] +domain = "publisher.com" +cookie_domain = ".publisher.com" +origin_url = "https://origin.publisher.com" +proxy_secret = "proxy-secret" + +[edge_cookie] +secret_key = "secret-key" +"#; + + let b = r#" +[publisher] +origin_url = "https://origin.publisher.com" +proxy_secret = "proxy-secret" +cookie_domain = ".publisher.com" +domain = "publisher.com" + +[edge_cookie] +secret_key = "secret-key" + +[[handlers]] +username = "admin" +password = "secret" +path = "^/admin" +"#; + + let loaded_a = load_runtime_config(a).expect("should load first config"); + let loaded_b = load_runtime_config(b).expect("should load reordered config"); + + assert_eq!( + loaded_a.config_hash, loaded_b.config_hash, + "should produce identical hashes for semantically identical config" + ); + assert_eq!( + loaded_a.canonical_toml, loaded_b.canonical_toml, + "should canonicalize reordered input to the same bytes" + ); + } +} diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index d851aff7..505912b2 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -1,3 +1,4 @@ +#[cfg(test)] use config::{Config, Environment, File, FileFormat}; use error_stack::{Report, ResultExt}; use regex::Regex; @@ -6,6 +7,7 @@ use serde_json::Value as JsonValue; use std::collections::HashMap; use std::ops::{Deref, DerefMut}; use std::sync::OnceLock; +use toml::Table as TomlTable; use url::Url; use validator::{Validate, ValidationError}; @@ -14,10 +16,28 @@ use crate::consent_config::ConsentConfig; use crate::error::TrustedServerError; use crate::redacted::Redacted; +/// Known top-level keys for the application configuration document. +pub const TOP_LEVEL_APPLICATION_CONFIG_KEYS: &[&str] = &[ + "auction", + "consent", + "debug", + "edge_cookie", + "handlers", + "integrations", + "proxy", + "publisher", + "request_signing", + "response_headers", + "rewrite", +]; + +#[cfg(test)] pub const ENVIRONMENT_VARIABLE_PREFIX: &str = "TRUSTED_SERVER"; +#[cfg(test)] pub const ENVIRONMENT_VARIABLE_SEPARATOR: &str = "__"; #[derive(Debug, Default, Clone, Deserialize, Serialize, Validate)] +#[serde(deny_unknown_fields)] pub struct Publisher { pub domain: String, #[validate(custom(function = validate_cookie_domain))] @@ -83,6 +103,36 @@ pub trait IntegrationConfig: DeserializeOwned + Validate { fn is_enabled(&self) -> bool; } +/// Parse the root TOML document and reject unknown top-level keys. +/// +/// # Errors +/// +/// Returns [`TrustedServerError::Configuration`] when the payload is malformed +/// TOML or contains unsupported top-level fields. +pub(crate) fn parse_toml_document(toml_str: &str) -> Result> { + let document: TomlTable = + toml::from_str(toml_str).change_context(TrustedServerError::Configuration { + message: "Failed to deserialize TOML configuration".to_string(), + })?; + + let unknown_keys: Vec = document + .keys() + .filter(|key| !TOP_LEVEL_APPLICATION_CONFIG_KEYS.contains(&key.as_str())) + .cloned() + .collect(); + + if unknown_keys.is_empty() { + return Ok(document); + } + + Err(Report::new(TrustedServerError::Configuration { + message: format!( + "Unknown top-level configuration field(s): {}", + unknown_keys.join(", ") + ), + })) +} + impl IntegrationSettings { /// Inserts a configuration value for an integration. /// @@ -206,6 +256,7 @@ impl DerefMut for IntegrationSettings { /// Edge Cookie configuration. #[allow(unused)] #[derive(Debug, Default, Clone, Deserialize, Serialize, Validate)] +#[serde(deny_unknown_fields)] pub struct EdgeCookie { #[validate(custom(function = EdgeCookie::validate_secret_key))] pub secret_key: Redacted, @@ -238,6 +289,7 @@ impl EdgeCookie { } #[derive(Debug, Default, Clone, Deserialize, Serialize, Validate)] +#[serde(deny_unknown_fields)] pub struct Rewrite { /// List of domains to exclude from rewriting. Supports wildcards (e.g., "*.example.com"). /// URLs from these domains will not be proxied through first-party endpoints. @@ -274,6 +326,7 @@ impl Rewrite { } #[derive(Debug, Default, Clone, Deserialize, Serialize, Validate)] +#[serde(deny_unknown_fields)] pub struct Handler { #[validate(length(min = 1), custom(function = validate_path))] pub path: String, @@ -322,6 +375,7 @@ impl Handler { } #[derive(Debug, Default, Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] pub struct RequestSigning { #[serde(default = "default_request_signing_enabled")] pub enabled: bool, @@ -334,6 +388,7 @@ fn default_request_signing_enabled() -> bool { } #[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] pub struct Proxy { /// Enable TLS certificate verification when proxying to HTTPS origins. /// Defaults to true for secure production use. @@ -441,15 +496,15 @@ pub struct Settings { #[allow(unused)] impl Settings { - /// Creates a new [`Settings`] instance from a pre-built TOML string. - /// - /// Use this for the runtime path where the TOML has already been - /// fully resolved (env vars baked in by build.rs). + /// Creates a new [`Settings`] instance from a runtime TOML string. /// /// # Errors /// - /// - [`TrustedServerError::Configuration`] if the TOML is invalid or missing required fields + /// - [`TrustedServerError::Configuration`] if the TOML is invalid, contains unknown fields, + /// or is missing required fields pub fn from_toml(toml_str: &str) -> Result> { + parse_toml_document(toml_str)?; + let mut settings: Self = toml::from_str(toml_str).change_context(TrustedServerError::Configuration { message: "Failed to deserialize TOML configuration".to_string(), @@ -466,13 +521,18 @@ impl Settings { /// Creates a new [`Settings`] instance from a TOML string, applying /// environment variable overrides using the `TRUSTED_SERVER__` prefix. /// - /// Used by build.rs to merge the base config with env vars before - /// baking the result into the binary. + /// Test-only compatibility helper retained for unit tests that still cover + /// legacy env-override parsing semantics. /// /// # Errors /// - /// - [`TrustedServerError::Configuration`] if the TOML is invalid or missing required fields + /// Returns [`TrustedServerError::Configuration`] if the TOML is invalid, + /// the merged configuration fails to deserialize, or required validation + /// checks fail. + #[cfg(test)] pub fn from_toml_and_env(toml_str: &str) -> Result> { + parse_toml_document(toml_str)?; + let environment = Environment::default() .prefix(ENVIRONMENT_VARIABLE_PREFIX) .separator(ENVIRONMENT_VARIABLE_SEPARATOR); @@ -495,13 +555,6 @@ impl Settings { settings.integrations.normalize(); settings.proxy.normalize(); settings.consent.validate(); - - settings.validate().map_err(|err| { - Report::new(TrustedServerError::Configuration { - message: format!("Build-time configuration validation failed: {err}"), - }) - })?; - settings.prepare_runtime()?; settings.validate_admin_coverage()?; @@ -1290,8 +1343,7 @@ mod tests { fn test_settings_extra_fields() { let toml_str = crate_test_settings_str() + "\nhello = 1"; - let settings = Settings::from_toml(&toml_str); - assert!(settings.is_ok(), "Extra fields should be ignored"); + let _error = Settings::from_toml(&toml_str).expect_err("should reject unknown field"); } #[test] diff --git a/crates/trusted-server-core/src/settings_data.rs b/crates/trusted-server-core/src/settings_data.rs deleted file mode 100644 index f69fc7ba..00000000 --- a/crates/trusted-server-core/src/settings_data.rs +++ /dev/null @@ -1,76 +0,0 @@ -use core::str; -use error_stack::{Report, ResultExt}; -use validator::Validate; - -use crate::error::TrustedServerError; -use crate::settings::{EdgeCookie, Publisher, Settings}; - -pub use crate::auction_config_types::AuctionConfig; - -const SETTINGS_DATA: &[u8] = include_bytes!("../../../target/trusted-server-out.toml"); - -/// Creates a new [`Settings`] instance from the embedded configuration file. -/// -/// Loads the pre-built TOML that was generated by `build.rs` (base config -/// merged with any `TRUSTED_SERVER__` environment variable overrides at -/// build time). Environment variables are **not** read at runtime. -/// -/// # Errors -/// -/// - [`TrustedServerError::InvalidUtf8`] if the embedded TOML file contains invalid UTF-8 -/// - [`TrustedServerError::Configuration`] if the configuration is invalid or missing required fields -pub fn get_settings() -> Result> { - let toml_bytes = SETTINGS_DATA; - let toml_str = str::from_utf8(toml_bytes).change_context(TrustedServerError::InvalidUtf8 { - message: "embedded trusted-server.toml file".to_string(), - })?; - - let settings = Settings::from_toml(toml_str)?; - - // Validate the settings - settings - .validate() - .change_context(TrustedServerError::Configuration { - message: "Failed to validate configuration".to_string(), - })?; - - if !settings.proxy.certificate_check { - log::warn!( - "INSECURE: proxy.certificate_check is disabled — TLS certificates will NOT be verified" - ); - } - - if EdgeCookie::is_placeholder_secret_key(settings.edge_cookie.secret_key.expose()) { - log::warn!( - "INSECURE: edge_cookie.secret_key is set to a default placeholder — \ - HMAC-SHA256 signatures can be forged. \ - Override via TRUSTED_SERVER__EDGE_COOKIE__SECRET_KEY at build time" - ); - } - - if Publisher::is_placeholder_proxy_secret(settings.publisher.proxy_secret.expose()) { - log::warn!( - "INSECURE: publisher.proxy_secret is set to a default placeholder — \ - XChaCha20-Poly1305 encrypted URLs can be decrypted by anyone. \ - Override via TRUSTED_SERVER__PUBLISHER__PROXY_SECRET at build time" - ); - } - - Ok(settings) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn get_settings_loads_embedded_toml_successfully() { - // The embedded TOML contains placeholder secrets (e.g. "trusted-server", - // "change-me-proxy-secret"). This is expected — production builds override - // them via TRUSTED_SERVER__* env vars at build time. - let settings = get_settings().expect("should load settings from embedded TOML"); - assert!(!settings.publisher.domain.is_empty()); - assert!(!settings.publisher.cookie_domain.is_empty()); - assert!(!settings.publisher.origin_url.is_empty()); - } -} diff --git a/crates/trusted-server-core/src/test_support.rs b/crates/trusted-server-core/src/test_support.rs index 8fdfaa85..b3a7c239 100644 --- a/crates/trusted-server-core/src/test_support.rs +++ b/crates/trusted-server-core/src/test_support.rs @@ -22,7 +22,6 @@ pub mod tests { [publisher] domain = "test-publisher.com" cookie_domain = ".test-publisher.com" - origin_backend = "publisher_origin" origin_url = "https://origin.test-publisher.com" proxy_secret = "unit-test-proxy-secret" diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index bfa655d9..0930e048 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -4,11 +4,11 @@ Learn how to configure Trusted Server for your deployment. ## Overview -Trusted Server uses a flexible configuration system based on: +Trusted Server uses a runtime configuration system based on: -1. **TOML Files** - `trusted-server.toml` for base configuration -2. **Environment Variables** - Build-time overrides with `TRUSTED_SERVER__` prefix (baked into the binary by `build.rs`) -3. **Fastly Stores** - KV/Config/Secret stores for runtime data +1. **TOML authoring** - `trusted-server.toml` for local authoring in development +2. **Config store deployment** - canonical TOML stored at runtime under the fixed key `ts-config` +3. **Fastly stores** - KV/Config/Secret stores for runtime data ## Quick Start @@ -27,18 +27,13 @@ proxy_secret = "your-secure-secret-here" secret_key = "your-hmac-secret" ``` -### Environment Variable Overrides +### Runtime loading model -Override any setting at build time. Environment variables are merged into the -config by `build.rs` and baked into the compiled binary — they are **not** read -at runtime. - -```bash -# Format: TRUSTED_SERVER__SECTION__FIELD -export TRUSTED_SERVER__PUBLISHER__DOMAIN=publisher.com -export TRUSTED_SERVER__PUBLISHER__ORIGIN_URL=https://origin.publisher.com -export TRUSTED_SERVER__EDGE_COOKIE__SECRET_KEY=your-secret -``` +- **Production** reads canonical TOML from the configured platform config store +- **Development** authors a local `trusted-server.toml`, then projects its + canonical form into the local simulated config store before startup +- Application settings are **not** merged from `TRUSTED_SERVER__*` environment + variables at build time anymore ### Generate Secure Secrets @@ -49,11 +44,12 @@ openssl rand -base64 32 ## Configuration Files -| File | Purpose | -| --------------------- | ------------------------------- | -| `trusted-server.toml` | Main application configuration | -| `fastly.toml` | Fastly Compute service settings | -| `.env.dev` | Local development overrides | +| File | Purpose | +| ----------------------------- | ----------------------------------------------------- | +| `trusted-server.toml` | Local development authoring file (untracked) | +| `trusted-server.example.toml` | Tracked template for authoring real config | +| `fastly.toml` | Fastly Compute service settings | +| `.env.dev` | Local development environment variables (non-app cfg) | ## Key Sections @@ -95,11 +91,15 @@ client_side_bidders = ["rubicon"] The sections below consolidate the full configuration reference on this page. -## Environment Variable Overrides (Build-Time) +## Runtime config store -Environment variables with the `TRUSTED_SERVER__` prefix are merged into the -base TOML configuration by `build.rs` at compile time. The resulting config is -embedded in the binary. Changing an environment variable requires a rebuild. +Application configuration is loaded at runtime from a config store entry whose +key is always `ts-config`. The payload stored under that key is canonical TOML. +Changing application settings no longer requires rebuilding the WASM binary. + +> **Note:** Older revisions of this guide referred to build-time +> `TRUSTED_SERVER__*` application-setting overrides. That mechanism has been +> removed for application configuration. ### Format @@ -1043,28 +1043,29 @@ trusted-server.dev.toml # Development overrides **Environment Variables Not Applied**: -- Env vars are applied at **build time** only — rebuild after changing them -- Verify prefix: `TRUSTED_SERVER__` -- Check separator: `__` (double underscore) -- Confirm variable is exported: `echo $VARIABLE_NAME` -- Try explicit string: `VARIABLE='value'` not `VARIABLE=value` +- Validate the authored TOML before projecting it into the config store +- Re-render the local Fastly/Viceroy config after changing `trusted-server.toml` +- Confirm the local rendered manifest contains `local_server.config_stores.ts_config_store` +- Confirm the `ts-config` key contains the expected canonical TOML ### Debug Configuration -**Print Loaded Config** (test only): - -```rust -use trusted_server_core::settings_data::get_settings; +**Canonicalize local config**: -let settings = get_settings()?; -println!("{:#?}", settings); +```bash +cargo run --target "$(rustc -vV | sed -n 's/^host: //p')" \ + -p trusted-server-core \ + --bin ts-config-canonicalize \ + -- trusted-server.toml ``` -**Check Environment**: +**Render local Fastly config**: ```bash -# List all TRUSTED_SERVER variables -env | grep TRUSTED_SERVER +python3 scripts/render-fastly-local-config.py \ + --app-config trusted-server.toml \ + --template fastly.toml \ + --output fastly.local.toml ``` **Validate TOML**: diff --git a/docs/guide/error-reference.md b/docs/guide/error-reference.md index 99f611aa..743f44e4 100644 --- a/docs/guide/error-reference.md +++ b/docs/guide/error-reference.md @@ -22,14 +22,16 @@ Common errors, their causes, and solutions when working with Trusted Server. Failed to load settings: ParseError ``` -**Cause:** Invalid TOML syntax in `trusted-server.toml` +**Cause:** Invalid application TOML in the runtime config store payload (`ts-config`), +or failure to read that payload from the configured config store. **Solution:** -1. Validate TOML syntax using an online validator -2. Check for missing quotes around strings -3. Ensure array syntax uses square brackets: `["item1", "item2"]` -4. Verify section headers use brackets: `[section]` +1. Validate the authored `trusted-server.toml` before deployment +2. Re-render local Fastly/Viceroy config after changing `trusted-server.toml` +3. Check for missing quotes around strings +4. Ensure array syntax uses square brackets: `["item1", "item2"]` +5. Verify section headers use brackets: `[section]` **Example Fix:** @@ -97,33 +99,36 @@ server_url = "https://prebid-server.example.com" --- -### Environment variable override failed +### Runtime config store payload failed **Error Message:** ``` -Failed to parse environment variable: TRUSTED_SERVER__PUBLISHER__DOMAIN +Failed to load settings ``` -**Cause:** Environment variable format doesn't match expected type +**Cause:** The runtime `ts-config` payload is missing, malformed, or does not +match the expected application-config schema. -**Solution:** Use correct format for the field type: +**Solution:** Validate and canonicalize the TOML before projecting it into the +config store: ```bash -# For strings -TRUSTED_SERVER__PUBLISHER__DOMAIN="example.com" - -# For numbers -TRUSTED_SERVER__INTEGRATIONS__PREBID__TIMEOUT_MS=1000 +cargo run --target "$(rustc -vV | sed -n 's/^host: //p')" \ + -p trusted-server-core \ + --bin ts-config-canonicalize \ + -- trusted-server.toml +``` -# For booleans -TRUSTED_SERVER__INTEGRATIONS__PREBID__ENABLED=true +For local Fastly/Viceroy development, regenerate the rendered manifest after +editing `trusted-server.toml`: -# For arrays (comma-separated) -TRUSTED_SERVER__INTEGRATIONS__PREBID__BIDDERS="appnexus,rubicon" +```bash +scripts/fastly-dev.sh --skip-build ``` -See [Configuration Reference](./configuration.md) for complete patterns. +Application settings are no longer loaded from `TRUSTED_SERVER__*` +configuration environment variables at build time. --- diff --git a/docs/guide/testing.md b/docs/guide/testing.md index cfa7ea51..b03dcf55 100644 --- a/docs/guide/testing.md +++ b/docs/guide/testing.md @@ -77,13 +77,18 @@ cargo test -- --test-threads=1 ```bash # Start local server -fastly compute serve +scripts/fastly-dev.sh # Test endpoints with curl curl http://localhost:7676/health curl http://localhost:7676/.well-known/trusted-server.json ``` +`GET /health` now depends on successful runtime config loading from the local +config store. If `trusted-server.toml` cannot be canonicalized and projected +into `ts-config`, the service is considered unhealthy and `/health` will return +an error instead of `200 ok`. + ## Real Test Examples ### EC ID Tests diff --git a/docs/superpowers/specs/2026-04-20-config-store-runtime-config-design.md b/docs/superpowers/specs/2026-04-20-config-store-runtime-config-design.md new file mode 100644 index 00000000..cd48b561 --- /dev/null +++ b/docs/superpowers/specs/2026-04-20-config-store-runtime-config-design.md @@ -0,0 +1,515 @@ +# Runtime Config Store Architecture for `trusted-server.toml` + +> **Status:** Proposal +> **Scope:** Config store architecture only +> **Date:** April 2026 + +## Summary + +This document specifies the application configuration changes required to move +Trusted Server from **build-time embedded configuration** to **runtime-loaded +configuration**. + +Today, Trusted Server reads `trusted-server.toml` during the build, merges +`TRUSTED_SERVER__*` environment variable overrides, writes a generated TOML +file to `target/trusted-server-out.toml`, and embeds that generated file into +the WASM binary with `include_bytes!`. + +This proposal replaces that model with the following target architecture: + +- **Production** loads application config from a platform config store at runtime +- **Development** uses a local TOML-authored workflow rooted at the repo root by default +- The platform config store uses a **fixed key**: `ts-config` +- The stored payload is **canonical TOML** representing the application config +- The config schema remains substantially the same as the existing + `trusted-server.toml` schema +- Build-time embedding and build-time environment-variable merging for + application settings are removed + +The goal is to make `trusted-server.toml` remain the single application config +source of truth while changing **how it is deployed and loaded**. + +This document is intentionally limited to config-store behavior and does not +specify attestation signatures, discovery endpoints, or a full CLI design. + +## Scope + +### In scope + +- Production runtime loading of application config from a platform config store +- Development loading of application config from a local TOML file +- Bootstrap contract for locating the platform config store in production +- Deterministic validation and canonicalization of application config +- Config hashing based on canonical TOML bytes +- Repository file ownership changes related to `trusted-server.toml` +- Minimal tooling responsibilities required to deploy config to the store +- Explicit removal of the current build-time merge-and-embed model + +### Out of scope + +- Config signatures, DSSE, or signature verification +- Runtime attestation endpoints or statement formats +- Full CLI command design or operator UX +- Hot reload, file watching, or runtime config refresh +- Runtime mutation of application config +- Broad config schema redesign +- Migration rollout sequencing or temporary dual-source compatibility +- Platform-specific implementation details beyond the abstract store contract + +## Current state + +Trusted Server currently treats application config as a **build input**. + +The current flow is: + +1. Read root `trusted-server.toml` +2. Merge `TRUSTED_SERVER__*` environment variable overrides at build time +3. Serialize the merged config to `target/trusted-server-out.toml` +4. Embed that generated file into the WASM binary with `include_bytes!` +5. Parse the embedded TOML at runtime into `Settings` + +In the current codebase, this behavior is centered in: + +- `crates/trusted-server-core/build.rs` +- `crates/trusted-server-core/src/settings_data.rs` + +This means the deployed binary contains both: + +- application code +- publisher/operator configuration + +## Problems with the current state + +### Code and config are tightly coupled + +A config-only change produces a different binary artifact. That makes it harder +to reason about code provenance separately from config changes. + +### Config changes require a rebuild + +Operators cannot update configuration independently from the WASM build. +Changing runtime behavior requires regenerating and redeploying the binary. + +### Build-time environment merging weakens the source-of-truth model + +The current model combines a TOML file with build-time environment variables. +That makes the effective config less obvious and weakens the idea of a single, +authoritative application config document. + +### Repository ownership is wrong for operator config + +A tracked root `trusted-server.toml` suggests application config should live in +source control as a committed repository artifact. In practice, application +config is operator-owned and deployment-specific. + +### Runtime behavior depends on build tooling decisions + +Because config is preprocessed during the build, the runtime is not the +authoritative point where config loading and validity are determined. + +## Goals + +- Keep **one application config document**: `trusted-server.toml` +- Make **production** load application config from a platform config store +- Make **development** load application config from a local TOML file +- Remove build-time embedding of application settings into the WASM binary +- Remove build-time `TRUSTED_SERVER__*` application-setting merges +- Define a deterministic canonical TOML representation suitable for hashing +- Preserve the existing application config schema as much as possible +- Keep the architecture generic enough to work with a future cross-platform + config store abstraction + +## Non-goals + +- Designing config signing or signature verification +- Designing a runtime attestation document +- Supporting multiple application config sources simultaneously in production +- Supporting hidden fallback from one source to another after source selection +- Supporting runtime writeback of application config +- Preserving comments or operator formatting in stored canonical payloads +- Supporting unknown fields in application config +- Introducing broad schema cleanup unrelated to config-store loading + +## Target architecture + +### High-level model + +`trusted-server.toml` remains the single application configuration document. +What changes is how that document is sourced: + +- **Production:** platform config store +- **Development:** local file at repo root by default + +The authoritative production payload is stored in the platform config store under +key `ts-config`. + +The value stored under `ts-config` is canonical TOML representing the application +config document. + +### Production behavior + +In production, Trusted Server: + +1. Obtains a platform config store reference from deployment/bootstrap wiring +2. Reads the fixed key `ts-config` +3. Parses the TOML payload using the existing application config schema +4. Rejects unknown fields +5. Validates semantic config rules +6. Produces a valid immutable `Settings` snapshot for request handling +7. Derives canonical TOML bytes and a config hash from that valid config + +Production application behavior must treat the platform config store payload as +authoritative. + +### Development behavior + +In development, `trusted-server.toml` remains the default local authoring file +at the repository root. + +A flag may be used to choose a different TOML file path. + +The local TOML file is the development authoring source of truth. Runtime +consumption may happen either directly from that file or via a +platform-specific projection step, depending on platform constraints. + +The development pipeline is: + +1. Load `trusted-server.toml` from the repository root by default, or the + explicitly selected TOML file +2. Parse the TOML payload using the existing application config schema +3. Reject unknown fields according to the rules defined in this document +4. Validate semantic config rules +5. Derive canonical TOML bytes and a config hash from that valid config +6. Project the canonical TOML into the local development runtime in the + platform-appropriate way when needed +7. Produce a valid immutable `Settings` snapshot for request handling + +On platforms such as Fastly/Viceroy, the preferred local-development approach is +to populate the local simulated config store with the canonical payload under +`ts-config` before request handling, rather than relying on direct host-file +reads from within the WASM guest. + +When running in development, tooling or runtime logs should identify which local +TOML file path was loaded. + +Development loading does **not** automatically rewrite the source file into +canonical form. + +### Request-level semantics + +Each request must be handled against **one valid, internally consistent, +immutable `Settings` snapshot**. + +This document intentionally does **not** require a specific fetch or caching +strategy. An implementation may fetch fresh config for each request or reuse +previously loaded state, as long as: + +- each request sees one coherent snapshot +- invalid or partially loaded config is never used +- correctness does not depend on cross-request in-memory persistence + +Because platform lifecycle behavior varies, the architecture must not assume that +in-memory state survives across requests. + +## Bootstrap contract + +Production requires a minimal bootstrap mechanism to locate the platform config +store. That bootstrap mechanism is **not** part of the application config schema. + +Bootstrap is deployment plumbing only. + +### Bootstrap responsibilities + +Production bootstrap must provide: + +- the platform config store reference needed to open/read the store + +Production bootstrap must **not**: + +- inject application settings values +- override application settings values +- provide an alternate key name for the application config payload +- create a secondary source of truth for runtime application behavior + +### Fixed key name + +The application config payload key is fixed globally: + +- `ts-config` + +The runtime always reads `ts-config` once the production store reference has been +resolved. + +The key name is not configurable. + +## Config loading and canonicalization pipeline + +This section defines the semantic pipeline for both production and development. + +### Step 1: Select the source by environment + +- **Production** uses the platform config store +- **Development** uses a local TOML-authored workflow rooted at + `trusted-server.toml` in the repository root by default, with a flag to + choose a different file + +This document does not define a separate first-class runtime `config_source` +mode. It defines source behavior in terms of production vs. development. + +### Step 2: Load the payload + +The selected source must yield a UTF-8 TOML payload. + +- Production loads the payload from store key `ts-config` +- Development loads the payload from the selected local TOML file path and may + then project the canonicalized result into a local platform-simulated config + store before runtime consumption + +### Step 3: Parse strictly using the existing schema + +The payload is parsed as the existing application configuration schema used to +produce `Settings`. + +This spec is schema-preserving by default. It does not redesign the +`trusted-server.toml` structure beyond what is necessary to support the new +loading model. + +### Step 4: Reject unknown fields + +Unknown fields are rejected. + +The system must not silently preserve, ignore, or drop unsupported keys. + +This applies to strongly typed configuration sections in the application config +schema. The `integrations` section continues to follow the existing integration +configuration model, where integration IDs are discovered dynamically and each +integration's typed validation rules govern the contents of its own config. + +### Step 5: Validate semantic constraints + +After parsing, the config must satisfy existing semantic validation rules. + +Examples include: + +- required fields must be present +- invalid field combinations must fail +- invalid regexes, store identifiers, or route coverage rules must fail + +### Step 6: Define the canonical TOML representation + +Valid config has a deterministic canonical TOML representation. + +Canonicalization is part of the config pipeline semantics, not merely a tooling +implementation detail. + +Canonicalization is defined as a dedicated transformation over valid config. It +must not rely on whatever output happens to fall out of naive derived +serialization alone. Implementations may need explicit canonicalization logic to +ensure the required output properties. + +Canonicalization is defined as: + +- parse valid config through the typed config model +- serialize it in a deterministic TOML form +- include explicitly declared settings only +- do **not** expand the config into a full dump of all effective defaulted runtime values +- define stable ordering for map-like structures so identical semantic config + produces identical canonical bytes + +As a consequence: + +- comments are not preserved in canonical form +- original formatting is not preserved in canonical form +- canonical stored payloads are intended to be tight and deterministic +- additional implementation work may be required beyond current derived + `Serialize` behavior to satisfy these guarantees + +### Step 7: Compute the config hash + +The config hash is computed over the canonical TOML bytes. + +This provides a stable hashable representation of application config suitable +for observability and future attestation work. + +### Step 8: Produce the runtime snapshot + +The runtime uses the validated config to produce the `Settings` snapshot used by +request handling. + +Implementations may materialize canonical bytes eagerly or lazily, but the +canonical form is part of the defined semantics. + +## Failure behavior + +The selected config source must produce one valid `Settings` snapshot. + +If loading, parsing, or validation fails, the runtime must **fail closed**. + +That includes failures such as: + +- missing store/bootstrap reference in production +- inability to read the selected store or file +- missing `ts-config` key in production +- invalid UTF-8 payloads +- malformed TOML +- unknown fields +- missing required fields +- semantic validation failures + +### No fallback after source selection + +Once the source has been selected by environment, the runtime must not fall back +to another source. + +Examples of disallowed behavior: + +- production store mode falling back to a local file +- development file mode falling back to embedded config +- falling back to a previously cached last-known-good config +- loading partial config and continuing with defaults beyond normal schema behavior + +### Availability behavior + +A config failure means the service is not healthy for serving application +traffic. + +This spec does not define a special "healthy but unusable" mode for config +failure. + +## Repository and file ownership changes + +The repository layout should change to reflect the new ownership model. + +### Required changes + +- Remove tracked `trusted-server.toml` from source control +- Add `trusted-server.toml` to `.gitignore` +- Add or retain `trusted-server.example.toml` as a tracked template file + +### File roles + +#### `trusted-server.toml` + +- operator-owned local/deployment artifact +- default local authoring file for development +- not tracked in git + +#### `trusted-server.example.toml` + +- tracked template file +- kept in sync with currently supported configuration features +- intended to help operators create a real `trusted-server.toml` + +## Minimal tooling contract + +This document does not define a full CLI specification. + +It does define the minimum tooling responsibilities required by the target +architecture. + +Tooling responsible for publishing production config must be able to: + +1. Load a local TOML file +2. Parse it using the application config schema +3. Reject unknown fields +4. Validate semantic config rules +5. Produce canonical TOML +6. Compute a hash over canonical TOML bytes +7. Write the canonical TOML payload to the platform config store under `ts-config` + +For local platform simulators such as `fastly compute serve`, tooling may also +materialize that canonical payload into the simulator's local config-store input +before starting the runtime. + +Tooling may support additional commands later, such as: + +- pull +- diff +- inspect +- dry-run deployment + +Those capabilities are explicitly out of scope for this document. + +## Hashing + +Config hashing is part of this architecture because it depends on deterministic +canonicalization. + +### Hash source + +The config hash is computed over the canonical TOML bytes. + +Because the hash is derived from canonical bytes, canonicalization must produce +stable field and map ordering for semantically identical config. + +### Purpose + +The config hash exists to support: + +- observability +- config comparison +- deterministic deployment artifacts +- future attestation and provenance work + +### Out of scope + +This document does not define: + +- signature formats +- signed envelopes +- signature verification behavior +- runtime signature enforcement + +## Explicit removals from the current design + +This proposal explicitly removes the current application-config build pipeline. + +### Removed behaviors + +- build-time embedding of application config into the WASM binary +- build-time generation of `target/trusted-server-out.toml` as the runtime app-config source +- build-time merging of `TRUSTED_SERVER__*` application-setting overrides +- production dependence on a repository-tracked `trusted-server.toml` +- runtime mutation of application config + +### Resulting source-of-truth model + +After this change: + +- `trusted-server.toml` remains the canonical application config document +- in production, the authoritative deployed copy is the platform-store payload under `ts-config` +- in development, the authoritative copy is the selected local TOML file + +## Implementation notes + +These notes are informative, not additional scope. + +- The concrete platform config-store API may still be evolving while this is + implemented +- Existing generic config-store abstractions in the codebase may be reused as + they mature +- Runtime caching strategy is intentionally unspecified by this document +- The settings schema should remain substantially unchanged except where minor + adjustments are necessary to support strict parsing or canonicalization + +## Future work enabled by this design + +This design is intended to enable, but not itself specify: + +- config attestation based on canonical payload hashes +- runtime reporting of config hash metadata +- richer deployment tooling around validation, diffing, and inspection +- broader cross-platform config-store support behind a generic API + +## Recommended next step + +After agreeing on this architecture, a follow-up spec should define the concrete +operator tooling used to: + +- validate local config +- canonicalize it +- compute hashes +- publish canonical TOML to the platform config store +- support development ergonomics around local file selection +- project local authored config into platform-specific local runtime inputs when + direct runtime file access is not available diff --git a/scripts/fastly-dev.sh b/scripts/fastly-dev.sh new file mode 100755 index 00000000..2282d354 --- /dev/null +++ b/scripts/fastly-dev.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "$0")/.." && pwd)" +config_path="${TRUSTED_SERVER_CONFIG_FILE:-$repo_root/trusted-server.toml}" +output_path="$repo_root/fastly.local.toml" +wasm_release_path="$repo_root/target/wasm32-wasip1/release/trusted-server-adapter-fastly.wasm" +wasm_debug_path="$repo_root/target/wasm32-wasip1/debug/trusted-server-adapter-fastly.wasm" + +if [[ $# -gt 0 && "$1" != -* ]]; then + config_path="$1" + shift +fi + +if [[ ! -f "$config_path" ]]; then + echo "error: config file not found: $config_path" >&2 + echo "hint: cp trusted-server.example.toml trusted-server.toml" >&2 + exit 1 +fi + +config_path="$(cd "$(dirname "$config_path")" && pwd)/$(basename "$config_path")" + +python3 "$repo_root/scripts/render-fastly-local-config.py" \ + --app-config "$config_path" \ + --template "$repo_root/fastly.toml" \ + --output "$output_path" + +fastly_args=(compute serve --dir "$repo_root" --env=local) +fastly_args+=("$@") + +has_skip_build=false +has_file=false +for arg in "$@"; do + if [[ "$arg" == "--skip-build" ]]; then + has_skip_build=true + fi + if [[ "$arg" == --file=* || "$arg" == "--file" ]]; then + has_file=true + fi +done + +if [[ "$has_skip_build" == true && "$has_file" == false ]]; then + if [[ -f "$wasm_release_path" ]]; then + fastly_args+=(--file "$wasm_release_path") + elif [[ -f "$wasm_debug_path" ]]; then + fastly_args+=(--file "$wasm_debug_path") + else + echo "error: --skip-build was passed but no built Wasm binary was found" >&2 + echo "hint: run cargo build --package trusted-server-adapter-fastly --release --target wasm32-wasip1" >&2 + exit 1 + fi +fi + +exec fastly "${fastly_args[@]}" diff --git a/scripts/render-fastly-local-config.py b/scripts/render-fastly-local-config.py new file mode 100755 index 00000000..92651ab5 --- /dev/null +++ b/scripts/render-fastly-local-config.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +import argparse +import json +import pathlib +import subprocess +import sys + +CONFIG_STORE_NAME = "ts_config_store" +CONFIG_KEY = "ts-config" + + +def host_target() -> str: + result = subprocess.run( + ["rustc", "-vV"], + check=True, + capture_output=True, + text=True, + ) + for line in result.stdout.splitlines(): + if line.startswith("host: "): + return line.removeprefix("host: ").strip() + raise RuntimeError("failed to determine rust host target") + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Render a Fastly/Viceroy local config with runtime app config projected into a config store." + ) + parser.add_argument("--app-config", required=True, help="Path to trusted-server TOML") + parser.add_argument("--template", required=True, help="Path to fastly/viceroy template TOML") + parser.add_argument("--output", required=True, help="Path to write rendered TOML") + args = parser.parse_args() + + repo_root = pathlib.Path(__file__).resolve().parent.parent + app_config = pathlib.Path(args.app_config).resolve(strict=False) + template = pathlib.Path(args.template).resolve(strict=False) + output = pathlib.Path(args.output).resolve(strict=False) + + try: + result = subprocess.run( + [ + "cargo", + "run", + "--quiet", + "--target", + host_target(), + "--package", + "trusted-server-core", + "--bin", + "ts-config-canonicalize", + "--", + str(app_config), + ], + cwd=repo_root, + check=True, + capture_output=True, + text=True, + ) + except subprocess.CalledProcessError as error: + if error.stderr: + print(error.stderr.strip(), file=sys.stderr) + if error.stdout: + print(error.stdout.strip(), file=sys.stderr) + return error.returncode + + canonical_toml = result.stdout + if result.stderr: + print(result.stderr.strip(), file=sys.stderr) + + rendered = template.read_text(encoding="utf-8") + rendered += "\n" + rendered += f"[local_server.config_stores.{CONFIG_STORE_NAME}]\n" + rendered += ' format = "inline-toml"\n' + rendered += f"[local_server.config_stores.{CONFIG_STORE_NAME}.contents]\n" + rendered += f" {CONFIG_KEY} = {json.dumps(canonical_toml)}\n" + + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(rendered, encoding="utf-8") + print(output) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/trusted-server.toml b/trusted-server.example.toml similarity index 97% rename from trusted-server.toml rename to trusted-server.example.toml index f57e9146..ad565c6e 100644 --- a/trusted-server.toml +++ b/trusted-server.example.toml @@ -19,11 +19,9 @@ secret_key = "trusted-server" # Custom headers to be included in every response # Allows publishers to include tags such as X-Robots-Tag: noindex +# Configure these directly in the runtime TOML payload stored under `ts-config`. # [response_headers] # X-Custom-Header = "custom header value" -# -# Or via environment variable (JSON preserves header name casing and hyphens): -# TRUSTED_SERVER__RESPONSE_HEADERS='{"X-Robots-Tag": "noindex", "X-Custom-Header": "custom value"}' # Request Signing Configuration # Enable signing of OpenRTB requests and other API calls