feat: add assistant response audio playback

Author: Wolfgang

src-tauri/src/bin/codex_monitor_daemon.rs

@@ -1329,6 +1329,25 @@ impl DaemonState {
         .await
     }
 
+    async fn generate_message_audio_summary(
+        &self,
+        workspace_id: String,
+        response_text: String,
+        model_id: Option<String>,
+    ) -> Result<String, String> {
+        codex_aux_core::generate_message_audio_summary_core(
+            &self.sessions,
+            &self.workspaces,
+            workspace_id,
+            &response_text,
+            model_id.as_deref(),
+            |workspace_id, thread_id| {
+                emit_background_thread_hide(&self.event_sink, workspace_id, thread_id);
+            },
+        )
+        .await
+    }
+
     async fn local_usage_snapshot(
         &self,
         days: Option<u32>,

src-tauri/src/bin/codex_monitor_daemon/rpc/codex.rs

@@ -505,6 +505,23 @@ pub(super) async fn try_handle(
                     .and_then(|value| serde_json::to_value(value).map_err(|err| err.to_string())),
             )
         }
+        "generate_message_audio_summary" => {
+            let workspace_id = match parse_string(params, "workspaceId") {
+                Ok(value) => value,
+                Err(err) => return Some(Err(err)),
+            };
+            let response_text = match parse_string(params, "responseText") {
+                Ok(value) => value,
+                Err(err) => return Some(Err(err)),
+            };
+            let model_id = parse_optional_string(params, "modelId");
+            Some(
+                state
+                    .generate_message_audio_summary(workspace_id, response_text, model_id)
+                    .await
+                    .and_then(|value| serde_json::to_value(value).map_err(|err| err.to_string())),
+            )
+        }
         _ => None,
     }
 }

src-tauri/src/codex/mod.rs

@@ -1022,3 +1022,51 @@ pub(crate) async fn generate_agent_description(
     )
     .await
 }
+
+#[tauri::command]
+pub(crate) async fn generate_message_audio_summary(
+    workspace_id: String,
+    response_text: String,
+    model_id: Option<String>,
+    state: State<'_, AppState>,
+    app: AppHandle,
+) -> Result<String, String> {
+    if remote_backend::is_remote_mode(&*state).await {
+        let value = remote_backend::call_remote(
+            &*state,
+            app,
+            "generate_message_audio_summary",
+            json!({
+                "workspaceId": workspace_id,
+                "responseText": response_text,
+                "modelId": model_id,
+            }),
+        )
+        .await?;
+        return serde_json::from_value(value).map_err(|err| err.to_string());
+    }
+
+    crate::shared::codex_aux_core::generate_message_audio_summary_core(
+        &state.sessions,
+        &state.workspaces,
+        workspace_id,
+        &response_text,
+        model_id.as_deref(),
+        |workspace_id, thread_id| {
+            let _ = app.emit(
+                "app-server-event",
+                AppServerEvent {
+                    workspace_id: workspace_id.to_string(),
+                    message: json!({
+                        "method": "codex/backgroundThread",
+                        "params": {
+                            "threadId": thread_id,
+                            "action": "hide"
+                        }
+                    }),
+                },
+            );
+        },
+    )
+    .await
+}

src-tauri/src/lib.rs

@@ -218,6 +218,7 @@ pub fn run() {
             codex::generate_commit_message,
             codex::generate_run_metadata,
             codex::generate_agent_description,
+            codex::generate_message_audio_summary,
             codex::resume_thread,
             codex::read_thread,
             codex::thread_live_subscribe,

src-tauri/src/shared/codex_aux_core.rs

@@ -20,6 +20,17 @@ Keep the summary line under 72 characters. \
 Only output the commit message, nothing else.\n\n\
 Changes:\n{diff}";
 
+const DEFAULT_MESSAGE_AUDIO_SUMMARY_PROMPT: &str =
+    "You are preparing audio playback for a coding assistant response.\n\
+Summarize the response below into short spoken prose for a developer.\n\n\
+Requirements:\n\
+- Return plain text only.\n\
+- Do not include markdown fences, bullets, or headings.\n\
+- Keep it concise and easy to listen to.\n\
+- Preserve important commands, file paths, errors, results, and next actions when they matter.\n\
+- If the response is mostly code or a table, summarize the outcome instead of reading every line.\n\n\
+Assistant response:\n{response}";
+
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
 #[serde(rename_all = "camelCase")]
 pub(crate) struct GeneratedAgentConfiguration {
@@ -50,6 +61,15 @@ pub(crate) fn build_commit_message_prompt_for_diff(
     Ok(build_commit_message_prompt(diff, template))
 }
 
+pub(crate) fn build_message_audio_summary_prompt(response_text: &str) -> Result<String, String> {
+    let cleaned_response = response_text.trim();
+    if cleaned_response.is_empty() {
+        return Err("Response text is required.".to_string());
+    }
+
+    Ok(DEFAULT_MESSAGE_AUDIO_SUMMARY_PROMPT.replace("{response}", cleaned_response))
+}
+
 pub(crate) fn build_run_metadata_prompt(cleaned_prompt: &str) -> String {
     format!(
         "You create concise run metadata for a coding task.\n\
@@ -198,6 +218,28 @@ pub(crate) fn parse_agent_description_value(
     Err("No valid agent configuration was generated".to_string())
 }
 
+pub(crate) fn normalize_message_audio_summary_value(raw: &str) -> Result<String, String> {
+    let normalized = raw
+        .lines()
+        .map(str::trim)
+        .filter(|line| !line.is_empty() && !line.starts_with("```"))
+        .map(|line| {
+            line.strip_prefix("- ")
+                .or_else(|| line.strip_prefix("* "))
+                .or_else(|| line.strip_prefix("• "))
+                .unwrap_or(line)
+        })
+        .collect::<Vec<_>>()
+        .join(" ");
+
+    let normalized = normalized.split_whitespace().collect::<Vec<_>>().join(" ");
+    if normalized.is_empty() {
+        return Err("No summary was generated".to_string());
+    }
+
+    Ok(normalized)
+}
+
 pub(crate) fn parse_run_metadata_value(raw: &str) -> Result<Value, String> {
     let trimmed = raw.trim();
     if trimmed.is_empty() {
@@ -649,10 +691,38 @@ where
     parse_agent_description_value(&response)
 }
 
+pub(crate) async fn generate_message_audio_summary_core<F>(
+    sessions: &Mutex<HashMap<String, Arc<WorkspaceSession>>>,
+    workspaces: &Mutex<HashMap<String, WorkspaceEntry>>,
+    workspace_id: String,
+    response_text: &str,
+    model: Option<&str>,
+    on_hide_thread: F,
+) -> Result<String, String>
+where
+    F: Fn(&str, &str),
+{
+    let prompt = build_message_audio_summary_prompt(response_text)?;
+    let response = run_background_prompt_core(
+        sessions,
+        workspaces,
+        workspace_id,
+        prompt,
+        model,
+        on_hide_thread,
+        "Timeout waiting for audio summary generation",
+        "Unknown error during audio summary generation",
+    )
+    .await?;
+
+    normalize_message_audio_summary_value(&response)
+}
+
 #[cfg(test)]
 mod tests {
     use super::{
-        build_commit_message_prompt_for_diff, parse_agent_description_value,
+        build_commit_message_prompt_for_diff, build_message_audio_summary_prompt,
+        normalize_message_audio_summary_value, parse_agent_description_value,
         parse_run_metadata_value,
     };
 
@@ -665,6 +735,22 @@ mod tests {
         );
     }
 
+    #[test]
+    fn build_message_audio_summary_prompt_requires_response_text() {
+        let result = build_message_audio_summary_prompt("   ");
+        assert_eq!(result.expect_err("should fail"), "Response text is required.");
+    }
+
+    #[test]
+    fn normalize_message_audio_summary_value_flattens_bullets_and_fences() {
+        let result = normalize_message_audio_summary_value(
+            "```md\n- Updated src/App.tsx\n- Ran npm run test\n```",
+        )
+        .expect("summary should parse");
+
+        assert_eq!(result, "Updated src/App.tsx Ran npm run test");
+    }
+
     #[test]
     fn parse_run_metadata_value_normalizes_worktree_name_alias() {
         let raw =

src/features/app/hooks/useMainAppLayoutSurfaces.ts

@@ -443,6 +443,7 @@ function buildPrimarySurface({
       workspacePath: activeWorkspace?.path ?? null,
       openTargets: appSettings.openAppTargets,
       selectedOpenAppId: appSettings.selectedOpenAppId,
+      selectedModelId,
       codeBlockCopyUseModifier: appSettings.composerCodeBlockCopyUseModifier,
       showMessageFilePath: appSettings.showMessageFilePath,
       userInputRequests,