refactor(sanitizer): harden JSON extraction with thought block and preamble handling

Sanitizer now strips both <think> (qwen3 native) and <thought> blocks
before JSON parsing, uses targeted "type" key search for reliable JSON
extraction from noisy LLM output, and detects conversational preambles
via VALID_TYPE_START_REGEX. Adds 9 regression tests covering thought
blocks, unclosed tags, preambles, and noisy JSON with braces.

Author: Sephyi

src/services/sanitizer.rs

@@ -27,6 +27,17 @@ static SCOPE_REGEX: LazyLock<Regex> =
 
 static CODE_FENCE_REGEX: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"```[\s\S]*?```").unwrap());
 
+static THOUGHT_BLOCK_REGEX: LazyLock<Regex> =
+    LazyLock::new(|| Regex::new(r"(?s)<(?:thought|think)>[\s\S]*?</(?:thought|think)>").unwrap());
+
+static UNCLOSED_THOUGHT_REGEX: LazyLock<Regex> =
+    LazyLock::new(|| Regex::new(r"(?i)^\s*<(?:thought|think)>\s*").unwrap());
+
+static VALID_TYPE_START_REGEX: LazyLock<Regex> = LazyLock::new(|| {
+    let types = CommitType::ALL.join("|");
+    Regex::new(&format!(r"(?m)(?:^|\s)({})(?:\(|!|:)", types)).unwrap()
+});
+
 static PREAMBLE_PATTERNS: &[&str] = &[
     "here's the commit message",
     "here is the commit message",
@@ -102,7 +113,7 @@ impl CommitSanitizer {
     ///   `  <continuation lines, indented two spaces>`
     ///
     /// `str::len()` is `const fn` since Rust 1.39 — `FIRST_LINE_BUDGET` is a
-    /// valid compile-time constant on MSRV 1.85.
+    /// valid compile-time constant on MSRV 1.94.
     fn format_breaking_footer(desc: &str) -> String {
         const PREFIX: &str = "BREAKING CHANGE: ";
         const FIRST_LINE_BUDGET: usize = 72 - PREFIX.len(); // 55
@@ -136,29 +147,34 @@ impl CommitSanitizer {
     }
 
     fn try_parse_json(raw: &str) -> std::result::Result<StructuredCommit, ()> {
-        let trimmed = raw.trim();
-
-        // Direct JSON
-        if trimmed.starts_with('{') {
-            return serde_json::from_str(trimmed).map_err(|_| ());
-        }
-
-        // JSON in code fence
-        if let Some(start) = trimmed.find("```json") {
-            let after_fence = &trimmed[start + 7..];
-            if let Some(end) = after_fence.find("```") {
-                let json = after_fence[..end].trim();
-                return serde_json::from_str(json).map_err(|_| ());
+        // Strip thought blocks first to avoid picking up braces inside thoughts
+        let stripped = THOUGHT_BLOCK_REGEX.replace_all(raw, "");
+        let stripped = UNCLOSED_THOUGHT_REGEX.replace(&stripped, "");
+        let trimmed = stripped.trim();
+
+        // 1. Look for the start of our specific JSON structure
+        // We look for "type" key because that's mandatory and highly specific.
+        if let Some(type_key_pos) = trimmed.find("type") {
+            // Find the '{' that starts this object (search backwards from "type")
+            if let Some(start_brace) = trimmed[..type_key_pos].rfind('{') {
+                let json_candidate = &trimmed[start_brace..];
+                // Find the last '}' to handle trailing text
+                if let Some(end_brace) = json_candidate.rfind('}') {
+                    let json = &json_candidate[..=end_brace];
+                    if let Ok(structured) = serde_json::from_str::<StructuredCommit>(json) {
+                        return Ok(structured);
+                    }
+                }
             }
         }
 
-        // Plain code fence
-        if let Some(start) = trimmed.find("```") {
-            let after_fence = &trimmed[start + 3..];
-            if let Some(end) = after_fence.find("```") {
-                let content = after_fence[..end].trim();
-                if content.starts_with('{') {
-                    return serde_json::from_str(content).map_err(|_| ());
+        // 2. Fallback to any brace if specific start not found
+        if let Some(start_brace) = trimmed.find('{') {
+            let json_candidate = &trimmed[start_brace..];
+            if let Some(end_brace) = json_candidate.rfind('}') {
+                let json = &json_candidate[..=end_brace];
+                if let Ok(structured) = serde_json::from_str::<StructuredCommit>(json) {
+                    return Ok(structured);
                 }
             }
         }
@@ -266,9 +282,18 @@ impl CommitSanitizer {
     fn clean_text(raw: &str, format: &CommitFormat) -> String {
         let mut cleaned = raw.to_string();
 
+        // Remove thought blocks
+        cleaned = THOUGHT_BLOCK_REGEX.replace_all(&cleaned, "").to_string();
+        cleaned = UNCLOSED_THOUGHT_REGEX.replace(&cleaned, "").to_string();
+
         // Remove code fences
         cleaned = CODE_FENCE_REGEX.replace_all(&cleaned, "").to_string();
 
+        // Find the actual start of the conventional commit (skip preambles/thoughts)
+        if let Some(mat) = VALID_TYPE_START_REGEX.find(&cleaned) {
+            cleaned = cleaned[mat.start()..].to_string();
+        }
+
         // Remove quotes at start/end
         cleaned = cleaned.trim().to_string();
         if cleaned.starts_with('"') && cleaned.ends_with('"') && cleaned.len() >= 2 {

tests/sanitizer.rs

@@ -603,3 +603,141 @@ fn validator_corrections_format() {
     assert!(corrections.contains("Type is wrong."));
     assert!(corrections.contains("Breaking change missing."));
 }
+
+#[test]
+fn sanitize_with_preceding_thought_block() {
+    let raw = r#"<thought>
+The core change is the addition of the CommitValidator struct to enforce subject specificity and evidence-based rules.
+</thought>
+{
+  "type": "feat",
+  "scope": "sanitizer",
+  "subject": "add CommitValidator for evidence-based validation",
+  "body": "Implements deterministic validation rules against code analysis signals.",
+  "breaking_change": null
+}"#;
+    let format = CommitFormat::default();
+    let result = CommitSanitizer::sanitize(raw, &format).unwrap();
+    assert!(
+        result.starts_with("feat(sanitizer): add CommitValidator for evidence-based validation")
+    );
+}
+
+#[test]
+fn sanitize_plain_text_with_thought_block() {
+    let raw = r#"<thought>
+The core change is renaming the function.
+</thought>
+refactor: rename process to process_all"#;
+    let format = CommitFormat::default();
+    let result = CommitSanitizer::sanitize(raw, &format).unwrap();
+    assert_eq!(result, "refactor: rename process to process_all");
+}
+
+#[test]
+fn sanitize_with_thought_block_containing_braces() {
+    let raw = r#"<thought>
+I should generate a JSON like this: { "foo": "bar" }
+</thought>
+{
+  "type": "refactor",
+  "scope": "splitter",
+  "subject": "upgrade clustering to hybrid Jaccard similarity",
+  "body": null,
+  "breaking_change": null
+}"#;
+    let format = CommitFormat::default();
+    let result = CommitSanitizer::sanitize(raw, &format).unwrap();
+    assert_eq!(
+        result,
+        "refactor(splitter): upgrade clustering to hybrid Jaccard similarity"
+    );
+}
+
+#[test]
+fn sanitize_with_unclosed_thought_block() {
+    // LLM might forget to close tag but still output the message
+    let raw = r#"<thought>
+I will refactor the splitter to use Jaccard similarity.
+
+refactor(splitter): upgrade clustering to hybrid Jaccard similarity"#;
+    let format = CommitFormat::default();
+    let result = CommitSanitizer::sanitize(raw, &format).unwrap();
+    assert_eq!(
+        result,
+        "refactor(splitter): upgrade clustering to hybrid Jaccard similarity"
+    );
+}
+
+#[test]
+fn sanitize_with_noise_containing_braces_before_json() {
+    let raw = r#"<thought>...</thought>
+The diff spans several files and adds a new field { "foo": 1 } to the config.
+{
+  "type": "refactor",
+  "scope": "sanitizer",
+  "subject": "harden JSON extraction",
+  "body": null,
+  "breaking_change": null
+}"#;
+    let format = CommitFormat::default();
+    let result = CommitSanitizer::sanitize(raw, &format).unwrap();
+    assert_eq!(result, "refactor(sanitizer): harden JSON extraction");
+}
+
+#[test]
+fn sanitize_with_noise_before_plain_text() {
+    let raw = r#"The diff spans several files. refactor: improve thing"#;
+    let format = CommitFormat::default();
+    let result = CommitSanitizer::sanitize(raw, &format).unwrap();
+    assert_eq!(result, "refactor: improve thing");
+}
+
+#[test]
+fn sanitize_with_think_block_json() {
+    // qwen3/ollama native thinking uses <think> tags, not <thought>
+    let raw = r#"<think>
+I need to analyze the diff. The main change is adding a new struct.
+</think>
+{
+  "type": "feat",
+  "scope": "core",
+  "subject": "add DiffFingerprint struct for similarity comparison",
+  "body": null,
+  "breaking_change": null
+}"#;
+    let format = CommitFormat::default();
+    let result = CommitSanitizer::sanitize(raw, &format).unwrap();
+    assert_eq!(
+        result,
+        "feat(core): add DiffFingerprint struct for similarity comparison"
+    );
+}
+
+#[test]
+fn sanitize_with_unclosed_think_block() {
+    let raw = r#"<think>
+I will analyze the changes...
+
+feat: add DiffFingerprint struct for similarity comparison"#;
+    let format = CommitFormat::default();
+    let result = CommitSanitizer::sanitize(raw, &format).unwrap();
+    assert_eq!(
+        result,
+        "feat: add DiffFingerprint struct for similarity comparison"
+    );
+}
+
+#[test]
+fn sanitize_conversational_preamble_with_json() {
+    // Claude/Anthropic sometimes outputs conversational text before JSON
+    let raw = r#"Let me analyze the changes in the diff.
+
+{"type": "refactor", "scope": "splitter", "subject": "upgrade clustering to hybrid Jaccard similarity", "body": null, "breaking_change": null}"#;
+    let format = CommitFormat::default();
+    let result = CommitSanitizer::sanitize(raw, &format).unwrap();
+    assert_eq!(
+        result,
+        "refactor(splitter): upgrade clustering to hybrid Jaccard similarity"
+    );
+}