fix: parsed json with extra tokens issue (#656)

Fixes #598 

* fix: parsed json with extra tokens issue

* Added unit test for json.ts

* fix the json unit test running issue

* Apply suggestions from code review

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update the code with code review suggestion

---------

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: Willem Jiang <143703838+willem-bd@users.noreply.github.com>

web/src/core/utils/json.ts

@@ -1,11 +1,72 @@
 import { parse } from "best-effort-json-parser";
 
+/**
+ * Extract valid JSON from content that may have extra tokens.
+ * Finds the last closing brace/bracket that could be valid JSON.
+ */
+function extractValidJSON(content: string): string {
+  let braceCount = 0;
+  let bracketCount = 0;
+  let inString = false;
+  let escapeNext = false;
+  let lastValidEnd = -1;
+
+  for (let i = 0; i < content.length; i++) {
+    const char = content[i];
+    
+    if (escapeNext) {
+      escapeNext = false;
+      continue;
+    }
+    
+    if (char === "\\") {
+      escapeNext = true;
+      continue;
+    }
+    
+    if (char === '"') {
+      inString = !inString;
+      continue;
+    }
+    
+    if (inString) {
+      continue;
+    }
+    
+    if (char === "{") {
+      braceCount++;
+    } else if (char === "}") {
+      if (braceCount > 0) {
+        braceCount--;
+        if (braceCount === 0) {
+          lastValidEnd = i;
+        }
+      }
+    } else if (char === "[") {
+      bracketCount++;
+    } else if (char === "]") {
+      if (bracketCount > 0) {
+        bracketCount--;
+        if (bracketCount === 0) {
+          lastValidEnd = i;
+        }
+      }
+    }
+  }
+  
+  if (lastValidEnd > 0) {
+    return content.substring(0, lastValidEnd + 1);
+  }
+  
+  return content;
+}
+
 export function parseJSON<T>(json: string | null | undefined, fallback: T) {
   if (!json) {
     return fallback;
   }
   try {
-    const raw = json
+    let raw = json
       .trim()
       .replace(/^```json\s*/, "")
       .replace(/^```js\s*/, "")
@@ -13,8 +74,17 @@ export function parseJSON<T>(json: string | null | undefined, fallback: T) {
       .replace(/^```plaintext\s*/, "")
       .replace(/^```\s*/, "")
       .replace(/\s*```$/, "");
+    
+    // First attempt: try to extract valid JSON to remove extra tokens
+    if (raw.startsWith("{") || raw.startsWith("[")) {
+      raw = extractValidJSON(raw);
+    }
+    
+    // Parse the cleaned content
     return parse(raw) as T;
   } catch {
+    // Fallback: try to extract meaningful content from malformed JSON
+    // This is a last-resort attempt to salvage partial data
     return fallback;
   }
 }