Improve file content parsing to correctly extract multiple files when they are wrapped within outer markdown code blocks

Author: robertpiosik

packages/vscode/src/commands/apply-chat-response-command/utils/clipboard-parser/clipboard-parser.spec.ts

@@ -300,6 +300,44 @@ describe('clipboard-parser', () => {
       )
     })
 
+    it('parses multiple files when each is wrapped in its own outer markdown code block', () => {
+      const test_case = 'markdown-wrappers'
+      const text = load_test_case_file(test_case, `${test_case}.txt`)
+      const result = parse_multiple_files({
+        response: text,
+        is_single_root_folder_workspace: true
+      })
+
+      expect(result).toHaveLength(2)
+      expect(result[0].file_path).toBe('src/lorem.ts')
+      expect(result[0].content).toBe(
+        load_test_case_file(test_case, 'file-1.txt')
+      )
+      expect(result[1].file_path).toBe('src/ipsum.ts')
+      expect(result[1].content).toBe(
+        load_test_case_file(test_case, 'file-2.txt')
+      )
+    })
+
+    it('parses multiple files when wrapped in a single outer markdown code block', () => {
+      const test_case = 'unclosed-markdown-wrapper'
+      const text = load_test_case_file(test_case, `${test_case}.txt`)
+      const result = parse_multiple_files({
+        response: text,
+        is_single_root_folder_workspace: true
+      })
+
+      expect(result).toHaveLength(2)
+      expect(result[0].file_path).toBe('src/lorem.ts')
+      expect(result[0].content).toBe(
+        load_test_case_file(test_case, 'file-1.txt')
+      )
+      expect(result[1].file_path).toBe('src/ipsum.ts')
+      expect(result[1].content).toBe(
+        load_test_case_file(test_case, 'file-2.txt')
+      )
+    })
+
     it('parses PHP files when the opening tag appears before the file path comment', () => {
       const test_case = 'php-opening-tag'
       const text = load_test_case_file(test_case, `${test_case}.txt`)

packages/vscode/src/commands/apply-chat-response-command/utils/clipboard-parser/clipboard-parser.ts

@@ -347,13 +347,12 @@ export const parse_multiple_files = (params: {
           ((current_language == 'markdown' || current_language == 'md') &&
             i > 0 &&
             (lines[i - 1].trim() == '' ||
-              lines[i - 1].trim().endsWith('```')))) &&
+              (current_file_name.endsWith('.md') &&
+                lines[i - 1].trim().endsWith('```'))))) &&
         (() => {
           for (let j = i + 1; j < lines.length; j++) {
             const next_line = lines[j].trim()
-            if (next_line !== '') {
-              return !next_line.startsWith('```')
-            }
+            if (next_line) return true
           }
           return false
         })()
@@ -363,6 +362,12 @@ export const parse_multiple_files = (params: {
         backtick_nesting_level++
       } else if (trimmed_line.endsWith('```')) {
         backtick_nesting_level--
+        if (
+          backtick_nesting_level === 1 &&
+          (current_language === 'markdown' || current_language === 'md')
+        ) {
+          is_first_content_line = true
+        }
       }
 
       if (backtick_nesting_level <= 0) {
@@ -500,6 +505,30 @@ export const parse_multiple_files = (params: {
           }
 
           if (extracted_filename) {
+            if (
+              (current_language === 'markdown' || current_language === 'md') &&
+              current_file_name
+            ) {
+              // This is a new file inside a markdown block. Save the previous one.
+              const cleaned_content = current_content.trim()
+              if (has_real_code(cleaned_content)) {
+                const file_key = `${
+                  current_workspace_name || ''
+                }:${current_file_name}`
+                if (files_map.has(file_key)) {
+                  const existing_file = files_map.get(file_key)!
+                  existing_file.content += '\n\n' + cleaned_content
+                } else {
+                  files_map.set(file_key, {
+                    file_path: current_file_name,
+                    content: cleaned_content,
+                    workspace_name: current_workspace_name
+                  })
+                }
+              }
+              current_content = ''
+            }
+
             const { workspace_name, relative_path } =
               extract_workspace_and_path(
                 extracted_filename,

packages/vscode/src/commands/apply-chat-response-command/utils/clipboard-parser/test-cases/markdown-wrappers/file-1.txt

@@ -0,0 +1 @@
+console.log('lorem');

packages/vscode/src/commands/apply-chat-response-command/utils/clipboard-parser/test-cases/markdown-wrappers/file-2.txt

@@ -0,0 +1 @@
+console.log('ipsum');

packages/vscode/src/commands/apply-chat-response-command/utils/clipboard-parser/test-cases/markdown-wrappers/markdown-wrappers.txt

@@ -0,0 +1,15 @@
+```markdown
+```ts
+// src/lorem.ts
+console.log('lorem');
+```
+```
+
+Lorem ipsum.
+
+```markdown
+```ts
+// src/ipsum.ts
+console.log('ipsum');
+```
+```

packages/vscode/src/commands/apply-chat-response-command/utils/clipboard-parser/test-cases/unclosed-markdown-wrapper/file-1.txt

@@ -0,0 +1 @@
+console.log('lorem');

packages/vscode/src/commands/apply-chat-response-command/utils/clipboard-parser/test-cases/unclosed-markdown-wrapper/file-2.txt

@@ -0,0 +1 @@
+console.log('ipsum');

packages/vscode/src/commands/apply-chat-response-command/utils/clipboard-parser/test-cases/unclosed-markdown-wrapper/unclosed-markdown-wrapper.txt

@@ -0,0 +1,10 @@
+```markdown
+```ts
+// src/lorem.ts
+console.log('lorem');
+```
+
+```ts
+// src/ipsum.ts
+console.log('ipsum');
+```