Implement support for pasting images in the chat panel, rendering them as interactive symbols, and processing them for multimodal model requests

Author: robertpiosik

apps/editor/src/commands/apply-context-command/sources/check-all-with-keywords-source.ts

@@ -17,7 +17,7 @@ export const handle_check_all_with_keywords_source = async (
       input_box.title = 'Keyword Search'
       input_box.prompt =
         'Enter keywords separated by comma (use & for AND, quotes for whole words)'
-      input_box.placeholder = 'e.g. user, login&auth, "ignoreFocus"'
+      input_box.placeholder = 'e.g. user, login&auth, "changePassword"'
       input_box.ignoreFocusOut = true
       input_box.buttons = [vscode.QuickInputButtons.Back]
 
@@ -221,6 +221,7 @@ export const handle_check_all_with_keywords_source = async (
             quick_pick_merge.items = quick_pick_options
             quick_pick_merge.placeholder = `How would you like to apply the ${selected_paths.length} selected files?`
             quick_pick_merge.buttons = [vscode.QuickInputButtons.Back]
+            quick_pick_merge.ignoreFocusOut = true
 
             if (last_choice_label) {
               const active_item = quick_pick_options.find(

apps/editor/src/commands/check-all-with-keywords.ts

@@ -20,14 +20,27 @@ export const check_all_with_keywords_command = (
         input_box.title = 'Keyword Search'
         input_box.prompt =
           'Enter keywords separated by comma (use & for AND, quotes for whole words)'
-        input_box.placeholder = 'e.g. user, login&auth, "ignoreFocus"'
+        input_box.placeholder = 'e.g. username, login&auth, "changePassword"'
+        input_box.ignoreFocusOut = true
+
+        const close_button = {
+          iconPath: new vscode.ThemeIcon('close'),
+          tooltip: 'Close'
+        }
+        input_box.buttons = [close_button]
 
         const keywords_input = await new Promise<string | undefined>(
           (resolve) => {
             let is_resolved = false
             const disposables: vscode.Disposable[] = []
 
             disposables.push(
+              input_box.onDidTriggerButton((button) => {
+                if (button === close_button) {
+                  resolve(undefined)
+                  input_box.hide()
+                }
+              }),
               input_box.onDidAccept(() => {
                 const value = input_box.value.trim()
                 if (value.length == 0) {
@@ -215,6 +228,7 @@ export const check_all_with_keywords_command = (
             const quick_pick_apply = vscode.window.createQuickPick()
             quick_pick_apply.items = quick_pick_options
             quick_pick_apply.placeholder = `How would you like to apply the ${selected_paths.length} selected files?`
+            quick_pick_apply.ignoreFocusOut = true
 
             if (last_choice_label) {
               const active_item = quick_pick_options.find(

apps/editor/src/utils/apply-reasoning-effort.ts

@@ -14,14 +14,13 @@ export const apply_reasoning_effort = (
       }
     }
   } else if (provider.name == 'Google') {
-    if (reasoning_effort) {
+    if (reasoning_effort == 'none') {
       body.reasoning_effort = reasoning_effort
-    }
-
-    if (reasoning_effort && reasoning_effort != 'none') {
+    } else {
       body.extra_body = {
         google: {
           thinking_config: {
+            thinking_level: reasoning_effort,
             include_thoughts: true
           }
         }

apps/editor/src/views/panel/backend/message-handlers/handle-code-completion.ts

@@ -26,6 +26,7 @@ import {
 } from '../utils/replace-git-symbols'
 import { replace_saved_context_symbol } from '@/views/panel/backend/utils/replace-saved-context-symbol'
 import { replace_skill_symbol } from '../utils/replace-skill-symbol'
+import { replace_image_symbol } from '../utils/replace-image-symbol'
 
 const get_code_completion_config = async (
   api_providers_manager: ModelProvidersManager,
@@ -404,6 +405,12 @@ export const handle_code_completion = async (
       skill_definitions += result.skill_definitions
     }
 
+    if (processed_completion_instructions.includes('#Image(')) {
+      processed_completion_instructions = await replace_image_symbol({
+        instruction: processed_completion_instructions
+      })
+    }
+
     const files_collector = new FilesCollector(
       panel_provider.workspace_provider,
       panel_provider.open_editors_provider
@@ -424,10 +431,33 @@ export const handle_code_completion = async (
         : '<missing_text>'
     }${payload.after}\n${main_instructions}`
 
+    let user_content: any = content
+
+    if (content.includes('<cwc-image>')) {
+      user_content = []
+      const parts = content.split(/<cwc-image>([\s\S]*?)<\/cwc-image>/)
+
+      for (let i = 0; i < parts.length; i++) {
+        const part = parts[i]
+        if (i % 2 == 0) {
+          if (part.length > 0) {
+            user_content.push({ type: 'text', text: part.trim() })
+          }
+        } else {
+          user_content.push({
+            type: 'image_url',
+            image_url: {
+              url: `data:image/png;base64,${part}`
+            }
+          })
+        }
+      }
+    }
+
     const messages = [
       {
         role: 'user',
-        content
+        content: user_content
       }
     ]
 

apps/editor/src/views/panel/backend/message-handlers/handle-copy-prompt.ts

@@ -9,6 +9,7 @@ import {
 } from '@/views/panel/backend/utils/replace-git-symbols'
 import { replace_saved_context_symbol } from '@/views/panel/backend/utils/replace-saved-context-symbol'
 import { replace_skill_symbol } from '@/views/panel/backend/utils/replace-skill-symbol'
+import { replace_image_symbol } from '@/views/panel/backend/utils/replace-image-symbol'
 import {
   code_at_cursor_instructions_for_panel,
   prune_context_instructions_prefix,
@@ -140,6 +141,13 @@ export const handle_copy_prompt = async (params: {
       skill_definitions += result.skill_definitions
     }
 
+    if (processed_completion_instructions.includes('#Image(')) {
+      processed_completion_instructions = await replace_image_symbol({
+        instruction: processed_completion_instructions,
+        remove: true
+      })
+    }
+
     const missing_text_tag = processed_completion_instructions
       ? `<missing_text>${processed_completion_instructions}</missing_text>`
       : '<missing_text>'
@@ -205,6 +213,13 @@ export const handle_copy_prompt = async (params: {
       skill_definitions += result.skill_definitions
     }
 
+    if (processed_instructions.includes('#Image(')) {
+      processed_instructions = await replace_image_symbol({
+        instruction: processed_instructions,
+        remove: true
+      })
+    }
+
     let system_instructions_xml = ''
 
     if (params.panel_provider.web_prompt_type == 'edit-context') {

apps/editor/src/views/panel/backend/message-handlers/handle-edit-context.ts

@@ -24,6 +24,7 @@ import { replace_saved_context_symbol } from '@/views/panel/backend/utils/replac
 import { replace_selection_symbol } from '@/views/panel/backend/utils/replace-selection-symbol'
 import { PanelProvider } from '@/views/panel/backend/panel-provider'
 import { replace_skill_symbol } from '@/views/panel/backend/utils/replace-skill-symbol'
+import { replace_image_symbol } from '@/views/panel/backend/utils/replace-image-symbol'
 import { apply_reasoning_effort } from '@/utils/apply-reasoning-effort'
 import { EditContextMessage } from '@/views/panel/types/messages'
 import { dictionary } from '@shared/constants/dictionary'
@@ -364,6 +365,12 @@ export const handle_edit_context = async (
     skill_definitions += result.skill_definitions
   }
 
+  if (processed_instructions.includes('#Image(')) {
+    processed_instructions = await replace_image_symbol({
+      instruction: processed_instructions
+    })
+  }
+
   const is_prune_context = panel_provider.api_prompt_type == 'prune-context'
   const collected_files = await files_collector.collect_files({
     compact: is_prune_context
@@ -465,6 +472,29 @@ export const handle_edit_context = async (
         .get<string>('editContextSystemInstructions') ||
       default_system_instructions
 
+    let user_content: any = content
+
+    if (content.includes('<cwc-image>')) {
+      user_content = []
+      const parts = content.split(/<cwc-image>([\s\S]*?)<\/cwc-image>/)
+
+      for (let i = 0; i < parts.length; i++) {
+        const part = parts[i]
+        if (i % 2 == 0) {
+          if (part.length > 0) {
+            user_content.push({ type: 'text', text: part.trim() })
+          }
+        } else {
+          user_content.push({
+            type: 'image_url',
+            image_url: {
+              url: `data:image/png;base64,${part}`
+            }
+          })
+        }
+      }
+    }
+
     const messages = [
       ...(system_instructions
         ? [
@@ -476,7 +506,7 @@ export const handle_edit_context = async (
         : []),
       {
         role: 'user',
-        content
+        content: user_content
       }
     ]
 

apps/editor/src/views/panel/backend/message-handlers/handle-open-temp-image.ts

@@ -0,0 +1,41 @@
+import * as vscode from 'vscode'
+import * as os from 'os'
+import * as path from 'path'
+import * as fs from 'fs'
+import { OpenTempImageMessage } from '../../types/messages'
+
+export const handle_open_temp_image = async (message: OpenTempImageMessage) => {
+  try {
+    const hash = message.hash
+    const txt_filename = `cwc-image-${hash}.txt`
+    const txt_path = path.join(os.tmpdir(), txt_filename)
+
+    // Read base64 content
+    let content_base64: string
+    try {
+      content_base64 = await fs.promises.readFile(txt_path, 'utf-8')
+    } catch (err: any) {
+      if (err.code == 'ENOENT') {
+        vscode.window.showErrorMessage(
+          'Image file not found (it may have been deleted).'
+        )
+        return
+      }
+      throw err
+    }
+
+    const buffer = Buffer.from(content_base64, 'base64')
+
+    // Create a png file for viewing
+    const png_filename = `cwc-image-${hash}.png`
+    const png_path = path.join(os.tmpdir(), png_filename)
+
+    await fs.promises.writeFile(png_path, buffer)
+
+    // Open the image
+    const uri = vscode.Uri.file(png_path)
+    await vscode.commands.executeCommand('vscode.open', uri)
+  } catch (error) {
+    vscode.window.showErrorMessage(`Failed to open image: ${error}`)
+  }
+}

apps/editor/src/views/panel/backend/message-handlers/handle-preview-preset.ts

@@ -11,6 +11,7 @@ import {
 } from '@/views/panel/backend/utils/replace-git-symbols'
 import { Preset } from '@shared/types/preset'
 import { replace_skill_symbol } from '@/views/panel/backend/utils/replace-skill-symbol'
+import { replace_image_symbol } from '@/views/panel/backend/utils/replace-image-symbol'
 import { apply_preset_affixes_to_instruction } from '@/utils/apply-preset-affixes'
 import { dictionary } from '@shared/constants/dictionary'
 import {
@@ -120,6 +121,13 @@ export const handle_preview_preset = async (
       skill_definitions += result.skill_definitions
     }
 
+    if (processed_completion_instructions.includes('#Image(')) {
+      processed_completion_instructions = await replace_image_symbol({
+        instruction: processed_completion_instructions,
+        remove: true
+      })
+    }
+
     const missing_text_tag = processed_completion_instructions
       ? `<missing_text>${processed_completion_instructions}</missing_text>`
       : '<missing_text>'
@@ -194,6 +202,13 @@ export const handle_preview_preset = async (
       skill_definitions += result.skill_definitions
     }
 
+    if (processed_instructions.includes('#Image(')) {
+      processed_instructions = await replace_image_symbol({
+        instruction: processed_instructions,
+        remove: true
+      })
+    }
+
     let system_instructions_xml = ''
     if (panel_provider.web_prompt_type == 'edit-context') {
       const config = vscode.workspace.getConfiguration('codeWebChat')

apps/editor/src/views/panel/backend/message-handlers/handle-prune-context.ts

@@ -22,6 +22,7 @@ import { replace_saved_context_symbol } from '@/views/panel/backend/utils/replac
 import { replace_selection_symbol } from '@/views/panel/backend/utils/replace-selection-symbol'
 import { PanelProvider } from '@/views/panel/backend/panel-provider'
 import { replace_skill_symbol } from '@/views/panel/backend/utils/replace-skill-symbol'
+import { replace_image_symbol } from '@/views/panel/backend/utils/replace-image-symbol'
 import { apply_reasoning_effort } from '@/utils/apply-reasoning-effort'
 import { PruneContextMessage } from '@/views/panel/types/messages'
 import { dictionary } from '@shared/constants/dictionary'
@@ -345,6 +346,12 @@ export const handle_prune_context = async (
     skill_definitions += result.skill_definitions
   }
 
+  if (processed_instructions.includes('#Image(')) {
+    processed_instructions = await replace_image_symbol({
+      instruction: processed_instructions
+    })
+  }
+
   const collected_files = await files_collector.collect_files({
     compact: true
   })
@@ -409,10 +416,33 @@ export const handle_prune_context = async (
 
     const content = `${system_instructions_xml}\n${skill_definitions}${files}\n${system_instructions_xml}\n${processed_instructions}`
 
+    let user_content: any = content
+
+    if (content.includes('<cwc-image>')) {
+      user_content = []
+      const parts = content.split(/<cwc-image>([\s\S]*?)<\/cwc-image>/)
+
+      for (let i = 0; i < parts.length; i++) {
+        const part = parts[i]
+        if (i % 2 == 0) {
+          if (part.length > 0) {
+            user_content.push({ type: 'text', text: part.trim() })
+          }
+        } else {
+          user_content.push({
+            type: 'image_url',
+            image_url: {
+              url: `data:image/png;base64,${part}`
+            }
+          })
+        }
+      }
+    }
+
     const messages = [
       {
         role: 'user',
-        content
+        content: user_content
       }
     ]
 

apps/editor/src/views/panel/backend/message-handlers/handle-save-temp-image.ts

@@ -0,0 +1,23 @@
+import * as vscode from 'vscode'
+import * as os from 'os'
+import * as path from 'path'
+import * as fs from 'fs'
+import * as crypto from 'crypto'
+import { SaveTempImageMessage } from '../../types/messages'
+import { PanelProvider } from '../panel-provider'
+
+export const handle_save_temp_image = async (
+  panel_provider: PanelProvider,
+  message: SaveTempImageMessage
+) => {
+  try {
+    const buffer = Buffer.from(message.content_base64, 'base64')
+    const hash = crypto.createHash('md5').update(buffer).digest('hex')
+    const filename = `cwc-image-${hash}.txt`
+    const file_path = path.join(os.tmpdir(), filename)
+    await fs.promises.writeFile(file_path, message.content_base64, 'utf-8')
+    panel_provider.add_text_at_cursor_position(`#Image(${hash})`)
+  } catch (error) {
+    vscode.window.showErrorMessage(`Failed to save image: ${error}`)
+  }
+}