Improve API manager modal responsiveness and caching with viewport-aware sizing, abbreviated labels, and request body-aware queueing

Author: robertpiosik

packages/ui/src/components/editor/panel/modals/ApiManagerModal/ApiManagerModal.module.scss

@@ -23,7 +23,7 @@
   border: 1px solid var(--vscode-focusBorder);
   box-shadow: 0 0 8px 2px var(--vscode-widget-shadow);
   width: 100%;
-  max-width: 260px;
+  max-width: min(260px, 70vw);
   overflow: hidden;
 }
 
@@ -94,7 +94,7 @@
     display: flex;
     height: 100%;
     align-items: center;
-    right: 4px;
+    right: 8px;
     font-size: 12px;
     background: linear-gradient(
       to left,

packages/ui/src/components/editor/panel/modals/ApiManagerModal/ApiManagerModal.tsx

@@ -27,6 +27,15 @@ export const ApiManagerModal: React.FC<Props> = (props) => {
   const [start_times, set_start_times] = useState<Record<string, number>>({})
   const [now, set_now] = useState(Date.now())
   const [is_scrolled, set_is_scrolled] = useState(false)
+  const [window_width, set_window_width] = useState(window.innerWidth)
+
+  useEffect(() => {
+    const handle_resize = () => set_window_width(window.innerWidth)
+    window.addEventListener('resize', handle_resize)
+    return () => {
+      window.removeEventListener('resize', handle_resize)
+    }
+  }, [])
 
   useEffect(() => {
     const interval = setInterval(() => {
@@ -87,7 +96,8 @@ export const ApiManagerModal: React.FC<Props> = (props) => {
                     <div className={styles.item__right}>
                       {item.tokens_per_second !== undefined && (
                         <div className={styles['tokens-per-second']}>
-                          {format_tokens(item.tokens_per_second)} tokens/s
+                          {format_tokens(item.tokens_per_second)}{' '}
+                          {window_width < 340 ? 't/s' : 'tokens/s'}
                         </div>
                       )}
                       {item.total_tokens !== undefined && (

packages/vscode/src/services/api-manager.ts

@@ -1,14 +1,17 @@
 import { PanelProvider } from '@/views/panel/backend/panel-provider'
 import { make_api_request } from '@/utils/make-api-request'
 import axios, { CancelTokenSource } from 'axios'
-import { randomUUID } from 'crypto'
+import { randomUUID, createHash } from 'crypto'
 import { Logger } from '@shared/utils/logger'
 
 export class ApiManager {
   private cancel_token_sources: Map<string, CancelTokenSource> = new Map()
   private next_allowed_finish_time = 0
   // Waiting pipeline to utilize KV cache for the same endpoint urls
-  private waiting_chain: Map<string, Promise<void>> = new Map()
+  private waiting_chain: Map<
+    string,
+    { promise: Promise<void>; resolve: () => void; body_hash: string }
+  > = new Map()
 
   constructor(private panel_provider: PanelProvider) {}
 
@@ -22,12 +25,23 @@ export class ApiManager {
     const cancel_token_source = axios.CancelToken.source()
     this.cancel_token_sources.set(request_id, cancel_token_source)
 
-    const previous_waiting_promise = this.waiting_chain.get(params.endpoint_url)
-    let resolve_waiting: () => void = () => {}
-    const waiting_promise = new Promise<void>((resolve) => {
-      resolve_waiting = resolve
+    const body_hash = createHash('md5')
+      .update(JSON.stringify(params.body))
+      .digest('hex')
+    const previous_waiting = this.waiting_chain.get(params.endpoint_url)
+
+    let resolve_current: () => void = () => {}
+    const current_promise = new Promise<void>((resolve) => {
+      resolve_current = resolve
     })
-    this.waiting_chain.set(params.endpoint_url, waiting_promise)
+
+    if (!previous_waiting || previous_waiting.body_hash !== body_hash) {
+      this.waiting_chain.set(params.endpoint_url, {
+        promise: current_promise,
+        resolve: resolve_current,
+        body_hash
+      })
+    }
 
     try {
       this.panel_provider.send_message({
@@ -36,8 +50,8 @@ export class ApiManager {
         title: 'Waiting...'
       })
 
-      if (previous_waiting_promise) {
-        await previous_waiting_promise
+      if (previous_waiting && previous_waiting.body_hash === body_hash) {
+        await previous_waiting.promise
       }
 
       const result = await make_api_request({
@@ -46,15 +60,23 @@ export class ApiManager {
         body: params.body,
         cancellation_token: cancel_token_source.token,
         on_thinking_chunk: () => {
-          resolve_waiting()
+          const chain_entry = this.waiting_chain.get(params.endpoint_url)
+          if (chain_entry && chain_entry.resolve === resolve_current) {
+            chain_entry.resolve()
+            this.waiting_chain.delete(params.endpoint_url)
+          }
           this.panel_provider.send_message({
             command: 'SHOW_API_MANAGER_PROGRESS',
             id: request_id,
             title: 'Thinking...'
           })
         },
         on_chunk: (tokens_per_second, total_tokens) => {
-          resolve_waiting()
+          const chain_entry = this.waiting_chain.get(params.endpoint_url)
+          if (chain_entry && chain_entry.resolve === resolve_current) {
+            chain_entry.resolve()
+            this.waiting_chain.delete(params.endpoint_url)
+          }
           this.panel_provider.send_message({
             command: 'SHOW_API_MANAGER_PROGRESS',
             id: request_id,
@@ -81,7 +103,13 @@ export class ApiManager {
       })
       return null
     } finally {
-      resolve_waiting()
+      // Unblock anyone waiting for this request if it was the one in the chain
+      const chain_entry = this.waiting_chain.get(params.endpoint_url)
+      if (chain_entry && chain_entry.resolve === resolve_current) {
+        chain_entry.resolve()
+        this.waiting_chain.delete(params.endpoint_url)
+      }
+
       this.panel_provider.send_message({
         command: 'HIDE_API_MANAGER_PROGRESS',
         id: request_id

packages/vscode/src/views/panel/backend/message-handlers/handle-undo.ts

@@ -71,7 +71,7 @@ export const handle_undo = async (
     panel_provider.set_undo_button_state(false)
     panel_provider.send_message({
       command: 'SHOW_AUTO_CLOSING_MODAL',
-      title: 'Changes undone'
+      title: 'Changes undone successfully'
     })
   } catch (error: any) {
     console.error('Error during undo:', error)