Implement persistent text storage for pasted text and website contents

- Refactor pasted text handling in the save/open handlers and symbol replacers to use `extension_context.globalStorageUri.fsPath + '/pasted-text'` instead of `os.tmpdir()`, replace the MD5 hash with a `Date.now()` timestamp for the file ID and `#PastedText` symbol, and add garbage collection to the save handler that deletes files in that directory with a timestamp older than 1 year (31536000000 ms).
- treat parsed websites similarly
- the one year garbage collection threshold should be kept in `apps/editor/src/constants/values.ts`
- pasted text/websites storage should be abstracted into a shared interace/util
- [Pasted text] should return id (so accepting in param is not necessary)
- the 'website' symbol should have website's title in it, not the generic 'Website'
- rename 'persistent storage' to 'persistent text storage'
- in parsed website, don't place URL. When displaying such website, in the titlebar, use it's title from the symbol.
- in parsed website preview: don't print page title in heading. in constructed prompt: place the website text in a markdown code block (```markdown...) prefixed by the title in **bold text** (taken from the symbol)
- analyze latest changes: [Changes] - keep storing url (before ) in the symbol and restore ability to open the website from preview action button.
- simplify `apps/editor/src/views/prompt/backend/utils/persistent-text-storage.ts` by storing pasted text and websites to unified folder in the storage (no need to store them in separate `pasted-text` and `websites` folders). Use folder name `prompt-text`.
- simplify file names, e.g. [Pasted text] to 1792446329395.txt
- instead of timestamps, use sha256 hashes for filenames. Garbage collect old files based on file metadata.
- shorten these hashes to reasonable lengths [Pasted text]

├── apps
│   └── editor
│       ├── package.json
│       └── src
│           ├── commands
│           │   └── save-context-command
│           │       ├── actions
│           │       │   └── save-to-workspace-state.ts
│           │       └── save-context-command.ts
│           ├── constants
│           │   └── values.ts
│           ├── features
│           │   └── context-restoration
│           │       └── utils
│           │           └── global-storage-utils.ts
│           ├── utils
│           │   ├── files-collector.ts
│           │   ├── normalize-path.ts
│           │   └── prompt-builder.ts
│           └── views
│               └── prompt
│                   ├── backend
│                   │   ├── message-handlers
│                   │   │   ├── handle-open-prompt-pasted-text.ts
│                   │   │   ├── handle-open-website.ts
│                   │   │   ├── handle-paste-url.ts
│                   │   │   ├── handle-save-prompt-pasted-text.ts
│                   │   │   ├── index.ts
│                   │   │   └── utils
│                   │   │       └── build-prompt-payload.ts
│                   │   ├── prompt-view-provider.ts
│                   │   └── utils
│                   │       ├── persistent-text-storage.ts
│                   │       ├── symbols
│                   │       │   ├── pasted-text
│                   │       │   │   └── replace-pasted-text-symbol.ts
│                   │       │   ├── replace-symbols.ts
│                   │       │   └── website
│                   │       │       └── replace-website-symbol.ts
│                   │       └── website-fetcher.ts
│                   ├── frontend
│                   │   ├── Main
│                   │   │   ├── Main.tsx
│                   │   │   └── MainView
│                   │   │       └── MainView.tsx
│                   │   ├── Prompt.tsx
│                   │   └── hooks
│                   │       └── use-prompt
│                   │           └── use-prompt.ts
│                   └── types
│                       └── messages.ts
└── packages
    ├── shared
    │   └── src
    │       └── utils
    │           └── simplify-prompt-symbols
    │               └── simplify-prompt-symbols.ts
    └── ui
        └── src
            └── components
                └── editor
                    └── common
                        └── prompts
                            ├── PromptField
                            │   ├── PromptField.tsx
                            │   └── hooks
                            │       └── use-handlers
                            │           ├── handle-paste.ts
                            │           └── utils.ts
                            └── shared
                                └── symbols
                                    ├── get-highlighted-text.tsx
                                    ├── position-mapping.ts
                                    ├── reconstruct-raw-value.ts
                                    └── use-symbol-deletion.ts

Author: robertpiosik

apps/editor/src/views/prompt/backend/message-handlers/handle-open-prompt-pasted-text.ts

@@ -1,14 +1,17 @@
-import * as os from 'os'
-import * as path from 'path'
 import * as vscode from 'vscode'
+import { OpenPromptPastedTextMessage } from '../../types/messages'
+import { PromptViewProvider } from '../prompt-view-provider'
+import { PersistentTextStorage } from '../utils/persistent-text-storage'
 
-export const handle_open_prompt_pasted_text = async (message: {
-  hash: string
-}) => {
+export const handle_open_prompt_pasted_text = async (
+  prompt_view_provider: PromptViewProvider,
+  message: OpenPromptPastedTextMessage
+) => {
   try {
-    const hash = message.hash
-    const filename = `cwc-paste-${hash}.txt`
-    const file_path = path.join(os.tmpdir(), filename)
+    const storage = new PersistentTextStorage(
+      prompt_view_provider.extension_context
+    )
+    const file_path = storage.get_file_path(message.id)
 
     const doc = await vscode.workspace.openTextDocument(file_path)
     await vscode.window.showTextDocument(doc)

apps/editor/src/views/prompt/backend/message-handlers/handle-open-website.ts

@@ -1,26 +1,50 @@
 import * as vscode from 'vscode'
 import { OpenWebsiteMessage } from '../../types/messages'
 import * as fs from 'fs'
-import { get_website_file_path } from '../utils/website-fetcher'
 import { marked } from 'marked'
+import { PromptViewProvider } from '../prompt-view-provider'
+import { PersistentTextStorage } from '../utils/persistent-text-storage'
 
 let current_preview_url: string | undefined
 
 export const get_current_preview_url = () => current_preview_url
 
-export const handle_open_website = async (message: OpenWebsiteMessage) => {
-  const file_path = get_website_file_path(message.url)
+export const handle_open_website = async (
+  prompt_view_provider: PromptViewProvider,
+  message: OpenWebsiteMessage
+) => {
+  let id: string | undefined
+  let title = 'Website'
 
-  if (fs.existsSync(file_path)) {
-    const markdown = fs.readFileSync(file_path, 'utf8')
-    const html = await marked.parse(markdown)
+  const escaped_url = message.url.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
+  const instructions = prompt_view_provider.current_instructions
+  const symbol_regex = new RegExp(
+    `#Website\\(${escaped_url} ([a-fA-F0-9]+):\\d+(?: "((?:\\\\.|[^"\\\\])*)")?\\)`
+  )
+
+  let match = instructions.match(symbol_regex)
+  if (!match) {
+    for (const task of prompt_view_provider.completed_tasks) {
+      match = task.match(symbol_regex)
+      if (match) break
+    }
+  }
 
-    const url_obj = new URL(message.url)
-    let hostname = url_obj.hostname
-    if (hostname.startsWith('www.')) {
-      hostname = hostname.substring(4)
+  if (match) {
+    id = match[1]
+    if (match[2]) {
+      title = match[2].replace(/\\"/g, '"').replace(/\\\\/g, '\\')
     }
-    const title = `${hostname}${url_obj.pathname == '/' ? '' : url_obj.pathname}`
+  }
+
+  const storage = new PersistentTextStorage(
+    prompt_view_provider.extension_context
+  )
+  const file_path = id ? storage.get_file_path(id) : ''
+
+  if (id && fs.existsSync(file_path)) {
+    const markdown = fs.readFileSync(file_path, 'utf8')
+    const html = await marked.parse(markdown)
 
     const webview_panel = vscode.window.createWebviewPanel(
       'websitePreview',
@@ -78,7 +102,7 @@ export const handle_open_website = async (message: OpenWebsiteMessage) => {
         <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.11.1/styles/vs2015.min.css">
         <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.11.1/highlight.min.js"></script>
         <script>document.addEventListener('DOMContentLoaded', () => hljs.highlightAll());</script>
-        <title>${message.url}</title>
+        <title>${title}</title>
         <style>
           body {
             line-height: 1.5;
@@ -101,6 +125,10 @@ export const handle_open_website = async (message: OpenWebsiteMessage) => {
       </html>
     `
   } else {
-    vscode.env.openExternal(vscode.Uri.parse(message.url))
+    if (message.url.startsWith('http')) {
+      vscode.env.openExternal(vscode.Uri.parse(message.url))
+    } else {
+      vscode.window.showErrorMessage('Website file not found.')
+    }
   }
 }

apps/editor/src/views/prompt/backend/message-handlers/handle-paste-url.ts

@@ -1,11 +1,8 @@
 import * as vscode from 'vscode'
 import { PasteUrlMessage } from '../../types/messages'
 import { PromptViewProvider } from '../prompt-view-provider'
-import * as fs from 'fs'
-import {
-  fetch_and_save_website,
-  get_website_file_path
-} from '../utils/website-fetcher'
+import { fetch_website } from '../utils/website-fetcher'
+import { PersistentTextStorage } from '../utils/persistent-text-storage'
 
 export const handle_paste_url = async (
   prompt_view_provider: PromptViewProvider,
@@ -23,13 +20,9 @@ export const handle_paste_url = async (
   }
 
   try {
-    const url = message.url
-    const file_path = get_website_file_path(url)
-
-    if (fs.existsSync(file_path)) {
-      prompt_view_provider.add_text_at_cursor_position(`#Website(${url})`)
-      return
-    }
+    const storage = new PersistentTextStorage(
+      prompt_view_provider.extension_context
+    )
 
     const abort_controller = new AbortController()
     prompt_view_provider.api_call_abort_controller = abort_controller
@@ -41,15 +34,40 @@ export const handle_paste_url = async (
     })
 
     let content: string | null = null
+    let title: string = ''
+    let id: string | null = null
     try {
-      content = await fetch_and_save_website(url, abort_controller.signal)
+      const result = await fetch_website(message.url, abort_controller.signal)
+      if (result) {
+        content = result.content
+        title = result.title
+        id = await storage.save(content)
+      }
     } finally {
       prompt_view_provider.send_message({ command: 'HIDE_PROGRESS' })
       prompt_view_provider.api_call_abort_controller = null
     }
 
-    if (content) {
-      prompt_view_provider.add_text_at_cursor_position(`#Website(${url})`)
+    if (content && id) {
+      const token_count = Math.ceil(content.length / 4)
+
+      let title_part = ''
+      if (title) {
+        title_part = ` "${title.replace(/\\/g, '\\\\').replace(/"/g, '\\"')}"`
+      } else {
+        try {
+          const parsed = new URL(message.url)
+          let hostname = parsed.hostname
+          if (hostname.startsWith('www.')) {
+            hostname = hostname.slice(4)
+          }
+          title_part = ` "${hostname}"`
+        } catch {}
+      }
+
+      prompt_view_provider.add_text_at_cursor_position(
+        `#Website(${message.url} ${id}:${token_count}${title_part})`
+      )
     } else {
       await paste_as_text()
     }

apps/editor/src/views/prompt/backend/message-handlers/handle-save-prompt-pasted-text.ts

@@ -1,23 +1,21 @@
-import * as crypto from 'crypto'
-import * as fs from 'fs'
-import * as os from 'os'
-import * as path from 'path'
 import * as vscode from 'vscode'
 import { SavePromptPastedTextMessage } from '../../types/messages'
 import { PromptViewProvider } from '../prompt-view-provider'
+import { PersistentTextStorage } from '../utils/persistent-text-storage'
 
 export const handle_save_prompt_pasted_text = async (
   prompt_view_provider: PromptViewProvider,
   message: SavePromptPastedTextMessage
 ) => {
   try {
-    const hash = crypto.createHash('md5').update(message.text).digest('hex')
-    const filename = `cwc-paste-${hash}.txt`
-    const file_path = path.join(os.tmpdir(), filename)
-    await fs.promises.writeFile(file_path, message.text, 'utf-8')
+    const storage = new PersistentTextStorage(
+      prompt_view_provider.extension_context
+    )
+    const id = await storage.save(message.text)
+
     const token_count = Math.ceil(message.text.length / 4)
     prompt_view_provider.add_text_at_cursor_position(
-      `#PastedText(${hash}:${token_count})`
+      `#PastedText(${id}:${token_count})`
     )
   } catch (error) {
     vscode.window.showErrorMessage(`Failed to save pasted text: ${error}`)

apps/editor/src/views/prompt/backend/prompt-view-provider.ts

@@ -864,11 +864,11 @@ export class PromptViewProvider implements vscode.WebviewViewProvider {
           } else if (message.command == 'SAVE_PROMPT_PASTED_TEXT') {
             await handle_save_prompt_pasted_text(this, message)
           } else if (message.command == 'OPEN_PROMPT_PASTED_TEXT') {
-            await handle_open_prompt_pasted_text(message)
+            await handle_open_prompt_pasted_text(this, message)
           } else if (message.command == 'PASTE_URL') {
             await handle_paste_url(this, message)
           } else if (message.command == 'OPEN_WEBSITE') {
-            await handle_open_website(message)
+            await handle_open_website(this, message)
           } else if (message.command == 'SET_RECORDING_STATE') {
             await handle_voice_input(this, message)
           } else if (message.command == 'GET_SETUP_PROGRESS') {

apps/editor/src/views/prompt/backend/utils/persistent-text-storage.ts

@@ -0,0 +1,80 @@
+import * as fs from 'fs'
+import * as path from 'path'
+import * as vscode from 'vscode'
+import * as crypto from 'crypto'
+
+export const GARBAGE_COLLECTION_THRESHOLD = 7884000000 // 3 months
+
+export class PersistentTextStorage {
+  private dir_path: string
+
+  constructor(extension_context: vscode.ExtensionContext) {
+    this.dir_path = path.join(
+      extension_context.globalStorageUri.fsPath,
+      'prompt-text'
+    )
+  }
+
+  async save(content: string): Promise<string> {
+    const id = crypto
+      .createHash('sha256')
+      .update(content)
+      .digest('hex')
+      .substring(0, 16)
+    if (!fs.existsSync(this.dir_path)) {
+      await fs.promises.mkdir(this.dir_path, { recursive: true })
+    }
+    const filename = `${id}.txt`
+    const file_path = path.join(this.dir_path, filename)
+
+    if (fs.existsSync(file_path)) {
+      const now = new Date()
+      await fs.promises.utimes(file_path, now, now)
+    } else {
+      await fs.promises.writeFile(file_path, content, 'utf-8')
+    }
+
+    await this.garbage_collect()
+
+    return id
+  }
+
+  async load(id: string): Promise<string> {
+    const filename = `${id}.txt`
+    const file_path = path.join(this.dir_path, filename)
+    return await fs.promises.readFile(file_path, 'utf-8')
+  }
+
+  get_file_path(id: string): string {
+    const filename = `${id}.txt`
+    return path.join(this.dir_path, filename)
+  }
+
+  private async garbage_collect() {
+    try {
+      if (!fs.existsSync(this.dir_path)) {
+        return
+      }
+      const files = await fs.promises.readdir(this.dir_path)
+      const threshold_time = Date.now() - GARBAGE_COLLECTION_THRESHOLD
+      for (const file of files) {
+        if (file.endsWith('.txt')) {
+          const file_path = path.join(this.dir_path, file)
+          try {
+            const stats = await fs.promises.stat(file_path)
+            if (stats.mtimeMs < threshold_time) {
+              await fs.promises.unlink(file_path)
+            }
+          } catch (err) {
+            // ignore
+          }
+        }
+      }
+    } catch (gc_error) {
+      console.error(
+        `Failed to garbage collect files in ${this.dir_path}:`,
+        gc_error
+      )
+    }
+  }
+}

apps/editor/src/views/prompt/backend/utils/symbols/pasted-text/replace-pasted-text-symbol.ts

@@ -1,10 +1,12 @@
-import * as fs from 'fs'
-import * as os from 'os'
-import * as path from 'path'
+import * as vscode from 'vscode'
+import { PersistentTextStorage } from '../../persistent-text-storage'
 
 export const replace_pasted_text_symbol = async (params: {
   instructions: string[]
+  extension_context: vscode.ExtensionContext
 }): Promise<string[]> => {
+  const storage = new PersistentTextStorage(params.extension_context)
+
   return Promise.all(
     params.instructions.map(async (instruction) => {
       const regex = /#PastedText\(([a-fA-F0-9]+)(?::\d+)?\)/g
@@ -17,11 +19,9 @@ export const replace_pasted_text_symbol = async (params: {
 
       const replacements = await Promise.all(
         matches.map(async (match) => {
-          const hash = match[1]
-          const filename = `cwc-paste-${hash}.txt`
-          const file_path = path.join(os.tmpdir(), filename)
+          const id = match[1]
           try {
-            const content = await fs.promises.readFile(file_path, 'utf-8')
+            const content = await storage.load(id)
             return {
               content,
               success: true

apps/editor/src/views/prompt/backend/utils/symbols/replace-symbols.ts

@@ -75,13 +75,15 @@ export const replace_symbols = async (params: {
 
   if (processed_instructions.some((i) => i.includes('#PastedText('))) {
     processed_instructions = await replace_pasted_text_symbol({
-      instructions: processed_instructions
+      instructions: processed_instructions,
+      extension_context: params.extension_context
     })
   }
 
   if (processed_instructions.some((i) => i.includes('#Website('))) {
     processed_instructions = await replace_website_symbol({
       instructions: processed_instructions,
+      extension_context: params.extension_context,
       symbols_cache: params.symbols_cache
     })
   }