Skip to content

Commit 930da94

Browse files
robertpiosikrobertpiosik
committed
Refactor token calculation caching, enhance file compaction across multiple languages, and update Hugging Chat integration
1 parent 9080e2e commit 930da94

15 files changed

Lines changed: 162 additions & 132 deletions

File tree

‎packages/browser/src/content-scripts/send-prompt-content-script/chatbots/hugging-chat.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -131,7 +131,7 @@ export const hugging_chat: Chatbot = {
131131
get_chat_turn: (f) => f.closest('div[data-message-role="assistant"]'),
132132
perform_copy: (f) => {
133133
const copy_button = f.querySelector(
134-
'button:nth-child(3)'
134+
'button:nth-child(2)'
135135
) as HTMLElement
136136
if (!copy_button) {
137137
report_initialization_error({
@@ -142,7 +142,7 @@ export const hugging_chat: Chatbot = {
142142
}
143143
copy_button.click()
144144
},
145-
insert_button: (f, b) => f.insertBefore(b, f.children[1])
145+
insert_button: (f, b) => f.insertBefore(b, f.children[0])
146146
})
147147
}
148148

‎packages/browser/src/manifest.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
"name": "Code Web Chat Connector",
55
"short_name": "CWC",
66
"description": "CWC makes it easy to construct coding prompts for chatbots and APIs.",
7-
"version": "1.86.0",
7+
"version": "1.87.0",
88
"homepage_url": "https://github.com/robertpiosik/CodeWebChat",
99
"icons": {
1010
"16": "icons/icon-16.png",

‎packages/ui/src/components/editor/panel/Tasks/Tasks.stories.tsx‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -151,7 +151,7 @@ export const Default = () => {
151151
on_add={handle_add}
152152
on_add_subtask={handle_add_subtask}
153153
on_delete={handle_delete}
154-
on_copy={handle_copy}
154+
on_forward={() => {}}
155155
/>
156156
</div>
157157
)
@@ -178,7 +178,7 @@ export const NoReorder = () => {
178178
on_add={() => {}}
179179
on_add_subtask={() => {}}
180180
on_delete={() => {}}
181-
on_copy={(text) => console.log('Copied:', text)}
181+
on_forward={() => {}}
182182
/>
183183
</div>
184184
)

‎packages/vscode/src/context/providers/workspace/modules/token-calculator.ts‎

Lines changed: 67 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -10,18 +10,14 @@ type TokenCountsCache = {
1010
[workspace_root: string]: {
1111
modified_at: number
1212
files: {
13-
[file_path: string]: {
14-
modified_at: number
15-
token_count: number
16-
compact_token_count?: number
17-
}
13+
[file_path: string]: [number, number, number]
1814
}
1915
}
2016
}
2117

2218
const SHOW_COUNTING_NOTIFICATION_DELAY_MS = 3000
2319

24-
const TOKEN_CACHE_FILE_NAME = 'token-count-cache.json'
20+
const TOKEN_CACHE_FILE_NAME = 'token-counts-cache.json'
2521

2622
export class TokenCalculator implements vscode.Disposable {
2723
private _file_token_counts: Map<string, number> = new Map()
@@ -32,6 +28,7 @@ export class TokenCalculator implements vscode.Disposable {
3228
private _directory_selected_compact_token_counts: Map<string, number> =
3329
new Map()
3430
private _token_cache: TokenCountsCache = {}
31+
private _session_cache: TokenCountsCache = {}
3532
private _token_cache_update_timeout: NodeJS.Timeout | null = null
3633
private _has_token_counts_cache_updated_once = false
3734

@@ -76,8 +73,8 @@ export class TokenCalculator implements vscode.Disposable {
7673
}
7774

7875
for (const root of this._provider.get_workspace_roots()) {
79-
if (this._token_cache[root]) {
80-
current_global_cache[root] = this._token_cache[root]
76+
if (this._session_cache[root]) {
77+
current_global_cache[root] = this._session_cache[root]
8178
}
8279
}
8380

@@ -107,7 +104,29 @@ export class TokenCalculator implements vscode.Disposable {
107104
data: error
108105
})
109106
}
110-
}, 10000)
107+
}, 5000)
108+
}
109+
110+
private _update_session_cache(
111+
workspace_root: string,
112+
relative_path: string,
113+
mtime: number,
114+
token_count: number,
115+
compact_token_count: number
116+
): void {
117+
if (!this._session_cache[workspace_root]) {
118+
this._session_cache[workspace_root] = {
119+
modified_at: Date.now(),
120+
files: {}
121+
}
122+
}
123+
this._session_cache[workspace_root].files[relative_path] = [
124+
mtime,
125+
token_count,
126+
compact_token_count
127+
]
128+
this._session_cache[workspace_root].modified_at = Date.now()
129+
this._update_token_counts_cache()
111130
}
112131

113132
public async with_token_counting_notification<T>(
@@ -219,19 +238,32 @@ export class TokenCalculator implements vscode.Disposable {
219238
if (workspace_root && !range) {
220239
try {
221240
const stats = await fs.promises.stat(file_path)
222-
mtime = stats.mtimeMs
241+
mtime = Math.floor(stats.mtimeMs)
242+
const relative_path = path
243+
.relative(workspace_root, file_path)
244+
.replace(/\\/g, '/')
245+
246+
const cached_file =
247+
this._token_cache[workspace_root]?.files?.[relative_path]
248+
223249
if (
224-
this._token_cache[workspace_root]?.files?.[file_path]?.modified_at ==
225-
mtime
250+
cached_file &&
251+
Array.isArray(cached_file) &&
252+
cached_file[0] == mtime
226253
) {
227-
const tokens =
228-
this._token_cache[workspace_root].files[file_path].token_count
229-
const compact_tokens =
230-
this._token_cache[workspace_root].files[file_path]
231-
.compact_token_count || tokens // Fallback for old cache
254+
const tokens = cached_file[1]
255+
const compact_tokens = cached_file[2]
232256

233257
this._file_token_counts.set(file_path, tokens)
234258
this._file_compact_token_counts.set(file_path, compact_tokens)
259+
260+
this._update_session_cache(
261+
workspace_root,
262+
relative_path,
263+
mtime,
264+
tokens,
265+
compact_tokens
266+
)
235267
return { total: tokens, compact: compact_tokens }
236268
}
237269
} catch {
@@ -285,13 +317,25 @@ export class TokenCalculator implements vscode.Disposable {
285317
files: {}
286318
}
287319
}
288-
this._token_cache[workspace_root].files[file_path] = {
289-
modified_at: mtime,
290-
token_count: token_count,
291-
compact_token_count: compact_token_count
292-
}
320+
321+
const relative_path = path
322+
.relative(workspace_root, file_path)
323+
.replace(/\\/g, '/')
324+
325+
this._token_cache[workspace_root].files[relative_path] = [
326+
mtime,
327+
token_count,
328+
compact_token_count
329+
]
293330
this._token_cache[workspace_root].modified_at = Date.now()
294-
this._update_token_counts_cache()
331+
332+
this._update_session_cache(
333+
workspace_root,
334+
relative_path,
335+
mtime,
336+
token_count,
337+
compact_token_count
338+
)
295339
}
296340

297341
return { total: token_count, compact: compact_token_count }

‎packages/vscode/src/context/utils/compact-file/compact-file.spec.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,11 @@ describe('compact_file', () => {
4646
expect(compact_file(input, '.css')).toBe(expected)
4747
})
4848

49+
it('should handle scss with nested BEM', () => {
50+
const { input, expected } = load_test_case('scss', '.')
51+
expect(compact_file(input, '.scss')).toBe(expected)
52+
})
53+
4954
it('should strip sql comments', () => {
5055
const { input, expected } = load_test_case('sql', '.')
5156
expect(compact_file(input, '.sql')).toBe(expected)

‎packages/vscode/src/context/utils/compact-file/languages/c-style.ts‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@ export const compact_c_style = (content: string): string => {
2020
const char = line[i]
2121
const next_char = line[i + 1]
2222

23-
// Handle Block Comments
2423
if (is_in_block_comment) {
2524
if (char == '*' && next_char == '/') {
2625
is_in_block_comment = false
@@ -31,10 +30,8 @@ export const compact_c_style = (content: string): string => {
3130
continue
3231
}
3332

34-
// Handle Strings (ignore comments inside strings)
3533
if (is_in_string) {
3634
if (char == '\\') {
37-
// Handle escaped characters inside string
3835
if (skip_body_depth == 0) {
3936
processed_line += char + (next_char || '')
4037
}
@@ -53,7 +50,6 @@ export const compact_c_style = (content: string): string => {
5350
continue
5451
}
5552

56-
// Start of String
5753
if (char == '"' || char === "'" || char == '`') {
5854
is_in_string = char
5955
if (skip_body_depth == 0) {
@@ -64,12 +60,10 @@ export const compact_c_style = (content: string): string => {
6460
continue
6561
}
6662

67-
// Start of Line Comment
6863
if (char == '/' && next_char == '/') {
6964
break // Ignore the rest of the line
7065
}
7166

72-
// Start of Block Comment
7367
if (char == '/' && next_char == '*') {
7468
is_in_block_comment = true
7569
i += 2
@@ -96,7 +90,6 @@ export const compact_c_style = (content: string): string => {
9690
const is_object_literal = /(=|:)\s*$/.test(last_code_buffer.trimEnd())
9791

9892
if (!is_keeper && !is_object_literal) {
99-
// Start skipping
10093
skip_body_depth = 1
10194
processed_line += `{\n${current_indent} // ...\n${current_indent}}`
10295
last_code_buffer += '{}'
@@ -116,7 +109,6 @@ export const compact_c_style = (content: string): string => {
116109
}
117110
}
118111

119-
// Regular character
120112
if (skip_body_depth == 0) {
121113
processed_line += char
122114
}

‎packages/vscode/src/context/utils/compact-file/languages/css.ts‎

Lines changed: 14 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,6 @@ export const compact_css = (content: string): string => {
33
const lines = content.split(/\r?\n/)
44
const result: string[] = []
55
let is_in_block_comment = false
6-
let skip_body_depth = 0
7-
// Buffer to track context for stripping bodies (e.g. "@media ... ")
8-
let last_code_buffer = ''
96

107
for (const line of lines) {
118
let processed_line = ''
@@ -16,7 +13,6 @@ export const compact_css = (content: string): string => {
1613
const char = line[i]
1714
const next_char = line[i + 1]
1815

19-
// Handle Block Comments
2016
if (is_in_block_comment) {
2117
if (char == '*' && next_char == '/') {
2218
is_in_block_comment = false
@@ -27,107 +23,55 @@ export const compact_css = (content: string): string => {
2723
continue
2824
}
2925

30-
// Handle Strings (ignore comments inside strings)
3126
if (is_in_string) {
3227
if (char == '\\') {
33-
// Handle escaped characters inside string
34-
if (skip_body_depth == 0) {
35-
processed_line += char + (next_char || '')
36-
}
37-
last_code_buffer += char + (next_char || '')
28+
processed_line += char + (next_char || '')
3829
i += 2
3930
continue
4031
}
4132
if (char === is_in_string) {
4233
is_in_string = false
4334
}
44-
if (skip_body_depth == 0) {
45-
processed_line += char
46-
}
47-
last_code_buffer += char
35+
processed_line += char
4836
i++
4937
continue
5038
}
5139

52-
// Start of String
5340
if (char == '"' || char === "'") {
5441
is_in_string = char
55-
if (skip_body_depth == 0) {
56-
processed_line += char
57-
}
58-
last_code_buffer += char
42+
processed_line += char
5943
i++
6044
continue
6145
}
6246

63-
// Start of Line Comment (Support // for SCSS/LESS, though not standard CSS)
47+
// Support // for SCSS/LESS, though not standard CSS
6448
if (char == '/' && next_char == '/') {
6549
break // Ignore the rest of the line
6650
}
6751

68-
// Start of Block Comment
6952
if (char == '/' && next_char == '*') {
7053
is_in_block_comment = true
7154
i += 2
7255
continue
7356
}
7457

75-
// Handle Body Stripping
76-
if (char == '{') {
77-
if (skip_body_depth > 0) {
78-
// Already skipping, just deepen the level
79-
skip_body_depth++
80-
last_code_buffer += char
81-
i++
82-
continue
83-
} else {
84-
// Check if we should preserve this block (at-rules like @media)
85-
const is_keeper =
86-
/(@media|@supports|@keyframes|@layer|@container)\b/.test(
87-
last_code_buffer
88-
)
89-
90-
if (!is_keeper) {
91-
// Start skipping
92-
skip_body_depth = 1
93-
processed_line += '{}'
94-
last_code_buffer += '{}'
95-
i++
96-
continue
97-
} else {
98-
// Keeper: reset buffer
99-
last_code_buffer = ''
100-
}
101-
}
102-
} else if (char == '}') {
103-
if (skip_body_depth > 0) {
104-
skip_body_depth--
105-
last_code_buffer += char
106-
i++
107-
continue
108-
}
109-
}
110-
111-
// Regular character
112-
if (skip_body_depth == 0) {
113-
processed_line += char
114-
}
115-
last_code_buffer += char
58+
processed_line += char
11659
i++
11760
}
11861

11962
// We trim the line to actually "compact" the file.
120-
// If a line contained only comments, it becomes empty and is skipped.
12163
const trimmed = processed_line.trim()
12264
if (trimmed) {
123-
result.push(processed_line.trimEnd())
124-
}
65+
// Filter Logic:
66+
// 1. Keep lines that preserve structure (containing { or })
67+
// 2. Strip lines that look like properties/variables (contain : or = and end in ;)
68+
// Also strip SCSS/LESS at-rules that function as properties (@include, @extend, etc.)
69+
const has_structure = /[{}]/.test(trimmed)
70+
const is_property = /([:=].*|@(?:include|extend|apply).*);$/.test(trimmed)
12571

126-
// Add a space to buffer for newline
127-
last_code_buffer += ' '
128-
// Limit buffer size
129-
if (last_code_buffer.length > 200) {
130-
last_code_buffer = last_code_buffer.slice(-200)
72+
if (has_structure || !is_property) {
73+
result.push(processed_line.trimEnd())
74+
}
13175
}
13276
}
13377

0 commit comments

Comments
 (0)