Skip to content

Commit 4ad35fa

Browse files
robertpiosikrobertpiosik
committed
Add semantic file search support using Semble and configurable result limits
- add another search mode: Semantic. Should use Semble [Website] - on 'semantic' mode show action with question codicon and linking to https://github.com/MinishLab/semble#quickstart - don't display score (thus simplify passed data) - in General > Context, add 'limit semantic search results', in description, emphasize it sets 'top-k'. Default in-settings stored value is not defined. Display value is [Fragment]. - 1) place the 'limit semantic search results' below 'find relevant files instructions' 2) in description of this setting emphasize it's for semble (https://github.com/MinishLab/semble#quickstart) - set top-k dynamically
1 parent c7370da commit 4ad35fa

19 files changed

Lines changed: 397 additions & 15 deletions

File tree

‎apps/editor/package.json‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -906,6 +906,10 @@
906906
"type": "number",
907907
"scope": "resource"
908908
},
909+
"codeWebChat.limitSemanticSearchResults": {
910+
"type": "number",
911+
"scope": "resource"
912+
},
909913
"codeWebChat.checkpointLifespan": {
910914
"type": "number",
911915
"scope": "resource"

‎apps/editor/src/constants/state-keys.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,8 @@ export const LAST_SEARCH_FILES_FILENAME_MATCH_MODE_STATE_KEY =
9393
'last-search-files-filename-match-mode'
9494
export const LAST_SEARCH_FILES_INTELLIGENT_QUERY_STATE_KEY =
9595
'last-search-files-intelligent-query'
96+
export const LAST_SEARCH_FILES_SEMANTIC_QUERY_STATE_KEY =
97+
'last-search-files-semantic-query'
9698

9799
export const LAST_SEARCH_FILES_FOR_CONTEXT_MODE_STATE_KEY =
98100
'last-search-files-for-context-mode'

‎apps/editor/src/constants/values.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,3 +3,4 @@ export const DEFAULT_CONTEXT_SIZE_WARNING_THRESHOLD = 60000
33
export const MAX_FILE_TOKENS_FOR_COMMIT_MESSAGE = 20000
44
export const MAX_PROMPT_CHARS_IN_COMMIT_MESSAGE = 1000
55
export const GIT_LOG_SINCE_DURATION = '6 months ago'
6+
export const LIMIT_SEMANTIC_SEARCH_RESULTS = 10

‎apps/editor/src/features/search-files/actions/search-files.ts‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,8 @@ import {
77
LAST_SEARCH_FILES_KEYWORDS_MATCH_MODE_STATE_KEY,
88
LAST_SEARCH_FILES_FILENAME_QUERY_STATE_KEY,
99
LAST_SEARCH_FILES_FILENAME_MATCH_MODE_STATE_KEY,
10-
LAST_SEARCH_FILES_FOR_CONTEXT_MODE_STATE_KEY
10+
LAST_SEARCH_FILES_FOR_CONTEXT_MODE_STATE_KEY,
11+
LAST_SEARCH_FILES_SEMANTIC_QUERY_STATE_KEY
1112
} from '@/constants/state-keys'
1213
import { prompt_for_search_mode } from '../utils/prompt-for-search-mode'
1314
import { prompt_for_keywords_match_mode } from '../utils/prompt-for-keywords-match-mode'
@@ -28,7 +29,7 @@ export const search_files = async (params: {
2829
> => {
2930
let initial_search_mode =
3031
params.extension_context.workspaceState.get<
31-
'phrase' | 'keywords' | 'filename' | 'intelligent'
32+
'phrase' | 'keywords' | 'filename' | 'intelligent' | 'semantic'
3233
>(LAST_SEARCH_FILES_FOR_CONTEXT_MODE_STATE_KEY) || 'phrase'
3334

3435
while (true) {
@@ -124,7 +125,9 @@ export const search_files = async (params: {
124125
? LAST_SEARCH_FILES_PHRASE_QUERY_STATE_KEY
125126
: search_mode == 'keywords'
126127
? LAST_SEARCH_FILES_KEYWORDS_QUERY_STATE_KEY
127-
: LAST_SEARCH_FILES_FILENAME_QUERY_STATE_KEY
128+
: search_mode == 'semantic'
129+
? LAST_SEARCH_FILES_SEMANTIC_QUERY_STATE_KEY
130+
: LAST_SEARCH_FILES_FILENAME_QUERY_STATE_KEY
128131

129132
const initial_search_term =
130133
params.extension_context.workspaceState.get<string>(state_key) || ''

‎apps/editor/src/features/search-files/utils/prompt-for-search-mode.ts‎

Lines changed: 29 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,24 @@ import * as vscode from 'vscode'
22
import { t } from '@/i18n'
33

44
export const prompt_for_search_mode = async (
5-
last_mode: 'phrase' | 'keywords' | 'filename' | 'intelligent',
5+
last_mode: 'phrase' | 'keywords' | 'filename' | 'intelligent' | 'semantic',
66
show_back_button?: boolean
77
): Promise<
8-
'phrase' | 'keywords' | 'filename' | 'intelligent' | undefined | 'back'
8+
| 'phrase'
9+
| 'keywords'
10+
| 'filename'
11+
| 'intelligent'
12+
| 'semantic'
13+
| undefined
14+
| 'back'
915
> => {
16+
const help_button: vscode.QuickInputButton = {
17+
iconPath: new vscode.ThemeIcon('question'),
18+
tooltip: 'Semble Quickstart'
19+
}
20+
1021
const items: (vscode.QuickPickItem & {
11-
mode: 'phrase' | 'keywords' | 'filename' | 'intelligent'
22+
mode: 'phrase' | 'keywords' | 'filename' | 'intelligent' | 'semantic'
1223
})[] = [
1324
{
1425
label: t('feature.search-files.mode.phrase'),
@@ -29,6 +40,12 @@ export const prompt_for_search_mode = async (
2940
label: t('feature.search-files.mode.intelligent'),
3041
description: t('feature.search-files.mode.intelligent-description'),
3142
mode: 'intelligent'
43+
},
44+
{
45+
label: t('feature.search-files.mode.semantic'),
46+
description: t('feature.search-files.mode.semantic-description'),
47+
mode: 'semantic',
48+
buttons: [help_button]
3249
}
3350
]
3451

@@ -41,7 +58,7 @@ export const prompt_for_search_mode = async (
4158

4259
const quick_pick = vscode.window.createQuickPick<
4360
vscode.QuickPickItem & {
44-
mode: 'phrase' | 'keywords' | 'filename' | 'intelligent'
61+
mode: 'phrase' | 'keywords' | 'filename' | 'intelligent' | 'semantic'
4562
}
4663
>()
4764
quick_pick.items = items
@@ -69,6 +86,14 @@ export const prompt_for_search_mode = async (
6986
}
7087
})
7188

89+
quick_pick.onDidTriggerItemButton((e) => {
90+
if (e.button === help_button) {
91+
vscode.env.openExternal(
92+
vscode.Uri.parse('https://github.com/MinishLab/semble#quickstart')
93+
)
94+
}
95+
})
96+
7297
quick_pick.onDidAccept(() => {
7398
const selected = quick_pick.selectedItems[0]
7499
if (selected) {

‎apps/editor/src/features/search-files/utils/prompt-for-search-results.ts‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ import { create_search_regex } from './create-search-regex'
88
export const prompt_for_search_results = async (params: {
99
matched_files: string[]
1010
search_term: string
11-
search_mode: 'phrase' | 'keywords' | 'filename' | 'intelligent'
11+
search_mode: 'phrase' | 'keywords' | 'filename' | 'intelligent' | 'semantic'
1212
workspace_provider: WorkspaceProvider
1313
}): Promise<
1414
| { selected_paths: string[]; matched_paths: string[] }
@@ -86,7 +86,9 @@ export const prompt_for_search_results = async (params: {
8686
? t('feature.search-files.results.filename')
8787
: params.search_mode == 'intelligent'
8888
? t('feature.search-files.results.intelligent')
89-
: t('feature.search-files.results.phrase')
89+
: params.search_mode == 'semantic'
90+
? t('feature.search-files.results.semantic')
91+
: t('feature.search-files.results.phrase')
9092

9193
quick_pick.ignoreFocusOut = true
9294
quick_pick.buttons = [
@@ -146,7 +148,10 @@ export const prompt_for_search_results = async (params: {
146148
.map((k) => k.trim())
147149
.filter((k) => k.length > 0)
148150
.map((k) => create_search_regex(k))
149-
} else if (params.search_mode == 'filename') {
151+
} else if (
152+
params.search_mode == 'filename' ||
153+
params.search_mode == 'semantic'
154+
) {
150155
regexes = []
151156
} else {
152157
regexes = [create_search_regex(params.search_term)]

‎apps/editor/src/features/search-files/utils/prompt-for-search-term.ts‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ import { t } from '@/i18n'
33

44
export const prompt_for_search_term = async (
55
initial_search_term: string,
6-
mode: 'phrase' | 'keywords' | 'filename' | 'intelligent'
6+
mode: 'phrase' | 'keywords' | 'filename' | 'intelligent' | 'semantic'
77
): Promise<{ value: string | undefined; back?: boolean }> => {
88
const close_button = {
99
iconPath: new vscode.ThemeIcon('close'),
@@ -18,15 +18,19 @@ export const prompt_for_search_term = async (
1818
? t('feature.search-files.title.filename')
1919
: mode == 'intelligent'
2020
? t('feature.search-files.title.intelligent')
21-
: t('feature.search-files.title.phrase')
21+
: mode == 'semantic'
22+
? t('feature.search-files.title.semantic')
23+
: t('feature.search-files.title.phrase')
2224
input_box.prompt =
2325
mode == 'keywords'
2426
? t('feature.search-files.prompt.keywords')
2527
: mode == 'filename'
2628
? t('feature.search-files.prompt.filename')
2729
: mode == 'intelligent'
2830
? t('feature.search-files.prompt.intelligent')
29-
: t('feature.search-files.prompt')
31+
: mode == 'semantic'
32+
? t('feature.search-files.prompt.semantic')
33+
: t('feature.search-files.prompt')
3034
input_box.placeholder = t('feature.search-files.placeholder')
3135
input_box.value = initial_search_term
3236
input_box.ignoreFocusOut = true

‎apps/editor/src/features/search-files/utils/search-files-by-term.ts‎

Lines changed: 56 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,72 @@
11
import * as path from 'path'
22
import * as fs from 'fs'
3+
import * as vscode from 'vscode'
4+
import { exec } from 'child_process'
5+
import { promisify } from 'util'
36
import { Logger } from '@shared/utils/logger'
47
import { create_search_regex } from './create-search-regex'
58
import { IGNORED_LOCK_FILES } from '@/constants/ignored-lock-files'
9+
import { LIMIT_SEMANTIC_SEARCH_RESULTS } from '@/constants/values'
10+
11+
const exec_promise = promisify(exec)
612

713
export const search_files_by_term = async (params: {
814
files: string[]
915
search_term: string
10-
search_mode: 'phrase' | 'keywords' | 'filename' | 'intelligent'
16+
search_mode: 'phrase' | 'keywords' | 'filename' | 'intelligent' | 'semantic'
1117
keywords_match_mode?: 'all' | 'some'
1218
}): Promise<string[]> => {
1319
const matched_files: string[] = []
1420

21+
if (params.search_mode === 'semantic') {
22+
const workspace_folders = vscode.workspace.workspaceFolders
23+
if (!workspace_folders) return []
24+
25+
const config = vscode.workspace.getConfiguration('codeWebChat')
26+
const top_k =
27+
config.get<number>('limitSemanticSearchResults') ||
28+
LIMIT_SEMANTIC_SEARCH_RESULTS
29+
30+
const matched_map = new Map<string, number | undefined>()
31+
32+
for (const folder of workspace_folders) {
33+
try {
34+
const safe_term = params.search_term.replace(/"/g, '\\"')
35+
const { stdout } = await exec_promise(
36+
`semble search "${safe_term}" "${folder.uri.fsPath}" --top-k ${top_k}`
37+
)
38+
39+
const lines = stdout.split('\n')
40+
41+
for (const file of params.files) {
42+
const relative_path = path.relative(folder.uri.fsPath, file)
43+
const matched_line = lines.find(
44+
(line) => line.includes(relative_path) || line.includes(file)
45+
)
46+
47+
if (matched_line) {
48+
const score_match = matched_line.match(
49+
/\b(0\.\d+|1\.0+|\d+\.\d+)\b/
50+
)
51+
const score = score_match ? parseFloat(score_match[1]) : undefined
52+
matched_map.set(file, score)
53+
}
54+
}
55+
} catch (error) {
56+
Logger.error({
57+
function_name: 'search_files_by_term',
58+
message: `Error running semble search in ${folder.uri.fsPath}`,
59+
data: error
60+
})
61+
}
62+
}
63+
64+
return Array.from(matched_map.entries())
65+
.map(([file_path, score]) => ({ file_path, score }))
66+
.sort((a, b) => (b.score || 0) - (a.score || 0))
67+
.map((item) => item.file_path)
68+
}
69+
1570
if (params.search_mode === 'filename') {
1671
const keywords = params.search_term
1772
.split(',')

‎apps/editor/src/i18n/translations/features/search-files.ts‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -580,6 +580,96 @@ export const translations = {
580580
hu: 'Természetes nyelvű lekérdezéshez releváns fájlok kiválasztása',
581581
bg: 'Изберете файлове, релевантни за заявка на естествен език'
582582
},
583+
'feature.search-files.mode.semantic': {
584+
en: 'Semantic',
585+
pl: 'Semantyczne',
586+
'zh-cn': '语义',
587+
ja: 'セマンティック',
588+
'zh-tw': '語義',
589+
de: 'Semantisch',
590+
es: 'Semántico',
591+
fr: 'Sémantique',
592+
'pt-br': 'Semântico',
593+
ru: 'Семантический',
594+
ko: '시맨틱',
595+
it: 'Semantico',
596+
tr: 'Anlamsal',
597+
cs: 'Sémantické',
598+
hu: 'Szemantikus',
599+
bg: 'Семантичен'
600+
},
601+
'feature.search-files.mode.semantic-description': {
602+
en: 'Search files semantically using Semble',
603+
pl: 'Wyszukuj pliki semantycznie używając Semble',
604+
'zh-cn': '使用 Semble 进行语义搜索文件',
605+
ja: 'Sembleを使用してファイルをセマンティック検索',
606+
'zh-tw': '使用 Semble 進行語義搜尋檔案',
607+
de: 'Dateien semantisch mit Semble suchen',
608+
es: 'Buscar archivos semánticamente usando Semble',
609+
fr: 'Rechercher des fichiers sémantiquement avec Semble',
610+
'pt-br': 'Buscar arquivos semanticamente usando Semble',
611+
ru: 'Семантический поиск файлов с помощью Semble',
612+
ko: 'Semble을 사용하여 시맨틱 파일 검색',
613+
it: 'Cerca file semanticamente usando Semble',
614+
tr: 'Semble kullanarak dosyaları anlamsal olarak ara',
615+
cs: 'Sémantické vyhledávání souborů pomocí Semble',
616+
hu: 'Fájlok szemantikus keresése a Semble használatával',
617+
bg: 'Семантично търсене на файлове с помощта на Semble'
618+
},
619+
'feature.search-files.prompt.semantic': {
620+
en: 'Enter semantic search query.',
621+
pl: 'Wprowadź zapytanie wyszukiwania semantycznego.',
622+
'zh-cn': '输入语义搜索查询。',
623+
ja: 'セマンティック検索クエリを入力してください。',
624+
'zh-tw': '輸入語義搜尋查詢。',
625+
de: 'Geben Sie eine semantische Suchanfrage ein.',
626+
es: 'Ingrese la consulta de búsqueda semántica.',
627+
fr: 'Entrez une requête de recherche sémantique.',
628+
'pt-br': 'Digite a consulta de pesquisa semântica.',
629+
ru: 'Введите запрос для семантического поиска.',
630+
ko: '시맨틱 검색 쿼리를 입력하세요.',
631+
it: 'Inserisci la query di ricerca semantica.',
632+
tr: 'Anlamsal arama sorgusu girin.',
633+
cs: 'Zadejte sémantický vyhledávací dotaz.',
634+
hu: 'Adja meg a szemantikus keresési lekérdezést.',
635+
bg: 'Въведете заявка за семантично търсене.'
636+
},
637+
'feature.search-files.title.semantic': {
638+
en: 'Semantic Search',
639+
pl: 'Wyszukiwanie semantyczne',
640+
'zh-cn': '语义搜索',
641+
ja: 'セマンティック検索',
642+
'zh-tw': '語義搜尋',
643+
de: 'Semantische Suche',
644+
es: 'Búsqueda semántica',
645+
fr: 'Recherche sémantique',
646+
'pt-br': 'Pesquisa semântica',
647+
ru: 'Семантический поиск',
648+
ko: '시맨틱 검색',
649+
it: 'Ricerca semantica',
650+
tr: 'Anlamsal Arama',
651+
cs: 'Sémantické hledání',
652+
hu: 'Szemantikus keresés',
653+
bg: 'Семантично търсене'
654+
},
655+
'feature.search-files.results.semantic': {
656+
en: 'Semantic Search Results',
657+
pl: 'Wyniki wyszukiwania semantycznego',
658+
'zh-cn': '语义搜索结果',
659+
ja: 'セマンティック検索結果',
660+
'zh-tw': '語義搜尋結果',
661+
de: 'Semantische Suchergebnisse',
662+
es: 'Resultados de búsqueda semántica',
663+
fr: 'Résultats de recherche sémantique',
664+
'pt-br': 'Resultados da pesquisa semântica',
665+
ru: 'Результаты семантического поиска',
666+
ko: '시맨틱 검색 결과',
667+
it: 'Risultati ricerca semantica',
668+
tr: 'Anlamsal Arama Sonuçları',
669+
cs: 'Výsledky sémantického hledání',
670+
hu: 'Szemantikus keresési eredmények',
671+
bg: 'Резултати от семантично търсене'
672+
},
583673
'feature.search-files.prompt.intelligent': {
584674
en: 'Enter natural language search query.',
585675
pl: 'Wprowadź zapytanie wyszukiwania w języku naturalnym.',

‎apps/editor/src/views/settings/backend/message-handlers/handle-get-limit-semantic-search-results.ts‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
import * as vscode from 'vscode'
2+
import { SettingsProvider } from '@/views/settings/backend/settings-provider'
3+
import { LIMIT_SEMANTIC_SEARCH_RESULTS } from '@/constants/values'
4+
5+
export const handle_get_limit_semantic_search_results = async (
6+
provider: SettingsProvider
7+
): Promise<void> => {
8+
const config = vscode.workspace.getConfiguration('codeWebChat')
9+
const limit =
10+
config.get<number>('limitSemanticSearchResults') ||
11+
LIMIT_SEMANTIC_SEARCH_RESULTS
12+
provider.postMessage({
13+
command: 'LIMIT_SEMANTIC_SEARCH_RESULTS',
14+
limit
15+
})
16+
}

0 commit comments

Comments
 (0)