Implement code compaction utility for C-style and Python files to strip comments and function bodies

Author: robertpiosik

packages/vscode/src/context/utils/compact-file/compact-file.spec.ts

@@ -0,0 +1,84 @@
+import { compact_file } from './compact-file'
+import * as fs from 'fs'
+import * as path from 'path'
+
+describe('compact_file', () => {
+  const load_test_case = (
+    test_case: string,
+    directory = 'c-style'
+  ): { input: string; expected: string } => {
+    const case_path = path.join(__dirname, 'test-cases', directory, test_case)
+    const input = fs.readFileSync(path.join(case_path, 'input.txt'), 'utf-8')
+    const expected = fs.readFileSync(
+      path.join(case_path, 'output.txt'),
+      'utf-8'
+    )
+    return { input, expected }
+  }
+
+  it('should remove single line comments', () => {
+    const { input, expected } = load_test_case('remove-single-line-comments')
+    expect(compact_file(input, '.ts')).toBe(expected)
+  })
+
+  it('should remove block comments on a single line', () => {
+    const { input, expected } = load_test_case(
+      'remove-block-comments-single-line'
+    )
+    expect(compact_file(input, '.ts')).toBe(expected)
+  })
+
+  it('should remove multi-line block comments', () => {
+    const { input, expected } = load_test_case(
+      'remove-multi-line-block-comments'
+    )
+    expect(compact_file(input, '.ts')).toBe(expected)
+  })
+
+  it('should preserve comments inside double quotes', () => {
+    const { input, expected } = load_test_case('preserve-double-quotes')
+    expect(compact_file(input, '.ts')).toBe(expected)
+  })
+
+  it('should preserve comments inside single quotes', () => {
+    const { input, expected } = load_test_case('preserve-single-quotes')
+    expect(compact_file(input, '.ts')).toBe(expected)
+  })
+
+  it('should preserve comments inside template literals', () => {
+    const { input, expected } = load_test_case('preserve-template-literals')
+    expect(compact_file(input, '.ts')).toBe(expected)
+  })
+
+  it('should preserve block comments inside strings', () => {
+    const { input, expected } = load_test_case(
+      'preserve-block-comments-in-strings'
+    )
+    expect(compact_file(input, '.ts')).toBe(expected)
+  })
+
+  it('should handle escaped quotes inside strings', () => {
+    const { input, expected } = load_test_case('handle-escaped-quotes')
+    expect(compact_file(input, '.ts')).toBe(expected)
+  })
+
+  it('should handle complex mixed content', () => {
+    const { input, expected } = load_test_case('complex-mixed-content')
+    expect(compact_file(input, '.ts')).toBe(expected)
+  })
+
+  it('should handle input with only comments', () => {
+    const { input, expected } = load_test_case('only-comments')
+    expect(compact_file(input, '.ts')).toBe(expected)
+  })
+
+  it('should strip function bodies when enabled', () => {
+    const { input, expected } = load_test_case('javascript-bodies')
+    expect(compact_file(input, '.ts')).toBe(expected)
+  })
+
+  it('should strip python function bodies', () => {
+    const { input, expected } = load_test_case('python', '.')
+    expect(compact_file(input, '.py')).toBe(expected)
+  })
+})

packages/vscode/src/context/utils/compact-file/compact-file.ts

@@ -0,0 +1,10 @@
+import { compact_c_style } from './languages/c-style'
+import { compact_python } from './languages/python'
+
+export const compact_file = (content: string, extension: string): string => {
+  if (extension == '.py') {
+    return compact_python(content)
+  }
+
+  return compact_c_style(content, true)
+}

packages/vscode/src/context/utils/compact-file/index.ts

@@ -0,0 +1 @@
+export * from './compact-file'

packages/vscode/src/context/utils/compact-file/languages/c-style.ts

@@ -0,0 +1,142 @@
+export const compact_c_style = (
+  content: string,
+  should_strip_bodies: boolean
+): string => {
+  // Split by newline to process line by line, handling both LF and CRLF
+  const lines = content.split(/\r?\n/)
+  const result: string[] = []
+  let is_in_block_comment = false
+  let skip_body_depth = 0
+  // Buffer to track context for stripping bodies (e.g. "class Foo ")
+  let last_code_buffer = ''
+
+  for (const line of lines) {
+    let processed_line = ''
+    let i = 0
+    let is_in_string: false | '"' | "'" | '`' = false
+
+    while (i < line.length) {
+      const char = line[i]
+      const next_char = line[i + 1]
+
+      // Handle Block Comments
+      if (is_in_block_comment) {
+        if (char === '*' && next_char === '/') {
+          is_in_block_comment = false
+          i += 2
+        } else {
+          i++
+        }
+        continue
+      }
+
+      // Handle Strings (ignore comments inside strings)
+      if (is_in_string) {
+        if (char == '\\') {
+          // Handle escaped characters inside string
+          if (skip_body_depth === 0) {
+            processed_line += char + (next_char || '')
+          }
+          last_code_buffer += char + (next_char || '')
+          i += 2
+          continue
+        }
+        if (char === is_in_string) {
+          is_in_string = false
+        }
+        if (skip_body_depth === 0) {
+          processed_line += char
+        }
+        last_code_buffer += char
+        i++
+        continue
+      }
+
+      // Start of String
+      if (char === '"' || char === "'" || char === '`') {
+        is_in_string = char
+        if (skip_body_depth === 0) {
+          processed_line += char
+        }
+        last_code_buffer += char
+        i++
+        continue
+      }
+
+      // Start of Line Comment
+      if (char == '/' && next_char == '/') {
+        break // Ignore the rest of the line
+      }
+
+      // Start of Block Comment
+      if (char == '/' && next_char == '*') {
+        is_in_block_comment = true
+        i += 2
+        continue
+      }
+
+      // Handle Function Body Stripping
+      if (should_strip_bodies && char == '{') {
+        if (skip_body_depth > 0) {
+          // Already skipping, just deepen the level
+          skip_body_depth++
+          last_code_buffer += char
+          i++
+          continue
+        } else {
+          // Check if we should preserve this block (class, interface, etc.)
+          // We look at last_code_buffer for keywords
+          const is_keeper =
+            /(^|\s)(class|interface|enum|namespace|type)(\s|$)/.test(
+              last_code_buffer
+            )
+          // Also preserve object literals (preceded by = or :)
+          const is_object_literal = /(=|:)\s*$/.test(last_code_buffer.trimEnd())
+
+          if (!is_keeper && !is_object_literal) {
+            // Start skipping
+            skip_body_depth = 1
+            const indent = processed_line.match(/^\s*/)?.[0] || ''
+            processed_line += `{\n${indent}  // ...\n${indent}}`
+            last_code_buffer += '{}'
+            i++
+            continue
+          } else {
+            // Keeper: reset buffer to avoid false positives in nested scopes
+            last_code_buffer = ''
+          }
+        }
+      } else if (should_strip_bodies && char == '}') {
+        if (skip_body_depth > 0) {
+          skip_body_depth--
+          last_code_buffer += char
+          i++
+          continue
+        }
+      }
+
+      // Regular character
+      if (skip_body_depth == 0) {
+        processed_line += char
+      }
+      last_code_buffer += char
+      i++
+    }
+
+    // We trim the line to actually "compact" the file.
+    // If a line contained only comments, it becomes empty and is skipped.
+    const trimmed = processed_line.trim()
+    if (trimmed) {
+      result.push(trimmed)
+    }
+
+    // Add a space to buffer for newline to avoid concatenation issues (e.g. "class Foo" + "{")
+    last_code_buffer += ' '
+    // Limit buffer size to avoid memory issues, keeping enough for context
+    if (last_code_buffer.length > 200) {
+      last_code_buffer = last_code_buffer.slice(-200)
+    }
+  }
+
+  return result.length > 0 ? result.join('\n') + '\n' : ''
+}

packages/vscode/src/context/utils/compact-file/languages/python.ts

@@ -0,0 +1,83 @@
+export const compact_python = (content: string): string => {
+  const lines = content.split(/\r?\n/)
+  const result: string[] = []
+  let skip_indent_threshold = -1
+
+  for (const line of lines) {
+    // 1. Calculate Indentation
+    const match = line.match(/^\s*/)
+    const current_indent = match ? match[0].length : 0
+
+    // 2. Handle Skipping (Body Stripping)
+    if (skip_indent_threshold !== -1) {
+      if (current_indent > skip_indent_threshold) {
+        continue
+      }
+      skip_indent_threshold = -1
+    }
+
+    // 3. Strip Comments (simplified char loop to handle strings)
+    let processed_line = ''
+    let in_string: string | false = false
+    let i = 0
+
+    while (i < line.length) {
+      const char = line[i]
+
+      if (in_string) {
+        if (char === '\\') {
+          processed_line += char + (line[i + 1] || '')
+          i += 2
+          continue
+        }
+        // Check for end of string (handling triple quotes)
+        if (
+          char === in_string[0] &&
+          (in_string.length === 1 ||
+            line.substring(i, i + in_string.length) === in_string)
+        ) {
+          processed_line += in_string
+          i += in_string.length
+          in_string = false
+          continue
+        }
+        processed_line += char
+        i++
+      } else {
+        // Start of String
+        if (char === '"' || char === "'") {
+          const is_triple = line.substring(i, i + 3) === char.repeat(3)
+          in_string = is_triple ? char.repeat(3) : char
+          processed_line += in_string
+          i += in_string.length
+          continue
+        }
+        // Start of Comment
+        if (char === '#') {
+          break // Ignore rest of line
+        }
+        processed_line += char
+        i++
+      }
+    }
+
+    const trimmed = processed_line.trim()
+    if (!trimmed) continue
+
+    // 4. Check for Block Start to Strip
+    if (trimmed.endsWith(':')) {
+      // Keep 'class', strip others (def, if, while, etc.)
+      const is_keeper = /^class\s/.test(trimmed)
+      if (!is_keeper) {
+        result.push(processed_line)
+        result.push(line.substring(0, current_indent) + '    ' + '# ...')
+        skip_indent_threshold = current_indent
+        continue
+      }
+    }
+
+    result.push(processed_line)
+  }
+
+  return result.join('\n') + '\n'
+}

packages/vscode/src/context/utils/compact-file/test-cases/c-style/complex-mixed-content/input.txt

@@ -0,0 +1,6 @@
+function test() {
+  const url = "http://example.com"; // URL with //
+  /* Start block
+  End block */
+  return `Result: ${url}`;
+}

packages/vscode/src/context/utils/compact-file/test-cases/c-style/complex-mixed-content/output.txt

@@ -0,0 +1,3 @@
+function test() {
+  // ...
+}

packages/vscode/src/context/utils/compact-file/test-cases/c-style/handle-escaped-quotes/input.txt

@@ -0,0 +1 @@
+const str = "String with \"// quote"; // real comment

packages/vscode/src/context/utils/compact-file/test-cases/c-style/handle-escaped-quotes/output.txt

@@ -0,0 +1 @@
+const str = "String with \"// quote";

packages/vscode/src/context/utils/compact-file/test-cases/c-style/javascript-bodies/input.txt

@@ -0,0 +1,27 @@
+function add(a, b) {
+  return a + b;
+}
+
+class Calculator {
+  constructor() {
+    this.value = 0;
+  }
+
+  add(n) {
+    this.value += n;
+  }
+}
+
+interface Item {
+  id: string;
+  name: string;
+}
+
+const double = (x) => {
+  return x * 2;
+}
+
+const config = {
+  env: "production",
+  debug: false
+};