From 402f0a515b344983404c3d8b99216b91c62758dd Mon Sep 17 00:00:00 2001 From: Franciszek Stachura Date: Tue, 18 Jun 2024 14:42:20 +0200 Subject: [PATCH 1/5] Add a list of tokens that are always indexed Add a list of always indexed prefixes Currently, the list consists of arbitrarily selected tokens from c_common_reswords list present in GCC source code. https://gcc.gnu.org/git/?p=gcc.git;a=blob;f=gcc/c-family/c-common.cc;h=0341c44a2cd99771c3eeb44878d3df7a9ae816ea;hb=HEAD#l385 --- elixir/lib.py | 5 ++ elixir/special_tokens.py | 189 +++++++++++++++++++++++++++++++++++++++ update.py | 14 +-- 3 files changed, 203 insertions(+), 5 deletions(-) create mode 100644 elixir/special_tokens.py diff --git a/elixir/lib.py b/elixir/lib.py index 8fcf2be3..89faeaf5 100755 --- a/elixir/lib.py +++ b/elixir/lib.py @@ -21,6 +21,7 @@ import sys import logging import subprocess, os +from .special_tokens import always_indexed_tokens, always_indexed_prefixes logger = logging.getLogger(__name__) @@ -181,6 +182,10 @@ def isIdent(bstr): else: return True +def isAlwaysIndexed(token): + return token in always_indexed_tokens or \ + any(token.startswith(pref) for pref in always_indexed_prefixes) + def autoBytes(arg): if type(arg) is str: arg = arg.encode() diff --git a/elixir/special_tokens.py b/elixir/special_tokens.py new file mode 100644 index 00000000..09bbc547 --- /dev/null +++ b/elixir/special_tokens.py @@ -0,0 +1,189 @@ +# list of tokens, that are indexed as references even if no definitions are known + +always_indexed_tokens = set([ +# gcc/c-family/c-common.cc + b'_Alignas', + b'_Alignof', + b'_Atomic', + b'_BitInt', + b'_Bool', + b'_Complex', + b'_Imaginary', + b'_Float16', + b'_Float32', + b'_Float64', + b'_Float128', + b'_Float32x', + b'_Float64x', + b'_Float128x', + b'_Decimal32', + b'_Decimal64', + b'_Decimal128', + b'_Fract', + b'_Accum', + b'_Sat', + b'_Static_assert', + b'_Noreturn', + b'_Generic', + b'_Thread_local', + b'__FUNCTION__', + b'__PRETTY_FUNCTION__', + b'__alignof', + b'__alignof__', + b'__asm', + b'__asm__', + b'__attribute', + b'__attribute__', + b'__auto_type', + b'__complex', + b'__complex__', + b'__const', + b'__const__', + b'__constinit', + b'__decltype', + b'__extension__', + b'__func__', + b'__imag', + b'__imag__', + b'__inline', + b'__inline__', + b'__label__', + b'__null', + b'__real', + b'__real__', + b'__restrict', + b'__restrict__', + b'__signed', + b'__signed__', + b'__thread', + b'__transaction_atomic', + b'__transaction_relaxed', + b'__transaction_cancel', + b'__typeof', + b'__typeof__', + b'__typeof_unqual', + b'__typeof_unqual__', + b'__volatile', + b'__volatile__', + b'__GIMPLE', + b'__PHI', + b'__RTL', + b'alignas', + b'alignof', + b'asm', + b'auto', + b'thread_local', + b'sizeof', + +# https://gcc.gnu.org/onlinedocs/gcc/_005f_005fint128.html + b'__int128', + +# https://gcc.gnu.org/onlinedocs/gcc/Floating-Types.html + b'__float80', + b'__ibm128', + +# https://gcc.gnu.org/onlinedocs/gcc/Half-Precision.html + b'__fp16', + +# https://gcc.gnu.org/onlinedocs/gcc/Named-Address-Spaces.html + b'__flash', + b'__flash1', + b'__flash2', + b'__flash3', + b'__flash4', + b'__flash5', + b'__memx', + b'__far', + b'__regio_symbol', + b'__seg_fs', + b'__seg_gs', + +# https://clang.llvm.org/docs/LanguageExtensions.html#feature-checking-macros + b'__is_identifier', + +# https://clang.llvm.org/docs/LanguageExtensions.html#include-file-checking-macros + b'__BASE_FILE__', + b'__FILE_NAME__', + b'__COUNTER__', + b'__INCLUDE_LEVEL__', + b'__TIMESTAMP__', + b'__clang__', + b'__clang_major__', + b'__clang_minor__', + b'__clang_patchlevel__', + b'__clang_version__', + b'__clang_literal_encoding__', + b'__clang_wide_literal_encoding__', + +# https://clang.llvm.org/docs/LanguageExtensions.html#include-file-checking-macros + b'__datasizeof', + +# https://clang.llvm.org/docs/LanguageExtensions.html#vectors-and-extended-vectors + b'__bf16', + +# https://clang.llvm.org/docs/LanguageExtensions.html#type-trait-primitives + b'__array_rank', + b'__array_extent', + b'__can_pass_in_regs', + b'__reference_binds_to_temporary', + b'__reference_constructs_from_temporary', + b'__reference_converts_from_temporary', + b'__underlying_type', + +# https://clang.llvm.org/docs/LanguageExtensions.html#opencl-features + b'__remove_address_space', + +# https://clang.llvm.org/docs/LanguageExtensions.html#source-location-builtins + b'__LINE__', + b'__FUNCSIG__', + b'__FILE__', + +# https://clang.llvm.org/docs/LanguageExtensions.html#arm-aarch64-language-extensions + b'__dmb', + b'__dsb', + b'__isb', + +# https://en.cppreference.com/w/c/language/attributes + b'deprecated', + b'fallthrough', + b'maybe_unused', + b'nodiscard', + b'noreturn', + b'_Noreturn', + b'unsequenced', + b'reproducible' + +# https://en.cppreference.com/w/cpp/language/attributes + b'noreturn', + b'carries_dependency', + b'likely', + b'unlikely', + b'no_unique_address', + b'assume', + b'indeterminate', + b'optimize_for_synchronized', +]) + +always_indexed_prefixes = ( + b'__builtin', + +# https://gcc.gnu.org/onlinedocs/gcc/_005f_005fsync-Builtins.html + b'__sync', + +# https://gcc.gnu.org/onlinedocs/gcc/_005f_005fatomic-Builtins.html + b'__atomic', + b'__ATOMIC', + +# https://clang.llvm.org/docs/LanguageExtensions.html#feature-checking-macros +# https://clang.llvm.org/docs/LanguageExtensions.html#include-file-checking-macros + b'__has', + +# https://clang.llvm.org/docs/LanguageExtensions.html#language-extensions-back-ported-to-previous-standards + b'__cpp', + +# https://clang.llvm.org/docs/LanguageExtensions.html#type-trait-primitives + b'__is', + +# https://clang.llvm.org/docs/LanguageExtensions.html#opencl-features + b'__cl', +) diff --git a/update.py b/update.py index 9d84ff31..9fbe1961 100755 --- a/update.py +++ b/update.py @@ -326,11 +326,15 @@ def update_references(self, idxes): if even: tok = prefix + tok - if (db.defs.exists(tok) and - not ( (idx*idx_key_mod + line_num) in defs_idxes and - defs_idxes[idx*idx_key_mod + line_num] == tok ) and - (family != 'M' or tok.startswith(b'CONFIG_'))): - # We only index CONFIG_??? in makefiles + ref_allowed = \ + db.defs.exists(tok) or \ + lib.isAlwaysIndexed(tok) + + # We only index CONFIG_??? in makefiles + config_or_not_makefile = family != 'M' or tok.startswith(b'CONFIG_') + i = idx*idx_key_mod + line_num + + if ref_allowed and defs_idxes.get(i) != tok and config_or_not_makefile: if tok in idents: idents[tok] += ',' + str(line_num) else: From 18941b97b89d0554207b6f6d37420e1495e5254a Mon Sep 17 00:00:00 2001 From: Franciszek Stachura Date: Sat, 29 Aug 2026 20:59:52 +0200 Subject: [PATCH 2/5] query: display references for tokens without a definition Tokens from always-indexed list do not have a definition. --- elixir/query.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/elixir/query.py b/elixir/query.py index e3e4ffd3..c35ec5d3 100755 --- a/elixir/query.py +++ b/elixir/query.py @@ -264,7 +264,7 @@ def get_idents_defs(self, version, ident, family): symbol_references = [] symbol_doccomments = [] - if not self.db.defs.exists(ident): + if not self.db.defs.exists(ident) and not self.db.refs.exists(ident): return symbol_definitions, symbol_references, symbol_doccomments, False if not self.db.vers.exists(version): @@ -272,8 +272,13 @@ def get_idents_defs(self, version, ident, family): files_this_version = self.db.vers.get(version).iter() this_ident = self.db.defs.get(ident) - defs_this_ident = this_ident.iter(dummy=True) - macros_this_ident = this_ident.get_macros() + if this_ident is not None: + defs_this_ident = this_ident.iter(dummy=True) + macros_this_ident = this_ident.get_macros() + else: + defs_this_ident = data.DefList().iter(dummy=True) + macros_this_ident = '' + # FIXME: see why we can have a discrepancy between defs_this_ident and refs if self.db.refs.exists(ident): refs = self.db.refs.get(ident).iter(dummy=True) From db96bb54708b8f3da0c5c64d6ccf8193763c19af Mon Sep 17 00:00:00 2001 From: Franciszek Stachura Date: Sat, 29 Aug 2026 21:03:07 +0200 Subject: [PATCH 3/5] query: highlight tokens from always indexed list --- elixir/query.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/elixir/query.py b/elixir/query.py index c35ec5d3..5d7537c9 100755 --- a/elixir/query.py +++ b/elixir/query.py @@ -123,7 +123,11 @@ def get_tokenized_file(self, version, path): for tok in tokens: even = not even tok2 = prefix + tok - if even and self.db.defs_cache[family].exists(tok2): + known_token = \ + self.db.defs_cache[family].exists(tok2) or \ + lib.isAlwaysIndexed(tok2) + + if even and known_token: tok = b'\033[31m' + tok2 + b'\033[0m' else: tok = lib.unescape(tok) From 040f75c29d18548792ffce717b960323c6f2a8a2 Mon Sep 17 00:00:00 2001 From: Franciszek Stachura Date: Sat, 29 Aug 2026 23:40:14 +0200 Subject: [PATCH 4/5] autocomplete: handle always indexed tokens Always indexed tokens/prefixes are not added to the definitions database. Autocomplete only queries definitions database. Make autocomplete also query references database and return an union of the results. --- elixir/autocomplete.py | 66 +++++++++++++++++++++++------------------- 1 file changed, 37 insertions(+), 29 deletions(-) diff --git a/elixir/autocomplete.py b/elixir/autocomplete.py index 6ce34078..bf33be66 100755 --- a/elixir/autocomplete.py +++ b/elixir/autocomplete.py @@ -21,13 +21,40 @@ import os import json from urllib import parse -from berkeleydb.db import DB_SET_RANGE +from berkeleydb.db import DB_SET_RANGE, DB import falcon from .lib import autoBytes, validFamily from .query import get_query from .web_utils import validate_project, validate_ident +def get_top_keys_with_prefix(db: DB, prefix: str, k: int): + cur = db.cursor() + i = 0 + query_bytes = autoBytes(parse.quote(prefix)) + keys = [] + + # Find "the smallest key greater than or equal to the specified key" + # https://docs.oracle.com/cd/E17276_01/html/api_reference/C/dbcget.html + # In practice this should mean "the key that starts with provided prefix" + # See docs about the default comparison function for B-Tree databases: + # https://docs.oracle.com/cd/E17276_01/html/api_reference/C/dbset_bt_compare.html + result = cur.get(query_bytes, DB_SET_RANGE) + while result is not None and i < k: + key, _ = result + if key.startswith(query_bytes): + # If found key starts with the prefix, add to response + # and move to the next key + i += 1 + keys.append(key.decode("utf-8")) + result = cur.next() + else: + # If found key does not start with the prefix, stop + break + + return keys + + class AutocompleteResource: def on_get(self, req, resp): ident_prefix = req.get_param('q') @@ -52,38 +79,19 @@ def on_get(self, req, resp): if family == 'B': # DTS identifiers are stored quoted - process = lambda x: parse.unquote(x) - db = query.db.comps + result = [ + parse.unquote(k) + for k + in get_top_keys_with_prefix(query.db.comps.db, ident_prefix, 10) + ] else: - process = lambda x: x - db = query.db.defs - - response = [] - - i = 0 - cur = db.db.cursor() - query_bytes = autoBytes(parse.quote(ident_prefix)) - # Find "the smallest key greater than or equal to the specified key" - # https://docs.oracle.com/cd/E17276_01/html/api_reference/C/dbcget.html - # In practice this should mean "the key that starts with provided prefix" - # See docs about the default comparison function for B-Tree databases: - # https://docs.oracle.com/cd/E17276_01/html/api_reference/C/dbset_bt_compare.html - result = cur.get(query_bytes, DB_SET_RANGE) - while result is not None and i < 10: - key, _ = result - if key.startswith(query_bytes): - # If found key starts with the prefix, add to response - # and move to the next key - i += 1 - response.append(process(key.decode("utf-8"))) - result = cur.next() - else: - # If found key does not start with the prefix, stop - break + result_defs = get_top_keys_with_prefix(query.db.defs.db, ident_prefix, 10) + result_refs = get_top_keys_with_prefix(query.db.refs.db, ident_prefix, 10) + result = sorted(set(result_defs).union(result_refs))[:10] resp.status = falcon.HTTP_200 resp.content_type = falcon.MEDIA_JSON - resp.media = response + resp.media = result query.close() From 923e68a4086c0b525a86162ee7ce8403fe15d0d9 Mon Sep 17 00:00:00 2001 From: Franciszek Stachura Date: Sat, 29 Aug 2026 23:43:26 +0200 Subject: [PATCH 5/5] lib: move token blacklist to special_tokens.py --- elixir/lib.py | 111 +------------------------------------- elixir/special_tokens.py | 113 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 114 insertions(+), 110 deletions(-) diff --git a/elixir/lib.py b/elixir/lib.py index 89faeaf5..5503a259 100755 --- a/elixir/lib.py +++ b/elixir/lib.py @@ -21,7 +21,7 @@ import sys import logging import subprocess, os -from .special_tokens import always_indexed_tokens, always_indexed_prefixes +from .special_tokens import always_indexed_tokens, always_indexed_prefixes, blacklist logger = logging.getLogger(__name__) @@ -65,115 +65,6 @@ def decode(byte_object): except UnicodeDecodeError: return byte_object.decode('iso-8859-1') -# List of tokens which we don't want to consider as identifiers -# Typically for very frequent variable names and things redefined by #define -# TODO: allow to have per project blacklists - -blacklist = ( - b'NULL', - b'__', - b'adapter', - b'addr', - b'arg', - b'attr', - b'base', - b'bp', - b'buf', - b'buffer', - b'c', - b'card', - b'char', - b'chip', - b'cmd', - b'codec', - b'const', - b'count', - b'cpu', - b'ctx', - b'data', - b'default', - b'define', - b'desc', - b'dev', - b'driver', - b'else', - b'end', - b'endif', - b'entry', - b'err', - b'error', - b'event', - b'extern', - b'failed', - b'flags', - b'h', - b'host', - b'hw', - b'i', - b'id', - b'idx', - b'if', - b'index', - b'info', - b'inline', - b'int', - b'irq', - b'j', - b'len', - b'length', - b'list', - b'lock', - b'long', - b'mask', - b'mode', - b'msg', - b'n', - b'name', - b'net', - b'next', - b'offset', - b'ops', - b'out', - b'p', - b'pdev', - b'port', - b'priv', - b'ptr', - b'q', - b'r', - b'rc', - b'rdev', - b'reg', - b'regs', - b'req', - b'res', - b'result', - b'ret', - b'return', - b'retval', - b'root', - b's', - b'sb', - b'size', - b'sizeof', - b'sk', - b'skb', - b'spec', - b'start', - b'state', - b'static', - b'status', - b'struct', - b't', - b'tmp', - b'tp', - b'type', - b'val', - b'value', - b'vcpu', - b'x' -) - def isIdent(bstr): if (len(bstr) < 2 or bstr in blacklist or diff --git a/elixir/special_tokens.py b/elixir/special_tokens.py index 09bbc547..b6837db3 100644 --- a/elixir/special_tokens.py +++ b/elixir/special_tokens.py @@ -164,6 +164,8 @@ b'optimize_for_synchronized', ]) + + always_indexed_prefixes = ( b'__builtin', @@ -187,3 +189,114 @@ # https://clang.llvm.org/docs/LanguageExtensions.html#opencl-features b'__cl', ) + + + +# List of tokens which we don't want to consider as identifiers +# Typically for very frequent variable names and things redefined by #define +# TODO: allow to have per project blacklists + +blacklist = ( + b'NULL', + b'__', + b'adapter', + b'addr', + b'arg', + b'attr', + b'base', + b'bp', + b'buf', + b'buffer', + b'c', + b'card', + b'char', + b'chip', + b'cmd', + b'codec', + b'const', + b'count', + b'cpu', + b'ctx', + b'data', + b'default', + b'define', + b'desc', + b'dev', + b'driver', + b'else', + b'end', + b'endif', + b'entry', + b'err', + b'error', + b'event', + b'extern', + b'failed', + b'flags', + b'h', + b'host', + b'hw', + b'i', + b'id', + b'idx', + b'if', + b'index', + b'info', + b'inline', + b'int', + b'irq', + b'j', + b'len', + b'length', + b'list', + b'lock', + b'long', + b'mask', + b'mode', + b'msg', + b'n', + b'name', + b'net', + b'next', + b'offset', + b'ops', + b'out', + b'p', + b'pdev', + b'port', + b'priv', + b'ptr', + b'q', + b'r', + b'rc', + b'rdev', + b'reg', + b'regs', + b'req', + b'res', + b'result', + b'ret', + b'return', + b'retval', + b'root', + b's', + b'sb', + b'size', + b'sizeof', + b'sk', + b'skb', + b'spec', + b'start', + b'state', + b'static', + b'status', + b'struct', + b't', + b'tmp', + b'tp', + b'type', + b'val', + b'value', + b'vcpu', + b'x' +)