Add a list of tokens that are always indexed

elixir/autocomplete.py

@@ -21,13 +21,40 @@
 import os
 import json
 from urllib import parse
-from berkeleydb.db import DB_SET_RANGE
+from berkeleydb.db import DB_SET_RANGE, DB
 import falcon
 
 from .lib import autoBytes, validFamily
 from .query import get_query
 from .web_utils import validate_project, validate_ident
 
+def get_top_keys_with_prefix(db: DB, prefix: str, k: int):
+    cur = db.cursor()
+    i = 0
+    query_bytes = autoBytes(parse.quote(prefix))
+    keys = []
+
+    # Find "the smallest key greater than or equal to the specified key"
+    # https://docs.oracle.com/cd/E17276_01/html/api_reference/C/dbcget.html
+    # In practice this should mean "the key that starts with provided prefix"
+    # See docs about the default comparison function for B-Tree databases:
+    # https://docs.oracle.com/cd/E17276_01/html/api_reference/C/dbset_bt_compare.html
+    result = cur.get(query_bytes, DB_SET_RANGE)
+    while result is not None and i < k:
+        key, _ = result
+        if key.startswith(query_bytes):
+            # If found key starts with the prefix, add to response
+            # and move to the next key
+            i += 1
+            keys.append(key.decode("utf-8"))
+            result = cur.next()
+        else:
+            # If found key does not start with the prefix, stop
+            break
+
+    return keys
+
+
 class AutocompleteResource:
     def on_get(self, req, resp):
         ident_prefix = req.get_param('q')
@@ -52,38 +79,19 @@ def on_get(self, req, resp):
 
         if family == 'B':
             # DTS identifiers are stored quoted
-            process = lambda x: parse.unquote(x)
-            db = query.db.comps
+            result = [
+                parse.unquote(k)
+                for k
+                in get_top_keys_with_prefix(query.db.comps.db, ident_prefix, 10)
+            ]
         else:
-            process = lambda x: x
-            db = query.db.defs
-
-        response = []
-
-        i = 0
-        cur = db.db.cursor()
-        query_bytes = autoBytes(parse.quote(ident_prefix))
-        # Find "the smallest key greater than or equal to the specified key"
-        # https://docs.oracle.com/cd/E17276_01/html/api_reference/C/dbcget.html
-        # In practice this should mean "the key that starts with provided prefix"
-        # See docs about the default comparison function for B-Tree databases:
-        # https://docs.oracle.com/cd/E17276_01/html/api_reference/C/dbset_bt_compare.html
-        result = cur.get(query_bytes, DB_SET_RANGE)
-        while result is not None and i < 10:
-            key, _ = result
-            if key.startswith(query_bytes):
-                # If found key starts with the prefix, add to response
-                # and move to the next key
-                i += 1
-                response.append(process(key.decode("utf-8")))
-                result = cur.next()
-            else:
-                # If found key does not start with the prefix, stop
-                break
+            result_defs = get_top_keys_with_prefix(query.db.defs.db, ident_prefix, 10)
+            result_refs = get_top_keys_with_prefix(query.db.refs.db, ident_prefix, 10)
+            result = sorted(set(result_defs).union(result_refs))[:10]
 
         resp.status = falcon.HTTP_200
         resp.content_type = falcon.MEDIA_JSON
-        resp.media = response
+        resp.media = result
 
         query.close()
 

elixir/lib.py

@@ -21,6 +21,7 @@
 import sys
 import logging
 import subprocess, os
+from .special_tokens import always_indexed_tokens, always_indexed_prefixes, blacklist
 
 logger = logging.getLogger(__name__)
 
@@ -64,115 +65,6 @@ def decode(byte_object):
     except UnicodeDecodeError:
         return byte_object.decode('iso-8859-1')
 
-# List of tokens which we don't want to consider as identifiers
-# Typically for very frequent variable names and things redefined by #define
-# TODO: allow to have per project blacklists
-
-blacklist = (
-    b'NULL',
-    b'__',
-    b'adapter',
-    b'addr',
-    b'arg',
-    b'attr',
-    b'base',
-    b'bp',
-    b'buf',
-    b'buffer',
-    b'c',
-    b'card',
-    b'char',
-    b'chip',
-    b'cmd',
-    b'codec',
-    b'const',
-    b'count',
-    b'cpu',
-    b'ctx',
-    b'data',
-    b'default',
-    b'define',
-    b'desc',
-    b'dev',
-    b'driver',
-    b'else',
-    b'end',
-    b'endif',
-    b'entry',
-    b'err',
-    b'error',
-    b'event',
-    b'extern',
-    b'failed',
-    b'flags',
-    b'h',
-    b'host',
-    b'hw',
-    b'i',
-    b'id',
-    b'idx',
-    b'if',
-    b'index',
-    b'info',
-    b'inline',
-    b'int',
-    b'irq',
-    b'j',
-    b'len',
-    b'length',
-    b'list',
-    b'lock',
-    b'long',
-    b'mask',
-    b'mode',
-    b'msg',
-    b'n',
-    b'name',
-    b'net',
-    b'next',
-    b'offset',
-    b'ops',
-    b'out',
-    b'p',
-    b'pdev',
-    b'port',
-    b'priv',
-    b'ptr',
-    b'q',
-    b'r',
-    b'rc',
-    b'rdev',
-    b'reg',
-    b'regs',
-    b'req',
-    b'res',
-    b'result',
-    b'ret',
-    b'return',
-    b'retval',
-    b'root',
-    b's',
-    b'sb',
-    b'size',
-    b'sizeof',
-    b'sk',
-    b'skb',
-    b'spec',
-    b'start',
-    b'state',
-    b'static',
-    b'status',
-    b'struct',
-    b't',
-    b'tmp',
-    b'tp',
-    b'type',
-    b'val',
-    b'value',
-    b'vcpu',
-    b'x'
-)
-
 def isIdent(bstr):
     if (len(bstr) < 2 or
         bstr in blacklist or
@@ -181,6 +73,10 @@ def isIdent(bstr):
     else:
         return True
 
+def isAlwaysIndexed(token):
+    return token in always_indexed_tokens or \
+        any(token.startswith(pref) for pref in always_indexed_prefixes)
+
 def autoBytes(arg):
     if type(arg) is str:
         arg = arg.encode()

elixir/query.py

@@ -123,7 +123,11 @@ def get_tokenized_file(self, version, path):
             for tok in tokens:
                 even = not even
                 tok2 = prefix + tok
-                if even and self.db.defs_cache[family].exists(tok2):
+                known_token = \
+                    self.db.defs_cache[family].exists(tok2) or \
+                    lib.isAlwaysIndexed(tok2)
+
+                if even and known_token:
                     tok = b'\033[31m' + tok2 + b'\033[0m'
                 else:
                     tok = lib.unescape(tok)
@@ -264,16 +268,21 @@ def get_idents_defs(self, version, ident, family):
         symbol_references = []
         symbol_doccomments = []
 
-        if not self.db.defs.exists(ident):
+        if not self.db.defs.exists(ident) and not self.db.refs.exists(ident):
             return symbol_definitions, symbol_references, symbol_doccomments, False
 
         if not self.db.vers.exists(version):
             return symbol_definitions, symbol_references, symbol_doccomments, True
 
         files_this_version = self.db.vers.get(version).iter()
         this_ident = self.db.defs.get(ident)
-        defs_this_ident = this_ident.iter(dummy=True)
-        macros_this_ident = this_ident.get_macros()
+        if this_ident is not None:
+            defs_this_ident = this_ident.iter(dummy=True)
+            macros_this_ident = this_ident.get_macros()
+        else:
+            defs_this_ident = data.DefList().iter(dummy=True)
+            macros_this_ident = ''
+
         # FIXME: see why we can have a discrepancy between defs_this_ident and refs
         if self.db.refs.exists(ident):
             refs = self.db.refs.get(ident).iter(dummy=True)