From 7279ad4500d625b6568633bb6c67f16ac6fb491e Mon Sep 17 00:00:00 2001
From: Eduardo Muniz Alves <82589615+devdudumuniz@users.noreply.github.com>
Date: Sat, 29 Aug 2026 22:23:22 -0300
Subject: [PATCH] perf(dbf): reduce per-value decoding overhead

---
 pysus/data/dbf_reader.py | 25 +++++++++++++------------
 1 file changed, 13 insertions(+), 12 deletions(-)

diff --git a/pysus/data/dbf_reader.py b/pysus/data/dbf_reader.py
index 832bd7a8..dbd44093 100644
--- a/pysus/data/dbf_reader.py
+++ b/pysus/data/dbf_reader.py
@@ -100,6 +100,17 @@ def _decode(val: bytes) -> str:
     return val.decode(_ENCODING, errors="replace").replace("\x00", "").strip()
 
 
+def _decode_column(column: np.ndarray) -> list[str]:
+    """Decode a NumPy byte-string column without per-item function calls."""
+    return [
+        (value if isinstance(value, bytes) else value.tobytes())
+        .decode(_ENCODING, errors="replace")
+        .replace("\x00", "")
+        .strip()
+        for value in column
+    ]
+
+
 def read_dbf_schema(path: str | Path) -> DBFSchema:
     """Return the schema of a DBF file without reading records."""
     return _parse_header(path)
@@ -158,12 +169,7 @@ def read_dbf_fast(
     data = {}
     for fld in target:
         col: np.ndarray = records[fld.name]
-        decoded = np.empty(n, dtype=object)
-        for i in range(n):
-            val = col[i]
-            b = val if isinstance(val, bytes) else val.tobytes()
-            decoded[i] = _decode(b)
-        data[fld.name] = decoded
+        data[fld.name] = _decode_column(col)
 
     return pd.DataFrame(data)
 
@@ -271,12 +277,7 @@ def stream_dbf_fast(
             data = {}
             for fld in schema.fields:
                 col: np.ndarray = records[fld.name]
-                decoded = np.empty(chunk_n, dtype=object)
-                for i in range(chunk_n):
-                    val = col[i]
-                    b = val if isinstance(val, bytes) else val.tobytes()
-                    decoded[i] = _decode(b)
-                data[fld.name] = decoded
+                data[fld.name] = _decode_column(col)
 
             yield pd.DataFrame(data)