diff --git a/.gitignore b/.gitignore index 48b007c..7c14077 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,6 @@ CMakeSettings.json scripts/*.exe scripts/appimagetool-x86_64.AppImage + +# generated test corpus (tests/gen_corpus.sh) +tests/data/ diff --git a/CMakeLists.txt b/CMakeLists.txt index 961217f..a4fbdb2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -60,6 +60,11 @@ if(UNIX AND NOT APPLE) set(LINUX TRUE) endif() +set(THREADS_PREFER_PTHREAD_FLAG ON) +find_package(Threads REQUIRED) + +option(TCRUNCHER_BUILD_TESTS "Build headless parser tests (no FLTK)" OFF) + string(TIMESTAMP NOW "%Y-%m-%dT%H:%M") string(TIMESTAMP TODAY "%Y-%m-%d") string(TIMESTAMP YEAR "%Y") @@ -100,6 +105,7 @@ set(SOURCES ${SRCDIR}/csvapplication.cpp ${SRCDIR}/csvdatastorage.cpp ${SRCDIR}/csvgrid.cpp + ${SRCDIR}/csvguess.cpp ${SRCDIR}/csvmenu.cpp ${SRCDIR}/csvparser.cpp ${SRCDIR}/csvtable.cpp @@ -108,6 +114,9 @@ set(SOURCES ${SRCDIR}/csvwindow.cpp ${SRCDIR}/helper.cpp ${SRCDIR}/macro.cpp + ${SRCDIR}/mappedfile.cpp + ${SRCDIR}/utf8validate.cpp + ${SRCDIR}/csvloader.cpp ${SRCDIR}/main.cpp ${EXTERNAL}/duktape/duktape.c ) @@ -152,11 +161,40 @@ elseif(LINUX) fontconfig Xrender X11 - pthread dl ) endif() +target_link_libraries(${PROJECT_NAME} PRIVATE Threads::Threads) + + +# +# Headless parser tests. Links WITHOUT FLTK, which is what keeps the data layer honest: +# if tc_parsecheck compiles, nothing in the parse path reaches into the UI layer. +# +if(TCRUNCHER_BUILD_TESTS) + add_executable(tc_parsecheck + tests/parsecheck.cpp + ${SRCDIR}/csvparser.cpp + ${SRCDIR}/csvdatastorage.cpp + ${SRCDIR}/helper.cpp + ${SRCDIR}/mappedfile.cpp + ${SRCDIR}/utf8validate.cpp + ${SRCDIR}/csvloader.cpp + ${SRCDIR}/csvguess.cpp + ) + # src/csvfsm.hh is header-only (templated) – no source entry + target_include_directories(tc_parsecheck PRIVATE ${SRCDIR} ${EXTERNAL}) + target_link_libraries(tc_parsecheck PRIVATE Threads::Threads) + if(APPLE) + # helper.cpp uses CFBundle* to locate the app bundle's resources + target_link_libraries(tc_parsecheck PRIVATE "-framework CoreFoundation") + endif() + set_target_properties(tc_parsecheck PROPERTIES + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}" + ) +endif() + file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/dist) diff --git a/src/csvapplication.cpp b/src/csvapplication.cpp index adb49bf..fb049fc 100644 --- a/src/csvapplication.cpp +++ b/src/csvapplication.cpp @@ -21,6 +21,7 @@ #include "csvapplication.hh" +#include "csvguess.hh" #include "csvmenu.hh" #include "icons/abouticon.xpm" @@ -175,6 +176,9 @@ CsvApplication::CsvApplication() { imWorkingButton = new Fl_Button(10,10,380,100,"Please stand by while I'm working ..."); imWorkingButton->box(FL_NO_BOX); imWorkingButton->visible_focus(0); + imWorkingCancelButton = new Fl_Button(150,78,100,26,"Cancel"); + imWorkingCancelButton->callback(imWorkingCancelCB, nullptr); + imWorkingCancelButton->hide(); imWorkingWindow->end(); imWorkingWindow->hide(); @@ -235,6 +239,7 @@ CsvApplication::~CsvApplication() { delete remainOpenWin; #endif + delete imWorkingCancelButton; delete imWorkingButton; delete imWorkingWindow; } @@ -533,8 +538,11 @@ void CsvApplication::openFile(std::string path, bool askUser) { windows[winIndex].readWindowPreferences(); windows[winIndex].grid->redraw(); Fl::check(); - // automatically arrange column widths + // Automatically arrange column widths. This used to run AFTER the progress window + // was gone, so on a wide table the user saw a frozen window with no indicator. + app.showImWorkingWindow("Arranging columns ...", true); app.arrangeColumnsCB(NULL, NULL); + app.hideImWorkingWindow(); // add file to recent files recentFiles.add(path); updateMenu(winIndex); @@ -867,140 +875,16 @@ int CsvApplication::getTopWindow() { /* * Guesses the definition of the given stream, returning some confidence value with it */ -std::pair CsvApplication::guessDefinition(std::istream *input) { - const int MAXLINES = 10; // number of lines to read for every test - float confidence; - CsvParser *parser = new CsvParser(); - CsvDataStorage localStorage; - - // Define definitions for probing - std::vector< std::tuple > definitions; - definitions.push_back( std::tuple(CsvDefinition(),0,0) ); - definitions.push_back( std::tuple(CsvDefinition(),0,0) ); - definitions.push_back( std::tuple(CsvDefinition(),0,0) ); - definitions.push_back( std::tuple(CsvDefinition(),0,0) ); - definitions.push_back( std::tuple(CsvDefinition(),0,0) ); - definitions.push_back( std::tuple(CsvDefinition(),0,0) ); - definitions.push_back( std::tuple(CsvDefinition(),0,0) ); - definitions.push_back( std::tuple(CsvDefinition(),0,0) ); - std::get<0>(definitions.at(0)).delimiter = ','; - std::get<0>(definitions.at(1)).delimiter = ';'; - std::get<0>(definitions.at(2)).delimiter = '\t'; - std::get<0>(definitions.at(3)).delimiter = '|'; - std::get<0>(definitions.at(4)).delimiter = ':'; - std::get<0>(definitions.at(5)).delimiter = ','; - std::get<0>(definitions.at(6)).delimiter = ';'; - std::get<0>(definitions.at(6)).delimiter = '*'; - std::get<0>(definitions.at(5)).escape = '\\'; - std::get<0>(definitions.at(6)).escape = '\\'; - - // get statistics for all probing definitions - for( size_t i = 0; i < definitions.size(); ++i ) { - std::pair statistics; - // reset stream - input->clear(); // WHY???? - input->seekg(0); - // clear localTable - localStorage.clear(); - parser->parseCsvStream( input, localStorage, &(std::get<0>(definitions.at(i))), MAXLINES, false ); - statistics = tableStatistics(localStorage); - // not so commonly used seperators and escape characters: decrease statistics value - if( - std::get<0>(definitions.at(i)).delimiter == ':' || - std::get<0>(definitions.at(i)).delimiter == '|' || - std::get<0>(definitions.at(i)).escape == '\\' || - std::get<0>(definitions.at(i)).escape == '*' - ) { - statistics.first = statistics.first * 70 / 100; - } - std::get<1>(definitions.at(i)) = statistics.first; - if( statistics.first <= 1 && statistics.second == 0) { - // if statistics is (1,0), sort it at the end - std::get<2>(definitions.at(i)) = 999; - } else { - std::get<2>(definitions.at(i)) = statistics.second; - } - #ifdef DEBUG - printf("CSV = '%c' => %d / %d\n", std::get<0>(definitions.at(i)).delimiter, statistics.first, statistics.second); - #endif - } - - // sort probes: 3rd element INC, 2nd element DESC (C++14) - std::sort(begin(definitions), end(definitions), [](auto &t1, auto &t2) { - if( std::get<2>(t1) == std::get<2>(t2) ) { - return std::get<1>(t1) > std::get<1>(t2); - } - return std::get<2>(t1) < std::get<2>(t2); - }); - - confidence = 1.0; - // if there's no definition with zero variance: reduce confidence - if( std::get<2>(definitions[0]) > 0 ) { - confidence /= 2; - } - // if there are at least two definitions with the same number of columns: reduce confidence - if( std::get<1>(definitions[0]) == std::get<1>(definitions[1]) ) { - confidence /= 2; - } - // improve confidence, if it's a typical CSV separator - if( std::get<0>(definitions.at(0)).delimiter == ',' || std::get<0>(definitions.at(0)).delimiter == '\t' ) { - confidence += (1.0 - confidence) * 0.5; - } - - // clear and reset - delete(parser); - input->clear(); - input->seekg(0); - return {std::get<0>(definitions.at(0)), confidence}; -} - - - -/* - * Returns {Number of Cols, Some kind of Variance} of the data in localStorage - * Variance: number of rows that are shorter than the longest row - * - * @return std::pair (Number of Columns, Number of rows that are shorter than longest row) - */ -std::pair CsvApplication::tableStatistics(CsvDataStorage localStorage) { - table_index_t maxCols = 0; - table_index_t shorterRows = 0; - - // table is empty - if( localStorage.rows() == 0 ) { - return {0, 0}; - } - // table has just one row - if( localStorage.rows() == 1) { - return { static_cast(localStorage.rawRow(0).size()), 0 }; - } - // get maximum columns - for( table_index_t r = 1; r < localStorage.rows(); ++r ) { - if( (table_index_t) localStorage.rawRow(r).size() > maxCols ) { - maxCols = localStorage.rawRow(r).size(); - } - } - // count number of shorter rows - for( table_index_t r = 1; r < localStorage.rows(); ++r ) { - if( (table_index_t) localStorage.rawRow(r).size() < maxCols ) { - ++shorterRows; - } - } - return {maxCols, shorterRows}; -} - - /* * Returns guessed encoding and the length of a BOM sequence – or zero if no BOM is present. * TODO remove 'bom' as it's not needed */ -std::pair CsvApplication::guessEncoding(std::istream *input, long streamLength) { - #ifdef DEBUG - CsvDefinition::BOMs bom = CsvDefinition::BOM_NONE; - #endif +std::pair CsvApplication::guessEncoding(std::istream *input, int64_t streamLength) { CsvDefinition::Encodings enc = CsvDefinition::ENC_NONE; int bomBytes = 0; - unsigned char octet[4]; + // zero-initialised on purpose: for files shorter than 4 bytes the read() calls below + // leave the tail untouched, and the BOM comparisons would read uninitialised memory + unsigned char octet[4] = {0, 0, 0, 0}; // reset stream @@ -1018,43 +902,11 @@ std::pair CsvApplication::guessEncoding(std::istr input->read((char *) &octet + 2, 1); input->read((char *) &octet + 3, 1); - if( octet[0] == 0xEF && octet[1] == 0xBB && octet[2] == 0xBF ) { - // UTF8 mit BOM - #ifdef DEBUG - bom = CsvDefinition::BOM_UTF8; - #endif - bomBytes = 3; - enc = CsvDefinition::ENC_UTF8; - } else if( octet[0] == 0x00 && octet[1] == 0x00 && octet[2] == 0xFE && octet[3] == 0xFF ) { - // UTF32BE - #ifdef DEBUG - bom = CsvDefinition::BOM_UTF32BE; - #endif - bomBytes = 4; - enc = CsvDefinition::ENC_UTF32BE; - } else if( octet[0] == 0xFF && octet[1] == 0xFE && octet[2] == 0x00 && octet[2] == 0x00 ) { - // UTF32LE - #ifdef DEBUG - bom = CsvDefinition::BOM_UTF32LE; - #endif - bomBytes = 4; - enc = CsvDefinition::ENC_UTF32LE; - } else if( octet[0] == 0xFE && octet[1] == 0xFF ) { - // UTF16BE - #ifdef DEBUG - bom = CsvDefinition::BOM_UTF16BE; - #endif - bomBytes = 2; - enc = CsvDefinition::ENC_UTF16BE; - } else if( octet[0] == 0xFF && octet[1] == 0xFE ) { - // UTF16LE - #ifdef DEBUG - bom = CsvDefinition::BOM_UTF16LE; - #endif - bomBytes = 2; - enc = CsvDefinition::ENC_UTF16LE; - } - + enc = CsvDefinition::fromBom(octet, bomBytes); + #ifdef DEBUG + printf("BOM bytes: %d\n", bomBytes); + #endif + // Falls kein BOM auf UTF-8 testen (falls Datei nicht zu groß ist) if( streamLength < TCRUNCHER_NUM_UTF8_TEST_BYTES && utf8::is_valid(it, eos) ) { enc = CsvDefinition::ENC_UTF8; @@ -1062,7 +914,7 @@ std::pair CsvApplication::guessEncoding(std::istr #ifdef DEBUG - printf("BOM: %d\nENC: %d\n", bom, enc); + printf("ENC: %d\n", enc); #endif @@ -1531,7 +1383,7 @@ void CsvApplication::paste(bool askUser, bool fillSelection) { } // guess properties - guessedDefinition = app.guessDefinition(&input); + guessedDefinition = CsvGuess::definition(&input); definition = guessedDefinition.first; definition.encoding = CsvDefinition::ENC_UTF8; @@ -2806,10 +2658,16 @@ void CsvApplication::arrangeColumnsCB(Fl_Widget *, void *) { std::vector content_length; std::vector content_length_relative; + // One pass over the probed rows for ALL columns – calling maximumContentLength() per + // column would rescan every row string from byte zero once per column. + std::vector> column_lengths = + windows[winIndex].table->columnContentLengths(TCRUNCHER_MAX_PROBE_ROWS_ARRANGE_COLS); + // Calculate width of all columns and find maximum width for each column for( table_index_t c = 0; c < windows[winIndex].table->getNumberCols(); ++c ) { all_columns_width += windows[winIndex].grid->col_width(c); - std::pair col_length_data = windows[winIndex].table->maximumContentLength(c, TCRUNCHER_MAX_PROBE_ROWS_ARRANGE_COLS); + std::pair col_length_data = + ( c < (table_index_t) column_lengths.size() ) ? column_lengths[(size_t) c] : std::make_pair(0, 0); int col_length = std::min( col_length_data.second, max_column_length); col_length = std::max(col_length, min_column_length); content_length.push_back(col_length); @@ -3383,10 +3241,19 @@ CsvMenu *CsvApplication::getAppMenuBar() { } -void CsvApplication::showImWorkingWindow(std::string message, bool showAlways) { +void CsvApplication::showImWorkingWindow(std::string message, bool showAlways, std::function onCancel) { int winIndex = getTopWindow(); if( windows[winIndex].table->getNumberRows() > 10000 || showAlways ) { windows[winIndex].grid->allowEvents(false); + imWorkingCancelHandler = onCancel; + if( onCancel ) { + // make room for the button and show it + imWorkingButton->resize(10, 10, 380, 62); + imWorkingCancelButton->show(); + } else { + imWorkingButton->resize(10, 10, 380, 100); + imWorkingCancelButton->hide(); + } imWorkingWindow->copy_label("Processing"); imWorkingWindow->color(ColorThemes::getColor(app.getTheme(), "win_bg")); imWorkingButton->copy_label(message.c_str()); @@ -3401,9 +3268,15 @@ void CsvApplication::showImWorkingWindow(std::string message, bool showAlways) { void CsvApplication::showImWorkingWindowCB(Fl_Widget *, long ) { // intentionally empty: showImWorkingWindow must not be closed by user } +void CsvApplication::imWorkingCancelCB(Fl_Widget *, void *) { + if( app.imWorkingCancelHandler ) + app.imWorkingCancelHandler(); +} void CsvApplication::hideImWorkingWindow() { int winIndex = getTopWindow(); windows[winIndex].grid->allowEvents(true); + imWorkingCancelHandler = nullptr; + imWorkingCancelButton->hide(); imWorkingWindow->hide(); Fl::check(); } diff --git a/src/csvapplication.hh b/src/csvapplication.hh index 86751ef..d31f6d5 100644 --- a/src/csvapplication.hh +++ b/src/csvapplication.hh @@ -27,6 +27,7 @@ #include "globals.hh" #include "colorthemes.hh" #include "csvdatastorage.hh" +#include "utf8validate.hh" #include "csvwindow.hh" #include "csvtable.hh" #include "csvgrid.hh" @@ -46,6 +47,7 @@ #include #include #include +#include #ifdef _WIN64 #include @@ -210,8 +212,7 @@ public: static void aboutCB(Fl_Widget *, void *); void changeFontSize(int changeMode); void setUndoMenuItem(bool ); - static std::pair guessDefinition(std::istream *input); // guesses the CSV definition - static std::pair guessEncoding(std::istream *input, long streamLength=0); + static std::pair guessEncoding(std::istream *input, int64_t streamLength=0); static CsvDefinition setTypeByUser(CsvDefinition guessedDefinition, std::istream *input, std::string buttonText = "Open"); bool isAlreadyOpened(std::string path); static void droppedFileCB(const char *path); @@ -235,7 +236,9 @@ public: void showOnboardingNextCB(Fl_Widget *w, void *data); void showOnboardingOkCB(Fl_Widget *w, void *data); void showOnboardingCancelCB(Fl_Widget *w, void *data); - void showImWorkingWindow(std::string message, bool showAlways = false); + // `onCancel`, when set, adds a Cancel button to the progress window and calls this back + // when it is pressed. Callers that pass nothing get the button-less window as before. + void showImWorkingWindow(std::string message, bool showAlways = false, std::function onCancel = nullptr); void hideImWorkingWindow(); void editSingleCell(); static void editSingleCellCB(Fl_Widget *, void *); @@ -304,6 +307,8 @@ private: bool checkUpdateAllowed = false; My_Fl_Small_Window *imWorkingWindow; Fl_Button *imWorkingButton; + Fl_Button *imWorkingCancelButton; + std::function imWorkingCancelHandler; int lastSingleEditWinWidth = 0; int lastSingleEditWinHeight = 0; std::string lastSplitString = ""; @@ -320,7 +325,6 @@ private: RecentFiles recentFiles; - static std::pair tableStatistics(CsvDataStorage localStorage); // Calculates the maximum number of columns and the variance of columns static void showPreview(struct previewTableStruct); // parses input and shows data // Callbacks for setTypeByUser() static void setTypeByUser_Done_CB(Fl_Widget *, long data); @@ -342,6 +346,7 @@ private: static void updateMacroBrowserList(Fl_Browser *macroList, int selected=0); static int selectMacroBrowserEntry(Fl_Browser *macroList, std::string ); static void showImWorkingWindowCB(Fl_Widget *, long ); + static void imWorkingCancelCB(Fl_Widget *, void *); static void dumpWindows(); }; diff --git a/src/csvdatastorage.cpp b/src/csvdatastorage.cpp index 745692c..8af5c65 100644 --- a/src/csvdatastorage.cpp +++ b/src/csvdatastorage.cpp @@ -62,8 +62,10 @@ void CsvDataStorage::resize(table_index_t R, table_index_t C) { table_index_t old_columns = columns(); if( C > old_columns ) { + // build the padding once – this used to allocate a fresh string for every row + const std::string pad = emptyCellsString(C - old_columns); for( table_index_t i = 0; i < old_rows; ++i) { - tableData.at(i) += emptyCellsString(C - old_columns); + tableData.at(i) += pad; } } if( R > 0 && R > rows() ) { @@ -78,6 +80,20 @@ void CsvDataStorage::resize(table_index_t R, table_index_t C) { } +/** + reserveRows(size_t n) + + Reserve capacity for `n` rows. Purely an optimisation: without it a bulk load + reallocates and moves millions of std::strings, which both costs time and spikes + peak RSS by ~1.5x. Never shrinks and never changes rows() or columns(). + */ +void CsvDataStorage::reserveRows(size_t n) { + if( n > tableData.capacity() ) { + tableData.reserve(n); + } +} + + /** clear() @@ -95,7 +111,7 @@ void CsvDataStorage::clear() { Returns number of rows */ -table_index_t CsvDataStorage::rows() { +table_index_t CsvDataStorage::rows() const { return (table_index_t) tableData.size(); } @@ -105,7 +121,7 @@ table_index_t CsvDataStorage::rows() { Returns number of columns */ -table_index_t CsvDataStorage::columns() { +table_index_t CsvDataStorage::columns() const { return numColumns; } @@ -122,14 +138,14 @@ void CsvDataStorage::sort(table_index_t column, bool ascending, int sortType) { } long counter = 0; - std::sort( tableData.begin(), tableData.end(), [&column, &ascending, &sortType, &counter](const auto& lhs, const auto& rhs) { + std::sort( tableData.begin(), tableData.end(), [this, &column, &ascending, &sortType, &counter](const auto& lhs, const auto& rhs) { std::string s1 = getColumn(lhs, column); std::string s2 = getColumn(rhs, column); double d1, d2; std::string lowerS1; std::string lowerS2; - if( counter % 50000 == 0 ) { - Fl::check(); + if( counter % 50000 == 0 && progressCallback ) { + progressCallback((size_t) counter); } ++counter; switch( sortType ) { @@ -177,6 +193,18 @@ void CsvDataStorage::sort(table_index_t column, bool ascending, int sortType) { +/** + setProgressCallback(std::function cb) + + Injected by the UI layer so that sort() can keep the event loop alive without + CsvDataStorage having to know about FLTK. Pass an empty std::function to detach. + */ +void CsvDataStorage::setProgressCallback(std::function cb) { + progressCallback = std::move(cb); +} + + + /** get(long R, long C) @@ -205,6 +233,41 @@ std::string CsvDataStorage::getRow(table_index_t R) { +/** + assembleFrom(std::vector& pieces, table_index_t columns) + + Concatenates the pieces of a parallel load into one table. + + Kept as a single operation because the intermediate states are not valid tables: appending + rows does not update numColumns, and setting numColumns does not pad rows. Exposing those + two halves separately would put a way to build an inconsistent CsvDataStorage — one whose + numColumns disagrees with its row contents — on the public API. + + Every row in every piece must already hold exactly `columns` glue-separated fields. + */ +void CsvDataStorage::assembleFrom(std::vector& pieces, table_index_t columns) { + size_t total = 0; + for( const CsvDataStorage& piece : pieces ) + total += piece.tableData.size(); + + tableData.clear(); + tableData.shrink_to_fit(); + tableData.reserve(total); + + for( CsvDataStorage& piece : pieces ) { + tableData.insert( tableData.end(), + std::make_move_iterator(piece.tableData.begin()), + std::make_move_iterator(piece.tableData.end()) ); + // release each piece's header array as we go, so peak overhead is one piece + piece.tableData.clear(); + piece.tableData.shrink_to_fit(); + piece.numColumns = 0; + } + + numColumns = columns; +} + + /** set(std::string content, long R, long C) @@ -265,11 +328,15 @@ std::vector CsvDataStorage::rawRow(table_index_t R) { Adds row to the end of the table data */ -void CsvDataStorage::push_back(std::string rowString) { +void CsvDataStorage::push_back(const std::string& rowString) { tableData.push_back(rowString); } -void CsvDataStorage::push_back(std::vector row) { +void CsvDataStorage::push_back(std::string&& rowString) { + tableData.push_back(std::move(rowString)); +} + +void CsvDataStorage::push_back(const std::vector& row) { tableData.push_back(mergeString(row)); } @@ -279,11 +346,11 @@ void CsvDataStorage::push_back(std::vector row) { push_front(std::vector > row) Adds row to the beginning of the table data */ -void CsvDataStorage::push_front(std::string row) { +void CsvDataStorage::push_front(const std::string& row) { // TODO edit length histogram!? tableData.insert(tableData.begin(), row); } -void CsvDataStorage::push_front(std::vector row) { +void CsvDataStorage::push_front(const std::vector& row) { // TODO edit length histogram!? tableData.insert(tableData.begin(), mergeString(row)); } @@ -410,12 +477,66 @@ bool CsvDataStorage::cellContainsLineBreak(table_index_t R, table_index_t C) { return get(R,C).find("\n") != std::string::npos; } + +/** + columnContentLengths(table_index_t maxProbeRows) + + Maximum and average content length, in bytes, for every column at once. + + The obvious formulation – ask get(R,C) for each cell – rescans each row string from byte + zero for every column, so probing C columns of a row costs O(L*C/2). On a 1000-column, + 10 KB-per-row table that is tens of billions of byte operations. This walks each row + exactly once instead. + + Parity with the per-cell version is deliberate: the average uses integer division by the + number of probed rows, and a row holding fewer than columns() fields contributes length 0 + for the missing tail, matching get()'s out-of-range "". + */ +std::vector> CsvDataStorage::columnContentLengths(table_index_t maxProbeRows) { + const table_index_t C = columns(); + std::vector> out; + if( C <= 0 ) + return out; + out.assign((size_t) C, {0,0}); + + table_index_t probeRows = rows(); + if( maxProbeRows > 0 ) + probeRows = std::min(probeRows, maxProbeRows); + if( probeRows <= 0 ) + return out; + + std::vector maxLen((size_t) C, 0); + std::vector sumLen((size_t) C, 0); + + for( table_index_t r = 0; r < probeRows; ++r ) { + const std::string& row = tableData[(size_t) r]; + const size_t n = row.size(); + table_index_t c = 0; + size_t fieldStart = 0; + for( size_t i = 0; i <= n; ++i ) { + if( i == n || static_cast(row[i]) == TCRUNCHER_UTF_8_DELIMITER ) { + if( c < C ) { + const int len = (int)(i - fieldStart); + if( len > maxLen[(size_t) c] ) maxLen[(size_t) c] = len; + sumLen[(size_t) c] += (uint64_t) len; + } + ++c; + fieldStart = i + 1; + } + } + } + + for( table_index_t c = 0; c < C; ++c ) + out[(size_t) c] = { maxLen[(size_t) c], (int)(sumLen[(size_t) c] / (uint64_t) probeRows) }; + return out; +} + /** splitString() Splits the given string at the defined internal CSV delimiter */ -std::vector CsvDataStorage::splitString(std::string str) { +std::vector CsvDataStorage::splitString(const std::string& str) { std::vector splitted; std::string tempStr = ""; size_t str_size = str.size(); @@ -437,8 +558,12 @@ std::vector CsvDataStorage::splitString(std::string str) { Merges the given vector-of-strings with the defined internal CSV delimiter */ -std::string CsvDataStorage::mergeString(std::vector row) { - std::string merged = ""; +std::string CsvDataStorage::mergeString(const std::vector& row) { + std::string merged; + size_t total = row.empty() ? 0 : row.size() - 1; // one glue byte between every pair of fields + for( auto const& field : row ) + total += field.size(); + merged.reserve(total); for(size_t i = 0; i < row.size(); ++i ) { if( i > 0 ) merged.push_back( static_cast(CsvDataStorage::TCRUNCHER_UTF_8_DELIMITER) ); @@ -455,7 +580,7 @@ std::string CsvDataStorage::mergeString(std::vector row) { a#bcd#ef#hij 0123456789ab */ -std::string CsvDataStorage::getColumn(std::string rowString, table_index_t column) { +std::string CsvDataStorage::getColumn(const std::string& rowString, table_index_t column) { std::pair fromTo = getColumnIndizes(rowString, column); if( fromTo.second - fromTo.first - 1 <= (table_index_t) rowString.size() && fromTo.second > fromTo.first ) { return rowString.substr(fromTo.first + 1, (fromTo.second - fromTo.first - 1)); @@ -464,7 +589,7 @@ std::string CsvDataStorage::getColumn(std::string rowString, table_index_t colum } } -std::pair CsvDataStorage::getColumnIndizes(std::string rowString, table_index_t column) { +std::pair CsvDataStorage::getColumnIndizes(const std::string& rowString, table_index_t column) { bool colFound = false; size_t str_size = rowString.size(); table_index_t fromIdx = -1; @@ -492,7 +617,7 @@ std::pair CsvDataStorage::getColumnIndizes(std::str } -std::string CsvDataStorage::setColumn(std::string rowString, table_index_t column, std::string content) { +std::string CsvDataStorage::setColumn(const std::string& rowString, table_index_t column, const std::string& content) { std::vector row = splitString(rowString); if( (table_index_t) row.size() < numColumns ) { row.resize(numColumns, ""); diff --git a/src/csvdatastorage.hh b/src/csvdatastorage.hh index 3944301..204f970 100644 --- a/src/csvdatastorage.hh +++ b/src/csvdatastorage.hh @@ -22,12 +22,11 @@ #ifndef _CSVDATASTORAGE_HH #define _CSVDATASTORAGE_HH -#include // called by sort() to update UI - #include #include #include #include +#include // sort() progress callback #include // needed for dump() #include // needed for dump() #include @@ -51,43 +50,60 @@ */ class CsvDataStorage { + static const unsigned char TCRUNCHER_UTF_8_DELIMITER = 0xFA; // this byte is used as a separator for fields within std::string (it's an invalid UTF-8 character) public: CsvDataStorage(); // Standard constructor CsvDataStorage(table_index_t R, table_index_t C); // Defined size constructor void resize(table_index_t R, table_index_t C = 0); // resizes the storage to dimensions R,C + void reserveRows(size_t n); // reserves capacity for n rows – avoids repeated reallocation while bulk loading void clear(); // clears the storage - table_index_t rows(); // returns number of rows - table_index_t columns(); // returns number of columns + table_index_t rows() const; // returns number of rows + table_index_t columns() const; // returns number of columns void sort(table_index_t column, bool ascending, int sortType); // sorts the table according to the given options + void setProgressCallback(std::function cb); // called by sort() every 50,000 comparisons; lets the UI layer pump events std::string get(table_index_t R, table_index_t C); // gets the cell content at R,C std::string getRow(table_index_t R); // returns the row string (including illegal UTF-8 glue character) bool set(std::string content, table_index_t R, table_index_t C); // sets the content of cell at R,C – true if succeeded std::vector row(table_index_t R); // returns a single row as a vector of strings with length `numColumns` std::vector rawRow(table_index_t R); // returns a single row as a vector of strings, length depends on content - void push_back(std::string rowString); // adds a row at end of the table - void push_back(std::vector row); // adds a row at end of the table - void push_front(std::string rowString); // adds a row at the beginning of the table - void push_front(std::vector row); // adds a row at beginning of the table + void push_back(const std::string& rowString); // adds a row at end of the table + void push_back(std::string&& rowString); // adds a row at end of the table, taking ownership + void push_back(const std::vector& row); // adds a row at end of the table + void push_front(const std::string& rowString); // adds a row at the beginning of the table + void push_front(const std::vector& row); // adds a row at beginning of the table void deleteRows(table_index_t rowFrom, table_index_t rowTo); // delete rows void deleteColumns(table_index_t colFrom, table_index_t colTo); // delete columns void insertRow(table_index_t R, table_index_t before = false); // inserts a row after (or before) row R void insertColumn(table_index_t C, bool before = false); // inserts a column after (or before) column C void moveColumns(table_index_t colFromStart, table_index_t colFromEnd, bool right); // move multiple columns to the right or left bool cellContainsLineBreak(table_index_t R, table_index_t C); // returns true if content of that cell contains line breaks (\n) + // (maximum, average) content length in bytes for EVERY column, in one pass per row. + // `maxProbeRows` <= 0 means "all rows". + std::vector> columnContentLengths(table_index_t maxProbeRows = 0); void dump(table_index_t numRows = 10, bool raw = false); // DEBUG: dumps content of tableData; if `raw`: strings are displayed + // The byte that glues fields together inside a row string. Exposed so that the parser + // can build finished row strings itself instead of going through mergeString(). + static char internalGlue() { return static_cast(TCRUNCHER_UTF_8_DELIMITER); } + + // Replaces this table with the concatenation of `pieces`, in order, and declares the + // result `columns` wide. For bulk loading only: every row in every piece must ALREADY + // hold exactly `columns` glue-separated fields, which is what the caller's padding pass + // guarantees. Rows are moved, not copied, and each piece is released as it is consumed. + void assembleFrom(std::vector& pieces, table_index_t columns); + private: std::vector tableData; // holds the data table_index_t numColumns = 0; // number of columns - static const unsigned char TCRUNCHER_UTF_8_DELIMITER = 0xFA; // this byte is used as a separator for fields within std::string (it's an invalid UTF-8 character) + std::function progressCallback; // optional: injected by the UI layer, see setProgressCallback() - static std::vector splitString(std::string str); // splits a string at the internal CSV delimiter - static std::string mergeString(std::vector row); // merges the vector to a string - static std::pair getColumnIndizes(std::string rowString, table_index_t column); // returns the positions of the surrounding bytes - static std::string getColumn(std::string row, table_index_t column); // gets the content of `column` in string row - std::string setColumn(std::string row, table_index_t column, std::string content); // sets the content of `column` in row and returns the new string + static std::vector splitString(const std::string& str); // splits a string at the internal CSV delimiter + static std::string mergeString(const std::vector& row); // merges the vector to a string + static std::pair getColumnIndizes(const std::string& rowString, table_index_t column); // returns the positions of the surrounding bytes + static std::string getColumn(const std::string& row, table_index_t column); // gets the content of `column` in string row + std::string setColumn(const std::string& row, table_index_t column, const std::string& content); // sets the content of `column` in row and returns the new string static std::string emptyCellsString(table_index_t num); // returns strings of delimiters }; diff --git a/src/csvfsm.hh b/src/csvfsm.hh new file mode 100644 index 0000000..1960f0d --- /dev/null +++ b/src/csvfsm.hh @@ -0,0 +1,314 @@ +/* + * SPDX-License-Identifier: GPL-3.0-or-later + * + * Copyright (C) 2025 Stefan Fischerländer + * + * This file is part of Tablecruncher. + * + * Tablecruncher is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Tablecruncher is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Tablecruncher. If not, see . + */ + +#ifndef _CSVFSM_HH +#define _CSVFSM_HH + +// +// The CSV finite state machine, extracted so that every consumer runs the SAME +// transitions: the serial parser, the parallel chunk prescan, and the parallel parse. +// Writing a second state machine for the prescan would let the two drift apart, and a +// drift here silently corrupts user data. +// +// Deliberately depends on nothing: no FLTK, no iostream, no project headers. This file +// is compiled into worker threads. +// + +#include +#include +#include +#include + + +/** + * The CSV dialect, reduced to what the state machine actually branches on. + */ +struct CsvDialect { + char delimiter = ','; + char quote = '"'; + char escape = '"'; + bool escapeActive = false; // == (quote != escape); when false the escape branch is dead + + // True for the (at most three) bytes the state machine has to look at individually. + // Everything else is ordinary content and can be handled in runs – see csvScanLine(). + bool special[256] = { false }; + + void buildSpecialTable() { + for( int i = 0; i < 256; ++i ) special[i] = false; + special[(unsigned char) delimiter] = true; + special[(unsigned char) quote] = true; + if( escapeActive ) + special[(unsigned char) escape] = true; + } +}; + + +/* + * Sink contract – see csvScanLine() below. + * + * void beginField(); a new field starts (the record's first, or one after a delimiter) + * void content(const char* p, size_t n); append n raw bytes to the current field + * void endField(); the current field is complete + * void lineContinuation(); a physical line ended while still inside a quoted field + * + * beginField() and endField() are always paired 1:1, so the number of either call is the + * record's field count. + */ + + +/** + * Discards everything. Used by the chunk prescan, which only cares about the `enclosed` + * bit at the chunk boundary – with this sink the whole content path is dead code and the + * scan collapses into a byte-set skip loop. + */ +struct StructuralSink { + void beginField() {} + void endField() {} + void content(const char*, size_t) {} + void lineContinuation() {} +}; + + +/** + * Builds the glued row string that `CsvDataStorage` stores, straight from the input bytes: + * no transient vector, no per-field allocation, no separate merge pass. The byte sequence + * produced is identical to `CsvDataStorage::mergeString(fields)` – fields joined by exactly + * one glue byte, none leading or trailing. + */ +class MergedRowSink { +public: + MergedRowSink(std::string* out, char glue) : out_(out), glue_(glue) {} + + // Begins a fresh record. `reserveHint` matters more than it looks: the finished string is + // MOVED into storage, so the next record starts from a moved-from (small-buffer) string + // and would otherwise re-cross the SSO boundary every row. The physical line length is an + // exact upper bound – each delimiter becomes exactly one glue byte, and quoting only ever + // removes bytes. + void startRecord(size_t reserveHint = 0) { + out_->clear(); + if( reserveHint ) + out_->reserve(reserveHint); + fields_ = 0; + beginField(); + } + + void beginField() { if( fields_++ > 0 ) out_->push_back(glue_); } + void endField() {} + void content(const char* p, size_t n) { out_->append(p, n); } + // LF is the right thing to do here even for a CRLF file: a CR would render as "^M" + // in the multiline cell editor. + void lineContinuation() { out_->push_back('\n'); } + + int32_t fields() const { return fields_; } + +private: + std::string* out_; + char glue_; + int32_t fields_ = 0; +}; + + +/** + * Scans ONE physical line [p, e) and drives `sink`. + * + * `enclosed` is in/out and is the ONLY state that crosses a line boundary – at a physical + * line start the machine's structural state is exactly this one bit, which is what makes + * the parallel chunk prescan composable. On return, enclosed == false means the record is + * complete and the caller should emit endField(); otherwise the caller emits + * lineContinuation() and feeds the next physical line into the same sink. + * + * PRECONDITION: [p, e) contains no NUL bytes. The serial reader strips them while + * splitting lines, and doing so BEFORE the scan is load-bearing: NUL removal shifts both + * the "is this the last character of the line" test and character adjacency, so `\` NUL `"` + * must behave exactly like `\"`. A caller working on raw bytes has to materialise a + * NUL-free line first. + * + * The transitions below reproduce the historical parser exactly, quirks included: + * - startField is unconditionally true at every physical line start, even mid-quote + * - a doubled quote while NOT enclosed opens a quote and consumes only ONE character + * - the escape lookahead is line-scoped: an escape char as the last byte of a line is + * literal content, not an escape + * - neither the escape branch nor the quote branch clears startField + */ +template +inline void csvScanLine(const char* p, const char* e, const CsvDialect& d, bool& enclosed, Sink& sink) { + const size_t lineLen = (size_t)(e - p); + bool startField = true; // always true at a physical line start + + for( size_t i = 0; i < lineLen; ++i ) { + const char c = p[i]; + + if( !d.special[(unsigned char) c] ) { + // Ordinary content. Take the whole run in one go: the per-byte path below would + // otherwise call into the sink once per byte, which is the single hottest thing + // in the parser. Semantics are unchanged – an ordinary byte only ever clears + // startField and appends itself. + size_t j = i + 1; + while( j < lineLen && !d.special[(unsigned char) p[j]] ) + ++j; + startField = false; + sink.content(p + i, j - i); + i = j - 1; + continue; + } + + if( d.escapeActive && c == d.escape ) { + if( i + 1 < lineLen ) { + // not the last character: write the following one back verbatim + sink.content(p + i + 1, 1); + ++i; + continue; + } + // last character of the line: falls through and is treated as plain content + } + + if( c == d.quote ) { + if( i + 1 < lineLen && p[i+1] == d.quote ) { + // doubled quote + if( enclosed ) { + sink.content(p + i, 1); + ++i; + continue; + } else { + enclosed = true; // consumes ONE character, not two + continue; + } + } else { + // single quote + if( enclosed ) { + enclosed = false; // quoting ends + } else if( startField ) { + enclosed = true; // not enclosed and at the start of a field + } else { + sink.content(p + i, 1); // not enclosed, mid-field: literal content + } + continue; + } + } + + if( startField ) { + startField = false; + } + + if( c == d.delimiter && !enclosed ) { + sink.endField(); + sink.beginField(); + startField = true; + continue; + } + + sink.content(p + i, 1); + } +} + + + +/** + * Walks the physical lines of a byte range exactly the way the historical stream reader + * (CsvParser::myGetline) does: LF, CRLF and a lone CR all terminate a line. + * + * Line splitting is part of the observable semantics, not an implementation detail: NUL + * removal shifts the escape lookahead, and the chunk prescan's one-bit carry is only + * composable if the prescan splits lines identically to the parse. So there is exactly one + * implementation, shared by both. + */ +class CsvLineCursor { +public: + CsvLineCursor(const char* data, uint64_t from, uint64_t to) + : data_(data), end_(data + to), to_(to), pos_(from < to ? from : to) {} + + uint64_t pos() const { return pos_; } + + // Consumes one line. Returns false once the range is exhausted. `reachedEnd` reports that + // the line ran into the end of the range with no terminator, which is what decides + // whether the reader would have set eofbit. + bool next(const char*& lineBegin, const char*& lineEnd, bool& reachedEnd) { + const char* s = data_ + pos_; + if( s >= end_ ) + return false; + + // next LF at or after s + if( !lfDone_ && (lfPos_ == nullptr || lfPos_ < s) ) { + lfPos_ = (const char*) memchr(s, '\n', (size_t)(end_ - s)); + if( lfPos_ == nullptr ) lfDone_ = true; + } + // Next CR at or after s – but never searched beyond the next LF, because a CR after it + // can never win the min() below. Without that bound a ten-line dialect probe on an + // LF-only file scans the entire buffer looking for a CR that isn't there, once per + // candidate dialect. + const char* crLimit = lfDone_ ? end_ : lfPos_; + if( crPos_ == nullptr || crPos_ < s ) + crPos_ = ( crLimit > s ) ? (const char*) memchr(s, '\r', (size_t)(crLimit - s)) : nullptr; + + const char* term = end_; + if( lfPos_ && lfPos_ < term ) term = lfPos_; + if( crPos_ && crPos_ < term ) term = crPos_; + + lineBegin = s; + lineEnd = term; + reachedEnd = ( term >= end_ ); + if( reachedEnd ) { + pos_ = to_; + } else if( *term == '\n' ) { + pos_ = (uint64_t)(term - data_) + 1; + } else { + pos_ = (uint64_t)(term - data_) + ((term + 1 < end_ && term[1] == '\n') ? 2 : 1); + } + return true; + } + +private: + const char* data_; + const char* end_; + uint64_t to_; + uint64_t pos_; + const char* lfPos_ = nullptr; + bool lfDone_ = false; + const char* crPos_ = nullptr; +}; + + +// True if [from, to) holds a NUL anywhere. Checking once per range keeps the per-line check +// off the hot path for the overwhelmingly common NUL-free file. +inline bool csvRangeHasNul(const char* data, uint64_t from, uint64_t to) { + return to > from && memchr(data + from, 0, (size_t)(to - from)) != nullptr; +} + + +// Materialises [lineBegin, lineEnd) into `scratch` without its NUL bytes and re-points +// [p, e) at it. Returns false – leaving p/e alone – when there is nothing to strip, which is +// what lets the state machine read straight out of the mapping. +inline bool csvStripNuls(const char* lineBegin, const char* lineEnd, std::string& scratch, + const char*& p, const char*& e) { + if( lineEnd <= lineBegin || memchr(lineBegin, 0, (size_t)(lineEnd - lineBegin)) == nullptr ) + return false; + scratch.clear(); + scratch.reserve((size_t)(lineEnd - lineBegin)); + for( const char* q = lineBegin; q < lineEnd; ++q ) { + if( *q != '\0' ) scratch.push_back(*q); + } + p = scratch.data(); + e = p + scratch.size(); + return true; +} + + +#endif diff --git a/src/csvguess.cpp b/src/csvguess.cpp new file mode 100644 index 0000000..ab108bf --- /dev/null +++ b/src/csvguess.cpp @@ -0,0 +1,272 @@ +/* + * SPDX-License-Identifier: GPL-3.0-or-later + * + * Copyright (C) 2025 Stefan Fischerländer + * + * This file is part of Tablecruncher. + * + * Tablecruncher is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Tablecruncher is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Tablecruncher. If not, see . + */ + +#include "csvguess.hh" + +#include +#include + +#include "csvparser.hh" +#include "utf8.h" + + +namespace { + const int MAXLINES = 10; // lines read per candidate dialect + +} + + +/* + * Returns {Number of Cols, Some kind of Variance} of the data in localStorage + * Variance: number of rows that are shorter than the longest row + */ +std::pair CsvGuess::tableStatistics(CsvDataStorage& localStorage) { + table_index_t maxCols = 0; + table_index_t shorterRows = 0; + + // table is empty + if( localStorage.rows() == 0 ) { + return {0, 0}; + } + // table has just one row + if( localStorage.rows() == 1) { + return { static_cast(localStorage.rawRow(0).size()), 0 }; + } + // get maximum columns + for( table_index_t r = 1; r < localStorage.rows(); ++r ) { + if( (table_index_t) localStorage.rawRow(r).size() > maxCols ) { + maxCols = localStorage.rawRow(r).size(); + } + } + // count number of shorter rows + for( table_index_t r = 1; r < localStorage.rows(); ++r ) { + if( (table_index_t) localStorage.rawRow(r).size() < maxCols ) { + ++shorterRows; + } + } + return {maxCols, shorterRows}; +} + + +/* + * Counts fields holding an ODD number of quote characters. + * + * This is what tells a separator apart from a character that merely occurs inside the data. + * Parsed with the right dialect, a quoted field has its quotes consumed and an unquoted one + * carries them in balanced pairs. Parsed with a delimiter that happens to appear INSIDE + * quoted content — ':' in a column of JSON, say — the parser slices through the quoting and + * leaves orphans behind in nearly every field. + */ +std::pair CsvGuess::quoteSanity(CsvDataStorage& localStorage, char quote) { + long totalFields = 0; + long orphanFields = 0; + for( table_index_t r = 0; r < localStorage.rows(); ++r ) { + for( const std::string& field : localStorage.rawRow(r) ) { + long quotes = 0; + for( char c : field ) { + if( c == quote ) ++quotes; + } + ++totalFields; + if( quotes & 1 ) ++orphanFields; + } + } + return {totalFields, orphanFields}; +} + + +/* + * The eight dialects that get probed. + * + * NOTE a long-standing quirk, preserved: entry 6's delimiter is assigned twice, so ';' with + * a backslash escape is never actually probed and entry 7 stays at the default (a duplicate + * of entry 0). Changing it would change which dialect wins for some files. + */ +std::vector CsvGuess::makeProbes() { + std::vector probes(8); + probes.at(0).definition.delimiter = ','; + probes.at(1).definition.delimiter = ';'; + probes.at(2).definition.delimiter = '\t'; + probes.at(3).definition.delimiter = '|'; + probes.at(4).definition.delimiter = ':'; + probes.at(5).definition.delimiter = ','; + probes.at(6).definition.delimiter = ';'; + probes.at(6).definition.delimiter = '*'; + probes.at(5).definition.escape = '\\'; + probes.at(6).definition.escape = '\\'; + return probes; +} + + +/* + * Measures one probe's parse and fills in everything the ranking looks at. + */ +void CsvGuess::scoreProbe(Probe& probe, CsvDataStorage& localStorage) { + std::pair statistics = tableStatistics(localStorage); + std::pair sanity = quoteSanity(localStorage, probe.definition.quote); + const int maxCols = statistics.first; // before the penalty below rewrites it + + // not so commonly used seperators and escape characters: decrease statistics value + if( + probe.definition.delimiter == ':' || + probe.definition.delimiter == '|' || + probe.definition.escape == '\\' || + probe.definition.escape == '*' + ) { + statistics.first = statistics.first * 70 / 100; + } + probe.score = statistics.first; + + if( statistics.first <= 1 && statistics.second == 0) { + // if statistics is (1,0), sort it at the end + probe.variance = 999; + } else { + probe.variance = statistics.second; + } + + // Does the first row hold as many fields as the rest? tableStatistics() deliberately skips + // row 0, so on its own it cannot see a delimiter that splits the header into one field and + // every data row into eighty-seven. Well-formed CSV agrees across all rows, header or not. + probe.headerMismatch = 0; + if( localStorage.rows() > 1 && (int) localStorage.rawRow(0).size() != maxCols ) { + probe.headerMismatch = 1; + } + + probe.orphanPercent = ( sanity.first > 0 ) + ? (int)( (sanity.second * 100) / sanity.first ) + : 0; + + #ifdef DEBUG + printf("CSV = '%c' => cols %d / var %d / header %s / orphan quotes %d%%\n", + probe.definition.delimiter, probe.score, probe.variance, + probe.headerMismatch ? "MISMATCH" : "ok", probe.orphanPercent); + #endif +} + + +/* + * Sorts the probes and derives a confidence value for the winner. + */ +std::pair CsvGuess::rankProbes(std::vector& probes) { + // + // Least variance first, exactly as before. The two signals after it only ever break a + // TIE, so no candidate that uniquely explains the row lengths can be displaced by them: + // - a delimiter that disagrees with the first row is separating something other than + // fields (this is what a column of JSON does to ':') + // - a delimiter that leaves orphaned quotes behind is slicing through quoted content + // Only then does the historical "more columns wins" tie-break apply. + // + std::sort(begin(probes), end(probes), [](const Probe& a, const Probe& b) { + if( a.variance != b.variance ) return a.variance < b.variance; + if( a.headerMismatch != b.headerMismatch ) return a.headerMismatch < b.headerMismatch; + if( a.orphanPercent != b.orphanPercent ) return a.orphanPercent < b.orphanPercent; + return a.score > b.score; + }); + + float confidence = 1.0; + // if there's no definition with zero variance: reduce confidence + if( probes[0].variance > 0 ) { + confidence /= 2; + } + // if there are at least two definitions with the same number of columns: reduce confidence + if( probes[0].score == probes[1].score ) { + confidence /= 2; + } + // improve confidence, if it's a typical CSV separator + if( probes.at(0).definition.delimiter == ',' || probes.at(0).definition.delimiter == '\t' ) { + confidence += (1.0 - confidence) * 0.5; + } + return { probes.at(0).definition, confidence }; +} + + +/* + * Guesses the dialect of an already-mapped buffer. Saves eight full re-reads of the file's + * head plus the seekg() churn, and is what the load path uses. + */ +std::pair CsvGuess::definition(const char* data, uint64_t len) { + CsvDataStorage localStorage; + std::vector probes = makeProbes(); + + for( Probe& probe : probes ) { + CsvParser parser; + localStorage.clear(); + parser.parseCsvBuffer( data, len, localStorage, &probe.definition, MAXLINES, false ); + scoreProbe( probe, localStorage ); + } + return rankProbes(probes); +} + + +/* + * Same, for a stream. Used when the file could not be mapped, and by the paste path. + */ +std::pair CsvGuess::definition(std::istream* input) { + CsvDataStorage localStorage; + std::vector probes = makeProbes(); + + for( Probe& probe : probes ) { + CsvParser parser; + input->clear(); + input->seekg(0); + localStorage.clear(); + parser.parseCsvStream( input, localStorage, &probe.definition, MAXLINES, false ); + scoreProbe( probe, localStorage ); + } + + input->clear(); + input->seekg(0); + return rankProbes(probes); +} + + +/* + * Encoding, and the length of any byte-order mark. + * + * Validation is multi-threaded here, so – unlike the istream overload – there is no + * TCRUNCHER_NUM_UTF8_TEST_BYTES size cap. A UTF-8 file above that cap used to be reported as + * ENC_NONE, which forced the "choose your format" modal on open; it now simply opens. The + * parsed bytes are unchanged either way, because ENC_NONE and ENC_UTF8 both route through + * Helper::fixUtf8(). + * + * The UTF-8 check deliberately starts at byte offset 4, reproducing the istream version, + * where four read() calls consume 4 bytes before utf8::is_valid() runs on the + * already-constructed iterator. That is why an ASCII UTF-16 file WITH a BOM is reported as + * UTF-8 – it then parses correctly only because the reader strips NUL bytes. Do not "fix" + * this; it changes how real files open. + */ +std::pair CsvGuess::encoding(const char* data, uint64_t len, + Utf8ValidationResult& validationOut) { + CsvDefinition::Encodings enc = CsvDefinition::ENC_NONE; + int bomBytes = 0; + unsigned char octet[4] = {0, 0, 0, 0}; + + for( uint64_t i = 0; i < 4 && i < len; ++i ) + octet[i] = (unsigned char) data[i]; + + enc = CsvDefinition::fromBom(octet, bomBytes); + + validationOut = validateUtf8Parallel(data, len, bomBytes); + if( validationOut.validFrom4 ) { + enc = CsvDefinition::ENC_UTF8; + } + + return std::make_pair(enc, bomBytes); +} diff --git a/src/csvguess.hh b/src/csvguess.hh new file mode 100644 index 0000000..4c1216f --- /dev/null +++ b/src/csvguess.hh @@ -0,0 +1,78 @@ +/* + * SPDX-License-Identifier: GPL-3.0-or-later + * + * Copyright (C) 2025 Stefan Fischerländer + * + * This file is part of Tablecruncher. + * + * Tablecruncher is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Tablecruncher is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Tablecruncher. If not, see . + */ + +#ifndef _CSVGUESS_HH +#define _CSVGUESS_HH + +// +// Working out a file's CSV dialect and encoding. +// +// This lived in CsvApplication, but none of it touches FLTK — and getting it wrong is how a +// file silently opens with the wrong number of columns, so it needs to be testable without +// a UI. +// + +#include +#include +#include +#include +#include + +#include "globals.hh" +#include "csvdatastorage.hh" +#include "utf8validate.hh" + + +namespace CsvGuess { + + // A candidate dialect together with the numbers it is ranked on, best first. + struct Probe { + CsvDefinition definition; + int variance = 0; // rows shorter than the longest; 999 means "sort me last" + int headerMismatch = 0; // 1 when row 0 holds a different number of fields than the rest + int orphanPercent = 0; // share of fields left holding an odd number of quote characters + int score = 0; // number of columns, after the unusual-delimiter penalty + }; + + // (number of columns, number of rows shorter than the longest) + std::pair tableStatistics(CsvDataStorage& localStorage); + + // How many of the probed fields carry an ODD number of quote characters. A delimiter that + // cuts through quoted content leaves orphaned quotes behind, which is the clearest signal + // that a candidate is slicing up a field rather than separating fields. + std::pair quoteSanity(CsvDataStorage& localStorage, char quote); + + std::vector makeProbes(); + // Measures `localStorage` — the result of parsing with this probe's dialect — and fills in + // everything the ranking looks at. + void scoreProbe(Probe& probe, CsvDataStorage& localStorage); + std::pair rankProbes(std::vector& probes); + + // The dialect the data most likely uses, with a confidence in [0, 1]. + std::pair definition(const char* data, uint64_t len); + std::pair definition(std::istream* input); + + // The encoding, and the length of the byte-order mark (0 when there is none). + std::pair encoding(const char* data, uint64_t len, + Utf8ValidationResult& validationOut); +} + +#endif diff --git a/src/csvloader.cpp b/src/csvloader.cpp new file mode 100644 index 0000000..5b684c4 --- /dev/null +++ b/src/csvloader.cpp @@ -0,0 +1,389 @@ +/* + * SPDX-License-Identifier: GPL-3.0-or-later + * + * Copyright (C) 2025 Stefan Fischerländer + * + * This file is part of Tablecruncher. + * + * Tablecruncher is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Tablecruncher is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Tablecruncher. If not, see . + */ + +#include "csvloader.hh" + +#include +#include +#include +#include +#include +#include + +#include "csvfsm.hh" +#include "csvparser.hh" +#include "parallelutil.hh" + + +namespace { + +const uint64_t NO_RECORD_START = UINT64_MAX; +const size_t MAX_CHUNKS = 256; +const uint64_t TARGET_CHUNK = 8ull << 20; +// Chunks per thread. Efficiency cores are ~2.5x slower, so one chunk per thread leaves the +// performance cores idle through the whole tail; oversubscribing lets work stealing even it +// out. Extra chunks are nearly free — prescan cost is per byte, not per chunk. +const size_t CHUNKS_PER_THREAD = 4; + + +/* + * First physical line start strictly after `from`. + * + * Unambiguous at byte level, which is what makes stateless splitting possible: a '\n' inside + * a quoted field is still a '\n' byte, and a nominal offset landing on the '\n' of a CRLF + * yields the same start the real reader would use. + */ +uint64_t nextLineStart(const char* data, uint64_t len, uint64_t from) { + uint64_t i = from; + while( i < len && data[i] != '\n' && data[i] != '\r' ) + ++i; + if( i >= len ) + return len; + if( data[i] == '\r' && i + 1 < len && data[i+1] == '\n' ) + return i + 2; + return i + 1; +} + + +struct ChunkPrescan { + uint64_t begin = 0; + uint64_t end = 0; + bool endEnclosed[2] = { false, false }; // indexed by the carry-in hypothesis + uint64_t firstRecordStart[2] = { NO_RECORD_START, NO_RECORD_START }; +}; + + +/* + * Runs the structural state machine over one chunk under BOTH carry-in hypotheses at once. + * + * The two runs converge almost immediately – a single quote character resynchronises them – + * so they are executed lock-step and the second is dropped the moment the two `enclosed` + * bits agree at a line boundary. Real cost is around 1.05x a single scan, fully parallel. + */ +void prescanChunk(const char* data, const CsvDialect& dialect, ChunkPrescan& out) { + StructuralSink sink; + std::string scratch; + + // NUL removal shifts both character adjacency and the "last character of the line" test, + // so the prescan has to strip them exactly like the parse does. Checking once per chunk + // keeps the common (NUL-free) case free. + const bool chunkHasNul = csvRangeHasNul(data, out.begin, out.end); + + bool encA = false; // hypothesis: carry-in NOT enclosed + bool encB = true; // hypothesis: carry-in enclosed + bool converged = false; + uint64_t rsA = NO_RECORD_START; + uint64_t rsB = NO_RECORD_START; + + CsvLineCursor cursor(data, out.begin, out.end); + const char* lineBegin = nullptr; + const char* lineEnd = nullptr; + bool reachedEnd = false; + + for(;;) { + const uint64_t lineStart = cursor.pos(); + // a record starts at any line start where the machine is not inside a quoted field + if( !encA && rsA == NO_RECORD_START ) rsA = lineStart; + if( !encB && rsB == NO_RECORD_START ) rsB = lineStart; + + if( !cursor.next(lineBegin, lineEnd, reachedEnd) ) + break; + + const char* p = lineBegin; + const char* e = lineEnd; + if( chunkHasNul ) + csvStripNuls(lineBegin, lineEnd, scratch, p, e); + + csvScanLine(p, e, dialect, encA, sink); + if( !converged ) { + csvScanLine(p, e, dialect, encB, sink); + if( encA == encB ) + converged = true; + } else { + encB = encA; + } + } + + out.endEnclosed[0] = encA; + out.endEnclosed[1] = encB; + out.firstRecordStart[0] = rsA; + out.firstRecordStart[1] = rsB; +} + + +} // namespace + + +/* + * One function, one place, one decision. + */ +LoadPlan CsvLoader::planLoad(const CsvDefinition& def, uint64_t fileLength, + const Utf8ValidationResult& validation, + bool haveBuffer, int maxLines, bool resizeRows, bool forceParallel) { + LoadPlan plan; + + // + // Stream vs buffer first. Deriving the parallel gate from bufferPathSupports() rather + // than repeating an encoding list keeps "parallel is a subset of buffer" structural. + // + if( !haveBuffer ) { + plan.reason = "no mapped buffer"; + return plan; + } + if( !CsvParser::bufferPathSupports(def.encoding) ) { + plan.reason = "encoding needs the code-unit reader"; + return plan; + } + plan.path = LoadPath::Buffer; + + // the escape hatch you tell a bug reporter to flip + if( std::getenv("TCRUNCHER_DISABLE_PARALLEL") ) { + plan.reason = "disabled by TCRUNCHER_DISABLE_PARALLEL"; + return plan; + } + // probing, preview and paste always stay serial – loadParallel() cannot honour maxLines + if( maxLines != 0 || !resizeRows ) { + plan.reason = "probing / preview path"; + return plan; + } + // Latin-1/Latin-9/Win1252 are buffer-capable but transcode by DROPPING bytes 0x80-0x9F, + // which changes line length and therefore the escape lookahead – a raw-byte prescan is + // not faithful for them + if( def.encoding != CsvDefinition::ENC_UTF8 && def.encoding != CsvDefinition::ENC_NONE ) { + plan.reason = "encoding transcodes by dropping bytes"; + return plan; + } + // The gate. Helper::fixUtf8() is provably the identity only when the parsed region is + // wholly valid UTF-8; otherwise utf8::replace_invalid can DISCARD up to three trailing + // bytes of a line, which can swallow a closing quote and flip the enclosed state. + if( !validation.validFromBom ) { + plan.reason = "parsed region is not valid UTF-8"; + return plan; + } + // structural characters must be plain ASCII and must not collide with a line terminator + const char specials[3] = { def.delimiter, def.quote, def.escape }; + for( char c : specials ) { + if( ((unsigned char) c) >= 0x80 || c == '\n' || c == '\r' || c == '\0' ) { + plan.reason = "delimiter/quote/escape is not a safe ASCII byte"; + return plan; + } + } + if( fileLength < MIN_PARALLEL_BYTES && !forceParallel ) { + plan.reason = "file too small to be worth splitting"; + return plan; + } + + const unsigned n = Parallel::threadCount(); + if( n < 2 && !forceParallel ) { + plan.reason = "not enough cores"; + return plan; + } + + plan.path = LoadPath::Parallel; + plan.threads = n; + plan.chunkCount = std::min(MAX_CHUNKS, + std::max(n * CHUNKS_PER_THREAD, (size_t)(fileLength / TARGET_CHUNK))); + return plan; +} + + +LoadResult CsvLoader::loadParallel(const char* data, uint64_t len, const CsvDefinition& definition, + const LoadPlan& plan, CsvDataStorage& storage, LoadProgress& progress) { + LoadResult result; + + // Whoever is pumping the UI is waiting on this, so it has to be set on EVERY exit path – + // including the cancellation returns below. + struct FinishGuard { + LoadProgress& p; + ~FinishGuard() { p.finished.store(true, std::memory_order_release); } + } finishGuard{progress}; + + // Set TCRUNCHER_LOADER_TIMING=1 to get a per-phase breakdown on stderr. Cheap enough to + // leave in: one getenv and a few clock reads per load. + const bool timing = ( std::getenv("TCRUNCHER_LOADER_TIMING") != nullptr ); + auto now = []() { return std::chrono::steady_clock::now(); }; + auto t0 = now(); + auto mark = [&](const char* what) { + if( !timing ) return; + auto t = now(); + fprintf(stderr, " [loader] %-14s %6.1f ms\n", what, + std::chrono::duration(t - t0).count()); + t0 = t; + }; + + CsvDefinition def = definition; + const CsvDialect dialect = CsvParser::makeDialect(def); + const uint64_t start = std::min((uint64_t) def.bomBytes, len); + + // + // 1. Split at line starts, statelessly. + // + std::vector bounds; + bounds.push_back(start); + if( !plan.forcedNominalOffsets.empty() ) { + for( uint64_t nominal : plan.forcedNominalOffsets ) { + if( nominal <= start || nominal >= len ) continue; + uint64_t b = nextLineStart(data, len, nominal); + if( b > bounds.back() && b < len ) + bounds.push_back(b); + } + } else { + const size_t k = std::max(1, plan.chunkCount); + for( size_t i = 1; i < k; ++i ) { + uint64_t nominal = start + ((len - start) * i) / k; + uint64_t b = nextLineStart(data, len, nominal); + if( b > bounds.back() && b < len ) + bounds.push_back(b); + } + } + bounds.push_back(len); + + std::vector chunks(bounds.size() - 1); + for( size_t i = 0; i + 1 < bounds.size(); ++i ) { + chunks[i].begin = bounds[i]; + chunks[i].end = bounds[i+1]; + } + mark("split"); + + // + // 2. Prescan every chunk in parallel, under both carry-in hypotheses. + // + if( !Parallel::parallelFor(chunks.size(), plan.threads, &progress.cancelRequested, + [&](size_t i) { prescanChunk(data, dialect, chunks[i]); }) ) { + result.cancelled = true; + return result; + } + mark("prescan"); + + // + // 3. Compose the carry chain serially. O(k), k <= 256. + // + std::vector recordStarts; + bool carry = false; + for( size_t i = 0; i < chunks.size(); ++i ) { + const int h = carry ? 1 : 0; + const uint64_t rs = chunks[i].firstRecordStart[h]; + if( rs != NO_RECORD_START ) + recordStarts.push_back(rs); // a chunk wholly inside one quoted field + // yields none and folds into its neighbour + carry = chunks[i].endEnclosed[h]; + } + if( recordStarts.empty() ) + recordStarts.push_back(start); + + // + // 4. Parse the ranges in parallel. Each starts at a record boundary with an empty state, + // so there is nothing to stitch afterwards. + // + const size_t rangeCount = recordStarts.size(); + std::vector pieces(rangeCount); + std::vector pieceRowLimitHit(rangeCount, 0); + std::vector> pieceHisto(rangeCount); + + if( !Parallel::parallelFor(rangeCount, plan.threads, &progress.cancelRequested, [&](size_t j) { + CsvParser::RangeOptions opt; + opt.from = recordStarts[j]; + opt.to = (j + 1 < rangeCount) ? recordStarts[j+1] : len; + opt.finalRange = ( j + 1 == rangeCount ); + opt.maxLines = 0; + opt.resizeRows = true; + // planLoad() has already required validation.validFromBom, so every line in this + // range is valid UTF-8 and re-checking it per line would be a wasted second pass + opt.assumeValidUtf8 = true; + + CsvParser parser; + parser.sharedRowCounter = &progress.rowsDone; + parser.cancelRequested = &progress.cancelRequested; + + // the piece's row vector would otherwise double its way up from nothing, on every + // worker at once + pieces[j].reserveRows( CsvParser::estimateRowCount(data + opt.from, opt.to - opt.from) ); + + CsvDefinition localDef = def; // each worker owns its copy + pieceHisto[j] = parser.parseCsvRange(data, opt, pieces[j], &localDef); + pieceRowLimitHit[j] = parser.rowLimitExceeded ? 1 : 0; + }) ) { + result.cancelled = true; + return result; + } + mark("parse"); + + // + // 5. Barrier: the global column count. Provably equal to the serial parser's final + // act_cols, since every committed record contributes its field count to some piece. + // + table_index_t globalColumns = 0; + uint64_t totalRows = 0; + for( size_t j = 0; j < rangeCount; ++j ) { + globalColumns = std::max(globalColumns, pieces[j].columns()); + totalRows += (uint64_t) pieces[j].rows(); + if( pieceRowLimitHit[j] ) + result.rowLimitExceeded = true; + } + if( totalRows > (uint64_t) INT_MAX ) { + // table_index_t is int: assembling these would make rows() wrap negative, and the + // table would be unusable. Refuse rather than hand back something that misreports its + // own size. + result.rowLimitExceeded = true; + return result; + } + + bool anyNeedsPadding = false; + for( size_t j = 0; j < rangeCount; ++j ) { + if( pieces[j].columns() < globalColumns ) { anyNeedsPadding = true; break; } + } + + // + // 6. Phase B: bring every piece up to the global width. This is what replaces the serial + // parser's quadratic re-padding of every earlier row. Usually there is nothing to do, + // and fanning out for nothing costs more than the work. + // + if( anyNeedsPadding ) { + if( !Parallel::parallelFor(rangeCount, plan.threads, &progress.cancelRequested, [&](size_t j) { + if( pieces[j].columns() < globalColumns ) + pieces[j].resize(0, globalColumns); + }) ) { + result.cancelled = true; + return result; + } + } + mark("pad"); + + // + // 7. Phase C: concatenate in file order. Pointer copying only – roughly 24 bytes a row. + // + storage.assembleFrom(pieces, globalColumns); + + // merge the per-range histograms + for( size_t j = 0; j < rangeCount; ++j ) { + for( auto const& kv : pieceHisto[j] ) + result.histogram[kv.first] += kv.second; + } + + // the serial parser drops the trailing row once, after everything has been appended + if( storage.rows() ) + storage.deleteRows( storage.rows() - 1, storage.rows() - 1 ); + + mark("assemble"); + + progress.rowsDone.store((long) storage.rows(), std::memory_order_relaxed); + return result; +} diff --git a/src/csvloader.hh b/src/csvloader.hh new file mode 100644 index 0000000..7b91ab2 --- /dev/null +++ b/src/csvloader.hh @@ -0,0 +1,112 @@ +/* + * SPDX-License-Identifier: GPL-3.0-or-later + * + * Copyright (C) 2025 Stefan Fischerländer + * + * This file is part of Tablecruncher. + * + * Tablecruncher is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Tablecruncher is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Tablecruncher. If not, see . + */ + +#ifndef _CSVLOADER_HH +#define _CSVLOADER_HH + +// +// Multi-threaded CSV load over a mapped buffer. +// +// The whole design turns on one observation: at a PHYSICAL LINE START the state machine's +// structural state is exactly one bit – `enclosed`. `startField` is unconditionally true +// there, and nothing else survives a line boundary. A one-bit transfer function composes in +// O(k), so each chunk can be prescanned independently under both hypotheses (carry-in false +// and true) and the true carry chain resolved serially afterwards. No guessing, no verify +// round, no probabilistic fallback. +// +// No FLTK and no iostream anywhere below – this runs on worker threads. +// + +#include +#include +#include +#include +#include + +#include "globals.hh" +#include "csvdatastorage.hh" +#include "utf8validate.hh" + + +/** + * Which engine reads the file. + * + * Ordered by capability: every Parallel-eligible file is also Buffer-eligible, and planLoad() + * derives it that way rather than keeping a second encoding whitelist in step with the first. + */ +enum class LoadPath { + Stream, // istream + the code-unit reader – UTF-16/32, or no mapping available + Buffer, // the mapped buffer, single threaded + Parallel // the mapped buffer, prescanned and parsed across all cores +}; + + +/** + * The single place that decides which engine runs, and why. + */ +struct LoadPlan { + LoadPath path = LoadPath::Stream; + unsigned threads = 1; + size_t chunkCount = 1; + const char* reason = ""; // why not the faster path – shown in DEBUG builds + + // Test hook, not used by the application: force these nominal split points instead of an + // even division. Lets the harness put a chunk boundary on every byte offset in turn. + std::vector forcedNominalOffsets; +}; + + +/** + * Shared between the coordinator thread and the main thread. The main thread owns every + * widget; workers only ever touch these atomics. + */ +struct LoadProgress { + std::atomic rowsDone{0}; + std::atomic cancelRequested{false}; + std::atomic finished{false}; // set by loadParallel() on every exit path +}; + + +struct LoadResult { + std::map histogram; + bool cancelled = false; + bool rowLimitExceeded = false; // more rows than table_index_t can address – nothing was loaded + bool failed = false; // ran out of memory, or the parse threw +}; + + +namespace CsvLoader { + + // Files below this stay serial: the serial fast path already finishes in well under + // 60 ms and the fixed costs of fanning out would dominate. + const uint64_t MIN_PARALLEL_BYTES = 8ull * 1024 * 1024; + + // `forceParallel` is a TEST HOOK: it waives the file-size and core-count thresholds. It + // deliberately does NOT waive the correctness gates (encoding, UTF-8 validity, ASCII + // structural characters) – those must refuse a file no matter who is asking. + LoadPlan planLoad(const CsvDefinition& def, uint64_t fileLength, const Utf8ValidationResult& validation, + bool haveBuffer, int maxLines, bool resizeRows, bool forceParallel = false); + + LoadResult loadParallel(const char* data, uint64_t len, const CsvDefinition& definition, + const LoadPlan& plan, CsvDataStorage& storage, LoadProgress& progress); +} + +#endif diff --git a/src/csvparser.cpp b/src/csvparser.cpp index acc96d2..12b6340 100644 --- a/src/csvparser.cpp +++ b/src/csvparser.cpp @@ -28,13 +28,10 @@ #include "csvparser.hh" +#include "csvfsm.hh" - -#ifndef TCRUNCHER_PARSER_WITHOUT_UPDATE -// needed for updateStatusbar() -extern CsvApplication app; -extern CsvWindow windows[]; -#endif +#include +#include /** @@ -57,21 +54,30 @@ extern CsvWindow windows[]; */ std::map CsvParser::parseCsvStream( std::istream *input, CsvDataStorage &storage, CsvDefinition *definition, int maxLines, bool resizeRows ) { std::string line; - std::stringstream sstr; - long act_rows = 0; - long act_cols = 0; - std::vector vec; - std::map rowLengths; - + CommitState st; + st.maxLines = maxLines; + st.resizeRows = resizeRows; + + const CsvDialect dialect = makeDialect(*definition); + + // The state machine writes finished, glued row strings straight into `rowStr`. There is + // no intermediate vector-of-fields and no merge pass. + std::string rowStr; + MergedRowSink sink(&rowStr, CsvDataStorage::internalGlue()); + + // Local, NOT a member: each call parses a complete input. CsvApplication::guessDefinition + // reuses one CsvParser across its eight dialect probes, and carrying "still inside a + // quoted field" from one probe into the next corrupted the next probe's field counts — + // and so, potentially, the dialect chosen for the file. + bool enclosed = false; + // skip bomBytes input->ignore(definition->bomBytes); // read lines from istream `input` into `line` while( myGetlineEncodings( *input, line, definition->encoding) ) { - if( parseCsvState == CSVPARSER_CONST_NOT_ENCLOSED ) { - // if this line has been fully parsed, clear previous line vector - vec.clear(); - } + if( !enclosed ) + sink.startRecord(line.size()); // translate encodings to valid UTF-8 switch( definition->encoding ) { @@ -80,186 +86,353 @@ std::map CsvParser::parseCsvStream( std::istream *input, CsvDataStora Helper::fixUtf8(line); // replaces invalid chars with replacement char break; case CsvDefinition::ENC_Latin1: - line = Helper::latin1toutf8(line); - break; case CsvDefinition::ENC_Win1252: - line = Helper::win1252toutf8(line); + line = transcodeToUtf8(definition->encoding, line); break; case CsvDefinition::ENC_UTF16LE: // translation happens in myGetlineEncodings() case CsvDefinition::ENC_UTF16BE: // translation happens in myGetlineEncodings() default: break; } - // parse `line` and put cells into `vec` - parseCsvLine(vec, line, definition); - if( parseCsvState == CSVPARSER_CONST_NOT_ENCLOSED ) { - // if this line has been fully parsed - - // calculate histogram data - size_t vecSize = vec.size(); - if( input->rdstate() != std::ios_base::eofbit ) { - // only record when we're not at the last line - rowLengths[vecSize]++; - } - - // resize rows - if( resizeRows && vec.size() ) { - if( act_cols < (long)vec.size() ) { - // parsed line is longer than columns(): resize storage - storage.resize(0, (long)vec.size()); - } else if( act_cols > (long)vec.size() ) { - // die aktuelle Zeile ist kürzer als die bisherigen Zeilen: die aktuelle Zeile verlängern - vec.resize( act_cols ); - } - act_cols = std::max( act_cols, (long)vec.size() ); - } - - -// #ifdef DEBUG -// printf("--------------------------------------\n"); -// printf("DEL: _%c_\n", definition->delimiter); -// printf("ENC: %s\n", CsvDefinition::getEncodingName(definition->encoding).c_str()); -// printf("ESC: %c\n", definition->escape); -// printf("QUO: %c\n", definition->quote); -// printf("ROW: %s\n", line.c_str()); -// printf("Columns: %ld\n", (long) vec.size()); -// for( size_t i = 0; i < vec.size(); ++i) printf("%ld: |%s|\n", (long) i, vec[i].c_str()); -// printf("**************************************\n"); -// #endif - - // add parsed line back to table storage - storage.push_back(vec); - ++act_rows; - // show an update every 25,000 rows in the status bar – very expensive! - if( act_rows % 25000 == 0 ) { - sstr.clear(); - sstr.str(""); - sstr << "Parsed " << act_rows << " lines."; - #ifndef TCRUNCHER_PARSER_WITHOUT_UPDATE - windows[app.getTopWindow()].updateStatusbar(sstr.str()); - #endif - } - if( maxLines && act_rows >= maxLines) { + + // run the shared state machine over this physical line + csvScanLine( line.data(), line.data() + line.size(), dialect, enclosed, sink ); + + const bool isLastLine = ( input->rdstate() == std::ios_base::eofbit ); + + if( enclosed ) { + // still inside a quoted field: the record continues on the next physical line + sink.lineContinuation(); + } else { + sink.endField(); + if( !commitRecord(storage, rowStr, (long) sink.fields(), isLastLine, st) ) break; - } } + // stop parsing if input has been fully consumed - if( input->rdstate() == std::ios_base::eofbit ) + if( isLastLine ) break; } // end of while( myGetlineEncodings() ) // Delete last row if storage is not empty if( storage.rows() ) storage.deleteRows( storage.rows() - 1, storage.rows() - 1 ); - - // #ifdef DEBUG - // printf("rowLengths: %zu\n", rowLengths.size()); - // for( auto const & kv : rowLengths) { - // printf("Length %ld: %ld\n", kv.first, kv.second); - // } - // #endif - - return rowLengths; + + // keys in flatHisto are unique by construction, so this is a faithful conversion + return std::map(st.flatHisto.begin(), st.flatHisto.end()); } /* - Expects 'line' to be in UTF8. - + * Which encodings the flat-buffer path can handle. UTF-16 and UTF-32 are read a code unit + * at a time and transcoded while splitting lines (myGetlineEncodings), so they keep using + * the stream path. */ -void CsvParser::parseCsvLine(std::vector& vector, const std::string &line, CsvDefinition *definition) { - bool enclosed = false; // true: Falls wir innerhalb von Quotes sind - bool startField = true; // true: immer zu Beginn eines Feldes +bool CsvParser::bufferPathSupports(CsvDefinition::Encodings enc) { + switch( enc ) { + case CsvDefinition::ENC_NONE: + case CsvDefinition::ENC_UTF8: + case CsvDefinition::ENC_Latin1: + case CsvDefinition::ENC_Latin9: + case CsvDefinition::ENC_Win1252: + return true; + default: + return false; + } +} - long lineLen = line.size(); - std::string real_field = ""; - - if( parseCsvState == CSVPARSER_CONST_ENCLOSED ) { - enclosed = true; - real_field = parseCsvRemaining + "\n"; // LF "\n" is the right thing to do, as CRLF "\r\n" would be shown as "^M" in CODE_INPUT_WIDGET TODO doesn't this change inlne "\r\n" to "\n"?? - } else { - vector.clear(); + +/* + * Transcodes one 8-bit-encoded line to UTF-8. Latin-9 and everything else pass through, as + * they always have. + */ +std::string CsvParser::transcodeToUtf8(CsvDefinition::Encodings enc, const std::string& line) { + switch( enc ) { + case CsvDefinition::ENC_Latin1: return Helper::latin1toutf8(line); + case CsvDefinition::ENC_Win1252: return Helper::win1252toutf8(line); + default: return line; } +} + + +/* + * Parses the whole buffer. Adds the BOM skip and the trailing-row delete on top of + * parseCsvRange(), which does the actual work. + */ +std::map CsvParser::parseCsvBuffer( const char* data, uint64_t len, CsvDataStorage &storage, + CsvDefinition *definition, int maxLines, bool resizeRows, + bool assumeValidUtf8 ) { + RangeOptions opt; + opt.from = std::min((uint64_t) definition->bomBytes, len); + opt.to = len; + opt.finalRange = true; + opt.maxLines = maxLines; + opt.resizeRows = resizeRows; + opt.assumeValidUtf8 = assumeValidUtf8; + + std::map histogram = parseCsvRange(data, opt, storage, definition); + + // Delete last row if storage is not empty + if( storage.rows() ) + storage.deleteRows( storage.rows() - 1, storage.rows() - 1 ); - real_field.reserve(10000); // reserve 10kb to increase parsing performance: boosts performance by factor of 4 + return histogram; +} - for( long i = 0; i < lineLen; i++ ) { -// #ifdef DEBUG -// printf(" Char: %c", line[i]); -// if( enclosed ) printf(" (enclosed)"); -// printf("\n"); -// #endif +/* + * Parses one slice of a flat buffer. + * + * Reproducing the stream reader exactly matters more than anything else here. CsvLineCursor + * handles the line splitting (LF / CRLF / lone CR); on top of that this loop reproduces: + * - NUL bytes dropped from the line BEFORE the state machine sees it, which shifts both + * character adjacency and the "last character of the line" test + * - reaching the end of input yielding one final line, with eofbit set only when that line + * is empty, which is what produces (or suppresses) the trailing phantom row + * + * This is the single implementation the serial buffer path and every parallel worker share, + * so the two cannot drift apart. + */ +std::map CsvParser::parseCsvRange( const char* data, const RangeOptions& opt, CsvDataStorage &storage, + CsvDefinition *definition, bool* endedEnclosed ) { + CommitState st; + st.maxLines = opt.maxLines; + st.resizeRows = opt.resizeRows; + + const CsvDialect dialect = makeDialect(*definition); + const CsvDefinition::Encodings enc = definition->encoding; + + std::string rowStr; + MergedRowSink sink(&rowStr, CsvDataStorage::internalGlue()); + std::string scratch; // only touched when a line must be materialised + + // One check for the whole range beats one per line – but only for a full parse. A dialect + // probe stops after ten lines, so scanning the whole buffer up front would cost far more + // than the per-line checks it saves (and guessDefinition() runs eight of them). + const bool checkNulPerLine = ( opt.maxLines != 0 ); + const bool rangeHasNul = checkNulPerLine || csvRangeHasNul(data, opt.from, opt.to); + + CsvLineCursor cursor(data, opt.from, opt.to); + bool enclosed = false; + + for(;;) { + const char* lineBegin = data + opt.to; + const char* lineEnd = lineBegin; + bool reachedEnd = false; + + if( !cursor.next(lineBegin, lineEnd, reachedEnd) ) { + if( !opt.finalRange ) + break; // a non-final slice just stops at its boundary + // the reader still yields one final, empty line and sets eofbit + reachedEnd = true; + } - if( definition->quote != definition->escape && line[i] == definition->escape ) { - if( i < lineLen - 1 ) { - // nicht das letzte Zeichen: folgendes Zeichen zurückschreiben - real_field.push_back(line[i+1]); - ++i; - continue; + // == "the stream reader would have set eofbit here": end of input reached, and the + // line is empty once its NUL bytes have been dropped + bool eofNow = false; + if( reachedEnd && opt.finalRange ) { + eofNow = true; + for( const char* q = lineBegin; q < lineEnd; ++q ) { + if( *q != '\0' ) { eofNow = false; break; } } } - if( line[i] == definition->quote ) { - // QUOTE gefunden - if( i < lineLen - 1 && line[i+1] == definition->quote ) { - // Doppelquote: "" - if( enclosed ) { - real_field.push_back(line[i]); - ++i; - continue; - } else { - enclosed = true; - continue; - } - } else { - // Einfaches Quote: " - if( enclosed ) { - // Quotierung endet - enclosed = false; - } else if( startField ) { - // Wir sind nicht enclosed, und am Beginn eines Feldes - enclosed = true; - } else { - // Wir sind nicht enclosed, aber mitten im Feld: einfaches Quote zurückschreiben - real_field.push_back(line[i]); + if( !enclosed ) + sink.startRecord((size_t)(lineEnd - lineBegin) + 1); + + // + // Produce the bytes the state machine should see. The fast path – valid UTF-8, no + // NULs – hands it a view straight into the mapping and copies nothing at all. + // + const char* p = lineBegin; + const char* e = lineEnd; + const bool hasNul = rangeHasNul && csvStripNuls(lineBegin, lineEnd, scratch, p, e); + + switch( enc ) { + case CsvDefinition::ENC_NONE: + case CsvDefinition::ENC_UTF8: + if( hasNul ) { + Helper::fixUtf8(scratch); + p = scratch.data(); e = p + scratch.size(); + } else if( !opt.assumeValidUtf8 && !utf8::is_valid(lineBegin, lineEnd) ) { + scratch.assign(lineBegin, lineEnd); + Helper::fixUtf8(scratch); + p = scratch.data(); e = p + scratch.size(); } - continue; - } + break; + case CsvDefinition::ENC_Latin1: + case CsvDefinition::ENC_Win1252: + scratch = transcodeToUtf8(enc, hasNul ? scratch : std::string(lineBegin, lineEnd)); + p = scratch.data(); e = p + scratch.size(); + break; + default: + // Latin-9 and anything else: no transcoding, exactly as on the stream path + break; } - if( startField ) { - // wir sind am Beginn eines Feldes, aber wir haben kein Quotierungszeichen - startField = false; + csvScanLine( p, e, dialect, enclosed, sink ); + + if( enclosed ) { + sink.lineContinuation(); + } else { + sink.endField(); + if( !commitRecord(storage, rowStr, (long) sink.fields(), eofNow, st) ) + break; } + if( eofNow ) + break; + } + + if( endedEnclosed ) + *endedEnclosed = enclosed; - if( line[i] == definition->delimiter && !enclosed ) { - // Zeichen ist ein Seperator und wir sind nicht quotiert: aktuelles Feld zurückschreiben - vector.push_back( real_field ); - real_field = ""; - enclosed = false; - startField = true; - continue; + return std::map(st.flatHisto.begin(), st.flatHisto.end()); +} + + +/* + * Appends one finished record to `storage`, maintaining the histogram, the column count and + * the progress/cancellation bookkeeping. Shared by both parse loops. + * + * `isLastLine` mirrors the stream reader's eofbit: the last record's length is deliberately + * left out of the histogram. + * + * @return false when parsing must stop (row cap, maxLines, or a cancellation request) + */ +bool CsvParser::commitRecord(CsvDataStorage &storage, std::string &rowStr, long fieldCount, + bool isLastLine, CommitState &st) { + // calculate histogram data + if( !isLastLine ) { + bool seen = false; + for( auto& kv : st.flatHisto ) { + if( kv.first == fieldCount ) { ++kv.second; seen = true; break; } } - - real_field.push_back(line[i]); - - } // END for - - if( !enclosed ) { - // Zeile abgearbeitet und wir sind nicht mehr quotiert: aktuelles Feld zurückschreiben - vector.push_back( real_field ); - parseCsvState = CSVPARSER_CONST_NOT_ENCLOSED; - } else { - // wir sind am Zeilenende quotiert!? neue Zeile quotiert beginnen und Zeile noch nicht an storage anhängen - parseCsvState = CSVPARSER_CONST_ENCLOSED; - parseCsvRemaining = real_field; + if( !seen ) { + st.flatHisto.emplace_back( fieldCount, 1L ); + } + } + + // resize rows + if( st.resizeRows && fieldCount ) { + if( st.act_cols < fieldCount ) { + // parsed line is longer than columns(): resize storage + storage.resize(0, fieldCount); + } else if( st.act_cols > fieldCount ) { + // this row is shorter than the ones before it: pad it out with empty fields + rowStr.append( (size_t)(st.act_cols - fieldCount), CsvDataStorage::internalGlue() ); + } + st.act_cols = std::max( st.act_cols, fieldCount ); } + // add parsed line back to table storage + storage.push_back( std::move(rowStr) ); + ++st.act_rows; + + // publish progress / poll for cancellation in coarse batches so that the counter's cache + // line doesn't ping-pong between threads + if( (st.act_rows & 0x0FFF) == 0 ) { + if( sharedRowCounter ) + sharedRowCounter->fetch_add(4096, std::memory_order_relaxed); + if( cancelRequested && cancelRequested->load(std::memory_order_relaxed) ) + return false; + } + // show an update every 25,000 rows in the status bar – very expensive! + // (only the owning thread ever installs onProgress, so workers pay nothing here) + if( onProgress && st.act_rows % 25000 == 0 ) { + onProgress(st.act_rows); + } + if( st.maxLines && st.act_rows >= st.maxLines ) { + return false; + } + if( st.act_rows >= (long) INT_MAX ) { + // table_index_t is int – one more row and rows() would silently wrap + rowLimitExceeded = true; + return false; + } + return true; +} + + +/* + * Reduces a CsvDefinition to what the state machine actually branches on. + */ +CsvDialect CsvParser::makeDialect(const CsvDefinition& definition) { + CsvDialect d; + d.delimiter = definition.delimiter; + d.quote = definition.quote; + d.escape = definition.escape; + d.escapeActive = ( definition.quote != definition.escape ); + d.buildSpecialTable(); + return d; +} + + +/* + * Estimates how many rows a buffer holds, by sampling the average line length over the + * first 64 KB. Used only to reserve storage up front – being wrong costs a little memory or + * a little reallocation, never correctness. + */ +size_t CsvParser::estimateRowCount(const char* data, uint64_t len) { + const uint64_t SAMPLE = 64 * 1024; + if( len == 0 || data == nullptr ) + return 0; + + const uint64_t got = std::min(SAMPLE, len); + + // count line ends the way myGetline() does: LF, CRLF and a lone CR all end one line + uint64_t lines = 0; + for( uint64_t i = 0; i < got; ++i ) { + if( data[i] == '\n' ) + ++lines; + else if( data[i] == '\r' && (i + 1 >= got || data[i+1] != '\n') ) + ++lines; + } + if( lines == 0 ) + return 1; + + double avgLineLen = (double) got / (double) lines; + return clampRowEstimate(((double) len / avgLineLen) * 1.05, len); // small headroom +} + + +/* + * Keeps a row-count estimate sane: at least one row, and never more than fits in + * max(256 MB, bytes/4) of row vector. + */ +size_t CsvParser::clampRowEstimate(double estimate, uint64_t bytes) { + if( estimate < 1 ) + return 1; + const double memBudget = std::max(256.0 * 1024 * 1024, (double) bytes / 4.0); + const double maxRows = memBudget / (double) sizeof(std::string); + return (size_t) std::min(estimate, maxRows); } +/* + * Same, for an istream. Leaves the stream rewound to 0. + */ +size_t CsvParser::estimateRowCount(std::istream& input, int64_t fileLength) { + const size_t SAMPLE = 64 * 1024; + if( fileLength <= 0 ) + return 0; + + std::vector buf(SAMPLE); + input.clear(); + input.seekg(0); + input.read(buf.data(), SAMPLE); + size_t got = (size_t) input.gcount(); + input.clear(); + input.seekg(0); + if( got == 0 ) + return 0; + + size_t rows = estimateRowCount(buf.data(), got); + if( rows == 0 ) + return 0; + // scale the sample's density up to the whole file + return clampRowEstimate((double) rows * ((double) fileLength / (double) got), (uint64_t) fileLength); +} // @@ -307,13 +480,18 @@ std::istream& CsvParser::myGetline(std::istream& is, std::string& t) { * TODO UTF-32LE and UTF-32BE */ std::istream& CsvParser::myGetlineEncodings(std::istream& is, std::string& t, CsvDefinition::Encodings enc) { - if( enc == CsvDefinition::ENC_UTF8 || + if( enc == CsvDefinition::ENC_NONE || // treated as UTF-8, exactly as parseCsvStream() does + enc == CsvDefinition::ENC_UTF8 || enc == CsvDefinition::ENC_Latin1 || enc == CsvDefinition::ENC_Latin9 || enc == CsvDefinition::ENC_Win1252 ) { return myGetline(is, t); } + // Anything below reads fixed-width code units. ENC_NONE used to land here and, with + // unitLength defaulting to 1, appended nothing at all – every ENC_NONE file parsed as an + // empty table. setTypeByUser() maps ENC_NONE to ENC_UTF8 before the load, so the app + // never hit it, but the two parse paths have to agree. codeUnitReturn_t nextCodeUnit; int unitLength = 1; bool bigEndian = true; @@ -371,7 +549,11 @@ std::istream& CsvParser::myGetlineEncodings(std::istream& is, std::string& t, Cs // returns a code unit (and the value of the last octet so that calling methods can see an EOF marker) CsvParser::codeUnitReturn_t CsvParser::getNextCodeUnit(std::streambuf *sb, int unitLength, bool bigEndian) { - uint8_t streamBytes[4]; + // MUST be zero-initialised: on EOF the loop below breaks early and leaves the remaining + // bytes unwritten. Reading them back was undefined behaviour, and the resulting garbage + // could compare equal to the 0x000A line terminator – which made UTF-16 line splitting + // depend on whatever happened to be on the stack. + uint8_t streamBytes[4] = {0, 0, 0, 0}; codeUnitReturn_t ret; int octet = 0; // write unitLength bytes into codeUnit32 in given order (BE or LE) diff --git a/src/csvparser.hh b/src/csvparser.hh index 62198c9..d9b330f 100644 --- a/src/csvparser.hh +++ b/src/csvparser.hh @@ -31,6 +31,8 @@ #include #include #include +#include +#include // for reading utf16 #include @@ -40,16 +42,8 @@ #include "helper.hh" #include "globals.hh" #include "csvdatastorage.hh" +#include "csvfsm.hh" -#ifndef TCRUNCHER_PARSER_WITHOUT_UPDATE -#include "csvapplication.hh" -#include "csvwindow.hh" -#endif - - -#define CSVPARSER_CONST_ENCLOSED 1 -#define CSVPARSER_CONST_NOT_ENCLOSED 0 -#define CSVPARSER_LINES_TO_TEST_FOR_CSV_TYPE 5 @@ -69,11 +63,73 @@ class CsvParser { } codeUnitReturn_t; public: std::map parseCsvStream( std::istream *input, CsvDataStorage &storage, CsvDefinition *definition, int maxLines=0, bool resizeRows=true ); + + // + // Same parse, but over a flat buffer (a memory mapping) instead of an istream. Produces + // byte-identical output for every encoding bufferPathSupports() accepts, while avoiding + // the byte-at-a-time streambuf reads and – for the common all-valid-UTF-8, no-NUL case – + // copying each line at all: the state machine reads straight out of the mapping. + // + // `assumeValidUtf8`: the caller has already proven [bomBytes, len) is valid UTF-8, so + // Helper::fixUtf8() is provably the identity and the per-line validity check – a second + // full pass over the file – can be skipped. + std::map parseCsvBuffer( const char* data, uint64_t len, CsvDataStorage &storage, CsvDefinition *definition, int maxLines=0, bool resizeRows=true, bool assumeValidUtf8=false ); + static bool bufferPathSupports(CsvDefinition::Encodings enc); // UTF-16/32 need the code-unit reader, so they stay on the stream + + /** + * One self-contained slice of a buffer. + * + * A slice always starts on a record boundary. Non-final slices stop cleanly at `to`, + * which must be a physical line start; only the final slice applies end-of-input + * semantics (the trailing phantom record, and leaving the last record out of the + * histogram). No slice ever deletes the trailing row – the caller does that once, after + * all slices have been concatenated. + */ + struct RangeOptions { + uint64_t from = 0; + uint64_t to = 0; // exclusive + bool finalRange = true; // `to` is the end of the input + int maxLines = 0; + bool resizeRows = true; + bool assumeValidUtf8 = false; // see parseCsvBuffer() + }; + // `endedEnclosed`, when given, reports whether the slice stopped inside a quoted field. + std::map parseCsvRange( const char* data, const RangeOptions& opt, CsvDataStorage &storage, + CsvDefinition *definition, bool* endedEnclosed = nullptr ); + + // + // Progress / cancellation hooks. + // + // CsvParser must stay free of FLTK so that it can also run on a worker thread (see + // docs/dev/parallel-loading-plan.md). The owning thread injects whatever it needs here. + // + std::function onProgress; // called every 25,000 rows – owning thread ONLY (serial path) + std::atomic* sharedRowCounter = nullptr; // incremented in batches of 4096 – safe to read from another thread + std::atomic* cancelRequested = nullptr; // polled every 4096 rows; parsing stops cooperatively when set + + bool rowLimitExceeded = false; // set when parsing stopped because the table hit INT_MAX rows + + static CsvDialect makeDialect(const CsvDefinition& definition); // CsvDefinition -> what the FSM branches on + static std::string transcodeToUtf8(CsvDefinition::Encodings enc, const std::string& line); // Latin-1 / Win1252 -> UTF-8; everything else passes through + static size_t estimateRowCount(std::istream& input, int64_t fileLength); // samples the first 64 KB; for reserveRows() + static size_t estimateRowCount(const char* data, uint64_t len); // ditto, over a mapped buffer + static size_t clampRowEstimate(double estimate, uint64_t bytes); // >= 1 row, and never more than the memory budget allows + private: - int parseCsvState = CSVPARSER_CONST_NOT_ENCLOSED; - std::string parseCsvRemaining = ""; - - void parseCsvLine(std::vector& vector, const std::string &line, CsvDefinition *definition); + // Everything a parse loop accumulates while walking records, plus the two settings that + // are fixed for the whole parse. Shared so the stream path and the buffer path cannot + // drift apart in their bookkeeping. + struct CommitState { + long act_rows = 0; + long act_cols = 0; + int maxLines = 0; + bool resizeRows = true; + std::vector> flatHisto; // row-length histogram; 1-3 entries typically + }; + // Appends one finished record. Returns false when parsing must stop. + bool commitRecord(CsvDataStorage &storage, std::string &rowStr, long fieldCount, + bool isLastLine, CommitState &st); + static std::istream& myGetline(std::istream& is, std::string& t); static std::istream& myGetlineEncodings(std::istream& is, std::string& t, CsvDefinition::Encodings enc); static codeUnitReturn_t getNextCodeUnit(std::streambuf *sb, int unitLength, bool bigEndian=true); diff --git a/src/csvtable.cpp b/src/csvtable.cpp index 5b830e8..b5a7e96 100644 --- a/src/csvtable.cpp +++ b/src/csvtable.cpp @@ -30,6 +30,8 @@ #include "csvtable.hh" #include "macro.hh" +#include // sortTable() injects an Fl::check() progress callback into CsvDataStorage + extern Macro macro; // to use JS for searching @@ -828,7 +830,11 @@ int CsvTable::exportJSON(std::string path, void (*cb)(const char*, void *), void * sortType 0:Numerical, 1:String, 2:String (ignore case) – default: 1 */ void CsvTable::sortTable(table_index_t column, bool ascending, int sortType) { + // CsvDataStorage knows nothing about FLTK; inject the event pump so that a long + // sort keeps repainting exactly as it did when sort() called Fl::check() itself. + storage.setProgressCallback( [](size_t){ Fl::check(); } ); storage.sort(column, ascending, sortType); + storage.setProgressCallback( nullptr ); } @@ -1101,25 +1107,14 @@ bool CsvTable::cellContainsLineBreak(table_index_t R, table_index_t C) { /* - Returns maximum length and average length of the contents of a given column (in std::string bytes) + Returns maximum and average content length for EVERY column, in one pass per row. + + The per-cell formulation this replaces rescanned each row string from byte zero for every + column — O(row length * columns) per row, which turned a wide table into a multi-minute + freeze after loading. */ -std::pair CsvTable::maximumContentLength(table_index_t col, table_index_t max_probe_rows) { - int max_length = 0, average_length = 0; - uint64_t sum_length = 0; - table_index_t probe_rows = getNumberRows(); - if( max_probe_rows > 0 ) - probe_rows = std::min(probe_rows, max_probe_rows); - if( probe_rows > 0 ) { - for(table_index_t r = 0; r < probe_rows; ++r) { - int cell_length = getCell(r, col).length(); - if( cell_length > max_length ) { - max_length = cell_length; - } - sum_length += cell_length; - } - average_length = (int)(sum_length / probe_rows); - } - return std::make_pair(max_length, average_length); +std::vector> CsvTable::columnContentLengths(table_index_t max_probe_rows) { + return storage.columnContentLengths(max_probe_rows); } diff --git a/src/csvtable.hh b/src/csvtable.hh index a058553..2df17be 100644 --- a/src/csvtable.hh +++ b/src/csvtable.hh @@ -141,7 +141,7 @@ public: CsvDataStorage &getStorage(); void setStorage(CsvDataStorage &storage); bool cellContainsLineBreak(table_index_t R, table_index_t C); // returns true if content of that cell contains line breaks (\n) - std::pair maximumContentLength(table_index_t col, table_index_t max_probe_rows = 0 ); // returns the maximum and average length of the content of the given column (in characters) + std::vector> columnContentLengths(table_index_t max_probe_rows = 0); // same, for every column at once – one pass per row instead of one per cell /************************************************************************************ diff --git a/src/csvwindow.cpp b/src/csvwindow.cpp index 9781abe..0078fc4 100644 --- a/src/csvwindow.cpp +++ b/src/csvwindow.cpp @@ -21,6 +21,11 @@ #include "csvwindow.hh" +#include "csvguess.hh" +#include "csvloader.hh" + +#include +#include @@ -331,9 +336,11 @@ void CsvWindow::setWindowSlotUsed(bool state) { */ bool CsvWindow::loadFile(std::string filename, bool askUser, bool reopen) { std::ifstream input; - long fileLength; + MappedFile mf; + Utf8ValidationResult validation; + int64_t fileLength; std::stringstream sstr; - CsvParser *parser = new CsvParser(); + CsvParser parser; std::pair guessedEncoding; std::pair guessedDefinition; CsvDefinition definition; @@ -352,11 +359,32 @@ bool CsvWindow::loadFile(std::string filename, bool askUser, bool reopen) { // Length of file: needed for guessEncoding fileLength = Helper::getFileSize(filename); - + + // + // Map the whole file once and let every consumer read the same bytes: dialect guessing, + // encoding detection and the parse itself. That removes eight seekg(0) re-reads and a + // full extra streaming pass over the file. + // + // NOTE (Windows): the mapping is raw bytes, while `input` is opened in text mode. CRLF + // handling is unaffected – the reader has always treated \r, \r\n and a lone \r + // identically – but a 0x1A (Ctrl-Z) byte no longer truncates the table. That is a + // silent-data-loss bug fixed, and it is observable: such files now open in full. + // + bool haveBuffer = mf.open(filename); + if( haveBuffer ) + mf.adviseSequential(); + // guess properties - guessedDefinition = app.guessDefinition(&input); + if( haveBuffer ) { + guessedDefinition = CsvGuess::definition(mf.data(), mf.size()); + // `validation` also tells us whether the parsed region is wholly valid UTF-8, which + // is what proves Helper::fixUtf8() is the identity for this file + guessedEncoding = CsvGuess::encoding(mf.data(), mf.size(), validation); + } else { + guessedDefinition = CsvGuess::definition(&input); + guessedEncoding = CsvApplication::guessEncoding(&input, fileLength); + } definition = guessedDefinition.first; - guessedEncoding = CsvApplication::guessEncoding(&input, fileLength); definition.encoding = guessedEncoding.first; definition.bomBytes = guessedEncoding.second; @@ -379,9 +407,107 @@ bool CsvWindow::loadFile(std::string filename, bool askUser, bool reopen) { // Tabelle leeren und geparste Daten laden table->clearTable(); - app.showImWorkingWindow("Opening file ...", true); - histogram = parser->parseCsvStream(&input, table->getStorage(), &definition); - app.hideImWorkingWindow(); + // The grid still holds the PREVIOUS table's dimensions, and both updateStatusbar() and + // Fl::wait() below dispatch redraws – draw_cell() would then call getCell() on a storage + // that is being rebuilt, on another thread for the parallel path. Shrink the grid to + // nothing first; it is resized to the real dimensions once the load is done. + grid->rows(0); + grid->cols(0); + + // planLoad() picks the engine: the istream reader, the mapped buffer, or the mapped + // buffer parsed across all cores. It is the only place that decision is made. + LoadPlan plan = CsvLoader::planLoad(definition, (uint64_t) std::max(fileLength, 0), + validation, haveBuffer, 0, true); + #ifdef DEBUG + if( plan.path != LoadPath::Parallel ) printf("not loading in parallel: %s\n", plan.reason); + #endif + + bool loadCancelled = false; + bool rowLimitExceeded = false; + + if( plan.path == LoadPath::Parallel ) { + // + // The load runs on a background coordinator thread that fans out to workers. NO + // worker ever touches a widget or calls Fl::check(), so FLTK stays effectively + // single-threaded and needs no thread-safe build, no Fl::lock and no Fl::awake. + // The modal progress window blocks input that could mutate the table underneath us, + // so "responsive" here means: the window keeps painting and Cancel works. + // + LoadProgress progress; + app.showImWorkingWindow("Opening file ...", true, [&progress]() { + progress.cancelRequested.store(true, std::memory_order_relaxed); + }); + + LoadResult loadResult; + std::thread coordinator([&]() { + // An escaping exception on a std::thread is an immediate std::terminate, and + // bad_alloc is a realistic outcome here. Report it instead of aborting. + try { + loadResult = CsvLoader::loadParallel(mf.data(), mf.size(), definition, plan, + table->getStorage(), progress); + } catch( const std::exception& ) { + loadResult.failed = true; + progress.finished.store(true, std::memory_order_release); + } + }); + while( !progress.finished.load(std::memory_order_acquire) ) { + updateStatusbar("Parsed " + Helper::groupedIntToString( + (int) std::min(progress.rowsDone.load(std::memory_order_relaxed), INT_MAX)) + " lines."); + Fl::wait(0.03); // a timeout, not Fl::check() – don't burn a core spinning + } + coordinator.join(); + + app.hideImWorkingWindow(); + histogram = loadResult.histogram; + loadCancelled = loadResult.cancelled; + rowLimitExceeded = loadResult.rowLimitExceeded; + + if( loadResult.failed ) { + table->clearTable(); + table->updateInternals(); + CsvApplication::myFlChoice("", "Ran out of memory while opening this file.", {"Okay"}); + return false; + } + } else { + const bool useBuffer = ( plan.path == LoadPath::Buffer ); + // reserve the row vector up front – saves both time and a ~1.5x peak-RSS spike + table->getStorage().reserveRows( useBuffer + ? CsvParser::estimateRowCount(mf.data(), mf.size()) + : CsvParser::estimateRowCount(input, fileLength) ); + app.showImWorkingWindow("Opening file ...", true); + // CsvParser no longer knows about windows – inject the status bar update it used to do itself + parser.onProgress = [this](long rows) { + this->updateStatusbar("Parsed " + std::to_string(rows) + " lines."); + }; + if( useBuffer ) { + // guessEncoding already proved whether the region is valid UTF-8; without passing + // that on, the parser would re-check every line and walk the whole file again + histogram = parser.parseCsvBuffer(mf.data(), mf.size(), table->getStorage(), &definition, + 0, true, validation.validFromBom); + } else { + histogram = parser.parseCsvStream(&input, table->getStorage(), &definition); + } + app.hideImWorkingWindow(); + rowLimitExceeded = parser.rowLimitExceeded; + } + + if( loadCancelled ) { + table->clearTable(); + table->updateInternals(); + updateStatusbar("Opening cancelled."); + return false; + } + if( rowLimitExceeded ) { + if( table->getNumberRows() <= 0 ) { + // the parallel loader refuses outright rather than hand back a wrapped row count + table->clearTable(); + table->updateInternals(); + CsvApplication::myFlChoice("", "This file has more rows than Tablecruncher can address.", {"Okay"}); + return false; + } + CsvApplication::myFlChoice("Warning", "This file has more rows than Tablecruncher can address. Only the first " + + Helper::groupedIntToString(INT_MAX) + " rows have been loaded.", {"OK"}); + } table->updateInternals(); if( table->getNumberRows() == 0 || table->getNumberCols() == 0 ) { if( askUser ) { @@ -432,7 +558,6 @@ bool CsvWindow::loadFile(std::string filename, bool askUser, bool reopen) { win->redraw(); win->flush(); Fl::check(); - delete(parser); // // Show warning on large table diff --git a/src/csvwindow.hh b/src/csvwindow.hh index e072ebf..96da989 100644 --- a/src/csvwindow.hh +++ b/src/csvwindow.hh @@ -57,6 +57,7 @@ #include "csvtable.hh" #include "csvapplication.hh" #include "csvparser.hh" +#include "mappedfile.hh" namespace ui_icons { diff --git a/src/globals.hh b/src/globals.hh index 36d47fd..0990e1c 100644 --- a/src/globals.hh +++ b/src/globals.hh @@ -165,6 +165,33 @@ public: bool cancelled = false; // Used by setTypeByUser() to signal a user abortion QuoteStyles quoteStyle = QUOTE_STYLE_RFC; // the quote style used for exports + /* + * Maps the first four bytes of a file to the encoding its byte-order mark declares, and + * reports the BOM's length. `octet` must be zero-padded when the file is shorter than + * four bytes. + * + * The UTF-32LE test used to compare octet[2] twice. That is not harmless: in UTF-16LE the + * low byte comes first, so octet[2] is the LOW byte of the first character — any UTF-16LE + * file starting with U+xx00 (\u4E00 and much of CJK) matched FF FE 00 00 and was reported + * as UTF-32LE, which the reader cannot decode, so the file opened as an empty table. + * Testing octet[3] is the actual UTF-32LE BOM and still matches every real one. + */ + static Encodings fromBom(const unsigned char octet[4], int& bomBytes) { + if( octet[0] == 0xEF && octet[1] == 0xBB && octet[2] == 0xBF ) { + bomBytes = 3; return ENC_UTF8; + } else if( octet[0] == 0x00 && octet[1] == 0x00 && octet[2] == 0xFE && octet[3] == 0xFF ) { + bomBytes = 4; return ENC_UTF32BE; + } else if( octet[0] == 0xFF && octet[1] == 0xFE && octet[2] == 0x00 && octet[3] == 0x00 ) { + bomBytes = 4; return ENC_UTF32LE; + } else if( octet[0] == 0xFE && octet[1] == 0xFF ) { + bomBytes = 2; return ENC_UTF16BE; + } else if( octet[0] == 0xFF && octet[1] == 0xFE ) { + bomBytes = 2; return ENC_UTF16LE; + } + bomBytes = 0; + return ENC_NONE; + } + static std::string getEncodingName(Encodings encoding) { std::string str = ""; switch( encoding ) { diff --git a/src/helper.cpp b/src/helper.cpp index dcb8cf6..d883bdd 100644 --- a/src/helper.cpp +++ b/src/helper.cpp @@ -129,6 +129,13 @@ std::string Helper::groupedIntToString( int num, std::string sep ) { // fixes UTF8 inplace – replaces invalid octets with replace character void Helper::fixUtf8(std::string& str) { + // Fast path: for valid UTF-8 the loop below is the identity, so skip the temporary + // string, the allocation and the two byte copies entirely. This is the common case – + // the parser calls this once per line. + if( utf8::is_valid(str.begin(), str.end()) ) { + return; + } + std::string temp; try { utf8::replace_invalid(str.begin(), str.end(), back_inserter(temp)); @@ -494,10 +501,17 @@ std::string Helper::padInteger(int num, int length) { } // https://stackoverflow.com/questions/5840148/how-can-i-get-a-files-size-in-c -long Helper::getFileSize(std::string filename) { +// _stat64 on Windows, where `long` is only 32 bit and a plain stat() would report a +// truncated (possibly negative) size for files larger than 2 GB. +int64_t Helper::getFileSize(const std::string& filename) { +#ifdef _WIN64 + struct __stat64 stat_buf; + int rc = _stat64(filename.c_str(), &stat_buf); +#else struct stat stat_buf; int rc = stat(filename.c_str(), &stat_buf); - return rc == 0 ? stat_buf.st_size : -1; +#endif + return rc == 0 ? (int64_t) stat_buf.st_size : -1; } diff --git a/src/helper.hh b/src/helper.hh index ba0f1fc..bf0a71a 100644 --- a/src/helper.hh +++ b/src/helper.hh @@ -80,7 +80,7 @@ public: static std::string getDirectory(const std::string& path); static std::pair getPathWithoutExtension(const std::string& path); static std::string padInteger(int num, int length); - static long getFileSize(std::string filename); + static int64_t getFileSize(const std::string& filename); // 64-bit safe: plain `long` is 32 bit on MSVC static bool guessHasHeader(std::vector firstRow); static std::vector splitString(std::string sep, std::string str, size_t maxSplits=0); static void dumpVecVec( std::vector> &vec, size_t maxRows = 10, int colWidth = 10 ); diff --git a/src/mappedfile.cpp b/src/mappedfile.cpp new file mode 100644 index 0000000..d6e65ba --- /dev/null +++ b/src/mappedfile.cpp @@ -0,0 +1,221 @@ +/* + * SPDX-License-Identifier: GPL-3.0-or-later + * + * Copyright (C) 2025 Stefan Fischerländer + * + * This file is part of Tablecruncher. + * + * Tablecruncher is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Tablecruncher is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Tablecruncher. If not, see . + */ + +#include "mappedfile.hh" + +#include + +#ifdef _WIN64 + #include + #include "helper.hh" // utf8_to_ws() for the wide path +#else + #include + #include + #include + #include +#endif + + +MappedFile::~MappedFile() { + close(); +} + + +MappedFile::MappedFile(MappedFile&& other) noexcept { + *this = std::move(other); +} + + +MappedFile& MappedFile::operator=(MappedFile&& other) noexcept { + if( this == &other ) + return *this; + close(); + data_ = other.data_; + size_ = other.size_; + mapped_ = other.mapped_; + heap_ = std::move(other.heap_); +#ifdef _WIN64 + fileHandle_ = other.fileHandle_; + mapHandle_ = other.mapHandle_; + other.fileHandle_ = nullptr; + other.mapHandle_ = nullptr; +#else + fd_ = other.fd_; + other.fd_ = -1; +#endif + // if the source was a heap fallback, data_ pointed into its vector – re-point it at ours + if( !mapped_ && !heap_.empty() ) + data_ = heap_.data(); + other.data_ = nullptr; + other.size_ = 0; + other.mapped_ = false; + return *this; +} + + +void MappedFile::close() { +#ifdef _WIN64 + if( mapped_ && data_ ) + UnmapViewOfFile((LPCVOID) data_); + if( mapHandle_ ) { + CloseHandle((HANDLE) mapHandle_); + mapHandle_ = nullptr; + } + if( fileHandle_ && fileHandle_ != INVALID_HANDLE_VALUE ) { + CloseHandle((HANDLE) fileHandle_); + fileHandle_ = nullptr; + } +#else + if( mapped_ && data_ && size_ > 0 ) + munmap((void*) data_, (size_t) size_); + if( fd_ >= 0 ) { + ::close(fd_); + fd_ = -1; + } +#endif + heap_.clear(); + heap_.shrink_to_fit(); + data_ = nullptr; + size_ = 0; + mapped_ = false; +} + + +#ifdef _WIN64 + +bool MappedFile::open(const std::string& path) { + close(); + + std::wstring wpath = Helper::utf8_to_ws(path); + fileHandle_ = (void*) CreateFileW(wpath.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, + OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr); + if( fileHandle_ == INVALID_HANDLE_VALUE ) { + fileHandle_ = nullptr; + return false; + } + + LARGE_INTEGER li; + if( !GetFileSizeEx((HANDLE) fileHandle_, &li) ) { + close(); + return false; + } + size_ = (uint64_t) li.QuadPart; + + if( size_ == 0 ) { + // CreateFileMapping refuses a zero-length file; an empty buffer is a valid result + static const char emptyByte = 0; + data_ = &emptyByte; + mapped_ = false; + return true; + } + + mapHandle_ = (void*) CreateFileMappingW((HANDLE) fileHandle_, nullptr, PAGE_READONLY, 0, 0, nullptr); + if( mapHandle_ ) { + void* view = MapViewOfFile((HANDLE) mapHandle_, FILE_MAP_READ, 0, 0, 0); + if( view ) { + data_ = (const char*) view; + mapped_ = true; + return true; + } + CloseHandle((HANDLE) mapHandle_); + mapHandle_ = nullptr; + } + + // mapping failed – fall back to a plain read for files small enough to justify it + if( size_ > MAX_HEAP_FALLBACK ) { + close(); + return false; + } + heap_.resize((size_t) size_); + DWORD got = 0; + if( !ReadFile((HANDLE) fileHandle_, heap_.data(), (DWORD) size_, &got, nullptr) || got != size_ ) { + close(); + return false; + } + data_ = heap_.data(); + mapped_ = false; + return true; +} + + +void MappedFile::adviseSequential() { + // no direct equivalent; FILE_FLAG_SEQUENTIAL_SCAN would have to be set at CreateFile time +} + +#else + +bool MappedFile::open(const std::string& path) { + close(); + + fd_ = ::open(path.c_str(), O_RDONLY); + if( fd_ < 0 ) + return false; + + struct stat st; + if( fstat(fd_, &st) != 0 || !S_ISREG(st.st_mode) ) { + close(); + return false; + } + size_ = (uint64_t) st.st_size; + + if( size_ == 0 ) { + // mmap of length 0 is an error; an empty buffer is a perfectly valid result + static const char emptyByte = 0; + data_ = &emptyByte; + mapped_ = false; + return true; + } + + void* p = mmap(nullptr, (size_t) size_, PROT_READ, MAP_PRIVATE, fd_, 0); + if( p != MAP_FAILED ) { + data_ = (const char*) p; + mapped_ = true; + return true; + } + + // mmap failed – fall back to a plain read for files small enough to justify it + if( size_ > MAX_HEAP_FALLBACK ) { + close(); + return false; + } + heap_.resize((size_t) size_); + ssize_t got = pread(fd_, heap_.data(), (size_t) size_, 0); + if( got < 0 || (uint64_t) got != size_ ) { + close(); + return false; + } + data_ = heap_.data(); + mapped_ = false; + return true; +} + + +void MappedFile::adviseSequential() { + if( !mapped_ || size_ == 0 ) + return; +#if defined(POSIX_MADV_SEQUENTIAL) + posix_madvise((void*) data_, (size_t) size_, POSIX_MADV_SEQUENTIAL); +#elif defined(MADV_SEQUENTIAL) + madvise((void*) data_, (size_t) size_, MADV_SEQUENTIAL); +#endif +} + +#endif diff --git a/src/mappedfile.hh b/src/mappedfile.hh new file mode 100644 index 0000000..0a8c3b3 --- /dev/null +++ b/src/mappedfile.hh @@ -0,0 +1,82 @@ +/* + * SPDX-License-Identifier: GPL-3.0-or-later + * + * Copyright (C) 2025 Stefan Fischerländer + * + * This file is part of Tablecruncher. + * + * Tablecruncher is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Tablecruncher is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Tablecruncher. If not, see . + */ + +#ifndef _MAPPEDFILE_HH +#define _MAPPEDFILE_HH + +// +// A whole file as one flat, read-only byte range. +// +// Every consumer of the file – dialect guessing, encoding detection, the parse – reads the +// same bytes, which removes the repeated seekg(0) re-reads and the separate streaming pass +// that UTF-8 validation used to make. mmap is preferred over reading into a std::string +// because the mapped pages are clean and evictable: the parsed representation of a 1 GB CSV +// is already ~1.3 GB, and adding a 1 GB read buffer on top would hurt exactly the users who +// care about large files. +// +// No FLTK, no iostream – this is used from worker threads. +// + +#include +#include +#include + + +class MappedFile { +public: + MappedFile() = default; + ~MappedFile(); + + MappedFile(const MappedFile&) = delete; + MappedFile& operator=(const MappedFile&) = delete; + MappedFile(MappedFile&& other) noexcept; + MappedFile& operator=(MappedFile&& other) noexcept; + + // Maps `path` read-only. Falls back to reading the whole file onto the heap when the + // mapping fails and the file is small enough to make that sane. Returns false if neither + // worked; an empty file counts as opened, with size 0. data() is null until it succeeds. + bool open(const std::string& path); + void close(); + + const char* data() const { return data_; } + uint64_t size() const { return size_; } + + void adviseSequential(); // hint the kernel: we will read this front to back + + // A heap fallback larger than this is refused – at that point streaming is the better + // answer than doubling peak memory. + static const uint64_t MAX_HEAP_FALLBACK = 256ull * 1024 * 1024; + +private: + const char* data_ = nullptr; + uint64_t size_ = 0; + bool mapped_ = false; + std::vector heap_; + +#ifdef _WIN64 + void* fileHandle_ = nullptr; + void* mapHandle_ = nullptr; +#else + int fd_ = -1; +#endif +}; + +#endif diff --git a/src/parallelutil.hh b/src/parallelutil.hh new file mode 100644 index 0000000..114070c --- /dev/null +++ b/src/parallelutil.hh @@ -0,0 +1,141 @@ +/* + * SPDX-License-Identifier: GPL-3.0-or-later + * + * Copyright (C) 2025 Stefan Fischerländer + * + * This file is part of Tablecruncher. + * + * Tablecruncher is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Tablecruncher is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Tablecruncher. If not, see . + */ + +#ifndef _PARALLELUTIL_HH +#define _PARALLELUTIL_HH + +// +// The worker-thread policy, in one place. +// +// Both the loader and the UTF-8 validator fan out; having them each carry their own copy of +// "how many threads" and "what quality of service" is how the two drift — the validator's +// threads originally missed the QoS bump the loader documents as essential. +// +// No FLTK, no iostream. +// + +#include +#include +#include +#include +#include + +#ifdef __APPLE__ + #include +#endif + + +namespace Parallel { + +// Beyond this, throughput stops improving and allocator contention starts growing. +const unsigned MAX_THREADS = 16; + + +/** + * How many workers to use. `hint` of 0 means "ask the machine". + */ +inline unsigned threadCount(unsigned hint = 0) { + if( hint == 0 ) { + hint = std::thread::hardware_concurrency(); + if( hint == 0 ) hint = 4; + } + return std::min(hint, MAX_THREADS); +} + + +/** + * Raises the calling thread's quality of service on Apple platforms. + * + * Without this a worker can inherit UTILITY or BACKGROUND from whoever spawned it, which + * parks it on the efficiency cores — losing to the serial path outright. There is + * deliberately no attempt to detect P vs E cores: no portable API exists, and the scheduler + * already does the right thing for a foreground app once the QoS is honest. + */ +inline void raiseWorkerQos() { +#ifdef __APPLE__ + pthread_set_qos_class_self_np(QOS_CLASS_USER_INITIATED, 0); +#endif +} + + +/** + * Runs `body(i)` for i in [0, count) across at most `threads` workers, handing out indices + * with an atomic counter so a slow item — or a slow core — does not stall the rest. + * + * Returns false if `cancel` was raised before every index had been processed, so that + * "did we actually finish?" is part of the mechanism rather than a check each caller has to + * remember to repeat. + */ +template +inline bool parallelFor(size_t count, unsigned threads, std::atomic* cancel, Body body) { + if( count == 0 ) + return true; + + // never spawn more workers than there is work for them + threads = (unsigned) std::min(threadCount(threads), count); + + if( threads <= 1 ) { + for( size_t i = 0; i < count; ++i ) { + if( cancel && cancel->load(std::memory_order_relaxed) ) return false; + body(i); + } + return !( cancel && cancel->load(std::memory_order_relaxed) ); + } + + std::atomic next{0}; + std::atomic completed{true}; + // `spawned` gates the QoS bump: the caller also takes a share of the work, and the caller + // may be the main UI thread (guessEncoding validates UTF-8 from there). Re-qualifying it + // as USER_INITIATED would permanently LOWER the interactive thread's priority. + auto run = [&](bool spawned) { + if( spawned ) + raiseWorkerQos(); + for(;;) { + size_t i = next.fetch_add(1, std::memory_order_relaxed); + if( i >= count ) break; + if( cancel && cancel->load(std::memory_order_relaxed) ) { + completed.store(false, std::memory_order_relaxed); + break; + } + body(i); + } + }; + + std::vector pool; + pool.reserve(threads - 1); + for( unsigned t = 1; t < threads; ++t ) + pool.emplace_back(run, true); + run(false); + for( auto& t : pool ) + t.join(); + + // Also check the flag itself, not just whether a worker happened to observe it at a + // dispatch point: a cancel raised while every worker is inside its LAST body() call would + // otherwise see them all exit through `i >= count` and report a clean finish — handing + // back partly-parsed output as a success. + if( cancel && cancel->load(std::memory_order_relaxed) ) + return false; + return completed.load(std::memory_order_relaxed); +} + +} // namespace Parallel + +#endif diff --git a/src/utf8validate.cpp b/src/utf8validate.cpp new file mode 100644 index 0000000..4f8495f --- /dev/null +++ b/src/utf8validate.cpp @@ -0,0 +1,148 @@ +/* + * SPDX-License-Identifier: GPL-3.0-or-later + * + * Copyright (C) 2025 Stefan Fischerländer + * + * This file is part of Tablecruncher. + * + * Tablecruncher is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Tablecruncher is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Tablecruncher. If not, see . + */ + +#include "utf8validate.hh" + +#include +#include + +#include "parallelutil.hh" +#include "utf8.h" + + +namespace { + +// Below this there is nothing to parallelise; the serial answer is already instant. +const uint64_t PARALLEL_THRESHOLD = 1ull << 20; // 1 MB +// Where the parallel region starts. The (tiny) head before it is validated once per start +// offset, which is why the three answers can share all the parallel work. +const uint64_t HEAD_END = 64; + + +/** + * One range's contribution. + * + * `s` is where this range's first code point actually begins: a range boundary can land in + * the middle of a multi-byte sequence, so the scan first skips forward over continuation + * bytes. `f` is where the scan stopped, which may overshoot `end` by up to three bytes + * because the last code point in the range is validated in full. + */ +struct RangeResult { + uint64_t s = 0; + uint64_t f = 0; + bool valid = true; +}; + + +RangeResult validateRange(const char* data, uint64_t len, uint64_t start, uint64_t end) { + RangeResult r; + + // resynchronise: skip the tail of whatever sequence straddles `start` + uint64_t s = start; + while( s < len && (((unsigned char) data[s]) & 0xC0) == 0x80 ) + ++s; + r.s = s; + r.f = s; + + if( s >= end ) { + // the whole range was continuation bytes; the neighbours' chain check covers it + return r; + } + + const char* p = data + s; + const char* eos = data + len; + const char* lim = data + end; + while( p < lim ) { + // validate_next() rather than a hand-rolled DFA, so that overlongs, surrogates and + // code points above U+10FFFF are accepted or rejected exactly as utf8::is_valid does + if( utf8::internal::validate_next(p, eos) != utf8::internal::UTF8_OK ) { + r.valid = false; + r.f = (uint64_t)(p - data); + return r; + } + } + r.f = (uint64_t)(p - data); + return r; +} + + +/** + * Stitches a head range onto the shared parallel ranges. + * + * The `f == next s` chain is what makes this EXACTLY equivalent to validating the whole + * region in one go. Without it, a run of orphan continuation bytes straddling a boundary + * would be skipped by the resynchronise step on one side and never reached on the other, + * and the buffer would be reported valid when it is not. + */ +bool chainValid(uint64_t startOffset, const RangeResult& head, const std::vector& parts) { + if( !head.valid || head.s != startOffset ) + return false; + uint64_t expect = head.f; + for( const RangeResult& r : parts ) { + if( !r.valid || r.s != expect ) + return false; + expect = r.f; + } + return true; +} + +} // namespace + + +Utf8ValidationResult validateUtf8Parallel(const char* data, uint64_t len, int bomBytes, unsigned threads, bool forceSplit) { + Utf8ValidationResult out; + if( data == nullptr ) + return out; + + const uint64_t bom = std::min( bomBytes < 0 ? 0 : (uint64_t) bomBytes, len ); + const uint64_t off4 = std::min(4, len); + + // small buffer, or nothing worth splitting: one range each, no threads + if( len <= HEAD_END || (len < PARALLEL_THRESHOLD && !forceSplit) ) { + std::vector none; + out.validFrom0 = chainValid(0, validateRange(data, len, 0, len), none); + out.validFrom4 = chainValid(off4, validateRange(data, len, off4, len), none); + out.validFromBom = chainValid(bom, validateRange(data, len, bom, len), none); + return out; + } + + threads = Parallel::threadCount(threads); + + // carve [HEAD_END, len) into one range per thread + const uint64_t body = len - HEAD_END; // >= 1: len > HEAD_END here + unsigned parts = (unsigned) std::min(threads, body); + + std::vector bounds(parts + 1); + for( unsigned i = 0; i <= parts; ++i ) + bounds[i] = HEAD_END + (body * i) / parts; + bounds[parts] = len; + + std::vector results(parts); + Parallel::parallelFor(parts, threads, nullptr, [&](size_t i) { + results[i] = validateRange(data, len, bounds[i], bounds[i+1]); + }); + + // the three answers differ only in their head range – the parallel work is shared + out.validFrom0 = chainValid(0, validateRange(data, len, 0, HEAD_END), results); + out.validFrom4 = chainValid(off4, validateRange(data, len, off4, HEAD_END), results); + out.validFromBom = chainValid(bom, validateRange(data, len, bom, HEAD_END), results); + return out; +} diff --git a/src/utf8validate.hh b/src/utf8validate.hh new file mode 100644 index 0000000..b300177 --- /dev/null +++ b/src/utf8validate.hh @@ -0,0 +1,61 @@ +/* + * SPDX-License-Identifier: GPL-3.0-or-later + * + * Copyright (C) 2025 Stefan Fischerländer + * + * This file is part of Tablecruncher. + * + * Tablecruncher is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or (at + * your option) any later version. + * + * Tablecruncher is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Tablecruncher. If not, see . + */ + +#ifndef _UTF8VALIDATE_HH +#define _UTF8VALIDATE_HH + +// +// Multi-threaded UTF-8 validation over a flat buffer. +// +// Answers three questions in one pass, because the load path needs all three and they only +// differ in where they start: +// - validFrom0 is the whole buffer valid? (nothing depends on this today, +// but it is the honest answer) +// - validFrom4 is [4, len) valid? the ENCODING DECISION, chosen to +// stay bit-compatible with the +// historical istream code path +// - validFromBom is [bomBytes, len) valid? the region actually parsed, and +// therefore the gate that proves +// Helper::fixUtf8() is the identity +// +// No FLTK, no iostream – this runs on worker threads. +// + +#include + +struct Utf8ValidationResult { + bool validFrom0 = false; + bool validFrom4 = false; + bool validFromBom = false; +}; + +// +// `threads` is a hint; 0 means "pick something sensible". The result is exactly what +// utf8::is_valid() would return for each of the three regions – see the note on chunk +// stitching in utf8validate.cpp for why that is not merely approximately true. +// +// `forceSplit` is a TEST HOOK: it skips the "too small to bother" shortcut so that a tiny +// buffer still gets carved into `threads` ranges. That is what lets the harness sweep range +// boundaries across every byte offset cheaply. +Utf8ValidationResult validateUtf8Parallel(const char* data, uint64_t len, int bomBytes, + unsigned threads = 0, bool forceSplit = false); + +#endif diff --git a/tests/gen_corpus.sh b/tests/gen_corpus.sh new file mode 100755 index 0000000..9cf7859 --- /dev/null +++ b/tests/gen_corpus.sh @@ -0,0 +1,199 @@ +#!/bin/sh +# +# Generates the test corpus into tests/data/ (gitignored). +# +# ./tests/gen_corpus.sh small correctness corpus + manifest +# ./tests/gen_corpus.sh --big also the ~120-160 MB benchmark files (bench_*) +# +set -e +DIR="$(cd "$(dirname "$0")" && pwd)/data" +mkdir -p "$DIR" +BIG=0 +[ "$1" = "--big" ] && BIG=1 +DIR="$DIR" BIG="$BIG" python3 - <<'PY' +import os, random + +D = os.environ["DIR"] +BIG = os.environ["BIG"] == "1" +manifest = [] + + +def w(name, data, delim=",", quote='"', escape='"', enc="AUTO", bom=0, in_manifest=True): + if isinstance(data, str): + data = data.encode("utf-8") + with open(os.path.join(D, name), "wb") as f: + f.write(data) + if in_manifest: + manifest.append((name, delim, quote, escape, enc, bom)) + + +def alias(name, delim=",", quote='"', escape='"', enc="AUTO", bom=0): + """Add another manifest row for an existing file, under a different dialect.""" + manifest.append((name, delim, quote, escape, enc, bom)) + + +# ---------------------------------------------------------------- line endings +w("plain_lf", "a,b,c\n1,2,3\n4,5,6\n") +w("plain_lf_no_trailing_nl", "a,b,c\n1,2,3\n4,5,6") +w("plain_crlf", "a,b,c\r\n1,2,3\r\n4,5,6\r\n") +w("plain_cr_only", "a,b,c\r1,2,3\r4,5,6\r") +w("mixed_endings", "a,b,c\n1,2,3\r\n4,5,6\r7,8,9\n") + +# -------------------------------------------------------------------- quoting +w("quoted_embedded_lf", 'a,"line1\nline2",c\nd,e,f\n') +w("quoted_embedded_crlf", 'a,"line1\r\nline2",c\r\nd,e,f\r\n') +w("doubled_quotes", 'a,"he said ""hi""",c\n1,2,3\n') +w("doubled_quotes_unenclosed", 'a,""x,c\n1,2,3\n') # "" opens a quote, consumes ONE char +w("quote_midfield", 'a,mid"quote,c\n1,2,3\n') +w("quote_after_space", 'a, "spaced",c\n1,2,3\n') +w("quote_at_eol", 'a,b,c"\n1,2,3\n') +w("unterminated_quote_at_eof", 'a,b,c\n1,"unterminated\n') # deleteRows() eats the last REAL row +w("all_quotes", '""""""""""""\n""""""\n') # the non-converging dual-hypothesis case +w("quote_then_eof_no_nl", 'a,"open') + +# --------------------------------------------------------- escape dialect (\) +w("bs_escaped_delim", "a,b\\,c,d\n1,2,3\n", escape="BSLASH") +w("bs_escaped_quote", 'a,b\\"c,d\n1,2,3\n', escape="BSLASH") +w("bs_at_eol", "a,b,c\\\n1,2,3\n", escape="BSLASH") # trailing \ is NOT an escape +w("bs_before_crlf", "a,b,c\\\r\n1,2,3\r\n", escape="BSLASH") +w("bs_double", "a,b\\\\c,d\n1,2,3\n", escape="BSLASH") +w("bs_before_nul", b"a,b\\\x00\"c,d\n1,2,3\n", escape="BSLASH") # NUL removed first -> \" + +# ------------------------------------------------------------------ raggedness +w("ragged_growing", "a\na,b\na,b,c\na,b,c,d\n") +w("ragged_shrinking", "a,b,c,d\na,b,c\na,b\na\n") +w("ragged_widest_last", "a,b\na,b\na,b\na,b,c,d,e\n") +w("ragged_widest_first","a,b,c,d,e\na,b\na,b\na,b\n") + +# ------------------------------------------------------------------------ NULs +w("nul_in_field", b"a,b\x00c,d\n1,2,3\n") +w("nul_in_quotes", b'a,"b\x00c",d\n1,2,3\n') +w("nul_adjacent_delim", b"a,\x00,b\n1,2,3\n") +w("bs_nul_quote", b'a,\\\x00"b,c\n1,2,3\n', escape="BSLASH") + +# ---------------------------------------------------------------- invalid UTF-8 +w("invalid_lone_continuation", b"a,b\x80c,d\n1,2,3\n") +w("invalid_truncated_lead_eats_delim", b'a,"abc\xf0"\nx,y\n') # NOT_ENOUGH_ROOM drops the closing quote +w("invalid_overlong", b"a,\xc0\xaf,c\n1,2,3\n") +w("invalid_surrogate", b"a,\xed\xa0\x80,c\n1,2,3\n") + +# ------------------------------------------------------------------ 8-bit encodings +w("latin1", b"a,caf\xe9,c\n1,2,3\n", enc="LATIN1") +w("latin1_c1_bytes", b"a,x\x80\x9fy,c\n1,2,3\n", enc="LATIN1") # 0x80-0x9F are DROPPED +w("win1252", b"a,caf\xe9 \x80,c\n1,2,3\n", enc="WIN1252") +w("latin9", b"a,caf\xe9,c\n1,2,3\n", enc="LATIN9") + +# ------------------------------------------------------------------- UTF-16/32 +u16 = "a,b,c\n1,2,3\n" +w("utf16le_bom", b"\xff\xfe" + u16.encode("utf-16-le"), enc="AUTO") +w("utf16be_bom", b"\xfe\xff" + u16.encode("utf-16-be"), enc="AUTO") +w("utf16le_nobom", u16.encode("utf-16-le"), enc="UTF16LE") +# the preserved quirk: ASCII UTF-16LE WITH a BOM is detected as UTF-8 and parsed via NUL stripping +w("utf16le_ascii_bom", b"\xff\xfe" + u16.encode("utf-16-le"), enc="AUTO", in_manifest=False) +alias("utf16le_ascii_bom", enc="AUTO") +w("utf16_nonascii", b"\xff\xfe" + "a,café,€\n1,2,3\n".encode("utf-16-le"), enc="AUTO") +# a UTF-16LE file whose first character is U+xx00: the low byte comes first, so the BOM test +# sees FF FE 00 00 and used to report UTF-32LE, which the reader cannot decode at all +w("utf16le_cjk", b"\xff\xfe" + "\u4e00,\u4e8c,\u4e09\n1,2,3\n".encode("utf-16-le"), enc="AUTO") +w("utf32le_bom", b"\xff\xfe\x00\x00" + u16.encode("utf-32-le")[4:], enc="AUTO") +w("utf8_bom", b"\xef\xbb\xbf" + u16.encode("utf-8"), enc="AUTO") + +# ------------------------------------------------------------------- degenerate +w("empty", "") +w("one_line", "a,b,c\n") +w("one_line_no_nl", "a,b,c") +w("only_newlines", "\n\n\n\n") +w("single_column", "a\nb\nc\n") +w("single_delimiter", ",") +w("header_only", "name,age,city\n") + +# ------------------------------------------------------ JSON inside a CSV column +# RFC-quoted JSON payloads. The JSON is full of ':' and ',', so a dialect guesser that just +# prefers "whichever delimiter yields the most columns" picks ':' and shreds every row. The +# giveaway is that ':' leaves the header as ONE field while exploding the data rows. +_req = '{""user"": {""id"": ""566674135""}, ""amount"": """", ""list"": [{""code"": ""ABC123""}], ""skip"": true}' +_resp = '{""errors"": [{""code"": 759, ""message"": ""redeem failed""}], ""customer"": {""id"": 566674135, ""profiles"": [{""fields"": {}, ""name"": ""chi que""}]}, ""status"": {""code"": 400, ""message"": ""series has expired""}}' +_err = '{""error"": ""ABC123:series has expired""}' +_rows = ["id,tenant_id,trace_id,req_body,resp_body,created_at,err_body,version"] +for _i in range(6): + _rows.append('%d,101,f4bce9098bf89a62,"%s","%s",2026-06-06 00:55:38.534355+00,"%s",v2' + % (5226 + _i, _req, _resp, _err)) +w("json_in_column", "\n".join(_rows) + "\n") + +# the same payloads in a semicolon-delimited file +_semi = ["id;req_body;created_at"] +for _i in range(6): + _semi.append('%d;"%s";2026-06-06 00:55:38+00' % (5226 + _i, _req)) +w("json_semicolon", "\n".join(_semi) + "\n", delim="SEMI") + + +# ------------------------------------------------------------------------ shape +# a single quoted field spanning many lines -> a chunk fully inside it yields NONE +giant = 'a,"' + ("filler line inside one giant quoted field\n" * 400) + '",z\nnext,row,here\n' +w("one_giant_field", giant) + +# moderately wide table, exercises arrangeColumns / per-row glue +rows = [] +rows.append(",".join("c%d" % c for c in range(300))) +for r in range(500): + rows.append(",".join("v%d_%d" % (r, c) for c in range(300))) +w("wide_300cols_500rows", "\n".join(rows) + "\n") + +# a deterministic pseudo-random soup, standard dialect +random.seed(20260826) +alphabet = ['a', 'b', 'Z', ',', '"', '\\', '\r', '\n', '\x00', ';', '\t', 'é', '€', '𝄞', ' '] +soup = "".join(random.choice(alphabet) for _ in range(20000)) +w("random_soup", soup) +alias("random_soup", delim="SEMI") +alias("random_soup", delim="TAB", escape="BSLASH") + +# a few extra dialect passes over existing files, for breadth +alias("plain_lf", delim="SEMI") +alias("doubled_quotes", escape="BSLASH") +alias("all_quotes", escape="BSLASH") +alias("nul_in_quotes", delim="PIPE") + +# ------------------------------------------------------------------ benchmarks +if BIG: + # (A) ~120 MB, 6 cols, ~15% quoted, some embedded newlines and multibyte UTF-8 + random.seed(7) + with open(os.path.join(D, "bench_mid.csv"), "w", encoding="utf-8") as f: + for i in range(2_500_000): + c = ["id%d" % i, "name-%d" % (i % 9973), "%.4f" % (i * 0.37)] + r = i % 20 + if r == 0: + c.append('"quoted, with comma"') + elif r == 1: + c.append('"two\nlines"') + elif r == 2: + c.append("caf\u00e9-\u20ac") + else: + c.append("plain%d" % r) + c.append("%d" % (i % 7)) + c.append("tail") + f.write(",".join(c) + "\n") + + # (B) no-quote TSV of comparable size – the best case, where the prescan is pure memchr + with open(os.path.join(D, "bench_tsv.tsv"), "w", encoding="utf-8") as f: + for i in range(2_500_000): + f.write("id%d\tname-%d\t%.4f\tplain\t%d\ttail\n" % (i, i % 9973, i * 0.37, i % 7)) + + # (C) 1000 columns x 20k rows – exercises the arrangeColumns scan and per-row glue cost + with open(os.path.join(D, "bench_wide.csv"), "w", encoding="utf-8") as f: + f.write(",".join("col%d" % c for c in range(1000)) + "\n") + for r in range(20_000): + f.write(",".join("v%d_%d" % (r % 97, c) for c in range(1000)) + "\n") + + manifest.append(("bench_mid.csv", ",", '"', '"', "AUTO", 0)) + manifest.append(("bench_tsv.tsv", "TAB", '"', '"', "AUTO", 0)) + manifest.append(("bench_wide.csv", ",", '"', '"', "AUTO", 0)) + +with open(os.path.join(D, "manifest.tsv"), "w") as f: + f.write("# name\tdelim\tquote\tescape\tencoding\tbomBytes\n") + for row in manifest: + f.write("\t".join(str(x) for x in row) + "\n") + +print("corpus: %d files, %d manifest rows -> %s" % ( + len([n for n in os.listdir(D) if n not in ("manifest.tsv", "goldens.tsv")]), + len(manifest), D)) +PY diff --git a/tests/parsecheck.cpp b/tests/parsecheck.cpp new file mode 100644 index 0000000..62e18f2 --- /dev/null +++ b/tests/parsecheck.cpp @@ -0,0 +1,1114 @@ +/* + * SPDX-License-Identifier: GPL-3.0-or-later + * + * tc_parsecheck — headless CSV parser test harness for Tablecruncher. + * + * Links WITHOUT FLTK. If this target compiles, the data layer is still properly + * decoupled from the UI layer (see docs/dev/parallel-loading-plan.md §7). + * + * Modes: + * --serial FILE parse serially, print rows/cols/histogram/row-hash + * --golden-write DIR parse every entry of DIR/manifest.tsv, write DIR/goldens.tsv + * --golden-check DIR re-parse and compare against DIR/goldens.tsv + * --bench FILE time the serial parse + * + * Dialect flags (used by --serial / --bench): + * --delim T --quote T --escape T --enc NAME --bom N + * where T is a single character or one of TAB COMMA SEMI PIPE COLON ASTER DQUOTE BSLASH + * and NAME is one of AUTO UTF8 NONE UTF16LE UTF16BE UTF32LE UTF32BE LATIN1 LATIN9 WIN1252 + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "csvparser.hh" +#include "csvdatastorage.hh" +#include "globals.hh" +#include "helper.hh" +#include "mappedfile.hh" +#include "utf8validate.hh" +#include "csvloader.hh" +#include "csvguess.hh" +#include "utf8.h" + + +/* ------------------------------------------------------------------ helpers */ + +static uint64_t fnv1a64(const std::string& s, uint64_t h = 1469598103934665603ULL) { + for( unsigned char c : s ) { + h ^= (uint64_t) c; + h *= 1099511628211ULL; + } + return h; +} + + +static bool parseCharToken(const std::string& tok, char& out) { + if( tok.size() == 1 ) { out = tok[0]; return true; } + if( tok == "TAB" ) { out = '\t'; return true; } + if( tok == "COMMA" ) { out = ','; return true; } + if( tok == "SEMI" ) { out = ';'; return true; } + if( tok == "PIPE" ) { out = '|'; return true; } + if( tok == "COLON" ) { out = ':'; return true; } + if( tok == "ASTER" ) { out = '*'; return true; } + if( tok == "DQUOTE" ) { out = '"'; return true; } + if( tok == "BSLASH" ) { out = '\\'; return true; } + return false; +} + + +static bool parseEncodingToken(const std::string& tok, CsvDefinition::Encodings& out, bool& autoDetect) { + autoDetect = false; + if( tok == "AUTO" ) { autoDetect = true; out = CsvDefinition::ENC_UTF8; return true; } + if( tok == "UTF8" ) { out = CsvDefinition::ENC_UTF8; return true; } + if( tok == "NONE" ) { out = CsvDefinition::ENC_NONE; return true; } + if( tok == "UTF16LE" ) { out = CsvDefinition::ENC_UTF16LE; return true; } + if( tok == "UTF16BE" ) { out = CsvDefinition::ENC_UTF16BE; return true; } + if( tok == "UTF32LE" ) { out = CsvDefinition::ENC_UTF32LE; return true; } + if( tok == "UTF32BE" ) { out = CsvDefinition::ENC_UTF32BE; return true; } + if( tok == "LATIN1" ) { out = CsvDefinition::ENC_Latin1; return true; } + if( tok == "LATIN9" ) { out = CsvDefinition::ENC_Latin9; return true; } + if( tok == "WIN1252" ) { out = CsvDefinition::ENC_Win1252; return true; } + return false; +} + + +/* + * Mirrors CsvApplication::guessEncoding(std::istream*) — which lives behind FLTK and so + * cannot be linked here — including its two quirks: + * - the UTF-8 validity test starts at byte offset 4, because four read() calls have + * already consumed 4 bytes when utf8::is_valid() runs + * - files >= TCRUNCHER_NUM_UTF8_TEST_BYTES are never validated, so they stay ENC_NONE + * Only the validation quirks are re-stated here; BOM detection comes from the shared + * CsvDefinition::fromBom() so there is no second copy of that table. + */ +static std::pair detectEncoding(const std::string& path, long fileLength) { + CsvDefinition::Encodings enc = CsvDefinition::ENC_NONE; + int bomBytes = 0; + unsigned char octet[4] = {0,0,0,0}; + + std::ifstream in(path, std::ios::binary); + if( !in ) return {enc, bomBytes}; + in.read((char*)octet, 4); + std::streamsize got = in.gcount(); + for( std::streamsize i = got; i < 4; ++i ) octet[i] = 0; + + enc = CsvDefinition::fromBom(octet, bomBytes); + + // the iterator was constructed before the 4 read() calls, but reads from the same + // streambuf – so validation effectively starts at offset 4 + std::istreambuf_iterator it(in.rdbuf()); + std::istreambuf_iterator eos; + if( fileLength < TCRUNCHER_NUM_UTF8_TEST_BYTES && utf8::is_valid(it, eos) ) { + enc = CsvDefinition::ENC_UTF8; + } + return {enc, bomBytes}; +} + + +struct Summary { + long rows = 0; + long cols = 0; + std::string histo; + uint64_t hash = 0; + bool ok = false; + bool refused = false; // planLoad() declined the parallel path + std::string refusedReason; +}; + + +static std::string histoToString(const std::map& h) { + std::string s; + for( auto const& kv : h ) { + if( !s.empty() ) s += ","; + s += std::to_string(kv.first) + ":" + std::to_string(kv.second); + } + if( s.empty() ) s = "-"; + return s; +} + + +static bool g_skipHash = false; // benchmarks measure parsing, not FNV over the result + +static Summary summarise(CsvDataStorage& storage, const std::map& histo, + std::vector* rowsOut) { + Summary sum; + sum.rows = storage.rows(); + sum.cols = storage.columns(); + sum.histo = histoToString(histo); + uint64_t h = 1469598103934665603ULL; + for( table_index_t r = 0; !g_skipHash && r < storage.rows(); ++r ) { + std::string row = storage.getRow(r); + h = fnv1a64(row, h); + h ^= 0xFF; h *= 1099511628211ULL; // row separator, so row splits are hashed distinctly + if( rowsOut ) rowsOut->push_back(row); + } + sum.hash = h; + sum.ok = true; + return sum; +} + + +static Summary runSerial(const std::string& path, CsvDefinition def, std::vector* rowsOut = nullptr) { + std::ifstream input(path, std::ios::binary); + if( !input ) { + fprintf(stderr, "cannot open %s\n", path.c_str()); + return Summary(); + } + CsvDataStorage storage; + CsvParser parser; + // mirror what CsvWindow::loadFile() does, so benchmarks reflect the real load path + storage.reserveRows( CsvParser::estimateRowCount(input, Helper::getFileSize(path)) ); + std::map histo = parser.parseCsvStream(&input, storage, &def); + return summarise(storage, histo, rowsOut); +} + + +static Summary runBufferMem(const char* data, uint64_t len, CsvDefinition def, + std::vector* rowsOut = nullptr) { + CsvDataStorage storage; + CsvParser parser; + storage.reserveRows( CsvParser::estimateRowCount(data, len) ); + std::map histo = parser.parseCsvBuffer(data, len, storage, &def); + return summarise(storage, histo, rowsOut); +} + + +static Summary runBuffer(const std::string& path, CsvDefinition def, std::vector* rowsOut = nullptr) { + MappedFile mf; + if( !mf.open(path) ) { + fprintf(stderr, "cannot map %s\n", path.c_str()); + return Summary(); + } + mf.adviseSequential(); + return runBufferMem(mf.data(), mf.size(), def, rowsOut); +} + + +static void printHexRow(const std::string& row) { + for( size_t i = 0; i < row.size(); ++i ) { + printf("%02X ", (unsigned char) row[i]); + if( (i % 16) == 15 ) printf("\n"); + } + printf("\n"); +} + + +/* ------------------------------------------------------------------ manifest */ + +struct ManifestEntry { + std::string name; + CsvDefinition def; + bool autoDetect = false; +}; + + +static bool readManifest(const std::string& dir, std::vector& out) { + std::string mpath = dir + "/manifest.tsv"; + std::ifstream in(mpath); + if( !in ) { + fprintf(stderr, "cannot read manifest %s\n", mpath.c_str()); + return false; + } + std::string line; + int lineNo = 0; + while( std::getline(in, line) ) { + ++lineNo; + if( line.empty() || line[0] == '#' ) continue; + std::vector f = Helper::splitString("\t", line); + if( f.size() < 6 ) { + fprintf(stderr, "%s:%d: expected 6 tab-separated fields\n", mpath.c_str(), lineNo); + return false; + } + ManifestEntry e; + e.name = f[0]; + if( !parseCharToken(f[1], e.def.delimiter) || + !parseCharToken(f[2], e.def.quote) || + !parseCharToken(f[3], e.def.escape) || + !parseEncodingToken(f[4], e.def.encoding, e.autoDetect) ) { + fprintf(stderr, "%s:%d: bad dialect token\n", mpath.c_str(), lineNo); + return false; + } + e.def.bomBytes = std::atoi(f[5].c_str()); + out.push_back(e); + } + return true; +} + + +/* + * Compares two summaries and, when rows were captured, reports the first differing row. + */ +static bool compareRuns(const std::string& label, const Summary& a, const Summary& b, + const std::vector& ra, const std::vector& rb) { + if( !a.ok || !b.ok ) { + fprintf(stderr, "%s: a run failed\n", label.c_str()); + return false; + } + if( a.rows == b.rows && a.cols == b.cols && a.histo == b.histo && a.hash == b.hash ) + return true; + + fprintf(stderr, "DIFF %s\n", label.c_str()); + fprintf(stderr, " rows %ld / %ld\n", a.rows, b.rows); + fprintf(stderr, " cols %ld / %ld\n", a.cols, b.cols); + fprintf(stderr, " histo %s / %s\n", a.histo.c_str(), b.histo.c_str()); + fprintf(stderr, " hash %016llx / %016llx\n", (unsigned long long) a.hash, (unsigned long long) b.hash); + size_t n = std::min(ra.size(), rb.size()); + for( size_t i = 0; i < n; ++i ) { + if( ra[i] != rb[i] ) { + fprintf(stderr, " first differing row %zu:\n A: ", i); + printHexRow(ra[i]); + fprintf(stderr, " B: "); + printHexRow(rb[i]); + break; + } + } + return false; +} + + +/* + * Runs both the stream path and the flat-buffer path over the whole manifest and demands + * byte-for-byte agreement. Encodings the buffer path does not claim to handle are skipped. + */ +static int diffBuffer(const std::string& dir) { + std::vector entries; + if( !readManifest(dir, entries) ) return 2; + + int bad = 0, ran = 0, skipped = 0; + for( auto& e : entries ) { + std::string fpath = dir + "/" + e.name; + CsvDefinition def = e.def; + if( e.autoDetect ) { + auto d = detectEncoding(fpath, Helper::getFileSize(fpath)); + def.encoding = d.first; + def.bomBytes = d.second; + } + if( !CsvParser::bufferPathSupports(def.encoding) ) { ++skipped; continue; } + std::vector ra, rb; + Summary a = runSerial(fpath, def, &ra); + Summary b = runBuffer(fpath, def, &rb); + ++ran; + if( !compareRuns(e.name + " [" + CsvDefinition::getEncodingName(def.encoding) + "]", a, b, ra, rb) ) + ++bad; + } + if( bad ) { fprintf(stderr, "%d of %d differ\n", bad, ran); return 1; } + printf("stream == buffer on all %d entries (%d skipped: encoding not on the buffer path)\n", ran, skipped); + return 0; +} + + +static Summary runParallelMem(const char* data, uint64_t len, CsvDefinition def, + const std::vector& forcedOffsets, size_t chunkCount, + unsigned threads, std::vector* rowsOut = nullptr) { + Utf8ValidationResult v = validateUtf8Parallel(data, len, def.bomBytes); + LoadPlan plan = CsvLoader::planLoad(def, len, v, true, 0, true, /*forceParallel*/ true); + if( plan.path != LoadPath::Parallel ) { + Summary s; + s.refused = true; + s.refusedReason = plan.reason; + return s; + } + if( chunkCount ) plan.chunkCount = chunkCount; + if( threads ) plan.threads = threads; + plan.forcedNominalOffsets = forcedOffsets; + + CsvDataStorage storage; + LoadProgress progress; + LoadResult r = CsvLoader::loadParallel(data, len, def, plan, storage, progress); + if( r.cancelled ) return Summary(); + return summarise(storage, r.histogram, rowsOut); +} + + +static Summary runParallel(const std::string& path, CsvDefinition def, + const std::vector& forcedOffsets, size_t chunkCount, + unsigned threads, std::vector* rowsOut = nullptr) { + MappedFile mf; + if( !mf.open(path) ) { + fprintf(stderr, "cannot map %s\n", path.c_str()); + return Summary(); + } + return runParallelMem(mf.data(), mf.size(), def, forcedOffsets, chunkCount, threads, rowsOut); +} + + +/* + * Resolves a manifest entry's dialect, applying AUTO detection when asked for. + */ +static CsvDefinition resolveDef(const std::string& fpath, const ManifestEntry& e) { + CsvDefinition def = e.def; + if( e.autoDetect ) { + auto d = detectEncoding(fpath, Helper::getFileSize(fpath)); + def.encoding = d.first; + def.bomBytes = d.second; + } + return def; +} + + +/* + * Runs the whole corpus through the parallel loader at a range of chunk counts and demands + * byte-for-byte agreement with the serial buffer path. + */ +static int diffParallel(const std::string& dir) { + std::vector entries; + if( !readManifest(dir, entries) ) return 2; + + const size_t chunkCounts[] = { 1, 2, 3, 5, 8, 17, 64 }; + int bad = 0, ran = 0, refused = 0; + for( auto& e : entries ) { + std::string fpath = dir + "/" + e.name; + CsvDefinition def = resolveDef(fpath, e); + + std::vector ra; + Summary a = runBuffer(fpath, def, &ra); + + for( size_t k : chunkCounts ) { + std::vector rb; + Summary b = runParallel(fpath, def, {}, k, 0, &rb); + if( b.refused ) { + if( k == chunkCounts[0] ) { + ++refused; + printf(" refused: %-36s %s\n", e.name.c_str(), b.refusedReason.c_str()); + } + break; + } + ++ran; + char label[256]; + snprintf(label, sizeof(label), "%s [%s, %zu chunks]", + e.name.c_str(), CsvDefinition::getEncodingName(def.encoding).c_str(), k); + if( !compareRuns(label, a, b, ra, rb) ) ++bad; + } + } + if( bad ) { fprintf(stderr, "%d of %d parallel runs differ\n", bad, ran); return 1; } + printf("parallel == serial on all %d runs (%d files refused the parallel path)\n", ran, refused); + return 0; +} + + +/* + * The highest-value test in the suite. + * + * Forces a chunk boundary at EVERY byte offset of the file and diffs each parallel parse + * against the serial one, then sweeps pairs and triples of simultaneous boundaries to + * exercise carry propagation and the "chunk contains no record start" absorption. This is + * what proves boundaries landing mid-quoted-field, between the CR and LF of a CRLF, inside a + * multi-byte sequence, on a NUL, right after an escape character and inside a doubled quote + * all produce identical output. + */ +static int sweep(const std::string& path, CsvDefinition def, bool quiet) { + int64_t sz = Helper::getFileSize(path); + if( sz < 0 ) { fprintf(stderr, "cannot stat %s\n", path.c_str()); return 2; } + const uint64_t len = (uint64_t) sz; + + std::vector ra; + Summary a = runBuffer(path, def, &ra); + if( !a.ok ) return 2; + + { // make sure the parallel path is even eligible before claiming the sweep proves anything + Summary probe = runParallel(path, def, {}, 1, 1); + if( probe.refused ) { + if( !quiet ) + printf(" sweep skipped %-30s (%s)\n", path.c_str(), probe.refusedReason.c_str()); + return 0; + } + } + + int bad = 0; + long cases = 0; + for( uint64_t off = 0; off <= len; ++off ) { + std::vector rb; + Summary b = runParallel(path, def, { off }, 0, 1, &rb); + ++cases; + char label[256]; + snprintf(label, sizeof(label), "%s @1 boundary %llu", path.c_str(), (unsigned long long) off); + if( !compareRuns(label, a, b, ra, rb) ) { ++bad; if( bad > 3 ) break; } + } + + // pairs and triples, on a coarser grid so the run stays quick + const uint64_t stride = std::max(1, len / 60); + for( uint64_t o1 = 0; o1 <= len && bad <= 3; o1 += stride ) { + for( uint64_t o2 = o1; o2 <= len && bad <= 3; o2 += stride ) { + std::vector rb; + Summary b = runParallel(path, def, { o1, o2 }, 0, 1, &rb); + ++cases; + char label[256]; + snprintf(label, sizeof(label), "%s @2 boundaries %llu,%llu", + path.c_str(), (unsigned long long) o1, (unsigned long long) o2); + if( !compareRuns(label, a, b, ra, rb) ) { ++bad; continue; } + + for( uint64_t o3 = o2; o3 <= len && bad <= 3; o3 += stride * 7 ) { + std::vector rc; + Summary c = runParallel(path, def, { o1, o2, o3 }, 0, 1, &rc); + ++cases; + snprintf(label, sizeof(label), "%s @3 boundaries %llu,%llu,%llu", + path.c_str(), (unsigned long long) o1, (unsigned long long) o2, + (unsigned long long) o3); + if( !compareRuns(label, a, c, ra, rc) ) ++bad; + } + } + } + + if( bad ) { fprintf(stderr, "%s: %d of %ld sweep cases differ\n", path.c_str(), bad, cases); return 1; } + if( !quiet ) printf(" %-40s %ld boundary placements, all identical\n", path.c_str(), cases); + return 0; +} + + +static int sweepCorpus(const std::string& dir) { + std::vector entries; + if( !readManifest(dir, entries) ) return 2; + int bad = 0; + for( auto& e : entries ) { + std::string fpath = dir + "/" + e.name; + if( Helper::getFileSize(fpath) > 20000 ) continue; // sweeping is O(len^2)-ish + if( sweep(fpath, resolveDef(fpath, e), false) ) ++bad; + } + if( bad ) { fprintf(stderr, "%d corpus files failed the boundary sweep\n", bad); return 1; } + printf("boundary sweep clean over the corpus\n"); + return 0; +} + + +/* + * Differential fuzz: random content over the bytes that actually drive the state machine, + * random dialects, random chunk counts. The hand-written corpus only catches what we already + * thought of; this is what would catch an FSM/prescan divergence nobody anticipated. + */ +static int fuzzParallel(int iterations) { + unsigned seed = 987654321u; + auto rnd = [&]() { seed = seed * 1103515245u + 12345u; return (seed >> 8); }; + + const char delims[6] = { ',', ';', '\t', '|', ':', '*' }; + const char escapes[3] = { '"', '\\', '*' }; + + int bad = 0; + long compared = 0, refused = 0; + for( int it = 0; it < iterations; ++it ) { + CsvDefinition def; + def.delimiter = delims[rnd() % 6]; + def.quote = '"'; + def.escape = escapes[rnd() % 3]; + def.encoding = CsvDefinition::ENC_UTF8; + def.bomBytes = 0; + + const size_t target = 40 + (rnd() % 900); + std::string buf; + buf.reserve(target + 8); + while( buf.size() < target ) { + switch( rnd() % 16 ) { + case 0: buf.push_back(def.delimiter); break; + case 1: buf.push_back(def.quote); break; + case 2: buf.push_back(def.escape); break; + case 3: buf.push_back(','); break; + case 4: buf.push_back('"'); break; + case 5: buf.push_back('\\'); break; + case 6: buf.push_back('\r'); break; + case 7: case 8: + buf.push_back('\n'); break; + case 9: buf.push_back('\0'); break; + case 10: buf.append("\xC3\xA9"); break; // 2-byte + case 11: buf.append("\xE2\x82\xAC"); break; // 3-byte + case 12: buf.append("\xF0\x9D\x84\x9E"); break; // 4-byte + default: buf.push_back((char)(0x20 + (rnd() % 0x5F))); // printable ASCII + } + } + + std::vector ra; + Summary a = runBufferMem(buf.data(), buf.size(), def, &ra); + + for( int rep = 0; rep < 3; ++rep ) { + size_t chunks = 1 + (rnd() % 8); + std::vector rb; + Summary b = runParallelMem(buf.data(), buf.size(), def, {}, chunks, 1, &rb); + if( b.refused ) { ++refused; break; } + ++compared; + char label[160]; + snprintf(label, sizeof(label), "fuzz#%d delim=%02X esc=%02X chunks=%zu", + it, (unsigned char) def.delimiter, (unsigned char) def.escape, chunks); + if( !compareRuns(label, a, b, ra, rb) ) { + ++bad; + // dump the offending input so the case can be reproduced + fprintf(stderr, " input (%zu bytes):", buf.size()); + for( size_t i = 0; i < buf.size(); ++i ) fprintf(stderr, " %02X", (unsigned char) buf[i]); + fprintf(stderr, "\n"); + break; + } + } + if( bad >= 3 ) break; + } + if( bad ) { fprintf(stderr, "%d fuzz divergences\n", bad); return 1; } + printf("differential fuzz: %ld parallel/serial comparisons identical (%ld inputs refused)\n", + compared, refused); + return 0; +} + + +/* + * Every parseCsvStream() call must be self-contained. + * + * CsvApplication::guessDefinition reuses ONE CsvParser across its eight dialect probes, so + * any state a parser keeps between calls leaks from one probe into the next and can change + * the dialect chosen for the file. This parses each corpus file twice through a single + * parser and demands the second result match a fresh one. + */ +static int reuseCheck(const std::string& dir) { + std::vector entries; + if( !readManifest(dir, entries) ) return 2; + + int bad = 0, ran = 0; + for( auto& e : entries ) { + const std::string fpath = dir + "/" + e.name; + if( Helper::getFileSize(fpath) > 200000 ) continue; + CsvDefinition def = resolveDef(fpath, e); + + // the reference: a parser that has never been used before + std::vector ra; + Summary fresh = runSerial(fpath, def, &ra); + + // the same file, through a parser that has already parsed something else + CsvParser parser; + for( const char* warmup : { "a,\"unterminated\n", "x,y,z\n" } ) { + std::istringstream warm(warmup); + CsvDataStorage scratchStorage; + CsvDefinition warmDef = def; + parser.parseCsvStream(&warm, scratchStorage, &warmDef, 10, false); + } + std::ifstream in(fpath, std::ios::binary); + if( !in ) continue; + CsvDataStorage storage; + CsvDefinition again = def; + std::map histo = parser.parseCsvStream(&in, storage, &again); + std::vector rb; + Summary reused = summarise(storage, histo, &rb); + + ++ran; + if( !compareRuns(e.name + " [reused parser]", fresh, reused, ra, rb) ) + ++bad; + } + if( bad ) { fprintf(stderr, "%d of %d files parse differently through a reused parser\n", bad, ran); return 1; } + printf("parseCsvStream is self-contained on all %d files\n", ran); + return 0; +} + + +/* ------------------------------------------------------------- dialect guessing */ + +static std::string charToken(char c) { + switch( c ) { + case ',': return "COMMA"; + case ';': return "SEMI"; + case '\t': return "TAB"; + case '|': return "PIPE"; + case ':': return "COLON"; + case '*': return "ASTER"; + case '"': return "DQUOTE"; + case '\\': return "BSLASH"; + default: return std::string(1, c); + } +} + + +/* + * What CsvGuess makes of one file, as a single stable line. + */ +static std::string guessLine(const std::string& path) { + MappedFile mf; + if( !mf.open(path) ) + return ""; + std::pair d = CsvGuess::definition(mf.data(), mf.size()); + Utf8ValidationResult v; + std::pair e = CsvGuess::encoding(mf.data(), mf.size(), v); + char buf[256]; + snprintf(buf, sizeof(buf), "%s\t%s\t%s\t%.2f\t%s\t%d", + charToken(d.first.delimiter).c_str(), charToken(d.first.quote).c_str(), + charToken(d.first.escape).c_str(), d.second, + CsvDefinition::getEncodingName(e.first).c_str(), e.second); + return buf; +} + + +/* + * Dialect guessing decides how every file opens, and it decides silently — a confident wrong + * answer never shows the format dialog. These goldens make any change to it visible. + */ +static int guesses(const std::string& dir, bool write) { + std::vector entries; + if( !readManifest(dir, entries) ) return 2; + + std::vector lines; + std::vector seen; + for( auto& e : entries ) { + if( std::find(seen.begin(), seen.end(), e.name) != seen.end() ) continue; // aliases + seen.push_back(e.name); + lines.push_back(e.name + "\t" + guessLine(dir + "/" + e.name)); + } + + const std::string gpath = dir + "/guesses.tsv"; + if( write ) { + std::ofstream out(gpath, std::ios::trunc); + if( !out ) { fprintf(stderr, "cannot write %s\n", gpath.c_str()); return 2; } + for( auto& l : lines ) out << l << "\n"; + printf("wrote %zu dialect guesses to %s\n", lines.size(), gpath.c_str()); + return 0; + } + + std::ifstream in(gpath); + if( !in ) { fprintf(stderr, "cannot read %s (run --guess-write first)\n", gpath.c_str()); return 2; } + std::vector expected; + std::string l; + while( std::getline(in, l) ) if( !l.empty() ) expected.push_back(l); + + int bad = 0; + if( expected.size() != lines.size() ) { + fprintf(stderr, "guess count mismatch: have %zu, expected %zu\n", lines.size(), expected.size()); + ++bad; + } + for( size_t i = 0; i < std::min(expected.size(), lines.size()); ++i ) { + if( expected[i] != lines[i] ) { + fprintf(stderr, "GUESS CHANGED\n was: %s\n now: %s\n", expected[i].c_str(), lines[i].c_str()); + ++bad; + } + } + if( bad ) { fprintf(stderr, "%d dialect guess(es) changed\n", bad); return 1; } + printf("all %zu dialect guesses unchanged\n", lines.size()); + return 0; +} + + +/* ------------------------------------------------------- column content lengths */ + +/* + * CsvDataStorage::columnContentLengths() replaces a per-cell get(R,C) loop that rescanned + * each row from byte zero for every column. This checks the two agree exactly – including + * the integer-division average and the ""-for-a-missing-field rule. + */ +static int colCheck(const std::string& dir) { + std::vector entries; + if( !readManifest(dir, entries) ) return 2; + + int bad = 0, ran = 0; + for( auto& e : entries ) { + std::string fpath = dir + "/" + e.name; + CsvDefinition def = resolveDef(fpath, e); + + MappedFile mf; + if( !mf.open(fpath) ) continue; + CsvDataStorage storage; + CsvParser parser; + if( CsvParser::bufferPathSupports(def.encoding) ) + parser.parseCsvBuffer(mf.data(), mf.size(), storage, &def); + else { + std::ifstream in(fpath, std::ios::binary); + parser.parseCsvStream(&in, storage, &def); + } + + for( table_index_t probe : { (table_index_t) 0, (table_index_t) 1, (table_index_t) 3, (table_index_t) 100000 } ) { + std::vector> fast = storage.columnContentLengths(probe); + + // the naive reference: exactly what CsvTable::maximumContentLength used to do + const table_index_t C = storage.columns(); + table_index_t probeRows = storage.rows(); + if( probe > 0 ) probeRows = std::min(probeRows, probe); + std::vector> slow; + if( C > 0 && probeRows > 0 ) { + for( table_index_t c = 0; c < C; ++c ) { + int maxLen = 0; + uint64_t sum = 0; + for( table_index_t r = 0; r < probeRows; ++r ) { + int l = (int) storage.get(r, c).length(); + if( l > maxLen ) maxLen = l; + sum += (uint64_t) l; + } + slow.push_back({ maxLen, (int)(sum / (uint64_t) probeRows) }); + } + } else if( C > 0 ) { + slow.assign((size_t) C, {0,0}); + } + + ++ran; + if( fast != slow ) { + fprintf(stderr, "COLUMN MISMATCH %s (probe %d): %zu vs %zu entries\n", + e.name.c_str(), (int) probe, fast.size(), slow.size()); + for( size_t c = 0; c < std::min(fast.size(), slow.size()); ++c ) { + if( fast[c] != slow[c] ) + fprintf(stderr, " col %zu: (%d,%d) vs (%d,%d)\n", + c, fast[c].first, fast[c].second, slow[c].first, slow[c].second); + } + ++bad; + } + } + } + if( bad ) { fprintf(stderr, "%d of %d column-length cases differ\n", bad, ran); return 1; } + printf("columnContentLengths matches the per-cell reference on all %d cases\n", ran); + return 0; +} + + +/* + * Times the one-pass column scan against the per-cell formulation it replaces. + */ +static int benchCols(const std::string& path, CsvDefinition def) { + MappedFile mf; + if( !mf.open(path) ) { fprintf(stderr, "cannot map %s\n", path.c_str()); return 2; } + CsvDataStorage storage; + CsvParser parser; + storage.reserveRows( CsvParser::estimateRowCount(mf.data(), mf.size()) ); + parser.parseCsvBuffer(mf.data(), mf.size(), storage, &def); + + const table_index_t probe = std::min(storage.rows(), TCRUNCHER_MAX_PROBE_ROWS_ARRANGE_COLS); + printf("table: %d rows x %d cols, probing %d rows\n", (int) storage.rows(), (int) storage.columns(), (int) probe); + + auto t0 = std::chrono::steady_clock::now(); + std::vector> fast = storage.columnContentLengths(probe); + auto t1 = std::chrono::steady_clock::now(); + printf("one-pass : %8.1f ms\n", std::chrono::duration(t1 - t0).count()); + + t0 = std::chrono::steady_clock::now(); + uint64_t sink = 0; + for( table_index_t c = 0; c < storage.columns(); ++c ) + for( table_index_t r = 0; r < probe; ++r ) + sink += storage.get(r, c).length(); + t1 = std::chrono::steady_clock::now(); + printf("per-cell : %8.1f ms (checksum %llu, %zu cols measured)\n", + std::chrono::duration(t1 - t0).count(), + (unsigned long long) sink, fast.size()); + return 0; +} + + +/* -------------------------------------------------------------- utf8 validation */ + +/* + * Checks validateUtf8Parallel() against utf8::is_valid() for one buffer, sweeping the + * thread count so that the range boundaries land on many different byte offsets. + */ +static int utf8CheckBuffer(const std::string& label, const char* data, uint64_t len, int bomBytes) { + auto oracle = [&](uint64_t from) { + uint64_t f = std::min(from, len); + return utf8::is_valid(data + f, data + len); + }; + const bool want0 = oracle(0); + const bool want4 = oracle(4); + const bool wantBom = oracle(bomBytes < 0 ? 0 : (uint64_t) bomBytes); + + int bad = 0; + for( unsigned t = 1; t <= 17; ++t ) { + Utf8ValidationResult v = validateUtf8Parallel(data, len, bomBytes, t, true); + if( v.validFrom0 != want0 || v.validFrom4 != want4 || v.validFromBom != wantBom ) { + fprintf(stderr, "UTF8 MISMATCH %s (len %llu, bom %d, threads %u)\n", + label.c_str(), (unsigned long long) len, bomBytes, t); + fprintf(stderr, " from0 got %d want %d\n", (int) v.validFrom0, (int) want0); + fprintf(stderr, " from4 got %d want %d\n", (int) v.validFrom4, (int) want4); + fprintf(stderr, " fromBom got %d want %d\n", (int) v.validFromBom, (int) wantBom); + ++bad; + break; + } + } + return bad; +} + + +static int utf8Check(const std::string& dir) { + std::vector entries; + if( !readManifest(dir, entries) ) return 2; + int bad = 0, ran = 0; + for( auto& e : entries ) { + std::ifstream in(dir + "/" + e.name, std::ios::binary); + if( !in ) continue; + std::string buf((std::istreambuf_iterator(in)), std::istreambuf_iterator()); + for( int bom = 0; bom <= 4; ++bom ) { + bad += utf8CheckBuffer(e.name, buf.data(), buf.size(), bom); + ++ran; + } + } + if( bad ) { fprintf(stderr, "%d utf8 mismatches over %d cases\n", bad, ran); return 1; } + printf("utf8 validation matches utf8::is_valid on all %d corpus cases\n", ran); + return 0; +} + + +/* + * Random buffers built from a mix of ASCII, well-formed multibyte sequences and every way a + * sequence can be malformed – then validated with forced range splitting so boundaries land + * inside sequences, inside orphan continuation runs, and on the buffer's edges. + */ +static int utf8Fuzz(int iterations) { + unsigned seed = 12345; + auto rnd = [&]() { seed = seed * 1103515245u + 12345u; return (seed >> 8); }; + + int bad = 0; + int validCases = 0; + for( int it = 0; it < iterations; ++it ) { + size_t len = 65 + (rnd() % 3000); + // half the buffers are built from well-formed pieces only, so the "valid" verdict is + // exercised as hard as the "invalid" one + const bool wellFormedOnly = ((it & 1) == 0); + std::string buf; + buf.reserve(len + 8); + while( buf.size() < len ) { + switch( rnd() % (wellFormedOnly ? 7u : 12u) ) { + case 0: case 1: case 2: case 3: + buf.push_back((char) (0x20 + (rnd() % 0x5F))); break; // ASCII + case 4: + buf.append("\xC3\xA9"); break; // 2-byte + case 5: + buf.append("\xE2\x82\xAC"); break; // 3-byte + case 6: + buf.append("\xF0\x9D\x84\x9E"); break; // 4-byte + case 7: + buf.push_back((char) (0x80 + (rnd() % 0x40))); break; // orphan continuation + case 8: + buf.append("\xC0\xAF"); break; // overlong + case 9: + buf.append("\xED\xA0\x80"); break; // surrogate + case 10: + buf.push_back((char) 0xF0); break; // truncated lead + default: + buf.push_back((char) 0x00); break; // NUL + } + } + int bom = (int) (rnd() % 5); + if( utf8::is_valid(buf.data(), buf.data() + buf.size()) ) ++validCases; + bad += utf8CheckBuffer("fuzz#" + std::to_string(it), buf.data(), buf.size(), bom); + if( bad > 5 ) break; + } + + // one buffer past the real PARALLEL_THRESHOLD, without the forceSplit hook, so the + // production code path (actual std::threads) is exercised too + { + std::string big; + big.reserve(3u << 20); + while( big.size() < (3u << 20) ) big.append("héllo,wörld,\xF0\x9D\x84\x9E,plain\n"); + Utf8ValidationResult v = validateUtf8Parallel(big.data(), big.size(), 0); + if( !v.validFrom0 || !v.validFrom4 || !v.validFromBom ) { + fprintf(stderr, "UTF8 MISMATCH: valid 3 MB buffer rejected on the threaded path\n"); + ++bad; + } + // A lead byte followed by a non-continuation byte is invalid no matter where it + // lands – either on its own, or by breaking the sequence it was written into. + big[2u << 20] = (char) 0xC3; + big[(2u << 20)+1] = (char) 0x41; + const bool wantBig = utf8::is_valid(big.data(), big.data() + big.size()); + v = validateUtf8Parallel(big.data(), big.size(), 0); + if( wantBig ) { + fprintf(stderr, "UTF8 TEST BROKEN: corrupted 3 MB buffer is still valid\n"); + ++bad; + } else if( v.validFrom0 ) { + fprintf(stderr, "UTF8 MISMATCH: corrupted 3 MB buffer accepted on the threaded path\n"); + ++bad; + } + } + + if( bad ) { fprintf(stderr, "%d utf8 fuzz mismatches\n", bad); return 1; } + printf("utf8 fuzz: %d random buffers x 17 thread counts (%d of them valid), all match utf8::is_valid;\n" + " threaded path over a 3 MB buffer OK both ways\n", iterations, validCases); + return 0; +} + + +static int goldens(const std::string& dir, bool write) { + std::vector entries; + if( !readManifest(dir, entries) ) return 2; + + std::string gpath = dir + "/goldens.tsv"; + std::vector lines; + for( auto& e : entries ) { + std::string fpath = dir + "/" + e.name; + CsvDefinition def = e.def; + if( e.autoDetect ) { + auto d = detectEncoding(fpath, Helper::getFileSize(fpath)); + def.encoding = d.first; + def.bomBytes = d.second; + } + Summary s = runSerial(fpath, def); + if( !s.ok ) return 2; + char buf[256]; + snprintf(buf, sizeof(buf), "%ld\t%ld\t%s\t%016llx", + s.rows, s.cols, s.histo.c_str(), (unsigned long long) s.hash); + lines.push_back(e.name + "\t" + buf); + } + + if( write ) { + std::ofstream out(gpath, std::ios::trunc); + if( !out ) { fprintf(stderr, "cannot write %s\n", gpath.c_str()); return 2; } + for( auto& l : lines ) out << l << "\n"; + printf("wrote %zu goldens to %s\n", lines.size(), gpath.c_str()); + return 0; + } + + std::ifstream in(gpath); + if( !in ) { fprintf(stderr, "cannot read %s (run --golden-write first)\n", gpath.c_str()); return 2; } + std::vector expected; + std::string l; + while( std::getline(in, l) ) if( !l.empty() ) expected.push_back(l); + + int bad = 0; + if( expected.size() != lines.size() ) { + fprintf(stderr, "golden count mismatch: have %zu, expected %zu\n", lines.size(), expected.size()); + ++bad; + } + size_t n = std::min(expected.size(), lines.size()); + for( size_t i = 0; i < n; ++i ) { + if( expected[i] != lines[i] ) { + fprintf(stderr, "MISMATCH\n expected: %s\n actual: %s\n", expected[i].c_str(), lines[i].c_str()); + ++bad; + } + } + if( bad ) { fprintf(stderr, "%d golden mismatch(es)\n", bad); return 1; } + printf("all %zu goldens match\n", lines.size()); + return 0; +} + + +/* ---------------------------------------------------------------------- main */ + +static void usage() { + printf( + "tc_parsecheck — headless Tablecruncher parser tests\n" + "\n" + " --serial FILE [--dump-rows] parse via the istream path and print a summary\n" + " --buffer FILE [--dump-rows] parse via the mmap path and print a summary\n" + " --diff-buffer DIR demand byte-for-byte agreement of both paths over the corpus\n" + " --golden-write DIR write DIR/goldens.tsv from DIR/manifest.tsv\n" + " --golden-check DIR compare against DIR/goldens.tsv\n" + " --bench FILE [--repeat N] time the istream parse\n" + " --bench-buffer FILE time the mmap parse\n" + " --utf8check DIR parallel UTF-8 validation vs utf8::is_valid over the corpus\n" + " --utf8fuzz N same, over N random malformed buffers\n" + " --parallel FILE [--dump-rows] parse via the parallel loader (forced on)\n" + " --diff-parallel DIR parallel vs serial over the corpus at 7 chunk counts\n" + " --sweep FILE|DIR force a chunk boundary at every byte offset and diff\n" + " --bench-parallel FILE time the parallel load\n" + " --fuzz N differential fuzz: N random inputs, serial vs parallel\n" + " --colcheck DIR columnContentLengths vs the per-cell reference\n" + " --reusecheck DIR parseCsvStream must be self-contained across calls\n" + " --guess FILE report the guessed dialect and encoding for one file\n" + " --guess-write DIR write DIR/guesses.tsv\n" + " --guess-check DIR the guessed dialect for every corpus file is unchanged\n" + " --bench-cols FILE time the column-width scan both ways\n" + "\n" + "dialect: --delim T --quote T --escape T --enc NAME --bom N\n" + " T single char | TAB COMMA SEMI PIPE COLON ASTER DQUOTE BSLASH\n" + " NAME AUTO UTF8 NONE UTF16LE UTF16BE UTF32LE UTF32BE LATIN1 LATIN9 WIN1252\n"); +} + + +int main(int argc, char** argv) { + std::string mode, arg; + CsvDefinition def; + bool autoDetect = true; // default: behave like the app and sniff the encoding + bool dumpRows = false; + int repeat = 1; + + for( int i = 1; i < argc; ++i ) { + std::string a = argv[i]; + auto need = [&](const char* what) -> std::string { + if( i + 1 >= argc ) { fprintf(stderr, "%s needs an argument\n", what); exit(2); } + return argv[++i]; + }; + if( a == "--serial" || a == "--buffer" || a == "--bench" || a == "--bench-buffer" + || a == "--golden-write" || a == "--golden-check" || a == "--diff-buffer" + || a == "--utf8check" || a == "--utf8fuzz" + || a == "--parallel" || a == "--diff-parallel" || a == "--sweep" || a == "--bench-parallel" + || a == "--fuzz" || a == "--colcheck" || a == "--bench-cols" || a == "--reusecheck" + || a == "--guess" || a == "--guess-write" || a == "--guess-check" ) { + mode = a; arg = need(a.c_str()); + } else if( a == "--delim" ) { if(!parseCharToken(need("--delim"), def.delimiter)) { fprintf(stderr,"bad --delim\n"); return 2; } } + else if( a == "--quote" ) { if(!parseCharToken(need("--quote"), def.quote)) { fprintf(stderr,"bad --quote\n"); return 2; } } + else if( a == "--escape" ) { if(!parseCharToken(need("--escape"), def.escape)) { fprintf(stderr,"bad --escape\n"); return 2; } } + else if( a == "--enc" ) { if(!parseEncodingToken(need("--enc"), def.encoding, autoDetect)) { fprintf(stderr,"bad --enc\n"); return 2; } } + else if( a == "--bom" ) { def.bomBytes = std::atoi(need("--bom").c_str()); autoDetect = false; } + else if( a == "--repeat" ) { repeat = std::atoi(need("--repeat").c_str()); } + else if( a == "--dump-rows" ) { dumpRows = true; } + else if( a == "--help" || a == "-h" ) { usage(); return 0; } + else { fprintf(stderr, "unknown option: %s\n", a.c_str()); usage(); return 2; } + } + + if( mode.empty() ) { usage(); return 2; } + + if( mode == "--golden-write" ) return goldens(arg, true); + if( mode == "--golden-check" ) return goldens(arg, false); + if( mode == "--diff-buffer" ) return diffBuffer(arg); + if( mode == "--utf8check" ) return utf8Check(arg); + if( mode == "--utf8fuzz" ) return utf8Fuzz(std::atoi(arg.c_str())); + if( mode == "--diff-parallel" ) return diffParallel(arg); + if( mode == "--fuzz" ) return fuzzParallel(std::atoi(arg.c_str())); + if( mode == "--colcheck" ) return colCheck(arg); + if( mode == "--reusecheck" ) return reuseCheck(arg); + if( mode == "--guess-write" ) return guesses(arg, true); + if( mode == "--guess-check" ) return guesses(arg, false); + if( mode == "--guess" ) { printf("%s\t%s\n", arg.c_str(), guessLine(arg).c_str()); return 0; } + + if( autoDetect ) { + auto d = detectEncoding(arg, Helper::getFileSize(arg)); + def.encoding = d.first; + def.bomBytes = d.second; + } + + if( mode == "--sweep" ) { + struct stat stbuf; + if( stat(arg.c_str(), &stbuf) == 0 && S_ISDIR(stbuf.st_mode) ) + return sweepCorpus(arg); + return sweep(arg, def, false); + } + + if( mode == "--bench-cols" ) return benchCols(arg, def); + + if( mode == "--serial" || mode == "--buffer" || mode == "--parallel" ) { + std::vector rows; + Summary s; + if( mode == "--buffer" ) s = runBuffer(arg, def, dumpRows ? &rows : nullptr); + else if( mode == "--parallel" ) s = runParallel(arg, def, {}, 0, 0, dumpRows ? &rows : nullptr); + else s = runSerial(arg, def, dumpRows ? &rows : nullptr); + if( s.refused ) { printf("parallel refused: %s\n", s.refusedReason.c_str()); return 3; } + if( !s.ok ) return 2; + printf("file: %s\n", arg.c_str()); + printf("enc: %s (bom %d)\n", CsvDefinition::getEncodingName(def.encoding).c_str(), def.bomBytes); + printf("rows: %ld\n", s.rows); + printf("cols: %ld\n", s.cols); + printf("histo: %s\n", s.histo.c_str()); + printf("hash: %016llx\n", (unsigned long long) s.hash); + if( dumpRows ) { + for( size_t r = 0; r < rows.size(); ++r ) { + printf("--- row %zu (%zu bytes)\n", r, rows[r].size()); + printHexRow(rows[r]); + } + } + return 0; + } + + if( mode == "--bench" || mode == "--bench-buffer" || mode == "--bench-parallel" ) { + g_skipHash = true; + double best = 1e30; + Summary s; + for( int i = 0; i < repeat; ++i ) { + auto t0 = std::chrono::steady_clock::now(); + if( mode == "--bench-buffer" ) s = runBuffer(arg, def); + else if( mode == "--bench-parallel" ) s = runParallel(arg, def, {}, 0, 0); + else s = runSerial(arg, def); + auto t1 = std::chrono::steady_clock::now(); + double secs = std::chrono::duration(t1 - t0).count(); + if( secs < best ) best = secs; + } + if( s.refused ) { printf("parallel refused: %s\n", s.refusedReason.c_str()); return 3; } + if( !s.ok ) return 2; + int64_t bytes = Helper::getFileSize(arg); + const char* label = (mode == "--bench-buffer") ? "buffer:" + : (mode == "--bench-parallel") ? "parallel:" : "serial:"; + printf("%-10s %.3f s %.1f MB/s %.0f rows/s (%ld rows x %ld cols)\n", + label, best, (bytes / (1024.0*1024.0)) / best, s.rows / best, s.rows, s.cols); + return 0; + } + + usage(); + return 2; +} diff --git a/tests/run_all.sh b/tests/run_all.sh new file mode 100755 index 0000000..8711650 --- /dev/null +++ b/tests/run_all.sh @@ -0,0 +1,36 @@ +#!/bin/sh +# +# Runs the whole parser test suite. Point BIN at a tc_parsecheck build +# (a sanitizer build works too, just slower). +# +# ./tests/run_all.sh +# BIN=./build/tsan/tc_parsecheck ./tests/run_all.sh +# +set -e +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +BIN="${BIN:-$ROOT/build/tests/tc_parsecheck}" +DATA="$ROOT/tests/data" + +[ -f "$DATA/manifest.tsv" ] || "$ROOT/tests/gen_corpus.sh" + +echo "== goldens (serial path regression)" +"$BIN" --golden-check "$DATA" +echo "== istream vs mmap" +"$BIN" --diff-buffer "$DATA" +echo "== serial vs parallel, 7 chunk counts" +"$BIN" --diff-parallel "$DATA" | tail -1 +echo "== chunk boundary sweep (every byte offset, plus pairs and triples)" +"$BIN" --sweep "$DATA" | tail -1 +echo "== differential fuzz" +"$BIN" --fuzz "${FUZZ:-100000}" +echo "== parallel UTF-8 validation" +"$BIN" --utf8check "$DATA" +"$BIN" --utf8fuzz "${UTF8FUZZ:-10000}" | head -1 +echo "== column width scan" +"$BIN" --colcheck "$DATA" +echo "== parser state does not leak between calls" +"$BIN" --reusecheck "$DATA" +echo "== dialect guessing (decides silently how every file opens)" +"$BIN" --guess-check "$DATA" +echo +echo "all green"