diff --git a/Core/GameEngine/Source/Common/System/ArchiveFile.cpp b/Core/GameEngine/Source/Common/System/ArchiveFile.cpp index 9e0db3f0ee2..ce82fa0b4a8 100644 --- a/Core/GameEngine/Source/Common/System/ArchiveFile.cpp +++ b/Core/GameEngine/Source/Common/System/ArchiveFile.cpp @@ -150,15 +150,17 @@ void ArchiveFile::getFileListInDirectory(const AsciiString& currentDirectory, co void ArchiveFile::getFileListInDirectory(const DetailedArchivedDirectoryInfo *dirInfo, const AsciiString& currentDirectory, const AsciiString& searchName, FilenameList &filenameList, Bool searchSubdirectories) const { - DetailedArchivedDirectoryInfoMap::const_iterator diriter = dirInfo->m_directories.begin(); - for (; diriter != dirInfo->m_directories.end(); ++diriter) { - const DetailedArchivedDirectoryInfo *tempDirInfo = &(diriter->second); - AsciiString tempDirName = currentDirectory; - if (!tempDirName.isEmpty() && !tempDirName.endsWith("\\")) { - tempDirName.concat('\\'); + if (searchSubdirectories) { + DetailedArchivedDirectoryInfoMap::const_iterator diriter = dirInfo->m_directories.begin(); + for (; diriter != dirInfo->m_directories.end(); ++diriter) { + const DetailedArchivedDirectoryInfo *tempDirInfo = &(diriter->second); + AsciiString tempDirName = currentDirectory; + if (!tempDirName.isEmpty() && !tempDirName.endsWith("\\")) { + tempDirName.concat('\\'); + } + tempDirName.concat(tempDirInfo->m_directoryName); + getFileListInDirectory(tempDirInfo, tempDirName, searchName, filenameList, searchSubdirectories); } - tempDirName.concat(tempDirInfo->m_directoryName); - getFileListInDirectory(tempDirInfo, tempDirName, searchName, filenameList, searchSubdirectories); } ArchivedFileInfoMap::const_iterator fileiter = dirInfo->m_files.begin(); diff --git a/Core/GameEngine/Source/GameClient/GameText.cpp b/Core/GameEngine/Source/GameClient/GameText.cpp index 9ada81db32c..3ba6b49ba1d 100644 --- a/Core/GameEngine/Source/GameClient/GameText.cpp +++ b/Core/GameEngine/Source/GameClient/GameText.cpp @@ -79,8 +79,6 @@ Bool g_useStringFile = TRUE; #define CSF_STRINGWITHWAVE ( ('S'<<24) | ('T'<<16) | ('R'<<8) | ('W') ) #define CSF_VERSION 3 -#define STRING_FILE 0 -#define CSF_FILE 1 #define MAX_UITEXT_LENGTH (10*1024) //---------------------------------------------------------------------------- // Private Types @@ -103,6 +101,39 @@ struct StringLookUp StringInfo *info; }; +typedef std::vector StringInfoVec; +typedef std::set > LabelSet; + +// Name order, .str before .csf of the same name. +struct StringFileOrder +{ + Bool operator()( const AsciiString& a, const AsciiString& b ) const + { + const char *extA = a.reverseFind( '.' ); + const char *extB = b.reverseFind( '.' ); + Int lenA = a.getLength(); + if ( extA ) + lenA = (Int)(extA - a.str()); + Int lenB = b.getLength(); + if ( extB ) + lenB = (Int)(extB - b.str()); + Int compareLength = lenA; + if ( lenB < compareLength ) + compareLength = lenB; + Int cmp = _strnicmp( a.str(), b.str(), compareLength ); + if ( cmp != 0 ) + return cmp < 0; + if ( lenA != lenB ) + return lenA < lenB; + if ( !extA || !extB ) + return FALSE; + Bool isStrA = stricmp( extA, ".str" ) == 0; + Bool isStrB = stricmp( extB, ".str" ) == 0; + return isStrA && !isStrB; + } +}; +typedef std::set StringFileList; + //=============================== // CSFHeader //=============================== @@ -187,16 +218,17 @@ class GameTextManager : public GameTextInterface void stripSpaces ( WideChar *string ); void removeLeadingAndTrailing ( Char *m_buffer ); - void readToEndOfQuote( File *file, Char *in, Char *out, Char *wavefile, Int maxBufLen ); + Bool readToEndOfQuote( File *file, Char *in, Char *out, Char *wavefile, Int maxBufLen ); void reverseWord ( Char *file, Char *lp ); void translateCopy( WideChar *outbuf, Char *inbuf ); - Bool getStringCount( const Char *filename, Int& textCount ); - Bool getCSFInfo ( const Char *filename ); - Bool parseCSF( const Char *filename ); - Bool parseStringFile( const char *filename ); - Bool parseMapStringFile( const char *filename ); + Bool getCSFInfo( const Char *filename, LanguageID& language ); + Bool parseCSF( const Char *filename, StringInfoVec& out ); + Bool parseStringFile( const char *filename, StringInfoVec& out, Bool filterLanguage ); Bool readLine( char *buffer, Int max, File *file ); Char readChar( File *file ); + + void collectStringFiles( const AsciiString& directory, StringFileList& files ); + Bool mergeStringFile( const AsciiString& filename, StringInfoVec& merged, LabelSet& labels ); }; static int __cdecl compareLUT ( const void *, const void*); @@ -288,7 +320,6 @@ void GameTextManager::init() { AsciiString csfFile; csfFile.format(g_csfFile, GetRegistryLanguage().str()); - Int format; if ( m_initialized ) { @@ -306,19 +337,39 @@ void GameTextManager::init() } #endif - if ( m_useStringFile && getStringCount( g_strFile, m_textCount ) ) + // TheSuperHackers @feature arcticdolphin 27/09/2026 Load every *.str and *.csf in data\\ and data\ before the string file. The first file to define a label wins. + StringInfoVec merged; + LabelSet labels; + + AsciiString languageDir; + languageDir.format( "data\\%s\\", GetRegistryLanguage().str() ); + StringFileList languageFiles, neutralFiles; + collectStringFiles( languageDir, languageFiles ); + collectStringFiles( "data\\", neutralFiles ); + languageFiles.erase( csfFile ); + neutralFiles.erase( AsciiString( g_strFile ) ); + + StringFileList::const_iterator it; + for ( it = languageFiles.begin(); it != languageFiles.end(); ++it ) { - format = STRING_FILE; + mergeStringFile( *it, merged, labels ); } - else if ( getCSFInfo ( csfFile.str() ) ) + for ( it = neutralFiles.begin(); it != neutralFiles.end(); ++it ) { - format = CSF_FILE; + mergeStringFile( *it, merged, labels ); } - else + + // Generals.str replaces generals.csf if it loads. + if ( !m_useStringFile || !mergeStringFile( AsciiString( g_strFile ), merged, labels ) ) { - return; + if ( !getCSFInfo( csfFile.str(), m_language ) || !mergeStringFile( csfFile, merged, labels ) ) + { + return; + } } + m_textCount = (Int)merged.size(); + if( m_textCount == 0 ) { return; @@ -334,22 +385,7 @@ void GameTextManager::init() return; } - if ( format == STRING_FILE ) - { - if( parseStringFile( g_strFile ) == FALSE ) - { - deinit(); - return; - } - } - else - { - if ( !parseCSF ( csfFile.str() ) ) - { - deinit(); - return; - } - } + std::copy( merged.begin(), merged.end(), m_stringInfo ); m_stringLUT = NEW StringLookUp[m_textCount]; @@ -368,6 +404,44 @@ void GameTextManager::init() } +void GameTextManager::collectStringFiles( const AsciiString& directory, StringFileList& files ) +{ + FilenameList found; + TheFileSystem->getFileListInDirectory( directory, "*.str", found, FALSE ); + TheFileSystem->getFileListInDirectory( directory, "*.csf", found, FALSE ); + files.insert( found.begin(), found.end() ); +} + +Bool GameTextManager::mergeStringFile( const AsciiString& filename, StringInfoVec& merged, LabelSet& labels ) +{ + const char *ext = filename.reverseFind('.'); + Bool isStr = ext && stricmp( ext, ".str" ) == 0; + + StringInfoVec parsed; + Bool ok = isStr ? parseStringFile( filename.str(), parsed, FALSE ) + : parseCSF( filename.str(), parsed ); + + if ( !ok ) + { + DEBUG_LOG(("GameText: Failed to parse string file '%s', skipping", filename.str())); + return FALSE; + } + + Int skippedCount = 0; + for ( StringInfoVec::const_iterator it = parsed.begin(); it != parsed.end(); ++it ) + { + if ( !labels.insert( it->label ).second ) + { + skippedCount++; + continue; + } + merged.push_back( *it ); + } + + DEBUG_LOG(("GameText: Loaded string file '%s' (%d entries, %d already defined)", filename.str(), (Int)parsed.size(), skippedCount)); + return TRUE; +} + //============================================================================ // GameTextManager::deinit //============================================================================ @@ -496,7 +570,7 @@ void GameTextManager::removeLeadingAndTrailing ( Char *buffer ) // GameTextManager::readToEndOfQuote //============================================================================ -void GameTextManager::readToEndOfQuote( File *file, Char *in, Char *out, Char *wavefile, Int maxBufLen ) +Bool GameTextManager::readToEndOfQuote( File *file, Char *in, Char *out, Char *wavefile, Int maxBufLen ) { Int slash = FALSE; Int state = 0; @@ -505,8 +579,11 @@ void GameTextManager::readToEndOfQuote( File *file, Char *in, Char *out, Char *w Int ccount = 0; Int len = 0; Int done = FALSE; + Bool closed = FALSE; + // Room for the 'e' suffix and the terminator. + const Int maxWaveLen = maxBufLen - 2; - while ( maxBufLen ) + while ( maxBufLen > 1 ) { // get next Char @@ -523,9 +600,11 @@ void GameTextManager::readToEndOfQuote( File *file, Char *in, Char *out, Char *w ch = readChar ( file ); } - if ( ch == EOF ) + if ( ch == 0 ) { - return ; + *out = 0; + *wavefile = 0; + return FALSE; } if ( ch == '\n' ) @@ -545,6 +624,7 @@ void GameTextManager::readToEndOfQuote( File *file, Char *in, Char *out, Char *w } else if ( ch == '"' && !slash ) { + closed = TRUE; break; // done } else @@ -563,6 +643,12 @@ void GameTextManager::readToEndOfQuote( File *file, Char *in, Char *out, Char *w *out = 0; + if ( !closed ) + { + *wavefile = 0; + return FALSE; + } + while ( !done ) { // get next Char @@ -580,7 +666,7 @@ void GameTextManager::readToEndOfQuote( File *file, Char *in, Char *out, Char *w ch = readChar ( file ); } - if ( ch == '\n' || ch == EOF ) + if ( ch == '\n' || ch == 0 ) { break; } @@ -599,6 +685,11 @@ void GameTextManager::readToEndOfQuote( File *file, Char *in, Char *out, Char *w case 1: if ( ( ch >= 'a' && ch <= 'z') || ( ch >= 'A' && ch <='Z') || (ch >= '0' && ch <= '9') || ch == '_' ) { + if ( len >= maxWaveLen ) + { + *wavefile = 0; + return FALSE; + } *wavefile++ = ch; len++; break; @@ -621,6 +712,7 @@ void GameTextManager::readToEndOfQuote( File *file, Char *in, Char *out, Char *w } } + return TRUE; } @@ -792,55 +884,11 @@ void GameTextManager::translateCopy( WideChar *outbuf, Char *inbuf ) *outbuf= 0; } -//============================================================================ -// GameTextManager::getStringCount -//============================================================================ - -Bool GameTextManager::getStringCount( const char *filename, Int& textCount ) -{ - Bool ok = TRUE; - - textCount = 0; - - File *file; - file = TheFileSystem->openFile(filename, File::READ | File::TEXT); - DEBUG_LOG(("Looking in %s for string file", filename)); - - if ( file == nullptr ) - { - return FALSE; - } - - while(ok) - { - if( !readLine( m_buffer, sizeof( m_buffer) -1, file ) ) - break; - removeLeadingAndTrailing ( m_buffer ); - - if( m_buffer[0] == '"' ) - { - Int len = strlen(m_buffer); - m_buffer[ len ] = '\n'; - m_buffer[ len+1] = 0; - readToEndOfQuote( file, &m_buffer[1], m_buffer2, m_buffer3, MAX_UITEXT_LENGTH ); - } - else if( stricmp( m_buffer, "END") == 0 ) - { - textCount++; - } - } - - textCount += 500; - file->close(); - file = nullptr; - return TRUE; -} - //============================================================================ // GameTextManager::getCSFInfo //============================================================================ -Bool GameTextManager::getCSFInfo ( const Char *filename ) +Bool GameTextManager::getCSFInfo( const Char *filename, LanguageID& language ) { CSFHeader header; Bool ok = FALSE; @@ -853,15 +901,13 @@ Bool GameTextManager::getCSFInfo ( const Char *filename ) { if ( header.id == CSF_ID ) { - m_textCount = header.num_labels; - if ( header.version >= 2 ) { - m_language = (LanguageID) header.langid; + language = (LanguageID) header.langid; } else { - m_language = LANGUAGE_ID_US; + language = LANGUAGE_ID_US; } ok = TRUE; @@ -879,12 +925,13 @@ Bool GameTextManager::getCSFInfo ( const Char *filename ) // GameTextManager::parseCSF //============================================================================ -Bool GameTextManager::parseCSF( const Char *filename ) +Bool GameTextManager::parseCSF( const Char *filename, StringInfoVec& out ) { File *file; Int id; Int len; Int listCount = 0; + Int stringCount = 0; Bool ok = FALSE; CSFHeader header; @@ -897,32 +944,61 @@ Bool GameTextManager::parseCSF( const Char *filename ) if ( file->read ( &header, sizeof ( CSFHeader)) != sizeof ( CSFHeader) ) { - return FALSE; + goto quit; } - while( file->read ( &id, sizeof (id)) == sizeof ( id) ) + // A label record is at least 12 bytes. + if ( header.id != CSF_ID || header.num_labels < 0 || header.num_labels > file->size() / 12 || + header.num_strings < 0 ) + { + goto quit; + } + + out.reserve( header.num_labels ); + + while ( TRUE ) { + Int bytesRead = file->read( &id, sizeof(id) ); + if ( bytesRead == 0 ) + break; + if ( bytesRead != sizeof(id) ) + goto quit; Int num; Int num_strings; + StringInfo info; if ( id != CSF_LABEL ) { goto quit; } - file->read ( &num_strings, sizeof ( Int )); + if ( file->read ( &num_strings, sizeof ( Int ) ) != (Int)(sizeof ( Int )) || + num_strings < 0 || num_strings > header.num_strings - stringCount ) + { + goto quit; + } - file->read ( &len, sizeof ( Int ) ); + if ( file->read ( &len, sizeof ( Int ) ) != (Int)(sizeof ( Int )) ) + { + goto quit; + } + + if ( len < 0 || len >= MAX_UITEXT_LENGTH ) + { + goto quit; + } if ( len ) { - file->read ( m_buffer, len ); + if ( file->read ( m_buffer, len ) != (Int)(len) ) + { + goto quit; + } } m_buffer[len] = 0; - m_stringInfo[listCount].label = m_buffer; - + info.label = m_buffer; if ( len > m_maxLabelLen ) { @@ -933,18 +1009,32 @@ Bool GameTextManager::parseCSF( const Char *filename ) while ( num < num_strings ) { - file->read ( &id, sizeof ( Int ) ); + if ( file->read ( &id, sizeof ( Int ) ) != (Int)(sizeof ( Int )) ) + { + goto quit; + } if ( id != CSF_STRING && id != CSF_STRINGWITHWAVE ) { goto quit; } - file->read ( &len, sizeof ( Int ) ); + if ( file->read ( &len, sizeof ( Int ) ) != (Int)(sizeof ( Int )) ) + { + goto quit; + } + + if ( len < 0 || len >= MAX_UITEXT_LENGTH*2 ) + { + goto quit; + } if ( len ) { - file->read ( m_tbuffer, len*sizeof(WideChar) ); + if ( file->read ( m_tbuffer, len*sizeof(WideChar) ) != (Int)(len*sizeof(WideChar)) ) + { + goto quit; + } } if ( num == 0 ) @@ -965,22 +1055,32 @@ Bool GameTextManager::parseCSF( const Char *filename ) } stripSpaces ( m_tbuffer ); - m_stringInfo[listCount].text = m_tbuffer; + info.text = m_tbuffer; } if ( id == CSF_STRINGWITHWAVE ) { - file->read ( &len, sizeof ( Int ) ); + if ( file->read ( &len, sizeof ( Int ) ) != (Int)(sizeof ( Int )) ) + { + goto quit; + } + if ( len < 0 || len >= MAX_UITEXT_LENGTH ) + { + goto quit; + } if ( len ) { - file->read ( m_buffer, len ); + if ( file->read ( m_buffer, len ) != (Int)(len) ) + { + goto quit; + } } m_buffer[len] = 0; if ( num == 0 && len ) { // only use the first string found - m_stringInfo[listCount].speech = m_buffer; + info.speech = m_buffer; } } @@ -988,10 +1088,12 @@ Bool GameTextManager::parseCSF( const Char *filename ) num++; } + out.push_back( info ); listCount++; + stringCount += num_strings; } - ok = TRUE; + ok = ( listCount == (Int)header.num_labels && stringCount == header.num_strings ); quit: @@ -1006,9 +1108,8 @@ Bool GameTextManager::parseCSF( const Char *filename ) // GameTextManager::parseStringFile //============================================================================ -Bool GameTextManager::parseStringFile( const char *filename ) +Bool GameTextManager::parseStringFile( const char *filename, StringInfoVec& out, Bool filterLanguage ) { - Int listCount = 0; Bool ok = TRUE; File *file = TheFileSystem->openFile(filename, File::READ | File::TEXT); @@ -1021,11 +1122,19 @@ Bool GameTextManager::parseStringFile( const char *filename ) while( ok ) { Int len; - if( !readLine( m_buffer, MAX_UITEXT_LENGTH, file )) + StringInfo info; + + if( !readLine( m_buffer, sizeof(m_buffer)-1, file )) { break; } + if ( strlen( m_buffer ) >= sizeof(m_buffer)-1 ) + { + ok = FALSE; + goto quit; + } + removeLeadingAndTrailing ( m_buffer ); if( ( *(unsigned short *)m_buffer == 0x2F2F) || !m_buffer[0]) // 0x2F2F is Hex for // @@ -1033,18 +1142,17 @@ Bool GameTextManager::parseStringFile( const char *filename ) // make sure label is unique - for ( Int i = 0; i < listCount; i++ ) + for ( StringInfoVec::const_iterator it = out.begin(); it != out.end(); ++it ) { - if ( stricmp ( m_stringInfo[i].label.str(), m_buffer ) == 0) + if ( stricmp ( it->label.str(), m_buffer ) == 0) { DEBUG_CRASH ( ("String label '%s' multiply defined!", m_buffer )); } } - m_stringInfo[listCount].label = m_buffer; + info.label = m_buffer; len = strlen ( m_buffer ); - if ( len > m_maxLabelLen ) { m_maxLabelLen = len; @@ -1053,13 +1161,19 @@ Bool GameTextManager::parseStringFile( const char *filename ) Bool readString = FALSE; while( ok ) { - if (!readLine ( m_buffer, sizeof(m_buffer)-1, file )) + if (!readLine ( m_buffer, sizeof(m_buffer)-2, file )) { DEBUG_CRASH (("Unexpected end of string file")); ok = FALSE; goto quit; } + if ( strlen( m_buffer ) >= sizeof(m_buffer)-2 ) + { + ok = FALSE; + goto quit; + } + removeLeadingAndTrailing ( m_buffer ); if( m_buffer[0] == '"' ) @@ -1067,22 +1181,27 @@ Bool GameTextManager::parseStringFile( const char *filename ) len = strlen(m_buffer); m_buffer[ len ] = '\n'; m_buffer[ len+1] = 0; - readToEndOfQuote( file, &m_buffer[1], m_buffer2, m_buffer3, MAX_UITEXT_LENGTH ); - + if ( !readToEndOfQuote( file, &m_buffer[1], m_buffer2, m_buffer3, MAX_UITEXT_LENGTH ) ) + { + ok = FALSE; + goto quit; + } if ( readString ) { - // only one string per label allows - DEBUG_CRASH ( ("String label '%s' has more than one string defined!", m_stringInfo[listCount].label.str())); + DEBUG_CRASH ( ("String label '%s' has more than one string defined!", info.label.str())); } else { - // Copy string into new home translateCopy( m_tbuffer, m_buffer2 ); stripSpaces ( m_tbuffer ); - m_stringInfo[listCount].text = m_tbuffer ; - m_stringInfo[listCount].speech = m_buffer3; + UnicodeString text = UnicodeString(m_tbuffer); + if ( filterLanguage && TheLanguageFilter ) + TheLanguageFilter->filterLine(text); + + info.text = text; + info.speech = m_buffer3; readString = TRUE; } } @@ -1092,7 +1211,13 @@ Bool GameTextManager::parseStringFile( const char *filename ) } } - listCount++; + if ( !readString ) + { + ok = FALSE; + goto quit; + } + + out.push_back( info ); } quit: @@ -1109,12 +1234,15 @@ Bool GameTextManager::parseStringFile( const char *filename ) void GameTextManager::initMapStringFile( const AsciiString& filename ) { - m_mapTextCount = 0; - getStringCount( filename.str(), m_mapTextCount ); + StringInfoVec parsed; + if ( !parseStringFile( filename.str(), parsed, TRUE ) ) + { + parsed.clear(); + } + m_mapTextCount = (Int)parsed.size(); m_mapStringInfo = NEW StringInfo[m_mapTextCount]; - - parseMapStringFile( filename.str() ); + std::copy( parsed.begin(), parsed.end(), m_mapStringInfo ); m_mapStringLUT = NEW StringLookUp[m_mapTextCount]; @@ -1132,111 +1260,6 @@ void GameTextManager::initMapStringFile( const AsciiString& filename ) qsort( m_mapStringLUT, m_mapTextCount, sizeof(StringLookUp), compareLUT ); } -//============================================================================ -// GameTextManager::parseMapStringFile -//============================================================================ - -Bool GameTextManager::parseMapStringFile( const char *filename ) -{ - Int listCount = 0; - Bool ok = TRUE; - - File *file; - - file = TheFileSystem->openFile(filename, File::READ | File::TEXT); - if ( file == nullptr ) - { - return FALSE; - } - - while( ok ) - { - Int len; - if( !readLine( m_buffer, MAX_UITEXT_LENGTH, file )) - { - break; - } - - removeLeadingAndTrailing ( m_buffer ); - - if( ( *(unsigned short *)m_buffer == 0x2F2F) || !m_buffer[0]) // 0x2F2F is Hex for // - continue; - - // make sure label is unique - - for ( Int i = 0; i < listCount; i++ ) - { - if ( stricmp ( m_mapStringInfo[i].label.str(), m_buffer ) == 0) - { - DEBUG_CRASH ( ("String label '%s' multiply defined!", m_buffer )); - } - } - - m_mapStringInfo[listCount].label = m_buffer; - len = strlen ( m_buffer ); - - - if ( len > m_maxLabelLen ) - { - m_maxLabelLen = len; - } - - Bool readString = FALSE; - while( ok ) - { - if (!readLine ( m_buffer, sizeof(m_buffer)-1, file )) - { - DEBUG_CRASH (("Unexpected end of string file")); - ok = FALSE; - goto quit; - } - - removeLeadingAndTrailing ( m_buffer ); - - if( m_buffer[0] == '"' ) - { - len = strlen(m_buffer); - m_buffer[ len ] = '\n'; - m_buffer[ len+1] = 0; - readToEndOfQuote( file, &m_buffer[1], m_buffer2, m_buffer3, MAX_UITEXT_LENGTH ); - - - if ( readString ) - { - // only one string per label allowed - DEBUG_CRASH ( ("String label '%s' has more than one string defined!", m_stringInfo[listCount].label.str())); - } - else - { - // Copy string into new home - translateCopy( m_tbuffer, m_buffer2 ); - stripSpaces ( m_tbuffer ); - - UnicodeString text = UnicodeString(m_tbuffer); - if (TheLanguageFilter) - TheLanguageFilter->filterLine(text); - - m_mapStringInfo[listCount].text = text; - m_mapStringInfo[listCount].speech = m_buffer3; - readString = TRUE; - } - } - else if ( stricmp ( m_buffer, "END" ) == 0) - { - break; - } - } - - listCount++; - } - -quit: - - file->close(); - file = nullptr; - - return ok; -} //============================================================================ // *GameTextManager::fetch