From ad10bfacf246a2aa3ce748bb27b61b5e1156645b Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 4 Nov 2026 11:26:22 +0000 Subject: [PATCH] Fold letters outside the BMP in OrdinalIgnoreCase [patch] OrdinalIgnoreCase folded one UTF-16 code unit at a time, and char.ToUpperInvariant leaves a surrogate unchanged, so a cased letter encoded as a surrogate pair (Adlam, Deseret, Osage, Warang Citi) was never folded: "\U00010428a" sorted after "\U00010400b", and names equal ignoring case did not sort next to each other. Each code unit is now folded as part of the code point it belongs to, with Rune.ToUpperInvariant on .NET and TextInfo.ToUpper on netstandard. Every text comparison still goes through the same fold, including text against a number's first digit and the prefix branch, so the order stays transitive. Folded text is still compared by code unit, the way Ordinal and the ordinal tiebreak order it, and the README says where that differs from OrdinalIgnoreCase (U+E000..U+FFFF against a pair). Fixes ktsu-dev/Sorting#77 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017vfJW3Ri1HBoN2geG4RgBi --- README.md | 2 + .../NaturalStringComparerIgnoreCaseTests.cs | 86 +++++++++++++++++ Sorting/NaturalStringComparer.cs | 93 ++++++++++++++++--- 3 files changed, 168 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 4f87dd0..d26408e 100644 --- a/README.md +++ b/README.md @@ -90,6 +90,8 @@ Array.Sort(files, NaturalStringComparer.OrdinalIgnoreCase); Strings that differ only in case are still never equal under `OrdinalIgnoreCase`. They are ordered next to each other, ordinally, so a `SortedSet` keeps both `file1` and `File1`. +Letters outside the Basic Multilingual Plane fold too, by code point, so names in Adlam, Deseret or Osage interleave the same way Latin names do. Folded text is still ordered by UTF-16 code unit, as `Ordinal` orders it, so U+E000 to U+FFFF sort after those letters, where `string.Compare(..., StringComparison.OrdinalIgnoreCase)` puts them before. + ## Features ### NaturalStringComparer diff --git a/Sorting.Test/NaturalStringComparerIgnoreCaseTests.cs b/Sorting.Test/NaturalStringComparerIgnoreCaseTests.cs index 7eff66f..37227e0 100644 --- a/Sorting.Test/NaturalStringComparerIgnoreCaseTests.cs +++ b/Sorting.Test/NaturalStringComparerIgnoreCaseTests.cs @@ -117,6 +117,91 @@ public void Compare_IgnoringCase_IsTransitiveOverMixedCaseDigitsAndText() } } + [TestMethod] + public void Compare_IgnoringCase_FoldsLettersOutsideTheBmp() + { + // Deseret small/capital long I, and Adlam small/capital alif: cased letters encoded as surrogate + // pairs, which StringComparison.OrdinalIgnoreCase folds by code point. + Assert.IsLessThan(0, string.Compare("\U00010428a", "\U00010400b", StringComparison.OrdinalIgnoreCase)); + Assert.IsLessThan(0, _comparer.Compare("\U00010428a", "\U00010400b")); + Assert.IsGreaterThan(0, _comparer.Compare("\U00010400b", "\U00010428a")); + Assert.IsLessThan(0, _comparer.Compare("\U0001E922a", "\U0001E900b")); + Assert.IsLessThan(0, _comparer.Compare("file\U0001E9222", "File\U0001E90010")); + } + + [TestMethod] + public void Sort_IgnoringCase_KeepsSupplementaryNamesEqualIgnoringCaseTogether() + { + string[] names = ["\U0001E900zzz", "\U0001E922file", "\U0001E900file", "\U0001E923a", "\U0001E901b"]; + + Array.Sort(names, _comparer); + + int lower = Array.IndexOf(names, "\U0001E922file"); + int upper = Array.IndexOf(names, "\U0001E900file"); + Assert.AreEqual(1, Math.Abs(lower - upper), string.Join(", ", names)); + Assert.AreNotEqual(0, _comparer.Compare("\U0001E922file", "\U0001E900file")); + Assert.AreEqual(Math.Sign(string.CompareOrdinal("\U0001E922file", "\U0001E900file")), Math.Sign(_comparer.Compare("\U0001E922file", "\U0001E900file"))); + } + + [TestMethod] + public void Compare_IgnoringCase_FoldsAsOrdinalIgnoreCaseDoes() + { + // U+E000..U+FFFF are left out: like Ordinal, the comparer orders them after a surrogate pair, by + // code unit, where OrdinalIgnoreCase orders them before it, by code point. + string[] values = + [ + "a", "A", "\U00010428", "\U00010400", "\U0001E922", "\U0001E900", "\U000104D8", "\U000104B0", + "\U0001F600", "\u00E9", "\u00C9", "_", "z", "\U00010428\U00010428", "\U00010400a", + ]; + + foreach (string a in values) + { + foreach (string b in values) + { + int expected = Math.Sign(string.Compare(a, b, StringComparison.OrdinalIgnoreCase)); + if (expected == 0) + { + expected = Math.Sign(string.CompareOrdinal(a, b)); + } + + Assert.AreEqual(expected, Math.Sign(_comparer.Compare(a, b)), $"'{a}' vs '{b}'"); + } + } + } + + [TestMethod] + public void Compare_IgnoringCase_IsTransitiveOverBmpAndSupplementaryLetters() + { + // A pair can straddle the end of the shorter text chunk ("a" + pair against "a" + digit), and + // U+E000..U+FFFF sort after surrogates by code unit, so mix both in with digits. + string[] values = + [ + "", "a", "A", "_", "\uE000", "\uFF41", "\uFF21", "\U00010428", "\U00010400", "\U0001E922", "\U0001E900", + "\U0001F600", "a\U00010428", "A\U00010400", "a\U0001E922", "a5", "A5", "a\U0001D7D7", "\U000104282", + "\U0001040010", "\U00010428_", "\U00010400a", "\U0001E922file", "\U0001E900file", "\U0001E900zzz", + "\uD801", "\uDC28", "a\uD801", "5\U00010428", + ]; + + foreach (string a in values) + { + foreach (string b in values) + { + int ab = Math.Sign(_comparer.Compare(a, b)); + Assert.AreEqual(ab, -Math.Sign(_comparer.Compare(b, a)), $"antisymmetry: '{a}' vs '{b}'"); + Assert.AreEqual(string.Equals(a, b, StringComparison.Ordinal), ab == 0, $"only equal strings tie: '{a}' vs '{b}'"); + + foreach (string c in values) + { + int bc = Math.Sign(_comparer.Compare(b, c)); + if (ab <= 0 && bc <= 0) + { + Assert.IsLessThanOrEqualTo(0, _comparer.Compare(a, c), $"transitivity: '{a}' <= '{b}' <= '{c}'"); + } + } + } + } + } + [TestMethod] public void Compare_IgnoringCase_AllocatesNothing() { @@ -129,6 +214,7 @@ public void Compare_IgnoringCase_AllocatesNothing() ("5A", "a5"), ("Same Prefix, Longer", "same prefix"), ("abc", "ABC"), + ("\U00010428a", "\U00010400b"), ]; foreach ((string x, string y) in pairs) diff --git a/Sorting/NaturalStringComparer.cs b/Sorting/NaturalStringComparer.cs index 4d8e893..4df89e6 100644 --- a/Sorting/NaturalStringComparer.cs +++ b/Sorting/NaturalStringComparer.cs @@ -106,8 +106,8 @@ public int Compare(string? x, string? y) // own code point here would sort it after letters while its ASCII equal sorts before // them, which makes the order intransitive. comparison = xIsNumeric - ? AsciiDigitAt(x, xIndex).CompareTo(Fold(y[yIndex])) - : Fold(x[xIndex]).CompareTo(AsciiDigitAt(y, yIndex)); + ? AsciiDigitAt(x, xIndex).CompareTo(Fold(y, yIndex)) + : Fold(x, xIndex).CompareTo(AsciiDigitAt(y, yIndex)); } else { @@ -236,16 +236,26 @@ private int CompareTextChunks(string x, int xStart, int xEnd, string y, int ySta // never holds a digit, and folding never makes a digit, so the two characters always differ. if (xLength < yLength) { - return xEnd < x.Length ? AsciiDigitAt(x, xEnd).CompareTo(Fold(y[yStart + xLength])) : -1; + return xEnd < x.Length ? AsciiDigitAt(x, xEnd).CompareTo(Fold(y, yStart + xLength)) : -1; } - return yEnd < y.Length ? Fold(x[xStart + yLength]).CompareTo(AsciiDigitAt(y, yEnd)) : 1; + return yEnd < y.Length ? Fold(x, xStart + yLength).CompareTo(AsciiDigitAt(y, yEnd)) : 1; } /// - /// Compares characters of two strings one at a time, each folded to upper - /// case as folds it. + /// Compares UTF-16 code units of two strings one at a time, each folded to + /// upper case as folds it. /// + /// + /// A surrogate pair is folded as the one code point it encodes, so letters outside the Basic + /// Multilingual Plane (Adlam, Deseret, Osage) fold too, and the folded pair is then compared a code + /// unit at a time, just as orders unfolded text. + /// That keeps U+E000 to U+FFFF after every surrogate pair, where + /// puts them before, but it means both modes and the + /// ordinal tiebreak order code points the same way. Folding never changes how many code units a code + /// point takes, so a pair that straddles the end of the shorter chunk compares the same way it would + /// within one chunk. + /// /// The first string. /// Where to start reading it. /// The second string. @@ -256,7 +266,7 @@ private static int CompareFolded(string x, int xStart, string y, int yStart, int { for (int offset = 0; offset < length; offset++) { - int comparison = char.ToUpperInvariant(x[xStart + offset]).CompareTo(char.ToUpperInvariant(y[yStart + offset])); + int comparison = FoldIgnoringCase(x, xStart + offset).CompareTo(FoldIgnoringCase(y, yStart + offset)); if (comparison != 0) { return comparison; @@ -267,13 +277,70 @@ private static int CompareFolded(string x, int xStart, string y, int yStart, int } /// - /// Returns the character text is compared by: itself, or its upper case - /// form when case is ignored. Every comparison of a text character goes through here, including - /// against a number's first digit, so the order stays transitive whichever way case is handled. + /// Returns the code unit text is compared by: the one at itself, or its + /// upper case form when case is ignored. Every comparison of a text character goes through here, + /// including against a number's first digit, so the order stays transitive whichever way case is + /// handled. + /// + /// The string to read. + /// The index of a code unit in a text chunk. + /// The code unit to compare. + private char Fold(string value, int index) => ignoreCase ? FoldIgnoringCase(value, index) : value[index]; + + /// + /// Returns the code unit at once the code point it belongs to is folded to + /// upper case, reading a surrogate pair as the one code point it encodes. That is how + /// folds text since .NET 5, so U+10428 (Deseret + /// small letter long I) folds to U+10400. A lone surrogate is left as it is. + /// + /// The string to read. + /// The index of the code unit. + /// The folded code unit. + private static char FoldIgnoringCase(string value, int index) + { + char unit = value[index]; + if (!char.IsSurrogate(unit)) + { + return char.ToUpperInvariant(unit); + } + + int pairStart = char.IsHighSurrogate(unit) ? index : index - 1; + if (pairStart < 0 || !char.IsSurrogatePair(value, pairStart)) + { + return unit; + } + + int folded = ToUpperInvariant(char.ConvertToUtf32(value, pairStart)); + + // No supplementary letter's upper case lies in the Basic Multilingual Plane, but should one ever + // do so, leaving the pair unfolded keeps both of its code units in step. + if (folded < 0x10000) + { + return unit; + } + + int offset = folded - 0x10000; + return pairStart == index + ? (char)(0xD800 + (offset >> 10)) + : (char)(0xDC00 + (offset & 0x3FF)); + } + + /// + /// Folds a code point outside the Basic Multilingual Plane to upper case, invariantly. /// - /// A character from a text chunk. - /// The character to compare. - private char Fold(char value) => ignoreCase ? char.ToUpperInvariant(value) : value; + /// A supplementary code point. + /// Its upper case form, or itself. + private static int ToUpperInvariant(int codePoint) + { +#if NET + return System.Text.Rune.ToUpperInvariant(new System.Text.Rune(codePoint)).Value; +#else + // netstandard has no Rune, so fold the pair as a string. That allocates, but only for a + // surrogate pair in a text chunk compared ignoring case. + string upper = CultureInfo.InvariantCulture.TextInfo.ToUpper(char.ConvertFromUtf32(codePoint)); + return upper.Length == 2 && char.IsSurrogatePair(upper, 0) ? char.ConvertToUtf32(upper, 0) : codePoint; +#endif + } /// /// Compares two runs of decimal digits by the numeric value they spell, rather than by their