Fold letters outside the BMP in OrdinalIgnoreCase [patch]

README.md

@@ -90,6 +90,8 @@ Array.Sort(files, NaturalStringComparer.OrdinalIgnoreCase);
 
 Strings that differ only in case are still never equal under `OrdinalIgnoreCase`. They are ordered next to each other, ordinally, so a `SortedSet<string>` keeps both `file1` and `File1`.
 
+Letters outside the Basic Multilingual Plane fold too, by code point, so names in Adlam, Deseret or Osage interleave the same way Latin names do. Folded text is still ordered by UTF-16 code unit, as `Ordinal` orders it, so U+E000 to U+FFFF sort after those letters, where `string.Compare(..., StringComparison.OrdinalIgnoreCase)` puts them before.
+
 ## Features
 
 ### NaturalStringComparer

Sorting.Test/NaturalStringComparerIgnoreCaseTests.cs

@@ -117,6 +117,91 @@ public void Compare_IgnoringCase_IsTransitiveOverMixedCaseDigitsAndText()
 		}
 	}
 
+	[TestMethod]
+	public void Compare_IgnoringCase_FoldsLettersOutsideTheBmp()
+	{
+		// Deseret small/capital long I, and Adlam small/capital alif: cased letters encoded as surrogate
+		// pairs, which StringComparison.OrdinalIgnoreCase folds by code point.
+		Assert.IsLessThan(0, string.Compare("\U00010428a", "\U00010400b", StringComparison.OrdinalIgnoreCase));
+		Assert.IsLessThan(0, _comparer.Compare("\U00010428a", "\U00010400b"));
+		Assert.IsGreaterThan(0, _comparer.Compare("\U00010400b", "\U00010428a"));
+		Assert.IsLessThan(0, _comparer.Compare("\U0001E922a", "\U0001E900b"));
+		Assert.IsLessThan(0, _comparer.Compare("file\U0001E9222", "File\U0001E90010"));
+	}
+
+	[TestMethod]
+	public void Sort_IgnoringCase_KeepsSupplementaryNamesEqualIgnoringCaseTogether()
+	{
+		string[] names = ["\U0001E900zzz", "\U0001E922file", "\U0001E900file", "\U0001E923a", "\U0001E901b"];
+
+		Array.Sort(names, _comparer);
+
+		int lower = Array.IndexOf(names, "\U0001E922file");
+		int upper = Array.IndexOf(names, "\U0001E900file");
+		Assert.AreEqual(1, Math.Abs(lower - upper), string.Join(", ", names));
+		Assert.AreNotEqual(0, _comparer.Compare("\U0001E922file", "\U0001E900file"));
+		Assert.AreEqual(Math.Sign(string.CompareOrdinal("\U0001E922file", "\U0001E900file")), Math.Sign(_comparer.Compare("\U0001E922file", "\U0001E900file")));
+	}
+
+	[TestMethod]
+	public void Compare_IgnoringCase_FoldsAsOrdinalIgnoreCaseDoes()
+	{
+		// U+E000..U+FFFF are left out: like Ordinal, the comparer orders them after a surrogate pair, by
+		// code unit, where OrdinalIgnoreCase orders them before it, by code point.
+		string[] values =
+		[
+			"a", "A", "\U00010428", "\U00010400", "\U0001E922", "\U0001E900", "\U000104D8", "\U000104B0",
+			"\U0001F600", "\u00E9", "\u00C9", "_", "z", "\U00010428\U00010428", "\U00010400a",
+		];
+
+		foreach (string a in values)
+		{
+			foreach (string b in values)
+			{
+				int expected = Math.Sign(string.Compare(a, b, StringComparison.OrdinalIgnoreCase));
+				if (expected == 0)
+				{
+					expected = Math.Sign(string.CompareOrdinal(a, b));
+				}
+
+				Assert.AreEqual(expected, Math.Sign(_comparer.Compare(a, b)), $"'{a}' vs '{b}'");
+			}
+		}
+	}
+
+	[TestMethod]
+	public void Compare_IgnoringCase_IsTransitiveOverBmpAndSupplementaryLetters()
+	{
+		// A pair can straddle the end of the shorter text chunk ("a" + pair against "a" + digit), and
+		// U+E000..U+FFFF sort after surrogates by code unit, so mix both in with digits.
+		string[] values =
+		[
+			"", "a", "A", "_", "\uE000", "\uFF41", "\uFF21", "\U00010428", "\U00010400", "\U0001E922", "\U0001E900",
+			"\U0001F600", "a\U00010428", "A\U00010400", "a\U0001E922", "a5", "A5", "a\U0001D7D7", "\U000104282",
+			"\U0001040010", "\U00010428_", "\U00010400a", "\U0001E922file", "\U0001E900file", "\U0001E900zzz",
+			"\uD801", "\uDC28", "a\uD801", "5\U00010428",
+		];
+
+		foreach (string a in values)
+		{
+			foreach (string b in values)
+			{
+				int ab = Math.Sign(_comparer.Compare(a, b));
+				Assert.AreEqual(ab, -Math.Sign(_comparer.Compare(b, a)), $"antisymmetry: '{a}' vs '{b}'");
+				Assert.AreEqual(string.Equals(a, b, StringComparison.Ordinal), ab == 0, $"only equal strings tie: '{a}' vs '{b}'");
+
+				foreach (string c in values)
+				{
+					int bc = Math.Sign(_comparer.Compare(b, c));
+					if (ab <= 0 && bc <= 0)
+					{
+						Assert.IsLessThanOrEqualTo(0, _comparer.Compare(a, c), $"transitivity: '{a}' <= '{b}' <= '{c}'");
+					}
+				}
+			}
+		}
+	}
+
 	[TestMethod]
 	public void Compare_IgnoringCase_AllocatesNothing()
 	{
@@ -129,6 +214,7 @@ public void Compare_IgnoringCase_AllocatesNothing()
 			("5A", "a5"),
 			("Same Prefix, Longer", "same prefix"),
 			("abc", "ABC"),
+			("\U00010428a", "\U00010400b"),
 		];
 
 		foreach ((string x, string y) in pairs)

Sorting/NaturalStringComparer.cs

@@ -59,7 +59,7 @@
 	/// ordinally.
 	/// Greater than zero if x is greater than y.
 	/// </returns>
 	public int Compare(string? x, string? y)
 	{
 		if (x == null && y == null)
 		{
@@ -106,8 +106,8 @@
 				// own code point here would sort it after letters while its ASCII equal sorts before
 				// them, which makes the order intransitive.
 				comparison = xIsNumeric
-					? AsciiDigitAt(x, xIndex).CompareTo(Fold(y[yIndex]))
-					: Fold(x[xIndex]).CompareTo(AsciiDigitAt(y, yIndex));
+					? AsciiDigitAt(x, xIndex).CompareTo(Fold(y, yIndex))
+					: Fold(x, xIndex).CompareTo(AsciiDigitAt(y, yIndex));
 			}
 			else
 			{
@@ -236,16 +236,26 @@
 		// never holds a digit, and folding never makes a digit, so the two characters always differ.
 		if (xLength < yLength)
 		{
-			return xEnd < x.Length ? AsciiDigitAt(x, xEnd).CompareTo(Fold(y[yStart + xLength])) : -1;
+			return xEnd < x.Length ? AsciiDigitAt(x, xEnd).CompareTo(Fold(y, yStart + xLength)) : -1;
 		}
 
-		return yEnd < y.Length ? Fold(x[xStart + yLength]).CompareTo(AsciiDigitAt(y, yEnd)) : 1;
+		return yEnd < y.Length ? Fold(x, xStart + yLength).CompareTo(AsciiDigitAt(y, yEnd)) : 1;
 	}
 
 	/// <summary>
-	/// Compares <paramref name="length"/> characters of two strings one at a time, each folded to upper
-	/// case as <see cref="StringComparison.OrdinalIgnoreCase"/> folds it.
+	/// Compares <paramref name="length"/> UTF-16 code units of two strings one at a time, each folded to
+	/// upper case as <see cref="StringComparison.OrdinalIgnoreCase"/> folds it.
 	/// </summary>
+	/// <remarks>
+	/// A surrogate pair is folded as the one code point it encodes, so letters outside the Basic
+	/// Multilingual Plane (Adlam, Deseret, Osage) fold too, and the folded pair is then compared a code
+	/// unit at a time, just as <see cref="string.CompareOrdinal(string, string)"/> orders unfolded text.
+	/// That keeps U+E000 to U+FFFF after every surrogate pair, where
+	/// <see cref="StringComparison.OrdinalIgnoreCase"/> puts them before, but it means both modes and the
+	/// ordinal tiebreak order code points the same way. Folding never changes how many code units a code
+	/// point takes, so a pair that straddles the end of the shorter chunk compares the same way it would
+	/// within one chunk.
+	/// </remarks>
 	/// <param name="x">The first string.</param>
 	/// <param name="xStart">Where to start reading it.</param>
 	/// <param name="y">The second string.</param>
@@ -256,7 +266,7 @@
 	{
 		for (int offset = 0; offset < length; offset++)
 		{
-			int comparison = char.ToUpperInvariant(x[xStart + offset]).CompareTo(char.ToUpperInvariant(y[yStart + offset]));
+			int comparison = FoldIgnoringCase(x, xStart + offset).CompareTo(FoldIgnoringCase(y, yStart + offset));
 			if (comparison != 0)
 			{
 				return comparison;
@@ -267,13 +277,70 @@
 	}
 
 	/// <summary>
-	/// Returns the character text is compared by: <paramref name="value"/> itself, or its upper case
-	/// form when case is ignored. Every comparison of a text character goes through here, including
-	/// against a number's first digit, so the order stays transitive whichever way case is handled.
+	/// Returns the code unit text is compared by: the one at <paramref name="index"/> itself, or its
+	/// upper case form when case is ignored. Every comparison of a text character goes through here,
+	/// including against a number's first digit, so the order stays transitive whichever way case is
+	/// handled.
+	/// </summary>
+	/// <param name="value">The string to read.</param>
+	/// <param name="index">The index of a code unit in a text chunk.</param>
+	/// <returns>The code unit to compare.</returns>
+	private char Fold(string value, int index) => ignoreCase ? FoldIgnoringCase(value, index) : value[index];
+
+	/// <summary>
+	/// Returns the code unit at <paramref name="index"/> once the code point it belongs to is folded to
+	/// upper case, reading a surrogate pair as the one code point it encodes. That is how
+	/// <see cref="StringComparison.OrdinalIgnoreCase"/> folds text since .NET 5, so U+10428 (Deseret
+	/// small letter long I) folds to U+10400. A lone surrogate is left as it is.
+	/// </summary>
+	/// <param name="value">The string to read.</param>
+	/// <param name="index">The index of the code unit.</param>
+	/// <returns>The folded code unit.</returns>
+	private static char FoldIgnoringCase(string value, int index)
+	{
+		char unit = value[index];
+		if (!char.IsSurrogate(unit))
+		{
+			return char.ToUpperInvariant(unit);
+		}
+
+		int pairStart = char.IsHighSurrogate(unit) ? index : index - 1;
+		if (pairStart < 0 || !char.IsSurrogatePair(value, pairStart))
+		{
+			return unit;
+		}
+
+		int folded = ToUpperInvariant(char.ConvertToUtf32(value, pairStart));
+
+		// No supplementary letter's upper case lies in the Basic Multilingual Plane, but should one ever
+		// do so, leaving the pair unfolded keeps both of its code units in step.
+		if (folded < 0x10000)
+		{
+			return unit;
+		}
+
+		int offset = folded - 0x10000;
+		return pairStart == index
+			? (char)(0xD800 + (offset >> 10))
+			: (char)(0xDC00 + (offset & 0x3FF));
+	}
+
+	/// <summary>
+	/// Folds a code point outside the Basic Multilingual Plane to upper case, invariantly.
 	/// </summary>
-	/// <param name="value">A character from a text chunk.</param>
-	/// <returns>The character to compare.</returns>
-	private char Fold(char value) => ignoreCase ? char.ToUpperInvariant(value) : value;
+	/// <param name="codePoint">A supplementary code point.</param>
+	/// <returns>Its upper case form, or <paramref name="codePoint"/> itself.</returns>
+	private static int ToUpperInvariant(int codePoint)
+	{
+#if NET
+		return System.Text.Rune.ToUpperInvariant(new System.Text.Rune(codePoint)).Value;
+#else
+		// netstandard has no Rune, so fold the pair as a string. That allocates, but only for a
+		// surrogate pair in a text chunk compared ignoring case.
+		string upper = CultureInfo.InvariantCulture.TextInfo.ToUpper(char.ConvertFromUtf32(codePoint));
+		return upper.Length == 2 && char.IsSurrogatePair(upper, 0) ? char.ConvertToUtf32(upper, 0) : codePoint;
+#endif
+	}
 
 	/// <summary>
 	/// Compares two runs of decimal digits by the numeric value they spell, rather than by their