ICU-22511 Handling unpaired surrogates and improved test cases.

Author: yumaoka

icu4c/source/i18n/utf16collationiterator.cpp

@@ -303,7 +303,7 @@ FCDUTF16CollationIterator::previousCodePoint(UErrorCode &errorCode) {
                         // To test possible trailing ccc, we need to check high surrogate
                         // (or previous character for broken surrogate pair).
                         if (pos - 1 != start) {
-                            normalizePrevSegment = CollationFCD::hasTccc(*(pos - 2));
+                            normalizePrevSegment = U16_IS_LEAD(*(pos - 2)) && CollationFCD::hasTccc(*(pos - 2));
                         }
                     } else {
                         normalizePrevSegment = CollationFCD::hasTccc(*(pos - 1));

icu4c/source/test/intltest/collationtest.cpp

@@ -2128,12 +2128,25 @@ void CollationTest::TestCollatorMap() {
 
 void CollationTest::TestColItrInfiniteLoop22511() {
     IcuTestErrorCode errorCode(*this, "TestColItrInfiniteLoop22511");
-    char16_t str1[] = {
-        0x0100, 0x032a, 0x01e0, 0xd804, 0xdd00, 0x031c
-    };
-    char16_t str2[] = {
-        0x0041, 0x0304, 0x032a, 0x01e0, 0xd804, 0xdd00, 0x031c
+    const char16_t* testCases[][2] = {
+        {
+            u"\u0100\u032a\u01e0\U00011100\u031c",
+            u"A\u0304\u032a\u01e0\U00011100\u031c"  // Equivalent to above, but U+0100 is decomposed to U+0041 U+0304.
+        },
+        {
+            u"\u0100\u032a\u01e0\xdd00\u031c",      // High surrogate 0xd804 is dropped
+            u"A\u0304\u032a\u01e0\xdd00\u031c"      // Equivalent to above, but U+0100 is decomposed
+        },
+        {
+            u"\u0100\u032a\u01e0\xd804\u031c",      // Low surrogate 0xdd00 is dropped
+            u"A\u0304\u032a\u01e0\xd804\u031c"      // Equivalent to above, but U+0100 is decomposed
+        },
+        {nullptr, nullptr}
     };
+
+    StringPiece sp1 = u8"\u0100\u032a\u01e0\U00011100\u031c";   // UTF-8 equivalent to str1a
+    StringPiece sp2 = u8"A\u0304\u032a\u01e0\U00011100\u031c";  // UTF-8 equivalent to str1b
+
     int32_t num_locales = 0;
     const icu::Locale* locales = icu::Locale::getAvailableLocales(num_locales);
     for (int32_t i = 0; i < num_locales; i++) {
@@ -2142,12 +2155,16 @@ void CollationTest::TestColItrInfiniteLoop22511() {
         LocalPointer<Collator> coll(Collator::createInstance(l, errorCode));
         errorCode.assertSuccess();
         coll->setStrength(icu::Collator::IDENTICAL);
-        UCollationResult result = coll->compare(
-            str1, sizeof(str1)/sizeof(char16_t),
-            str2, sizeof(str2)/sizeof(char16_t),
-            errorCode);
+        for (int j = 0; testCases[j][0] != nullptr; j++) {
+            UCollationResult result = coll->compare(testCases[j][0], -1, testCases[j][1], -1, errorCode);
+            errorCode.assertSuccess();
+            assertEquals(UnicodeString("Locale ") + l.getName() + "UTF16 case:" + j, UCOL_EQUAL, result);
+        }
+
+        // Also test the UTF-8 versions
+        UCollationResult result = coll->compareUTF8(sp1, sp2, errorCode);
         errorCode.assertSuccess();
-        assertEquals(UnicodeString("Locale ") + l.getName(), UCOL_EQUAL, result);
+        assertEquals(UnicodeString("Locale ") + l.getName() + " UTF-8", UCOL_EQUAL, result);
     }
 }
 

icu4j/main/collate/src/main/java/com/ibm/icu/impl/coll/FCDUTF16CollationIterator.java

@@ -144,7 +144,7 @@ public int previousCodePoint() {
                             // To test possible trailing ccc, we need to check high surrogate
                             // (or previous character for broken surrogate pair).
                             if (pos - 1 != start) {
-                                normalizePrevSegment = CollationFCD.hasTccc(seq.charAt(pos - 2));
+                                normalizePrevSegment = Character.isHighSurrogate(seq.charAt(pos - 2)) && CollationFCD.hasTccc(seq.charAt(pos - 2));
                             }
                         } else {
                             normalizePrevSegment = CollationFCD.hasTccc(seq.charAt(pos - 1));

icu4j/main/collate/src/test/java/com/ibm/icu/dev/test/collator/CollationTest.java

@@ -1965,15 +1965,30 @@ public void TestBuilderContextsOverflow() {
     public void TestColItrInfiniteLoop22511() {
         // ICU-22511 Locale vi and wo triggers infinite loop for getting
         // collation key for these strings.
-        final String str1 = "\u0100\u032a\u01e0\ud804\udd00\u031c";
-        final String str2 = "\u0041\u0304\u032a\u01e0\ud804\udd00\u031c";
+        String[][] testCases = {
+            {
+                "\u0100\u032a\u01e0\ud804\udd00\u031c",
+                "\u0041\u0304\u032a\u01e0\ud804\udd00\u031c"    // Equivalent to above, but U+0100 is decomposed to U+0041 U+0304
+            },
+            {
+                "\u0100\u032a\u01e0\udd00\u031c",       // High surrogate 0xd804 is dropped
+                "\u0041\u0304\u032a\u01e0\udd00\u031c"  // Equivalent to above, but U+0100 is decomposed
+            },
+            {
+                "\u0100\u032a\u01e0\ud804\u031c",       // Low surrogate 0xdd00 is dropped
+                "\u0041\u0304\u032a\u01e0\ud804\u031c"  // Equivalent to above, but U+0100 is decomposed
+            }
+        };
 
         ULocale[] locales = ULocale.getAvailableLocales();
         for (ULocale loc : locales) {
             Collator coll = Collator.getInstance(loc);
             coll.setStrength(Collator.IDENTICAL);
-            int cmp = coll.compare(str1, str2);
-            assertEquals("Locale " + loc.toString(), 0, cmp);
+            for (int i = 0; i < testCases.length; i++) {
+                String[] testCase = testCases[i];
+                int cmp = coll.compare(testCase[0], testCase[1]);
+                assertEquals("Locale " + loc.toString() + " case:" + i, 0, cmp);
+            }
         }
     }
 }