3 * Tests for UtfNormal::cleanUp() function.
5 * Copyright © 2004 Brion Vibber <brion@pobox.com>
6 * https://www.mediawiki.org/
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2 of the License, or
11 * (at your option) any later version.
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
18 * You should have received a copy of the GNU General Public License along
19 * with this program; if not, write to the Free Software Foundation, Inc.,
20 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
21 * http://www.gnu.org/copyleft/gpl.html
27 * Additional tests for UtfNormal::cleanUp() function, inclusion
28 * regression checks for known problems.
34 * @todo covers tags, will be UtfNormal::cleanUp once the below is resolved
35 * @todo split me into test methods and providers per the below comment
37 * We ignore code coverage for this test suite until they are rewritten
38 * to use data providers (bug 46561).
41 class CleanUpTest
extends MediaWikiTestCase
{
43 public function testAscii() {
44 $text = 'This is plain ASCII text.';
45 $this->assertEquals( $text, UtfNormal
::cleanUp( $text ) );
49 public function testNull() {
50 $text = "a \x00 null";
51 $expect = "a \xef\xbf\xbd null";
54 bin2hex( UtfNormal
::cleanUp( $text ) ) );
58 public function testLatin() {
59 $text = "L'\xc3\xa9cole";
60 $this->assertEquals( $text, UtfNormal
::cleanUp( $text ) );
64 public function testLatinNormal() {
65 $text = "L'e\xcc\x81cole";
66 $expect = "L'\xc3\xa9cole";
67 $this->assertEquals( $expect, UtfNormal
::cleanUp( $text ) );
71 * This test is *very* expensive!
74 function XtestAllChars() {
75 $rep = UTF8_REPLACEMENT
;
76 for ( $i = 0x0; $i < UNICODE_MAX
; $i++
) {
77 $char = codepointToUtf8( $i );
78 $clean = UtfNormal
::cleanUp( $char );
79 $x = sprintf( "%04X", $i );
81 if ( $i %
0x1000 == 0 ) {
88 ( $i > 0x001f && $i < UNICODE_SURROGATE_FIRST
) ||
89 ( $i > UNICODE_SURROGATE_LAST
&& $i < 0xfffe ) ||
90 ( $i > 0xffff && $i <= UNICODE_MAX
)
92 if ( isset( UtfNormal
::$utfCanonicalComp[$char] )
93 ||
isset( UtfNormal
::$utfCanonicalDecomp[$char] )
95 $comp = UtfNormal
::NFC( $char );
99 "U+$x should be decomposed" );
104 "U+$x should be intact" );
107 $this->assertEquals( bin2hex( $rep ), bin2hex( $clean ), $x );
112 /** @todo document */
113 public function provideAllBytes() {
123 * @dataProvider provideAllBytes
126 function testBytes( $head, $tail ) {
127 for ( $i = 0x0; $i < 256; $i++
) {
128 $char = $head . chr( $i ) . $tail;
129 $clean = UtfNormal
::cleanUp( $char );
130 $x = sprintf( "%02X", $i );
135 ( $i > 0x001f && $i < 0x80 )
140 "ASCII byte $x should be intact" );
141 if ( $char != $clean ) {
145 $norm = $head . UTF8_REPLACEMENT
. $tail;
149 "Forbidden byte $x should be rejected" );
150 if ( $norm != $clean ) {
158 * @dataProvider provideAllBytes
161 function testDoubleBytes( $head, $tail ) {
162 for ( $first = 0xc0; $first < 0x100; $first +
= 2 ) {
163 for ( $second = 0x80; $second < 0x100; $second +
= 2 ) {
164 $char = $head . chr( $first ) . chr( $second ) . $tail;
165 $clean = UtfNormal
::cleanUp( $char );
166 $x = sprintf( "%02X,%02X", $first, $second );
167 if ( $first > 0xc1 &&
171 $norm = UtfNormal
::NFC( $char );
175 "Pair $x should be intact" );
176 if ( $norm != $clean ) {
179 } elseif ( $first > 0xfd ||
$second > 0xbf ) {
180 # fe and ff are not legal head bytes -- expect two replacement chars
181 $norm = $head . UTF8_REPLACEMENT
. UTF8_REPLACEMENT
. $tail;
185 "Forbidden pair $x should be rejected" );
186 if ( $norm != $clean ) {
190 $norm = $head . UTF8_REPLACEMENT
. $tail;
194 "Forbidden pair $x should be rejected" );
195 if ( $norm != $clean ) {
204 * @dataProvider provideAllBytes
207 function testTripleBytes( $head, $tail ) {
208 for ( $first = 0xc0; $first < 0x100; $first +
= 2 ) {
209 for ( $second = 0x80; $second < 0x100; $second +
= 2 ) {
210 #for( $third = 0x80; $third < 0x100; $third++ ) {
211 for ( $third = 0x80; $third < 0x81; $third++
) {
212 $char = $head . chr( $first ) . chr( $second ) . chr( $third ) . $tail;
213 $clean = UtfNormal
::cleanUp( $char );
214 $x = sprintf( "%02X,%02X,%02X", $first, $second, $third );
216 if ( $first >= 0xe0 &&
221 if ( $first == 0xe0 && $second < 0xa0 ) {
223 bin2hex( $head . UTF8_REPLACEMENT
. $tail ),
225 "Overlong triplet $x should be rejected" );
226 } elseif ( $first == 0xed &&
227 ( chr( $first ) . chr( $second ) . chr( $third ) ) >= UTF8_SURROGATE_FIRST
230 bin2hex( $head . UTF8_REPLACEMENT
. $tail ),
232 "Surrogate triplet $x should be rejected" );
235 bin2hex( UtfNormal
::NFC( $char ) ),
237 "Triplet $x should be intact" );
239 } elseif ( $first > 0xc1 && $first < 0xe0 && $second < 0xc0 ) {
241 bin2hex( UtfNormal
::NFC( $head . chr( $first ) .
242 chr( $second ) ) . UTF8_REPLACEMENT
. $tail ),
244 "Valid 2-byte $x + broken tail" );
245 } elseif ( $second > 0xc1 && $second < 0xe0 && $third < 0xc0 ) {
247 bin2hex( $head . UTF8_REPLACEMENT
.
248 UtfNormal
::NFC( chr( $second ) . chr( $third ) . $tail ) ),
250 "Broken head + valid 2-byte $x" );
251 } elseif ( ( $first > 0xfd ||
$second > 0xfd ) &&
252 ( ( $second > 0xbf && $third > 0xbf ) ||
253 ( $second < 0xc0 && $third < 0xc0 ) ||
254 ( $second > 0xfd ) ||
257 # fe and ff are not legal head bytes -- expect three replacement chars
259 bin2hex( $head . UTF8_REPLACEMENT
. UTF8_REPLACEMENT
. UTF8_REPLACEMENT
. $tail ),
261 "Forbidden triplet $x should be rejected" );
262 } elseif ( $first > 0xc2 && $second < 0xc0 && $third < 0xc0 ) {
264 bin2hex( $head . UTF8_REPLACEMENT
. $tail ),
266 "Forbidden triplet $x should be rejected" );
269 bin2hex( $head . UTF8_REPLACEMENT
. UTF8_REPLACEMENT
. $tail ),
271 "Forbidden triplet $x should be rejected" );
278 /** @todo document */
279 public function testChunkRegression() {
280 # Check for regression against a chunking bug
281 $text = "\x46\x55\xb8" .
288 $expect = "\x46\x55\xef\xbf\xbd" .
298 bin2hex( UtfNormal
::cleanUp( $text ) ) );
301 /** @todo document */
302 public function testInterposeRegression() {
317 $expect = "\x4e\x30" .
333 bin2hex( UtfNormal
::cleanUp( $text ) ) );
336 /** @todo document */
337 public function testOverlongRegression() {
339 "\x1a" . # forbidden ascii
341 "\xc1\xa6" . # overlong sequence
343 "\x1c" . # forbidden ascii
358 bin2hex( UtfNormal
::cleanUp( $text ) ) );
361 /** @todo document */
362 public function testSurrogateRegression() {
363 $text = "\xed\xb4\x96" . # surrogate 0xDD16
367 $expect = "\xef\xbf\xbd" .
373 bin2hex( UtfNormal
::cleanUp( $text ) ) );
376 /** @todo document */
377 public function testBomRegression() {
378 $text = "\xef\xbf\xbe" . # U+FFFE, illegal char
382 $expect = "\xef\xbf\xbd" .
388 bin2hex( UtfNormal
::cleanUp( $text ) ) );
391 /** @todo document */
392 public function testForbiddenRegression() {
393 $text = "\xef\xbf\xbf"; # U+FFFF, illegal char
394 $expect = "\xef\xbf\xbd";
397 bin2hex( UtfNormal
::cleanUp( $text ) ) );
400 /** @todo document */
401 public function testHangulRegression() {
402 $text = "\xed\x9c\xaf" . # Hangul char
403 "\xe1\x87\x81"; # followed by another final jamo
404 $expect = $text; # Should *not* change.
407 bin2hex( UtfNormal
::cleanUp( $text ) ) );