99ec05ddcb0382ac8af5cf986e24b06b7448b1aa
3 * Tests for UtfNormal::cleanUp() function.
5 * Copyright © 2004 Brion Vibber <brion@pobox.com>
6 * http://www.mediawiki.org/
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2 of the License, or
11 * (at your option) any later version.
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
18 * You should have received a copy of the GNU General Public License along
19 * with this program; if not, write to the Free Software Foundation, Inc.,
20 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
21 * http://www.gnu.org/copyleft/gpl.html
27 * Additional tests for UtfNormal::cleanUp() function, inclusion
28 * regression checks for known problems.
34 class CleanUpTest
extends MediaWikiTestCase
{
36 function testAscii() {
37 $text = 'This is plain ASCII text.';
38 $this->assertEquals( $text, UtfNormal
::cleanUp( $text ) );
43 $text = "a \x00 null";
44 $expect = "a \xef\xbf\xbd null";
47 bin2hex( UtfNormal
::cleanUp( $text ) ) );
51 function testLatin() {
52 $text = "L'\xc3\xa9cole";
53 $this->assertEquals( $text, UtfNormal
::cleanUp( $text ) );
57 function testLatinNormal() {
58 $text = "L'e\xcc\x81cole";
59 $expect = "L'\xc3\xa9cole";
60 $this->assertEquals( $expect, UtfNormal
::cleanUp( $text ) );
64 * This test is *very* expensive!
67 function XtestAllChars() {
68 $rep = UTF8_REPLACEMENT
;
69 for ( $i = 0x0; $i < UNICODE_MAX
; $i++
) {
70 $char = codepointToUtf8( $i );
71 $clean = UtfNormal
::cleanUp( $char );
72 $x = sprintf( "%04X", $i );
74 if ( $i %
0x1000 == 0 ) {
81 ( $i > 0x001f && $i < UNICODE_SURROGATE_FIRST
) ||
82 ( $i > UNICODE_SURROGATE_LAST
&& $i < 0xfffe ) ||
83 ( $i > 0xffff && $i <= UNICODE_MAX
)
85 if ( isset( UtfNormal
::$utfCanonicalComp[$char] ) ||
isset( UtfNormal
::$utfCanonicalDecomp[$char] ) ) {
86 $comp = UtfNormal
::NFC( $char );
90 "U+$x should be decomposed" );
95 "U+$x should be intact" );
98 $this->assertEquals( bin2hex( $rep ), bin2hex( $clean ), $x );
103 /** @todo document */
104 function testAllBytes() {
105 $this->doTestBytes( '', '' );
106 $this->doTestBytes( 'x', '' );
107 $this->doTestBytes( '', 'x' );
108 $this->doTestBytes( 'x', 'x' );
111 /** @todo document */
112 function doTestBytes( $head, $tail ) {
113 for ( $i = 0x0; $i < 256; $i++
) {
114 $char = $head . chr( $i ) . $tail;
115 $clean = UtfNormal
::cleanUp( $char );
116 $x = sprintf( "%02X", $i );
121 ( $i > 0x001f && $i < 0x80 )
126 "ASCII byte $x should be intact" );
127 if ( $char != $clean ) {
131 $norm = $head . UTF8_REPLACEMENT
. $tail;
135 "Forbidden byte $x should be rejected" );
136 if ( $norm != $clean ) {
143 /** @todo document */
144 function testDoubleBytes() {
145 $this->doTestDoubleBytes( '', '' );
146 $this->doTestDoubleBytes( 'x', '' );
147 $this->doTestDoubleBytes( '', 'x' );
148 $this->doTestDoubleBytes( 'x', 'x' );
154 function doTestDoubleBytes( $head, $tail ) {
155 for ( $first = 0xc0; $first < 0x100; $first +
= 2 ) {
156 for ( $second = 0x80; $second < 0x100; $second +
= 2 ) {
157 $char = $head . chr( $first ) . chr( $second ) . $tail;
158 $clean = UtfNormal
::cleanUp( $char );
159 $x = sprintf( "%02X,%02X", $first, $second );
160 if ( $first > 0xc1 &&
164 $norm = UtfNormal
::NFC( $char );
168 "Pair $x should be intact" );
169 if ( $norm != $clean ) {
172 } elseif ( $first > 0xfd ||
$second > 0xbf ) {
173 # fe and ff are not legal head bytes -- expect two replacement chars
174 $norm = $head . UTF8_REPLACEMENT
. UTF8_REPLACEMENT
. $tail;
178 "Forbidden pair $x should be rejected" );
179 if ( $norm != $clean ) {
183 $norm = $head . UTF8_REPLACEMENT
. $tail;
187 "Forbidden pair $x should be rejected" );
188 if ( $norm != $clean ) {
196 /** @todo document */
197 function testTripleBytes() {
198 $this->doTestTripleBytes( '', '' );
199 $this->doTestTripleBytes( 'x', '' );
200 $this->doTestTripleBytes( '', 'x' );
201 $this->doTestTripleBytes( 'x', 'x' );
204 /** @todo document */
205 function doTestTripleBytes( $head, $tail ) {
206 for ( $first = 0xc0; $first < 0x100; $first +
= 2 ) {
207 for ( $second = 0x80; $second < 0x100; $second +
= 2 ) {
208 #for( $third = 0x80; $third < 0x100; $third++ ) {
209 for ( $third = 0x80; $third < 0x81; $third++
) {
210 $char = $head . chr( $first ) . chr( $second ) . chr( $third ) . $tail;
211 $clean = UtfNormal
::cleanUp( $char );
212 $x = sprintf( "%02X,%02X,%02X", $first, $second, $third );
214 if ( $first >= 0xe0 &&
219 if ( $first == 0xe0 && $second < 0xa0 ) {
221 bin2hex( $head . UTF8_REPLACEMENT
. $tail ),
223 "Overlong triplet $x should be rejected" );
224 } elseif ( $first == 0xed &&
225 ( chr( $first ) . chr( $second ) . chr( $third ) ) >= UTF8_SURROGATE_FIRST
228 bin2hex( $head . UTF8_REPLACEMENT
. $tail ),
230 "Surrogate triplet $x should be rejected" );
233 bin2hex( UtfNormal
::NFC( $char ) ),
235 "Triplet $x should be intact" );
237 } elseif ( $first > 0xc1 && $first < 0xe0 && $second < 0xc0 ) {
239 bin2hex( UtfNormal
::NFC( $head . chr( $first ) . chr( $second ) ) . UTF8_REPLACEMENT
. $tail ),
241 "Valid 2-byte $x + broken tail" );
242 } elseif ( $second > 0xc1 && $second < 0xe0 && $third < 0xc0 ) {
244 bin2hex( $head . UTF8_REPLACEMENT
. UtfNormal
::NFC( chr( $second ) . chr( $third ) . $tail ) ),
246 "Broken head + valid 2-byte $x" );
247 } elseif ( ( $first > 0xfd ||
$second > 0xfd ) &&
248 ( ( $second > 0xbf && $third > 0xbf ) ||
249 ( $second < 0xc0 && $third < 0xc0 ) ||
250 ( $second > 0xfd ) ||
253 # fe and ff are not legal head bytes -- expect three replacement chars
255 bin2hex( $head . UTF8_REPLACEMENT
. UTF8_REPLACEMENT
. UTF8_REPLACEMENT
. $tail ),
257 "Forbidden triplet $x should be rejected" );
258 } elseif ( $first > 0xc2 && $second < 0xc0 && $third < 0xc0 ) {
260 bin2hex( $head . UTF8_REPLACEMENT
. $tail ),
262 "Forbidden triplet $x should be rejected" );
265 bin2hex( $head . UTF8_REPLACEMENT
. UTF8_REPLACEMENT
. $tail ),
267 "Forbidden triplet $x should be rejected" );
274 /** @todo document */
275 function testChunkRegression() {
276 # Check for regression against a chunking bug
277 $text = "\x46\x55\xb8" .
284 $expect = "\x46\x55\xef\xbf\xbd" .
294 bin2hex( UtfNormal
::cleanUp( $text ) ) );
297 /** @todo document */
298 function testInterposeRegression() {
313 $expect = "\x4e\x30" .
329 bin2hex( UtfNormal
::cleanUp( $text ) ) );
332 /** @todo document */
333 function testOverlongRegression() {
335 "\x1a" . # forbidden ascii
337 "\xc1\xa6" . # overlong sequence
339 "\x1c" . # forbidden ascii
354 bin2hex( UtfNormal
::cleanUp( $text ) ) );
357 /** @todo document */
358 function testSurrogateRegression() {
359 $text = "\xed\xb4\x96" . # surrogate 0xDD16
363 $expect = "\xef\xbf\xbd" .
369 bin2hex( UtfNormal
::cleanUp( $text ) ) );
372 /** @todo document */
373 function testBomRegression() {
374 $text = "\xef\xbf\xbe" . # U+FFFE, illegal char
378 $expect = "\xef\xbf\xbd" .
384 bin2hex( UtfNormal
::cleanUp( $text ) ) );
387 /** @todo document */
388 function testForbiddenRegression() {
389 $text = "\xef\xbf\xbf"; # U+FFFF, illegal char
390 $expect = "\xef\xbf\xbd";
393 bin2hex( UtfNormal
::cleanUp( $text ) ) );
396 /** @todo document */
397 function testHangulRegression() {
398 $text = "\xed\x9c\xaf" . # Hangul char
399 "\xe1\x87\x81"; # followed by another final jamo
400 $expect = $text; # Should *not* change.
403 bin2hex( UtfNormal
::cleanUp( $text ) ) );