tests/phpunit/includes/normal/CleanUpTest.php

   1 <?php
   2 /**
   3  * Tests for UtfNormal::cleanUp() function.
   4  *
   5  * Copyright © 2004 Brion Vibber <brion@pobox.com>
   6  * https://www.mediawiki.org/
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License as published by
  10  * the Free Software Foundation; either version 2 of the License, or
  11  * (at your option) any later version.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  16  * GNU General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU General Public License along
  19  * with this program; if not, write to the Free Software Foundation, Inc.,
  20  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  21  * http://www.gnu.org/copyleft/gpl.html
  22  *
  23  * @file
  24  */
  25
  26 /**
  27  * Additional tests for UtfNormal::cleanUp() function, inclusion
  28  * regression checks for known problems.
  29  * Requires PHPUnit.
  30  *
  31  * @ingroup UtfNormal
  32  * @group Large
  33  *
  34  * @todo covers tags, will be UtfNormal::cleanUp once the below is resolved
  35  * @todo split me into test methods and providers per the below comment
  36  *
  37  * We ignore code coverage for this test suite until they are rewritten
  38  * to use data providers (bug 46561).
  39  * @codeCoverageIgnore
  40  */
  41 class CleanUpTest extends MediaWikiTestCase {
  42         /** @todo document */
  43         public function testAscii() {
  44                 $text = 'This is plain ASCII text.';
  45                 $this->assertEquals( $text, UtfNormal::cleanUp( $text ) );
  46         }
  47
  48         /** @todo document */
  49         public function testNull() {
  50                 $text = "a \x00 null";
  51                 $expect = "a \xef\xbf\xbd null";
  52                 $this->assertEquals(
  53                         bin2hex( $expect ),
  54                         bin2hex( UtfNormal::cleanUp( $text ) ) );
  55         }
  56
  57         /** @todo document */
  58         public function testLatin() {
  59                 $text = "L'\xc3\xa9cole";
  60                 $this->assertEquals( $text, UtfNormal::cleanUp( $text ) );
  61         }
  62
  63         /** @todo document */
  64         public function testLatinNormal() {
  65                 $text = "L'e\xcc\x81cole";
  66                 $expect = "L'\xc3\xa9cole";
  67                 $this->assertEquals( $expect, UtfNormal::cleanUp( $text ) );
  68         }
  69
  70         /**
  71          * This test is *very* expensive!
  72          * @todo document
  73          */
  74         function XtestAllChars() {
  75                 $rep = UTF8_REPLACEMENT;
  76                 for ( $i = 0x0; $i < UNICODE_MAX; $i++ ) {
  77                         $char = codepointToUtf8( $i );
  78                         $clean = UtfNormal::cleanUp( $char );
  79                         $x = sprintf( "%04X", $i );
  80
  81                         if ( $i % 0x1000 == 0 ) {
  82                                 echo "U+$x\n";
  83                         }
  84
  85                         if ( $i == 0x0009 ||
  86                                 $i == 0x000a ||
  87                                 $i == 0x000d ||
  88                                 ( $i > 0x001f && $i < UNICODE_SURROGATE_FIRST ) ||
  89                                 ( $i > UNICODE_SURROGATE_LAST && $i < 0xfffe ) ||
  90                                 ( $i > 0xffff && $i <= UNICODE_MAX )
  91                         ) {
  92                                 if ( isset( UtfNormal::$utfCanonicalComp[$char] )
  93                                         || isset( UtfNormal::$utfCanonicalDecomp[$char] )
  94                                 ) {
  95                                         $comp = UtfNormal::NFC( $char );
  96                                         $this->assertEquals(
  97                                                 bin2hex( $comp ),
  98                                                 bin2hex( $clean ),
  99                                                 "U+$x should be decomposed" );
 100                                 } else {
 101                                         $this->assertEquals(
 102                                                 bin2hex( $char ),
 103                                                 bin2hex( $clean ),
 104                                                 "U+$x should be intact" );
 105                                 }
 106                         } else {
 107                                 $this->assertEquals( bin2hex( $rep ), bin2hex( $clean ), $x );
 108                         }
 109                 }
 110         }
 111
 112         /** @todo document */
 113         public function testAllBytes() {
 114                 $this->doTestBytes( '', '' );
 115                 $this->doTestBytes( 'x', '' );
 116                 $this->doTestBytes( '', 'x' );
 117                 $this->doTestBytes( 'x', 'x' );
 118         }
 119
 120         /** @todo document */
 121         function doTestBytes( $head, $tail ) {
 122                 for ( $i = 0x0; $i < 256; $i++ ) {
 123                         $char = $head . chr( $i ) . $tail;
 124                         $clean = UtfNormal::cleanUp( $char );
 125                         $x = sprintf( "%02X", $i );
 126
 127                         if ( $i == 0x0009 ||
 128                                 $i == 0x000a ||
 129                                 $i == 0x000d ||
 130                                 ( $i > 0x001f && $i < 0x80 )
 131                         ) {
 132                                 $this->assertEquals(
 133                                         bin2hex( $char ),
 134                                         bin2hex( $clean ),
 135                                         "ASCII byte $x should be intact" );
 136                                 if ( $char != $clean ) {
 137                                         return;
 138                                 }
 139                         } else {
 140                                 $norm = $head . UTF8_REPLACEMENT . $tail;
 141                                 $this->assertEquals(
 142                                         bin2hex( $norm ),
 143                                         bin2hex( $clean ),
 144                                         "Forbidden byte $x should be rejected" );
 145                                 if ( $norm != $clean ) {
 146                                         return;
 147                                 }
 148                         }
 149                 }
 150         }
 151
 152         /** @todo document */
 153         public function testDoubleBytes() {
 154                 $this->doTestDoubleBytes( '', '' );
 155                 $this->doTestDoubleBytes( 'x', '' );
 156                 $this->doTestDoubleBytes( '', 'x' );
 157                 $this->doTestDoubleBytes( 'x', 'x' );
 158         }
 159
 160         /**
 161          * @todo document
 162          */
 163         function doTestDoubleBytes( $head, $tail ) {
 164                 for ( $first = 0xc0; $first < 0x100; $first += 2 ) {
 165                         for ( $second = 0x80; $second < 0x100; $second += 2 ) {
 166                                 $char = $head . chr( $first ) . chr( $second ) . $tail;
 167                                 $clean = UtfNormal::cleanUp( $char );
 168                                 $x = sprintf( "%02X,%02X", $first, $second );
 169                                 if ( $first > 0xc1 &&
 170                                         $first < 0xe0 &&
 171                                         $second < 0xc0
 172                                 ) {
 173                                         $norm = UtfNormal::NFC( $char );
 174                                         $this->assertEquals(
 175                                                 bin2hex( $norm ),
 176                                                 bin2hex( $clean ),
 177                                                 "Pair $x should be intact" );
 178                                         if ( $norm != $clean ) {
 179                                                 return;
 180                                         }
 181                                 } elseif ( $first > 0xfd || $second > 0xbf ) {
 182                                         # fe and ff are not legal head bytes -- expect two replacement chars
 183                                         $norm = $head . UTF8_REPLACEMENT . UTF8_REPLACEMENT . $tail;
 184                                         $this->assertEquals(
 185                                                 bin2hex( $norm ),
 186                                                 bin2hex( $clean ),
 187                                                 "Forbidden pair $x should be rejected" );
 188                                         if ( $norm != $clean ) {
 189                                                 return;
 190                                         }
 191                                 } else {
 192                                         $norm = $head . UTF8_REPLACEMENT . $tail;
 193                                         $this->assertEquals(
 194                                                 bin2hex( $norm ),
 195                                                 bin2hex( $clean ),
 196                                                 "Forbidden pair $x should be rejected" );
 197                                         if ( $norm != $clean ) {
 198                                                 return;
 199                                         }
 200                                 }
 201                         }
 202                 }
 203         }
 204
 205         /** @todo document */
 206         public function testTripleBytes() {
 207                 $this->doTestTripleBytes( '', '' );
 208                 $this->doTestTripleBytes( 'x', '' );
 209                 $this->doTestTripleBytes( '', 'x' );
 210                 $this->doTestTripleBytes( 'x', 'x' );
 211         }
 212
 213         /** @todo document */
 214         function doTestTripleBytes( $head, $tail ) {
 215                 for ( $first = 0xc0; $first < 0x100; $first += 2 ) {
 216                         for ( $second = 0x80; $second < 0x100; $second += 2 ) {
 217                                 #for( $third = 0x80; $third < 0x100; $third++ ) {
 218                                 for ( $third = 0x80; $third < 0x81; $third++ ) {
 219                                         $char = $head . chr( $first ) . chr( $second ) . chr( $third ) . $tail;
 220                                         $clean = UtfNormal::cleanUp( $char );
 221                                         $x = sprintf( "%02X,%02X,%02X", $first, $second, $third );
 222
 223                                         if ( $first >= 0xe0 &&
 224                                                 $first < 0xf0 &&
 225                                                 $second < 0xc0 &&
 226                                                 $third < 0xc0
 227                                         ) {
 228                                                 if ( $first == 0xe0 && $second < 0xa0 ) {
 229                                                         $this->assertEquals(
 230                                                                 bin2hex( $head . UTF8_REPLACEMENT . $tail ),
 231                                                                 bin2hex( $clean ),
 232                                                                 "Overlong triplet $x should be rejected" );
 233                                                 } elseif ( $first == 0xed &&
 234                                                         ( chr( $first ) . chr( $second ) . chr( $third ) ) >= UTF8_SURROGATE_FIRST
 235                                                 ) {
 236                                                         $this->assertEquals(
 237                                                                 bin2hex( $head . UTF8_REPLACEMENT . $tail ),
 238                                                                 bin2hex( $clean ),
 239                                                                 "Surrogate triplet $x should be rejected" );
 240                                                 } else {
 241                                                         $this->assertEquals(
 242                                                                 bin2hex( UtfNormal::NFC( $char ) ),
 243                                                                 bin2hex( $clean ),
 244                                                                 "Triplet $x should be intact" );
 245                                                 }
 246                                         } elseif ( $first > 0xc1 && $first < 0xe0 && $second < 0xc0 ) {
 247                                                 $this->assertEquals(
 248                                                         bin2hex( UtfNormal::NFC( $head . chr( $first ) .
 249                                                                         chr( $second ) ) . UTF8_REPLACEMENT . $tail ),
 250                                                         bin2hex( $clean ),
 251                                                         "Valid 2-byte $x + broken tail" );
 252                                         } elseif ( $second > 0xc1 && $second < 0xe0 && $third < 0xc0 ) {
 253                                                 $this->assertEquals(
 254                                                         bin2hex( $head . UTF8_REPLACEMENT .
 255                                                                 UtfNormal::NFC( chr( $second ) . chr( $third ) . $tail ) ),
 256                                                         bin2hex( $clean ),
 257                                                         "Broken head + valid 2-byte $x" );
 258                                         } elseif ( ( $first > 0xfd || $second > 0xfd ) &&
 259                                                 ( ( $second > 0xbf && $third > 0xbf ) ||
 260                                                         ( $second < 0xc0 && $third < 0xc0 ) ||
 261                                                         ( $second > 0xfd ) ||
 262                                                         ( $third > 0xfd ) )
 263                                         ) {
 264                                                 # fe and ff are not legal head bytes -- expect three replacement chars
 265                                                 $this->assertEquals(
 266                                                         bin2hex( $head . UTF8_REPLACEMENT . UTF8_REPLACEMENT . UTF8_REPLACEMENT . $tail ),
 267                                                         bin2hex( $clean ),
 268                                                         "Forbidden triplet $x should be rejected" );
 269                                         } elseif ( $first > 0xc2 && $second < 0xc0 && $third < 0xc0 ) {
 270                                                 $this->assertEquals(
 271                                                         bin2hex( $head . UTF8_REPLACEMENT . $tail ),
 272                                                         bin2hex( $clean ),
 273                                                         "Forbidden triplet $x should be rejected" );
 274                                         } else {
 275                                                 $this->assertEquals(
 276                                                         bin2hex( $head . UTF8_REPLACEMENT . UTF8_REPLACEMENT . $tail ),
 277                                                         bin2hex( $clean ),
 278                                                         "Forbidden triplet $x should be rejected" );
 279                                         }
 280                                 }
 281                         }
 282                 }
 283         }
 284
 285         /** @todo document */
 286         public function testChunkRegression() {
 287                 # Check for regression against a chunking bug
 288                 $text = "\x46\x55\xb8" .
 289                         "\xdc\x96" .
 290                         "\xee" .
 291                         "\xe7" .
 292                         "\x44" .
 293                         "\xaa" .
 294                         "\x2f\x25";
 295                 $expect = "\x46\x55\xef\xbf\xbd" .
 296                         "\xdc\x96" .
 297                         "\xef\xbf\xbd" .
 298                         "\xef\xbf\xbd" .
 299                         "\x44" .
 300                         "\xef\xbf\xbd" .
 301                         "\x2f\x25";
 302
 303                 $this->assertEquals(
 304                         bin2hex( $expect ),
 305                         bin2hex( UtfNormal::cleanUp( $text ) ) );
 306         }
 307
 308         /** @todo document */
 309         public function testInterposeRegression() {
 310                 $text = "\x4e\x30" .
 311                         "\xb1" . # bad tail
 312                         "\x3a" .
 313                         "\x92" . # bad tail
 314                         "\x62\x3a" .
 315                         "\x84" . # bad tail
 316                         "\x43" .
 317                         "\xc6" . # bad head
 318                         "\x3f" .
 319                         "\x92" . # bad tail
 320                         "\xad" . # bad tail
 321                         "\x7d" .
 322                         "\xd9\x95";
 323
 324                 $expect = "\x4e\x30" .
 325                         "\xef\xbf\xbd" .
 326                         "\x3a" .
 327                         "\xef\xbf\xbd" .
 328                         "\x62\x3a" .
 329                         "\xef\xbf\xbd" .
 330                         "\x43" .
 331                         "\xef\xbf\xbd" .
 332                         "\x3f" .
 333                         "\xef\xbf\xbd" .
 334                         "\xef\xbf\xbd" .
 335                         "\x7d" .
 336                         "\xd9\x95";
 337
 338                 $this->assertEquals(
 339                         bin2hex( $expect ),
 340                         bin2hex( UtfNormal::cleanUp( $text ) ) );
 341         }
 342
 343         /** @todo document */
 344         public function testOverlongRegression() {
 345                 $text = "\x67" .
 346                         "\x1a" . # forbidden ascii
 347                         "\xea" . # bad head
 348                         "\xc1\xa6" . # overlong sequence
 349                         "\xad" . # bad tail
 350                         "\x1c" . # forbidden ascii
 351                         "\xb0" . # bad tail
 352                         "\x3c" .
 353                         "\x9e"; # bad tail
 354                 $expect = "\x67" .
 355                         "\xef\xbf\xbd" .
 356                         "\xef\xbf\xbd" .
 357                         "\xef\xbf\xbd" .
 358                         "\xef\xbf\xbd" .
 359                         "\xef\xbf\xbd" .
 360                         "\xef\xbf\xbd" .
 361                         "\x3c" .
 362                         "\xef\xbf\xbd";
 363                 $this->assertEquals(
 364                         bin2hex( $expect ),
 365                         bin2hex( UtfNormal::cleanUp( $text ) ) );
 366         }
 367
 368         /** @todo document */
 369         public function testSurrogateRegression() {
 370                 $text = "\xed\xb4\x96" . # surrogate 0xDD16
 371                         "\x83" . # bad tail
 372                         "\xb4" . # bad tail
 373                         "\xac"; # bad head
 374                 $expect = "\xef\xbf\xbd" .
 375                         "\xef\xbf\xbd" .
 376                         "\xef\xbf\xbd" .
 377                         "\xef\xbf\xbd";
 378                 $this->assertEquals(
 379                         bin2hex( $expect ),
 380                         bin2hex( UtfNormal::cleanUp( $text ) ) );
 381         }
 382
 383         /** @todo document */
 384         public function testBomRegression() {
 385                 $text = "\xef\xbf\xbe" . # U+FFFE, illegal char
 386                         "\xb2" . # bad tail
 387                         "\xef" . # bad head
 388                         "\x59";
 389                 $expect = "\xef\xbf\xbd" .
 390                         "\xef\xbf\xbd" .
 391                         "\xef\xbf\xbd" .
 392                         "\x59";
 393                 $this->assertEquals(
 394                         bin2hex( $expect ),
 395                         bin2hex( UtfNormal::cleanUp( $text ) ) );
 396         }
 397
 398         /** @todo document */
 399         public function testForbiddenRegression() {
 400                 $text = "\xef\xbf\xbf"; # U+FFFF, illegal char
 401                 $expect = "\xef\xbf\xbd";
 402                 $this->assertEquals(
 403                         bin2hex( $expect ),
 404                         bin2hex( UtfNormal::cleanUp( $text ) ) );
 405         }
 406
 407         /** @todo document */
 408         public function testHangulRegression() {
 409                 $text = "\xed\x9c\xaf" . # Hangul char
 410                         "\xe1\x87\x81"; # followed by another final jamo
 411                 $expect = $text; # Should *not* change.
 412                 $this->assertEquals(
 413                         bin2hex( $expect ),
 414                         bin2hex( UtfNormal::cleanUp( $text ) ) );
 415         }
 416 }