includes/normal/CleanUpTest.php

   1 <?php
   2
   3 #ini_set( 'memory_limit', '40M' );
   4
   5 require_once( 'PHPUnit.php' );
   6 require_once( 'UtfNormal.php' );
   7
   8 class CleanUpTest extends PHPUnit_TestCase {
   9         function CleanUpTest( $name ) {
  10                 $this->PHPUnit_TestCase( $name );
  11         }
  12
  13         function setUp() {
  14         }
  15
  16         function tearDown() {
  17         }
  18
  19         function testAscii() {
  20                 $text = 'This is plain ASCII text.';
  21                 $this->assertEquals( $text, UtfNormal::cleanUp( $text ) );
  22         }
  23
  24         function testNull() {
  25                 $text = "a \x00 null";
  26                 $expect = "a \xef\xbf\xbd null";
  27                 $this->assertEquals(
  28                         bin2hex( $expect ),
  29                         bin2hex( UtfNormal::cleanUp( $text ) ) );
  30         }
  31
  32         function testLatin() {
  33                 $text = "L'\xc3\xa9cole";
  34                 $this->assertEquals( $text, UtfNormal::cleanUp( $text ) );
  35         }
  36
  37         function testLatinNormal() {
  38                 $text = "L'e\xcc\x81cole";
  39                 $expect = "L'\xc3\xa9cole";
  40                 $this->assertEquals( $expect, UtfNormal::cleanUp( $text ) );
  41         }
  42
  43         # This test is *very* expensive!
  44         function XtestAllChars() {
  45                 $rep = UTF8_REPLACEMENT;
  46                 global $utfCanonicalComp, $utfCanonicalDecomp;
  47                 for( $i = 0x0; $i < UNICODE_MAX; $i++ ) {
  48                         $char = codepointToUtf8( $i );
  49                         $clean = UtfNormal::cleanUp( $char );
  50                         $x = sprintf( "%04X", $i );
  51                         if( $i % 0x1000 == 0 ) echo "U+$x\n";
  52                         if( $i == 0x0009 ||
  53                             $i == 0x000a ||
  54                             $i == 0x000d ||
  55                             ($i > 0x001f && $i < UNICODE_SURROGATE_FIRST) ||
  56                             ($i > UNICODE_SURROGATE_LAST && $i < 0xfffe ) ||
  57                             ($i > 0xffff && $i <= UNICODE_MAX ) ) {
  58                                 if( isset( $utfCanonicalComp[$char] ) || isset( $utfCanonicalDecomp[$char] ) ) {
  59                                     $comp = UtfNormal::NFC( $char );
  60                                         $this->assertEquals(
  61                                                 bin2hex( $comp ),
  62                                                 bin2hex( $clean ),
  63                                                 "U+$x should be decomposed" );
  64                                 } else {
  65                                         $this->assertEquals(
  66                                                 bin2hex( $char ),
  67                                                 bin2hex( $clean ),
  68                                                 "U+$x should be intact" );
  69                                 }
  70                         } else {
  71                                 $this->assertEquals( bin2hex( $rep ), bin2hex( $clean ), $x );
  72                         }
  73                 }
  74         }
  75
  76         function testAllBytes() {
  77                 $this->doTestBytes( '', '' );
  78                 $this->doTestBytes( 'x', '' );
  79                 $this->doTestBytes( '', 'x' );
  80                 $this->doTestBytes( 'x', 'x' );
  81         }
  82
  83         function doTestBytes( $head, $tail ) {
  84                 for( $i = 0x0; $i < 256; $i++ ) {
  85                         $char = $head . chr( $i ) . $tail;
  86                         $clean = UtfNormal::cleanUp( $char );
  87                         $x = sprintf( "%02X", $i );
  88                         if( $i == 0x0009 ||
  89                             $i == 0x000a ||
  90                             $i == 0x000d ||
  91                             ($i > 0x001f && $i < 0x80) ) {
  92                                 $this->assertEquals(
  93                                         bin2hex( $char ),
  94                                         bin2hex( $clean ),
  95                                         "ASCII byte $x should be intact" );
  96                                 if( $char != $clean ) return;
  97                         } else {
  98                                 $norm = $head . UTF8_REPLACEMENT . $tail;
  99                                 $this->assertEquals(
 100                                         bin2hex( $norm ),
 101                                         bin2hex( $clean ),
 102                                         "Forbidden byte $x should be rejected" );
 103                                 if( $norm != $clean ) return;
 104                         }
 105                 }
 106         }
 107
 108         function testDoubleBytes() {
 109                 $this->doTestDoubleBytes( '', '' );
 110                 $this->doTestDoubleBytes( 'x', '' );
 111                 $this->doTestDoubleBytes( '', 'x' );
 112                 $this->doTestDoubleBytes( 'x', 'x' );
 113         }
 114
 115         function doTestDoubleBytes( $head, $tail ) {
 116                 for( $first = 0xc0; $first < 0x100; $first++ ) {
 117                         for( $second = 0x80; $second < 0x100; $second++ ) {
 118                                 $char = $head . chr( $first ) . chr( $second ) . $tail;
 119                                 $clean = UtfNormal::cleanUp( $char );
 120                                 $x = sprintf( "%02X,%02X", $first, $second );
 121                                 if( $first > 0xc1 &&
 122                                     $first < 0xe0 &&
 123                                     $second < 0xc0 ) {
 124                                     $norm = UtfNormal::NFC( $char );
 125                                         $this->assertEquals(
 126                                                 bin2hex( $norm ),
 127                                                 bin2hex( $clean ),
 128                                                 "Pair $x should be intact" );
 129                                     if( $norm != $clean ) return;
 130                                 } elseif( $first > 0xfd || $second > 0xbf ) {
 131                                         # fe and ff are not legal head bytes -- expect two replacement chars
 132                                         $norm = $head . UTF8_REPLACEMENT . UTF8_REPLACEMENT . $tail;
 133                                         $this->assertEquals(
 134                                                 bin2hex( $norm ),
 135                                                 bin2hex( $clean ),
 136                                                 "Forbidden pair $x should be rejected" );
 137                                         if( $norm != $clean ) return;
 138                                 } else {
 139                                         $norm = $head . UTF8_REPLACEMENT . $tail;
 140                                         $this->assertEquals(
 141                                                 bin2hex( $norm ),
 142                                                 bin2hex( $clean ),
 143                                                 "Forbidden pair $x should be rejected" );
 144                                         if( $norm != $clean ) return;
 145                                 }
 146                         }
 147                 }
 148         }
 149
 150         function testTripleBytes() {
 151                 $this->doTestTripleBytes( '', '' );
 152                 $this->doTestTripleBytes( 'x', '' );
 153                 $this->doTestTripleBytes( '', 'x' );
 154                 $this->doTestTripleBytes( 'x', 'x' );
 155         }
 156
 157         function doTestTripleBytes( $head, $tail ) {
 158                 for( $first = 0xc0; $first < 0x100; $first++ ) {
 159                         for( $second = 0x80; $second < 0x100; $second++ ) {
 160                                 #for( $third = 0x80; $third < 0x100; $third++ ) {
 161                                 for( $third = 0x80; $third < 0x81; $third++ ) {
 162                                         $char = $head . chr( $first ) . chr( $second ) . chr( $third ) . $tail;
 163                                         $clean = UtfNormal::cleanUp( $char );
 164                                         $x = sprintf( "%02X,%02X,%02X", $first, $second, $third );
 165                                         if( $first >= 0xe0 &&
 166                                                 $first < 0xf0 &&
 167                                                 $second < 0xc0 &&
 168                                                 $third < 0xc0 ) {
 169                                                 if( $first == 0xe0 && $second < 0xa0 ) {
 170                                                         $this->assertEquals(
 171                                                                 bin2hex( $head . UTF8_REPLACEMENT . $tail ),
 172                                                                 bin2hex( $clean ),
 173                                                                 "Overlong triplet $x should be rejected" );
 174                                                 } elseif( $first == 0xed &&
 175                                                         ( chr( $first ) . chr( $second ) . chr( $third ))  >= UTF8_SURROGATE_FIRST ) {
 176                                                         $this->assertEquals(
 177                                                                 bin2hex( $head . UTF8_REPLACEMENT . $tail ),
 178                                                                 bin2hex( $clean ),
 179                                                                 "Surrogate triplet $x should be rejected" );
 180                                                 } else {
 181                                                         $this->assertEquals(
 182                                                                 bin2hex( UtfNormal::NFC( $char ) ),
 183                                                                 bin2hex( $clean ),
 184                                                                 "Triplet $x should be intact" );
 185                                                 }
 186                                         } elseif( $first > 0xc1 && $first < 0xe0 && $second < 0xc0 ) {
 187                                                 $this->assertEquals(
 188                                                         bin2hex( UtfNormal::NFC( $head . chr( $first ) . chr( $second ) ) . UTF8_REPLACEMENT . $tail ),
 189                                                         bin2hex( $clean ),
 190                                                         "Valid 2-byte $x + broken tail" );
 191                                         } elseif( $second > 0xc1 && $second < 0xe0 && $third < 0xc0 ) {
 192                                                 $this->assertEquals(
 193                                                         bin2hex( $head . UTF8_REPLACEMENT . UtfNormal::NFC( chr( $second ) . chr( $third ) . $tail ) ),
 194                                                         bin2hex( $clean ),
 195                                                         "Broken head + valid 2-byte $x" );
 196                                         } elseif( $first > 0xfd && ( ( $second > 0xbf && $third > 0xbf ) || ($second < 0xc0 && $third < 0xc0 ) || ($second > 0xfd ) || ($third > 0xfd) ) ) {
 197                                                 # fe and ff are not legal head bytes -- expect three replacement chars
 198                                                 $this->assertEquals(
 199                                                         bin2hex( $head . UTF8_REPLACEMENT . UTF8_REPLACEMENT . UTF8_REPLACEMENT . $tail ),
 200                                                         bin2hex( $clean ),
 201                                                         "Forbidden triplet $x should be rejected" );
 202                                         } elseif( $second < 0xc0 && $second < 0xc0 ) {
 203                                                 $this->assertEquals(
 204                                                         bin2hex( $head . UTF8_REPLACEMENT . $tail ),
 205                                                         bin2hex( $clean ),
 206                                                         "Forbidden triplet $x should be rejected" );
 207                                         } else {
 208                                                 $this->assertEquals(
 209                                                         bin2hex( $head . UTF8_REPLACEMENT . UTF8_REPLACEMENT . $tail ),
 210                                                         bin2hex( $clean ),
 211                                                         "Forbidden triplet $x should be rejected" );
 212                                         }
 213                                 }
 214                         }
 215                 }
 216         }
 217
 218 }
 219
 220
 221 $suite =& new PHPUnit_TestSuite( 'CleanUpTest' );
 222 $result = PHPUnit::run( $suite );
 223 echo $result->toString();
 224
 225 ?>