From: Chad Horohoe Date: Wed, 4 May 2011 23:05:25 +0000 (+0000) Subject: Move CleanUpTest, fixes in next commit X-Git-Tag: 1.31.0-rc.0~30402 X-Git-Url: http://git.cyclocoop.org/%22.%24info%5B?a=commitdiff_plain;h=6944e565815d917145e9db80093c5767034392c6;p=lhc%2Fweb%2Fwiklou.git Move CleanUpTest, fixes in next commit --- diff --git a/includes/normal/CleanUpTest.php b/includes/normal/CleanUpTest.php deleted file mode 100644 index 549a04062e..0000000000 --- a/includes/normal/CleanUpTest.php +++ /dev/null @@ -1,425 +0,0 @@ - - * http://www.mediawiki.org/ - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - * http://www.gnu.org/copyleft/gpl.html - * - * @file - * @ingroup UtfNormal - */ - - -if( php_sapi_name() != 'cli' ) { - die( "Run me from the command line please.\n" ); -} - -/** */ -if( isset( $_SERVER['argv'] ) && in_array( '--icu', $_SERVER['argv'] ) ) { - dl( 'php_utfnormal.so' ); -} - -#ini_set( 'memory_limit', '40M' ); - -require_once( 'PHPUnit/Runner/Version.php' ); -if( version_compare( PHPUnit_Runner_Version::id(), '3.5.0', '>=' ) ) { - # PHPUnit 3.5.0 introduced a nice autoloader based on class name - require_once( 'PHPUnit/Autoload.php' ); -} else { - # Keep the old pre PHPUnit 3.5.0 behaviour for compatibility - require_once 'PHPUnit/Framework.php'; -} -require_once 'PHPUnit/TextUI/TestRunner.php'; - -require_once 'UtfNormal.php'; - -/** - * Additional tests for UtfNormal::cleanUp() function, inclusion - * regression checks for known problems. - * Requires PHPUnit. - * - * @ingroup UtfNormal - * @private - */ -class CleanUpTest extends PHPUnit_Framework_TestCase { - /** @todo document */ - function setUp() { - } - - /** @todo document */ - function tearDown() { - } - - /** @todo document */ - function testAscii() { - $text = 'This is plain ASCII text.'; - $this->assertEquals( $text, UtfNormal::cleanUp( $text ) ); - } - - /** @todo document */ - function testNull() { - $text = "a \x00 null"; - $expect = "a \xef\xbf\xbd null"; - $this->assertEquals( - bin2hex( $expect ), - bin2hex( UtfNormal::cleanUp( $text ) ) ); - } - - /** @todo document */ - function testLatin() { - $text = "L'\xc3\xa9cole"; - $this->assertEquals( $text, UtfNormal::cleanUp( $text ) ); - } - - /** @todo document */ - function testLatinNormal() { - $text = "L'e\xcc\x81cole"; - $expect = "L'\xc3\xa9cole"; - $this->assertEquals( $expect, UtfNormal::cleanUp( $text ) ); - } - - /** - * This test is *very* expensive! - * @todo document - */ - function XtestAllChars() { - $rep = UTF8_REPLACEMENT; - for( $i = 0x0; $i < UNICODE_MAX; $i++ ) { - $char = codepointToUtf8( $i ); - $clean = UtfNormal::cleanUp( $char ); - $x = sprintf( "%04X", $i ); - if( $i % 0x1000 == 0 ) echo "U+$x\n"; - if( $i == 0x0009 || - $i == 0x000a || - $i == 0x000d || - ($i > 0x001f && $i < UNICODE_SURROGATE_FIRST) || - ($i > UNICODE_SURROGATE_LAST && $i < 0xfffe ) || - ($i > 0xffff && $i <= UNICODE_MAX ) ) { - if( isset( UtfNormal::$utfCanonicalComp[$char] ) || isset( UtfNormal::$utfCanonicalDecomp[$char] ) ) { - $comp = UtfNormal::NFC( $char ); - $this->assertEquals( - bin2hex( $comp ), - bin2hex( $clean ), - "U+$x should be decomposed" ); - } else { - $this->assertEquals( - bin2hex( $char ), - bin2hex( $clean ), - "U+$x should be intact" ); - } - } else { - $this->assertEquals( bin2hex( $rep ), bin2hex( $clean ), $x ); - } - } - } - - /** @todo document */ - function testAllBytes() { - $this->doTestBytes( '', '' ); - $this->doTestBytes( 'x', '' ); - $this->doTestBytes( '', 'x' ); - $this->doTestBytes( 'x', 'x' ); - } - - /** @todo document */ - function doTestBytes( $head, $tail ) { - for( $i = 0x0; $i < 256; $i++ ) { - $char = $head . chr( $i ) . $tail; - $clean = UtfNormal::cleanUp( $char ); - $x = sprintf( "%02X", $i ); - if( $i == 0x0009 || - $i == 0x000a || - $i == 0x000d || - ($i > 0x001f && $i < 0x80) ) { - $this->assertEquals( - bin2hex( $char ), - bin2hex( $clean ), - "ASCII byte $x should be intact" ); - if( $char != $clean ) return; - } else { - $norm = $head . UTF8_REPLACEMENT . $tail; - $this->assertEquals( - bin2hex( $norm ), - bin2hex( $clean ), - "Forbidden byte $x should be rejected" ); - if( $norm != $clean ) return; - } - } - } - - /** @todo document */ - function testDoubleBytes() { - $this->doTestDoubleBytes( '', '' ); - $this->doTestDoubleBytes( 'x', '' ); - $this->doTestDoubleBytes( '', 'x' ); - $this->doTestDoubleBytes( 'x', 'x' ); - } - - /** - * @todo document - */ - function doTestDoubleBytes( $head, $tail ) { - for( $first = 0xc0; $first < 0x100; $first++ ) { - for( $second = 0x80; $second < 0x100; $second++ ) { - $char = $head . chr( $first ) . chr( $second ) . $tail; - $clean = UtfNormal::cleanUp( $char ); - $x = sprintf( "%02X,%02X", $first, $second ); - if( $first > 0xc1 && - $first < 0xe0 && - $second < 0xc0 ) { - $norm = UtfNormal::NFC( $char ); - $this->assertEquals( - bin2hex( $norm ), - bin2hex( $clean ), - "Pair $x should be intact" ); - if( $norm != $clean ) return; - } elseif( $first > 0xfd || $second > 0xbf ) { - # fe and ff are not legal head bytes -- expect two replacement chars - $norm = $head . UTF8_REPLACEMENT . UTF8_REPLACEMENT . $tail; - $this->assertEquals( - bin2hex( $norm ), - bin2hex( $clean ), - "Forbidden pair $x should be rejected" ); - if( $norm != $clean ) return; - } else { - $norm = $head . UTF8_REPLACEMENT . $tail; - $this->assertEquals( - bin2hex( $norm ), - bin2hex( $clean ), - "Forbidden pair $x should be rejected" ); - if( $norm != $clean ) return; - } - } - } - } - - /** @todo document */ - function testTripleBytes() { - $this->doTestTripleBytes( '', '' ); - $this->doTestTripleBytes( 'x', '' ); - $this->doTestTripleBytes( '', 'x' ); - $this->doTestTripleBytes( 'x', 'x' ); - } - - /** @todo document */ - function doTestTripleBytes( $head, $tail ) { - for( $first = 0xc0; $first < 0x100; $first++ ) { - for( $second = 0x80; $second < 0x100; $second++ ) { - #for( $third = 0x80; $third < 0x100; $third++ ) { - for( $third = 0x80; $third < 0x81; $third++ ) { - $char = $head . chr( $first ) . chr( $second ) . chr( $third ) . $tail; - $clean = UtfNormal::cleanUp( $char ); - $x = sprintf( "%02X,%02X,%02X", $first, $second, $third ); - if( $first >= 0xe0 && - $first < 0xf0 && - $second < 0xc0 && - $third < 0xc0 ) { - if( $first == 0xe0 && $second < 0xa0 ) { - $this->assertEquals( - bin2hex( $head . UTF8_REPLACEMENT . $tail ), - bin2hex( $clean ), - "Overlong triplet $x should be rejected" ); - } elseif( $first == 0xed && - ( chr( $first ) . chr( $second ) . chr( $third )) >= UTF8_SURROGATE_FIRST ) { - $this->assertEquals( - bin2hex( $head . UTF8_REPLACEMENT . $tail ), - bin2hex( $clean ), - "Surrogate triplet $x should be rejected" ); - } else { - $this->assertEquals( - bin2hex( UtfNormal::NFC( $char ) ), - bin2hex( $clean ), - "Triplet $x should be intact" ); - } - } elseif( $first > 0xc1 && $first < 0xe0 && $second < 0xc0 ) { - $this->assertEquals( - bin2hex( UtfNormal::NFC( $head . chr( $first ) . chr( $second ) ) . UTF8_REPLACEMENT . $tail ), - bin2hex( $clean ), - "Valid 2-byte $x + broken tail" ); - } elseif( $second > 0xc1 && $second < 0xe0 && $third < 0xc0 ) { - $this->assertEquals( - bin2hex( $head . UTF8_REPLACEMENT . UtfNormal::NFC( chr( $second ) . chr( $third ) . $tail ) ), - bin2hex( $clean ), - "Broken head + valid 2-byte $x" ); - } elseif( ( $first > 0xfd || $second > 0xfd ) && - ( ( $second > 0xbf && $third > 0xbf ) || - ( $second < 0xc0 && $third < 0xc0 ) || - ( $second > 0xfd ) || - ( $third > 0xfd ) ) ) { - # fe and ff are not legal head bytes -- expect three replacement chars - $this->assertEquals( - bin2hex( $head . UTF8_REPLACEMENT . UTF8_REPLACEMENT . UTF8_REPLACEMENT . $tail ), - bin2hex( $clean ), - "Forbidden triplet $x should be rejected" ); - } elseif( $first > 0xc2 && $second < 0xc0 && $third < 0xc0 ) { - $this->assertEquals( - bin2hex( $head . UTF8_REPLACEMENT . $tail ), - bin2hex( $clean ), - "Forbidden triplet $x should be rejected" ); - } else { - $this->assertEquals( - bin2hex( $head . UTF8_REPLACEMENT . UTF8_REPLACEMENT . $tail ), - bin2hex( $clean ), - "Forbidden triplet $x should be rejected" ); - } - } - } - } - } - - /** @todo document */ - function testChunkRegression() { - # Check for regression against a chunking bug - $text = "\x46\x55\xb8" . - "\xdc\x96" . - "\xee" . - "\xe7" . - "\x44" . - "\xaa" . - "\x2f\x25"; - $expect = "\x46\x55\xef\xbf\xbd" . - "\xdc\x96" . - "\xef\xbf\xbd" . - "\xef\xbf\xbd" . - "\x44" . - "\xef\xbf\xbd" . - "\x2f\x25"; - - $this->assertEquals( - bin2hex( $expect ), - bin2hex( UtfNormal::cleanUp( $text ) ) ); - } - - /** @todo document */ - function testInterposeRegression() { - $text = "\x4e\x30" . - "\xb1" . # bad tail - "\x3a" . - "\x92" . # bad tail - "\x62\x3a" . - "\x84" . # bad tail - "\x43" . - "\xc6" . # bad head - "\x3f" . - "\x92" . # bad tail - "\xad" . # bad tail - "\x7d" . - "\xd9\x95"; - - $expect = "\x4e\x30" . - "\xef\xbf\xbd" . - "\x3a" . - "\xef\xbf\xbd" . - "\x62\x3a" . - "\xef\xbf\xbd" . - "\x43" . - "\xef\xbf\xbd" . - "\x3f" . - "\xef\xbf\xbd" . - "\xef\xbf\xbd" . - "\x7d" . - "\xd9\x95"; - - $this->assertEquals( - bin2hex( $expect ), - bin2hex( UtfNormal::cleanUp( $text ) ) ); - } - - /** @todo document */ - function testOverlongRegression() { - $text = "\x67" . - "\x1a" . # forbidden ascii - "\xea" . # bad head - "\xc1\xa6" . # overlong sequence - "\xad" . # bad tail - "\x1c" . # forbidden ascii - "\xb0" . # bad tail - "\x3c" . - "\x9e"; # bad tail - $expect = "\x67" . - "\xef\xbf\xbd" . - "\xef\xbf\xbd" . - "\xef\xbf\xbd" . - "\xef\xbf\xbd" . - "\xef\xbf\xbd" . - "\xef\xbf\xbd" . - "\x3c" . - "\xef\xbf\xbd"; - $this->assertEquals( - bin2hex( $expect ), - bin2hex( UtfNormal::cleanUp( $text ) ) ); - } - - /** @todo document */ - function testSurrogateRegression() { - $text = "\xed\xb4\x96" . # surrogate 0xDD16 - "\x83" . # bad tail - "\xb4" . # bad tail - "\xac"; # bad head - $expect = "\xef\xbf\xbd" . - "\xef\xbf\xbd" . - "\xef\xbf\xbd" . - "\xef\xbf\xbd"; - $this->assertEquals( - bin2hex( $expect ), - bin2hex( UtfNormal::cleanUp( $text ) ) ); - } - - /** @todo document */ - function testBomRegression() { - $text = "\xef\xbf\xbe" . # U+FFFE, illegal char - "\xb2" . # bad tail - "\xef" . # bad head - "\x59"; - $expect = "\xef\xbf\xbd" . - "\xef\xbf\xbd" . - "\xef\xbf\xbd" . - "\x59"; - $this->assertEquals( - bin2hex( $expect ), - bin2hex( UtfNormal::cleanUp( $text ) ) ); - } - - /** @todo document */ - function testForbiddenRegression() { - $text = "\xef\xbf\xbf"; # U+FFFF, illegal char - $expect = "\xef\xbf\xbd"; - $this->assertEquals( - bin2hex( $expect ), - bin2hex( UtfNormal::cleanUp( $text ) ) ); - } - - /** @todo document */ - function testHangulRegression() { - $text = "\xed\x9c\xaf" . # Hangul char - "\xe1\x87\x81"; # followed by another final jamo - $expect = $text; # Should *not* change. - $this->assertEquals( - bin2hex( $expect ), - bin2hex( UtfNormal::cleanUp( $text ) ) ); - } -} - - -$suite = new PHPUnit_Framework_TestSuite( 'CleanUpTest' ); -$result = PHPUnit_TextUI_TestRunner::run( $suite ); - -if( !$result->wasSuccessful() ) { - exit( -1 ); -} -exit( 0 ); diff --git a/tests/phpunit/includes/normal/CleanUpTest.php b/tests/phpunit/includes/normal/CleanUpTest.php new file mode 100644 index 0000000000..549a04062e --- /dev/null +++ b/tests/phpunit/includes/normal/CleanUpTest.php @@ -0,0 +1,425 @@ + + * http://www.mediawiki.org/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup UtfNormal + */ + + +if( php_sapi_name() != 'cli' ) { + die( "Run me from the command line please.\n" ); +} + +/** */ +if( isset( $_SERVER['argv'] ) && in_array( '--icu', $_SERVER['argv'] ) ) { + dl( 'php_utfnormal.so' ); +} + +#ini_set( 'memory_limit', '40M' ); + +require_once( 'PHPUnit/Runner/Version.php' ); +if( version_compare( PHPUnit_Runner_Version::id(), '3.5.0', '>=' ) ) { + # PHPUnit 3.5.0 introduced a nice autoloader based on class name + require_once( 'PHPUnit/Autoload.php' ); +} else { + # Keep the old pre PHPUnit 3.5.0 behaviour for compatibility + require_once 'PHPUnit/Framework.php'; +} +require_once 'PHPUnit/TextUI/TestRunner.php'; + +require_once 'UtfNormal.php'; + +/** + * Additional tests for UtfNormal::cleanUp() function, inclusion + * regression checks for known problems. + * Requires PHPUnit. + * + * @ingroup UtfNormal + * @private + */ +class CleanUpTest extends PHPUnit_Framework_TestCase { + /** @todo document */ + function setUp() { + } + + /** @todo document */ + function tearDown() { + } + + /** @todo document */ + function testAscii() { + $text = 'This is plain ASCII text.'; + $this->assertEquals( $text, UtfNormal::cleanUp( $text ) ); + } + + /** @todo document */ + function testNull() { + $text = "a \x00 null"; + $expect = "a \xef\xbf\xbd null"; + $this->assertEquals( + bin2hex( $expect ), + bin2hex( UtfNormal::cleanUp( $text ) ) ); + } + + /** @todo document */ + function testLatin() { + $text = "L'\xc3\xa9cole"; + $this->assertEquals( $text, UtfNormal::cleanUp( $text ) ); + } + + /** @todo document */ + function testLatinNormal() { + $text = "L'e\xcc\x81cole"; + $expect = "L'\xc3\xa9cole"; + $this->assertEquals( $expect, UtfNormal::cleanUp( $text ) ); + } + + /** + * This test is *very* expensive! + * @todo document + */ + function XtestAllChars() { + $rep = UTF8_REPLACEMENT; + for( $i = 0x0; $i < UNICODE_MAX; $i++ ) { + $char = codepointToUtf8( $i ); + $clean = UtfNormal::cleanUp( $char ); + $x = sprintf( "%04X", $i ); + if( $i % 0x1000 == 0 ) echo "U+$x\n"; + if( $i == 0x0009 || + $i == 0x000a || + $i == 0x000d || + ($i > 0x001f && $i < UNICODE_SURROGATE_FIRST) || + ($i > UNICODE_SURROGATE_LAST && $i < 0xfffe ) || + ($i > 0xffff && $i <= UNICODE_MAX ) ) { + if( isset( UtfNormal::$utfCanonicalComp[$char] ) || isset( UtfNormal::$utfCanonicalDecomp[$char] ) ) { + $comp = UtfNormal::NFC( $char ); + $this->assertEquals( + bin2hex( $comp ), + bin2hex( $clean ), + "U+$x should be decomposed" ); + } else { + $this->assertEquals( + bin2hex( $char ), + bin2hex( $clean ), + "U+$x should be intact" ); + } + } else { + $this->assertEquals( bin2hex( $rep ), bin2hex( $clean ), $x ); + } + } + } + + /** @todo document */ + function testAllBytes() { + $this->doTestBytes( '', '' ); + $this->doTestBytes( 'x', '' ); + $this->doTestBytes( '', 'x' ); + $this->doTestBytes( 'x', 'x' ); + } + + /** @todo document */ + function doTestBytes( $head, $tail ) { + for( $i = 0x0; $i < 256; $i++ ) { + $char = $head . chr( $i ) . $tail; + $clean = UtfNormal::cleanUp( $char ); + $x = sprintf( "%02X", $i ); + if( $i == 0x0009 || + $i == 0x000a || + $i == 0x000d || + ($i > 0x001f && $i < 0x80) ) { + $this->assertEquals( + bin2hex( $char ), + bin2hex( $clean ), + "ASCII byte $x should be intact" ); + if( $char != $clean ) return; + } else { + $norm = $head . UTF8_REPLACEMENT . $tail; + $this->assertEquals( + bin2hex( $norm ), + bin2hex( $clean ), + "Forbidden byte $x should be rejected" ); + if( $norm != $clean ) return; + } + } + } + + /** @todo document */ + function testDoubleBytes() { + $this->doTestDoubleBytes( '', '' ); + $this->doTestDoubleBytes( 'x', '' ); + $this->doTestDoubleBytes( '', 'x' ); + $this->doTestDoubleBytes( 'x', 'x' ); + } + + /** + * @todo document + */ + function doTestDoubleBytes( $head, $tail ) { + for( $first = 0xc0; $first < 0x100; $first++ ) { + for( $second = 0x80; $second < 0x100; $second++ ) { + $char = $head . chr( $first ) . chr( $second ) . $tail; + $clean = UtfNormal::cleanUp( $char ); + $x = sprintf( "%02X,%02X", $first, $second ); + if( $first > 0xc1 && + $first < 0xe0 && + $second < 0xc0 ) { + $norm = UtfNormal::NFC( $char ); + $this->assertEquals( + bin2hex( $norm ), + bin2hex( $clean ), + "Pair $x should be intact" ); + if( $norm != $clean ) return; + } elseif( $first > 0xfd || $second > 0xbf ) { + # fe and ff are not legal head bytes -- expect two replacement chars + $norm = $head . UTF8_REPLACEMENT . UTF8_REPLACEMENT . $tail; + $this->assertEquals( + bin2hex( $norm ), + bin2hex( $clean ), + "Forbidden pair $x should be rejected" ); + if( $norm != $clean ) return; + } else { + $norm = $head . UTF8_REPLACEMENT . $tail; + $this->assertEquals( + bin2hex( $norm ), + bin2hex( $clean ), + "Forbidden pair $x should be rejected" ); + if( $norm != $clean ) return; + } + } + } + } + + /** @todo document */ + function testTripleBytes() { + $this->doTestTripleBytes( '', '' ); + $this->doTestTripleBytes( 'x', '' ); + $this->doTestTripleBytes( '', 'x' ); + $this->doTestTripleBytes( 'x', 'x' ); + } + + /** @todo document */ + function doTestTripleBytes( $head, $tail ) { + for( $first = 0xc0; $first < 0x100; $first++ ) { + for( $second = 0x80; $second < 0x100; $second++ ) { + #for( $third = 0x80; $third < 0x100; $third++ ) { + for( $third = 0x80; $third < 0x81; $third++ ) { + $char = $head . chr( $first ) . chr( $second ) . chr( $third ) . $tail; + $clean = UtfNormal::cleanUp( $char ); + $x = sprintf( "%02X,%02X,%02X", $first, $second, $third ); + if( $first >= 0xe0 && + $first < 0xf0 && + $second < 0xc0 && + $third < 0xc0 ) { + if( $first == 0xe0 && $second < 0xa0 ) { + $this->assertEquals( + bin2hex( $head . UTF8_REPLACEMENT . $tail ), + bin2hex( $clean ), + "Overlong triplet $x should be rejected" ); + } elseif( $first == 0xed && + ( chr( $first ) . chr( $second ) . chr( $third )) >= UTF8_SURROGATE_FIRST ) { + $this->assertEquals( + bin2hex( $head . UTF8_REPLACEMENT . $tail ), + bin2hex( $clean ), + "Surrogate triplet $x should be rejected" ); + } else { + $this->assertEquals( + bin2hex( UtfNormal::NFC( $char ) ), + bin2hex( $clean ), + "Triplet $x should be intact" ); + } + } elseif( $first > 0xc1 && $first < 0xe0 && $second < 0xc0 ) { + $this->assertEquals( + bin2hex( UtfNormal::NFC( $head . chr( $first ) . chr( $second ) ) . UTF8_REPLACEMENT . $tail ), + bin2hex( $clean ), + "Valid 2-byte $x + broken tail" ); + } elseif( $second > 0xc1 && $second < 0xe0 && $third < 0xc0 ) { + $this->assertEquals( + bin2hex( $head . UTF8_REPLACEMENT . UtfNormal::NFC( chr( $second ) . chr( $third ) . $tail ) ), + bin2hex( $clean ), + "Broken head + valid 2-byte $x" ); + } elseif( ( $first > 0xfd || $second > 0xfd ) && + ( ( $second > 0xbf && $third > 0xbf ) || + ( $second < 0xc0 && $third < 0xc0 ) || + ( $second > 0xfd ) || + ( $third > 0xfd ) ) ) { + # fe and ff are not legal head bytes -- expect three replacement chars + $this->assertEquals( + bin2hex( $head . UTF8_REPLACEMENT . UTF8_REPLACEMENT . UTF8_REPLACEMENT . $tail ), + bin2hex( $clean ), + "Forbidden triplet $x should be rejected" ); + } elseif( $first > 0xc2 && $second < 0xc0 && $third < 0xc0 ) { + $this->assertEquals( + bin2hex( $head . UTF8_REPLACEMENT . $tail ), + bin2hex( $clean ), + "Forbidden triplet $x should be rejected" ); + } else { + $this->assertEquals( + bin2hex( $head . UTF8_REPLACEMENT . UTF8_REPLACEMENT . $tail ), + bin2hex( $clean ), + "Forbidden triplet $x should be rejected" ); + } + } + } + } + } + + /** @todo document */ + function testChunkRegression() { + # Check for regression against a chunking bug + $text = "\x46\x55\xb8" . + "\xdc\x96" . + "\xee" . + "\xe7" . + "\x44" . + "\xaa" . + "\x2f\x25"; + $expect = "\x46\x55\xef\xbf\xbd" . + "\xdc\x96" . + "\xef\xbf\xbd" . + "\xef\xbf\xbd" . + "\x44" . + "\xef\xbf\xbd" . + "\x2f\x25"; + + $this->assertEquals( + bin2hex( $expect ), + bin2hex( UtfNormal::cleanUp( $text ) ) ); + } + + /** @todo document */ + function testInterposeRegression() { + $text = "\x4e\x30" . + "\xb1" . # bad tail + "\x3a" . + "\x92" . # bad tail + "\x62\x3a" . + "\x84" . # bad tail + "\x43" . + "\xc6" . # bad head + "\x3f" . + "\x92" . # bad tail + "\xad" . # bad tail + "\x7d" . + "\xd9\x95"; + + $expect = "\x4e\x30" . + "\xef\xbf\xbd" . + "\x3a" . + "\xef\xbf\xbd" . + "\x62\x3a" . + "\xef\xbf\xbd" . + "\x43" . + "\xef\xbf\xbd" . + "\x3f" . + "\xef\xbf\xbd" . + "\xef\xbf\xbd" . + "\x7d" . + "\xd9\x95"; + + $this->assertEquals( + bin2hex( $expect ), + bin2hex( UtfNormal::cleanUp( $text ) ) ); + } + + /** @todo document */ + function testOverlongRegression() { + $text = "\x67" . + "\x1a" . # forbidden ascii + "\xea" . # bad head + "\xc1\xa6" . # overlong sequence + "\xad" . # bad tail + "\x1c" . # forbidden ascii + "\xb0" . # bad tail + "\x3c" . + "\x9e"; # bad tail + $expect = "\x67" . + "\xef\xbf\xbd" . + "\xef\xbf\xbd" . + "\xef\xbf\xbd" . + "\xef\xbf\xbd" . + "\xef\xbf\xbd" . + "\xef\xbf\xbd" . + "\x3c" . + "\xef\xbf\xbd"; + $this->assertEquals( + bin2hex( $expect ), + bin2hex( UtfNormal::cleanUp( $text ) ) ); + } + + /** @todo document */ + function testSurrogateRegression() { + $text = "\xed\xb4\x96" . # surrogate 0xDD16 + "\x83" . # bad tail + "\xb4" . # bad tail + "\xac"; # bad head + $expect = "\xef\xbf\xbd" . + "\xef\xbf\xbd" . + "\xef\xbf\xbd" . + "\xef\xbf\xbd"; + $this->assertEquals( + bin2hex( $expect ), + bin2hex( UtfNormal::cleanUp( $text ) ) ); + } + + /** @todo document */ + function testBomRegression() { + $text = "\xef\xbf\xbe" . # U+FFFE, illegal char + "\xb2" . # bad tail + "\xef" . # bad head + "\x59"; + $expect = "\xef\xbf\xbd" . + "\xef\xbf\xbd" . + "\xef\xbf\xbd" . + "\x59"; + $this->assertEquals( + bin2hex( $expect ), + bin2hex( UtfNormal::cleanUp( $text ) ) ); + } + + /** @todo document */ + function testForbiddenRegression() { + $text = "\xef\xbf\xbf"; # U+FFFF, illegal char + $expect = "\xef\xbf\xbd"; + $this->assertEquals( + bin2hex( $expect ), + bin2hex( UtfNormal::cleanUp( $text ) ) ); + } + + /** @todo document */ + function testHangulRegression() { + $text = "\xed\x9c\xaf" . # Hangul char + "\xe1\x87\x81"; # followed by another final jamo + $expect = $text; # Should *not* change. + $this->assertEquals( + bin2hex( $expect ), + bin2hex( UtfNormal::cleanUp( $text ) ) ); + } +} + + +$suite = new PHPUnit_Framework_TestSuite( 'CleanUpTest' ); +$result = PHPUnit_TextUI_TestRunner::run( $suite ); + +if( !$result->wasSuccessful() ) { + exit( -1 ); +} +exit( 0 );