includes/normal/UtfNormalTest.php

   1 <?php
   2 # Copyright (C) 2004 Brion Vibber <brion@pobox.com>
   3 # http://www.mediawiki.org/
   4 #
   5 # This program is free software; you can redistribute it and/or modify
   6 # it under the terms of the GNU General Public License as published by
   7 # the Free Software Foundation; either version 2 of the License, or
   8 # (at your option) any later version.
   9 #
  10 # This program is distributed in the hope that it will be useful,
  11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13 # GNU General Public License for more details.
  14 #
  15 # You should have received a copy of the GNU General Public License along
  16 # with this program; if not, write to the Free Software Foundation, Inc.,
  17 # 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  18 # http://www.gnu.org/copyleft/gpl.html
  19
  20 /**
  21  * Implements the conformance test at:
  22  * http://www.unicode.org/Public/UNIDATA/NormalizationTest.txt
  23  * @package MediaWiki
  24  */
  25
  26 /** */
  27 $verbose = true;
  28 #define( 'PRETTY_UTF8', true );
  29
  30 if( defined( 'PRETTY_UTF8' ) ) {
  31         function pretty( $string ) {
  32                 return preg_replace( '/([\x00-\xff])/e',
  33                         'sprintf("%02X", ord("$1"))',
  34                         $string );
  35         }
  36 } else {
  37         /**
  38          * @ignore
  39          */
  40         function pretty( $string ) {
  41                 return trim( preg_replace( '/(.)/use',
  42                         'sprintf("%04X ", utf8ToCodepoint("$1"))',
  43                         $string ) );
  44         }
  45 }
  46
  47 require_once 'UtfNormalUtil.php';
  48 require_once 'UtfNormal.php';
  49
  50 if( php_sapi_name() != 'cli' ) {
  51         die( "Run me from the command line please.\n" );
  52 }
  53
  54 $in = fopen("NormalizationTest.txt", "rt");
  55 if( !$in ) {
  56         print "Couldn't open NormalizationTest.txt -- can't run tests.\n";
  57         print "If necessary, manually download this file. It can be obtained at\n";
  58         print "http://www.unicode.org/Public/UNIDATA/NormalizationTest.txt";
  59         exit(-1);
  60 }
  61
  62 $normalizer = new UtfNormal;
  63
  64 $total = 0;
  65 $success = 0;
  66 $failure = 0;
  67 $ok = true;
  68 $testedChars = array();
  69 while( false !== ( $line = fgets( $in ) ) ) {
  70         list( $data, $comment ) = explode( '#', $line );
  71         if( $data === '' ) continue;
  72         if( preg_match( '/@Part([\d])/', $data, $matches ) ) {
  73                 if( $matches[1] > 0 ) {
  74                         $ok = reportResults( $total, $success, $failure ) && $ok;
  75                 }
  76                 print "Part {$matches[1]}: $comment";
  77                 continue;
  78         }
  79
  80         $columns = array_map( "hexSequenceToUtf8", explode( ";", $data ) );
  81         array_unshift( $columns, '' );
  82
  83         $testedChars[$columns[1]] = true;
  84         $total++;
  85         if( testNormals( $normalizer, $columns, $comment ) ) {
  86                 $success++;
  87         } else {
  88                 $failure++;
  89                 # print "FAILED: $comment";
  90         }
  91         if( $total % 100 == 0 ) print "$total ";
  92 }
  93 fclose( $in );
  94
  95 $ok = reportResults( $total, $success, $failure ) && $ok;
  96
  97 $in = fopen("UnicodeData.txt", "rt" );
  98 if( !$in ) {
  99         print "Can't open UnicodeData.txt for reading.\n";
 100         print "If necessary, fetch this file from the internet:\n";
 101         print "http://www.unicode.org/Public/UNIDATA/UnicodeData.txt\n";
 102         exit(-1);
 103 }
 104 print "Now testing invariants...\n";
 105 while( false !== ($line = fgets( $in ) ) ) {
 106         $cols = explode( ';', $line );
 107         $char = codepointToUtf8( hexdec( $cols[0] ) );
 108         $desc = $cols[0] . ": " . $cols[1];
 109         if( $char >= UTF8_SURROGATE_FIRST && $char <= UTF8_SURROGATE_LAST ) {
 110                 # Surrogates are illegal on their own or in UTF-8, ignore.
 111                 continue;
 112         }
 113         if( empty( $testedChars[$char] ) ) {
 114                 $total++;
 115                 if( testInvariant( $normalizer, $char, $desc ) ) {
 116                         $success++;
 117                 } else {
 118                         $failure++;
 119                 }
 120                 if( $total % 100 == 0 ) print "$total ";
 121         }
 122 }
 123 fclose( $in );
 124
 125 $ok = reportResults( $total, $success, $failure ) && $ok;
 126
 127 if( $ok ) {
 128         print "TEST SUCCEEDED!\n";
 129         exit(0);
 130 } else {
 131         print "TEST FAILED!\n";
 132         exit(-1);
 133 }
 134
 135 ## ------
 136
 137 function reportResults( &$total, &$success, &$failure ) {
 138         $percSucc = IntVal( $success * 100 / $total );
 139         $percFail = IntVal( $failure * 100 / $total );
 140         print "\n";
 141         print "$success tests successful ($percSucc%)\n";
 142         print "$failure tests failed ($percFail%)\n\n";
 143         $ok = ($success > 0 && $failure == 0);
 144         $total = 0;
 145         $success = 0;
 146         $failure = 0;
 147         return $ok;
 148 }
 149
 150 function testNormals( &$u, $c, $comment, $reportFailure = false ) {
 151         $result = testNFC( $u, $c, $comment, $reportFailure );
 152         $result = testNFD( $u, $c, $comment, $reportFailure ) && $result;
 153         $result = testNFKC( $u, $c, $comment, $reportFailure ) && $result;
 154         $result = testNFKD( $u, $c, $comment, $reportFailure ) && $result;
 155
 156         global $verbose;
 157         if( $verbose && !$result && !$reportFailure ) {
 158                 print $comment;
 159                 testNormals( $u, $c, $comment, true );
 160         }
 161         return $result;
 162 }
 163
 164 function verbosify( $a, $b, $col, $form, $verbose ) {
 165         #$result = ($a === $b);
 166         $result = (strcmp( $a, $b ) == 0);
 167         if( $verbose ) {
 168                 $aa = pretty( $a );
 169                 $bb = pretty( $b );
 170                 $ok = $result ? "succeed" : " failed";
 171                 $eq = $result ? "==" : "!=";
 172                 print "  $ok $form c$col '$aa' $eq '$bb'\n";
 173         }
 174         return $result;
 175 }
 176
 177 function testNFC( &$u, $c, $comment, $verbose ) {
 178         $result = verbosify( $c[2], $u->toNFC( $c[1] ), 1, 'NFC', $verbose );
 179         $result = verbosify( $c[2], $u->toNFC( $c[2] ), 2, 'NFC', $verbose ) && $result;
 180         $result = verbosify( $c[2], $u->toNFC( $c[3] ), 3, 'NFC', $verbose ) && $result;
 181         $result = verbosify( $c[4], $u->toNFC( $c[4] ), 4, 'NFC', $verbose ) && $result;
 182         $result = verbosify( $c[4], $u->toNFC( $c[5] ), 5, 'NFC', $verbose ) && $result;
 183         return $result;
 184 }
 185
 186 function testNFD( &$u, $c, $comment, $verbose ) {
 187         $result = verbosify( $c[3], $u->toNFD( $c[1] ), 1, 'NFD', $verbose );
 188         $result = verbosify( $c[3], $u->toNFD( $c[2] ), 2, 'NFD', $verbose ) && $result;
 189         $result = verbosify( $c[3], $u->toNFD( $c[3] ), 3, 'NFD', $verbose ) && $result;
 190         $result = verbosify( $c[5], $u->toNFD( $c[4] ), 4, 'NFD', $verbose ) && $result;
 191         $result = verbosify( $c[5], $u->toNFD( $c[5] ), 5, 'NFD', $verbose ) && $result;
 192         return $result;
 193 }
 194
 195 function testNFKC( &$u, $c, $comment, $verbose ) {
 196         $result = verbosify( $c[4], $u->toNFKC( $c[1] ), 1, 'NFKC', $verbose );
 197         $result = verbosify( $c[4], $u->toNFKC( $c[2] ), 2, 'NFKC', $verbose ) && $result;
 198         $result = verbosify( $c[4], $u->toNFKC( $c[3] ), 3, 'NFKC', $verbose ) && $result;
 199         $result = verbosify( $c[4], $u->toNFKC( $c[4] ), 4, 'NFKC', $verbose ) && $result;
 200         $result = verbosify( $c[4], $u->toNFKC( $c[5] ), 5, 'NFKC', $verbose ) && $result;
 201         return $result;
 202 }
 203
 204 function testNFKD( &$u, $c, $comment, $verbose ) {
 205         $result = verbosify( $c[5], $u->toNFKD( $c[1] ), 1, 'NFKD', $verbose );
 206         $result = verbosify( $c[5], $u->toNFKD( $c[2] ), 2, 'NFKD', $verbose ) && $result;
 207         $result = verbosify( $c[5], $u->toNFKD( $c[3] ), 3, 'NFKD', $verbose ) && $result;
 208         $result = verbosify( $c[5], $u->toNFKD( $c[4] ), 4, 'NFKD', $verbose ) && $result;
 209         $result = verbosify( $c[5], $u->toNFKD( $c[5] ), 5, 'NFKD', $verbose ) && $result;
 210         return $result;
 211 }
 212
 213 function testInvariant( &$u, $char, $desc, $reportFailure = false ) {
 214         $result = verbosify( $char, $u->toNFC( $char ), 1, 'NFC', $reportFailure );
 215         $result = verbosify( $char, $u->toNFD( $char ), 1, 'NFD', $reportFailure ) && $result;
 216         $result = verbosify( $char, $u->toNFKC( $char ), 1, 'NFKC', $reportFailure ) && $result;
 217         $result = verbosify( $char, $u->toNFKD( $char ), 1, 'NFKD', $reportFailure ) && $result;
 218         global $verbose;
 219         if( $verbose && !$result && !$reportFailure ) {
 220                 print $desc;
 221                 testInvariant( $u, $char, $desc, true );
 222         }
 223         return $result;
 224 }
 225
 226 ?>