includes/normal/UtfNormalGenerate.php

   1 <?php
   2 # Copyright (C) 2004 Brion Vibber <brion@pobox.com>
   3 # http://www.mediawiki.org/
   4 #
   5 # This program is free software; you can redistribute it and/or modify
   6 # it under the terms of the GNU General Public License as published by
   7 # the Free Software Foundation; either version 2 of the License, or
   8 # (at your option) any later version.
   9 #
  10 # This program is distributed in the hope that it will be useful,
  11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13 # GNU General Public License for more details.
  14 #
  15 # You should have received a copy of the GNU General Public License along
  16 # with this program; if not, write to the Free Software Foundation, Inc.,
  17 # 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  18 # http://www.gnu.org/copyleft/gpl.html
  19
  20 /**
  21  * This script generates UniNormalData.inc from the Unicode Character Database
  22  * and supplementary files.
  23  *
  24  * @package MediaWiki
  25  */
  26
  27 /** */
  28 require_once 'UtfNormalUtil.php';
  29
  30 $in = fopen("DerivedNormalizationProps.txt", "rt" );
  31 if( !$in ) {
  32         print "Can't open DerivedNormalizationProps.txt for reading.\n";
  33         print "If necessary, fetch this file from the internet:\n";
  34         print "http://www.unicode.org/Public/UNIDATA/CompositionExclusions.txt\n";
  35         exit(-1);
  36 }
  37 print "Initializing normalization quick check tables...\n";
  38 $checkNFC = array();
  39 while( false !== ($line = fgets( $in ) ) ) {
  40         if( preg_match( '/^([0-9A-F]+)(?:..([0-9A-F]+))?\s*;\s*(NFC_QC)\s*;\s*([MN])/', $line, $matches ) ) {
  41                 list( $junk, $first, $last, $prop, $value ) = $matches;
  42                 #print "$first $last $prop $value\n";
  43                 if( !$last ) $last = $first;
  44                 for( $i = hexdec( $first ); $i <= hexdec( $last ); $i++) {
  45                         $char = codepointToUtf8( $i );
  46                         $checkNFC[$char] = $value;
  47                 }
  48         }
  49 }
  50 fclose( $in );
  51
  52 $in = fopen("CompositionExclusions.txt", "rt" );
  53 if( !$in ) {
  54         print "Can't open CompositionExclusions.txt for reading.\n";
  55         print "If necessary, fetch this file from the internet:\n";
  56         print "http://www.unicode.org/Public/UNIDATA/CompositionExclusions.txt\n";
  57         exit(-1);
  58 }
  59 $exclude = array();
  60 while( false !== ($line = fgets( $in ) ) ) {
  61         if( preg_match( '/^([0-9A-F]+)/i', $line, $matches ) ) {
  62                 $codepoint = $matches[1];
  63                 $source = codepointToUtf8( hexdec( $codepoint ) );
  64                 $exclude[$source] = true;
  65         }
  66 }
  67 fclose($in);
  68
  69 $in = fopen("UnicodeData.txt", "rt" );
  70 if( !$in ) {
  71         print "Can't open UnicodeData.txt for reading.\n";
  72         print "If necessary, fetch this file from the internet:\n";
  73         print "http://www.unicode.org/Public/UNIDATA/UnicodeData.txt\n";
  74         exit(-1);
  75 }
  76
  77 $compatibilityDecomp = array();
  78 $canonicalDecomp = array();
  79 $canonicalComp = array();
  80 $combiningClass = array();
  81 $total = 0;
  82 $compat = 0;
  83 $canon = 0;
  84
  85 print "Reading character definitions...\n";
  86 while( false !== ($line = fgets( $in ) ) ) {
  87         $columns = split(';', $line);
  88         $codepoint = $columns[0];
  89         $name = $columns[1];
  90         $canonicalCombiningClass = $columns[3];
  91         $decompositionMapping = $columns[5];
  92
  93         $source = codepointToUtf8( hexdec( $codepoint ) );
  94
  95         if( $canonicalCombiningClass != 0 ) {
  96                 $combiningClass[$source] = IntVal( $canonicalCombiningClass );
  97         }
  98
  99         if( $decompositionMapping === '' ) continue;
 100         if( preg_match( '/^<(.+)> (.*)$/', $decompositionMapping, $matches ) ) {
 101                 # Compatibility decomposition
 102                 $canonical = false;
 103                 $decompositionMapping = $matches[2];
 104                 $compat++;
 105         } else {
 106                 $canonical = true;
 107                 $canon++;
 108         }
 109         $total++;
 110         $dest = hexSequenceToUtf8( $decompositionMapping );
 111
 112         $compatibilityDecomp[$source] = $dest;
 113         if( $canonical ) {
 114                 $canonicalDecomp[$source] = $dest;
 115                 if( empty( $exclude[$source] ) ) {
 116                         $canonicalComp[$dest] = $source;
 117                 }
 118         }
 119         #print "$codepoint | $canonicalCombiningClasses | $decompositionMapping\n";
 120 }
 121 fclose( $in );
 122
 123 print "Recursively expanding canonical mappings...\n";
 124 $changed = 42;
 125 $pass = 1;
 126 while( $changed > 0 ) {
 127         print "pass $pass\n";
 128         $changed = 0;
 129         foreach( $canonicalDecomp as $source => $dest ) {
 130                 $newDest = preg_replace_callback(
 131                         '/([\xc0-\xff][\x80-\xbf]+)/',
 132                         'callbackCanonical',
 133                         $dest);
 134                 if( $newDest === $dest ) continue;
 135                 $changed++;
 136                 $canonicalDecomp[$source] = $newDest;
 137         }
 138         $pass++;
 139 }
 140
 141 print "Recursively expanding compatibility mappings...\n";
 142 $changed = 42;
 143 $pass = 1;
 144 while( $changed > 0 ) {
 145         print "pass $pass\n";
 146         $changed = 0;
 147         foreach( $compatibilityDecomp as $source => $dest ) {
 148                 $newDest = preg_replace_callback(
 149                         '/([\xc0-\xff][\x80-\xbf]+)/',
 150                         'callbackCompat',
 151                         $dest);
 152                 if( $newDest === $dest ) continue;
 153                 $changed++;
 154                 $compatibilityDecomp[$source] = $newDest;
 155         }
 156         $pass++;
 157 }
 158
 159 print "$total decomposition mappings ($canon canonical, $compat compatibility)\n";
 160
 161 $out = fopen("UtfNormalData.inc", "wt");
 162 if( $out ) {
 163         $serCombining = escapeSingleString( serialize( $combiningClass ) );
 164         $serComp = escapeSingleString( serialize( $canonicalComp ) );
 165         $serCanon = escapeSingleString( serialize( $canonicalDecomp ) );
 166         $serCheckNFC = escapeSingleString( serialize( $checkNFC ) );
 167         $outdata = "<" . "?php
 168 /**
 169  * This file was automatically generated -- do not edit!
 170  * Run UtfNormalGenerate.php to create this file again (make clean && make)
 171  * @package MediaWiki
 172  */
 173 /** */
 174 global \$utfCombiningClass, \$utfCanonicalComp, \$utfCanonicalDecomp;
 175 \$utfCombiningClass = unserialize( '$serCombining' );
 176 \$utfCanonicalComp = unserialize( '$serComp' );
 177 \$utfCanonicalDecomp = unserialize( '$serCanon' );
 178 \$utfCheckNFC = unserialize( '$serCheckNFC' );
 179 ?" . ">\n";
 180         fputs( $out, $outdata );
 181         fclose( $out );
 182         print "Wrote out UtfNormalData.inc\n";
 183 } else {
 184         print "Can't create file UtfNormalData.inc\n";
 185         exit(-1);
 186 }
 187
 188
 189 $out = fopen("UtfNormalDataK.inc", "wt");
 190 if( $out ) {
 191         $serCompat = escapeSingleString( serialize( $compatibilityDecomp ) );
 192         $outdata = "<" . "?php
 193 /**
 194  * This file was automatically generated -- do not edit!
 195  * Run UtfNormalGenerate.php to create this file again (make clean && make)
 196  * @package MediaWiki
 197  */
 198 /** */
 199 global \$utfCompatibilityDecomp;
 200 \$utfCompatibilityDecomp = unserialize( '$serCompat' );
 201 ?" . ">\n";
 202         fputs( $out, $outdata );
 203         fclose( $out );
 204         print "Wrote out UtfNormalDataK.inc\n";
 205         exit(0);
 206 } else {
 207         print "Can't create file UtfNormalDataK.inc\n";
 208         exit(-1);
 209 }
 210
 211 # ---------------
 212
 213 function callbackCanonical( $matches ) {
 214         global $canonicalDecomp;
 215         if( isset( $canonicalDecomp[$matches[1]] ) ) {
 216                 return $canonicalDecomp[$matches[1]];
 217         }
 218         return $matches[1];
 219 }
 220
 221 function callbackCompat( $matches ) {
 222         global $compatibilityDecomp;
 223         if( isset( $compatibilityDecomp[$matches[1]] ) ) {
 224                 return $compatibilityDecomp[$matches[1]];
 225         }
 226         return $matches[1];
 227 }
 228
 229 ?>