includes/normal/UtfNormalGenerate.php

   1 <?php
   2 # Copyright (C) 2004 Brion Vibber <brion@pobox.com>
   3 # http://www.mediawiki.org/
   4 #
   5 # This program is free software; you can redistribute it and/or modify
   6 # it under the terms of the GNU General Public License as published by
   7 # the Free Software Foundation; either version 2 of the License, or
   8 # (at your option) any later version.
   9 #
  10 # This program is distributed in the hope that it will be useful,
  11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13 # GNU General Public License for more details.
  14 #
  15 # You should have received a copy of the GNU General Public License along
  16 # with this program; if not, write to the Free Software Foundation, Inc.,
  17 # 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  18 # http://www.gnu.org/copyleft/gpl.html
  19
  20 # This script generates UniNormalData.inc from the Unicode Character Database
  21 # and supplementary files.
  22
  23 require_once 'UtfNormalUtil.php';
  24
  25 $in = fopen("DerivedNormalizationProps.txt", "rt" );
  26 if( !$in ) {
  27         print "Can't open DerivedNormalizationProps.txt for reading.\n";
  28         print "If necessary, fetch this file from the internet:\n";
  29         print "http://www.unicode.org/Public/UNIDATA/CompositionExclusions.txt\n";
  30         exit(-1);
  31 }
  32 print "Initializing normalization quick check tables...\n";
  33 $checkNFC = array();
  34 while( false !== ($line = fgets( $in ) ) ) {
  35         if( preg_match( '/^([0-9A-F]+)(?:..([0-9A-F]+))?\s*;\s*(NFC_QC)\s*;\s*([MN])/', $line, $matches ) ) {
  36                 list( $junk, $first, $last, $prop, $value ) = $matches;
  37                 #print "$first $last $prop $value\n";
  38                 if( !$last ) $last = $first;
  39                 for( $i = hexdec( $first ); $i <= hexdec( $last ); $i++) {
  40                         $char = codepointToUtf8( $i );
  41                         $checkNFC[$char] = $value;
  42                 }
  43         }
  44 }
  45 fclose( $in );
  46
  47 $in = fopen("CompositionExclusions.txt", "rt" );
  48 if( !$in ) {
  49         print "Can't open CompositionExclusions.txt for reading.\n";
  50         print "If necessary, fetch this file from the internet:\n";
  51         print "http://www.unicode.org/Public/UNIDATA/CompositionExclusions.txt\n";
  52         exit(-1);
  53 }
  54 $exclude = array();
  55 while( false !== ($line = fgets( $in ) ) ) {
  56         if( preg_match( '/^([0-9A-F]+)/i', $line, $matches ) ) {
  57                 $codepoint = $matches[1];
  58                 $source = codepointToUtf8( hexdec( $codepoint ) );
  59                 $exclude[$source] = true;
  60         }
  61 }
  62 fclose($in);
  63
  64 $in = fopen("UnicodeData.txt", "rt" );
  65 if( !$in ) {
  66         print "Can't open UnicodeData.txt for reading.\n";
  67         print "If necessary, fetch this file from the internet:\n";
  68         print "http://www.unicode.org/Public/UNIDATA/UnicodeData.txt\n";
  69         exit(-1);
  70 }
  71
  72 $compatibilityDecomp = array();
  73 $canonicalDecomp = array();
  74 $canonicalComp = array();
  75 $combiningClass = array();
  76 $total = 0;
  77 $compat = 0;
  78 $canon = 0;
  79
  80 print "Reading character definitions...\n";
  81 while( false !== ($line = fgets( $in ) ) ) {
  82         $columns = split(';', $line);
  83         $codepoint = $columns[0];
  84         $name = $columns[1];
  85         $canonicalCombiningClass = $columns[3];
  86         $decompositionMapping = $columns[5];
  87
  88         $source = codepointToUtf8( hexdec( $codepoint ) );
  89
  90         if( $canonicalCombiningClass != 0 ) {
  91                 $combiningClass[$source] = IntVal( $canonicalCombiningClass );
  92         }
  93
  94         if( $decompositionMapping === '' ) continue;
  95         if( preg_match( '/^<(.+)> (.*)$/', $decompositionMapping, $matches ) ) {
  96                 # Compatibility decomposition
  97                 $canonical = false;
  98                 $decompositionMapping = $matches[2];
  99                 $compat++;
 100         } else {
 101                 $canonical = true;
 102                 $canon++;
 103         }
 104         $total++;
 105         $dest = hexSequenceToUtf8( $decompositionMapping );
 106
 107         $compatibilityDecomp[$source] = $dest;
 108         if( $canonical ) {
 109                 $canonicalDecomp[$source] = $dest;
 110                 if( empty( $exclude[$source] ) ) {
 111                         $canonicalComp[$dest] = $source;
 112                 }
 113         }
 114         #print "$codepoint | $canonicalCombiningClasses | $decompositionMapping\n";
 115 }
 116 fclose( $in );
 117
 118 print "Recursively expanding canonical mappings...\n";
 119 $changed = 42;
 120 $pass = 1;
 121 while( $changed > 0 ) {
 122         print "pass $pass\n";
 123         $changed = 0;
 124         foreach( $canonicalDecomp as $source => $dest ) {
 125                 $newDest = preg_replace_callback(
 126                         '/([\xc0-\xff][\x80-\xbf]+)/',
 127                         'callbackCanonical',
 128                         $dest);
 129                 if( $newDest === $dest ) continue;
 130                 $changed++;
 131                 $canonicalDecomp[$source] = $newDest;
 132         }
 133         $pass++;
 134 }
 135
 136 print "Recursively expanding compatibility mappings...\n";
 137 $changed = 42;
 138 $pass = 1;
 139 while( $changed > 0 ) {
 140         print "pass $pass\n";
 141         $changed = 0;
 142         foreach( $compatibilityDecomp as $source => $dest ) {
 143                 $newDest = preg_replace_callback(
 144                         '/([\xc0-\xff][\x80-\xbf]+)/',
 145                         'callbackCompat',
 146                         $dest);
 147                 if( $newDest === $dest ) continue;
 148                 $changed++;
 149                 $compatibilityDecomp[$source] = $newDest;
 150         }
 151         $pass++;
 152 }
 153
 154 print "$total decomposition mappings ($canon canonical, $compat compatibility)\n";
 155
 156 $out = fopen("UtfNormalData.inc", "wt");
 157 if( $out ) {
 158         $serCombining = escapeSingleString( serialize( $combiningClass ) );
 159         $serComp = escapeSingleString( serialize( $canonicalComp ) );
 160         $serCanon = escapeSingleString( serialize( $canonicalDecomp ) );
 161         $serCompat = escapeSingleString( serialize( $compatibilityDecomp ) );
 162         $serCheckNFC = escapeSingleString( serialize( $checkNFC ) );
 163         $outdata = "<" . "?php
 164 # This file was automatically generated -- do not edit!
 165 # Run UtfNormalGenerate.php to create this file again (make clean && make)
 166
 167 global \$utfCombiningClass, \$utfCanonicalComp, \$utfCanonicalDecomp, \$utfCompatibilityDecomp;
 168 \$utfCombiningClass = unserialize( '$serCombining' );
 169 \$utfCanonicalComp = unserialize( '$serComp' );
 170 \$utfCanonicalDecomp = unserialize( '$serCanon' );
 171 \$utfCompatibilityDecomp = unserialize( '$serCompat' );
 172 \$utfCheckNFC = unserialize( '$serCheckNFC' );
 173 ?" . ">\n";
 174         fputs( $out, $outdata );
 175         fclose( $out );
 176         print "Wrote out UtfNormalData.inc\n";
 177         exit(0);
 178 } else {
 179         print "Can't create file UtfNormalData.inc\n";
 180         exit(-1);
 181 }
 182
 183 # ---------------
 184
 185 function callbackCanonical( $matches ) {
 186         global $canonicalDecomp;
 187         if( isset( $canonicalDecomp[$matches[1]] ) ) {
 188                 return $canonicalDecomp[$matches[1]];
 189         }
 190         return $matches[1];
 191 }
 192
 193 function callbackCompat( $matches ) {
 194         global $compatibilityDecomp;
 195         if( isset( $compatibilityDecomp[$matches[1]] ) ) {
 196                 return $compatibilityDecomp[$matches[1]];
 197         }
 198         return $matches[1];
 199 }
 200
 201 ?>