a5e2885ae87d17c73ad9afc10be2b5a4b8528804
3 * This script generates UniNormalData.inc from the Unicode Character Database
4 * and supplementary files.
6 * Copyright (C) 2004 Brion Vibber <brion@pobox.com>
7 * http://www.mediawiki.org/
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License as published by
11 * the Free Software Foundation; either version 2 of the License, or
12 * (at your option) any later version.
14 * This program is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 * GNU General Public License for more details.
19 * You should have received a copy of the GNU General Public License along
20 * with this program; if not, write to the Free Software Foundation, Inc.,
21 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
22 * http://www.gnu.org/copyleft/gpl.html
28 if( php_sapi_name() != 'cli' ) {
29 die( "Run me from the command line please.\n" );
32 require_once 'UtfNormalUtil.php';
34 $in = fopen("DerivedNormalizationProps.txt", "rt" );
36 print "Can't open DerivedNormalizationProps.txt for reading.\n";
37 print "If necessary, fetch this file from the internet:\n";
38 print "http://www.unicode.org/Public/UNIDATA/DerivedNormalizationProps.txt\n";
41 print "Initializing normalization quick check tables...\n";
43 while( false !== ($line = fgets( $in ) ) ) {
45 if( preg_match( '/^([0-9A-F]+)(?:..([0-9A-F]+))?\s*;\s*(NFC_QC)\s*;\s*([MN])/', $line, $matches ) ) {
46 list( $junk, $first, $last, $prop, $value ) = $matches;
47 #print "$first $last $prop $value\n";
48 if( !$last ) $last = $first;
49 for( $i = hexdec( $first ); $i <= hexdec( $last ); $i++
) {
50 $char = codepointToUtf8( $i );
51 $checkNFC[$char] = $value;
57 $in = fopen("CompositionExclusions.txt", "rt" );
59 print "Can't open CompositionExclusions.txt for reading.\n";
60 print "If necessary, fetch this file from the internet:\n";
61 print "http://www.unicode.org/Public/UNIDATA/CompositionExclusions.txt\n";
65 while( false !== ($line = fgets( $in ) ) ) {
66 if( preg_match( '/^([0-9A-F]+)/i', $line, $matches ) ) {
67 $codepoint = $matches[1];
68 $source = codepointToUtf8( hexdec( $codepoint ) );
69 $exclude[$source] = true;
74 $in = fopen("UnicodeData.txt", "rt" );
76 print "Can't open UnicodeData.txt for reading.\n";
77 print "If necessary, fetch this file from the internet:\n";
78 print "http://www.unicode.org/Public/UNIDATA/UnicodeData.txt\n";
82 $compatibilityDecomp = array();
83 $canonicalDecomp = array();
84 $canonicalComp = array();
85 $combiningClass = array();
90 print "Reading character definitions...\n";
91 while( false !== ($line = fgets( $in ) ) ) {
92 $columns = explode(';', $line);
93 $codepoint = $columns[0];
95 $canonicalCombiningClass = $columns[3];
96 $decompositionMapping = $columns[5];
98 $source = codepointToUtf8( hexdec( $codepoint ) );
100 if( $canonicalCombiningClass != 0 ) {
101 $combiningClass[$source] = intval( $canonicalCombiningClass );
104 if( $decompositionMapping === '' ) continue;
105 if( preg_match( '/^<(.+)> (.*)$/', $decompositionMapping, $matches ) ) {
106 # Compatibility decomposition
108 $decompositionMapping = $matches[2];
115 $dest = hexSequenceToUtf8( $decompositionMapping );
117 $compatibilityDecomp[$source] = $dest;
119 $canonicalDecomp[$source] = $dest;
120 if( empty( $exclude[$source] ) ) {
121 $canonicalComp[$dest] = $source;
124 #print "$codepoint | $canonicalCombiningClasses | $decompositionMapping\n";
128 print "Recursively expanding canonical mappings...\n";
131 while( $changed > 0 ) {
132 print "pass $pass\n";
134 foreach( $canonicalDecomp as $source => $dest ) {
135 $newDest = preg_replace_callback(
136 '/([\xc0-\xff][\x80-\xbf]+)/',
139 if( $newDest === $dest ) continue;
141 $canonicalDecomp[$source] = $newDest;
146 print "Recursively expanding compatibility mappings...\n";
149 while( $changed > 0 ) {
150 print "pass $pass\n";
152 foreach( $compatibilityDecomp as $source => $dest ) {
153 $newDest = preg_replace_callback(
154 '/([\xc0-\xff][\x80-\xbf]+)/',
157 if( $newDest === $dest ) continue;
159 $compatibilityDecomp[$source] = $newDest;
164 print "$total decomposition mappings ($canon canonical, $compat compatibility)\n";
166 $out = fopen("UtfNormalData.inc", "wt");
168 $serCombining = escapeSingleString( serialize( $combiningClass ) );
169 $serComp = escapeSingleString( serialize( $canonicalComp ) );
170 $serCanon = escapeSingleString( serialize( $canonicalDecomp ) );
171 $serCheckNFC = escapeSingleString( serialize( $checkNFC ) );
172 $outdata = "<" . "?php
174 * This file was automatically generated -- do not edit!
175 * Run UtfNormalGenerate.php to create this file again (make clean && make)
180 UtfNormal::\$utfCombiningClass = unserialize( '$serCombining' );
181 UtfNormal::\$utfCanonicalComp = unserialize( '$serComp' );
182 UtfNormal::\$utfCanonicalDecomp = unserialize( '$serCanon' );
183 UtfNormal::\$utfCheckNFC = unserialize( '$serCheckNFC' );
185 fputs( $out, $outdata );
187 print "Wrote out UtfNormalData.inc\n";
189 print "Can't create file UtfNormalData.inc\n";
194 $out = fopen("UtfNormalDataK.inc", "wt");
196 $serCompat = escapeSingleString( serialize( $compatibilityDecomp ) );
197 $outdata = "<" . "?php
199 * This file was automatically generated -- do not edit!
200 * Run UtfNormalGenerate.php to create this file again (make clean && make)
205 UtfNormal::\$utfCompatibilityDecomp = unserialize( '$serCompat' );
207 fputs( $out, $outdata );
209 print "Wrote out UtfNormalDataK.inc\n";
212 print "Can't create file UtfNormalDataK.inc\n";
218 function callbackCanonical( $matches ) {
219 global $canonicalDecomp;
220 if( isset( $canonicalDecomp[$matches[1]] ) ) {
221 return $canonicalDecomp[$matches[1]];
226 function callbackCompat( $matches ) {
227 global $compatibilityDecomp;
228 if( isset( $compatibilityDecomp[$matches[1]] ) ) {
229 return $compatibilityDecomp[$matches[1]];