b638b63bfa0432c04bbbf3d43ae50192b97c47ba
3 * Generates normalizer data files for Arabic and Malayalam.
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License along
16 * with this program; if not, write to the Free Software Foundation, Inc.,
17 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18 * http://www.gnu.org/copyleft/gpl.html
21 * @ingroup MaintenanceLanguage
24 require_once __DIR__
. '/../../includes/normal/UtfNormalUtil.php';
26 require_once __DIR__
. '/../Maintenance.php';
29 * Generates normalizer data files for Arabic and Malayalam.
30 * For NFC see includes/normal.
32 * @ingroup MaintenanceLanguage
34 class GenerateNormalizerData
extends Maintenance
{
37 public function __construct() {
38 parent
::__construct();
39 $this->addOption( 'unicode-data-file', 'The local location of the data file ' .
40 'from http://unicode.org/Public/UNIDATA/UnicodeData.txt', false, true );
43 public function execute() {
44 if ( !$this->hasOption( 'unicode-data-file' ) ) {
45 $this->dataFile
= 'UnicodeData.txt';
46 if ( !file_exists( $this->dataFile
) ) {
47 $this->error( "Unable to find UnicodeData.txt. Please specify " .
48 "its location with --unicode-data-file=<FILE>" );
52 $this->dataFile
= $this->getOption( 'unicode-data-file' );
53 if ( !file_exists( $this->dataFile
) ) {
54 $this->error( 'Unable to find the specified data file.' );
59 $this->generateArabic();
60 $this->generateMalayalam();
63 function generateArabic() {
64 $file = fopen( $this->dataFile
, 'r' );
66 $this->error( 'Unable to open the data file.' );
70 // For the file format, see http://www.unicode.org/reports/tr44/
75 'Canonical_Combining_Class',
77 'Decomposition_Type_Mapping',
82 'Simple_Uppercase_Mapping',
83 'Simple_Lowercase_Mapping',
84 'Simple_Titlecase_Mapping'
90 while ( false !== ( $line = fgets( $file ) ) ) {
94 $line = trim( substr( $line, 0, strcspn( $line, '#' ) ) );
100 $numberedData = explode( ';', $line );
102 foreach ( $fieldNames as $number => $name ) {
103 $data[$name] = $numberedData[$number];
106 $code = base_convert( $data['Code'], 16, 10 );
107 if ( ( $code >= 0xFB50 && $code <= 0xFDFF ) # Arabic presentation forms A
108 ||
( $code >= 0xFE70 && $code <= 0xFEFF ) # Arabic presentation forms B
110 if ( $data['Decomposition_Type_Mapping'] === '' ) {
114 if ( !preg_match( '/^ *(<\w*>) +([0-9A-F ]*)$/',
115 $data['Decomposition_Type_Mapping'], $m )
117 $this->error( "Can't parse Decomposition_Type/Mapping on line $lineNum" );
118 $this->error( $line );
122 $source = hexSequenceToUtf8( $data['Code'] );
123 $dest = hexSequenceToUtf8( $m[2] );
124 $pairs[$source] = $dest;
129 file_put_contents( "$IP/serialized/normalize-ar.ser", serialize( $pairs ) );
130 echo "ar: " . count( $pairs ) . " pairs written.\n";
133 function generateMalayalam() {
135 # From http://unicode.org/versions/Unicode5.1.0/#Malayalam_Chillu_Characters
136 '0D23 0D4D 200D' => '0D7A',
137 '0D28 0D4D 200D' => '0D7B',
138 '0D30 0D4D 200D' => '0D7C',
139 '0D32 0D4D 200D' => '0D7D',
140 '0D33 0D4D 200D' => '0D7E',
142 # From http://permalink.gmane.org/gmane.science.linguistics.wikipedia.technical/46413
143 '0D15 0D4D 200D' => '0D7F',
147 foreach ( $hexPairs as $hexSource => $hexDest ) {
148 $source = hexSequenceToUtf8( $hexSource );
149 $dest = hexSequenceToUtf8( $hexDest );
150 $pairs[$source] = $dest;
154 file_put_contents( "$IP/serialized/normalize-ml.ser", serialize( $pairs ) );
155 echo "ml: " . count( $pairs ) . " pairs written.\n";
159 $maintClass = 'GenerateNormalizerData';
160 require_once RUN_MAINTENANCE_IF_MAIN
;