includes/StringUtils.php

   1 <?php
   2 /**
   3  * Methods to play with strings.
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  * http://www.gnu.org/copyleft/gpl.html
  19  *
  20  * @file
  21  */
  22
  23 /**
  24  * A collection of static methods to play with strings.
  25  */
  26 class StringUtils {
  27
  28         /**
  29          * Test whether a string is valid UTF-8.
  30          *
  31          * The function check for invalid byte sequences, overlong encoding but
  32          * not for different normalisations.
  33          *
  34          * This relies internally on the mbstring function mb_check_encoding()
  35          * hardcoded to check against UTF-8. Whenever the function is not available
  36          * we fallback to a pure PHP implementation. Setting $disableMbstring to
  37          * true will skip the use of mb_check_encoding, this is mostly intended for
  38          * unit testing our internal implementation.
  39          *
  40          * @since 1.21
  41          * @note In MediaWiki 1.21, this function did not provide proper UTF-8 validation.
  42          * In particular, the pure PHP code path did not in fact check for overlong forms.
  43          * Beware of this when backporting code to that version of MediaWiki.
  44          *
  45          * @param string $value String to check
  46          * @param boolean $disableMbstring Whether to use the pure PHP
  47          * implementation instead of trying mb_check_encoding. Intended for unit
  48          * testing. Default: false
  49          *
  50          * @return boolean Whether the given $value is a valid UTF-8 encoded string
  51          */
  52         static function isUtf8( $value, $disableMbstring = false ) {
  53                 $value = (string)$value;
  54                 if ( preg_match( "/[\x80-\xff]/S", $value ) === 0 ) {
  55                         // String contains only ASCII characters, has to be valid
  56                         return true;
  57                 }
  58
  59                 // If the mbstring extension is loaded, use it. However, before PHP 5.4, values above
  60                 // U+10FFFF are incorrectly allowed, so we have to check for them separately.
  61                 if ( !$disableMbstring && function_exists( 'mb_check_encoding' ) ) {
  62                         static $newPHP;
  63                         if ( $newPHP === null ) {
  64                                 $newPHP = !mb_check_encoding( "\xf4\x90\x80\x80", 'UTF-8' );
  65                         }
  66
  67                         return mb_check_encoding( $value, 'UTF-8' ) &&
  68                                 ( $newPHP || preg_match( "/\xf4[\x90-\xbf]|[\xf5-\xff]/S", $value ) === 0 );
  69                 }
  70
  71                 // PCRE implements repetition using recursion; to avoid a stack overflow (and segfault)
  72                 // for large input, we check for invalid sequences (<= 5 bytes) rather than valid
  73                 // sequences, which can be as long as the input string is. Multiple short regexes are
  74                 // used rather than a single long regex for performance.
  75                 static $regexes;
  76                 if ( $regexes === null ) {
  77                         $cont = "[\x80-\xbf]";
  78                         $after = "(?!$cont)"; // "(?:[^\x80-\xbf]|$)" would work here
  79                         $regexes = array(
  80                                 // Continuation byte at the start
  81                                 "/^$cont/",
  82
  83                                 // ASCII byte followed by a continuation byte
  84                                 "/[\\x00-\x7f]$cont/S",
  85
  86                                 // Illegal byte
  87                                 "/[\xc0\xc1\xf5-\xff]/S",
  88
  89                                 // Invalid 2-byte sequence, or valid one then an extra continuation byte
  90                                 "/[\xc2-\xdf](?!$cont$after)/S",
  91
  92                                 // Invalid 3-byte sequence, or valid one then an extra continuation byte
  93                                 "/\xe0(?![\xa0-\xbf]$cont$after)/",
  94                                 "/[\xe1-\xec\xee\xef](?!$cont{2}$after)/S",
  95                                 "/\xed(?![\x80-\x9f]$cont$after)/",
  96
  97                                 // Invalid 4-byte sequence, or valid one then an extra continuation byte
  98                                 "/\xf0(?![\x90-\xbf]$cont{2}$after)/",
  99                                 "/[\xf1-\xf3](?!$cont{3}$after)/S",
 100                                 "/\xf4(?![\x80-\x8f]$cont{2}$after)/",
 101                         );
 102                 }
 103
 104                 foreach ( $regexes as $regex ) {
 105                         if ( preg_match( $regex, $value ) !== 0 ) {
 106                                 return false;
 107                         }
 108                 }
 109                 return true;
 110         }
 111
 112         /**
 113          * Perform an operation equivalent to
 114          *
 115          *     preg_replace( "!$startDelim(.*?)$endDelim!", $replace, $subject );
 116          *
 117          * except that it's worst-case O(N) instead of O(N^2)
 118          *
 119          * Compared to delimiterReplace(), this implementation is fast but memory-
 120          * hungry and inflexible. The memory requirements are such that I don't
 121          * recommend using it on anything but guaranteed small chunks of text.
 122          *
 123          * @param $startDelim
 124          * @param $endDelim
 125          * @param $replace
 126          * @param $subject
 127          *
 128          * @return string
 129          */
 130         static function hungryDelimiterReplace( $startDelim, $endDelim, $replace, $subject ) {
 131                 $segments = explode( $startDelim, $subject );
 132                 $output = array_shift( $segments );
 133                 foreach ( $segments as $s ) {
 134                         $endDelimPos = strpos( $s, $endDelim );
 135                         if ( $endDelimPos === false ) {
 136                                 $output .= $startDelim . $s;
 137                         } else {
 138                                 $output .= $replace . substr( $s, $endDelimPos + strlen( $endDelim ) );
 139                         }
 140                 }
 141                 return $output;
 142         }
 143
 144         /**
 145          * Perform an operation equivalent to
 146          *
 147          *   preg_replace_callback( "!$startDelim(.*)$endDelim!s$flags", $callback, $subject )
 148          *
 149          * This implementation is slower than hungryDelimiterReplace but uses far less
 150          * memory. The delimiters are literal strings, not regular expressions.
 151          *
 152          * If the start delimiter ends with an initial substring of the end delimiter,
 153          * e.g. in the case of C-style comments, the behavior differs from the model
 154          * regex. In this implementation, the end must share no characters with the
 155          * start, so e.g. /*\/ is not considered to be both the start and end of a
 156          * comment. /*\/xy/*\/ is considered to be a single comment with contents /xy/.
 157          *
 158          * @param string $startDelim start delimiter
 159          * @param string $endDelim end delimiter
 160          * @param $callback Callback: function to call on each match
 161          * @param $subject String
 162          * @param string $flags regular expression flags
 163          * @throws MWException
 164          * @return string
 165          */
 166         static function delimiterReplaceCallback( $startDelim, $endDelim, $callback, $subject, $flags = '' ) {
 167                 $inputPos = 0;
 168                 $outputPos = 0;
 169                 $output = '';
 170                 $foundStart = false;
 171                 $encStart = preg_quote( $startDelim, '!' );
 172                 $encEnd = preg_quote( $endDelim, '!' );
 173                 $strcmp = strpos( $flags, 'i' ) === false ? 'strcmp' : 'strcasecmp';
 174                 $endLength = strlen( $endDelim );
 175                 $m = array();
 176
 177                 while ( $inputPos < strlen( $subject ) &&
 178                         preg_match( "!($encStart)|($encEnd)!S$flags", $subject, $m, PREG_OFFSET_CAPTURE, $inputPos ) )
 179                 {
 180                         $tokenOffset = $m[0][1];
 181                         if ( $m[1][0] != '' ) {
 182                                 if ( $foundStart &&
 183                                         $strcmp( $endDelim, substr( $subject, $tokenOffset, $endLength ) ) == 0 )
 184                                 {
 185                                         # An end match is present at the same location
 186                                         $tokenType = 'end';
 187                                         $tokenLength = $endLength;
 188                                 } else {
 189                                         $tokenType = 'start';
 190                                         $tokenLength = strlen( $m[0][0] );
 191                                 }
 192                         } elseif ( $m[2][0] != '' ) {
 193                                 $tokenType = 'end';
 194                                 $tokenLength = strlen( $m[0][0] );
 195                         } else {
 196                                 throw new MWException( 'Invalid delimiter given to ' . __METHOD__ );
 197                         }
 198
 199                         if ( $tokenType == 'start' ) {
 200                                 # Only move the start position if we haven't already found a start
 201                                 # This means that START START END matches outer pair
 202                                 if ( !$foundStart ) {
 203                                         # Found start
 204                                         $inputPos = $tokenOffset + $tokenLength;
 205                                         # Write out the non-matching section
 206                                         $output .= substr( $subject, $outputPos, $tokenOffset - $outputPos );
 207                                         $outputPos = $tokenOffset;
 208                                         $contentPos = $inputPos;
 209                                         $foundStart = true;
 210                                 } else {
 211                                         # Move the input position past the *first character* of START,
 212                                         # to protect against missing END when it overlaps with START
 213                                         $inputPos = $tokenOffset + 1;
 214                                 }
 215                         } elseif ( $tokenType == 'end' ) {
 216                                 if ( $foundStart ) {
 217                                         # Found match
 218                                         $output .= call_user_func( $callback, array(
 219                                                 substr( $subject, $outputPos, $tokenOffset + $tokenLength - $outputPos ),
 220                                                 substr( $subject, $contentPos, $tokenOffset - $contentPos )
 221                                         ));
 222                                         $foundStart = false;
 223                                 } else {
 224                                         # Non-matching end, write it out
 225                                         $output .= substr( $subject, $inputPos, $tokenOffset + $tokenLength - $outputPos );
 226                                 }
 227                                 $inputPos = $outputPos = $tokenOffset + $tokenLength;
 228                         } else {
 229                                 throw new MWException( 'Invalid delimiter given to ' . __METHOD__ );
 230                         }
 231                 }
 232                 if ( $outputPos < strlen( $subject ) ) {
 233                         $output .= substr( $subject, $outputPos );
 234                 }
 235                 return $output;
 236         }
 237
 238         /**
 239          * Perform an operation equivalent to
 240          *
 241          *   preg_replace( "!$startDelim(.*)$endDelim!$flags", $replace, $subject )
 242          *
 243          * @param string $startDelim start delimiter regular expression
 244          * @param string $endDelim end delimiter regular expression
 245          * @param string $replace replacement string. May contain $1, which will be
 246          *                 replaced by the text between the delimiters
 247          * @param string $subject to search
 248          * @param string $flags regular expression flags
 249          * @return String: The string with the matches replaced
 250          */
 251         static function delimiterReplace( $startDelim, $endDelim, $replace, $subject, $flags = '' ) {
 252                 $replacer = new RegexlikeReplacer( $replace );
 253                 return self::delimiterReplaceCallback( $startDelim, $endDelim,
 254                         $replacer->cb(), $subject, $flags );
 255         }
 256
 257         /**
 258          * More or less "markup-safe" explode()
 259          * Ignores any instances of the separator inside <...>
 260          * @param $separator String
 261          * @param $text String
 262          * @return array
 263          */
 264         static function explodeMarkup( $separator, $text ) {
 265                 $placeholder = "\x00";
 266
 267                 // Remove placeholder instances
 268                 $text = str_replace( $placeholder, '', $text );
 269
 270                 // Replace instances of the separator inside HTML-like tags with the placeholder
 271                 $replacer = new DoubleReplacer( $separator, $placeholder );
 272                 $cleaned = StringUtils::delimiterReplaceCallback( '<', '>', $replacer->cb(), $text );
 273
 274                 // Explode, then put the replaced separators back in
 275                 $items = explode( $separator, $cleaned );
 276                 foreach ( $items as $i => $str ) {
 277                         $items[$i] = str_replace( $placeholder, $separator, $str );
 278                 }
 279
 280                 return $items;
 281         }
 282
 283         /**
 284          * Escape a string to make it suitable for inclusion in a preg_replace()
 285          * replacement parameter.
 286          *
 287          * @param $string String
 288          * @return String
 289          */
 290         static function escapeRegexReplacement( $string ) {
 291                 $string = str_replace( '\\', '\\\\', $string );
 292                 $string = str_replace( '$', '\\$', $string );
 293                 return $string;
 294         }
 295
 296         /**
 297          * Workalike for explode() with limited memory usage.
 298          * Returns an Iterator
 299          * @param $separator
 300          * @param $subject
 301          * @return ArrayIterator|ExplodeIterator
 302          */
 303         static function explode( $separator, $subject ) {
 304                 if ( substr_count( $subject, $separator ) > 1000 ) {
 305                         return new ExplodeIterator( $separator, $subject );
 306                 } else {
 307                         return new ArrayIterator( explode( $separator, $subject ) );
 308                 }
 309         }
 310 }
 311
 312 /**
 313  * Base class for "replacers", objects used in preg_replace_callback() and
 314  * StringUtils::delimiterReplaceCallback()
 315  */
 316 class Replacer {
 317
 318         /**
 319          * @return array
 320          */
 321         function cb() {
 322                 return array( &$this, 'replace' );
 323         }
 324 }
 325
 326 /**
 327  * Class to replace regex matches with a string similar to that used in preg_replace()
 328  */
 329 class RegexlikeReplacer extends Replacer {
 330         var $r;
 331
 332         /**
 333          * @param $r string
 334          */
 335         function __construct( $r ) {
 336                 $this->r = $r;
 337         }
 338
 339         /**
 340          * @param $matches array
 341          * @return string
 342          */
 343         function replace( $matches ) {
 344                 $pairs = array();
 345                 foreach ( $matches as $i => $match ) {
 346                         $pairs["\$$i"] = $match;
 347                 }
 348                 return strtr( $this->r, $pairs );
 349         }
 350
 351 }
 352
 353 /**
 354  * Class to perform secondary replacement within each replacement string
 355  */
 356 class DoubleReplacer extends Replacer {
 357
 358         /**
 359          * @param $from
 360          * @param $to
 361          * @param $index int
 362          */
 363         function __construct( $from, $to, $index = 0 ) {
 364                 $this->from = $from;
 365                 $this->to = $to;
 366                 $this->index = $index;
 367         }
 368
 369         /**
 370          * @param $matches array
 371          * @return mixed
 372          */
 373         function replace( $matches ) {
 374                 return str_replace( $this->from, $this->to, $matches[$this->index] );
 375         }
 376 }
 377
 378 /**
 379  * Class to perform replacement based on a simple hashtable lookup
 380  */
 381 class HashtableReplacer extends Replacer {
 382         var $table, $index;
 383
 384         /**
 385          * @param $table
 386          * @param $index int
 387          */
 388         function __construct( $table, $index = 0 ) {
 389                 $this->table = $table;
 390                 $this->index = $index;
 391         }
 392
 393         /**
 394          * @param $matches array
 395          * @return mixed
 396          */
 397         function replace( $matches ) {
 398                 return $this->table[$matches[$this->index]];
 399         }
 400 }
 401
 402 /**
 403  * Replacement array for FSS with fallback to strtr()
 404  * Supports lazy initialisation of FSS resource
 405  */
 406 class ReplacementArray {
 407         /*mostly private*/ var $data = false;
 408         /*mostly private*/ var $fss = false;
 409
 410         /**
 411          * Create an object with the specified replacement array
 412          * The array should have the same form as the replacement array for strtr()
 413          * @param array $data
 414          */
 415         function __construct( $data = array() ) {
 416                 $this->data = $data;
 417         }
 418
 419         /**
 420          * @return array
 421          */
 422         function __sleep() {
 423                 return array( 'data' );
 424         }
 425
 426         function __wakeup() {
 427                 $this->fss = false;
 428         }
 429
 430         /**
 431          * Set the whole replacement array at once
 432          */
 433         function setArray( $data ) {
 434                 $this->data = $data;
 435                 $this->fss = false;
 436         }
 437
 438         /**
 439          * @return array|bool
 440          */
 441         function getArray() {
 442                 return $this->data;
 443         }
 444
 445         /**
 446          * Set an element of the replacement array
 447          * @param $from string
 448          * @param $to string
 449          */
 450         function setPair( $from, $to ) {
 451                 $this->data[$from] = $to;
 452                 $this->fss = false;
 453         }
 454
 455         /**
 456          * @param $data array
 457          */
 458         function mergeArray( $data ) {
 459                 $this->data = array_merge( $this->data, $data );
 460                 $this->fss = false;
 461         }
 462
 463         /**
 464          * @param $other
 465          */
 466         function merge( $other ) {
 467                 $this->data = array_merge( $this->data, $other->data );
 468                 $this->fss = false;
 469         }
 470
 471         /**
 472          * @param $from string
 473          */
 474         function removePair( $from ) {
 475                 unset( $this->data[$from] );
 476                 $this->fss = false;
 477         }
 478
 479         /**
 480          * @param $data array
 481          */
 482         function removeArray( $data ) {
 483                 foreach ( $data as $from => $to ) {
 484                         $this->removePair( $from );
 485                 }
 486                 $this->fss = false;
 487         }
 488
 489         /**
 490          * @param $subject string
 491          * @return string
 492          */
 493         function replace( $subject ) {
 494                 if ( function_exists( 'fss_prep_replace' ) ) {
 495                         wfProfileIn( __METHOD__ . '-fss' );
 496                         if ( $this->fss === false ) {
 497                                 $this->fss = fss_prep_replace( $this->data );
 498                         }
 499                         $result = fss_exec_replace( $this->fss, $subject );
 500                         wfProfileOut( __METHOD__ . '-fss' );
 501                 } else {
 502                         wfProfileIn( __METHOD__ . '-strtr' );
 503                         $result = strtr( $subject, $this->data );
 504                         wfProfileOut( __METHOD__ . '-strtr' );
 505                 }
 506                 return $result;
 507         }
 508 }
 509
 510 /**
 511  * An iterator which works exactly like:
 512  *
 513  * foreach ( explode( $delim, $s ) as $element ) {
 514  *    ...
 515  * }
 516  *
 517  * Except it doesn't use 193 byte per element
 518  */
 519 class ExplodeIterator implements Iterator {
 520         // The subject string
 521         var $subject, $subjectLength;
 522
 523         // The delimiter
 524         var $delim, $delimLength;
 525
 526         // The position of the start of the line
 527         var $curPos;
 528
 529         // The position after the end of the next delimiter
 530         var $endPos;
 531
 532         // The current token
 533         var $current;
 534
 535         /**
 536          * Construct a DelimIterator
 537          * @param $delim string
 538          * @param $s string
 539          */
 540         function __construct( $delim, $s ) {
 541                 $this->subject = $s;
 542                 $this->delim = $delim;
 543
 544                 // Micro-optimisation (theoretical)
 545                 $this->subjectLength = strlen( $s );
 546                 $this->delimLength = strlen( $delim );
 547
 548                 $this->rewind();
 549         }
 550
 551         function rewind() {
 552                 $this->curPos = 0;
 553                 $this->endPos = strpos( $this->subject, $this->delim );
 554                 $this->refreshCurrent();
 555         }
 556
 557         function refreshCurrent() {
 558                 if ( $this->curPos === false ) {
 559                         $this->current = false;
 560                 } elseif ( $this->curPos >= $this->subjectLength ) {
 561                         $this->current = '';
 562                 } elseif ( $this->endPos === false ) {
 563                         $this->current = substr( $this->subject, $this->curPos );
 564                 } else {
 565                         $this->current = substr( $this->subject, $this->curPos, $this->endPos - $this->curPos );
 566                 }
 567         }
 568
 569         function current() {
 570                 return $this->current;
 571         }
 572
 573         function key() {
 574                 return $this->curPos;
 575         }
 576
 577         /**
 578          * @return string
 579          */
 580         function next() {
 581                 if ( $this->endPos === false ) {
 582                         $this->curPos = false;
 583                 } else {
 584                         $this->curPos = $this->endPos + $this->delimLength;
 585                         if ( $this->curPos >= $this->subjectLength ) {
 586                                 $this->endPos = false;
 587                         } else {
 588                                 $this->endPos = strpos( $this->subject, $this->delim, $this->curPos );
 589                         }
 590                 }
 591                 $this->refreshCurrent();
 592                 return $this->current;
 593         }
 594
 595         /**
 596          * @return bool
 597          */
 598         function valid() {
 599                 return $this->curPos !== false;
 600         }
 601 }