Repaired rendering of ''emph ''' bold emph '' bold ''' to produce valid HTML.

[lhc/web/wiklou.git] / includes / Tokenizer.php

diff --git a/includes/Tokenizer.php b/includes/Tokenizer.php
index 9ef18b0..0b313d5 100644 (file)
--- a/includes/Tokenizer.php
+++ b/includes/Tokenizer.php
@@ -8,8 +8,11 @@ class Tokenizer {
 
        /* private */ function Tokenizer()
        {
+               global $wgLang;
+
                $this->mPos=0;
                $this->mTokenQueue=array();
+               $this->linkPrefixExtension = $wgLang->linkPrefixExtension();
        }
 
        # factory function
@@ -54,8 +57,7 @@ class Tokenizer {
        // proceeds character by character through the text, looking for characters needing
        // special attention. Those are currently: I, R, ', [, ], newline
        //
-       // TODO: prefixed links for Arabic wikipedia not implemented yet
-       //       handling of French blanks not yet implemented
+       // TODO:  handling of French blanks not yet implemented
        function nextToken()
        {
                $fname = "Tokenizer::nextToken";
@@ -64,8 +66,8 @@ class Tokenizer {
                if ( count( $this->mQueuedToken ) != 0 ) {
                        // still one token from the last round around. Return that one first.
                        $token = array_shift( $this->mQueuedToken );
-               } else if ( $this->mPos > $this->mTextLength )
-               {       // If no text is left, return "false".
+               } else if ( $this->mPos > $this->mTextLength ) {
+                       // If no text is left, return "false".
                        $token = false;
                } else {
 
@@ -98,8 +100,21 @@ class Tokenizer {
                                                        $this->mPos += 3;
                                                        break 2; // switch + while
                                                } else if ( $this->continues("[") ) {
-                                                       $queueToken["type"] = "[[";
+                                                       $queueToken["type"] = "[[";
                                                        $queueToken["text"] = "";
+                                                       // Check for a "prefixed link", e.g. Al[[Khazar]]
+                                                       // Mostly for arabic wikipedia
+                                                       if ( $this->linkPrefixExtension ) {
+                                                               while (    $this->linkPrefixExtension
+                                                                       && ($len = strlen( $token["text"] ) ) > 0 
+                                                                       && !ctype_space( $token["text"][$len-1] ) )
+                                                               {
+                                                                       //prepend the character to the link's open tag
+                                                                       $queueToken["text"] = $token["text"][$len-1] . $queueToken["text"];
+                                                                       //remove character from the end of the text token
+                                                                       $token["text"] = substr( $token["text"], 0, -1);
+                                                               }
+                                                       }
                                                        $this->mQueuedToken[] = $queueToken;
                                                        $this->mPos += 2;
                                                        break 2; // switch + while 
@@ -122,6 +137,7 @@ class Tokenizer {
                                                               && $this->mText[$this->mPos+1] == "'" )
                                                        {
                                                                $queueToken["type"] .= "'";
+                                                               $queueToken["pos"] = $this->mPos;
                                                                $this->mPos ++;
                                                        }
                                                        
@@ -145,6 +161,75 @@ class Tokenizer {
                                                        }
                                                        break 2;
                                                }
+                                               break;
+                                       case "!": // French spacing rules have a space before exclamation
+                                       case "?": // and question marks. Those have to become  
+                                       case ":": // And colons, Hashar says ...
+                                               if ( $this->preceeded( " " ) )
+                                               {
+                                                       // strip blank from Token
+                                                       $token["text"] = substr( $token["text"], 0, -1 );
+                                                       $queueToken["type"] = "blank";
+                                                       $queueToken["text"] = " {$ch}";
+                                                       $this->mQueuedToken[] = $queueToken;
+                                                       $this->mPos ++;
+                                                       break 2; // switch + while
+                                               }
+                                               break;
+                                       case "0": // A space between two numbers is used to ease reading
+                                       case "1": // of big numbers, e.g. 1 000 000. Those spaces need
+                                       case "2": // to be unbreakable
+                                       case "3":
+                                       case "4":
+                                       case "5":
+                                       case "6":
+                                       case "7":
+                                       case "8":
+                                       case "9":
+                                               if (    ($this->mTextLength >= $this->mPos +2)
+                                                    && ($this->mText[$this->mPos+1] == " ")
+                                                    && ctype_digit( $this->mText[$this->mPos+2] ) )
+                                               {
+                                                       $queueToken["type"] = "blank";
+                                                       $queueToken["text"] = $ch . " ";
+                                                       $this->mQueuedToken[] = $queueToken;
+                                                       $this->mPos += 2;
+                                                       break 2; // switch + while
+                                               }
+                                               break;
+                                       case "\302": // first byte of UTF-8 Character Guillemet-left
+                                               if ( $this->continues( "\253 ") ) // second byte and a blank
+                                               {
+                                                       $queueToken["type"] = "blank";
+                                                       $queueToken["text"] = "\302\253 ";
+                                                       $this->mQueuedToken[] = $queueToken;
+                                                       $this->mPos += 3;
+                                                       break 2; // switch + while
+                                               }
+                                               break;
+                                       case "\273": //last byte of UTF-8 Character Guillemet-right
+                                               if ( $this->preceeded( " \302" ) )
+                                               {
+                                                       $queueToken["type"] = "blank";
+                                                       $queueToken["text"] = " \302\273";
+                                                       $token["text"] = substr( $token["text"], 0, -2 );
+                                                       $this->mQueuedToken[] = $queueToken;
+                                                       $this->mPos ++;
+                                                       break 2; // switch + while
+                                               }
+                                               break;
+                                       case "&": //extensions like <timeline>, since HTML stripping has already been done, 
+                                                 //those look like &lt;timeline&gt;
+                                               if ( $this->continues( "lt;timeline&gt;" ) )
+                                               {
+                                                       $queueToken["type"] = "<timeline>";
+                                                       $queueToken["text"] = "&lt;timeline&gt;";
+                                                       $this->mQueuedToken[] = $queueToken;
+                                                       $this->mPos += 16;
+                                                       break 2; // switch + while
+                                               }
+                                               break;
+
                                } /* switch */
                                $token["text"].=$ch;
                                $this->mPos ++;
@@ -158,7 +243,7 @@ class Tokenizer {
 
        // function continues
        // checks whether the mText continues with $cont from mPos+1
-       function continues( $cont )
+       /* private */ function continues( $cont )
        {
                // If string is not long enough to contain $cont, return false
                if ( $this->mTextLength < $this->mPos + strlen( $cont ) )
@@ -170,6 +255,27 @@ class Tokenizer {
                }
                return true;
        }
-               
+
+       // function preceeded
+       // checks whether the mText is preceeded by $prec at position mPos
+       /* private */ function preceeded( $prec )
+       {
+               $len = strlen( $prec );
+               // if $prec is longer than the text up to mPos, return false
+               if ( $this->mPos < $len )
+                       return false;
+               return ( 0 == strcmp( $prec, substr($this->mText, $this->mPos-$len, $len) ) );
+       }
+
+       function readAllUntil( $border )
+       {
+               $n = strpos( $this->mText, $border, $this->mPos );
+               if ( $n === false )
+                       return "";
+               $ret = substr( $this->mText, $this->mPos, $n - $this->mPos );
+               $this->mPos = $n + strlen( $border ) + 1;
+               return $ret;
+       }
+
 }