more cssification of diff rendering, some " -> ' in diff engine

[lhc/web/wiklou.git] / includes / Tokenizer.php

diff --git a/includes/Tokenizer.php b/includes/Tokenizer.php
index f23df7f..ab18b10 100644 (file)
--- a/includes/Tokenizer.php
+++ b/includes/Tokenizer.php
@@ -137,6 +137,7 @@ class Tokenizer {
                                                               && $this->mText[$this->mPos+1] == "'" )
                                                        {
                                                                $queueToken["type"] .= "'";
+                                                               $queueToken["pos"] = $this->mPos;
                                                                $this->mPos ++;
                                                        }
                                                        
@@ -146,7 +147,7 @@ class Tokenizer {
                                                }
                                                break;
                                        case "\n": // for block levels, actually, only "----" is handled.
-                                       case "\r":
+                                       case "\r": // headings are detected to close any unbalanced em or strong tags in a section
                                                if ( $this->continues( "----" ) )
                                                {
                                                        $queueToken["type"] = "----";
@@ -159,6 +160,21 @@ class Tokenizer {
                                                                $this->mPos ++;
                                                        }
                                                        break 2;
+                                               } else if ( 
+                                                       $this->continues( "<h" ) and (
+                                                               $this->continues( "<h1" ) or
+                                                               $this->continues( "<h2" ) or 
+                                                               $this->continues( "<h3" ) or 
+                                                               $this->continues( "<h4" ) or 
+                                                               $this->continues( "<h5" ) or
+                                                               $this->continues( "<h6" ) 
+                                                       )
+                                               ) { // heading
+                                                       $queueToken["type"] = "h";
+                                                       $queueToken["text"] = "";
+                                                       $this->mQueuedToken[] = $queueToken;
+                                                       $this->mPos ++;
+                                                       break 2; // switch + while
                                                }
                                                break;
                                        case "!": // French spacing rules have a space before exclamation
@@ -217,6 +233,17 @@ class Tokenizer {
                                                        break 2; // switch + while
                                                }
                                                break;
+                                       case "&": //extensions like <timeline>, since HTML stripping has already been done, 
+                                                 //those look like &lt;timeline&gt;
+                                               if ( $this->continues( "lt;timeline&gt;" ) )
+                                               {
+                                                       $queueToken["type"] = "<timeline>";
+                                                       $queueToken["text"] = "&lt;timeline&gt;";
+                                                       $this->mQueuedToken[] = $queueToken;
+                                                       $this->mPos += 16;
+                                                       break 2; // switch + while
+                                               }
+                                               break;
 
                                } /* switch */
                                $token["text"].=$ch;
@@ -254,5 +281,16 @@ class Tokenizer {
                        return false;
                return ( 0 == strcmp( $prec, substr($this->mText, $this->mPos-$len, $len) ) );
        }
+
+       function readAllUntil( $border )
+       {
+               $n = strpos( $this->mText, $border, $this->mPos );
+               if ( $n === false )
+                       return "";
+               $ret = substr( $this->mText, $this->mPos, $n - $this->mPos );
+               $this->mPos = $n + strlen( $border ) + 1;
+               return $ret;
+       }
+
 }