restore section folding
[lhc/web/wiklou.git] / includes / Parser.php
index 3a1d830..dfba14d 100644 (file)
@@ -1,9 +1,12 @@
 <?php
 
-include_once('Tokenizer.php');
+require_once('Tokenizer.php');
 
 if( $GLOBALS['wgUseWikiHiero'] ){
-       include_once('wikihiero.php');
+       require_once('extensions/wikihiero/wikihiero.php');
+}
+if( $GLOBALS['wgUseTimeline'] ){
+       require_once('extensions/timeline/Timeline.php');
 }
 
 # PHP Parser
@@ -41,6 +44,12 @@ define( "OT_HTML", 1 );
 define( "OT_WIKI", 2 );
 define( "OT_MSG", 3 );
 
+# string parameter for extractTags which will cause it
+# to strip HTML comments in addition to regular
+# <XML>-style tags. This should not be anything we
+# may want to use in wikisyntax
+define( "STRIP_COMMENTS", "HTMLCommentStrip" );
+
 # prefix for escaping, used in two functions at least
 define( "UNIQ_PREFIX", "NaodW29");
 
@@ -77,6 +86,7 @@ class Parser
        #
        function parse( $text, &$title, $options, $linestart = true, $clearState = true )
        {
+               global $wgUseTidy;
                $fname = "Parser::parse";
                wfProfileIn( $fname );
 
@@ -93,20 +103,29 @@ class Parser
                $text = $this->internalParse( $text, $linestart );
                $text = $this->unstrip( $text, $this->mStripState );
                # Clean up special characters, only run once, next-to-last before doBlockLevels
-               $fixtags = array(
-                       "/<hr *>/i" => '<hr/>',
-                       "/<br *>/i" => '<br/>',
-                       "/<center *>/i"=>'<div class="center">',
-                       "/<\\/center *>/i" => '</div>',
-                       # Clean up spare ampersands; note that we probably ought to be
-                       # more careful about named entities.
-                       '/&(?!:amp;|#[Xx][0-9A-fa-f]+;|#[0-9]+;|[a-zA-Z0-9]+;)/' => '&amp;'
-               );
-               $text = preg_replace( array_keys($fixtags), array_values($fixtags), $text );
-
+               if(!$wgUseTidy) {
+                       $fixtags = array(
+                               "/<hr *>/i" => '<hr/>',
+                               "/<br *>/i" => '<br/>',
+                               "/<center *>/i"=>'<div class="center">',
+                               "/<\\/center *>/i" => '</div>',
+                               # Clean up spare ampersands; note that we probably ought to be
+                               # more careful about named entities.
+                               '/&(?!:amp;|#[Xx][0-9A-fa-f]+;|#[0-9]+;|[a-zA-Z0-9]+;)/' => '&amp;'
+                       );
+                       $text = preg_replace( array_keys($fixtags), array_values($fixtags), $text );
+               } else {
+                       $fixtags = array(
+                               "/<center *>/i"=>'<div class="center">',
+                               "/<\\/center *>/i" => '</div>'
+                       );
+                       $text = preg_replace( array_keys($fixtags), array_values($fixtags), $text );
+               }
                # only once and last
                $text = $this->doBlockLevels( $text, $linestart );
-
+               if($wgUseTidy) {
+                       $text = $this->tidy($text);
+               }
                $this->mOutput->setText( $text );
                wfProfileOut( $fname );
                return $this->mOutput;
@@ -124,6 +143,9 @@ class Parser
 
        # If $content is already set, the additional entries will be appended
 
+       # If $tag is set to STRIP_COMMENTS, the function will extract
+       # <!-- HTML comments -->
+
        /* static */ function extractTags($tag, $text, &$content, $uniq_prefix = ""){
                $rnd = $uniq_prefix . '-' . $tag . Parser::getRandomString();
                if ( !$content ) {
@@ -133,12 +155,20 @@ class Parser
                $stripped = "";
 
                while ( "" != $text ) {
-                       $p = preg_split( "/<\\s*$tag\\s*>/i", $text, 2 );
+                       if($tag==STRIP_COMMENTS) {
+                               $p = preg_split( "/<!--/i", $text, 2 );
+                       } else {
+                               $p = preg_split( "/<\\s*$tag\\s*>/i", $text, 2 );
+                       }
                        $stripped .= $p[0];
                        if ( ( count( $p ) < 2 ) || ( "" == $p[1] ) ) {
                                $text = "";
                        } else {
-                               $q = preg_split( "/<\\/\\s*$tag\\s*>/i", $p[1], 2 );
+                               if($tag==STRIP_COMMENTS) {
+                                       $q = preg_split( "/-->/i", $p[1], 2 );
+                               } else {
+                                       $q = preg_split( "/<\\/\\s*$tag\\s*>/i", $p[1], 2 );
+                               }
                                $marker = $rnd . sprintf("%08X", $n++);
                                $content[$marker] = $q[0];
                                $stripped .= $marker;
@@ -148,18 +178,23 @@ class Parser
                return $stripped;
        }
 
-       # Strips <nowiki>, <pre> and <math>
+       # Strips and renders <nowiki>, <pre>, <math>, <hiero>
+       # If $render is set, performs necessary rendering operations on plugins
        # Returns the text, and fills an array with data needed in unstrip()
        # If the $state is already a valid strip state, it adds to the state
-       #
-       function strip( $text, &$state )
+
+       # When $stripcomments is set, HTML comments <!-- like this -->
+       # will be stripped in addition to other tags. This is important
+       # for section editing, where these comments cause confusion when
+       # counting the sections in the wikisource
+       function strip( $text, &$state, $stripcomments = false )
        {
                $render = ($this->mOutputType == OT_HTML);
                $nowiki_content = array();
                $hiero_content = array();
                $math_content = array();
                $pre_content = array();
-               $item_content = array();
+               $comment_content = array();
 
                # Replace any instances of the placeholders
                $uniq_prefix = UNIQ_PREFIX;
@@ -174,25 +209,25 @@ class Parser
                        }
                }
 
-               if( $GLOBALS['wgUseWikiHiero'] ){
-                       $text = Parser::extractTags("hiero", $text, $hiero_content, $uniq_prefix);
-                       foreach( $hiero_content as $marker => $content ){
-                               if( $render ){
-                                       $hiero_content[$marker] = WikiHiero( $content, WH_MODE_HTML);
-                               } else {
-                                       $hiero_content[$marker] = "<hiero>$content</hiero>";
-                               }
+               $text = Parser::extractTags("hiero", $text, $hiero_content, $uniq_prefix);
+               foreach( $hiero_content as $marker => $content ){
+                       if( $render && $GLOBALS['wgUseWikiHiero']){
+                               $hiero_content[$marker] = WikiHiero( $content, WH_MODE_HTML);
+                       } else {
+                               $hiero_content[$marker] = "<hiero>$content</hiero>";
                        }
                }
 
-               if( $this->mOptions->getUseTeX() ){
-                       $text = Parser::extractTags("math", $text, $math_content, $uniq_prefix);
-                       foreach( $math_content as $marker => $content ){
-                               if( $render ){
+               $text = Parser::extractTags("math", $text, $math_content, $uniq_prefix);
+               foreach( $math_content as $marker => $content ){
+                       if( $render ) {
+                               if( $this->mOptions->getUseTeX() ) {
                                        $math_content[$marker] = renderMath( $content );
                                } else {
-                                       $math_content[$marker] = "<math>$content</math>";
+                                       $math_content[$marker] = "&lt;math&gt;$content&lt;math&gt;";
                                }
+                       } else {
+                               $math_content[$marker] = "<math>$content</math>";
                        }
                }
 
@@ -204,6 +239,12 @@ class Parser
                                $pre_content[$marker] = "<pre>$content</pre>";
                        }
                }
+               if($stripcomments) {
+                       $text = Parser::extractTags(STRIP_COMMENTS, $text, $comment_content, $uniq_prefix);
+                       foreach( $comment_content as $marker => $content ){
+                               $comment_content[$marker] = "<!--$content-->";
+                       }
+               }
 
                # Merge state with the pre-existing state, if there is one
                if ( $state ) {
@@ -211,13 +252,14 @@ class Parser
                        $state['hiero'] = $state['hiero'] + $hiero_content;
                        $state['math'] = $state['math'] + $math_content;
                        $state['pre'] = $state['pre'] + $pre_content;
+                       $state['comment'] = $state['comment'] + $comment_content;
                } else {
                        $state = array(
                          'nowiki' => $nowiki_content,
                          'hiero' => $hiero_content,
                          'math' => $math_content,
                          'pre' => $pre_content,
-                         'item' => $item_content
+                         'comment' => $comment_content
                        );
                }
                return $text;
@@ -248,8 +290,7 @@ class Parser
                          'nowiki' => array(),
                          'hiero' => array(),
                          'math' => array(),
-                         'pre' => array(),
-                         'item' => array()
+                         'pre' => array()
                        );
                }
                $state['item'][$rnd] = $text;
@@ -365,6 +406,47 @@ class Parser
                return trim ( $t ) ;
        }
 
+       /* interface with html tidy, used if $wgUseTidy = true */
+       function tidy ( $text ) {
+               global $wgTidyConf, $wgTidyBin, $wgTidyOpts;
+               global $wgInputEncoding, $wgOutputEncoding;
+               $cleansource = '';
+               switch(strtoupper($wgOutputEncoding)) {
+                       case 'ISO-8859-1':
+                               $wgTidyOpts .= ($wgInputEncoding == $wgOutputEncoding)? ' -latin1':' -raw';
+                               break;
+                       case 'UTF-8':
+                               $wgTidyOpts .= ($wgInputEncoding == $wgOutputEncoding)? ' -utf8':' -raw';
+                               break;
+                       default:
+                               $wgTidyOpts .= ' -raw';
+                       }
+
+               $text = '<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"'.
+' "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"><html>'.
+'<head><title>test</title></head><body>'.$text.'</body></html>';
+               $descriptorspec = array(
+                       0 => array("pipe", "r"),
+                       1 => array("pipe", "w"),
+                       2 => array("file", "/dev/null", "a")
+               );
+               $process = proc_open("$wgTidyBin -config $wgTidyConf $wgTidyOpts", $descriptorspec, $pipes);
+               if (is_resource($process)) {
+                       fwrite($pipes[0], $text);
+                       fclose($pipes[0]);
+                       while (!feof($pipes[1])) {
+                               $cleansource .= fgets($pipes[1], 1024);
+                       }
+                       fclose($pipes[1]);
+                       $return_value = proc_close($process);
+               }
+               if( $cleansource == '' && $text != '') {
+                       return '<h2>'.wfMsg('seriousxhtmlerrors').'</h2><pre>'.htmlspecialchars($text).'</pre>';
+               } else {
+                       return $cleansource;
+               }
+       }
+
        function doTableStuff ( $t )
        {
                $t = explode ( "\n" , $t ) ;
@@ -465,7 +547,9 @@ class Parser
                return $t ;
        }
 
-       function internalParse( $text, $linestart, $args = array() )
+       // set isMain=false if you call from a template etc. and don't want to do stuff
+       // like TOC insertion for that content
+       function internalParse( $text, $linestart, $args = array(), $isMain=true )
        {
                $fname = "Parser::internalParse";
                wfProfileIn( $fname );
@@ -483,7 +567,7 @@ class Parser
                $text = $this->replaceExternalLinks( $text );
                $text = $this->doTokenizedParser ( $text );
                $text = $this->doTableStuff ( $text ) ;
-               $text = $this->formatHeadings( $text );
+               $text = $this->formatHeadings( $text, $isMain );
                $sk =& $this->mOptions->getSkin();
                $text = $sk->transformContent( $text );
 
@@ -658,6 +742,7 @@ class Parser
        /* private */ function doTokenizedParser( $str )
        {
                global $wgLang; # for language specific parser hook
+               global $wgUploadDirectory, $wgUseTimeline;
 
                $tokenizer=Tokenizer::newFromString( $str );
                $tokenStack = array();
@@ -683,7 +768,7 @@ class Parser
                                        # non-breakable spaces for French.
                                        # U+202F NARROW NO-BREAK SPACE might be a better choice, but
                                        # browser support for Unicode spacing is poor.
-                                       $txt = str_replace( " ", "<u>&nbsp;</u>", $token["text"] );
+                                       $txt = str_replace( " ", "&nbsp;", $token["text"] );
                                        break;
                                case "[[[":
                                        # remember the tag opened with 3 [
@@ -774,6 +859,15 @@ class Parser
                                                $txt = $this->doMagicISBN( $tokenizer );
                                        }
                                        break;
+                               case "<timeline>":
+                                       if ( $wgUseTimeline && 
+                                            "" != ( $timelinesrc = $tokenizer->readAllUntil("&lt;/timeline&gt;") ) )
+                                       {
+                                               $txt = renderTimeline( $timelinesrc );
+                                       } else {
+                                               $txt=$token["text"];
+                                       }
+                                       break;
                                default:
                                        # Call language specific Hook.
                                        $txt = $wgLang->processToken( $token, $tokenStack );
@@ -842,7 +936,7 @@ class Parser
                if ( !$image ) { $image = Namespace::getImage(); }
                if ( !$special ) { $special = Namespace::getSpecial(); }
                if ( !$media ) { $media = Namespace::getMedia(); }
-               if ( !$category ) { $category = Namespace::getCategory(); }
+               if ( !$category ) { $category = Namespace::getCategory(); }
 
                $nottalk = !Namespace::isTalk( $this->mTitle->getNamespace() );
 
@@ -898,6 +992,7 @@ class Parser
                if( $noforce ) {
                        if( $iw && $this->mOptions->getInterwikiMagic() && $nottalk && $wgLang->getLanguageName( $iw ) ) {
                                array_push( $this->mOutput->mLanguageLinks, $nt->getPrefixedText() );
+                               $s .= $prefix . $trail ;
                                return (trim($s) == '')? '': $s;
                        }
                        if( $ns == $image ) {
@@ -1013,81 +1108,94 @@ class Parser
                return $text."\n";
        }
 
-       /* private */ function doBlockLevels( $text, $linestart )
-       {
+       /* private */ function doBlockLevels( $text, $linestart ) {
                $fname = "Parser::doBlockLevels";
                wfProfileIn( $fname );
+               
                # Parsing through the text line by line.  The main thing
                # happening here is handling of block-level elements p, pre,
                # and making lists from lines starting with * # : etc.
                #
-               $a = explode( "\n", $text );
+               $textLines = explode( "\n", $text );
 
-               $lastPref = $text = $lastLine = '';
+               $lastPrefix = $output = $lastLine = '';
                $this->mDTopen = $inBlockElem = false;
-               $npl = 0;
-               $pstack = false;
-
-               if ( ! $linestart ) { $text .= array_shift( $a ); }
-               foreach ( $a as $t ) {
-                       $oLine = $t;
-                       $opl = strlen( $lastPref );
-                       $preCloseMatch = preg_match("/<\\/pre/i", $t );
-                       $preOpenMatch = preg_match("/<pre/i", $t );
+               $prefixLength = 0;
+               $paragraphStack = false;
+
+               if ( !$linestart ) {
+                       $output .= array_shift( $textLines );
+               }
+               foreach ( $textLines as $oLine ) {
+                       $lastPrefixLength = strlen( $lastPrefix );
+                       $preCloseMatch = preg_match("/<\\/pre/i", $oLine );
+                       $preOpenMatch = preg_match("/<pre/i", $oLine );
                        if (!$this->mInPre) {
                                $this->mInPre = !empty($preOpenMatch);
                        }
                        if ( !$this->mInPre ) {
-                               $npl = strspn( $t, "*#:;" );
-                               $pref = substr( $t, 0, $npl );
+                               # Multiple prefixes may abut each other for nested lists.
+                               $prefixLength = strspn( $oLine, "*#:;" );
+                               $pref = substr( $oLine, 0, $prefixLength );
+                               
+                               # eh?
                                $pref2 = str_replace( ";", ":", $pref );
-                               $t = substr( $t, $npl );
+                               $t = substr( $oLine, $prefixLength );
                        } else {
-                               $npl = 0;
+                               # Don't interpret any other prefixes in preformatted text
+                               $prefixLength = 0;
                                $pref = $pref2 = '';
+                               $t = $oLine;
                        }
 
-                       // list generation
-                       if ( 0 != $npl && 0 == strcmp( $lastPref, $pref2 ) ) {
-                               $text .= $this->nextItem( substr( $pref, -1 ) );
-                               if ( $pstack ) { $pstack = false; }
+                       # List generation
+                       if( $prefixLength && 0 == strcmp( $lastPrefix, $pref2 ) ) {
+                               # Same as the last item, so no need to deal with nesting or opening stuff
+                               $output .= $this->nextItem( substr( $pref, -1 ) );
+                               $paragraphStack = false;
 
                                if ( ";" == substr( $pref, -1 ) ) {
-                                       $cpos = strpos( $t, ":" );
-                                       if ( false !== $cpos ) {
-                                               $term = substr( $t, 0, $cpos );
-                                               $text .= $term . $this->nextItem( ":" );
-                                               $t = substr( $t, $cpos + 1 );
+                                       # The one nasty exception: definition lists work like this:
+                                       # ; title : definition text
+                                       # So we check for : in the remainder text to split up the
+                                       # title and definition, without b0rking links.
+                                       # FIXME: This is not foolproof. Something better in Tokenizer might help.
+                                       if( preg_match( '/^(.*?(?:\s|&nbsp;)):(.*)$/', $t, $match ) ) {
+                                               $term = $match[1];
+                                               $output .= $term . $this->nextItem( ":" );
+                                               $t = $match[2];
                                        }
                                }
-                       } else if (0 != $npl || 0 != $opl) {
-                               $cpl = $this->getCommon( $pref, $lastPref );
-                               if ( $pstack ) { $pstack = false; }
-
-                               while ( $cpl < $opl ) {
-                                       $text .= $this->closeList( $lastPref{$opl-1} );
-                                       --$opl;
+                       } elseif( $prefixLength || $lastPrefixLength ) {
+                               # Either open or close a level...
+                               $commonPrefixLength = $this->getCommon( $pref, $lastPrefix );
+                               $paragraphStack = false;
+
+                               while( $commonPrefixLength < $lastPrefixLength ) {
+                                       $output .= $this->closeList( $lastPrefix{$lastPrefixLength-1} );
+                                       --$lastPrefixLength;
                                }
-                               if ( $npl <= $cpl && $cpl > 0 ) {
-                                       $text .= $this->nextItem( $pref{$cpl-1} );
+                               if ( $prefixLength <= $commonPrefixLength && $commonPrefixLength > 0 ) {
+                                       $output .= $this->nextItem( $pref{$commonPrefixLength-1} );
                                }
-                               while ( $npl > $cpl ) {
-                                       $char = substr( $pref, $cpl, 1 );
-                                       $text .= $this->openList( $char );
+                               while ( $prefixLength > $commonPrefixLength ) {
+                                       $char = substr( $pref, $commonPrefixLength, 1 );
+                                       $output .= $this->openList( $char );
 
                                        if ( ";" == $char ) {
-                                               $cpos = strpos( $t, ":" );
-                                               if ( ! ( false === $cpos ) ) {
-                                                       $term = substr( $t, 0, $cpos );
-                                                       $text .= $term . $this->nextItem( ":" );
-                                                       $t = substr( $t, $cpos + 1 );
+                                               # FIXME: This is dupe of code above
+                                               if( preg_match( '/^(.*?(?:\s|&nbsp;)):(.*)$/', $t, $match ) ) {
+                                                       $term = $match[1];
+                                                       $output .= $term . $this->nextItem( ":" );
+                                                       $t = $match[2];
                                                }
                                        }
-                                       ++$cpl;
+                                       ++$commonPrefixLength;
                                }
-                               $lastPref = $pref2;
+                               $lastPrefix = $pref2;
                        }
-                       if ( 0 == $npl ) { # No prefix (not in list)--go to paragraph mode
+                       if( 0 == $prefixLength ) {
+                               # No prefix (not in list)--go to paragraph mode
                                $uniq_prefix = UNIQ_PREFIX;
                                // XXX: use a stack for nestable elements like span, table and div
                                $openmatch = preg_match("/(<table|<blockquote|<h1|<h2|<h3|<h4|<h5|<h6|<div|<pre|<tr|<td|<p|<ul|<li)/i", $t );
@@ -1095,8 +1203,8 @@ class Parser
                                        "/(<\\/table|<\\/blockquote|<\\/h1|<\\/h2|<\\/h3|<\\/h4|<\\/h5|<\\/h6|".
                                        "<\\/div|<hr|<\\/td|<\\/pre|<\\/p|".$uniq_prefix."-pre|<\\/li|<\\/ul)/i", $t );
                                if ( $openmatch or $closematch ) {
-                                       if ( $pstack ) { $pstack = false; }
-                                       $text .= $this->closeParagraph();
+                                       $paragraphStack = false;
+                                       $output .= $this->closeParagraph();
                                        if($preOpenMatch and !$preCloseMatch) {
                                                $this->mInPre = true;   
                                        }
@@ -1109,54 +1217,54 @@ class Parser
                                        if ( " " == $t{0} ) {
                                                // pre
                                                if ($this->mLastSection != 'pre') {
-                                                       $pstack = false;
-                                                       $text .= $this->closeParagraph().'<pre>';
+                                                       $paragraphStack = false;
+                                                       $output .= $this->closeParagraph().'<pre>';
                                                        $this->mLastSection = 'pre';
                                                }
                                        } else {
                                                // paragraph
                                                if ( '' == trim($t) ) {
-                                                       if ( $pstack ) {
-                                                               $text .= $pstack.'<br/>';
-                                                               $pstack = false;
+                                                       if ( $paragraphStack ) {
+                                                               $output .= $paragraphStack.'<br/>';
+                                                               $paragraphStack = false;
                                                                $this->mLastSection = 'p';
                                                        } else {
                                                                if ($this->mLastSection != 'p' ) {
-                                                                       $text .= $this->closeParagraph();
+                                                                       $output .= $this->closeParagraph();
                                                                        $this->mLastSection = '';
-                                                                       $pstack = "<p>";
+                                                                       $paragraphStack = "<p>";
                                                                } else {
-                                                                       $pstack = '</p><p>';
+                                                                       $paragraphStack = '</p><p>';
                                                                }
                                                        }
                                                } else {
-                                                       if ( $pstack ) {
-                                                               $text .= $pstack;
-                                                               $pstack = false;
+                                                       if ( $paragraphStack ) {
+                                                               $output .= $paragraphStack;
+                                                               $paragraphStack = false;
                                                                $this->mLastSection = 'p';
                                                        } else if ($this->mLastSection != 'p') {
-                                                               $text .= $this->closeParagraph().'<p>';
+                                                               $output .= $this->closeParagraph().'<p>';
                                                                $this->mLastSection = 'p';
                                                        }
                                                }
                                        }
                                }
                        }
-                       if ($pstack === false) {
-                               $text .= $t."\n";
+                       if ($paragraphStack === false) {
+                               $output .= $t."\n";
                        }
                }
-               while ( $npl ) {
-                       $text .= $this->closeList( $pref2{$npl-1} );
-                       --$npl;
+               while ( $prefixLength ) {
+                       $output .= $this->closeList( $pref2{$prefixLength-1} );
+                       --$prefixLength;
                }
                if ( "" != $this->mLastSection ) {
-                       $text .= "</" . $this->mLastSection . ">";
+                       $output .= "</" . $this->mLastSection . ">";
                        $this->mLastSection = "";
                }
 
                wfProfileOut( $fname );
-               return $text;
+               return $output;
        }
 
        function getVariableValue( $index ) {
@@ -1171,6 +1279,11 @@ class Parser
                                return $wgLang->getMonthNameGen( date("n") );
                        case MAG_CURRENTDAY:
                                return date("j");
+                       case MAG_PAGENAME:
+                               return $this->mTitle->getText();
+                       case MAG_NAMESPACE:
+                               # return Namespace::getCanonicalName($this->mTitle->getNamespace());
+                               return $wgLang->getNsText($this->mTitle->getNamespace()); // Patch  by Dori
                        case MAG_CURRENTDAYNAME:
                                return $wgLang->getWeekdayName( date("w")+1 );
                        case MAG_CURRENTYEAR:
@@ -1396,7 +1509,8 @@ class Parser
 
                        # Run full parser on the included text
                        $text = $this->strip( $text, $this->mStripState );
-                       $text = $this->internalParse( $text, (bool)$newline, $assocArgs );
+                       $text = $this->internalParse( $text, (bool)$newline, $assocArgs, false );
+                       if(!empty($newline)) $text = "\n".$text;
 
                        # Add the result to the strip state for re-inclusion after
                        # the rest of the processing
@@ -1412,7 +1526,7 @@ class Parser
                if ( !$found ) {
                        return $matches[0];
                } else {
-                       return $newline . $text;
+                       return $text;
                }
        }
 
@@ -1433,25 +1547,34 @@ class Parser
        # Cleans up HTML, removes dangerous tags and attributes
        /* private */ function removeHTMLtags( $text )
        {
+               global $wgUseTidy, $wgUserHtml;
                $fname = "Parser::removeHTMLtags";
                wfProfileIn( $fname );
-               $htmlpairs = array( # Tags that must be closed
-                       "b", "del", "i", "ins", "u", "font", "big", "small", "sub", "sup", "h1",
-                       "h2", "h3", "h4", "h5", "h6", "cite", "code", "em", "s",
-                       "strike", "strong", "tt", "var", "div", "center",
-                       "blockquote", "ol", "ul", "dl", "table", "caption", "pre",
-                       "ruby", "rt" , "rb" , "rp", "p"
-               );
-               $htmlsingle = array(
-                       "br", "hr", "li", "dt", "dd"
-               );
-               $htmlnest = array( # Tags that can be nested--??
-                       "table", "tr", "td", "th", "div", "blockquote", "ol", "ul",
-                       "dl", "font", "big", "small", "sub", "sup"
-               );
-               $tabletags = array( # Can only appear inside table
-                       "td", "th", "tr"
-               );
+               
+               if( $wgUserHtml ) {
+                       $htmlpairs = array( # Tags that must be closed
+                               "b", "del", "i", "ins", "u", "font", "big", "small", "sub", "sup", "h1",
+                               "h2", "h3", "h4", "h5", "h6", "cite", "code", "em", "s",
+                               "strike", "strong", "tt", "var", "div", "center",
+                               "blockquote", "ol", "ul", "dl", "table", "caption", "pre",
+                               "ruby", "rt" , "rb" , "rp", "p"
+                       );
+                       $htmlsingle = array(
+                               "br", "hr", "li", "dt", "dd"
+                       );
+                       $htmlnest = array( # Tags that can be nested--??
+                               "table", "tr", "td", "th", "div", "blockquote", "ol", "ul",
+                               "dl", "font", "big", "small", "sub", "sup"
+                       );
+                       $tabletags = array( # Can only appear inside table
+                               "td", "th", "tr"
+                       );
+               } else {
+                       $htmlpairs = array();
+                       $htmlsingle = array();
+                       $htmlnest = array();
+                       $tabletags = array();
+               }
 
                $htmlsingle = array_merge( $tabletags, $htmlsingle );
                $htmlelements = array_merge( $htmlsingle, $htmlpairs );
@@ -1459,70 +1582,86 @@ class Parser
                $htmlattrs = $this->getHTMLattrs () ;
 
                # Remove HTML comments
-               $text = preg_replace( "/<!--.*-->/sU", "", $text );
+               $text = preg_replace( "/(\\n *<!--.*--> *(?=\\n)|<!--.*-->)/sU", "$2", $text );
 
                $bits = explode( "<", $text );
                $text = array_shift( $bits );
-               $tagstack = array(); $tablestack = array();
-
-               foreach ( $bits as $x ) {
-                       $prev = error_reporting( E_ALL & ~( E_NOTICE | E_WARNING ) );
-                       preg_match( "/^(\\/?)(\\w+)([^>]*)(\\/{0,1}>)([^<]*)$/",
-                         $x, $regs );
-                       list( $qbar, $slash, $t, $params, $brace, $rest ) = $regs;
-                       error_reporting( $prev );
-
-                       $badtag = 0 ;
-                       if ( in_array( $t = strtolower( $t ), $htmlelements ) ) {
-                               # Check our stack
-                               if ( $slash ) {
-                                       # Closing a tag...
-                                       if ( ! in_array( $t, $htmlsingle ) &&
-                                         ( $ot = array_pop( $tagstack ) ) != $t ) {
-                                               array_push( $tagstack, $ot );
-                                               $badtag = 1;
+               if(!$wgUseTidy) {
+                       $tagstack = array(); $tablestack = array();
+                       foreach ( $bits as $x ) {
+                               $prev = error_reporting( E_ALL & ~( E_NOTICE | E_WARNING ) );
+                               preg_match( "/^(\\/?)(\\w+)([^>]*)(\\/{0,1}>)([^<]*)$/",
+                               $x, $regs );
+                               list( $qbar, $slash, $t, $params, $brace, $rest ) = $regs;
+                               error_reporting( $prev );
+
+                               $badtag = 0 ;
+                               if ( in_array( $t = strtolower( $t ), $htmlelements ) ) {
+                                       # Check our stack
+                                       if ( $slash ) {
+                                               # Closing a tag...
+                                               if ( ! in_array( $t, $htmlsingle ) &&
+                                               ( count($tagstack) && $ot = array_pop( $tagstack ) ) != $t ) {
+                                                       if(!empty($ot)) array_push( $tagstack, $ot );
+                                                       $badtag = 1;
+                                               } else {
+                                                       if ( $t == "table" ) {
+                                                               $tagstack = array_pop( $tablestack );
+                                                       }
+                                                       $newparams = "";
+                                               }
                                        } else {
-                                               if ( $t == "table" ) {
-                                                       $tagstack = array_pop( $tablestack );
+                                               # Keep track for later
+                                               if ( in_array( $t, $tabletags ) &&
+                                               ! in_array( "table", $tagstack ) ) {
+                                                       $badtag = 1;
+                                               } else if ( in_array( $t, $tagstack ) &&
+                                               ! in_array ( $t , $htmlnest ) ) {
+                                                       $badtag = 1 ;
+                                               } else if ( ! in_array( $t, $htmlsingle ) ) {
+                                                       if ( $t == "table" ) {
+                                                               array_push( $tablestack, $tagstack );
+                                                               $tagstack = array();
+                                                       }
+                                                       array_push( $tagstack, $t );
                                                }
-                                               $newparams = "";
+                                               # Strip non-approved attributes from the tag
+                                               $newparams = $this->fixTagAttributes($params);
+
                                        }
-                               } else {
-                                       # Keep track for later
-                                       if ( in_array( $t, $tabletags ) &&
-                                         ! in_array( "table", $tagstack ) ) {
-                                               $badtag = 1;
-                                       } else if ( in_array( $t, $tagstack ) &&
-                                         ! in_array ( $t , $htmlnest ) ) {
-                                               $badtag = 1 ;
-                                       } else if ( ! in_array( $t, $htmlsingle ) ) {
-                                               if ( $t == "table" ) {
-                                                       array_push( $tablestack, $tagstack );
-                                                       $tagstack = array();
-                                               }
-                                               array_push( $tagstack, $t );
+                                       if ( ! $badtag ) {
+                                               $rest = str_replace( ">", "&gt;", $rest );
+                                               $text .= "<$slash$t $newparams$brace$rest";
+                                               continue;
                                        }
-                                       # Strip non-approved attributes from the tag
-                                       $newparams = $this->fixTagAttributes($params);
-
                                }
-                               if ( ! $badtag ) {
+                               $text .= "&lt;" . str_replace( ">", "&gt;", $x);
+                       }
+                       # Close off any remaining tags
+                       while ( $t = array_pop( $tagstack ) ) {
+                               $text .= "</$t>\n";
+                               if ( $t == "table" ) { $tagstack = array_pop( $tablestack ); }
+                       }
+               } else {
+                       # this might be possible using tidy itself
+                       foreach ( $bits as $x ) {
+                               preg_match( "/^(\\/?)(\\w+)([^>]*)(\\/{0,1}>)([^<]*)$/",
+                               $x, $regs );
+                               @list( $qbar, $slash, $t, $params, $brace, $rest ) = $regs;
+                               if ( in_array( $t = strtolower( $t ), $htmlelements ) ) {
+                                       $newparams = $this->fixTagAttributes($params);
                                        $rest = str_replace( ">", "&gt;", $rest );
                                        $text .= "<$slash$t $newparams$brace$rest";
-                                       continue;
+                               } else {
+                                       $text .= "&lt;" . str_replace( ">", "&gt;", $x);
                                }
-                       }
-                       $text .= "&lt;" . str_replace( ">", "&gt;", $x);
-               }
-               # Close off any remaining tags
-               while ( $t = array_pop( $tagstack ) ) {
-                       $text .= "</$t>\n";
-                       if ( $t == "table" ) { $tagstack = array_pop( $tablestack ); }
+                       }       
                }
                wfProfileOut( $fname );
                return $text;
        }
 
+
 /*
  *
  * This function accomplishes several tasks:
@@ -1536,8 +1675,12 @@ class Parser
  *
  */
 
-       /* private */ function formatHeadings( $text )
+       /* private */ function formatHeadings( $text, $isMain=true )
        {
+               global $wgInputEncoding,$wgRequest,$wgOut;
+               
+               $startsection=$wgRequest->getVal('section');    
+               if($startsection) { $startsection--;}
                $doNumberHeadings = $this->mOptions->getNumberHeadings();
                $doShowToc = $this->mOptions->getShowToc();
                if( !$this->mTitle->userCanEdit() ) {
@@ -1638,7 +1781,7 @@ class Parser
                        # strip out HTML
                        $canonized_headline = preg_replace( "/<.*?" . ">/","",$canonized_headline );
                        $tocline = trim( $canonized_headline );
-                       $canonized_headline = preg_replace("/[ \\?&\\/<>\\(\\)\\[\\]=,+']+/", '_', html_entity_decode( $tocline));
+                       $canonized_headline = preg_replace("/[ \\?&\\/<>\\(\\)\\[\\]=,+']+/", '_', urlencode( do_html_entity_decode( $tocline, ENT_COMPAT, $wgInputEncoding ) ) );
                        $refer[$headlineCount] = $canonized_headline;
 
                        # count how many in assoc. array so we can track dupes in anchors
@@ -1669,12 +1812,12 @@ class Parser
                                if ( empty( $head[$headlineCount] ) ) {
                                        $head[$headlineCount] = "";
                                }
-                               $head[$headlineCount] .= $sk->editSectionLink($headlineCount+1);
+                               $head[$headlineCount] .= $sk->editSectionLink($startsection+$headlineCount+1);
                        }
 
                        # Add the edit section span
                        if( $rightClickHack ) {
-                               $headline = $sk->editSectionScript($headlineCount+1,$headline);
+                               $headline = $sk->editSectionScript($startsection+$headlineCount+1,$headline);
                        }
 
                        # give headline the correct <h#> tag
@@ -1708,6 +1851,14 @@ class Parser
                        # Top anchor now in skin
                                $full = $full.$toc;
                        }
+                       
+                       # If a page is viewed in collapsed mode, a TOC generated
+                       # from the wikisource is stored in the title object.
+                       # This TOC is now fetched and inserted here if it exists.
+                       $collapsedtoc=$wgOut->getToc();
+                       if ($collapsedtoc && !$i && $isMain) {
+                               $full = $full.$collapsedtoc;            
+                       }                       
 
                        if( !empty( $head[$i] ) ) {
                                $full .= $head[$i];
@@ -1718,6 +1869,115 @@ class Parser
                return $full;
        }
 
+       /* Generates a HTML-formatted table of contents which links to individual sections 
+          from the wikisource. Used for collapsing long pages.
+        */        
+       /* static */ function getTocFromSource( $text ) {               
+               
+               global $wgUser,$wgInputEncoding,$wgTitle,$wgOut,$wgParser;              
+               $sk = $wgUser->getSkin();               
+               
+               $striparray=array();
+               $oldtype=$wgParser->mOutputType;
+               $wgParser->mOutputType=OT_WIKI;
+               $text=$wgParser->strip($text, $striparray, true);
+               $wgParser->mOutputType=$oldtype;        
+               
+               $numMatches = preg_match_all( "/^(=+)(.*?)=+|^<h([1-6]).*?>(.*?)<\/h[1-6].*?>/mi",$text,$matches);
+               
+               # no headings: text cannot be collapsed
+               if( $numMatches == 0 ) {
+                       return "";
+               }
+               
+               # We combine the headlines into a bundle and convert them to HTML
+               # in order to make stripping out the wikicrap easier.
+               $combined=implode("!@@@!",$matches[2]);
+               $myout=$wgParser->parse($combined,$wgTitle,$wgOut->mParserOptions);                     
+               $combined_html=$myout->getText();               
+               $headlines=array();
+               $headlines=explode("!@@@!",$combined_html);
+               
+               # headline counter
+               $headlineCount = 0;             
+               $toclevel = 0;
+               $toc = "";
+               $full = "";
+               $head = array();
+               $sublevelCount = array();
+               $level = 0;
+               $prevlevel = 0;
+               foreach( $headlines as $headline ) {                    
+                       $headline=trim($headline);
+                       $numbering = "";
+                       if( $level ) {
+                               $prevlevel = $level;
+                       }
+                       $level = $matches[1][$headlineCount];
+                       
+                       # wikisource headings need to be converted into numbers
+                       # =foo= equals <h1>foo</h1>, ==foo== equals <h2>foo</h2> etc.
+                       if(strpos($level,"=")!==false) {
+                               $level=strlen($level);                  
+                       }
+                       
+                       if(  $prevlevel && $level > $prevlevel ) {
+                               # reset when we enter a new level
+                               $sublevelCount[$level] = 0;
+                               $toc .= $sk->tocIndent( $level - $prevlevel );
+                               $toclevel += $level - $prevlevel;
+                       }
+                       if( $level < $prevlevel ) {
+                               # reset when we step back a level
+                               $sublevelCount[$level+1]=0;
+                               $toc .= $sk->tocUnindent( $prevlevel - $level );
+                               $toclevel -= $prevlevel - $level;
+                       }
+                       # count number of headlines for each level
+                       @$sublevelCount[$level]++;                      
+                       $dot = 0;
+                       for( $i = 1; $i <= $level; $i++ ) {
+                               if( !empty( $sublevelCount[$i] ) ) {
+                                       if( $dot ) {
+                                               $numbering .= ".";
+                                       }
+                                       $numbering .= $sublevelCount[$i];
+                                       $dot = 1;
+                               }
+                       }
+                       
+
+                       # The canonized header is a version of the header text safe to use for links
+                       # Avoid insertion of weird stuff like <math> by expanding the relevant sections
+                       $state=array();
+                       $canonized_headline = Parser::unstrip( $headline, $state);                      
+                       
+                       # strip out HTML
+                       $canonized_headline = preg_replace( "/<.*?" . ">/","",$canonized_headline );
+                       $tocline = trim( $canonized_headline );
+                       $canonized_headline = preg_replace("/[ \\?&\\/<>\\(\\)\\[\\]=,+']+/", '_', urlencode( do_html_entity_decode( $tocline, ENT_COMPAT, $wgInputEncoding ) ) );
+                       $refer[$headlineCount] = $canonized_headline;
+
+                       # count how many in assoc. array so we can track dupes in anchors
+                       @$refers[$canonized_headline]++;
+                       $refcount[$headlineCount]=$refers[$canonized_headline];
+                       $tocline = $numbering . " " . $tocline;
+
+                       # Create the anchor for linking from the TOC to the section
+                       $anchor = trim($canonized_headline);
+                       
+                       if($refcount[$headlineCount] > 1 ) {
+                               $anchor .= "_" . $refcount[$headlineCount];
+                       }                       
+                       $headlineCount++;
+                       $toc .= $sk->tocLine($anchor,$tocline,$toclevel,$headlineCount);
+               }
+               $toclines = $headlineCount;
+               $toc .= $sk->tocUnindent( $toclevel );
+               $toc = $sk->tocTable( $toc );
+               return $toc;
+       
+       }
        /* private */ function doMagicISBN( &$tokenizer )
        {
                global $wgLang;