Copy querycache: encoding and a table definition change
[lhc/web/wiklou.git] / maintenance / parserTests.php
index a6bcb33..adb341b 100644 (file)
  */
 
 /** */
-$optionsWithArgs = array('regex');
+$options = array( 'quick', 'color', 'quiet', 'help' );
+$optionsWithArgs = array( 'regex' );
 
 require_once( 'commandLine.inc' );
-require_once( 'languages/LanguageUtf8.php' );
+require_once( "$IP/includes/ObjectCache.php" );
+require_once( "$IP/includes/BagOStuff.php" );
+require_once( "$IP/languages/LanguageUtf8.php" );
 
-/** */
+/**
+ * @package MediaWiki
+ * @subpackage Maintenance
+ */
 class ParserTest {
        /**
         * boolean $color whereas output should be colorized
@@ -52,33 +58,33 @@ class ParserTest {
         */
        function ParserTest() {
                global $options;
+               
+               # Only colorize output if stdout is a terminal.
                $this->lightcolor = false;
-               if( isset( $_SERVER['argv'] ) && in_array( '--color', $_SERVER['argv'] ) ) {
-                       $this->color = true;
-               } elseif( isset( $_SERVER['argv'] ) && in_array( '--color=yes', $_SERVER['argv'] ) ) {
-                       $this->color = true;
-               } elseif( isset( $_SERVER['argv'] ) && in_array( '--color=light', $_SERVER['argv'] ) ) {
-                       $this->color = true;
-                       $this->lightcolor = true;
-               } elseif( isset( $_SERVER['argv'] ) && in_array( '--color=no', $_SERVER['argv'] ) ) {
-                       $this->color = false;
-               } elseif( wfIsWindows() ) {
-                       $this->color = false;
-               } else {
-                       # Only colorize output if stdout is a terminal.
-                       $this->color = posix_isatty(1);
-               }
+               $this->color = !wfIsWindows() && posix_isatty(1);
                
-               if( isset( $_SERVER['argv'] ) && in_array( '--quick', $_SERVER['argv'] ) ) {
-                       $this->showDiffs = false;
-               } else {
-                       $this->showDiffs = true;
+               if( isset( $options['color'] ) ) {
+                       switch( $options['color'] ) {
+                       case 'no':
+                               $this->color = false;
+                               break;
+                       case 'light':
+                               $this->lightcolor = true;
+                               # Fall through
+                       case 'yes':
+                       default:
+                               $this->color = true;
+                               break;
+                       }
                }
+               
+               $this->showDiffs = !isset( $options['quick'] );
+               
+               $this->quiet = isset( $options['quiet'] );
 
                if (isset($options['regex'])) {
                        $this->regex = $options['regex'];
-               }
-               else {
+               } else {
                        # Matches anything
                        $this->regex = '';
                }
@@ -182,7 +188,7 @@ class ParserTest {
                        }
                }
                if( $total > 0 ) {
-                       $ratio = IntVal( 100.0 * $success / $total );
+                       $ratio = sprintf( "%.2f", 100 * $success / $total );
                        print $this->termColor( 1 ) . "\nPassed $success of $total tests ($ratio%) ";
                        if( $success == $total ) {
                                print $this->termColor( 32 ) . "PASSED!";
@@ -206,7 +212,9 @@ class ParserTest {
         * @return bool
         */
        function runTest( $desc, $input, $result, $opts ) {
-               print "Running test $desc... ";
+               if( !$this->quiet ) {
+                       $this->showTesting( $desc );
+               }
 
                $this->setupGlobals($opts);
 
@@ -232,33 +240,24 @@ class ParserTest {
 
                if (preg_match('/\\bpst\\b/i', $opts)) {
                        $out = $parser->preSaveTransform( $input, $title, $user, $options );
-               }
-               else if (preg_match('/\\bmsg\\b/i', $opts)) {
+               } elseif (preg_match('/\\bmsg\\b/i', $opts)) {
                        $out = $parser->transformMsg( $input, $options );
-               }
-               else {
+               } else {
                        $output =& $parser->parse( $input, $title, $options );
                        $out = $output->getText();
 
-                       $op = new OutputPage();
-                       $op->replaceLinkHolders($out);
-
                        if (preg_match('/\\bill\\b/i', $opts)) {
-                               $out .= implode( ' ', $output->getLanguageLinks() );
-                       }       
-                       if (preg_match('/\\bcat\\b/i', $opts)) {
-                               $out .= implode( ' ', $output->getCategoryLinks() );
+                               $out = $this->tidy( implode( ' ', $output->getLanguageLinks() ) );
+                       } else if (preg_match('/\\bcat\\b/i', $opts)) {
+                               $out = $this->tidy ( implode( ' ', $output->getCategoryLinks() ) );
                        }
 
-                       if ($GLOBALS['wgUseTidy']) {
-                               $out = Parser::tidy($out);
-                               $result = Parser::tidy($result);
-                       }
+                       $result = $this->tidy($result);
                }
                
                $this->teardownGlobals();
                
-               if( $result === $out ) {
+               if( $result === $out && $this->wellFormed( $out ) ) {
                        return $this->showSuccess( $desc );
                } else {
                        return $this->showFailure( $desc, $result, $out );
@@ -278,6 +277,9 @@ class ParserTest {
                foreach( $this->listTables() as $table ) {
                        $this->oldTableNames[$table] = $db->tableName( $table );
                }
+               if( !isset( $this->uploadDir ) ) {
+                       $this->uploadDir = $this->setupUploadDir();
+               }
                
                $settings = array(
                        'wgServer' => 'http://localhost',
@@ -285,15 +287,22 @@ class ParserTest {
                        'wgScriptPath' => '/',
                        'wgArticlePath' => '/wiki/$1',
                        'wgUploadPath' => '/images',
+                       'wgUploadDirectory' => $this->uploadDir,
+                       'wgStyleSheetPath' => '/skins',
                        'wgSitename' => 'MediaWiki',
                        'wgLanguageCode' => 'en',
-                       'wgUseLatin1' => false,
+                       'wgContLanguageCode' => 'en',
                        'wgDBprefix' => 'parsertest',
+                       'wgDefaultUserOptions' => array(),
                        
-                       'wgLoadBalancer' => LoadBalancer::newFromParams( $GLOBALS['wgDBservers'] ),
                        'wgLang' => new LanguageUtf8(),
+                       'wgContLang' => new LanguageUtf8(),
                        'wgNamespacesWithSubpages' => array( 0 => preg_match('/\\bsubpage\\b/i', $opts)),
                        'wgMaxTocLevel' => 999,
+                       'wgCapitalLinks' => true,
+                       'wgDefaultUserOptions' => array(),
+                       'wgNoFollowLinks' => true,
+                       'wgThumbnailScriptPath' => false,
                        );
                $this->savedGlobals = array();
                foreach( $settings as $var => $val ) {
@@ -301,15 +310,19 @@ class ParserTest {
                        $GLOBALS[$var] = $val;
                }
                $GLOBALS['wgLoadBalancer']->loadMasterPos();
+               $GLOBALS['wgMessageCache']->initialise( new BagOStuff(), false, 0, $GLOBALS['wgDBname'] );
                $this->setupDatabase();
+               
+               global $wgUser;
+               $wgUser = new User();
        }
        
        # List of temporary tables to create, without prefix
        # Some of these probably aren't necessary
        function listTables() {
-               return array('user', 'cur', 'old', 'links',
-                       'brokenlinks', 'imagelinks', 'categorylinks',
-                       'linkscc', 'site_stats', 'hitcounter',
+               return array('user', 'page', 'revision', 'text',
+                       'pagelinks', 'imagelinks', 'categorylinks',
+                       'site_stats', 'hitcounter',
                        'ipblocks', 'image', 'oldimage',
                        'recentchanges',
                        'watchlist', 'math', 'searchindex',
@@ -331,16 +344,24 @@ class ParserTest {
 
                # Make sure we don't mess with the live DB
                if (!$setupDB && $wgDBprefix === 'parsertest') {
+                       # oh teh horror
+                       $GLOBALS['wgLoadBalancer'] = LoadBalancer::newFromParams( $GLOBALS['wgDBservers'] );
                        $db =& wfGetDB( DB_MASTER );
 
                        $tables = $this->listTables();
 
                        if (!(strcmp($db->getServerVersion(), '4.1') < 0 and stristr($db->getSoftwareLink(), 'MySQL'))) {
                                # Database that supports CREATE TABLE ... LIKE
+                               global $wgDBtype;
+                               if( $wgDBtype == 'PostgreSQL' ) {
+                                       $def = 'INCLUDING DEFAULTS';
+                               } else {
+                                       $def = '';
+                               }
                                foreach ($tables as $tbl) {
                                        $newTableName = $db->tableName( $tbl );
                                        $tableName = $this->oldTableNames[$tbl];
-                                       $db->query("CREATE TEMPORARY TABLE $newTableName (LIKE $tableName INCLUDING DEFAULTS)");
+                                       $db->query("CREATE TEMPORARY TABLE $newTableName (LIKE $tableName $def)");
                                }
                        } else {
                                # Hack for MySQL versions < 4.1, which don't support
@@ -348,11 +369,11 @@ class ParserTest {
                                # "CREATE TEMPORARY TABLE ... SELECT * FROM ... LIMIT 0"
                                # would not create the indexes we need....
                                foreach ($tables as $tbl) {
-                                       $res = $db->query("SHOW CREATE TABLE $tbl");
+                                       $res = $db->query("SHOW CREATE TABLE {$this->oldTableNames[$tbl]}");
                                        $row = $db->fetchRow($res);
                                        $create = $row[1];
                                        $create_tmp = preg_replace('/CREATE TABLE `(.*?)`/', 'CREATE TEMPORARY TABLE `'
-                                               . $wgDBprefix . '\\1`', $create);
+                                               . $wgDBprefix . $tbl .'`', $create);
                                        if ($create === $create_tmp) {
                                                # Couldn't do replacement
                                                die("could not create temporary table $tbl");
@@ -364,7 +385,7 @@ class ParserTest {
 
                        # Hack: insert a few Wikipedia in-project interwiki prefixes,
                        # for testing inter-language links
-                       $db->insertArray( 'interwiki', array(
+                       $db->insert( 'interwiki', array(
                                array( 'iw_prefix' => 'Wikipedia',
                                       'iw_url'    => 'http://en.wikipedia.org/wiki/$1',
                                       'iw_local'  => 0 ),
@@ -379,13 +400,57 @@ class ParserTest {
                                       'iw_local'  => 1 ),
                                array( 'iw_prefix' => 'fr',
                                       'iw_url'    => 'http://fr.wikipedia.org/wiki/$1',
-                                      'iw_local'  => 1 ) ) );
-
-
+                                      'iw_local'  => 1 ),
+                               array( 'iw_prefix' => 'ru',
+                                      'iw_url'    => 'http://ru.wikipedia.org/wiki/$1',
+                                      'iw_local'  => 1 ),
+                               ) );
+
+                       # Hack: Insert an image to work with
+                       $db->insert( 'image', array(
+                               'img_name'        => 'Foobar.jpg',
+                               'img_size'        => 12345,
+                               'img_description' => 'Some lame file',
+                               'img_user'        => 1,
+                               'img_user_text'   => 'WikiSysop',
+                               'img_timestamp'   => $db->timestamp( '20010115123500' ),
+                               'img_width'       => 1941,
+                               'img_height'      => 220,
+                               'img_bits'        => 24,
+                               'img_media_type'  => MEDIATYPE_BITMAP, 
+                               'img_major_mime'  => "image", 
+                               'img_minor_mime'  => "jpeg", 
+                               ) );
+                       
                        $setupDB = true;
                }
        }
        
+       /**
+        * Create a dummy uploads directory which will contain a couple
+        * of files in order to pass existence tests.
+        * @return string The directory
+        * @access private
+        */
+       function setupUploadDir() {
+               global $IP;
+               
+               $dir = wfTempDir() . "/mwParser-" . mt_rand() . "-images";
+               mkdir( $dir );
+               mkdir( $dir . '/3' );
+               mkdir( $dir . '/3/3a' );
+               
+               $img = "$IP/skins/monobook/headbg.jpg";
+               $h = fopen($img, 'r');
+               $c = fread($h, filesize($img));
+               fclose($h);
+               
+               $f = fopen( $dir . '/3/3a/Foobar.jpg', 'wb' );
+               fwrite( $f, $c );
+               fclose( $f );
+               return $dir;
+       }
+       
        /**
         * Restore default values and perform any necessary clean-up
         * after each test runs.
@@ -396,6 +461,36 @@ class ParserTest {
                foreach( $this->savedGlobals as $var => $val ) {
                        $GLOBALS[$var] = $val;
                }
+               if( isset( $this->uploadDir ) ) {
+                       $this->teardownUploadDir( $this->uploadDir );
+                       unset( $this->uploadDir );
+               }
+       }
+       
+       /**
+        * Remove the dummy uploads directory
+        * @access private
+        */
+       function teardownUploadDir( $dir ) {
+               unlink( "$dir/3/3a/Foobar.jpg" );
+               rmdir( "$dir/3/3a" );
+               rmdir( "$dir/3" );
+
+               @unlink( "$dir/thumb/3/3a/Foobar.jpg/180px-Foobar.jpg" );
+               @rmdir( "$dir/thumb/3/3a/Foobar.jpg" );
+               @rmdir( "$dir/thumb/3/3a" );
+               @rmdir( "$dir/thumb/3/39" ); # wtf?
+               @rmdir( "$dir/thumb/3" );
+               @rmdir( "$dir/thumb" );
+               rmdir( "$dir" );
+       }
+       
+       /**
+        * "Running test $desc..."
+        * @access private
+        */
+       function showTesting( $desc ) {
+               print "Running test $desc... ";
        }
        
        /**
@@ -406,7 +501,9 @@ class ParserTest {
         * @access private
         */
        function showSuccess( $desc ) {
-               print $this->termColor( '1;32' ) . 'PASSED' . $this->termReset() . "\n";
+               if( !$this->quiet ) {
+                       print $this->termColor( '1;32' ) . 'PASSED' . $this->termReset() . "\n";
+               }
                return true;
        }
        
@@ -421,10 +518,18 @@ class ParserTest {
         * @access private
         */
        function showFailure( $desc, $result, $html ) {
+               if( $this->quiet ) {
+                       # In quiet mode we didn't show the 'Testing' message before the
+                       # test, in case it succeeded. Show it now:
+                       $this->showTesting( $desc );
+               }
                print $this->termColor( '1;31' ) . 'FAILED!' . $this->termReset() . "\n";
                if( $this->showDiffs ) {
                        print $this->quickDiff( $result, $html );
                }
+               if( !$this->wellFormed( $html ) ) {
+                       print "XML error: $this->mXmlError\n";
+               }
                return false;
        }
        
@@ -438,7 +543,7 @@ class ParserTest {
         * @access private
         */
        function quickDiff( $input, $output ) {
-               $prefix = "/tmp/mwParser-" . mt_rand();
+               $prefix = wfTempDir() . "/mwParser-" . mt_rand();
                
                $infile = "$prefix-expected";
                $this->dumpToFile( $input, $infile );
@@ -511,9 +616,9 @@ class ParserTest {
 
        /**
         * Insert a temporary test article
-        * @param $name string the title, including any prefix
-        * @param $text string the article text
-        * @param $line int the input line number, for reporting errors
+        * @param string $name the title, including any prefix
+        * @param string $text the article text
+        * @param int $line the input line number, for reporting errors
         * @static
         * @access private
         */
@@ -533,9 +638,97 @@ class ParserTest {
                $art->insertNewArticle($text, '', false, false );
                $this->teardownGlobals();
        }
+
+       /*
+        * Run the "tidy" command on text if the $wgUseTidy
+        * global is true
+        *
+        * @param string $text the text to tidy
+        * @return string
+        * @static
+        * @access private
+        */
+       function tidy( $text ) {
+               global $wgUseTidy;
+               if ($wgUseTidy) {
+                       $text = Parser::tidy($text);
+               }
+               return $text;
+       }
+       
+       function wellFormed( $text ) {
+               $html = 
+                       '<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" ' .
+                       '"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">' .
+                       '<html>' .
+                       $text .
+                       '</html>';
+               
+               $parser = xml_parser_create( "UTF-8" );
+               
+               # case folding violates XML standard, turn it off
+               xml_parser_set_option( $parser, XML_OPTION_CASE_FOLDING, false );
+               
+               if( !xml_parse( $parser, $html, true ) ) {
+                       $err = xml_error_string( xml_get_error_code( $parser ) );
+                       $position = xml_get_current_byte_index( $parser );
+                       $fragment = $this->extractFragment( $html, $position );
+                       $this->mXmlError = "$err at byte $position:\n$fragment";
+                       xml_parser_free( $parser );
+                       return false;
+               }
+               xml_parser_free( $parser );
+               return true;
+       }
+       
+       function extractFragment( $text, $position ) {
+               $start = max( 0, $position - 10 );
+               $before = $position - $start;
+               $fragment = '...' .
+                       $this->termColor( 34 ) .
+                       substr( $text, $start, $before ) .
+                       $this->termColor( 0 ) .
+                       $this->termColor( 31 ) .
+                       $this->termColor( 1 ) .
+                       substr( $text, $position, 1 ) .
+                       $this->termColor( 0 ) .
+                       $this->termColor( 34 ) .
+                       substr( $text, $position + 1, 9 ) .
+                       $this->termColor( 0 ) .
+                       '...';
+               $display = str_replace( "\n", ' ', $fragment );
+               $caret = '   ' .
+                       str_repeat( ' ', $before ) .
+                       $this->termColor( 31 ) .
+                       '^' .
+                       $this->termColor( 0 );
+               return "$display\n$caret";
+       }
+       
+}
+
+if( isset( $options['help'] ) ) {
+       echo <<<END
+MediaWiki $wgVersion parser test suite
+Usage: php parserTests.php [--quick] [--quiet] [--color[=(yes|no|light)]]
+                           [--regex <expression>] [--help]
+Options:
+  --quick  Suppress diff output of failed tests
+  --quiet  Suppress notification of passed tests (shows only failed tests)
+  --color  Override terminal detection and force color output on or off
+           'light' option is similar to 'yes' but with color for dark backgrounds
+  --regex  Only run tests whose descriptions which match given regex
+  --help   Show this help message
+
+
+END;
+       exit( 0 );
 }
 
-$wgTitle = Title::newFromText( 'Parser test script' );
+# There is a convention that the parser should never
+# refer to $wgTitle directly, but instead use the title
+# passed to it.
+$wgTitle = Title::newFromText( 'Parser test script do not use' );
 $tester =& new ParserTest();
 
 # Note: the command line setup changes the current working directory