Merge "Run some parser tests with tidy."
authorjenkins-bot <jenkins-bot@gerrit.wikimedia.org>
Wed, 13 Aug 2014 21:02:56 +0000 (21:02 +0000)
committerGerrit Code Review <gerrit@wikimedia.org>
Wed, 13 Aug 2014 21:02:56 +0000 (21:02 +0000)
1  2 
tests/TestsAutoLoader.php
tests/parser/parserTest.inc
tests/phpunit/includes/parser/NewParserTest.php
tests/testHelpers.inc

@@@ -35,11 -35,13 +35,12 @@@ $wgAutoloadClasses += array
        'TestRecorder' => "$testDir/testHelpers.inc",
        'ITestRecorder' => "$testDir/testHelpers.inc",
        'DjVuSupport' => "$testDir/testHelpers.inc",
+       'TidySupport' => "$testDir/testHelpers.inc",
  
        # tests/phpunit
        'MediaWikiTestCase' => "$testDir/phpunit/MediaWikiTestCase.php",
        'MediaWikiPHPUnitTestListener' => "$testDir/phpunit/MediaWikiPHPUnitTestListener.php",
        'MediaWikiLangTestCase' => "$testDir/phpunit/MediaWikiLangTestCase.php",
 -      'MediaWikiPasswordTestCase' => "$testDir/phpunit/MediaWikiPasswordTestCase.php",
        'ResourceLoaderTestCase' => "$testDir/phpunit/ResourceLoaderTestCase.php",
        'ResourceLoaderTestModule' => "$testDir/phpunit/ResourceLoaderTestCase.php",
        'ResourceLoaderFileModuleTestModule' => "$testDir/phpunit/ResourceLoaderTestCase.php",
@@@ -77,9 -79,6 +78,9 @@@
        'PageORMTableForTesting' => "$testDir/phpunit/includes/db/ORMTableTest.php",
        'DatabaseTestHelper' => "$testDir/phpunit/includes/db/DatabaseTestHelper.php",
  
 +      # tests/phpunit/includes/passwords
 +      'PasswordTestCase' => "$testDir/phpunit/includes/password/PasswordTestCase.php",
 +
        # tests/phpunit/languages
        'LanguageClassesTestCase' => "$testDir/phpunit/languages/LanguageClassesTestCase.php",
  
@@@ -69,6 -69,11 +69,11 @@@ class ParserTest 
         */
        private $djVuSupport;
  
+       /**
+        * @var TidySupport
+        */
+       private $tidySupport;
        private $maxFuzzTestLength = 300;
        private $fuzzSeed = 0;
        private $memoryLimit = 50;
                $this->runParsoid = isset( $options['run-parsoid'] );
  
                $this->djVuSupport = new DjVuSupport();
+               $this->tidySupport = new TidySupport();
+               if ( !$this->tidySupport->isEnabled() ) {
+                       echo "Warning: tidy is not installed, skipping some tests\n";
+               }
  
                $this->hooks = array();
                $this->functionHooks = array();
                        $output = $parser->parse( $input, $title, $options, true, true, 1337 );
                        $output->setTOCEnabled( !isset( $opts['notoc'] ) );
                        $out = $output->getText();
+                       if ( isset( $opts['tidy'] ) ) {
+                               if ( !$this->tidySupport->isEnabled() ) {
+                                       return $this->showSkipped();
+                               }
+                               $out = MWTidy::tidy( $out );
+                               $out = preg_replace( '/\s+$/', '', $out);
+                       }
  
                        if ( isset( $opts['showtitle'] ) ) {
                                if ( $output->getTitleText() ) {
                        }
  
                        if ( isset( $opts['ill'] ) ) {
-                               $out = $this->tidy( implode( ' ', $output->getLanguageLinks() ) );
+                               $out = implode( ' ', $output->getLanguageLinks() );
                        } elseif ( isset( $opts['cat'] ) ) {
                                $outputPage = $context->getOutput();
                                $outputPage->addCategoryLinks( $output->getCategories() );
                                $cats = $outputPage->getCategoryLinks();
  
                                if ( isset( $cats['normal'] ) ) {
-                                       $out = $this->tidy( implode( ' ', $cats['normal'] ) );
+                                       $out = implode( ' ', $cats['normal'] );
                                } else {
                                        $out = '';
                                }
                        }
-                       $result = $this->tidy( $result );
                }
  
                $this->teardownGlobals();
         * @param string $config
         */
        private function setupGlobals( $opts = '', $config = '' ) {
+               global $IP;
                # Find out values for some special options.
                $lang =
                        self::getOptionValue( 'language', $opts, 'en' );
                                ) )
                        ),
                        'wgEnableUploads' => self::getOptionValue( 'wgEnableUploads', $opts, true ),
 +                      'wgUploadNavigationUrl' => false,
                        'wgStylePath' => '/skins',
                        'wgSitename' => 'MediaWiki',
                        'wgLanguageCode' => $lang,
                        'wgLocaltimezone' => 'UTC',
                        'wgAllowExternalImages' => self::getOptionValue( 'wgAllowExternalImages', $opts, true ),
                        'wgThumbLimits' => array( self::getOptionValue( 'thumbsize', $opts, 180 ) ),
-                       'wgUseTidy' => false,
                        'wgDefaultLanguageVariant' => $variant,
                        'wgVariantArticlePath' => false,
                        'wgGroupPermissions' => array( '*' => array(
                        'wgLinkHolderBatchSize' => $linkHolderBatchSize,
                        'wgExperimentalHtmlIds' => false,
                        'wgExternalLinkTarget' => false,
-                       'wgAlwaysUseTidy' => false,
                        'wgHtml5' => true,
                        'wgWellFormedXml' => true,
                        'wgAllowMicrodataAttributes' => true,
                        'wgAdaptiveMessageCache' => true,
                        'wgDisableLangConversion' => false,
                        'wgDisableTitleConversion' => false,
+                       // Tidy options.
+                       // We always set 'wgUseTidy' to false when parsing, but certain
+                       // test-running modes still use tidy if available, so ensure
+                       // that the tidy-related options are all set to their defaults.
+                       'wgUseTidy' => false,
+                       'wgAlwaysUseTidy' => false,
+                       'wgDebugTidy' => false,
+                       'wgTidyConf' => $IP . '/includes/tidy.conf',
+                       'wgTidyOpts' => '',
+                       'wgTidyInternal' => $this->tidySupport->isInternal(),
                );
  
                if ( $config ) {
                return true;
        }
  
-       /**
-        * Run the "tidy" command on text if the $wgUseTidy
-        * global is true
-        *
-        * @param string $text The text to tidy
-        * @return string
-        */
-       private function tidy( $text ) {
-               global $wgUseTidy;
-               if ( $wgUseTidy ) {
-                       $text = MWTidy::tidy( $text );
-               }
-               return $text;
-       }
        private function wellFormed( $text ) {
                $html =
                        Sanitizer::hackDocType() .
@@@ -36,6 -36,10 +36,10 @@@ class NewParserTest extends MediaWikiTe
         * @var DjVuSupport
         */
        private $djVuSupport;
+       /**
+        * @var TidySupport
+        */
+       private $tidySupport;
  
        protected $file = false;
  
@@@ -75,7 -79,6 +79,7 @@@
                $tmpGlobals['wgExtensionAssetsPath'] = '/extensions';
                $tmpGlobals['wgStylePath'] = '/skins';
                $tmpGlobals['wgEnableUploads'] = true;
 +              $tmpGlobals['wgUploadNavigationUrl'] = false;
                $tmpGlobals['wgThumbnailScriptPath'] = false;
                $tmpGlobals['wgLocalFileRepo'] = array(
                        'class' => 'LocalRepo',
                $tmpGlobals['wgUseImageResize'] = true;
                $tmpGlobals['wgAllowExternalImages'] = true;
                $tmpGlobals['wgRawHtml'] = false;
-               $tmpGlobals['wgUseTidy'] = false;
-               $tmpGlobals['wgAlwaysUseTidy'] = false;
                $tmpGlobals['wgWellFormedXml'] = true;
                $tmpGlobals['wgAllowMicrodataAttributes'] = true;
                $tmpGlobals['wgExperimentalHtmlIds'] = false;
                # see https://gerrit.wikimedia.org/r/111390
                $tmpGlobals['wgExtraInterlanguageLinkPrefixes'] = array( 'mul' );
  
-               //DjVu support
+               // DjVu support
                $this->djVuSupport = new DjVuSupport();
+               // Tidy support
+               $this->tidySupport = new TidySupport();
+               // We always set 'wgUseTidy' to false when parsing, but certain
+               // test-running modes still use tidy if available, so ensure
+               // that the tidy-related options are all set to their defaults.
+               $tmpGlobals['wgUseTidy'] = false;
+               $tmpGlobals['wgAlwaysUseTidy'] = false;
+               $tmpGlobals['wgDebugTidy'] = false;
+               $tmpGlobals['wgTidyConf'] = $IP . '/includes/tidy.conf';
+               $tmpGlobals['wgTidyOpts'] = '';
+               $tmpGlobals['wgTidyInternal'] = $this->tidySupport->isInternal();
  
                $this->setMwGlobals( $tmpGlobals );
  
                        $output = $parser->parse( $input, $title, $options, true, true, 1337 );
                        $output->setTOCEnabled( !isset( $opts['notoc'] ) );
                        $out = $output->getText();
+                       if ( isset( $opts['tidy'] ) ) {
+                               if ( !$this->tidySupport->isEnabled() ) {
+                                       $this->markTestSkipped( "SKIPPED: tidy extension is not installed.\n" );
+                               } else {
+                                       $out = MWTidy::tidy( $out );
+                                       $out = preg_replace( '/\s+$/', '', $out);
+                               }
+                       }
  
                        if ( isset( $opts['showtitle'] ) ) {
                                if ( $output->getTitleText() ) {
                        }
  
                        if ( isset( $opts['ill'] ) ) {
-                               $out = $this->tidy( implode( ' ', $output->getLanguageLinks() ) );
+                               $out = implode( ' ', $output->getLanguageLinks() );
                        } elseif ( isset( $opts['cat'] ) ) {
                                $outputPage = $context->getOutput();
                                $outputPage->addCategoryLinks( $output->getCategories() );
                                $cats = $outputPage->getCategoryLinks();
  
                                if ( isset( $cats['normal'] ) ) {
-                                       $out = $this->tidy( implode( ' ', $cats['normal'] ) );
+                                       $out = implode( ' ', $cats['normal'] );
                                } else {
                                        $out = '';
                                }
                        }
                        $parser->mPreprocessor = null;
-                       $result = $this->tidy( $result );
                }
  
                $this->teardownGlobals();
  
        //Various "cleanup" functions
  
-       /**
-        * Run the "tidy" command on text if the $wgUseTidy
-        * global is true
-        *
-        * @param string $text The text to tidy
-        * @return string
-        */
-       protected function tidy( $text ) {
-               global $wgUseTidy;
-               if ( $wgUseTidy ) {
-                       $text = MWTidy::tidy( $text );
-               }
-               return $text;
-       }
        /**
         * Remove last character if it is a newline
         * @param string $s
diff --combined tests/testHelpers.inc
@@@ -364,6 -364,10 +364,10 @@@ class TestFileIterator implements Itera
        private $sectionData = array();
        private $lineNum;
        private $eof;
+       # Create a fake parser tests which never run anything unless
+       # asked to do so. This will avoid running hooks for a disabled test
+       private $delayedParserTest;
+       private $nextSubTest = 0;
  
        function __construct( $file, $parserTest ) {
                $this->file = $file;
                }
  
                $this->parserTest = $parserTest;
+               $this->delayedParserTest = new DelayedParserTest();
  
                $this->lineNum = $this->index = 0;
        }
                return $this->eof != true;
        }
  
+       function setupCurrentTest() {
+               // "input" and "result" are old section names allowed
+               // for backwards-compatibility.
+               $input = $this->checkSection( array( 'wikitext', 'input' ), false );
+               $result = $this->checkSection( array( 'html/php', 'html/*', 'html', 'result' ), false );
+               // some tests have "with tidy" and "without tidy" variants
+               $tidy = $this->checkSection( array( 'html/php+tidy', 'html+tidy'), false );
+               if ( $tidy != false ) {
+                       if ( $this->nextSubTest == 0 ) {
+                               if ( $result != false ) {
+                                       $this->nextSubTest = 1; // rerun non-tidy variant later
+                               }
+                               $result = $tidy;
+                       } else {
+                               $this->nextSubTest = 0; // go on to next test after this
+                               $tidy = false;
+                       }
+               }
+               if ( !isset( $this->sectionData['options'] ) ) {
+                       $this->sectionData['options'] = '';
+               }
+               if ( !isset( $this->sectionData['config'] ) ) {
+                       $this->sectionData['config'] = '';
+               }
+               $isDisabled = preg_match( '/\\bdisabled\\b/i', $this->sectionData['options'] ) && !$this->parserTest->runDisabled;
+               $isParsoidOnly = preg_match( '/\\bparsoid\\b/i', $this->sectionData['options'] ) && $result == 'html' && !$this->parserTest->runParsoid;
+               $isFiltered = !preg_match( "/" . $this->parserTest->regex . "/i", $this->sectionData['test'] );
+               if ( $input == false || $result == false || $isDisabled || $isParsoidOnly || $isFiltered ) {
+                       # disabled test
+                       return false;
+               }
+               # We are really going to run the test, run pending hooks and hooks function
+               wfDebug( __METHOD__ . " unleashing delayed test for: {$this->sectionData['test']}" );
+               $hooksResult = $this->delayedParserTest->unleash( $this->parserTest );
+               if ( !$hooksResult ) {
+                       # Some hook reported an issue. Abort.
+                       throw new MWException( "Problem running hook" );
+               }
+               $this->test = array(
+                       'test' => ParserTest::chomp( $this->sectionData['test'] ),
+                       'input' => ParserTest::chomp( $this->sectionData[$input] ),
+                       'result' => ParserTest::chomp( $this->sectionData[$result] ),
+                       'options' => ParserTest::chomp( $this->sectionData['options'] ),
+                       'config' => ParserTest::chomp( $this->sectionData['config'] ),
+               );
+               if ( $tidy != false ) {
+                       $this->test['options'] .= " tidy";
+               }
+               return true;
+       }
        function readNextTest() {
-               $this->clearSection();
+               # Run additional subtests of previous test
+               while ( $this->nextSubTest > 0 )
+                       if ( $this->setupCurrentTest() )
+                               return true;
  
-               # Create a fake parser tests which never run anything unless
-               # asked to do so. This will avoid running hooks for a disabled test
-               $delayedParserTest = new DelayedParserTest();
+               $this->clearSection();
+               # Reset hooks for the delayed test object
+               $this->delayedParserTest->reset();
  
                while ( false !== ( $line = fgets( $this->fh ) ) ) {
                        $this->lineNum++;
                                                $line = trim( $line );
  
                                                if ( $line ) {
-                                                       $delayedParserTest->requireHook( $line );
+                                                       $this->delayedParserTest->requireHook( $line );
                                                }
                                        }
  
                                                $line = trim( $line );
  
                                                if ( $line ) {
-                                                       $delayedParserTest->requireFunctionHook( $line );
+                                                       $this->delayedParserTest->requireFunctionHook( $line );
                                                }
                                        }
  
  
                                if ( $this->section == 'end' ) {
                                        $this->checkSection( 'test' );
-                                       // "input" and "result" are old section names allowed
-                                       // for backwards-compatibility.
-                                       $input = $this->checkSection( array( 'wikitext', 'input' ), false );
-                                       $result = $this->checkSection( array( 'html/php', 'html/*', 'html', 'result' ), false );
-                                       if ( !isset( $this->sectionData['options'] ) ) {
-                                               $this->sectionData['options'] = '';
-                                       }
-                                       if ( !isset( $this->sectionData['config'] ) ) {
-                                               $this->sectionData['config'] = '';
-                                       }
-                                       if ( $input == false || $result == false ||
-                                               ( ( preg_match( '/\\bdisabled\\b/i', $this->sectionData['options'] )
-                                                       && !$this->parserTest->runDisabled )
-                                               || ( preg_match( '/\\bparsoid\\b/i', $this->sectionData['options'] )
-                                                       && $result != 'html/php' && !$this->parserTest->runParsoid )
-                                               || !preg_match( "/" . $this->parserTest->regex . "/i", $this->sectionData['test'] ) )
-                                       ) {
-                                               # disabled test
-                                               $this->clearSection();
-                                               # Forget any pending hooks call since test is disabled
-                                               $delayedParserTest->reset();
-                                               continue;
-                                       }
-                                       # We are really going to run the test, run pending hooks and hooks function
-                                       wfDebug( __METHOD__ . " unleashing delayed test for: {$this->sectionData['test']}" );
-                                       $hooksResult = $delayedParserTest->unleash( $this->parserTest );
-                                       if ( !$hooksResult ) {
-                                               # Some hook reported an issue. Abort.
-                                               return false;
-                                       }
-                                       $this->test = array(
-                                               'test' => ParserTest::chomp( $this->sectionData['test'] ),
-                                               'input' => ParserTest::chomp( $this->sectionData[$input] ),
-                                               'result' => ParserTest::chomp( $this->sectionData[$result] ),
-                                               'options' => ParserTest::chomp( $this->sectionData['options'] ),
-                                               'config' => ParserTest::chomp( $this->sectionData['config'] ),
-                                       );
-                                       return true;
+                                       do {
+                                               if ( $this->setupCurrentTest() )
+                                                       return true;
+                                       } while ( $this->nextSubTest > 0 );
+                                       # go on to next test (since this was disabled)
+                                       $this->clearSection();
+                                       $this->delayedParserTest->reset();
+                                       continue;
                                }
  
                                if ( isset( $this->sectionData[$this->section] ) ) {
         * Throw an exception if it is not set, referencing current section
         * and adding the current file name and line number
         *
 -       * @param string|array $token Expected token(s) that should have been
 +       * @param string|array $tokens Expected token(s) that should have been
         * mentioned before closing this section
         * @param bool $fatal True iff an exception should be thrown if
         * the section is not found.
@@@ -702,7 -728,7 +728,7 @@@ class DelayedParserTest 
        /**
         * Similar to ParserTest object but does not run anything
         * Use unleash() to really execute the hook function
 -       * @param string $fnHook
 +       * @param string $hook
         */
        public function requireTransparentHook( $hook ) {
                $this->transparentHooks[] = $hook;
@@@ -732,7 -758,7 +758,7 @@@ class DjVuSupport 
        }
  
        /**
-        * Returns if the DjVu tools are usable
+        * Returns true if the DjVu tools are usable
         *
         * @return bool
         */
                        && is_executable( $wgDjvuTxt );
        }
  }
+ /**
+  * Initialize and detect the tidy support
+  */
+ class TidySupport {
+       private $internalTidy;
+       private $externalTidy;
+       /**
+        * Determine if there is a usable tidy.
+        */
+       public function __construct() {
+               global $wgTidyBin;
+               $this->internalTidy = extension_loaded( 'tidy' ) &&
+                       class_exists( 'tidy' );
+               $this->externalTidy = is_executable( $wgTidyBin ) ||
+                       Installer::locateExecutableInDefaultPaths( array( $wgTidyBin ) )
+                       !== false;
+       }
+       /**
+        * Returns true if we should use internal tidy.
+        *
+        * @return bool
+        */
+       public function isInternal() {
+               return $this->internalTidy;
+       }
+       /**
+        * Returns true if tidy is usable
+        *
+        * @return bool
+        */
+       public function isEnabled() {
+               return $this->internalTidy || $this->externalTidy;
+       }
+ }