tests/parser/ParserTestRunner.php

   1 <?php
   2 /**
   3  * Generic backend for the MediaWiki parser test suite, used by both the
   4  * standalone parserTests.php and the PHPUnit "parsertests" suite.
   5  *
   6  * Copyright © 2004, 2010 Brion Vibber <brion@pobox.com>
   7  * https://www.mediawiki.org/
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License as published by
  11  * the Free Software Foundation; either version 2 of the License, or
  12  * (at your option) any later version.
  13  *
  14  * This program is distributed in the hope that it will be useful,
  15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  17  * GNU General Public License for more details.
  18  *
  19  * You should have received a copy of the GNU General Public License along
  20  * with this program; if not, write to the Free Software Foundation, Inc.,
  21  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  22  * http://www.gnu.org/copyleft/gpl.html
  23  *
  24  * @todo Make this more independent of the configuration (and if possible the database)
  25  * @file
  26  * @ingroup Testing
  27  */
  28 use Wikimedia\Rdbms\IDatabase;
  29 use MediaWiki\MediaWikiServices;
  30 use MediaWiki\Tidy\TidyDriverBase;
  31 use Wikimedia\ScopedCallback;
  32 use Wikimedia\TestingAccessWrapper;
  33
  34 /**
  35  * @ingroup Testing
  36  */
  37 class ParserTestRunner {
  38
  39         /**
  40          * MediaWiki core parser test files, paths
  41          * will be prefixed with __DIR__ . '/'
  42          *
  43          * @var array
  44          */
  45         private static $coreTestFiles = [
  46                 'parserTests.txt',
  47                 'extraParserTests.txt',
  48         ];
  49
  50         /**
  51          * @var bool $useTemporaryTables Use temporary tables for the temporary database
  52          */
  53         private $useTemporaryTables = true;
  54
  55         /**
  56          * @var array $setupDone The status of each setup function
  57          */
  58         private $setupDone = [
  59                 'staticSetup' => false,
  60                 'perTestSetup' => false,
  61                 'setupDatabase' => false,
  62                 'setDatabase' => false,
  63                 'setupUploads' => false,
  64         ];
  65
  66         /**
  67          * Our connection to the database
  68          * @var Database
  69          */
  70         private $db;
  71
  72         /**
  73          * Database clone helper
  74          * @var CloneDatabase
  75          */
  76         private $dbClone;
  77
  78         /**
  79          * @var TidySupport
  80          */
  81         private $tidySupport;
  82
  83         /**
  84          * @var TidyDriverBase
  85          */
  86         private $tidyDriver = null;
  87
  88         /**
  89          * @var TestRecorder
  90          */
  91         private $recorder;
  92
  93         /**
  94          * The upload directory, or null to not set up an upload directory
  95          *
  96          * @var string|null
  97          */
  98         private $uploadDir = null;
  99
 100         /**
 101          * The name of the file backend to use, or null to use MockFileBackend.
 102          * @var string|null
 103          */
 104         private $fileBackendName;
 105
 106         /**
 107          * A complete regex for filtering tests.
 108          * @var string
 109          */
 110         private $regex;
 111
 112         /**
 113          * A list of normalization functions to apply to the expected and actual
 114          * output.
 115          * @var array
 116          */
 117         private $normalizationFunctions = [];
 118
 119         /**
 120          * Run disabled parser tests
 121          * @var bool
 122          */
 123         private $runDisabled;
 124
 125         /**
 126          * Run tests intended only for parsoid
 127          * @var bool
 128          */
 129         private $runParsoid;
 130
 131         /**
 132          * Disable parse on article insertion
 133          * @var bool
 134          */
 135         private $disableSaveParse;
 136
 137         /**
 138          * Reuse upload directory
 139          * @var bool
 140          */
 141         private $keepUploads;
 142
 143         /**
 144          * @param TestRecorder $recorder
 145          * @param array $options
 146          */
 147         public function __construct( TestRecorder $recorder, $options = [] ) {
 148                 $this->recorder = $recorder;
 149
 150                 if ( isset( $options['norm'] ) ) {
 151                         foreach ( $options['norm'] as $func ) {
 152                                 if ( in_array( $func, [ 'removeTbody', 'trimWhitespace' ] ) ) {
 153                                         $this->normalizationFunctions[] = $func;
 154                                 } else {
 155                                         $this->recorder->warning(
 156                                                 "Warning: unknown normalization option \"$func\"\n" );
 157                                 }
 158                         }
 159                 }
 160
 161                 if ( isset( $options['regex'] ) && $options['regex'] !== false ) {
 162                         $this->regex = $options['regex'];
 163                 } else {
 164                         # Matches anything
 165                         $this->regex = '//';
 166                 }
 167
 168                 $this->keepUploads = !empty( $options['keep-uploads'] );
 169
 170                 $this->fileBackendName = $options['file-backend'] ?? false;
 171
 172                 $this->runDisabled = !empty( $options['run-disabled'] );
 173                 $this->runParsoid = !empty( $options['run-parsoid'] );
 174
 175                 $this->disableSaveParse = !empty( $options['disable-save-parse'] );
 176
 177                 $this->tidySupport = new TidySupport( !empty( $options['use-tidy-config'] ) );
 178                 if ( !$this->tidySupport->isEnabled() ) {
 179                         $this->recorder->warning(
 180                                 "Warning: tidy is not installed, skipping some tests\n" );
 181                 }
 182
 183                 if ( isset( $options['upload-dir'] ) ) {
 184                         $this->uploadDir = $options['upload-dir'];
 185                 }
 186         }
 187
 188         /**
 189          * Get list of filenames to extension and core parser tests
 190          *
 191          * @return array
 192          */
 193         public static function getParserTestFiles() {
 194                 global $wgParserTestFiles;
 195
 196                 // Add core test files
 197                 $files = array_map( function ( $item ) {
 198                         return __DIR__ . "/$item";
 199                 }, self::$coreTestFiles );
 200
 201                 // Plus legacy global files
 202                 $files = array_merge( $files, $wgParserTestFiles );
 203
 204                 // Auto-discover extension parser tests
 205                 $registry = ExtensionRegistry::getInstance();
 206                 foreach ( $registry->getAllThings() as $info ) {
 207                         $dir = dirname( $info['path'] ) . '/tests/parser';
 208                         if ( !file_exists( $dir ) ) {
 209                                 continue;
 210                         }
 211                         $counter = 1;
 212                         $dirIterator = new RecursiveIteratorIterator(
 213                                 new RecursiveDirectoryIterator( $dir )
 214                         );
 215                         foreach ( $dirIterator as $fileInfo ) {
 216                                 /** @var SplFileInfo $fileInfo */
 217                                 if ( substr( $fileInfo->getFilename(), -4 ) === '.txt' ) {
 218                                         $name = $info['name'] . $counter;
 219                                         while ( isset( $files[$name] ) ) {
 220                                                 $name = $info['name'] . '_' . $counter++;
 221                                         }
 222                                         $files[$name] = $fileInfo->getPathname();
 223                                 }
 224                         }
 225                 }
 226
 227                 return array_unique( $files );
 228         }
 229
 230         public function getRecorder() {
 231                 return $this->recorder;
 232         }
 233
 234         /**
 235          * Do any setup which can be done once for all tests, independent of test
 236          * options, except for database setup.
 237          *
 238          * Public setup functions in this class return a ScopedCallback object. When
 239          * this object is destroyed by going out of scope, teardown of the
 240          * corresponding test setup is performed.
 241          *
 242          * Teardown objects may be chained by passing a ScopedCallback from a
 243          * previous setup stage as the $nextTeardown parameter. This enforces the
 244          * convention that teardown actions are taken in reverse order to the
 245          * corresponding setup actions. When $nextTeardown is specified, a
 246          * ScopedCallback will be returned which first tears down the current
 247          * setup stage, and then tears down the previous setup stage which was
 248          * specified by $nextTeardown.
 249          *
 250          * @param ScopedCallback|null $nextTeardown
 251          * @return ScopedCallback
 252          */
 253         public function staticSetup( $nextTeardown = null ) {
 254                 // A note on coding style:
 255
 256                 // The general idea here is to keep setup code together with
 257                 // corresponding teardown code, in a fine-grained manner. We have two
 258                 // arrays: $setup and $teardown. The code snippets in the $setup array
 259                 // are executed at the end of the method, before it returns, and the
 260                 // code snippets in the $teardown array are executed in reverse order
 261                 // when the Wikimedia\ScopedCallback object is consumed.
 262
 263                 // Because it is a common operation to save, set and restore global
 264                 // variables, we have an additional convention: when the array key of
 265                 // $setup is a string, the string is taken to be the name of the global
 266                 // variable, and the element value is taken to be the desired new value.
 267
 268                 // It's acceptable to just do the setup immediately, instead of adding
 269                 // a closure to $setup, except when the setup action depends on global
 270                 // variable initialisation being done first. In this case, you have to
 271                 // append a closure to $setup after the global variable is appended.
 272
 273                 // When you add to setup functions in this class, please keep associated
 274                 // setup and teardown actions together in the source code, and please
 275                 // add comments explaining why the setup action is necessary.
 276
 277                 $setup = [];
 278                 $teardown = [];
 279
 280                 $teardown[] = $this->markSetupDone( 'staticSetup' );
 281
 282                 // Some settings which influence HTML output
 283                 $setup['wgSitename'] = 'MediaWiki';
 284                 $setup['wgServer'] = 'http://example.org';
 285                 $setup['wgServerName'] = 'example.org';
 286                 $setup['wgScriptPath'] = '';
 287                 $setup['wgScript'] = '/index.php';
 288                 $setup['wgResourceBasePath'] = '';
 289                 $setup['wgStylePath'] = '/skins';
 290                 $setup['wgExtensionAssetsPath'] = '/extensions';
 291                 $setup['wgArticlePath'] = '/wiki/$1';
 292                 $setup['wgActionPaths'] = [];
 293                 $setup['wgVariantArticlePath'] = false;
 294                 $setup['wgUploadNavigationUrl'] = false;
 295                 $setup['wgCapitalLinks'] = true;
 296                 $setup['wgNoFollowLinks'] = true;
 297                 $setup['wgNoFollowDomainExceptions'] = [ 'no-nofollow.org' ];
 298                 $setup['wgExternalLinkTarget'] = false;
 299                 $setup['wgLocaltimezone'] = 'UTC';
 300                 $setup['wgHtml5'] = true;
 301                 $setup['wgDisableLangConversion'] = false;
 302                 $setup['wgDisableTitleConversion'] = false;
 303                 $setup['wgMediaInTargetLanguage'] = false;
 304
 305                 // "extra language links"
 306                 // see https://gerrit.wikimedia.org/r/111390
 307                 $setup['wgExtraInterlanguageLinkPrefixes'] = [ 'mul' ];
 308
 309                 // All FileRepo changes should be done here by injecting services,
 310                 // there should be no need to change global variables.
 311                 RepoGroup::setSingleton( $this->createRepoGroup() );
 312                 $teardown[] = function () {
 313                         RepoGroup::destroySingleton();
 314                 };
 315
 316                 // Set up null lock managers
 317                 $setup['wgLockManagers'] = [ [
 318                         'name' => 'fsLockManager',
 319                         'class' => NullLockManager::class,
 320                 ], [
 321                         'name' => 'nullLockManager',
 322                         'class' => NullLockManager::class,
 323                 ] ];
 324                 $reset = function () {
 325                         LockManagerGroup::destroySingletons();
 326                 };
 327                 $setup[] = $reset;
 328                 $teardown[] = $reset;
 329
 330                 // This allows article insertion into the prefixed DB
 331                 $setup['wgDefaultExternalStore'] = false;
 332
 333                 // This might slightly reduce memory usage
 334                 $setup['wgAdaptiveMessageCache'] = true;
 335
 336                 // This is essential and overrides disabling of database messages in TestSetup
 337                 $setup['wgUseDatabaseMessages'] = true;
 338                 $reset = function () {
 339                         MessageCache::destroyInstance();
 340                 };
 341                 $setup[] = $reset;
 342                 $teardown[] = $reset;
 343
 344                 // It's not necessary to actually convert any files
 345                 $setup['wgSVGConverter'] = 'null';
 346                 $setup['wgSVGConverters'] = [ 'null' => 'echo "1">$output' ];
 347
 348                 // Fake constant timestamp
 349                 Hooks::register( 'ParserGetVariableValueTs', function ( &$parser, &$ts ) {
 350                         $ts = $this->getFakeTimestamp();
 351                         return true;
 352                 } );
 353                 $teardown[] = function () {
 354                         Hooks::clear( 'ParserGetVariableValueTs' );
 355                 };
 356
 357                 $this->appendNamespaceSetup( $setup, $teardown );
 358
 359                 // Set up interwikis and append teardown function
 360                 $teardown[] = $this->setupInterwikis();
 361
 362                 // This affects title normalization in links. It invalidates
 363                 // MediaWikiTitleCodec objects.
 364                 $setup['wgLocalInterwikis'] = [ 'local', 'mi' ];
 365                 $reset = function () {
 366                         $this->resetTitleServices();
 367                 };
 368                 $setup[] = $reset;
 369                 $teardown[] = $reset;
 370
 371                 // Set up a mock MediaHandlerFactory
 372                 MediaWikiServices::getInstance()->disableService( 'MediaHandlerFactory' );
 373                 MediaWikiServices::getInstance()->redefineService(
 374                         'MediaHandlerFactory',
 375                         function ( MediaWikiServices $services ) {
 376                                 $handlers = $services->getMainConfig()->get( 'ParserTestMediaHandlers' );
 377                                 return new MediaHandlerFactory( $handlers );
 378                         }
 379                 );
 380                 $teardown[] = function () {
 381                         MediaWikiServices::getInstance()->resetServiceForTesting( 'MediaHandlerFactory' );
 382                 };
 383
 384                 // SqlBagOStuff broke when using temporary tables on r40209 (T17892).
 385                 // It seems to have been fixed since (r55079?), but regressed at some point before r85701.
 386                 // This works around it for now...
 387                 global $wgObjectCaches;
 388                 $setup['wgObjectCaches'] = [ CACHE_DB => $wgObjectCaches['hash'] ] + $wgObjectCaches;
 389                 if ( isset( ObjectCache::$instances[CACHE_DB] ) ) {
 390                         $savedCache = ObjectCache::$instances[CACHE_DB];
 391                         ObjectCache::$instances[CACHE_DB] = new HashBagOStuff;
 392                         $teardown[] = function () use ( $savedCache ) {
 393                                 ObjectCache::$instances[CACHE_DB] = $savedCache;
 394                         };
 395                 }
 396
 397                 $teardown[] = $this->executeSetupSnippets( $setup );
 398
 399                 // Schedule teardown snippets in reverse order
 400                 return $this->createTeardownObject( $teardown, $nextTeardown );
 401         }
 402
 403         private function appendNamespaceSetup( &$setup, &$teardown ) {
 404                 // Add a namespace shadowing a interwiki link, to test
 405                 // proper precedence when resolving links. (T53680)
 406                 $setup['wgExtraNamespaces'] = [
 407                         100 => 'MemoryAlpha',
 408                         101 => 'MemoryAlpha_talk'
 409                 ];
 410                 // Changing wgExtraNamespaces invalidates caches in MWNamespace and
 411                 // any live Language object, both on setup and teardown
 412                 $reset = function () {
 413                         MWNamespace::clearCaches();
 414                         MediaWikiServices::getInstance()->getContentLanguage()->resetNamespaces();
 415                 };
 416                 $setup[] = $reset;
 417                 $teardown[] = $reset;
 418         }
 419
 420         /**
 421          * Create a RepoGroup object appropriate for the current configuration
 422          * @return RepoGroup
 423          */
 424         protected function createRepoGroup() {
 425                 if ( $this->uploadDir ) {
 426                         if ( $this->fileBackendName ) {
 427                                 throw new MWException( 'You cannot specify both use-filebackend and upload-dir' );
 428                         }
 429                         $backend = new FSFileBackend( [
 430                                 'name' => 'local-backend',
 431                                 'wikiId' => wfWikiID(),
 432                                 'basePath' => $this->uploadDir,
 433                                 'tmpDirectory' => wfTempDir()
 434                         ] );
 435                 } elseif ( $this->fileBackendName ) {
 436                         global $wgFileBackends;
 437                         $name = $this->fileBackendName;
 438                         $useConfig = false;
 439                         foreach ( $wgFileBackends as $conf ) {
 440                                 if ( $conf['name'] === $name ) {
 441                                         $useConfig = $conf;
 442                                 }
 443                         }
 444                         if ( $useConfig === false ) {
 445                                 throw new MWException( "Unable to find file backend \"$name\"" );
 446                         }
 447                         $useConfig['name'] = 'local-backend'; // swap name
 448                         unset( $useConfig['lockManager'] );
 449                         unset( $useConfig['fileJournal'] );
 450                         $class = $useConfig['class'];
 451                         $backend = new $class( $useConfig );
 452                 } else {
 453                         # Replace with a mock. We do not care about generating real
 454                         # files on the filesystem, just need to expose the file
 455                         # informations.
 456                         $backend = new MockFileBackend( [
 457                                 'name' => 'local-backend',
 458                                 'wikiId' => wfWikiID()
 459                         ] );
 460                 }
 461
 462                 return new RepoGroup(
 463                         [
 464                                 'class' => MockLocalRepo::class,
 465                                 'name' => 'local',
 466                                 'url' => 'http://example.com/images',
 467                                 'hashLevels' => 2,
 468                                 'transformVia404' => false,
 469                                 'backend' => $backend
 470                         ],
 471                         []
 472                 );
 473         }
 474
 475         /**
 476          * Execute an array in which elements with integer keys are taken to be
 477          * callable objects, and other elements are taken to be global variable
 478          * set operations, with the key giving the variable name and the value
 479          * giving the new global variable value. A closure is returned which, when
 480          * executed, sets the global variables back to the values they had before
 481          * this function was called.
 482          *
 483          * @see staticSetup
 484          *
 485          * @param array $setup
 486          * @return closure
 487          */
 488         protected function executeSetupSnippets( $setup ) {
 489                 $saved = [];
 490                 foreach ( $setup as $name => $value ) {
 491                         if ( is_int( $name ) ) {
 492                                 $value();
 493                         } else {
 494                                 $saved[$name] = $GLOBALS[$name] ?? null;
 495                                 $GLOBALS[$name] = $value;
 496                         }
 497                 }
 498                 return function () use ( $saved ) {
 499                         $this->executeSetupSnippets( $saved );
 500                 };
 501         }
 502
 503         /**
 504          * Take a setup array in the same format as the one given to
 505          * executeSetupSnippets(), and return a ScopedCallback which, when consumed,
 506          * executes the snippets in the setup array in reverse order. This is used
 507          * to create "teardown objects" for the public API.
 508          *
 509          * @see staticSetup
 510          *
 511          * @param array $teardown The snippet array
 512          * @param ScopedCallback|null $nextTeardown A ScopedCallback to consume
 513          * @return ScopedCallback
 514          */
 515         protected function createTeardownObject( $teardown, $nextTeardown = null ) {
 516                 return new ScopedCallback( function () use ( $teardown, $nextTeardown ) {
 517                         // Schedule teardown snippets in reverse order
 518                         $teardown = array_reverse( $teardown );
 519
 520                         $this->executeSetupSnippets( $teardown );
 521                         if ( $nextTeardown ) {
 522                                 ScopedCallback::consume( $nextTeardown );
 523                         }
 524                 } );
 525         }
 526
 527         /**
 528          * Set a setupDone flag to indicate that setup has been done, and return
 529          * the teardown closure. If the flag was already set, throw an exception.
 530          *
 531          * @param string $funcName The setup function name
 532          * @return closure
 533          */
 534         protected function markSetupDone( $funcName ) {
 535                 if ( $this->setupDone[$funcName] ) {
 536                         throw new MWException( "$funcName is already done" );
 537                 }
 538                 $this->setupDone[$funcName] = true;
 539                 return function () use ( $funcName ) {
 540                         $this->setupDone[$funcName] = false;
 541                 };
 542         }
 543
 544         /**
 545          * Ensure a given setup stage has been done, throw an exception if it has
 546          * not.
 547          * @param string $funcName
 548          * @param string|null $funcName2
 549          */
 550         protected function checkSetupDone( $funcName, $funcName2 = null ) {
 551                 if ( !$this->setupDone[$funcName]
 552                         && ( $funcName === null || !$this->setupDone[$funcName2] )
 553                 ) {
 554                         throw new MWException( "$funcName must be called before calling " .
 555                                 wfGetCaller() );
 556                 }
 557         }
 558
 559         /**
 560          * Determine whether a particular setup function has been run
 561          *
 562          * @param string $funcName
 563          * @return bool
 564          */
 565         public function isSetupDone( $funcName ) {
 566                 return $this->setupDone[$funcName] ?? false;
 567         }
 568
 569         /**
 570          * Insert hardcoded interwiki in the lookup table.
 571          *
 572          * This function insert a set of well known interwikis that are used in
 573          * the parser tests. They can be considered has fixtures are injected in
 574          * the interwiki cache by using the 'InterwikiLoadPrefix' hook.
 575          * Since we are not interested in looking up interwikis in the database,
 576          * the hook completely replace the existing mechanism (hook returns false).
 577          *
 578          * @return closure for teardown
 579          */
 580         private function setupInterwikis() {
 581                 # Hack: insert a few Wikipedia in-project interwiki prefixes,
 582                 # for testing inter-language links
 583                 Hooks::register( 'InterwikiLoadPrefix', function ( $prefix, &$iwData ) {
 584                         static $testInterwikis = [
 585                                 'local' => [
 586                                         'iw_url' => 'http://doesnt.matter.org/$1',
 587                                         'iw_api' => '',
 588                                         'iw_wikiid' => '',
 589                                         'iw_local' => 0 ],
 590                                 'wikipedia' => [
 591                                         'iw_url' => 'http://en.wikipedia.org/wiki/$1',
 592                                         'iw_api' => '',
 593                                         'iw_wikiid' => '',
 594                                         'iw_local' => 0 ],
 595                                 'meatball' => [
 596                                         'iw_url' => 'http://www.usemod.com/cgi-bin/mb.pl?$1',
 597                                         'iw_api' => '',
 598                                         'iw_wikiid' => '',
 599                                         'iw_local' => 0 ],
 600                                 'memoryalpha' => [
 601                                         'iw_url' => 'http://www.memory-alpha.org/en/index.php/$1',
 602                                         'iw_api' => '',
 603                                         'iw_wikiid' => '',
 604                                         'iw_local' => 0 ],
 605                                 'zh' => [
 606                                         'iw_url' => 'http://zh.wikipedia.org/wiki/$1',
 607                                         'iw_api' => '',
 608                                         'iw_wikiid' => '',
 609                                         'iw_local' => 1 ],
 610                                 'es' => [
 611                                         'iw_url' => 'http://es.wikipedia.org/wiki/$1',
 612                                         'iw_api' => '',
 613                                         'iw_wikiid' => '',
 614                                         'iw_local' => 1 ],
 615                                 'fr' => [
 616                                         'iw_url' => 'http://fr.wikipedia.org/wiki/$1',
 617                                         'iw_api' => '',
 618                                         'iw_wikiid' => '',
 619                                         'iw_local' => 1 ],
 620                                 'ru' => [
 621                                         'iw_url' => 'http://ru.wikipedia.org/wiki/$1',
 622                                         'iw_api' => '',
 623                                         'iw_wikiid' => '',
 624                                         'iw_local' => 1 ],
 625                                 'mi' => [
 626                                         'iw_url' => 'http://mi.wikipedia.org/wiki/$1',
 627                                         'iw_api' => '',
 628                                         'iw_wikiid' => '',
 629                                         'iw_local' => 1 ],
 630                                 'mul' => [
 631                                         'iw_url' => 'http://wikisource.org/wiki/$1',
 632                                         'iw_api' => '',
 633                                         'iw_wikiid' => '',
 634                                         'iw_local' => 1 ],
 635                         ];
 636                         if ( array_key_exists( $prefix, $testInterwikis ) ) {
 637                                 $iwData = $testInterwikis[$prefix];
 638                         }
 639
 640                         // We only want to rely on the above fixtures
 641                         return false;
 642                 } );// hooks::register
 643
 644                 // Reset the service in case any other tests already cached some prefixes.
 645                 MediaWikiServices::getInstance()->resetServiceForTesting( 'InterwikiLookup' );
 646
 647                 return function () {
 648                         // Tear down
 649                         Hooks::clear( 'InterwikiLoadPrefix' );
 650                         MediaWikiServices::getInstance()->resetServiceForTesting( 'InterwikiLookup' );
 651                 };
 652         }
 653
 654         /**
 655          * Reset the Title-related services that need resetting
 656          * for each test
 657          */
 658         private function resetTitleServices() {
 659                 $services = MediaWikiServices::getInstance();
 660                 $services->resetServiceForTesting( 'TitleFormatter' );
 661                 $services->resetServiceForTesting( 'TitleParser' );
 662                 $services->resetServiceForTesting( '_MediaWikiTitleCodec' );
 663                 $services->resetServiceForTesting( 'LinkRenderer' );
 664                 $services->resetServiceForTesting( 'LinkRendererFactory' );
 665         }
 666
 667         /**
 668          * Remove last character if it is a newline
 669          * @param string $s
 670          * @return string
 671          */
 672         public static function chomp( $s ) {
 673                 if ( substr( $s, -1 ) === "\n" ) {
 674                         return substr( $s, 0, -1 );
 675                 } else {
 676                         return $s;
 677                 }
 678         }
 679
 680         /**
 681          * Run a series of tests listed in the given text files.
 682          * Each test consists of a brief description, wikitext input,
 683          * and the expected HTML output.
 684          *
 685          * Prints status updates on stdout and counts up the total
 686          * number and percentage of passed tests.
 687          *
 688          * Handles all setup and teardown.
 689          *
 690          * @param array $filenames Array of strings
 691          * @return bool True if passed all tests, false if any tests failed.
 692          */
 693         public function runTestsFromFiles( $filenames ) {
 694                 $ok = false;
 695
 696                 $teardownGuard = $this->staticSetup();
 697                 $teardownGuard = $this->setupDatabase( $teardownGuard );
 698                 $teardownGuard = $this->setupUploads( $teardownGuard );
 699
 700                 $this->recorder->start();
 701                 try {
 702                         $ok = true;
 703
 704                         foreach ( $filenames as $filename ) {
 705                                 $testFileInfo = TestFileReader::read( $filename, [
 706                                         'runDisabled' => $this->runDisabled,
 707                                         'runParsoid' => $this->runParsoid,
 708                                         'regex' => $this->regex ] );
 709
 710                                 // Don't start the suite if there are no enabled tests in the file
 711                                 if ( !$testFileInfo['tests'] ) {
 712                                         continue;
 713                                 }
 714
 715                                 $this->recorder->startSuite( $filename );
 716                                 $ok = $this->runTests( $testFileInfo ) && $ok;
 717                                 $this->recorder->endSuite( $filename );
 718                         }
 719
 720                         $this->recorder->report();
 721                 } catch ( DBError $e ) {
 722                         $this->recorder->warning( $e->getMessage() );
 723                 }
 724                 $this->recorder->end();
 725
 726                 ScopedCallback::consume( $teardownGuard );
 727
 728                 return $ok;
 729         }
 730
 731         /**
 732          * Determine whether the current parser has the hooks registered in it
 733          * that are required by a file read by TestFileReader.
 734          * @param array $requirements
 735          * @return bool
 736          */
 737         public function meetsRequirements( $requirements ) {
 738                 foreach ( $requirements as $requirement ) {
 739                         switch ( $requirement['type'] ) {
 740                                 case 'hook':
 741                                         $ok = $this->requireHook( $requirement['name'] );
 742                                         break;
 743                                 case 'functionHook':
 744                                         $ok = $this->requireFunctionHook( $requirement['name'] );
 745                                         break;
 746                                 case 'transparentHook':
 747                                         $ok = $this->requireTransparentHook( $requirement['name'] );
 748                                         break;
 749                         }
 750                         if ( !$ok ) {
 751                                 return false;
 752                         }
 753                 }
 754                 return true;
 755         }
 756
 757         /**
 758          * Run the tests from a single file. staticSetup() and setupDatabase()
 759          * must have been called already.
 760          *
 761          * @param array $testFileInfo Parsed file info returned by TestFileReader
 762          * @return bool True if passed all tests, false if any tests failed.
 763          */
 764         public function runTests( $testFileInfo ) {
 765                 $ok = true;
 766
 767                 $this->checkSetupDone( 'staticSetup' );
 768
 769                 // Don't add articles from the file if there are no enabled tests from the file
 770                 if ( !$testFileInfo['tests'] ) {
 771                         return true;
 772                 }
 773
 774                 // If any requirements are not met, mark all tests from the file as skipped
 775                 if ( !$this->meetsRequirements( $testFileInfo['requirements'] ) ) {
 776                         foreach ( $testFileInfo['tests'] as $test ) {
 777                                 $this->recorder->startTest( $test );
 778                                 $this->recorder->skipped( $test, 'required extension not enabled' );
 779                         }
 780                         return true;
 781                 }
 782
 783                 // Add articles
 784                 $this->addArticles( $testFileInfo['articles'] );
 785
 786                 // Run tests
 787                 foreach ( $testFileInfo['tests'] as $test ) {
 788                         $this->recorder->startTest( $test );
 789                         $result =
 790                                 $this->runTest( $test );
 791                         if ( $result !== false ) {
 792                                 $ok = $ok && $result->isSuccess();
 793                                 $this->recorder->record( $test, $result );
 794                         }
 795                 }
 796
 797                 return $ok;
 798         }
 799
 800         /**
 801          * Get a Parser object
 802          *
 803          * @param string|null $preprocessor
 804          * @return Parser
 805          */
 806         function getParser( $preprocessor = null ) {
 807                 global $wgParserConf;
 808
 809                 $class = $wgParserConf['class'];
 810                 $parser = new $class( [ 'preprocessorClass' => $preprocessor ] + $wgParserConf );
 811                 ParserTestParserHook::setup( $parser );
 812
 813                 return $parser;
 814         }
 815
 816         /**
 817          * Run a given wikitext input through a freshly-constructed wiki parser,
 818          * and compare the output against the expected results.
 819          * Prints status and explanatory messages to stdout.
 820          *
 821          * staticSetup() and setupWikiData() must be called before this function
 822          * is entered.
 823          *
 824          * @param array $test The test parameters:
 825          *  - test: The test name
 826          *  - desc: The subtest description
 827          *  - input: Wikitext to try rendering
 828          *  - options: Array of test options
 829          *  - config: Overrides for global variables, one per line
 830          *
 831          * @return ParserTestResult|false false if skipped
 832          */
 833         public function runTest( $test ) {
 834                 wfDebug( __METHOD__ . ": running {$test['desc']}" );
 835                 $opts = $this->parseOptions( $test['options'] );
 836                 $teardownGuard = $this->perTestSetup( $test );
 837
 838                 $context = RequestContext::getMain();
 839                 $user = $context->getUser();
 840                 $options = ParserOptions::newFromContext( $context );
 841                 $options->setTimestamp( $this->getFakeTimestamp() );
 842
 843                 if ( isset( $opts['tidy'] ) ) {
 844                         if ( !$this->tidySupport->isEnabled() ) {
 845                                 $this->recorder->skipped( $test, 'tidy extension is not installed' );
 846                                 return false;
 847                         } else {
 848                                 $options->setTidy( true );
 849                         }
 850                 }
 851
 852                 if ( isset( $opts['title'] ) ) {
 853                         $titleText = $opts['title'];
 854                 } else {
 855                         $titleText = 'Parser test';
 856                 }
 857
 858                 if ( isset( $opts['maxincludesize'] ) ) {
 859                         $options->setMaxIncludeSize( $opts['maxincludesize'] );
 860                 }
 861                 if ( isset( $opts['maxtemplatedepth'] ) ) {
 862                         $options->setMaxTemplateDepth( $opts['maxtemplatedepth'] );
 863                 }
 864
 865                 $local = isset( $opts['local'] );
 866                 $preprocessor = $opts['preprocessor'] ?? null;
 867                 $parser = $this->getParser( $preprocessor );
 868                 $title = Title::newFromText( $titleText );
 869
 870                 if ( isset( $opts['styletag'] ) ) {
 871                         // For testing the behavior of <style> (including those deduplicated
 872                         // into <link> tags), add tag hooks to allow them to be generated.
 873                         $parser->setHook( 'style', function ( $content, $attributes, $parser ) {
 874                                 $marker = Parser::MARKER_PREFIX . '-style-' . md5( $content ) . Parser::MARKER_SUFFIX;
 875                                 $parser->mStripState->addNoWiki( $marker, $content );
 876                                 return Html::inlineStyle( $marker, 'all', $attributes );
 877                         } );
 878                         $parser->setHook( 'link', function ( $content, $attributes, $parser ) {
 879                                 return Html::element( 'link', $attributes );
 880                         } );
 881                 }
 882
 883                 if ( isset( $opts['pst'] ) ) {
 884                         $out = $parser->preSaveTransform( $test['input'], $title, $user, $options );
 885                         $output = $parser->getOutput();
 886                 } elseif ( isset( $opts['msg'] ) ) {
 887                         $out = $parser->transformMsg( $test['input'], $options, $title );
 888                 } elseif ( isset( $opts['section'] ) ) {
 889                         $section = $opts['section'];
 890                         $out = $parser->getSection( $test['input'], $section );
 891                 } elseif ( isset( $opts['replace'] ) ) {
 892                         $section = $opts['replace'][0];
 893                         $replace = $opts['replace'][1];
 894                         $out = $parser->replaceSection( $test['input'], $section, $replace );
 895                 } elseif ( isset( $opts['comment'] ) ) {
 896                         $out = Linker::formatComment( $test['input'], $title, $local );
 897                 } elseif ( isset( $opts['preload'] ) ) {
 898                         $out = $parser->getPreloadText( $test['input'], $title, $options );
 899                 } else {
 900                         $output = $parser->parse( $test['input'], $title, $options, true, true, 1337 );
 901                         $out = $output->getText( [
 902                                 'allowTOC' => !isset( $opts['notoc'] ),
 903                                 'unwrap' => !isset( $opts['wrap'] ),
 904                         ] );
 905                         if ( isset( $opts['tidy'] ) ) {
 906                                 $out = preg_replace( '/\s+$/', '', $out );
 907                         }
 908
 909                         if ( isset( $opts['showtitle'] ) ) {
 910                                 if ( $output->getTitleText() ) {
 911                                         $title = $output->getTitleText();
 912                                 }
 913
 914                                 $out = "$title\n$out";
 915                         }
 916
 917                         if ( isset( $opts['showindicators'] ) ) {
 918                                 $indicators = '';
 919                                 foreach ( $output->getIndicators() as $id => $content ) {
 920                                         $indicators .= "$id=$content\n";
 921                                 }
 922                                 $out = $indicators . $out;
 923                         }
 924
 925                         if ( isset( $opts['ill'] ) ) {
 926                                 $out = implode( ' ', $output->getLanguageLinks() );
 927                         } elseif ( isset( $opts['cat'] ) ) {
 928                                 $out = '';
 929                                 foreach ( $output->getCategories() as $name => $sortkey ) {
 930                                         if ( $out !== '' ) {
 931                                                 $out .= "\n";
 932                                         }
 933                                         $out .= "cat=$name sort=$sortkey";
 934                                 }
 935                         }
 936                 }
 937
 938                 if ( isset( $output ) && isset( $opts['showflags'] ) ) {
 939                         $actualFlags = array_keys( TestingAccessWrapper::newFromObject( $output )->mFlags );
 940                         sort( $actualFlags );
 941                         $out .= "\nflags=" . implode( ', ', $actualFlags );
 942                 }
 943
 944                 ScopedCallback::consume( $teardownGuard );
 945
 946                 $expected = $test['result'];
 947                 if ( count( $this->normalizationFunctions ) ) {
 948                         $expected = ParserTestResultNormalizer::normalize(
 949                                 $test['expected'], $this->normalizationFunctions );
 950                         $out = ParserTestResultNormalizer::normalize( $out, $this->normalizationFunctions );
 951                 }
 952
 953                 $testResult = new ParserTestResult( $test, $expected, $out );
 954                 return $testResult;
 955         }
 956
 957         /**
 958          * Use a regex to find out the value of an option
 959          * @param string $key Name of option val to retrieve
 960          * @param array $opts Options array to look in
 961          * @param mixed $default Default value returned if not found
 962          * @return mixed
 963          */
 964         private static function getOptionValue( $key, $opts, $default ) {
 965                 $key = strtolower( $key );
 966
 967                 if ( isset( $opts[$key] ) ) {
 968                         return $opts[$key];
 969                 } else {
 970                         return $default;
 971                 }
 972         }
 973
 974         /**
 975          * Given the options string, return an associative array of options.
 976          * @todo Move this to TestFileReader
 977          *
 978          * @param string $instring
 979          * @return array
 980          */
 981         private function parseOptions( $instring ) {
 982                 $opts = [];
 983                 // foo
 984                 // foo=bar
 985                 // foo="bar baz"
 986                 // foo=[[bar baz]]
 987                 // foo=bar,"baz quux"
 988                 // foo={...json...}
 989                 $defs = '(?(DEFINE)
 990                         (?<qstr>                                        # Quoted string
 991                                 "
 992                                 (?:[^\\\\"] | \\\\.)*
 993                                 "
 994                         )
 995                         (?<json>
 996                                 \{              # Open bracket
 997                                 (?:
 998                                         [^"{}] |                                # Not a quoted string or object, or
 999                                         (?&qstr) |                              # A quoted string, or
1000                                         (?&json)                                # A json object (recursively)
1001                                 )*
1002                                 \}              # Close bracket
1003                         )
1004                         (?<value>
1005                                 (?:
1006                                         (?&qstr)                        # Quoted val
1007                                 |
1008                                         \[\[
1009                                                 [^]]*                   # Link target
1010                                         \]\]
1011                                 |
1012                                         [\w-]+                          # Plain word
1013                                 |
1014                                         (?&json)                        # JSON object
1015                                 )
1016                         )
1017                 )';
1018                 $regex = '/' . $defs . '\b
1019                         (?<k>[\w-]+)                            # Key
1020                         \b
1021                         (?:\s*
1022                                 =                                               # First sub-value
1023                                 \s*
1024                                 (?<v>
1025                                         (?&value)
1026                                         (?:\s*
1027                                                 ,                               # Sub-vals 1..N
1028                                                 \s*
1029                                                 (?&value)
1030                                         )*
1031                                 )
1032                         )?
1033                         /x';
1034                 $valueregex = '/' . $defs . '(?&value)/x';
1035
1036                 if ( preg_match_all( $regex, $instring, $matches, PREG_SET_ORDER ) ) {
1037                         foreach ( $matches as $bits ) {
1038                                 $key = strtolower( $bits['k'] );
1039                                 if ( !isset( $bits['v'] ) ) {
1040                                         $opts[$key] = true;
1041                                 } else {
1042                                         preg_match_all( $valueregex, $bits['v'], $vmatches );
1043                                         $opts[$key] = array_map( [ $this, 'cleanupOption' ], $vmatches[0] );
1044                                         if ( count( $opts[$key] ) == 1 ) {
1045                                                 $opts[$key] = $opts[$key][0];
1046                                         }
1047                                 }
1048                         }
1049                 }
1050                 return $opts;
1051         }
1052
1053         private function cleanupOption( $opt ) {
1054                 if ( substr( $opt, 0, 1 ) == '"' ) {
1055                         return stripcslashes( substr( $opt, 1, -1 ) );
1056                 }
1057
1058                 if ( substr( $opt, 0, 2 ) == '[[' ) {
1059                         return substr( $opt, 2, -2 );
1060                 }
1061
1062                 if ( substr( $opt, 0, 1 ) == '{' ) {
1063                         return FormatJson::decode( $opt, true );
1064                 }
1065                 return $opt;
1066         }
1067
1068         /**
1069          * Do any required setup which is dependent on test options.
1070          *
1071          * @see staticSetup() for more information about setup/teardown
1072          *
1073          * @param array $test Test info supplied by TestFileReader
1074          * @param callable|null $nextTeardown
1075          * @return ScopedCallback
1076          */
1077         public function perTestSetup( $test, $nextTeardown = null ) {
1078                 $teardown = [];
1079
1080                 $this->checkSetupDone( 'setupDatabase', 'setDatabase' );
1081                 $teardown[] = $this->markSetupDone( 'perTestSetup' );
1082
1083                 $opts = $this->parseOptions( $test['options'] );
1084                 $config = $test['config'];
1085
1086                 // Find out values for some special options.
1087                 $langCode =
1088                         self::getOptionValue( 'language', $opts, 'en' );
1089                 $variant =
1090                         self::getOptionValue( 'variant', $opts, false );
1091                 $maxtoclevel =
1092                         self::getOptionValue( 'wgMaxTocLevel', $opts, 999 );
1093                 $linkHolderBatchSize =
1094                         self::getOptionValue( 'wgLinkHolderBatchSize', $opts, 1000 );
1095
1096                 // Default to fallback skin, but allow it to be overridden
1097                 $skin = self::getOptionValue( 'skin', $opts, 'fallback' );
1098
1099                 $setup = [
1100                         'wgEnableUploads' => self::getOptionValue( 'wgEnableUploads', $opts, true ),
1101                         'wgLanguageCode' => $langCode,
1102                         'wgRawHtml' => self::getOptionValue( 'wgRawHtml', $opts, false ),
1103                         'wgNamespacesWithSubpages' => array_fill_keys(
1104                                 MWNamespace::getValidNamespaces(), isset( $opts['subpage'] )
1105                         ),
1106                         'wgMaxTocLevel' => $maxtoclevel,
1107                         'wgAllowExternalImages' => self::getOptionValue( 'wgAllowExternalImages', $opts, true ),
1108                         'wgThumbLimits' => [ self::getOptionValue( 'thumbsize', $opts, 180 ) ],
1109                         'wgDefaultLanguageVariant' => $variant,
1110                         'wgLinkHolderBatchSize' => $linkHolderBatchSize,
1111                         // Set as a JSON object like:
1112                         // wgEnableMagicLinks={"ISBN":false, "PMID":false, "RFC":false}
1113                         'wgEnableMagicLinks' => self::getOptionValue( 'wgEnableMagicLinks', $opts, [] )
1114                                 + [ 'ISBN' => true, 'PMID' => true, 'RFC' => true ],
1115                         // Test with legacy encoding by default until HTML5 is very stable and default
1116                         'wgFragmentMode' => [ 'legacy' ],
1117                         'wgMediaInTargetLanguage' => self::getOptionValue( 'wgMediaInTargetLanguage', $opts, false ),
1118                 ];
1119
1120                 $nonIncludable = self::getOptionValue( 'wgNonincludableNamespaces', $opts, false );
1121                 if ( $nonIncludable !== false ) {
1122                         $setup['wgNonincludableNamespaces'] = [ $nonIncludable ];
1123                 }
1124
1125                 if ( $config ) {
1126                         $configLines = explode( "\n", $config );
1127
1128                         foreach ( $configLines as $line ) {
1129                                 list( $var, $value )  = explode( '=', $line, 2 );
1130                                 $setup[$var] = eval( "return $value;" );
1131                         }
1132                 }
1133
1134                 /** @since 1.20 */
1135                 Hooks::run( 'ParserTestGlobals', [ &$setup ] );
1136
1137                 // Create tidy driver
1138                 if ( isset( $opts['tidy'] ) ) {
1139                         // Cache a driver instance
1140                         if ( $this->tidyDriver === null ) {
1141                                 $this->tidyDriver = MWTidy::factory( $this->tidySupport->getConfig() );
1142                         }
1143                         $tidy = $this->tidyDriver;
1144                 } else {
1145                         $tidy = false;
1146                 }
1147                 MWTidy::setInstance( $tidy );
1148                 $teardown[] = function () {
1149                         MWTidy::destroySingleton();
1150                 };
1151
1152                 // Set content language. This invalidates the magic word cache and title services
1153                 $lang = Language::factory( $langCode );
1154                 $lang->resetNamespaces();
1155                 $setup['wgContLang'] = $lang;
1156                 $setup[] = function () use ( $lang ) {
1157                         MediaWikiServices::getInstance()->disableService( 'ContentLanguage' );
1158                         MediaWikiServices::getInstance()->redefineService(
1159                                 'ContentLanguage',
1160                                 function () use ( $lang ) {
1161                                         return $lang;
1162                                 }
1163                         );
1164                 };
1165                 $teardown[] = function () {
1166                         MediaWikiServices::getInstance()->resetServiceForTesting( 'ContentLanguage' );
1167                 };
1168                 $reset = function () {
1169                         MediaWikiServices::getInstance()->resetServiceForTesting( 'MagicWordFactory' );
1170                         $this->resetTitleServices();
1171                 };
1172                 $setup[] = $reset;
1173                 $teardown[] = $reset;
1174
1175                 // Make a user object with the same language
1176                 $user = new User;
1177                 $user->setOption( 'language', $langCode );
1178                 $setup['wgLang'] = $lang;
1179
1180                 // We (re)set $wgThumbLimits to a single-element array above.
1181                 $user->setOption( 'thumbsize', 0 );
1182
1183                 $setup['wgUser'] = $user;
1184
1185                 // And put both user and language into the context
1186                 $context = RequestContext::getMain();
1187                 $context->setUser( $user );
1188                 $context->setLanguage( $lang );
1189                 // And the skin!
1190                 $oldSkin = $context->getSkin();
1191                 $skinFactory = MediaWikiServices::getInstance()->getSkinFactory();
1192                 $context->setSkin( $skinFactory->makeSkin( $skin ) );
1193                 $context->setOutput( new OutputPage( $context ) );
1194                 $setup['wgOut'] = $context->getOutput();
1195                 $teardown[] = function () use ( $context, $oldSkin ) {
1196                         // Clear language conversion tables
1197                         $wrapper = TestingAccessWrapper::newFromObject(
1198                                 $context->getLanguage()->getConverter()
1199                         );
1200                         $wrapper->reloadTables();
1201                         // Reset context to the restored globals
1202                         $context->setUser( $GLOBALS['wgUser'] );
1203                         $context->setLanguage( $GLOBALS['wgContLang'] );
1204                         $context->setSkin( $oldSkin );
1205                         $context->setOutput( $GLOBALS['wgOut'] );
1206                 };
1207
1208                 $teardown[] = $this->executeSetupSnippets( $setup );
1209
1210                 return $this->createTeardownObject( $teardown, $nextTeardown );
1211         }
1212
1213         /**
1214          * List of temporary tables to create, without prefix.
1215          * Some of these probably aren't necessary.
1216          * @return array
1217          */
1218         private function listTables() {
1219                 global $wgCommentTableSchemaMigrationStage, $wgActorTableSchemaMigrationStage;
1220
1221                 $tables = [ 'user', 'user_properties', 'user_former_groups', 'page', 'page_restrictions',
1222                         'protected_titles', 'revision', 'ip_changes', 'text', 'pagelinks', 'imagelinks',
1223                         'categorylinks', 'templatelinks', 'externallinks', 'langlinks', 'iwlinks',
1224                         'site_stats', 'ipblocks', 'image', 'oldimage',
1225                         'recentchanges', 'watchlist', 'interwiki', 'logging', 'log_search',
1226                         'querycache', 'objectcache', 'job', 'l10n_cache', 'redirect', 'querycachetwo',
1227                         'archive', 'user_groups', 'page_props', 'category',
1228                         'slots', 'content', 'slot_roles', 'content_models',
1229                 ];
1230
1231                 if ( $wgCommentTableSchemaMigrationStage >= MIGRATION_WRITE_BOTH ) {
1232                         // The new tables for comments are in use
1233                         $tables[] = 'comment';
1234                         $tables[] = 'revision_comment_temp';
1235                         $tables[] = 'image_comment_temp';
1236                 }
1237
1238                 if ( $wgActorTableSchemaMigrationStage & SCHEMA_COMPAT_WRITE_NEW ) {
1239                         // The new tables for actors are in use
1240                         $tables[] = 'actor';
1241                         $tables[] = 'revision_actor_temp';
1242                 }
1243
1244                 if ( in_array( $this->db->getType(), [ 'mysql', 'sqlite', 'oracle' ] ) ) {
1245                         array_push( $tables, 'searchindex' );
1246                 }
1247
1248                 // Allow extensions to add to the list of tables to duplicate;
1249                 // may be necessary if they hook into page save or other code
1250                 // which will require them while running tests.
1251                 Hooks::run( 'ParserTestTables', [ &$tables ] );
1252
1253                 return $tables;
1254         }
1255
1256         public function setDatabase( IDatabase $db ) {
1257                 $this->db = $db;
1258                 $this->setupDone['setDatabase'] = true;
1259         }
1260
1261         /**
1262          * Set up temporary DB tables.
1263          *
1264          * For best performance, call this once only for all tests. However, it can
1265          * be called at the start of each test if more isolation is desired.
1266          *
1267          * @todo This is basically an unrefactored copy of
1268          * MediaWikiTestCase::setupAllTestDBs. They should be factored out somehow.
1269          *
1270          * Do not call this function from a MediaWikiTestCase subclass, since
1271          * MediaWikiTestCase does its own DB setup. Instead use setDatabase().
1272          *
1273          * @see staticSetup() for more information about setup/teardown
1274          *
1275          * @param ScopedCallback|null $nextTeardown The next teardown object
1276          * @return ScopedCallback The teardown object
1277          */
1278         public function setupDatabase( $nextTeardown = null ) {
1279                 global $wgDBprefix;
1280
1281                 $this->db = wfGetDB( DB_MASTER );
1282                 $dbType = $this->db->getType();
1283
1284                 if ( $dbType == 'oracle' ) {
1285                         $suspiciousPrefixes = [ 'pt_', MediaWikiTestCase::ORA_DB_PREFIX ];
1286                 } else {
1287                         $suspiciousPrefixes = [ 'parsertest_', MediaWikiTestCase::DB_PREFIX ];
1288                 }
1289                 if ( in_array( $wgDBprefix, $suspiciousPrefixes ) ) {
1290                         throw new MWException( "\$wgDBprefix=$wgDBprefix suggests DB setup is already done" );
1291                 }
1292
1293                 $teardown = [];
1294
1295                 $teardown[] = $this->markSetupDone( 'setupDatabase' );
1296
1297                 # CREATE TEMPORARY TABLE breaks if there is more than one server
1298                 if ( MediaWikiServices::getInstance()->getDBLoadBalancer()->getServerCount() != 1 ) {
1299                         $this->useTemporaryTables = false;
1300                 }
1301
1302                 $temporary = $this->useTemporaryTables || $dbType == 'postgres';
1303                 $prefix = $dbType != 'oracle' ? 'parsertest_' : 'pt_';
1304
1305                 $this->dbClone = new CloneDatabase( $this->db, $this->listTables(), $prefix );
1306                 $this->dbClone->useTemporaryTables( $temporary );
1307                 $this->dbClone->cloneTableStructure();
1308                 CloneDatabase::changePrefix( $prefix );
1309
1310                 if ( $dbType == 'oracle' ) {
1311                         $this->db->query( 'BEGIN FILL_WIKI_INFO; END;' );
1312                         # Insert 0 user to prevent FK violations
1313
1314                         # Anonymous user
1315                         $this->db->insert( 'user', [
1316                                 'user_id' => 0,
1317                                 'user_name' => 'Anonymous' ] );
1318                 }
1319
1320                 $teardown[] = function () {
1321                         $this->teardownDatabase();
1322                 };
1323
1324                 // Wipe some DB query result caches on setup and teardown
1325                 $reset = function () {
1326                         MediaWikiServices::getInstance()->getLinkCache()->clear();
1327
1328                         // Clear the message cache
1329                         MessageCache::singleton()->clear();
1330                 };
1331                 $reset();
1332                 $teardown[] = $reset;
1333                 return $this->createTeardownObject( $teardown, $nextTeardown );
1334         }
1335
1336         /**
1337          * Add data about uploads to the new test DB, and set up the upload
1338          * directory. This should be called after either setDatabase() or
1339          * setupDatabase().
1340          *
1341          * @param ScopedCallback|null $nextTeardown The next teardown object
1342          * @return ScopedCallback The teardown object
1343          */
1344         public function setupUploads( $nextTeardown = null ) {
1345                 $teardown = [];
1346
1347                 $this->checkSetupDone( 'setupDatabase', 'setDatabase' );
1348                 $teardown[] = $this->markSetupDone( 'setupUploads' );
1349
1350                 // Create the files in the upload directory (or pretend to create them
1351                 // in a MockFileBackend). Append teardown callback.
1352                 $teardown[] = $this->setupUploadBackend();
1353
1354                 // Create a user
1355                 $user = User::createNew( 'WikiSysop' );
1356
1357                 // Register the uploads in the database
1358
1359                 $image = wfLocalFile( Title::makeTitle( NS_FILE, 'Foobar.jpg' ) );
1360                 # note that the size/width/height/bits/etc of the file
1361                 # are actually set by inspecting the file itself; the arguments
1362                 # to recordUpload2 have no effect.  That said, we try to make things
1363                 # match up so it is less confusing to readers of the code & tests.
1364                 $image->recordUpload2( '', 'Upload of some lame file', 'Some lame file', [
1365                         'size' => 7881,
1366                         'width' => 1941,
1367                         'height' => 220,
1368                         'bits' => 8,
1369                         'media_type' => MEDIATYPE_BITMAP,
1370                         'mime' => 'image/jpeg',
1371                         'metadata' => serialize( [] ),
1372                         'sha1' => Wikimedia\base_convert( '1', 16, 36, 31 ),
1373                         'fileExists' => true
1374                 ], $this->db->timestamp( '20010115123500' ), $user );
1375
1376                 $image = wfLocalFile( Title::makeTitle( NS_FILE, 'Thumb.png' ) );
1377                 # again, note that size/width/height below are ignored; see above.
1378                 $image->recordUpload2( '', 'Upload of some lame thumbnail', 'Some lame thumbnail', [
1379                         'size' => 22589,
1380                         'width' => 135,
1381                         'height' => 135,
1382                         'bits' => 8,
1383                         'media_type' => MEDIATYPE_BITMAP,
1384                         'mime' => 'image/png',
1385                         'metadata' => serialize( [] ),
1386                         'sha1' => Wikimedia\base_convert( '2', 16, 36, 31 ),
1387                         'fileExists' => true
1388                 ], $this->db->timestamp( '20130225203040' ), $user );
1389
1390                 $image = wfLocalFile( Title::makeTitle( NS_FILE, 'Foobar.svg' ) );
1391                 $image->recordUpload2( '', 'Upload of some lame SVG', 'Some lame SVG', [
1392                                 'size'        => 12345,
1393                                 'width'       => 240,
1394                                 'height'      => 180,
1395                                 'bits'        => 0,
1396                                 'media_type'  => MEDIATYPE_DRAWING,
1397                                 'mime'        => 'image/svg+xml',
1398                                 'metadata'    => serialize( [
1399                                         'version'        => SvgHandler::SVG_METADATA_VERSION,
1400                                         'width'          => 240,
1401                                         'height'         => 180,
1402                                         'originalWidth'  => '100%',
1403                                         'originalHeight' => '100%',
1404                                         'translations'   => [
1405                                                 'en' => SVGReader::LANG_FULL_MATCH,
1406                                                 'ru' => SVGReader::LANG_FULL_MATCH,
1407                                         ],
1408                                 ] ),
1409                                 'sha1'        => Wikimedia\base_convert( '', 16, 36, 31 ),
1410                                 'fileExists'  => true
1411                 ], $this->db->timestamp( '20010115123500' ), $user );
1412
1413                 # This image will be blacklisted in [[MediaWiki:Bad image list]]
1414                 $image = wfLocalFile( Title::makeTitle( NS_FILE, 'Bad.jpg' ) );
1415                 $image->recordUpload2( '', 'zomgnotcensored', 'Borderline image', [
1416                         'size' => 12345,
1417                         'width' => 320,
1418                         'height' => 240,
1419                         'bits' => 24,
1420                         'media_type' => MEDIATYPE_BITMAP,
1421                         'mime' => 'image/jpeg',
1422                         'metadata' => serialize( [] ),
1423                         'sha1' => Wikimedia\base_convert( '3', 16, 36, 31 ),
1424                         'fileExists' => true
1425                 ], $this->db->timestamp( '20010115123500' ), $user );
1426
1427                 $image = wfLocalFile( Title::makeTitle( NS_FILE, 'Video.ogv' ) );
1428                 $image->recordUpload2( '', 'A pretty movie', 'Will it play', [
1429                         'size' => 12345,
1430                         'width' => 320,
1431                         'height' => 240,
1432                         'bits' => 0,
1433                         'media_type' => MEDIATYPE_VIDEO,
1434                         'mime' => 'application/ogg',
1435                         'metadata' => serialize( [] ),
1436                         'sha1' => Wikimedia\base_convert( '', 16, 36, 31 ),
1437                         'fileExists' => true
1438                 ], $this->db->timestamp( '20010115123500' ), $user );
1439
1440                 $image = wfLocalFile( Title::makeTitle( NS_FILE, 'Audio.oga' ) );
1441                 $image->recordUpload2( '', 'An awesome hitsong', 'Will it play', [
1442                         'size' => 12345,
1443                         'width' => 0,
1444                         'height' => 0,
1445                         'bits' => 0,
1446                         'media_type' => MEDIATYPE_AUDIO,
1447                         'mime' => 'application/ogg',
1448                         'metadata' => serialize( [] ),
1449                         'sha1' => Wikimedia\base_convert( '', 16, 36, 31 ),
1450                         'fileExists' => true
1451                 ], $this->db->timestamp( '20010115123500' ), $user );
1452
1453                 # A DjVu file
1454                 $image = wfLocalFile( Title::makeTitle( NS_FILE, 'LoremIpsum.djvu' ) );
1455                 $image->recordUpload2( '', 'Upload a DjVu', 'A DjVu', [
1456                         'size' => 3249,
1457                         'width' => 2480,
1458                         'height' => 3508,
1459                         'bits' => 0,
1460                         'media_type' => MEDIATYPE_BITMAP,
1461                         'mime' => 'image/vnd.djvu',
1462                         'metadata' => '<?xml version="1.0" ?>
1463 <!DOCTYPE DjVuXML PUBLIC "-//W3C//DTD DjVuXML 1.1//EN" "pubtext/DjVuXML-s.dtd">
1464 <DjVuXML>
1465 <HEAD></HEAD>
1466 <BODY><OBJECT height="3508" width="2480">
1467 <PARAM name="DPI" value="300" />
1468 <PARAM name="GAMMA" value="2.2" />
1469 </OBJECT>
1470 <OBJECT height="3508" width="2480">
1471 <PARAM name="DPI" value="300" />
1472 <PARAM name="GAMMA" value="2.2" />
1473 </OBJECT>
1474 <OBJECT height="3508" width="2480">
1475 <PARAM name="DPI" value="300" />
1476 <PARAM name="GAMMA" value="2.2" />
1477 </OBJECT>
1478 <OBJECT height="3508" width="2480">
1479 <PARAM name="DPI" value="300" />
1480 <PARAM name="GAMMA" value="2.2" />
1481 </OBJECT>
1482 <OBJECT height="3508" width="2480">
1483 <PARAM name="DPI" value="300" />
1484 <PARAM name="GAMMA" value="2.2" />
1485 </OBJECT>
1486 </BODY>
1487 </DjVuXML>',
1488                         'sha1' => Wikimedia\base_convert( '', 16, 36, 31 ),
1489                         'fileExists' => true
1490                 ], $this->db->timestamp( '20010115123600' ), $user );
1491
1492                 return $this->createTeardownObject( $teardown, $nextTeardown );
1493         }
1494
1495         /**
1496          * Helper for database teardown, called from the teardown closure. Destroy
1497          * the database clone and fix up some things that CloneDatabase doesn't fix.
1498          *
1499          * @todo Move most things here to CloneDatabase
1500          */
1501         private function teardownDatabase() {
1502                 $this->checkSetupDone( 'setupDatabase' );
1503
1504                 $this->dbClone->destroy();
1505
1506                 if ( $this->useTemporaryTables ) {
1507                         if ( $this->db->getType() == 'sqlite' ) {
1508                                 # Under SQLite the searchindex table is virtual and need
1509                                 # to be explicitly destroyed. See T31912
1510                                 # See also MediaWikiTestCase::destroyDB()
1511                                 wfDebug( __METHOD__ . " explicitly destroying sqlite virtual table parsertest_searchindex\n" );
1512                                 $this->db->query( "DROP TABLE `parsertest_searchindex`" );
1513                         }
1514                         # Don't need to do anything
1515                         return;
1516                 }
1517
1518                 $tables = $this->listTables();
1519
1520                 foreach ( $tables as $table ) {
1521                         if ( $this->db->getType() == 'oracle' ) {
1522                                 $this->db->query( "DROP TABLE pt_$table DROP CONSTRAINTS" );
1523                         } else {
1524                                 $this->db->query( "DROP TABLE `parsertest_$table`" );
1525                         }
1526                 }
1527
1528                 if ( $this->db->getType() == 'oracle' ) {
1529                         $this->db->query( 'BEGIN FILL_WIKI_INFO; END;' );
1530                 }
1531         }
1532
1533         /**
1534          * Upload test files to the backend created by createRepoGroup().
1535          *
1536          * @return callable The teardown callback
1537          */
1538         private function setupUploadBackend() {
1539                 global $IP;
1540
1541                 $repo = RepoGroup::singleton()->getLocalRepo();
1542                 $base = $repo->getZonePath( 'public' );
1543                 $backend = $repo->getBackend();
1544                 $backend->prepare( [ 'dir' => "$base/3/3a" ] );
1545                 $backend->store( [
1546                         'src' => "$IP/tests/phpunit/data/parser/headbg.jpg",
1547                         'dst' => "$base/3/3a/Foobar.jpg"
1548                 ] );
1549                 $backend->prepare( [ 'dir' => "$base/e/ea" ] );
1550                 $backend->store( [
1551                         'src' => "$IP/tests/phpunit/data/parser/wiki.png",
1552                         'dst' => "$base/e/ea/Thumb.png"
1553                 ] );
1554                 $backend->prepare( [ 'dir' => "$base/0/09" ] );
1555                 $backend->store( [
1556                         'src' => "$IP/tests/phpunit/data/parser/headbg.jpg",
1557                         'dst' => "$base/0/09/Bad.jpg"
1558                 ] );
1559                 $backend->prepare( [ 'dir' => "$base/5/5f" ] );
1560                 $backend->store( [
1561                         'src' => "$IP/tests/phpunit/data/parser/LoremIpsum.djvu",
1562                         'dst' => "$base/5/5f/LoremIpsum.djvu"
1563                 ] );
1564
1565                 // No helpful SVG file to copy, so make one ourselves
1566                 $data = '<?xml version="1.0" encoding="utf-8"?>' .
1567                         '<svg xmlns="http://www.w3.org/2000/svg"' .
1568                         ' version="1.1" width="240" height="180"/>';
1569
1570                 $backend->prepare( [ 'dir' => "$base/f/ff" ] );
1571                 $backend->quickCreate( [
1572                         'content' => $data, 'dst' => "$base/f/ff/Foobar.svg"
1573                 ] );
1574
1575                 return function () use ( $backend ) {
1576                         if ( $backend instanceof MockFileBackend ) {
1577                                 // In memory backend, so dont bother cleaning them up.
1578                                 return;
1579                         }
1580                         $this->teardownUploadBackend();
1581                 };
1582         }
1583
1584         /**
1585          * Remove the dummy uploads directory
1586          */
1587         private function teardownUploadBackend() {
1588                 if ( $this->keepUploads ) {
1589                         return;
1590                 }
1591
1592                 $repo = RepoGroup::singleton()->getLocalRepo();
1593                 $public = $repo->getZonePath( 'public' );
1594
1595                 $this->deleteFiles(
1596                         [
1597                                 "$public/3/3a/Foobar.jpg",
1598                                 "$public/e/ea/Thumb.png",
1599                                 "$public/0/09/Bad.jpg",
1600                                 "$public/5/5f/LoremIpsum.djvu",
1601                                 "$public/f/ff/Foobar.svg",
1602                                 "$public/0/00/Video.ogv",
1603                                 "$public/4/41/Audio.oga",
1604                         ]
1605                 );
1606         }
1607
1608         /**
1609          * Delete the specified files and their parent directories
1610          * @param array $files File backend URIs mwstore://...
1611          */
1612         private function deleteFiles( $files ) {
1613                 // Delete the files
1614                 $backend = RepoGroup::singleton()->getLocalRepo()->getBackend();
1615                 foreach ( $files as $file ) {
1616                         $backend->delete( [ 'src' => $file ], [ 'force' => 1 ] );
1617                 }
1618
1619                 // Delete the parent directories
1620                 foreach ( $files as $file ) {
1621                         $tmp = FileBackend::parentStoragePath( $file );
1622                         while ( $tmp ) {
1623                                 if ( !$backend->clean( [ 'dir' => $tmp ] )->isOK() ) {
1624                                         break;
1625                                 }
1626                                 $tmp = FileBackend::parentStoragePath( $tmp );
1627                         }
1628                 }
1629         }
1630
1631         /**
1632          * Add articles to the test DB.
1633          *
1634          * @param array $articles Article info array from TestFileReader
1635          */
1636         public function addArticles( $articles ) {
1637                 $setup = [];
1638                 $teardown = [];
1639
1640                 // Be sure ParserTestRunner::addArticle has correct language set,
1641                 // so that system messages get into the right language cache
1642                 if ( MediaWikiServices::getInstance()->getContentLanguage()->getCode() !== 'en' ) {
1643                         $setup['wgLanguageCode'] = 'en';
1644                         $lang = Language::factory( 'en' );
1645                         $setup['wgContLang'] = $lang;
1646                         $setup[] = function () use ( $lang ) {
1647                                 $services = MediaWikiServices::getInstance();
1648                                 $services->disableService( 'ContentLanguage' );
1649                                 $services->redefineService( 'ContentLanguage', function () use ( $lang ) {
1650                                         return $lang;
1651                                 } );
1652                         };
1653                         $teardown[] = function () {
1654                                 MediaWikiServices::getInstance()->resetServiceForTesting( 'ContentLanguage' );
1655                         };
1656                 }
1657
1658                 // Add special namespaces, in case that hasn't been done by staticSetup() yet
1659                 $this->appendNamespaceSetup( $setup, $teardown );
1660
1661                 // wgCapitalLinks obviously needs initialisation
1662                 $setup['wgCapitalLinks'] = true;
1663
1664                 $teardown[] = $this->executeSetupSnippets( $setup );
1665
1666                 foreach ( $articles as $info ) {
1667                         $this->addArticle( $info['name'], $info['text'], $info['file'], $info['line'] );
1668                 }
1669
1670                 // Wipe WANObjectCache process cache, which is invalidated by article insertion
1671                 // due to T144706
1672                 ObjectCache::getMainWANInstance()->clearProcessCache();
1673
1674                 $this->executeSetupSnippets( $teardown );
1675         }
1676
1677         /**
1678          * Insert a temporary test article
1679          * @param string $name The title, including any prefix
1680          * @param string $text The article text
1681          * @param string $file The input file name
1682          * @param int|string $line The input line number, for reporting errors
1683          * @throws Exception
1684          * @throws MWException
1685          */
1686         private function addArticle( $name, $text, $file, $line ) {
1687                 $text = self::chomp( $text );
1688                 $name = self::chomp( $name );
1689
1690                 $title = Title::newFromText( $name );
1691                 wfDebug( __METHOD__ . ": adding $name" );
1692
1693                 if ( is_null( $title ) ) {
1694                         throw new MWException( "invalid title '$name' at $file:$line\n" );
1695                 }
1696
1697                 $newContent = ContentHandler::makeContent( $text, $title );
1698
1699                 $page = WikiPage::factory( $title );
1700                 $page->loadPageData( 'fromdbmaster' );
1701
1702                 if ( $page->exists() ) {
1703                         $content = $page->getContent( Revision::RAW );
1704                         // Only reject the title, if the content/content model is different.
1705                         // This makes it easier to create Template:(( or Template:)) in different extensions
1706                         if ( $newContent->equals( $content ) ) {
1707                                 return;
1708                         }
1709                         throw new MWException(
1710                                 "duplicate article '$name' with different content at $file:$line\n"
1711                         );
1712                 }
1713
1714                 // Optionally use mock parser, to make debugging of actual parser tests simpler.
1715                 // But initialise the MessageCache clone first, don't let MessageCache
1716                 // get a reference to the mock object.
1717                 if ( $this->disableSaveParse ) {
1718                         MessageCache::singleton()->getParser();
1719                         $restore = $this->executeSetupSnippets( [ 'wgParser' => new ParserTestMockParser ] );
1720                 } else {
1721                         $restore = false;
1722                 }
1723                 try {
1724                         $status = $page->doEditContent(
1725                                 $newContent,
1726                                 '',
1727                                 EDIT_NEW | EDIT_INTERNAL
1728                         );
1729                 } finally {
1730                         if ( $restore ) {
1731                                 $restore();
1732                         }
1733                 }
1734
1735                 if ( !$status->isOK() ) {
1736                         throw new MWException( $status->getWikiText( false, false, 'en' ) );
1737                 }
1738
1739                 // The RepoGroup cache is invalidated by the creation of file redirects
1740                 if ( $title->inNamespace( NS_FILE ) ) {
1741                         RepoGroup::singleton()->clearCache( $title );
1742                 }
1743         }
1744
1745         /**
1746          * Check if a hook is installed
1747          *
1748          * @param string $name
1749          * @return bool True if tag hook is present
1750          */
1751         public function requireHook( $name ) {
1752                 global $wgParser;
1753
1754                 $wgParser->firstCallInit(); // make sure hooks are loaded.
1755                 if ( isset( $wgParser->mTagHooks[$name] ) ) {
1756                         return true;
1757                 } else {
1758                         $this->recorder->warning( "   This test suite requires the '$name' hook " .
1759                                 "extension, skipping." );
1760                         return false;
1761                 }
1762         }
1763
1764         /**
1765          * Check if a function hook is installed
1766          *
1767          * @param string $name
1768          * @return bool True if function hook is present
1769          */
1770         public function requireFunctionHook( $name ) {
1771                 global $wgParser;
1772
1773                 $wgParser->firstCallInit(); // make sure hooks are loaded.
1774
1775                 if ( isset( $wgParser->mFunctionHooks[$name] ) ) {
1776                         return true;
1777                 } else {
1778                         $this->recorder->warning( "   This test suite requires the '$name' function " .
1779                                 "hook extension, skipping." );
1780                         return false;
1781                 }
1782         }
1783
1784         /**
1785          * Check if a transparent tag hook is installed
1786          *
1787          * @param string $name
1788          * @return bool True if function hook is present
1789          */
1790         public function requireTransparentHook( $name ) {
1791                 global $wgParser;
1792
1793                 $wgParser->firstCallInit(); // make sure hooks are loaded.
1794
1795                 if ( isset( $wgParser->mTransparentTagHooks[$name] ) ) {
1796                         return true;
1797                 } else {
1798                         $this->recorder->warning( "   This test suite requires the '$name' transparent " .
1799                                 "hook extension, skipping.\n" );
1800                         return false;
1801                 }
1802         }
1803
1804         /**
1805          * Fake constant timestamp to make sure time-related parser
1806          * functions give a persistent value.
1807          *
1808          * - Parser::getVariableValue (via ParserGetVariableValueTs hook)
1809          * - Parser::preSaveTransform (via ParserOptions)
1810          */
1811         private function getFakeTimestamp() {
1812                 // parsed as '1970-01-01T00:02:03Z'
1813                 return 123;
1814         }
1815 }