maintenance/tests/testHelpers.inc

   1 <?php
   2
   3 /**
   4  * @ingroup Maintenance
   5  *
   6  * Set of classes to help with test output and such. Right now pretty specific
   7  * to the parser tests but could be more useful one day :)
   8  *
   9  * @todo @fixme Make this more generic
  10  */
  11
  12 class AnsiTermColorer {
  13         function __construct() {
  14         }
  15
  16         /**
  17          * Return ANSI terminal escape code for changing text attribs/color
  18          *
  19          * @param $color String: semicolon-separated list of attribute/color codes
  20          * @return String
  21          */
  22         public function color( $color ) {
  23                 global $wgCommandLineDarkBg;
  24
  25                 $light = $wgCommandLineDarkBg ? "1;" : "0;";
  26
  27                 return "\x1b[{$light}{$color}m";
  28         }
  29
  30         /**
  31          * Return ANSI terminal escape code for restoring default text attributes
  32          *
  33          * @return String
  34          */
  35         public function reset() {
  36                 return $this->color( 0 );
  37         }
  38 }
  39
  40 /* A colour-less terminal */
  41 class DummyTermColorer {
  42         public function color( $color ) {
  43                 return '';
  44         }
  45
  46         public function reset() {
  47                 return '';
  48         }
  49 }
  50
  51 class TestRecorder {
  52         var $parent;
  53         var $term;
  54
  55         function __construct( $parent ) {
  56                 $this->parent = $parent;
  57                 $this->term = $parent->term;
  58         }
  59
  60         function start() {
  61                 $this->total = 0;
  62                 $this->success = 0;
  63         }
  64
  65         function record( $test, $result, $recorder ) {
  66                 $this->total++;
  67                 if( $result ) {
  68                         $recorder->recordTest( $test );
  69                         $this->success += ( $result ? 1 : 0 );
  70                 }
  71         }
  72
  73         function end() {
  74                 // dummy
  75         }
  76
  77         function report() {
  78                 if ( $this->total > 0 ) {
  79                         $this->reportPercentage( $this->success, $this->total );
  80                 } else {
  81                         wfDie( "No tests found.\n" );
  82                 }
  83         }
  84
  85         function reportPercentage( $success, $total ) {
  86                 $ratio = wfPercent( 100 * $success / $total );
  87                 print $this->term->color( 1 ) . "Passed $success of $total tests ($ratio)... ";
  88
  89                 if ( $success == $total ) {
  90                         print $this->term->color( 32 ) . "ALL TESTS PASSED!";
  91                 } else {
  92                         $failed = $total - $success ;
  93                         print $this->term->color( 31 ) . "$failed tests failed!";
  94                 }
  95
  96                 print $this->term->reset() . "\n";
  97
  98                 return ( $success == $total );
  99         }
 100 }
 101
 102 class DbTestPreviewer extends TestRecorder      {
 103         protected $lb;          // /< Database load balancer
 104         protected $db;          // /< Database connection to the main DB
 105         protected $curRun;      // /< run ID number for the current run
 106         protected $prevRun; // /< run ID number for the previous run, if any
 107         protected $results; // /< Result array
 108
 109         /**
 110          * This should be called before the table prefix is changed
 111          */
 112         function __construct( $parent ) {
 113                 parent::__construct( $parent );
 114
 115                 $this->lb = wfGetLBFactory()->newMainLB();
 116                 // This connection will have the wiki's table prefix, not parsertest_
 117                 $this->db = $this->lb->getConnection( DB_MASTER );
 118         }
 119
 120         /**
 121          * Set up result recording; insert a record for the run with the date
 122          * and all that fun stuff
 123          */
 124         function start() {
 125                 parent::start();
 126
 127                 if ( ! $this->db->tableExists( 'testrun' )
 128                         or ! $this->db->tableExists( 'testitem' ) )
 129                 {
 130                         print "WARNING> `testrun` table not found in database.\n";
 131                         $this->prevRun = false;
 132                 } else {
 133                         // We'll make comparisons against the previous run later...
 134                         $this->prevRun = $this->db->selectField( 'testrun', 'MAX(tr_id)' );
 135                 }
 136
 137                 $this->results = array();
 138         }
 139
 140         function record( $test, $result ) {
 141                 parent::record( $test, $result );
 142                 $this->results[$test] = $result;
 143         }
 144
 145         function report() {
 146                 if ( $this->prevRun ) {
 147                         // f = fail, p = pass, n = nonexistent
 148                         // codes show before then after
 149                         $table = array(
 150                                 'fp' => 'previously failing test(s) now PASSING! :)',
 151                                 'pn' => 'previously PASSING test(s) removed o_O',
 152                                 'np' => 'new PASSING test(s) :)',
 153
 154                                 'pf' => 'previously passing test(s) now FAILING! :(',
 155                                 'fn' => 'previously FAILING test(s) removed O_o',
 156                                 'nf' => 'new FAILING test(s) :(',
 157                                 'ff' => 'still FAILING test(s) :(',
 158                         );
 159
 160                         $prevResults = array();
 161
 162                         $res = $this->db->select( 'testitem', array( 'ti_name', 'ti_success' ),
 163                                 array( 'ti_run' => $this->prevRun ), __METHOD__ );
 164
 165                         foreach ( $res as $row ) {
 166                                 if ( !$this->parent->regex
 167                                         || preg_match( "/{$this->parent->regex}/i", $row->ti_name ) )
 168                                 {
 169                                         $prevResults[$row->ti_name] = $row->ti_success;
 170                                 }
 171                         }
 172
 173                         $combined = array_keys( $this->results + $prevResults );
 174
 175                         # Determine breakdown by change type
 176                         $breakdown = array();
 177                         foreach ( $combined as $test ) {
 178                                 if ( !isset( $prevResults[$test] ) ) {
 179                                         $before = 'n';
 180                                 } elseif ( $prevResults[$test] == 1 ) {
 181                                         $before = 'p';
 182                                 } else /* if ( $prevResults[$test] == 0 )*/ {
 183                                         $before = 'f';
 184                                 }
 185
 186                                 if ( !isset( $this->results[$test] ) ) {
 187                                         $after = 'n';
 188                                 } elseif ( $this->results[$test] == 1 ) {
 189                                         $after = 'p';
 190                                 } else /*if ( $this->results[$test] == 0 ) */ {
 191                                         $after = 'f';
 192                                 }
 193
 194                                 $code = $before . $after;
 195
 196                                 if ( isset( $table[$code] ) ) {
 197                                         $breakdown[$code][$test] = $this->getTestStatusInfo( $test, $after );
 198                                 }
 199                         }
 200
 201                         # Write out results
 202                         foreach ( $table as $code => $label ) {
 203                                 if ( !empty( $breakdown[$code] ) ) {
 204                                         $count = count( $breakdown[$code] );
 205                                         printf( "\n%4d %s\n", $count, $label );
 206
 207                                         foreach ( $breakdown[$code] as $differing_test_name => $statusInfo ) {
 208                                                 print "          * $differing_test_name  [$statusInfo]\n";
 209                                         }
 210                                 }
 211                         }
 212                 } else {
 213                         print "No previous test runs to compare against.\n";
 214                 }
 215
 216                 print "\n";
 217                 parent::report();
 218         }
 219
 220         /**
 221          * Returns a string giving information about when a test last had a status change.
 222          * Could help to track down when regressions were introduced, as distinct from tests
 223          * which have never passed (which are more change requests than regressions).
 224          */
 225         private function getTestStatusInfo( $testname, $after ) {
 226                 // If we're looking at a test that has just been removed, then say when it first appeared.
 227                 if ( $after == 'n' ) {
 228                         $changedRun = $this->db->selectField ( 'testitem',
 229                                 'MIN(ti_run)',
 230                                 array( 'ti_name' => $testname ),
 231                                 __METHOD__ );
 232                         $appear = $this->db->selectRow ( 'testrun',
 233                                 array( 'tr_date', 'tr_mw_version' ),
 234                                 array( 'tr_id' => $changedRun ),
 235                                 __METHOD__ );
 236
 237                         return "First recorded appearance: "
 238                                    . date( "d-M-Y H:i:s",  strtotime ( $appear->tr_date ) )
 239                                    .  ", " . $appear->tr_mw_version;
 240                 }
 241
 242                 // Otherwise, this test has previous recorded results.
 243                 // See when this test last had a different result to what we're seeing now.
 244                 $conds = array(
 245                         'ti_name'        => $testname,
 246                         'ti_success' => ( $after == 'f' ? "1" : "0" ) );
 247
 248                 if ( $this->curRun ) {
 249                         $conds[] = "ti_run != " . $this->db->addQuotes ( $this->curRun );
 250                 }
 251
 252                 $changedRun = $this->db->selectField ( 'testitem', 'MAX(ti_run)', $conds, __METHOD__ );
 253
 254                 // If no record of ever having had a different result.
 255                 if ( is_null ( $changedRun ) ) {
 256                         if ( $after == "f" ) {
 257                                 return "Has never passed";
 258                         } else {
 259                                 return "Has never failed";
 260                         }
 261                 }
 262
 263                 // Otherwise, we're looking at a test whose status has changed.
 264                 // (i.e. it used to work, but now doesn't; or used to fail, but is now fixed.)
 265                 // In this situation, give as much info as we can as to when it changed status.
 266                 $pre  = $this->db->selectRow ( 'testrun',
 267                         array( 'tr_date', 'tr_mw_version' ),
 268                         array( 'tr_id' => $changedRun ),
 269                         __METHOD__ );
 270                 $post = $this->db->selectRow ( 'testrun',
 271                         array( 'tr_date', 'tr_mw_version' ),
 272                         array( "tr_id > " . $this->db->addQuotes ( $changedRun ) ),
 273                         __METHOD__,
 274                         array( "LIMIT" => 1, "ORDER BY" => 'tr_id' )
 275                 );
 276
 277                 if ( $post ) {
 278                         $postDate = date( "d-M-Y H:i:s",  strtotime ( $post->tr_date  ) ) . ", {$post->tr_mw_version}";
 279                 } else {
 280                         $postDate = 'now';
 281                 }
 282
 283                 return ( $after == "f" ? "Introduced" : "Fixed" ) . " between "
 284                                 . date( "d-M-Y H:i:s",  strtotime ( $pre->tr_date ) ) .  ", " . $pre->tr_mw_version
 285                                 . " and $postDate";
 286
 287         }
 288
 289         /**
 290          * Commit transaction and clean up for result recording
 291          */
 292         function end() {
 293                 $this->lb->commitMasterChanges();
 294                 $this->lb->closeAll();
 295                 parent::end();
 296         }
 297
 298 }
 299
 300 class DbTestRecorder extends DbTestPreviewer  {
 301         var $version;
 302
 303         /**
 304          * Set up result recording; insert a record for the run with the date
 305          * and all that fun stuff
 306          */
 307         function start() {
 308                 global $wgDBtype;
 309                 $this->db->begin();
 310
 311                 if ( ! $this->db->tableExists( 'testrun' )
 312                         or ! $this->db->tableExists( 'testitem' ) )
 313                 {
 314                         print "WARNING> `testrun` table not found in database. Trying to create table.\n";
 315                         if ( $wgDBtype === 'postgres' ) {
 316                                 $this->db->sourceFile( dirname( __FILE__ ) . '/testRunner.postgres.sql' );
 317                         } elseif ( $wgDBtype === 'oracle' ) {
 318                                 $this->db->sourceFile( dirname( __FILE__ ) . '/testRunner.ora.sql' );
 319                         } else {
 320                                 $this->db->sourceFile( dirname( __FILE__ ) . '/testRunner.sql' );
 321                         }
 322
 323                         echo "OK, resuming.\n";
 324                 }
 325
 326                 parent::start();
 327
 328                 $this->db->insert( 'testrun',
 329                         array(
 330                                 'tr_date'                => $this->db->timestamp(),
 331                                 'tr_mw_version'  => $this->version,
 332                                 'tr_php_version' => phpversion(),
 333                                 'tr_db_version'  => $this->db->getServerVersion(),
 334                                 'tr_uname'               => php_uname()
 335                         ),
 336                         __METHOD__ );
 337                         if ( $wgDBtype === 'postgres' ) {
 338                                 $this->curRun = $this->db->currentSequenceValue( 'testrun_id_seq' );
 339                         } else {
 340                                 $this->curRun = $this->db->insertId();
 341                         }
 342         }
 343
 344         /**
 345          * Record an individual test item's success or failure to the db
 346          *
 347          * @param $test String
 348          * @param $result Boolean
 349          */
 350         function record( $test, $result ) {
 351                 parent::record( $test, $result );
 352
 353                 $this->db->insert( 'testitem',
 354                         array(
 355                                 'ti_run'         => $this->curRun,
 356                                 'ti_name'        => $test,
 357                                 'ti_success' => $result ? 1 : 0,
 358                         ),
 359                         __METHOD__ );
 360         }
 361 }
 362
 363 class RemoteTestRecorder extends TestRecorder {
 364         function start() {
 365                 parent::start();
 366
 367                 $this->results = array();
 368                 $this->ping( 'running' );
 369         }
 370
 371         function record( $test, $result ) {
 372                 parent::record( $test, $result );
 373                 $this->results[$test] = (bool)$result;
 374         }
 375
 376         function end() {
 377                 $this->ping( 'complete', $this->results );
 378                 parent::end();
 379         }
 380
 381         /**
 382          * Inform a CodeReview instance that we've started or completed a test run...
 383          *
 384          * @param $status string: "running" - tell it we've started
 385          *                                                "complete" - provide test results array
 386          *                                                "abort" - something went horribly awry
 387          * @param $results array of test name => true/false
 388          */
 389         function ping( $status, $results = false ) {
 390                 global $wgParserTestRemote, $IP;
 391
 392                 $remote = $wgParserTestRemote;
 393                 $revId = SpecialVersion::getSvnRevision( $IP );
 394                 $jsonResults = FormatJson::encode( $results );
 395
 396                 if ( !$remote ) {
 397                         print "Can't do remote upload without configuring \$wgParserTestRemote!\n";
 398                         exit( 1 );
 399                 }
 400
 401                 // Generate a hash MAC to validate our credentials
 402                 $message = array(
 403                         $remote['repo'],
 404                         $remote['suite'],
 405                         $revId,
 406                         $status,
 407                 );
 408
 409                 if ( $status == "complete" ) {
 410                         $message[] = $jsonResults;
 411                 }
 412                 $hmac = hash_hmac( "sha1", implode( "|", $message ), $remote['secret'] );
 413
 414                 $postData = array(
 415                         'action' => 'codetestupload',
 416                         'format' => 'json',
 417                         'repo'   => $remote['repo'],
 418                         'suite'  => $remote['suite'],
 419                         'rev'    => $revId,
 420                         'status' => $status,
 421                         'hmac'   => $hmac,
 422                 );
 423
 424                 if ( $status == "complete" ) {
 425                         $postData['results'] = $jsonResults;
 426                 }
 427
 428                 $response = $this->post( $remote['api-url'], $postData );
 429
 430                 if ( $response === false ) {
 431                         print "CodeReview info upload failed to reach server.\n";
 432                         exit( 1 );
 433                 }
 434
 435                 $responseData = FormatJson::decode( $response, true );
 436
 437                 if ( !is_array( $responseData ) ) {
 438                         print "CodeReview API response not recognized...\n";
 439                         wfDebug( "Unrecognized CodeReview API response: $response\n" );
 440                         exit( 1 );
 441                 }
 442
 443                 if ( isset( $responseData['error'] ) ) {
 444                         $code = $responseData['error']['code'];
 445                         $info = $responseData['error']['info'];
 446                         print "CodeReview info upload failed: $code $info\n";
 447                         exit( 1 );
 448                 }
 449         }
 450
 451         function post( $url, $data ) {
 452                 return Http::post( $url, array( 'postData' => $data ) );
 453         }
 454 }
 455
 456 class TestFileIterator implements Iterator {
 457         private $file;
 458         private $fh;
 459         private $parser;
 460         private $index = 0;
 461         private $test;
 462         private $lineNum;
 463         private $eof;
 464
 465         function __construct( $file, $parser = null ) {
 466                 global $IP;
 467
 468                 $this->file = $file;
 469                 $this->fh = fopen( $this->file, "rt" );
 470
 471                 if ( !$this->fh ) {
 472                         wfDie( "Couldn't open file '$file'\n" );
 473                 }
 474
 475                 $this->parser = $parser;
 476
 477                 if ( $this->parser ) {
 478                         $this->parser->showRunFile( wfRelativePath( $this->file, $IP ) );
 479                 }
 480
 481                 $this->lineNum = $this->index = 0;
 482         }
 483
 484         function setParser( MediaWikiParserTest $parser ) {
 485                 $this->parser = $parser;
 486         }
 487
 488         function rewind() {
 489                 if ( fseek( $this->fh, 0 ) ) {
 490                         wfDie( "Couldn't fseek to the start of '$this->file'\n" );
 491                 }
 492
 493                 $this->index = -1;
 494                 $this->lineNum = 0;
 495                 $this->eof = false;
 496                 $this->next();
 497
 498                 return true;
 499         }
 500
 501         function current() {
 502                 return $this->test;
 503         }
 504
 505         function key() {
 506                 return $this->index;
 507         }
 508
 509         function next() {
 510                 if ( $this->readNextTest() ) {
 511                         $this->index++;
 512                         return true;
 513                 } else {
 514                         $this->eof = true;
 515                 }
 516         }
 517
 518         function valid() {
 519                 return $this->eof != true;
 520         }
 521
 522         function readNextTest() {
 523                 $data = array();
 524                 $section = null;
 525
 526                 while ( false !== ( $line = fgets( $this->fh ) ) ) {
 527                         $this->lineNum++;
 528                         $matches = array();
 529
 530                         if ( preg_match( '/^!!\s*(\w+)/', $line, $matches ) ) {
 531                                 $section = strtolower( $matches[1] );
 532
 533                                 if ( $section == 'endarticle' ) {
 534                                         if ( !isset( $data['text'] ) ) {
 535                                                 wfDie( "'endarticle' without 'text' at line {$this->lineNum} of $this->file\n" );
 536                                         }
 537
 538                                         if ( !isset( $data['article'] ) ) {
 539                                                 wfDie( "'endarticle' without 'article' at line {$this->lineNum} of $this->file\n" );
 540                                         }
 541
 542                                         if ( $this->parser ) {
 543                                                 $this->parser->addArticle( $this->parser->chomp( $data['article'] ), $this->parser->chomp( $data['text'] ),
 544                                                         $this->lineNum );
 545                                         }
 546
 547                                         $data = array();
 548                                         $section = null;
 549
 550                                         continue;
 551                                 }
 552
 553                                 if ( $section == 'endhooks' ) {
 554                                         if ( !isset( $data['hooks'] ) ) {
 555                                                 wfDie( "'endhooks' without 'hooks' at line {$this->lineNum} of $this->file\n" );
 556                                         }
 557
 558                                         foreach ( explode( "\n", $data['hooks'] ) as $line ) {
 559                                                 $line = trim( $line );
 560
 561                                                 if ( $line ) {
 562                                                         if ( $this->parser && !$this->parser->requireHook( $line ) ) {
 563                                                                 return false;
 564                                                         }
 565                                                 }
 566                                         }
 567
 568                                         $data = array();
 569                                         $section = null;
 570
 571                                         continue;
 572                                 }
 573
 574                                 if ( $section == 'endfunctionhooks' ) {
 575                                         if ( !isset( $data['functionhooks'] ) ) {
 576                                                 wfDie( "'endfunctionhooks' without 'functionhooks' at line {$this->lineNum} of $this->file\n" );
 577                                         }
 578
 579                                         foreach ( explode( "\n", $data['functionhooks'] ) as $line ) {
 580                                                 $line = trim( $line );
 581
 582                                                 if ( $line ) {
 583                                                         if ( $this->parser && !$this->parser->requireFunctionHook( $line ) ) {
 584                                                                 return false;
 585                                                         }
 586                                                 }
 587                                         }
 588
 589                                         $data = array();
 590                                         $section = null;
 591
 592                                         continue;
 593                                 }
 594
 595                                 if ( $section == 'end' ) {
 596                                         if ( !isset( $data['test'] ) ) {
 597                                                 wfDie( "'end' without 'test' at line {$this->lineNum} of $this->file\n" );
 598                                         }
 599
 600                                         if ( !isset( $data['input'] ) ) {
 601                                                 wfDie( "'end' without 'input' at line {$this->lineNum} of $this->file\n" );
 602                                         }
 603
 604                                         if ( !isset( $data['result'] ) ) {
 605                                                 wfDie( "'end' without 'result' at line {$this->lineNum} of $this->file\n" );
 606                                         }
 607
 608                                         if ( !isset( $data['options'] ) ) {
 609                                                 $data['options'] = '';
 610                                         }
 611
 612                                         if ( !isset( $data['config'] ) )
 613                                                 $data['config'] = '';
 614
 615                                         if ( $this->parser
 616                                                  && ( ( preg_match( '/\\bdisabled\\b/i', $data['options'] ) && !$this->parser->runDisabled )
 617                                                          || !preg_match( "/" . $this->parser->regex . "/i", $data['test'] ) )  ) {
 618                                                 # disabled test
 619                                                 $data = array();
 620                                                 $section = null;
 621
 622                                                 continue;
 623                                         }
 624
 625                                         global $wgUseTeX;
 626
 627                                         if ( $this->parser &&
 628                                                  preg_match( '/\\bmath\\b/i', $data['options'] ) && !$wgUseTeX ) {
 629                                                 # don't run math tests if $wgUseTeX is set to false in LocalSettings
 630                                                 $data = array();
 631                                                 $section = null;
 632
 633                                                 continue;
 634                                         }
 635
 636                                         if ( $this->parser ) {
 637                                                 $this->test = array(
 638                                                         'test' => $this->parser->chomp( $data['test'] ),
 639                                                         'input' => $this->parser->chomp( $data['input'] ),
 640                                                         'result' => $this->parser->chomp( $data['result'] ),
 641                                                         'options' => $this->parser->chomp( $data['options'] ),
 642                                                         'config' => $this->parser->chomp( $data['config'] ) );
 643                                         } else {
 644                                                 $this->test['test'] = $data['test'];
 645                                         }
 646
 647                                         return true;
 648                                 }
 649
 650                                 if ( isset ( $data[$section] ) ) {
 651                                         wfDie( "duplicate section '$section' at line {$this->lineNum} of $this->file\n" );
 652                                 }
 653
 654                                 $data[$section] = '';
 655
 656                                 continue;
 657                         }
 658
 659                         if ( $section ) {
 660                                 $data[$section] .= $line;
 661                         }
 662                 }
 663
 664                 return false;
 665         }
 666 }