maintenance/tests/testHelpers.inc

   1 <?php
   2
   3 /**
   4  * @ingroup Maintenance
   5  *
   6  * Set of classes to help with test output and such. Right now pretty specific
   7  * to the parser tests but could be more useful one day :)
   8  *
   9  * @todo @fixme Make this more generic
  10  */
  11
  12 class AnsiTermColorer {
  13         function __construct() {
  14         }
  15
  16         /**
  17          * Return ANSI terminal escape code for changing text attribs/color
  18          *
  19          * @param $color String: semicolon-separated list of attribute/color codes
  20          * @return String
  21          */
  22         public function color( $color ) {
  23                 global $wgCommandLineDarkBg;
  24
  25                 $light = $wgCommandLineDarkBg ? "1;" : "0;";
  26
  27                 return "\x1b[{$light}{$color}m";
  28         }
  29
  30         /**
  31          * Return ANSI terminal escape code for restoring default text attributes
  32          *
  33          * @return String
  34          */
  35         public function reset() {
  36                 return $this->color( 0 );
  37         }
  38 }
  39
  40 /* A colour-less terminal */
  41 class DummyTermColorer {
  42         public function color( $color ) {
  43                 return '';
  44         }
  45
  46         public function reset() {
  47                 return '';
  48         }
  49 }
  50
  51 class TestRecorder {
  52         var $parent;
  53         var $term;
  54
  55         function __construct( $parent ) {
  56                 $this->parent = $parent;
  57                 $this->term = $parent->term;
  58         }
  59
  60         function start() {
  61                 $this->total = 0;
  62                 $this->success = 0;
  63         }
  64
  65         function record( $test, $result ) {
  66                 $this->total++;
  67                 $this->success += ( $result ? 1 : 0 );
  68         }
  69
  70         function end() {
  71                 // dummy
  72         }
  73
  74         function report() {
  75                 if ( $this->total > 0 ) {
  76                         $this->reportPercentage( $this->success, $this->total );
  77                 } else {
  78                         wfDie( "No tests found.\n" );
  79                 }
  80         }
  81
  82         function reportPercentage( $success, $total ) {
  83                 $ratio = wfPercent( 100 * $success / $total );
  84                 print $this->term->color( 1 ) . "Passed $success of $total tests ($ratio)... ";
  85
  86                 if ( $success == $total ) {
  87                         print $this->term->color( 32 ) . "ALL TESTS PASSED!";
  88                 } else {
  89                         $failed = $total - $success ;
  90                         print $this->term->color( 31 ) . "$failed tests failed!";
  91                 }
  92
  93                 print $this->term->reset() . "\n";
  94
  95                 return ( $success == $total );
  96         }
  97 }
  98
  99 class DbTestPreviewer extends TestRecorder  {
 100         protected $lb;      // /< Database load balancer
 101         protected $db;      // /< Database connection to the main DB
 102         protected $curRun;  // /< run ID number for the current run
 103         protected $prevRun; // /< run ID number for the previous run, if any
 104         protected $results; // /< Result array
 105
 106         /**
 107          * This should be called before the table prefix is changed
 108          */
 109         function __construct( $parent ) {
 110                 parent::__construct( $parent );
 111
 112                 $this->lb = wfGetLBFactory()->newMainLB();
 113                 // This connection will have the wiki's table prefix, not parsertest_
 114                 $this->db = $this->lb->getConnection( DB_MASTER );
 115         }
 116
 117         /**
 118          * Set up result recording; insert a record for the run with the date
 119          * and all that fun stuff
 120          */
 121         function start() {
 122                 parent::start();
 123
 124                 if ( ! $this->db->tableExists( 'testrun' )
 125                         or ! $this->db->tableExists( 'testitem' ) )
 126                 {
 127                         print "WARNING> `testrun` table not found in database.\n";
 128                         $this->prevRun = false;
 129                 } else {
 130                         // We'll make comparisons against the previous run later...
 131                         $this->prevRun = $this->db->selectField( 'testrun', 'MAX(tr_id)' );
 132                 }
 133
 134                 $this->results = array();
 135         }
 136
 137         function record( $test, $result ) {
 138                 parent::record( $test, $result );
 139                 $this->results[$test] = $result;
 140         }
 141
 142         function report() {
 143                 if ( $this->prevRun ) {
 144                         // f = fail, p = pass, n = nonexistent
 145                         // codes show before then after
 146                         $table = array(
 147                                 'fp' => 'previously failing test(s) now PASSING! :)',
 148                                 'pn' => 'previously PASSING test(s) removed o_O',
 149                                 'np' => 'new PASSING test(s) :)',
 150
 151                                 'pf' => 'previously passing test(s) now FAILING! :(',
 152                                 'fn' => 'previously FAILING test(s) removed O_o',
 153                                 'nf' => 'new FAILING test(s) :(',
 154                                 'ff' => 'still FAILING test(s) :(',
 155                         );
 156
 157                         $prevResults = array();
 158
 159                         $res = $this->db->select( 'testitem', array( 'ti_name', 'ti_success' ),
 160                                 array( 'ti_run' => $this->prevRun ), __METHOD__ );
 161
 162                         foreach ( $res as $row ) {
 163                                 if ( !$this->parent->regex
 164                                         || preg_match( "/{$this->parent->regex}/i", $row->ti_name ) )
 165                                 {
 166                                         $prevResults[$row->ti_name] = $row->ti_success;
 167                                 }
 168                         }
 169
 170                         $combined = array_keys( $this->results + $prevResults );
 171
 172                         # Determine breakdown by change type
 173                         $breakdown = array();
 174                         foreach ( $combined as $test ) {
 175                                 if ( !isset( $prevResults[$test] ) ) {
 176                                         $before = 'n';
 177                                 } elseif ( $prevResults[$test] == 1 ) {
 178                                         $before = 'p';
 179                                 } else /* if ( $prevResults[$test] == 0 )*/ {
 180                                         $before = 'f';
 181                                 }
 182
 183                                 if ( !isset( $this->results[$test] ) ) {
 184                                         $after = 'n';
 185                                 } elseif ( $this->results[$test] == 1 ) {
 186                                         $after = 'p';
 187                                 } else /*if ( $this->results[$test] == 0 ) */ {
 188                                         $after = 'f';
 189                                 }
 190
 191                                 $code = $before . $after;
 192
 193                                 if ( isset( $table[$code] ) ) {
 194                                         $breakdown[$code][$test] = $this->getTestStatusInfo( $test, $after );
 195                                 }
 196                         }
 197
 198                         # Write out results
 199                         foreach ( $table as $code => $label ) {
 200                                 if ( !empty( $breakdown[$code] ) ) {
 201                                         $count = count( $breakdown[$code] );
 202                                         printf( "\n%4d %s\n", $count, $label );
 203
 204                                         foreach ( $breakdown[$code] as $differing_test_name => $statusInfo ) {
 205                                                 print "      * $differing_test_name  [$statusInfo]\n";
 206                                         }
 207                                 }
 208                         }
 209                 } else {
 210                         print "No previous test runs to compare against.\n";
 211                 }
 212
 213                 print "\n";
 214                 parent::report();
 215         }
 216
 217         /**
 218          * Returns a string giving information about when a test last had a status change.
 219          * Could help to track down when regressions were introduced, as distinct from tests
 220          * which have never passed (which are more change requests than regressions).
 221          */
 222         private function getTestStatusInfo( $testname, $after ) {
 223                 // If we're looking at a test that has just been removed, then say when it first appeared.
 224                 if ( $after == 'n' ) {
 225                         $changedRun = $this->db->selectField ( 'testitem',
 226                                 'MIN(ti_run)',
 227                                 array( 'ti_name' => $testname ),
 228                                 __METHOD__ );
 229                         $appear = $this->db->selectRow ( 'testrun',
 230                                 array( 'tr_date', 'tr_mw_version' ),
 231                                 array( 'tr_id' => $changedRun ),
 232                                 __METHOD__ );
 233
 234                         return "First recorded appearance: "
 235                                    . date( "d-M-Y H:i:s",  strtotime ( $appear->tr_date ) )
 236                                    .  ", " . $appear->tr_mw_version;
 237                 }
 238
 239                 // Otherwise, this test has previous recorded results.
 240                 // See when this test last had a different result to what we're seeing now.
 241                 $conds = array(
 242                         'ti_name'    => $testname,
 243                         'ti_success' => ( $after == 'f' ? "1" : "0" ) );
 244
 245                 if ( $this->curRun ) {
 246                         $conds[] = "ti_run != " . $this->db->addQuotes ( $this->curRun );
 247                 }
 248
 249                 $changedRun = $this->db->selectField ( 'testitem', 'MAX(ti_run)', $conds, __METHOD__ );
 250
 251                 // If no record of ever having had a different result.
 252                 if ( is_null ( $changedRun ) ) {
 253                         if ( $after == "f" ) {
 254                                 return "Has never passed";
 255                         } else {
 256                                 return "Has never failed";
 257                         }
 258                 }
 259
 260                 // Otherwise, we're looking at a test whose status has changed.
 261                 // (i.e. it used to work, but now doesn't; or used to fail, but is now fixed.)
 262                 // In this situation, give as much info as we can as to when it changed status.
 263                 $pre  = $this->db->selectRow ( 'testrun',
 264                         array( 'tr_date', 'tr_mw_version' ),
 265                         array( 'tr_id' => $changedRun ),
 266                         __METHOD__ );
 267                 $post = $this->db->selectRow ( 'testrun',
 268                         array( 'tr_date', 'tr_mw_version' ),
 269                         array( "tr_id > " . $this->db->addQuotes ( $changedRun ) ),
 270                         __METHOD__,
 271                         array( "LIMIT" => 1, "ORDER BY" => 'tr_id' )
 272                 );
 273
 274                 if ( $post ) {
 275                         $postDate = date( "d-M-Y H:i:s",  strtotime ( $post->tr_date  ) ) . ", {$post->tr_mw_version}";
 276                 } else {
 277                         $postDate = 'now';
 278                 }
 279
 280                 return ( $after == "f" ? "Introduced" : "Fixed" ) . " between "
 281                                 . date( "d-M-Y H:i:s",  strtotime ( $pre->tr_date ) ) .  ", " . $pre->tr_mw_version
 282                                 . " and $postDate";
 283
 284         }
 285
 286         /**
 287          * Commit transaction and clean up for result recording
 288          */
 289         function end() {
 290                 $this->lb->commitMasterChanges();
 291                 $this->lb->closeAll();
 292                 parent::end();
 293         }
 294
 295 }
 296
 297 class DbTestRecorder extends DbTestPreviewer  {
 298         var $version;
 299
 300         /**
 301          * Set up result recording; insert a record for the run with the date
 302          * and all that fun stuff
 303          */
 304         function start() {
 305                 global $wgDBtype;
 306                 $this->db->begin();
 307
 308                 if ( ! $this->db->tableExists( 'testrun' )
 309                         or ! $this->db->tableExists( 'testitem' ) )
 310                 {
 311                         print "WARNING> `testrun` table not found in database. Trying to create table.\n";
 312                         if ( $wgDBtype === 'postgres' ) {
 313                                 $this->db->sourceFile( dirname( __FILE__ ) . '/testRunner.postgres.sql' );
 314                         } elseif ( $wgDBtype === 'oracle' ) {
 315                                 $this->db->sourceFile( dirname( __FILE__ ) . '/testRunner.ora.sql' );
 316                         } else {
 317                                 $this->db->sourceFile( dirname( __FILE__ ) . '/testRunner.sql' );
 318                         }
 319
 320                         echo "OK, resuming.\n";
 321                 }
 322
 323                 parent::start();
 324
 325                 $this->db->insert( 'testrun',
 326                         array(
 327                                 'tr_date'        => $this->db->timestamp(),
 328                                 'tr_mw_version'  => $this->version,
 329                                 'tr_php_version' => phpversion(),
 330                                 'tr_db_version'  => $this->db->getServerVersion(),
 331                                 'tr_uname'       => php_uname()
 332                         ),
 333                         __METHOD__ );
 334                         if ( $wgDBtype === 'postgres' ) {
 335                                 $this->curRun = $this->db->currentSequenceValue( 'testrun_id_seq' );
 336                         } else {
 337                                 $this->curRun = $this->db->insertId();
 338                         }
 339         }
 340
 341         /**
 342          * Record an individual test item's success or failure to the db
 343          *
 344          * @param $test String
 345          * @param $result Boolean
 346          */
 347         function record( $test, $result ) {
 348                 parent::record( $test, $result );
 349
 350                 $this->db->insert( 'testitem',
 351                         array(
 352                                 'ti_run'     => $this->curRun,
 353                                 'ti_name'    => $test,
 354                                 'ti_success' => $result ? 1 : 0,
 355                         ),
 356                         __METHOD__ );
 357         }
 358 }
 359
 360 class RemoteTestRecorder extends TestRecorder {
 361         function start() {
 362                 parent::start();
 363
 364                 $this->results = array();
 365                 $this->ping( 'running' );
 366         }
 367
 368         function record( $test, $result ) {
 369                 parent::record( $test, $result );
 370                 $this->results[$test] = (bool)$result;
 371         }
 372
 373         function end() {
 374                 $this->ping( 'complete', $this->results );
 375                 parent::end();
 376         }
 377
 378         /**
 379          * Inform a CodeReview instance that we've started or completed a test run...
 380          *
 381          * @param $status string: "running" - tell it we've started
 382          *                        "complete" - provide test results array
 383          *                        "abort" - something went horribly awry
 384          * @param $results array of test name => true/false
 385          */
 386         function ping( $status, $results = false ) {
 387                 global $wgParserTestRemote, $IP;
 388
 389                 $remote = $wgParserTestRemote;
 390                 $revId = SpecialVersion::getSvnRevision( $IP );
 391                 $jsonResults = FormatJson::encode( $results );
 392
 393                 if ( !$remote ) {
 394                         print "Can't do remote upload without configuring \$wgParserTestRemote!\n";
 395                         exit( 1 );
 396                 }
 397
 398                 // Generate a hash MAC to validate our credentials
 399                 $message = array(
 400                         $remote['repo'],
 401                         $remote['suite'],
 402                         $revId,
 403                         $status,
 404                 );
 405
 406                 if ( $status == "complete" ) {
 407                         $message[] = $jsonResults;
 408                 }
 409                 $hmac = hash_hmac( "sha1", implode( "|", $message ), $remote['secret'] );
 410
 411                 $postData = array(
 412                         'action' => 'codetestupload',
 413                         'format' => 'json',
 414                         'repo'   => $remote['repo'],
 415                         'suite'  => $remote['suite'],
 416                         'rev'    => $revId,
 417                         'status' => $status,
 418                         'hmac'   => $hmac,
 419                 );
 420
 421                 if ( $status == "complete" ) {
 422                         $postData['results'] = $jsonResults;
 423                 }
 424
 425                 $response = $this->post( $remote['api-url'], $postData );
 426
 427                 if ( $response === false ) {
 428                         print "CodeReview info upload failed to reach server.\n";
 429                         exit( 1 );
 430                 }
 431
 432                 $responseData = FormatJson::decode( $response, true );
 433
 434                 if ( !is_array( $responseData ) ) {
 435                         print "CodeReview API response not recognized...\n";
 436                         wfDebug( "Unrecognized CodeReview API response: $response\n" );
 437                         exit( 1 );
 438                 }
 439
 440                 if ( isset( $responseData['error'] ) ) {
 441                         $code = $responseData['error']['code'];
 442                         $info = $responseData['error']['info'];
 443                         print "CodeReview info upload failed: $code $info\n";
 444                         exit( 1 );
 445                 }
 446         }
 447
 448         function post( $url, $data ) {
 449                 return Http::post( $url, array( 'postData' => $data ) );
 450         }
 451 }
 452
 453 class TestFileIterator implements Iterator {
 454         private $file;
 455         private $fh;
 456         private $parser;
 457         private $index = 0;
 458         private $test;
 459         private $lineNum;
 460         private $eof;
 461
 462         function __construct( $file, $parser = null ) {
 463                 global $IP;
 464
 465                 $this->file = $file;
 466                 $this->fh = fopen( $this->file, "rt" );
 467
 468                 if ( !$this->fh ) {
 469                         wfDie( "Couldn't open file '$file'\n" );
 470                 }
 471
 472                 $this->parser = $parser;
 473
 474                 if ( $this->parser ) {
 475                         $this->parser->showRunFile( wfRelativePath( $this->file, $IP ) );
 476                 }
 477
 478                 $this->lineNum = $this->index = 0;
 479         }
 480
 481         function setParser( MediaWikiParserTest $parser ) {
 482                 $this->parser = $parser;
 483         }
 484
 485         function rewind() {
 486                 if ( fseek( $this->fh, 0 ) ) {
 487                         wfDie( "Couldn't fseek to the start of '$this->file'\n" );
 488                 }
 489
 490                 $this->index = -1;
 491                 $this->lineNum = 0;
 492                 $this->eof = false;
 493                 $this->next();
 494
 495                 return true;
 496         }
 497
 498         function current() {
 499                 return $this->test;
 500         }
 501
 502         function key() {
 503                 return $this->index;
 504         }
 505
 506         function next() {
 507                 if ( $this->readNextTest() ) {
 508                         $this->index++;
 509                         return true;
 510                 } else {
 511                         $this->eof = true;
 512                 }
 513         }
 514
 515         function valid() {
 516                 return $this->eof != true;
 517         }
 518
 519         function readNextTest() {
 520                 $data = array();
 521                 $section = null;
 522
 523                 while ( false !== ( $line = fgets( $this->fh ) ) ) {
 524                         $this->lineNum++;
 525                         $matches = array();
 526
 527                         if ( preg_match( '/^!!\s*(\w+)/', $line, $matches ) ) {
 528                                 $section = strtolower( $matches[1] );
 529
 530                                 if ( $section == 'endarticle' ) {
 531                                         if ( !isset( $data['text'] ) ) {
 532                                                 wfDie( "'endarticle' without 'text' at line {$this->lineNum} of $this->file\n" );
 533                                         }
 534
 535                                         if ( !isset( $data['article'] ) ) {
 536                                                 wfDie( "'endarticle' without 'article' at line {$this->lineNum} of $this->file\n" );
 537                                         }
 538
 539                                         $title = Title::newFromText( ParserTest::chomp( $data['article'] ) );
 540                                         $aid = $title->getArticleID( Title::GAID_FOR_UPDATE );
 541                                         if ( $aid == 0 )
 542                                                 ParserTest::addArticle( $data['article'], $data['text'], $this->lineNum );
 543                                         $data = array();
 544                                         $section = null;
 545
 546                                         continue;
 547                                 }
 548
 549                                 if ( $section == 'endhooks' ) {
 550                                         if ( !isset( $data['hooks'] ) ) {
 551                                                 wfDie( "'endhooks' without 'hooks' at line {$this->lineNum} of $this->file\n" );
 552                                         }
 553
 554                                         foreach ( explode( "\n", $data['hooks'] ) as $line ) {
 555                                                 $line = trim( $line );
 556
 557                                                 if ( $line ) {
 558                                                         if ( $this->parser && !$this->parser->requireHook( $line ) ) {
 559                                                                 return false;
 560                                                         }
 561                                                 }
 562                                         }
 563
 564                                         $data = array();
 565                                         $section = null;
 566
 567                                         continue;
 568                                 }
 569
 570                                 if ( $section == 'endfunctionhooks' ) {
 571                                         if ( !isset( $data['functionhooks'] ) ) {
 572                                                 wfDie( "'endfunctionhooks' without 'functionhooks' at line {$this->lineNum} of $this->file\n" );
 573                                         }
 574
 575                                         foreach ( explode( "\n", $data['functionhooks'] ) as $line ) {
 576                                                 $line = trim( $line );
 577
 578                                                 if ( $line ) {
 579                                                         if ( $this->parser && !$this->parser->requireFunctionHook( $line ) ) {
 580                                                                 return false;
 581                                                         }
 582                                                 }
 583                                         }
 584
 585                                         $data = array();
 586                                         $section = null;
 587
 588                                         continue;
 589                                 }
 590
 591                                 if ( $section == 'end' ) {
 592                                         if ( !isset( $data['test'] ) ) {
 593                                                 wfDie( "'end' without 'test' at line {$this->lineNum} of $this->file\n" );
 594                                         }
 595
 596                                         if ( !isset( $data['input'] ) ) {
 597                                                 wfDie( "'end' without 'input' at line {$this->lineNum} of $this->file\n" );
 598                                         }
 599
 600                                         if ( !isset( $data['result'] ) ) {
 601                                                 wfDie( "'end' without 'result' at line {$this->lineNum} of $this->file\n" );
 602                                         }
 603
 604                                         if ( !isset( $data['options'] ) ) {
 605                                                 $data['options'] = '';
 606                                         }
 607
 608                                         if ( !isset( $data['config'] ) )
 609                                                 $data['config'] = '';
 610
 611                                         if ( $this->parser
 612                                                  && ( ( preg_match( '/\\bdisabled\\b/i', $data['options'] ) && !$this->parser->runDisabled )
 613                                                          || !preg_match( "/" . $this->parser->regex . "/i", $data['test'] ) )  ) {
 614                                                 # disabled test
 615                                                 $data = array();
 616                                                 $section = null;
 617
 618                                                 continue;
 619                                         }
 620
 621                                         global $wgUseTeX;
 622
 623                                         if ( $this->parser &&
 624                                                  preg_match( '/\\bmath\\b/i', $data['options'] ) && !$wgUseTeX ) {
 625                                                 # don't run math tests if $wgUseTeX is set to false in LocalSettings
 626                                                 $data = array();
 627                                                 $section = null;
 628
 629                                                 continue;
 630                                         }
 631
 632                                         if ( $this->parser ) {
 633                                                 $this->test = array(
 634                                                         'test' => ParserTest::chomp( $data['test'] ),
 635                                                         'input' => ParserTest::chomp( $data['input'] ),
 636                                                         'result' => ParserTest::chomp( $data['result'] ),
 637                                                         'options' => ParserTest::chomp( $data['options'] ),
 638                                                         'config' => ParserTest::chomp( $data['config'] ) );
 639                                         } else {
 640                                                 $this->test['test'] = $data['test'];
 641                                         }
 642
 643                                         return true;
 644                                 }
 645
 646                                 if ( isset ( $data[$section] ) ) {
 647                                         wfDie( "duplicate section '$section' at line {$this->lineNum} of $this->file\n" );
 648                                 }
 649
 650                                 $data[$section] = '';
 651
 652                                 continue;
 653                         }
 654
 655                         if ( $section ) {
 656                                 $data[$section] .= $line;
 657                         }
 658                 }
 659
 660                 return false;
 661         }
 662 }