From 957c9a76a2c37785acf71bca1a2b7a312cacded1 Mon Sep 17 00:00:00 2001 From: Platonides Date: Sat, 26 Feb 2011 23:36:30 +0000 Subject: [PATCH] Readding parserTests.php and testHelpers.inc with history back to r73884 when they had also been moved without telling svn about it. Follow up to r78389 and r82873. --- tests/parserTests.php | 91 ++++++ tests/testHelpers.inc | 651 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 742 insertions(+) create mode 100644 tests/parserTests.php create mode 100644 tests/testHelpers.inc diff --git a/tests/parserTests.php b/tests/parserTests.php new file mode 100644 index 0000000000..bc3281624e --- /dev/null +++ b/tests/parserTests.php @@ -0,0 +1,91 @@ + + * http://www.mediawiki.org/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Maintenance + */ + +$options = array( 'quick', 'color', 'quiet', 'help', 'show-output', 'record', 'run-disabled' ); +$optionsWithArgs = array( 'regex', 'seed', 'setversion' ); + +require_once( dirname( __FILE__ ) . '/../commandLine.inc' ); + +if ( isset( $options['help'] ) ) { + echo << Run test cases from a custom file instead of parserTests.txt + --record Record tests in database + --compare Compare with recorded results, without updating the database. + --setversion When using --record, set the version string to use (useful + with git-svn so that you can get the exact revision) + --keep-uploads Re-use the same upload directory for each test, don't delete it + --fuzz Do a fuzz test instead of a normal test + --seed Start the fuzz test from the specified seed + --help Show this help message + --run-disabled run disabled tests + --upload Upload test results to remote wiki (per \$wgParserTestRemote) + +ENDS; + exit( 0 ); +} + +# Cases of weird db corruption were encountered when running tests on earlyish +# versions of SQLite +if ( wfGetDB( DB_MASTER )->getType() == 'sqlite' ) { + $version = wfGetDB( DB_MASTER )->getServerVersion(); + if ( version_compare( $version, '3.6' ) < 0 ) { + die( "Parser tests require SQLite version 3.6 or later, you have $version\n" ); + } +} + +# There is a convention that the parser should never +# refer to $wgTitle directly, but instead use the title +# passed to it. +$wgTitle = Title::newFromText( 'Parser test script do not use' ); +$tester = new ParserTest($options); + +if ( isset( $options['file'] ) ) { + $files = array( $options['file'] ); +} else { + // Default parser tests and any set from extensions or local config + $files = $wgParserTestFiles; +} + +# Print out software version to assist with locating regressions +$version = SpecialVersion::getVersion(); +echo( "This is MediaWiki version {$version}.\n\n" ); + +if ( isset( $options['fuzz'] ) ) { + $tester->fuzzTest( $files ); +} else { + $ok = $tester->runTestsFromFiles( $files ); + exit ( $ok ? 0 : 1 ); +} diff --git a/tests/testHelpers.inc b/tests/testHelpers.inc new file mode 100644 index 0000000000..dc8b0fa8c2 --- /dev/null +++ b/tests/testHelpers.inc @@ -0,0 +1,651 @@ +color( 0 ); + } +} + +/* A colour-less terminal */ +class DummyTermColorer { + public function color( $color ) { + return ''; + } + + public function reset() { + return ''; + } +} + +class TestRecorder { + var $parent; + var $term; + + function __construct( $parent ) { + $this->parent = $parent; + $this->term = $parent->term; + } + + function start() { + $this->total = 0; + $this->success = 0; + } + + function record( $test, $result ) { + $this->total++; + $this->success += ( $result ? 1 : 0 ); + } + + function end() { + // dummy + } + + function report() { + if ( $this->total > 0 ) { + $this->reportPercentage( $this->success, $this->total ); + } else { + wfDie( "No tests found.\n" ); + } + } + + function reportPercentage( $success, $total ) { + $ratio = wfPercent( 100 * $success / $total ); + print $this->term->color( 1 ) . "Passed $success of $total tests ($ratio)... "; + + if ( $success == $total ) { + print $this->term->color( 32 ) . "ALL TESTS PASSED!"; + } else { + $failed = $total - $success ; + print $this->term->color( 31 ) . "$failed tests failed!"; + } + + print $this->term->reset() . "\n"; + + return ( $success == $total ); + } +} + +class DbTestPreviewer extends TestRecorder { + protected $lb; // /< Database load balancer + protected $db; // /< Database connection to the main DB + protected $curRun; // /< run ID number for the current run + protected $prevRun; // /< run ID number for the previous run, if any + protected $results; // /< Result array + + /** + * This should be called before the table prefix is changed + */ + function __construct( $parent ) { + parent::__construct( $parent ); + + $this->lb = wfGetLBFactory()->newMainLB(); + // This connection will have the wiki's table prefix, not parsertest_ + $this->db = $this->lb->getConnection( DB_MASTER ); + } + + /** + * Set up result recording; insert a record for the run with the date + * and all that fun stuff + */ + function start() { + parent::start(); + + if ( ! $this->db->tableExists( 'testrun' ) + or ! $this->db->tableExists( 'testitem' ) ) + { + print "WARNING> `testrun` table not found in database.\n"; + $this->prevRun = false; + } else { + // We'll make comparisons against the previous run later... + $this->prevRun = $this->db->selectField( 'testrun', 'MAX(tr_id)' ); + } + + $this->results = array(); + } + + function record( $test, $result ) { + parent::record( $test, $result ); + $this->results[$test] = $result; + } + + function report() { + if ( $this->prevRun ) { + // f = fail, p = pass, n = nonexistent + // codes show before then after + $table = array( + 'fp' => 'previously failing test(s) now PASSING! :)', + 'pn' => 'previously PASSING test(s) removed o_O', + 'np' => 'new PASSING test(s) :)', + + 'pf' => 'previously passing test(s) now FAILING! :(', + 'fn' => 'previously FAILING test(s) removed O_o', + 'nf' => 'new FAILING test(s) :(', + 'ff' => 'still FAILING test(s) :(', + ); + + $prevResults = array(); + + $res = $this->db->select( 'testitem', array( 'ti_name', 'ti_success' ), + array( 'ti_run' => $this->prevRun ), __METHOD__ ); + + foreach ( $res as $row ) { + if ( !$this->parent->regex + || preg_match( "/{$this->parent->regex}/i", $row->ti_name ) ) + { + $prevResults[$row->ti_name] = $row->ti_success; + } + } + + $combined = array_keys( $this->results + $prevResults ); + + # Determine breakdown by change type + $breakdown = array(); + foreach ( $combined as $test ) { + if ( !isset( $prevResults[$test] ) ) { + $before = 'n'; + } elseif ( $prevResults[$test] == 1 ) { + $before = 'p'; + } else /* if ( $prevResults[$test] == 0 )*/ { + $before = 'f'; + } + + if ( !isset( $this->results[$test] ) ) { + $after = 'n'; + } elseif ( $this->results[$test] == 1 ) { + $after = 'p'; + } else /*if ( $this->results[$test] == 0 ) */ { + $after = 'f'; + } + + $code = $before . $after; + + if ( isset( $table[$code] ) ) { + $breakdown[$code][$test] = $this->getTestStatusInfo( $test, $after ); + } + } + + # Write out results + foreach ( $table as $code => $label ) { + if ( !empty( $breakdown[$code] ) ) { + $count = count( $breakdown[$code] ); + printf( "\n%4d %s\n", $count, $label ); + + foreach ( $breakdown[$code] as $differing_test_name => $statusInfo ) { + print " * $differing_test_name [$statusInfo]\n"; + } + } + } + } else { + print "No previous test runs to compare against.\n"; + } + + print "\n"; + parent::report(); + } + + /** + * Returns a string giving information about when a test last had a status change. + * Could help to track down when regressions were introduced, as distinct from tests + * which have never passed (which are more change requests than regressions). + */ + private function getTestStatusInfo( $testname, $after ) { + // If we're looking at a test that has just been removed, then say when it first appeared. + if ( $after == 'n' ) { + $changedRun = $this->db->selectField ( 'testitem', + 'MIN(ti_run)', + array( 'ti_name' => $testname ), + __METHOD__ ); + $appear = $this->db->selectRow ( 'testrun', + array( 'tr_date', 'tr_mw_version' ), + array( 'tr_id' => $changedRun ), + __METHOD__ ); + + return "First recorded appearance: " + . date( "d-M-Y H:i:s", strtotime ( $appear->tr_date ) ) + . ", " . $appear->tr_mw_version; + } + + // Otherwise, this test has previous recorded results. + // See when this test last had a different result to what we're seeing now. + $conds = array( + 'ti_name' => $testname, + 'ti_success' => ( $after == 'f' ? "1" : "0" ) ); + + if ( $this->curRun ) { + $conds[] = "ti_run != " . $this->db->addQuotes ( $this->curRun ); + } + + $changedRun = $this->db->selectField ( 'testitem', 'MAX(ti_run)', $conds, __METHOD__ ); + + // If no record of ever having had a different result. + if ( is_null ( $changedRun ) ) { + if ( $after == "f" ) { + return "Has never passed"; + } else { + return "Has never failed"; + } + } + + // Otherwise, we're looking at a test whose status has changed. + // (i.e. it used to work, but now doesn't; or used to fail, but is now fixed.) + // In this situation, give as much info as we can as to when it changed status. + $pre = $this->db->selectRow ( 'testrun', + array( 'tr_date', 'tr_mw_version' ), + array( 'tr_id' => $changedRun ), + __METHOD__ ); + $post = $this->db->selectRow ( 'testrun', + array( 'tr_date', 'tr_mw_version' ), + array( "tr_id > " . $this->db->addQuotes ( $changedRun ) ), + __METHOD__, + array( "LIMIT" => 1, "ORDER BY" => 'tr_id' ) + ); + + if ( $post ) { + $postDate = date( "d-M-Y H:i:s", strtotime ( $post->tr_date ) ) . ", {$post->tr_mw_version}"; + } else { + $postDate = 'now'; + } + + return ( $after == "f" ? "Introduced" : "Fixed" ) . " between " + . date( "d-M-Y H:i:s", strtotime ( $pre->tr_date ) ) . ", " . $pre->tr_mw_version + . " and $postDate"; + + } + + /** + * Commit transaction and clean up for result recording + */ + function end() { + $this->lb->commitMasterChanges(); + $this->lb->closeAll(); + parent::end(); + } + +} + +class DbTestRecorder extends DbTestPreviewer { + var $version; + + /** + * Set up result recording; insert a record for the run with the date + * and all that fun stuff + */ + function start() { + $this->db->begin(); + + if ( ! $this->db->tableExists( 'testrun' ) + or ! $this->db->tableExists( 'testitem' ) ) + { + print "WARNING> `testrun` table not found in database. Trying to create table.\n"; + $this->db->sourceFile( $this->db->patchPath( 'patch-testrun.sql' ) ); + echo "OK, resuming.\n"; + } + + parent::start(); + + $this->db->insert( 'testrun', + array( + 'tr_date' => $this->db->timestamp(), + 'tr_mw_version' => $this->version, + 'tr_php_version' => phpversion(), + 'tr_db_version' => $this->db->getServerVersion(), + 'tr_uname' => php_uname() + ), + __METHOD__ ); + if ( $this->db->getType() === 'postgres' ) { + $this->curRun = $this->db->currentSequenceValue( 'testrun_id_seq' ); + } else { + $this->curRun = $this->db->insertId(); + } + } + + /** + * Record an individual test item's success or failure to the db + * + * @param $test String + * @param $result Boolean + */ + function record( $test, $result ) { + parent::record( $test, $result ); + + $this->db->insert( 'testitem', + array( + 'ti_run' => $this->curRun, + 'ti_name' => $test, + 'ti_success' => $result ? 1 : 0, + ), + __METHOD__ ); + } +} + +class RemoteTestRecorder extends TestRecorder { + function start() { + parent::start(); + + $this->results = array(); + $this->ping( 'running' ); + } + + function record( $test, $result ) { + parent::record( $test, $result ); + $this->results[$test] = (bool)$result; + } + + function end() { + $this->ping( 'complete', $this->results ); + parent::end(); + } + + /** + * Inform a CodeReview instance that we've started or completed a test run... + * + * @param $status string: "running" - tell it we've started + * "complete" - provide test results array + * "abort" - something went horribly awry + * @param $results array of test name => true/false + */ + function ping( $status, $results = false ) { + global $wgParserTestRemote, $IP; + + $remote = $wgParserTestRemote; + $revId = SpecialVersion::getSvnRevision( $IP ); + $jsonResults = FormatJson::encode( $results ); + + if ( !$remote ) { + print "Can't do remote upload without configuring \$wgParserTestRemote!\n"; + exit( 1 ); + } + + // Generate a hash MAC to validate our credentials + $message = array( + $remote['repo'], + $remote['suite'], + $revId, + $status, + ); + + if ( $status == "complete" ) { + $message[] = $jsonResults; + } + $hmac = hash_hmac( "sha1", implode( "|", $message ), $remote['secret'] ); + + $postData = array( + 'action' => 'codetestupload', + 'format' => 'json', + 'repo' => $remote['repo'], + 'suite' => $remote['suite'], + 'rev' => $revId, + 'status' => $status, + 'hmac' => $hmac, + ); + + if ( $status == "complete" ) { + $postData['results'] = $jsonResults; + } + + $response = $this->post( $remote['api-url'], $postData ); + + if ( $response === false ) { + print "CodeReview info upload failed to reach server.\n"; + exit( 1 ); + } + + $responseData = FormatJson::decode( $response, true ); + + if ( !is_array( $responseData ) ) { + print "CodeReview API response not recognized...\n"; + wfDebug( "Unrecognized CodeReview API response: $response\n" ); + exit( 1 ); + } + + if ( isset( $responseData['error'] ) ) { + $code = $responseData['error']['code']; + $info = $responseData['error']['info']; + print "CodeReview info upload failed: $code $info\n"; + exit( 1 ); + } + } + + function post( $url, $data ) { + return Http::post( $url, array( 'postData' => $data ) ); + } +} + +class TestFileIterator implements Iterator { + private $file; + private $fh; + private $parserTest; /* An instance of ParserTest (parserTests.php) or MediaWikiParserTest (phpunit) */ + private $index = 0; + private $test; + private $lineNum; + private $eof; + + function __construct( $file, $parserTest = null ) { + global $IP; + + $this->file = $file; + $this->fh = fopen( $this->file, "rt" ); + + if ( !$this->fh ) { + wfDie( "Couldn't open file '$file'\n" ); + } + + $this->parserTest = $parserTest; + + if ( $this->parserTest ) { + $this->parserTest->showRunFile( wfRelativePath( $this->file, $IP ) ); + } + + $this->lineNum = $this->index = 0; + } + + function rewind() { + if ( fseek( $this->fh, 0 ) ) { + wfDie( "Couldn't fseek to the start of '$this->file'\n" ); + } + + $this->index = -1; + $this->lineNum = 0; + $this->eof = false; + $this->next(); + + return true; + } + + function current() { + return $this->test; + } + + function key() { + return $this->index; + } + + function next() { + if ( $this->readNextTest() ) { + $this->index++; + return true; + } else { + $this->eof = true; + } + } + + function valid() { + return $this->eof != true; + } + + function readNextTest() { + $data = array(); + $section = null; + + while ( false !== ( $line = fgets( $this->fh ) ) ) { + $this->lineNum++; + $matches = array(); + + if ( preg_match( '/^!!\s*(\w+)/', $line, $matches ) ) { + $section = strtolower( $matches[1] ); + + if ( $section == 'endarticle' ) { + if ( !isset( $data['text'] ) ) { + wfDie( "'endarticle' without 'text' at line {$this->lineNum} of $this->file\n" ); + } + + if ( !isset( $data['article'] ) ) { + wfDie( "'endarticle' without 'article' at line {$this->lineNum} of $this->file\n" ); + } + + if ( $this->parserTest ) { + $this->parserTest->addArticle( ParserTest::chomp( $data['article'] ), $data['text'], $this->lineNum ); + } else {wfDie("JAJA"); + ParserTest::addArticle( $data['article'], $data['text'], $this->lineNum ); + } + $data = array(); + $section = null; + + continue; + } + + if ( $section == 'endhooks' ) { + if ( !isset( $data['hooks'] ) ) { + wfDie( "'endhooks' without 'hooks' at line {$this->lineNum} of $this->file\n" ); + } + + foreach ( explode( "\n", $data['hooks'] ) as $line ) { + $line = trim( $line ); + + if ( $line ) { + if ( $this->parserTest && !$this->parserTest->requireHook( $line ) ) { + return false; + } + } + } + + $data = array(); + $section = null; + + continue; + } + + if ( $section == 'endfunctionhooks' ) { + if ( !isset( $data['functionhooks'] ) ) { + wfDie( "'endfunctionhooks' without 'functionhooks' at line {$this->lineNum} of $this->file\n" ); + } + + foreach ( explode( "\n", $data['functionhooks'] ) as $line ) { + $line = trim( $line ); + + if ( $line ) { + if ( $this->parserTest && !$this->parserTest->requireFunctionHook( $line ) ) { + return false; + } + } + } + + $data = array(); + $section = null; + + continue; + } + + if ( $section == 'end' ) { + if ( !isset( $data['test'] ) ) { + wfDie( "'end' without 'test' at line {$this->lineNum} of $this->file\n" ); + } + + if ( !isset( $data['input'] ) ) { + wfDie( "'end' without 'input' at line {$this->lineNum} of $this->file\n" ); + } + + if ( !isset( $data['result'] ) ) { + wfDie( "'end' without 'result' at line {$this->lineNum} of $this->file\n" ); + } + + if ( !isset( $data['options'] ) ) { + $data['options'] = ''; + } + + if ( !isset( $data['config'] ) ) + $data['config'] = ''; + + if ( $this->parserTest + && ( ( preg_match( '/\\bdisabled\\b/i', $data['options'] ) && !$this->parserTest->runDisabled ) + || !preg_match( "/" . $this->parserTest->regex . "/i", $data['test'] ) ) ) { + # disabled test + $data = array(); + $section = null; + + continue; + } + + global $wgUseTeX; + + if ( $this->parserTest && + preg_match( '/\\bmath\\b/i', $data['options'] ) && !$wgUseTeX ) { + # don't run math tests if $wgUseTeX is set to false in LocalSettings + $data = array(); + $section = null; + + continue; + } + + if ( $this->parserTest ) { + $this->test = array( + 'test' => ParserTest::chomp( $data['test'] ), + 'input' => ParserTest::chomp( $data['input'] ), + 'result' => ParserTest::chomp( $data['result'] ), + 'options' => ParserTest::chomp( $data['options'] ), + 'config' => ParserTest::chomp( $data['config'] ) ); + } else { + $this->test['test'] = $data['test']; + } + + return true; + } + + if ( isset ( $data[$section] ) ) { + wfDie( "duplicate section '$section' at line {$this->lineNum} of $this->file\n" ); + } + + $data[$section] = ''; + + continue; + } + + if ( $section ) { + $data[$section] .= $line; + } + } + + return false; + } +} -- 2.20.1