From e9d2c09560dd51f7e386b184ceca34f6bd79242c Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Sat, 11 Nov 2006 12:33:46 +0000 Subject: [PATCH] * Add experimental recording/reporting mode to parser tests runner, to compare changes against the previous run. Additional tables 'testrun' and 'testitem' are in maintenance/testRunner.sql, source this and pass --record option to parserTests.php --- RELEASE-NOTES | 5 ++ maintenance/parserTests.inc | 159 +++++++++++++++++++++++++++++++++++- maintenance/testRunner.sql | 35 ++++++++ 3 files changed, 195 insertions(+), 4 deletions(-) create mode 100644 maintenance/testRunner.sql diff --git a/RELEASE-NOTES b/RELEASE-NOTES index b70d7162c9..d315ecdbb4 100644 --- a/RELEASE-NOTES +++ b/RELEASE-NOTES @@ -163,6 +163,11 @@ it from source control: http://www.mediawiki.org/wiki/Download_from_SVN * Approximate height for client-side scaling fallback instead of passing -1 into the HTML output. * Make the DNSBL to check for proxy blocking configurable via $wgSorbsUrl +* Add experimental recording/reporting mode to parser tests runner, to + compare changes against the previous run. + Additional tables 'testrun' and 'testitem' are in maintenance/testRunner.sql, + source this and pass --record option to parserTests.php + == Languages updated == diff --git a/maintenance/parserTests.inc b/maintenance/parserTests.inc index c34ba20b1b..5c4465fdd5 100644 --- a/maintenance/parserTests.inc +++ b/maintenance/parserTests.inc @@ -25,7 +25,7 @@ */ /** */ -$options = array( 'quick', 'color', 'quiet', 'help', 'show-output' ); +$options = array( 'quick', 'color', 'quiet', 'help', 'show-output', 'record' ); $optionsWithArgs = array( 'regex' ); require_once( 'commandLine.inc' ); @@ -98,6 +98,12 @@ class ParserTest { $this->regex = ''; } + if( isset( $options['record'] ) ) { + $this->recorder = new TestRecorder(); + } else { + $this->recorder = new DummyTestRecorder(); + } + $this->hooks = array(); $this->functionHooks = array(); } @@ -138,6 +144,7 @@ class ParserTest { $success = 0; $total = 0; $n = 0; + $this->recorder->start(); while( false !== ($line = fgets( $infile ) ) ) { $n++; if( preg_match( '/^!!\s*(\w+)/', $line, $matches ) ) { @@ -205,14 +212,16 @@ class ParserTest { $section = null; continue; } - if( $this->runTest( + $result = $this->runTest( $this->chomp( $data['test'] ), $this->chomp( $data['input'] ), $this->chomp( $data['result'] ), - $this->chomp( $data['options'] ) ) ) { + $this->chomp( $data['options'] ) ); + if( $result ) { $success++; } $total++; + $this->recorder->record( $this->chomp( $data['test'] ), $result ); $data = array(); $section = null; continue; @@ -227,9 +236,13 @@ class ParserTest { $data[$section] .= $line; } } + $this->recorder->end(); + + print "\n"; + $this->recorder->report(); if( $total > 0 ) { $ratio = wfPercent( 100 * $success / $total ); - print $this->termColor( 1 ) . "\nPassed $success of $total tests ($ratio) "; + print $this->termColor( 1 ) . "Passed $success of $total tests ($ratio) "; if( $success == $total ) { print $this->termColor( 32 ) . "PASSED!"; } else { @@ -827,6 +840,144 @@ class ParserTest { $this->termColor( 0 ); return "$display\n$caret"; } + +} + +class DummyTestRecorder { + function start() { + // dummy + } + + function record( $test, $result ) { + // dummy + } + + function end() { + // dummy + } + + function report() { + // dummy + } +} + +class TestRecorder { + var $db; ///< Database connection to the main DB + var $curRun; ///< run ID number for the current run + var $prevRun; ///< run ID number for the previous run, if any + + function __construct() { + $this->db = wfGetDB( DB_MASTER ); + } + + /** + * Set up result recording; insert a record for the run with the date + * and all that fun stuff + */ + function start() { + $this->db->begin(); + + // We'll make comparisons against the previous run later... + $this->prevRun = $this->db->selectField( 'testrun', 'MAX(tr_id)' ); + + $this->db->insert( 'testrun', + array( + 'tr_date' => $this->db->timestamp(), + 'tr_mw_version' => SpecialVersion::getVersion(), + 'tr_php_version' => phpversion(), + 'tr_db_version' => $this->db->getServerVersion(), + 'tr_uname' => php_uname() + ), + __METHOD__ ); + $this->curRun = $this->db->insertId(); + } + + /** + * Record an individual test item's success or failure to the db + * @param string $test + * @param bool $result + */ + function record( $test, $result ) { + $this->db->insert( 'testitem', + array( + 'ti_run' => $this->curRun, + 'ti_name' => $test, + 'ti_success' => $result ? 1 : 0, + ), + __METHOD__ ); + } + + /** + * Commit transaction and clean up for result recording + */ + function end() { + $this->db->commit(); + } + + function report() { + if( $this->prevRun ) { + $table = array( + array( 'previously failing test(s) now PASSING! :)', 0, 1 ), + array( 'previously PASSING test(s) removed o_O', 1, null ), + array( 'new PASSING test(s) :)', null, 1 ), + + array( 'previously passing test(s) now FAILING! :(', 1, 0 ), + array( 'previously FAILING test(s) removed O_o', 0, null ), + array( 'new FAILING test(s) :(', null, 0 ), + ); + foreach( $table as $blah ) { + list( $label, $before, $after ) = $blah; + $count = $this->comparisonCount( $before, $after ); + if( $count ) { + printf( "%4d %s\n", $count, $label ); + } + } + } else { + print "No previous test runs to compare against.\n"; + } + } + + /** + * :P + */ + private function comparisonCount( $before, $after ) { + $testitem = $this->db->tableName( 'testitem' ); + $prevRun = intval( $this->prevRun ); + $curRun = intval( $this->curRun ); + $prevStatus = $this->condition( $before ); + $curStatus = $this->condition( $after ); + + // note: requires a current mysql for subselects + if( is_null( $after ) ) { + $sql = " + select count(*) as c from $testitem as prev + where prev.ti_run=$prevRun and + prev.ti_success $prevStatus and + (select current.ti_success from testitem as current + where current.ti_run=$curRun + and prev.ti_name=current.ti_name) $curStatus"; + } else { + $sql = " + select count(*) as c from $testitem as current + where current.ti_run=$curRun and + current.ti_success $curStatus and + (select prev.ti_success from testitem as prev + where prev.ti_run=$prevRun + and prev.ti_name=current.ti_name) $prevStatus"; + } + $result = $this->db->query( $sql, __METHOD__ ); + $row = $this->db->fetchObject( $result ); + $this->db->freeResult( $result ); + return $row->c; + } + + private function condition( $value ) { + if( is_null( $value ) ) { + return 'IS NULL'; + } else { + return '=' . intval( $value ); + } + } } diff --git a/maintenance/testRunner.sql b/maintenance/testRunner.sql new file mode 100644 index 0000000000..8591d81df4 --- /dev/null +++ b/maintenance/testRunner.sql @@ -0,0 +1,35 @@ +-- +-- Optional tables for parserTests recording mode +-- With --record option, success data will be saved to these tables, +-- and comparisons of what's changed from the previous run will be +-- displayed at the end of each run. +-- +-- These tables currently require MySQL 5 (or maybe 4.1?) for subselects. +-- + +drop table if exists /*$wgDBprefix*/testitem; +drop table if exists /*$wgDBprefix*/testrun; + +create table /*$wgDBprefix*/testrun ( + tr_id int not null auto_increment, + + tr_date char(14) binary, + tr_mw_version blob, + tr_php_version blob, + tr_db_version blob, + tr_uname blob, + + primary key (tr_id) +) engine=InnoDB; + +create table /*$wgDBprefix*/testitem ( + ti_run int not null, + ti_name varchar(255), + ti_success bool, + + unique key (ti_run, ti_name), + key (ti_run, ti_success), + + foreign key (ti_run) references /*$wgDBprefix*/testrun(tr_id) + on delete cascade +) engine=InnoDB; -- 2.20.1