From d831844364ce5a34bac672b838271b138ce77337 Mon Sep 17 00:00:00 2001 From: Tim Starling Date: Wed, 28 Sep 2016 16:32:17 +1000 Subject: [PATCH] Parser test file editor Add editTests.php, which provides an interactive interface to the parser tests, with semi-automated editing. Change-Id: I1a20d007ba4627d562a16c03849bbad7aec0e516 --- maintenance/Maintenance.php | 30 ++ tests/common/TestsAutoLoader.php | 1 + tests/parser/TestFileEditor.php | 196 +++++++++++++ tests/parser/TestFileReader.php | 6 + tests/parser/TestRecorder.php | 2 +- tests/parser/editTests.php | 490 +++++++++++++++++++++++++++++++ tests/parser/parserTests.php | 3 +- 7 files changed, 726 insertions(+), 2 deletions(-) create mode 100644 tests/parser/TestFileEditor.php create mode 100644 tests/parser/editTests.php diff --git a/maintenance/Maintenance.php b/maintenance/Maintenance.php index 208cf1769c..c74cae2ada 100644 --- a/maintenance/Maintenance.php +++ b/maintenance/Maintenance.php @@ -1502,6 +1502,36 @@ abstract class Maintenance { return fgets( STDIN, 1024 ); } + /** + * Get the terminal size as a two-element array where the first element + * is the width (number of columns) and the second element is the height + * (number of rows). + * + * @return array + */ + public static function getTermSize() { + $default = [ 80, 50 ]; + if ( wfIsWindows() ) { + return $default; + } + // It's possible to get the screen size with VT-100 terminal escapes, + // but reading the responses is not possible without setting raw mode + // (unless you want to require the user to press enter), and that + // requires an ioctl(), which we can't do. So we have to shell out to + // something that can do the relevant syscalls. There are a few + // options. Linux and Mac OS X both have "stty size" which does the + // job directly. + $retval = false; + $size = wfShellExec( 'stty size', $retval ); + if ( $retval !== 0 ) { + return $default; + } + if ( !preg_match( '/^(\d+) (\d+)$/', $size, $m ) ) { + return $default; + } + return [ intval( $m[2] ), intval( $m[1] ) ]; + } + /** * Call this to set up the autoloader to allow classes to be used from the * tests directory. diff --git a/tests/common/TestsAutoLoader.php b/tests/common/TestsAutoLoader.php index a19fea1bd0..0bfa318081 100644 --- a/tests/common/TestsAutoLoader.php +++ b/tests/common/TestsAutoLoader.php @@ -41,6 +41,7 @@ $wgAutoloadClasses += [ 'ParserTestResult' => "$testDir/parser/ParserTestResult.php", 'ParserTestResultNormalizer' => "$testDir/parser/ParserTestResultNormalizer.php", 'PhpunitTestRecorder' => "$testDir/parser/PhpunitTestRecorder.php", + 'TestFileEditor' => "$testDir/parser/TestFileEditor.php", 'TestFileReader' => "$testDir/parser/TestFileReader.php", 'TestRecorder' => "$testDir/parser/TestRecorder.php", 'TidySupport' => "$testDir/parser/TidySupport.php", diff --git a/tests/parser/TestFileEditor.php b/tests/parser/TestFileEditor.php new file mode 100644 index 0000000000..05b1216fe6 --- /dev/null +++ b/tests/parser/TestFileEditor.php @@ -0,0 +1,196 @@ +execute(); + return $editor->result; + } + + private function __construct( $text, array $deletions, array $changes, $warningCallback ) { + $this->lines = explode( "\n", $text ); + $this->numLines = count( $this->lines ); + $this->deletions = array_flip( $deletions ); + $this->changes = $changes; + $this->pos = 0; + $this->warningCallback = $warningCallback; + $this->result = ''; + } + + private function execute() { + while ( $this->pos < $this->numLines ) { + $line = $this->lines[$this->pos]; + switch ( $this->getHeading( $line ) ) { + case 'test': + $this->parseTest(); + break; + case 'hooks': + case 'functionhooks': + case 'transparenthooks': + $this->parseHooks(); + break; + default: + if ( $this->pos < $this->numLines - 1 ) { + $line .= "\n"; + } + $this->emitComment( $line ); + $this->pos++; + } + } + foreach ( $this->deletions as $deletion => $unused ) { + $this->warning( "Could not find test \"$deletion\" to delete it" ); + } + foreach ( $this->changes as $test => $sectionChanges ) { + foreach ( $sectionChanges as $section => $change ) { + $this->warning( "Could not find section \"$section\" in test \"$test\" " . + "to {$change['op']} it" ); + } + } + } + + private function warning( $text ) { + $cb = $this->warningCallback; + if ( $cb ) { + $cb( $text ); + } + } + + private function getHeading( $line ) { + if ( preg_match( '/^!!\s*(\S+)/', $line, $m ) ) { + return $m[1]; + } else { + return false; + } + } + + private function parseTest() { + $test = []; + $line = $this->lines[$this->pos++]; + $heading = $this->getHeading( $line ); + $section = [ + 'name' => $heading, + 'headingLine' => $line, + 'contents' => '' + ]; + + while ( $this->pos < $this->numLines ) { + $line = $this->lines[$this->pos++]; + $nextHeading = $this->getHeading( $line ); + if ( $nextHeading === 'end' ) { + $test[] = $section; + + // Add trailing line breaks to the "end" section, to allow for neat deletions + $trail = ''; + for ( $i = 0; $i < $this->numLines - $this->pos - 1; $i++ ) { + if ( $this->lines[$this->pos + $i] === '' ) { + $trail .= "\n"; + } else { + break; + } + } + $this->pos += strlen( $trail ); + + $test[] = [ + 'name' => 'end', + 'headingLine' => $line, + 'contents' => $trail + ]; + $this->emitTest( $test ); + return; + } elseif ( $nextHeading !== false ) { + $test[] = $section; + $heading = $nextHeading; + $section = [ + 'name' => $heading, + 'headingLine' => $line, + 'contents' => '' + ]; + } else { + $section['contents'] .= "$line\n"; + } + } + + throw new Exception( 'Unexpected end of file' ); + } + + private function parseHooks() { + $line = $this->lines[$this->pos++]; + $heading = $this->getHeading( $line ); + $expectedEnd = 'end' . $heading; + $contents = $line; + + do { + $line = $this->lines[$this->pos++]; + $nextHeading = $this->getHeading( $line ); + $contents .= "$line\n"; + } while ( $this->pos < $this->numLines && $nextHeading !== $expectedEnd ); + + if ( $nextHeading !== $expectedEnd ) { + throw new Exception( 'Unexpected end of file' ); + } + $this->emitHooks( $heading, $contents ); + } + + protected function emitComment( $contents ) { + $this->result .= $contents; + } + + protected function emitTest( $test ) { + $testName = false; + foreach ( $test as $section ) { + if ( $section['name'] === 'test' ) { + $testName = rtrim( $section['contents'], "\n" ); + } + } + if ( isset( $this->deletions[$testName] ) ) { + // Acknowledge deletion + unset( $this->deletions[$testName] ); + return; + } + if ( isset( $this->changes[$testName] ) ) { + $changes =& $this->changes[$testName]; + foreach ( $test as $i => $section ) { + $sectionName = $section['name']; + if ( isset( $changes[$sectionName] ) ) { + $change = $changes[$sectionName]; + switch ( $change['op'] ) { + case 'rename': + $test[$i]['name'] = $change['value']; + $test[$i]['headingLine'] = "!! {$change['value']}"; + break; + case 'update': + $test[$i]['contents'] = $change['value']; + break; + case 'delete': + $test[$i]['deleted'] = true; + break; + default: + throw new Exception( "Unknown op: ${change['op']}" ); + } + // Acknowledge + // Note that we use the old section name for the rename op + unset( $changes[$sectionName] ); + } + } + } + foreach ( $test as $section ) { + if ( isset( $section['deleted'] ) ) { + continue; + } + $this->result .= $section['headingLine'] . "\n"; + $this->result .= $section['contents']; + } + } + + protected function emitHooks( $heading, $contents ) { + $this->result .= $contents; + } +} diff --git a/tests/parser/TestFileReader.php b/tests/parser/TestFileReader.php index 6279d68fd8..59f88c9325 100644 --- a/tests/parser/TestFileReader.php +++ b/tests/parser/TestFileReader.php @@ -130,12 +130,15 @@ class TestFileReader { 'input' => $data[$input], 'options' => $data['options'], 'config' => $data['config'], + 'line' => $this->sectionLineNum['test'], + 'file' => $this->file ]; if ( $nonTidySection !== false ) { // Add non-tidy test $this->tests[] = [ 'result' => $data[$nonTidySection], + 'resultSection' => $nonTidySection ] + $commonInfo; if ( $tidySection !== false ) { @@ -143,13 +146,16 @@ class TestFileReader { $this->tests[] = [ 'desc' => $data['test'] . ' (with tidy)', 'result' => $data[$tidySection], + 'resultSection' => $tidySection, 'options' => $data['options'] . ' tidy', + 'isSubtest' => true, ] + $commonInfo; } } elseif ( $tidySection !== false ) { // No need to override desc when there is no subtest $this->tests[] = [ 'result' => $data[$tidySection], + 'resultSection' => $tidySection, 'options' => $data['options'] . ' tidy' ] + $commonInfo; } else { diff --git a/tests/parser/TestRecorder.php b/tests/parser/TestRecorder.php index 70215b6efe..4b816991a3 100644 --- a/tests/parser/TestRecorder.php +++ b/tests/parser/TestRecorder.php @@ -32,7 +32,7 @@ * * @since 1.22 */ -abstract class TestRecorder { +class TestRecorder { /** * Called at beginning of the parser test run diff --git a/tests/parser/editTests.php b/tests/parser/editTests.php new file mode 100644 index 0000000000..a9704e69e2 --- /dev/null +++ b/tests/parser/editTests.php @@ -0,0 +1,490 @@ +addOption( 'session-data', 'internal option, do not use', false, true ); + $this->addOption( 'use-tidy-config', + 'Use the wiki\'s Tidy configuration instead of known-good' . + 'defaults.' ); + } + + public function finalSetup() { + parent::finalSetup(); + self::requireTestsAutoloader(); + TestSetup::applyInitialConfig(); + } + + public function execute() { + $this->termWidth = $this->getTermSize()[0] - 1; + + $this->recorder = new TestRecorder(); + $this->setupFileData(); + + if ( $this->hasOption( 'session-data' ) ) { + $this->session = json_decode( $this->getOption( 'session-data' ), true ); + } else { + $this->session = [ 'options' => [] ]; + } + if ( $this->hasOption( 'use-tidy-config' ) ) { + $this->session['options']['use-tidy-config'] = true; + } + $this->runner = new ParserTestRunner( $this->recorder, $this->session['options'] ); + + $this->runTests(); + + if ( $this->numFailed === 0 ) { + if ( $this->numSkipped === 0 ) { + print "All tests passed!\n"; + } else { + print "All tests passed (but skipped {$this->numSkipped})\n"; + } + return; + } + print "{$this->numFailed} test(s) failed.\n"; + $this->showResults(); + } + + protected function setupFileData() { + global $wgParserTestFiles; + $this->testFiles = []; + $this->testCount = 0; + foreach ( $wgParserTestFiles as $file ) { + $fileInfo = TestFileReader::read( $file ); + $this->testFiles[$file] = $fileInfo; + $this->testCount += count( $fileInfo['tests'] ); + } + } + + protected function runTests() { + $teardown = $this->runner->staticSetup(); + $teardown = $this->runner->setupDatabase( $teardown ); + $teardown = $this->runner->setupUploads( $teardown ); + + print "Running tests...\n"; + $this->results = []; + $this->numExecuted = 0; + $this->numSkipped = 0; + $this->numFailed = 0; + foreach ( $this->testFiles as $fileName => $fileInfo ) { + $this->runner->addArticles( $fileInfo['articles'] ); + foreach ( $fileInfo['tests'] as $testInfo ) { + $result = $this->runner->runTest( $testInfo ); + if ( $result === false ) { + $this->numSkipped++; + } elseif ( !$result->isSuccess() ) { + $this->results[$fileName][$testInfo['desc']] = $result; + $this->numFailed++; + } + $this->numExecuted++; + $this->showProgress(); + } + } + print "\n"; + } + + protected function showProgress() { + $done = $this->numExecuted; + $total = $this->testCount; + $width = $this->termWidth - 9; + $pos = round( $width * $done / $total ); + printf( '│' . str_repeat( '█', $pos ) . str_repeat( '-', $width - $pos ) . + "│ %5.1f%%\r", $done / $total * 100 ); + } + + protected function showResults() { + if ( isset( $this->session['startFile'] ) ) { + $startFile = $this->session['startFile']; + $startTest = $this->session['startTest']; + $foundStart = false; + } else { + $startFile = false; + $startTest = false; + $foundStart = true; + } + + $testIndex = 0; + foreach ( $this->testFiles as $fileName => $fileInfo ) { + if ( !isset( $this->results[$fileName] ) ) { + continue; + } + if ( !$foundStart && $startFile !== false && $fileName !== $startFile ) { + $testIndex += count( $this->results[$fileName] ); + continue; + } + foreach ( $fileInfo['tests'] as $testInfo ) { + if ( !isset( $this->results[$fileName][$testInfo['desc']] ) ) { + continue; + } + $result = $this->results[$fileName][$testInfo['desc']]; + $testIndex++; + if ( !$foundStart && $startTest !== false ) { + if ( $testInfo['desc'] !== $startTest ) { + continue; + } + $foundStart = true; + } + + $this->handleFailure( $testIndex, $testInfo, $result ); + } + } + + if ( !$foundStart ) { + print "Could not find the test after a restart, did you rename it?"; + unset( $this->session['startFile'] ); + unset( $this->session['startTest'] ); + $this->showResults(); + } + print "All done\n"; + } + + protected function heading( $text ) { + $term = new AnsiTermColorer; + $heading = "─── $text "; + $heading .= str_repeat( '─', $this->termWidth - mb_strlen( $heading ) ); + $heading = $term->color( 34 ) . $heading . $term->reset() . "\n"; + return $heading; + } + + protected function unifiedDiff( $left, $right ) { + $fromLines = explode( "\n", $left ); + $toLines = explode( "\n", $right ); + $formatter = new UnifiedDiffFormatter; + return $formatter->format( new Diff( $fromLines, $toLines ) ); + } + + protected function handleFailure( $index, $testInfo, $result ) { + $term = new AnsiTermColorer; + $div1 = $term->color( 34 ) . str_repeat( '━', $this->termWidth ) . + $term->reset() . "\n"; + $div2 = $term->color( 34 ) . str_repeat( '─', $this->termWidth ) . + $term->reset() . "\n"; + + print $div1; + print "Failure $index/{$this->numFailed}: {$testInfo['file']} line {$testInfo['line']}\n" . + "{$testInfo['desc']}\n"; + + print $this->heading( 'Input' ); + print "{$testInfo['input']}\n"; + + print $this->heading( 'Alternating expected/actual output' ); + print $this->alternatingAligned( $result->expected, $result->actual ); + + print $this->heading( 'Diff' ); + + $dwdiff = $this->dwdiff( $result->expected, $result->actual ); + if ( $dwdiff !== false ) { + $diff = $dwdiff; + } else { + $diff = $this->unifiedDiff( $result->expected, $result->actual ); + } + print $diff; + + if ( $testInfo['options'] || $testInfo['config'] ) { + print $this->heading( 'Options / Config' ); + if ( $testInfo['options'] ) { + print $testInfo['options'] . "\n"; + } + if ( $testInfo['config'] ) { + print $testInfo['config'] . "\n"; + } + } + + print $div2; + print "What do you want to do?\n"; + $specs = [ + '[R]eload code and run again', + '[U]pdate source file, copy actual to expected', + '[I]gnore' ]; + + if ( strpos( $testInfo['options'], ' tidy' ) === false ) { + if ( empty( $testInfo['isSubtest'] ) ) { + $specs[] = "Enable [T]idy"; + } + } else { + $specs[] = 'Disable [T]idy'; + } + + if ( !empty( $testInfo['isSubtest'] ) ) { + $specs[] = 'Delete [s]ubtest'; + } + $specs[] = '[D]elete test'; + $specs[] = '[Q]uit'; + + $options = []; + foreach ( $specs as $spec ) { + if ( !preg_match( '/^(.*\[)(.)(\].*)$/', $spec, $m ) ) { + throw new MWException( 'Invalid option spec: ' . $spec ); + } + print '* ' . $m[1] . $term->color( 35 ) . $m[2] . $term->color( 0 ) . $m[3] . "\n"; + $options[strtoupper( $m[2] )] = true; + } + + do { + $response = $this->readconsole(); + $cmdResult = false; + if ( $response === false ) { + exit( 0 ); + } + + $response = strtoupper( trim( $response ) ); + if ( !isset( $options[$response] ) ) { + print "Invalid response, please enter a single letter from the list above\n"; + continue; + } + + switch ( strtoupper( trim( $response ) ) ) { + case 'R': + $cmdResult = $this->reload( $testInfo ); + break; + case 'U': + $cmdResult = $this->update( $testInfo, $result ); + break; + case 'I': + return; + case 'T': + $cmdResult = $this->switchTidy( $testInfo ); + break; + case 'S': + $cmdResult = $this->deleteSubtest( $testInfo ); + break; + case 'D': + $cmdResult = $this->deleteTest( $testInfo ); + break; + case 'Q': + exit( 0 ); + } + } while ( !$cmdResult ); + } + + protected function dwdiff( $expected, $actual ) { + if ( !is_executable( '/usr/bin/dwdiff' ) ) { + return false; + } + + $markers = [ + "\n" => '¶', + ' ' => '·', + "\t" => '→' + ]; + $markedExpected = strtr( $expected, $markers ); + $markedActual = strtr( $actual, $markers ); + $diff = $this->unifiedDiff( $markedExpected, $markedActual ); + + $tempFile = tmpfile(); + fwrite( $tempFile, $diff ); + fseek( $tempFile, 0 ); + $pipes = []; + $proc = proc_open( '/usr/bin/dwdiff -Pc --diff-input', + [ 0 => $tempFile, 1 => [ 'pipe', 'w' ], 2 => STDERR ], + $pipes ); + + if ( !$proc ) { + return false; + } + + $result = stream_get_contents( $pipes[1] ); + proc_close( $proc ); + fclose( $tempFile ); + return $result; + } + + protected function alternatingAligned( $expectedStr, $actualStr ) { + $expectedLines = explode( "\n", $expectedStr ); + $actualLines = explode( "\n", $actualStr ); + $maxLines = max( count( $expectedLines ), count( $actualLines ) ); + $result = ''; + for ( $i = 0; $i < $maxLines; $i++ ) { + if ( $i < count( $expectedLines ) ) { + $expectedLine = $expectedLines[$i]; + $expectedChunks = str_split( $expectedLine, $this->termWidth - 3 ); + } else { + $expectedChunks = []; + } + + if ( $i < count( $actualLines ) ) { + $actualLine = $actualLines[$i]; + $actualChunks = str_split( $actualLine, $this->termWidth - 3 ); + } else { + $actualChunks = []; + } + + $maxChunks = max( count( $expectedChunks ), count( $actualChunks ) ); + + for ( $j = 0; $j < $maxChunks; $j++ ) { + if ( isset( $expectedChunks[$j] ) ) { + $result .= "E: " . $expectedChunks[$j]; + if ( $j === count( $expectedChunks ) - 1 ) { + $result .= "¶"; + } + $result .= "\n"; + } else { + $result .= "E:\n"; + } + $result .= "\33[4m" . // underline + "A: "; + if ( isset( $actualChunks[$j] ) ) { + $result .= $actualChunks[$j]; + if ( $j === count( $actualChunks ) - 1 ) { + $result .= "¶"; + } + } + $result .= "\33[0m\n"; // reset + } + } + return $result; + } + + protected function reload( $testInfo ) { + global $argv; + pcntl_exec( PHP_BINARY, [ + $argv[0], + '--session-data', + json_encode( [ + 'startFile' => $testInfo['file'], + 'startTest' => $testInfo['desc'] + ] + $this->session ) ] ); + + print "pcntl_exec() failed\n"; + return false; + } + + protected function findTest( $file, $testInfo ) { + $initialPart = ''; + for ( $i = 1; $i < $testInfo['line']; $i++ ) { + $line = fgets( $file ); + if ( $line === false ) { + print "Error reading from file\n"; + return false; + } + $initialPart .= $line; + } + + $line = fgets( $file ); + if ( !preg_match( '/^!!\s*test/', $line ) ) { + print "Test has moved, cannot edit\n"; + return false; + } + + $testPart = $line; + + $desc = fgets( $file ); + if ( trim( $desc ) !== $testInfo['desc'] ) { + print "Description does not match, cannot edit\n"; + return false; + } + $testPart .= $desc; + return [ $initialPart, $testPart ]; + } + + protected function getOutputFileName( $inputFileName ) { + if ( is_writable( $inputFileName ) ) { + $outputFileName = $inputFileName; + } else { + $outputFileName = wfTempDir() . '/' . basename( $inputFileName ); + print "Cannot write to input file, writing to $outputFileName instead\n"; + } + return $outputFileName; + } + + protected function editTest( $fileName, $deletions, $changes ) { + $text = file_get_contents( $fileName ); + if ( $text === false ) { + print "Unable to open test file!"; + return false; + } + $result = TestFileEditor::edit( $text, $deletions, $changes, + function ( $msg ) { + print "$msg\n"; + } + ); + if ( is_writable( $fileName ) ) { + file_put_contents( $fileName, $result ); + print "Wrote updated file\n"; + } else { + print "Cannot write updated file, here is a patch you can paste:\n\n"; + print + "--- {$fileName}\n" . + "+++ {$fileName}~\n" . + $this->unifiedDiff( $text, $result ) . + "\n"; + } + } + + protected function update( $testInfo, $result ) { + $this->editTest( $testInfo['file'], + [], // deletions + [ // changes + $testInfo['test'] => [ + $testInfo['resultSection'] => [ + 'op' => 'update', + 'value' => $result->actual . "\n" + ] + ] + ] + ); + } + + protected function deleteTest( $testInfo ) { + $this->editTest( $testInfo['file'], + [ $testInfo['test'] ], // deletions + [] // changes + ); + } + + protected function switchTidy( $testInfo ) { + $resultSection = $testInfo['resultSection']; + if ( in_array( $resultSection, [ 'html/php', 'html/*', 'html', 'result' ] ) ) { + $newSection = 'html+tidy'; + } elseif ( in_array( $resultSection, [ 'html/php+tidy', 'html+tidy' ] ) ) { + $newSection = 'html'; + } else { + print "Unrecognised result section name \"$resultSection\""; + return; + } + + $this->editTest( $testInfo['file'], + [], // deletions + [ // changes + $testInfo['test'] => [ + $resultSection => [ + 'op' => 'rename', + 'value' => $newSection + ] + ] + ] + ); + } + + protected function deleteSubtest( $testInfo ) { + $this->editTest( $testInfo['file'], + [], // deletions + [ // changes + $testInfo['test'] => [ + $testInfo['resultSection'] => [ + 'op' => 'delete' + ] + ] + ] + ); + } +} + +$maintClass = 'ParserEditTests'; +require RUN_MAINTENANCE_IF_MAIN; diff --git a/tests/parser/parserTests.php b/tests/parser/parserTests.php index 38923f0494..1d0867abf0 100644 --- a/tests/parser/parserTests.php +++ b/tests/parser/parserTests.php @@ -68,7 +68,8 @@ class ParserTestsMaintenance extends Maintenance { 'are: removeTbody to remove tags; and trimWhitespace ' . 'to trim whitespace from the start and end of text nodes.', false, true ); - $this->addOption( 'use-tidy-config', 'Use the wiki\'s Tidy configuration instead of known-good' . + $this->addOption( 'use-tidy-config', + 'Use the wiki\'s Tidy configuration instead of known-good' . 'defaults.' ); } -- 2.20.1