From e1a23c9aad323dba775918b894bc2630f81122cd Mon Sep 17 00:00:00 2001 From: Amir Sarabadani Date: Sun, 18 Mar 2018 00:34:32 +0100 Subject: [PATCH] Introduce deleteAutoPatrolLogs maintenance script It's useful to delete old patrol logs that are not useful Bug: T189594 Change-Id: I605bb85f172eb25df45ed83ce50a3d1044f1c281 --- autoload.php | 1 + maintenance/deleteAutoPatrolLogs.php | 196 +++++++++++++ .../maintenance/deleteAutoPatrolLogsTest.php | 261 ++++++++++++++++++ 3 files changed, 458 insertions(+) create mode 100644 maintenance/deleteAutoPatrolLogs.php create mode 100644 tests/phpunit/maintenance/deleteAutoPatrolLogsTest.php diff --git a/autoload.php b/autoload.php index b5f3e4a067..36c9b1734c 100644 --- a/autoload.php +++ b/autoload.php @@ -365,6 +365,7 @@ $wgAutoloadLocalClasses = [ 'DeleteAction' => __DIR__ . '/includes/actions/DeleteAction.php', 'DeleteArchivedFiles' => __DIR__ . '/maintenance/deleteArchivedFiles.php', 'DeleteArchivedRevisions' => __DIR__ . '/maintenance/deleteArchivedRevisions.php', + 'DeleteAutoPatrolLogs' => __DIR__ . '/maintenance/deleteAutoPatrolLogs.php', 'DeleteBatch' => __DIR__ . '/maintenance/deleteBatch.php', 'DeleteDefaultMessages' => __DIR__ . '/maintenance/deleteDefaultMessages.php', 'DeleteEqualMessages' => __DIR__ . '/maintenance/deleteEqualMessages.php', diff --git a/maintenance/deleteAutoPatrolLogs.php b/maintenance/deleteAutoPatrolLogs.php new file mode 100644 index 0000000000..3082d60a02 --- /dev/null +++ b/maintenance/deleteAutoPatrolLogs.php @@ -0,0 +1,196 @@ +addDescription( 'Remove autopatrol logs in the logging table' ); + $this->addOption( 'dry-run', 'Print debug info instead of actually deleting' ); + $this->addOption( + 'check-old', + 'Check old patrol logs (for deleting old format autopatrols).' . + 'Note that this will not delete rows older than 2011 (MediaWiki 1.18).' + ); + $this->addOption( + 'before', + 'Timestamp to delete only before that time, all MediaWiki timestamp formats are accepted', + false, + true + ); + $this->addOption( + 'from-id', + 'First row (log id) to start updating from', + false, + true + ); + $this->addOption( + 'sleep', + 'Sleep time (in seconds) between every batch', + false, + true + ); + $this->setBatchSize( 1000 ); + } + + public function execute() { + $sleep = (int)$this->getOption( 'sleep', 10 ); + $fromId = $this->getOption( 'from-id', null ); + $this->countDown( 5 ); + while ( true ) { + if ( $this->hasOption( 'check-old' ) ) { + $rowsData = $this->getRowsOld( $fromId ); + // We reached end of the table + if ( !$rowsData ) { + break; + } + $rows = $rowsData['rows']; + $fromId = $rowsData['lastId']; + + // There is nothing to delete in this batch + if ( !$rows ) { + continue; + } + } else { + $rows = $this->getRows( $fromId ); + if ( !$rows ) { + break; + } + $fromId = end( $rows ); + } + + if ( $this->hasOption( 'dry-run' ) ) { + $this->output( 'These rows will get deleted: ' . implode( ', ', $rows ) . "\n" ); + } else { + $this->deleteRows( $rows ); + $this->output( 'Processed up to row id ' . end( $rows ) . "\n" ); + } + + if ( $sleep > 0 ) { + sleep( $sleep ); + } + } + } + + private function getRows( $fromId ) { + $dbr = MediaWiki\MediaWikiServices::getInstance()->getDBLoadBalancer()->getConnection( + DB_REPLICA + ); + $before = $this->getOption( 'before', false ); + + $conds = [ + 'log_type' => 'patrol', + 'log_action' => 'autopatrol', + ]; + + if ( $fromId ) { + $conds[] = 'log_id > ' . $dbr->addQuotes( $fromId ); + } + + if ( $before ) { + $conds[] = 'log_timestamp < ' . $dbr->addQuotes( $dbr->timestamp( $before ) ); + } + + return $dbr->selectFieldValues( + 'logging', + 'log_id', + $conds, + __METHOD__, + [ 'LIMIT' => $this->getBatchSize() ] + ); + } + + private function getRowsOld( $fromId ) { + $dbr = MediaWiki\MediaWikiServices::getInstance()->getDBLoadBalancer()->getConnection( + DB_REPLICA + ); + $batchSize = $this->getBatchSize(); + $before = $this->getOption( 'before', false ); + + $conds = [ + 'log_type' => 'patrol', + 'log_action' => 'patrol', + ]; + + if ( $fromId ) { + $conds[] = 'log_id > ' . $dbr->addQuotes( $fromId ); + } + + if ( $before ) { + $conds[] = 'log_timestamp < ' . $dbr->addQuotes( $dbr->timestamp( $before ) ); + } + + $result = $dbr->select( + 'logging', + [ 'log_id', 'log_params' ], + $conds, + __METHOD__, + [ 'LIMIT' => $batchSize ] + ); + + $last = null; + $autopatrolls = []; + foreach ( $result as $row ) { + $last = $row->log_id; + Wikimedia\suppressWarnings(); + $params = unserialize( $row->log_params ); + Wikimedia\restoreWarnings(); + + // Skipping really old rows, before 2011 + if ( is_array( $params ) && !array_key_exists( '6::auto', $params ) ) { + continue; + } + + $auto = $params['6::auto']; + if ( $auto ) { + $autopatrolls[] = $row->log_id; + } + } + + if ( $last === null ) { + return null; + } + + return [ 'rows' => $autopatrolls, 'lastId' => $last ]; + } + + private function deleteRows( array $rows ) { + $dbw = MediaWiki\MediaWikiServices::getInstance()->getDBLoadBalancer()->getConnection( + DB_MASTER + ); + + $dbw->delete( + 'logging', + [ 'log_id' => $rows ], + __METHOD__ + ); + + MediaWiki\MediaWikiServices::getInstance()->getDBLoadBalancerFactory()->waitForReplication(); + } + +} + +$maintClass = DeleteAutoPatrolLogs::class; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/tests/phpunit/maintenance/deleteAutoPatrolLogsTest.php b/tests/phpunit/maintenance/deleteAutoPatrolLogsTest.php new file mode 100644 index 0000000000..af29ff9ca7 --- /dev/null +++ b/tests/phpunit/maintenance/deleteAutoPatrolLogsTest.php @@ -0,0 +1,261 @@ +tablesUsed = [ 'logging' ]; + + $this->cleanLoggingTable(); + $this->insertLoggingData(); + } + + private function cleanLoggingTable() { + wfGetDB( DB_MASTER )->delete( 'logging', '*' ); + } + + private function insertLoggingData() { + $logs = []; + + // Manual patrolling + $logs[] = [ + 'log_type' => 'patrol', + 'log_action' => 'patrol', + 'log_user' => 7251, + 'log_params' => '', + 'log_timestamp' => 20041223210426 + ]; + + // Autopatrol #1 + $logs[] = [ + 'log_type' => 'patrol', + 'log_action' => 'autopatrol', + 'log_user' => 7252, + 'log_params' => '', + 'log_timestamp' => 20051223210426 + ]; + + // Block + $logs[] = [ + 'log_type' => 'block', + 'log_action' => 'block', + 'log_user' => 7253, + 'log_params' => '', + 'log_timestamp' => 20061223210426 + ]; + + // Autopatrol #2 + $logs[] = [ + 'log_type' => 'patrol', + 'log_action' => 'autopatrol', + 'log_user' => 7254, + 'log_params' => '', + 'log_timestamp' => 20071223210426 + ]; + + // Autopatrol #3 old way + $logs[] = [ + 'log_type' => 'patrol', + 'log_action' => 'patrol', + 'log_user' => 7255, + 'log_params' => serialize( [ '6::auto' => true ] ), + 'log_timestamp' => 20081223210426 + ]; + + // Manual patrol #2 old way + $logs[] = [ + 'log_type' => 'patrol', + 'log_action' => 'patrol', + 'log_user' => 7256, + 'log_params' => serialize( [ '6::auto' => false ] ), + 'log_timestamp' => 20091223210426 + ]; + + wfGetDB( DB_MASTER )->insert( 'logging', $logs ); + } + + public function testBasicRun() { + $this->maintenance->loadWithArgv( [ '--sleep', '0', '-q' ] ); + + $this->maintenance->execute(); + + $remainingLogs = wfGetDB( DB_REPLICA )->select( + [ 'logging' ], + [ 'log_type', 'log_action', 'log_user' ], + [], + __METHOD__, + [ 'ORDER BY' => 'log_id' ] + ); + + $expected = [ + (object)[ + 'log_type' => 'patrol', + 'log_action' => 'patrol', + 'log_user' => '7251', + ], + (object)[ + 'log_type' => 'block', + 'log_action' => 'block', + 'log_user' => '7253', + ], + (object)[ + 'log_type' => 'patrol', + 'log_action' => 'patrol', + 'log_user' => '7255', + ], + (object)[ + 'log_type' => 'patrol', + 'log_action' => 'patrol', + 'log_user' => '7256', + ], + ]; + $this->assertEquals( $expected, iterator_to_array( $remainingLogs, false ) ); + } + + public function testDryRun() { + $this->maintenance->loadWithArgv( [ '--sleep', '0', '--dry-run', '-q' ] ); + + $this->maintenance->execute(); + + $remainingLogs = wfGetDB( DB_REPLICA )->select( + [ 'logging' ], + [ 'log_type', 'log_action', 'log_user' ], + [], + __METHOD__, + [ 'ORDER BY' => 'log_id' ] + ); + + $expected = [ + (object)[ + 'log_type' => 'patrol', + 'log_action' => 'patrol', + 'log_user' => '7251', + ], + (object)[ + 'log_type' => 'patrol', + 'log_action' => 'autopatrol', + 'log_user' => '7252', + ], + (object)[ + 'log_type' => 'block', + 'log_action' => 'block', + 'log_user' => '7253', + ], + (object)[ + 'log_type' => 'patrol', + 'log_action' => 'autopatrol', + 'log_user' => '7254', + ], + (object)[ + 'log_type' => 'patrol', + 'log_action' => 'patrol', + 'log_user' => '7255', + ], + (object)[ + 'log_type' => 'patrol', + 'log_action' => 'patrol', + 'log_user' => '7256', + ], + ]; + $this->assertEquals( $expected, iterator_to_array( $remainingLogs, false ) ); + } + + public function testRunWithTimestamp() { + $this->maintenance->loadWithArgv( [ '--sleep', '0', '--before', '20060123210426', '-q' ] ); + + $this->maintenance->execute(); + + $remainingLogs = wfGetDB( DB_REPLICA )->select( + [ 'logging' ], + [ 'log_type', 'log_action', 'log_user' ], + [], + __METHOD__, + [ 'ORDER BY' => 'log_id' ] + ); + + $expected = [ + (object)[ + 'log_type' => 'patrol', + 'log_action' => 'patrol', + 'log_user' => '7251', + ], + (object)[ + 'log_type' => 'block', + 'log_action' => 'block', + 'log_user' => '7253', + ], + (object)[ + 'log_type' => 'patrol', + 'log_action' => 'autopatrol', + 'log_user' => '7254', + ], + (object)[ + 'log_type' => 'patrol', + 'log_action' => 'patrol', + 'log_user' => '7255', + ], + (object)[ + 'log_type' => 'patrol', + 'log_action' => 'patrol', + 'log_user' => '7256', + ] + ]; + $this->assertEquals( $expected, iterator_to_array( $remainingLogs, false ) ); + } + + public function testRunWithCheckOld() { + $this->maintenance->loadWithArgv( [ '--sleep', '0', '--check-old', '-q' ] ); + + $this->maintenance->execute(); + + $remainingLogs = wfGetDB( DB_REPLICA )->select( + [ 'logging' ], + [ 'log_type', 'log_action', 'log_user' ], + [], + __METHOD__, + [ 'ORDER BY' => 'log_id' ] + ); + + $expected = [ + (object)[ + 'log_type' => 'patrol', + 'log_action' => 'patrol', + 'log_user' => '7251', + ], + (object)[ + 'log_type' => 'patrol', + 'log_action' => 'autopatrol', + 'log_user' => '7252', + ], + (object)[ + 'log_type' => 'block', + 'log_action' => 'block', + 'log_user' => '7253', + ], + (object)[ + 'log_type' => 'patrol', + 'log_action' => 'autopatrol', + 'log_user' => '7254', + ], + (object)[ + 'log_type' => 'patrol', + 'log_action' => 'patrol', + 'log_user' => '7256', + ] + ]; + $this->assertEquals( $expected, iterator_to_array( $remainingLogs, false ) ); + } + +} -- 2.20.1