3 use Wikimedia\Rdbms\IDatabase
;
5 require_once __DIR__
. '/Maintenance.php';
8 * Maintenance script that cleans up archive rows with duplicated ar_rev_id,
9 * both within archive and between archive and revision.
11 * @ingroup Maintenance
14 class DeduplicateArchiveRevId
extends LoggedUpdateMaintenance
{
16 /** @var array|null */
17 private $arActorQuery = null;
20 private $reassigned = 0;
22 public function __construct() {
23 parent
::__construct();
24 $this->addDescription(
25 'Clean up duplicate ar_rev_id, both within archive and between archive and revision.'
27 $this->setBatchSize( 10000 );
30 protected function getUpdateKey() {
34 protected function doDBUpdates() {
35 $this->output( "Deduplicating ar_rev_id...\n" );
36 $dbw = $this->getDB( DB_MASTER
);
37 // Sanity check. If this is a new install, we don't need to do anything here.
38 if ( PopulateArchiveRevId
::isNewInstall( $dbw ) ) {
39 $this->output( "New install, nothing to do here.\n" );
43 PopulateArchiveRevId
::checkMysqlAutoIncrementBug( $dbw );
45 $minId = $dbw->selectField( 'archive', 'MIN(ar_rev_id)', [], __METHOD__
);
46 $maxId = $dbw->selectField( 'archive', 'MAX(ar_rev_id)', [], __METHOD__
);
47 $batchSize = $this->getBatchSize();
49 $this->arActorQuery
= ActorMigration
::newMigration()->getJoin( 'ar_user' );
50 $revActorQuery = ActorMigration
::newMigration()->getJoin( 'rev_user' );
52 for ( $id = $minId; $id <= $maxId; $id +
= $batchSize ) {
53 $endId = min( $maxId, $id +
$batchSize - 1 );
55 $this->beginTransaction( $dbw, __METHOD__
);
57 // Lock the archive and revision table rows for the IDs we're checking
58 // to try to prevent deletions or undeletions from confusing things.
62 [ 'ar_rev_id >= ' . (int)$id, 'ar_rev_id <= ' . (int)$endId ],
69 [ 'rev_id >= ' . (int)$id, 'rev_id <= ' . (int)$endId ],
71 [ 'LOCK IN SHARE MODE' ]
74 // Figure out the ar_rev_ids we actually need to look at
76 [ 'archive', 'revision' ] +
$revActorQuery['tables'],
77 [ 'rev_id', 'rev_timestamp', 'rev_sha1' ] +
$revActorQuery['fields'],
78 [ 'ar_rev_id >= ' . (int)$id, 'ar_rev_id <= ' . (int)$endId ],
81 [ 'revision' => [ 'JOIN', 'ar_rev_id = rev_id' ] ] +
$revActorQuery['joins']
84 foreach ( $res as $row ) {
85 $revRows[$row->rev_id
] = $row;
88 $arRevIds = $dbw->selectFieldValues(
91 [ 'ar_rev_id >= ' . (int)$id, 'ar_rev_id <= ' . (int)$endId ],
93 [ 'GROUP BY' => 'ar_rev_id', 'HAVING' => 'COUNT(*) > 1' ]
95 $arRevIds = array_values( array_unique( array_merge( $arRevIds, array_keys( $revRows ) ) ) );
98 $this->processArRevIds( $dbw, $arRevIds, $revRows );
101 $this->output( "... $id-$endId\n" );
102 $this->commitTransaction( $dbw, __METHOD__
);
106 "Finished deduplicating ar_rev_id. $this->deleted rows deleted, "
107 . "$this->reassigned assigned new IDs.\n"
113 * Process a set of ar_rev_ids
114 * @param IDatabase $dbw
115 * @param int[] $arRevIds IDs to process
116 * @param object[] $revRows Existing revision-table row data
118 private function processArRevIds( IDatabase
$dbw, array $arRevIds, array $revRows ) {
119 // Select all the data we need for deduplication
121 [ 'archive' ] +
$this->arActorQuery
['tables'],
122 [ 'ar_id', 'ar_rev_id', 'ar_namespace', 'ar_title', 'ar_timestamp', 'ar_sha1' ]
123 +
$this->arActorQuery
['fields'],
124 [ 'ar_rev_id' => $arRevIds ],
127 $this->arActorQuery
['joins']
130 // Determine which rows we need to delete or reassign
134 foreach ( $res as $row ) {
135 // Revision-table row exists?
136 if ( isset( $revRows[$row->ar_rev_id
] ) ) {
137 $revRow = $revRows[$row->ar_rev_id
];
139 // Record the rev_id as seen, so the code below will always delete or reassign.
140 if ( !isset( $seen[$revRow->rev_id
] ) ) {
141 $seen[$revRow->rev_id
] = [
142 'first' => "revision row",
146 // Delete the archive row if it seems to be the same regardless
147 // of page, because moves can change IDs and titles.
148 if ( $row->ar_timestamp
=== $revRow->rev_timestamp
&&
149 $row->ar_sha1
=== $revRow->rev_sha1
&&
150 $row->ar_user
=== $revRow->rev_user
&&
151 $row->ar_user_text
=== $revRow->rev_user_text
154 "Row $row->ar_id duplicates revision row for rev_id $revRow->rev_id, deleting\n"
156 $toDelete[] = $row->ar_id
;
161 $key = $this->getSeenKey( $row );
162 if ( !isset( $seen[$row->ar_rev_id
] ) ) {
163 // This rev_id hasn't even been seen yet, nothing to do besides record it.
164 $seen[$row->ar_rev_id
] = [
165 'first' => "archive row $row->ar_id",
168 } elseif ( !isset( $seen[$row->ar_rev_id
][$key] ) ) {
169 // The rev_id was seen, but not this particular change. Reassign it.
170 $seen[$row->ar_rev_id
][$key] = $row->ar_id
;
172 "Row $row->ar_id conflicts with {$seen[$row->ar_rev_id]['first']} "
173 . "for rev_id $row->ar_rev_id, reassigning\n"
175 $toReassign[] = $row->ar_id
;
177 // The rev_id was seen with a row that matches this change. Delete it.
179 "Row $row->ar_id duplicates archive row {$seen[$row->ar_rev_id][$key]} "
180 . "for rev_id $row->ar_rev_id, deleting\n"
182 $toDelete[] = $row->ar_id
;
186 // Perform the updates
188 $dbw->delete( 'archive', [ 'ar_id' => $toDelete ], __METHOD__
);
189 $this->deleted +
= $dbw->affectedRows();
192 $this->reassigned +
= PopulateArchiveRevId
::reassignArRevIds( $dbw, $toReassign );
197 * Make a key identifying a "unique" change from a row
201 private function getSeenKey( $row ) {
202 return implode( "\n", [
214 $maintClass = "DeduplicateArchiveRevId";
215 require_once RUN_MAINTENANCE_IF_MAIN
;