3 use Wikimedia\Rdbms\IDatabase
;
5 require_once __DIR__
. '/Maintenance.php';
8 * Maintenance script that cleans up archive rows with duplicated ar_rev_id,
9 * both within archive and between archive and revision.
11 * @ingroup Maintenance
14 class DeduplicateArchiveRevId
extends LoggedUpdateMaintenance
{
16 /** @var array|null */
17 private $arActorQuery = null;
20 private $reassigned = 0;
22 public function __construct() {
23 parent
::__construct();
24 $this->addDescription(
25 'Clean up duplicate ar_rev_id, both within archive and between archive and revision.'
27 $this->setBatchSize( 10000 );
30 protected function getUpdateKey() {
34 protected function doDBUpdates() {
35 $this->output( "Deduplicating ar_rev_id...\n" );
37 $dbw = $this->getDB( DB_MASTER
);
39 $minId = $dbw->selectField( 'archive', 'MIN(ar_rev_id)', [], __METHOD__
);
40 $maxId = $dbw->selectField( 'archive', 'MAX(ar_rev_id)', [], __METHOD__
);
41 $batchSize = $this->getBatchSize();
43 $this->arActorQuery
= ActorMigration
::newMigration()->getJoin( 'ar_user' );
44 $revActorQuery = ActorMigration
::newMigration()->getJoin( 'rev_user' );
46 for ( $id = $minId; $id <= $maxId; $id +
= $batchSize ) {
47 $endId = min( $maxId, $id +
$batchSize - 1 );
49 $this->beginTransaction( $dbw, __METHOD__
);
51 // Lock the archive and revision table rows for the IDs we're checking
52 // to try to prevent deletions or undeletions from confusing things.
56 [ 'ar_rev_id >= ' . (int)$id, 'ar_rev_id <= ' . (int)$endId ],
63 [ 'rev_id >= ' . (int)$id, 'rev_id <= ' . (int)$endId ],
65 [ 'LOCK IN SHARE MODE' ]
68 // Figure out the ar_rev_ids we actually need to look at
70 [ 'archive', 'revision' ] +
$revActorQuery['tables'],
71 [ 'rev_id', 'rev_timestamp', 'rev_sha1' ] +
$revActorQuery['fields'],
72 [ 'ar_rev_id >= ' . (int)$id, 'ar_rev_id <= ' . (int)$endId ],
75 [ 'revision' => [ 'JOIN', 'ar_rev_id = rev_id' ] ] +
$revActorQuery['joins']
78 foreach ( $res as $row ) {
79 $revRows[$row->rev_id
] = $row;
82 $arRevIds = $dbw->selectFieldValues(
85 [ 'ar_rev_id >= ' . (int)$id, 'ar_rev_id <= ' . (int)$endId ],
87 [ 'GROUP BY' => 'ar_rev_id', 'HAVING' => 'COUNT(*) > 1' ]
89 $arRevIds = array_values( array_unique( array_merge( $arRevIds, array_keys( $revRows ) ) ) );
92 $this->processArRevIds( $dbw, $arRevIds, $revRows );
95 $this->output( "... $id-$endId\n" );
96 $this->commitTransaction( $dbw, __METHOD__
);
100 "Finished deduplicating ar_rev_id. $this->deleted rows deleted, "
101 . "$this->reassigned assigned new IDs.\n"
107 * Process a set of ar_rev_ids
108 * @param IDatabase $dbw
109 * @param int[] $arRevIds IDs to process
110 * @param object[] $revRows Existing revision-table row data
112 private function processArRevIds( IDatabase
$dbw, array $arRevIds, array $revRows ) {
113 // Select all the data we need for deduplication
115 [ 'archive' ] +
$this->arActorQuery
['tables'],
116 [ 'ar_id', 'ar_rev_id', 'ar_namespace', 'ar_title', 'ar_timestamp', 'ar_sha1' ]
117 +
$this->arActorQuery
['fields'],
118 [ 'ar_rev_id' => $arRevIds ],
121 $this->arActorQuery
['joins']
124 // Determine which rows we need to delete or reassign
128 foreach ( $res as $row ) {
129 // Revision-table row exists?
130 if ( isset( $revRows[$row->ar_rev_id
] ) ) {
131 $revRow = $revRows[$row->ar_rev_id
];
133 // Record the rev_id as seen, so the code below will always delete or reassign.
134 if ( !isset( $seen[$revRow->rev_id
] ) ) {
135 $seen[$revRow->rev_id
] = [
136 'first' => "revision row",
140 // Delete the archive row if it seems to be the same regardless
141 // of page, because moves can change IDs and titles.
142 if ( $row->ar_timestamp
=== $revRow->rev_timestamp
&&
143 $row->ar_sha1
=== $revRow->rev_sha1
&&
144 $row->ar_user
=== $revRow->rev_user
&&
145 $row->ar_user_text
=== $revRow->rev_user_text
148 "Row $row->ar_id duplicates revision row for rev_id $revRow->rev_id, deleting\n"
150 $toDelete[] = $row->ar_id
;
155 $key = $this->getSeenKey( $row );
156 if ( !isset( $seen[$row->ar_rev_id
] ) ) {
157 // This rev_id hasn't even been seen yet, nothing to do besides record it.
158 $seen[$row->ar_rev_id
] = [
159 'first' => "archive row $row->ar_id",
162 } elseif ( !isset( $seen[$row->ar_rev_id
][$key] ) ) {
163 // The rev_id was seen, but not this particular change. Reassign it.
164 $seen[$row->ar_rev_id
][$key] = $row->ar_id
;
166 "Row $row->ar_id conflicts with {$seen[$row->ar_rev_id]['first']} "
167 . "for rev_id $row->ar_rev_id, reassigning\n"
169 $toReassign[] = $row->ar_id
;
171 // The rev_id was seen with a row that matches this change. Delete it.
173 "Row $row->ar_id duplicates archive row {$seen[$row->ar_rev_id][$key]} "
174 . "for rev_id $row->ar_rev_id, deleting\n"
176 $toDelete[] = $row->ar_id
;
180 // Perform the updates
182 $dbw->delete( 'archive', [ 'ar_id' => $toDelete ], __METHOD__
);
183 $this->deleted +
= $dbw->affectedRows();
186 $this->reassigned +
= PopulateArchiveRevId
::reassignArRevIds( $dbw, $toReassign );
191 * Make a key identifying a "unique" change from a row
195 private function getSeenKey( $row ) {
196 return implode( "\n", [
208 $maintClass = "DeduplicateArchiveRevId";
209 require_once RUN_MAINTENANCE_IF_MAIN
;