3 * This program is free software; you can redistribute it and/or modify
4 * it under the terms of the GNU General Public License as published by
5 * the Free Software Foundation; either version 2 of the License, or
6 * (at your option) any later version.
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
13 * You should have received a copy of the GNU General Public License along
14 * with this program; if not, write to the Free Software Foundation, Inc.,
15 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
16 * http://www.gnu.org/copyleft/gpl.html
19 * @ingroup Maintenance
22 use MediaWiki\MediaWikiServices
;
23 use MediaWiki\Revision\SlotRecord
;
24 use MediaWiki\Storage\BlobStore
;
25 use MediaWiki\Storage\NameTableStore
;
26 use MediaWiki\Storage\SqlBlobStore
;
27 use Wikimedia\Assert\Assert
;
28 use Wikimedia\Rdbms\IDatabase
;
29 use Wikimedia\Rdbms\IResultWrapper
;
31 require_once __DIR__
. '/Maintenance.php';
34 * Populate the content and slot tables.
37 class PopulateContentTables
extends Maintenance
{
42 /** @var NameTableStore */
43 private $contentModelStore;
45 /** @var NameTableStore */
46 private $slotRoleStore;
54 /** @var array|null Map "{$modelId}:{$address}" to content_id */
55 private $contentRowMap = null;
57 private $count = 0, $totalCount = 0;
59 public function __construct() {
60 parent
::__construct();
62 $this->addDescription( 'Populate content and slot tables' );
63 $this->addOption( 'table', 'revision or archive table, or `all` to populate both', false,
65 $this->addOption( 'reuse-content',
66 'Reuse content table rows when the address and model are the same. '
67 . 'This will increase the script\'s time and memory usage, perhaps significantly.',
69 $this->addOption( 'start-revision', 'The rev_id to start at', false, true );
70 $this->addOption( 'start-archive', 'The ar_rev_id to start at', false, true );
71 $this->setBatchSize( 500 );
74 private function initServices() {
75 $this->dbw
= $this->getDB( DB_MASTER
);
76 $this->contentModelStore
= MediaWikiServices
::getInstance()->getContentModelStore();
77 $this->slotRoleStore
= MediaWikiServices
::getInstance()->getSlotRoleStore();
78 $this->blobStore
= MediaWikiServices
::getInstance()->getBlobStore();
80 // Don't trust the cache for the NameTableStores, in case something went
81 // wrong during a previous run (see T224949#5325895).
82 $this->contentModelStore
->reloadMap();
83 $this->slotRoleStore
->reloadMap();
84 $this->mainRoleId
= $this->slotRoleStore
->acquireId( SlotRecord
::MAIN
);
87 public function execute() {
88 $multiContentRevisionSchemaMigrationStage =
89 $this->getConfig()->get( 'MultiContentRevisionSchemaMigrationStage' );
91 $t0 = microtime( true );
93 if ( ( $multiContentRevisionSchemaMigrationStage & SCHEMA_COMPAT_WRITE_NEW
) === 0 ) {
95 '...cannot update while \$wgMultiContentRevisionSchemaMigrationStage '
96 . 'does not have the SCHEMA_COMPAT_WRITE_NEW bit set.'
101 $this->initServices();
103 if ( $this->getOption( 'reuse-content', false ) ) {
104 $this->loadContentMap();
107 foreach ( $this->getTables() as $table ) {
108 $this->populateTable( $table );
111 $elapsed = microtime( true ) - $t0;
112 $this->writeln( "Done. Processed $this->totalCount rows in $elapsed seconds" );
119 private function getTables() {
120 $table = $this->getOption( 'table', 'all' );
121 $validTableOptions = [ 'all', 'revision', 'archive' ];
123 if ( !in_array( $table, $validTableOptions ) ) {
124 $this->fatalError( 'Invalid table. Must be either `revision` or `archive` or `all`' );
127 if ( $table === 'all' ) {
128 $tables = [ 'revision', 'archive' ];
130 $tables = [ $table ];
136 private function loadContentMap() {
137 $t0 = microtime( true );
138 $this->writeln( "Loading existing content table rows..." );
139 $this->contentRowMap
= [];
140 $dbr = $this->getDB( DB_REPLICA
);
145 [ 'content_id', 'content_address', 'content_model' ],
146 $from ?
"content_id > $from" : '',
148 [ 'ORDER BY' => 'content_id', 'LIMIT' => $this->getBatchSize() ]
150 if ( !$res ||
!$res->numRows() ) {
153 foreach ( $res as $row ) {
154 $from = $row->content_id
;
155 $this->contentRowMap
["{$row->content_model}:{$row->content_address}"] = $row->content_id
;
158 $elapsed = microtime( true ) - $t0;
159 $this->writeln( "Loaded " . count( $this->contentRowMap
) . " rows in $elapsed seconds" );
163 * @param string $table
165 private function populateTable( $table ) {
166 $t0 = microtime( true );
168 $this->writeln( "Populating $table..." );
170 if ( $table === 'revision' ) {
172 $tables = [ 'revision', 'slots', 'page' ];
176 'sha1' => 'rev_sha1',
177 'text_id' => 'rev_text_id',
178 'content_model' => 'rev_content_model',
179 'namespace' => 'page_namespace',
180 'title' => 'page_title',
183 'slots' => [ 'LEFT JOIN', 'rev_id=slot_revision_id' ],
184 'page' => [ 'LEFT JOIN', 'rev_page=page_id' ],
186 $startOption = 'start-revision';
188 $idField = 'ar_rev_id';
189 $tables = [ 'archive', 'slots' ];
191 'rev_id' => 'ar_rev_id',
194 'text_id' => 'ar_text_id',
195 'content_model' => 'ar_content_model',
196 'namespace' => 'ar_namespace',
197 'title' => 'ar_title',
200 'slots' => [ 'LEFT JOIN', 'ar_rev_id=slot_revision_id' ],
202 $startOption = 'start-archive';
205 if ( !$this->dbw
->fieldExists( $table, $fields['text_id'], __METHOD__
) ) {
206 $this->writeln( "No need to populate, $table.{$fields['text_id']} field does not exist" );
210 $minmax = $this->dbw
->selectRow(
212 [ 'min' => "MIN( $idField )", 'max' => "MAX( $idField )" ],
216 if ( $this->hasOption( $startOption ) ) {
217 $minmax->min
= (int)$this->getOption( $startOption );
219 if ( !$minmax ||
!is_numeric( $minmax->min
) ||
!is_numeric( $minmax->max
) ) {
221 $minmax = (object)[ 'min' => 1, 'max' => 0 ];
224 $batchSize = $this->getBatchSize();
226 for ( $startId = $minmax->min
; $startId <= $minmax->max
; $startId +
= $batchSize ) {
227 $endId = min( $startId +
$batchSize - 1, $minmax->max
);
228 $rows = $this->dbw
->select(
232 "$idField >= $startId",
233 "$idField <= $endId",
234 'slot_revision_id IS NULL',
237 [ 'ORDER BY' => 'rev_id' ],
240 if ( $rows->numRows() !== 0 ) {
241 $this->populateContentTablesForRowBatch( $rows, $startId, $table );
244 $elapsed = microtime( true ) - $t0;
246 "... $table processed up to revision id $endId of {$minmax->max}"
247 . " ($this->count rows in $elapsed seconds)"
251 $elapsed = microtime( true ) - $t0;
252 $this->writeln( "Done populating $table table. Processed $this->count rows in $elapsed seconds" );
256 * @param IResultWrapper $rows
257 * @param int $startId
258 * @param string $table
261 private function populateContentTablesForRowBatch( IResultWrapper
$rows, $startId, $table ) {
262 $this->beginTransaction( $this->dbw
, __METHOD__
);
264 if ( $this->contentRowMap
=== null ) {
267 $map = &$this->contentRowMap
;
272 // Step 1: Figure out content rows needing insertion.
274 foreach ( $rows as $row ) {
275 $revisionId = $row->rev_id
;
277 Assert
::invariant( $revisionId !== null, 'rev_id must not be null' );
279 $model = $this->getContentModel( $row );
280 $modelId = $this->contentModelStore
->acquireId( $model );
281 $address = SqlBlobStore
::makeAddressFromTextId( $row->text_id
);
283 $key = "{$modelId}:{$address}";
284 $contentKeys[$revisionId] = $key;
286 if ( !isset( $map[$key] ) ) {
287 $this->fillMissingFields( $row, $model, $address );
291 'content_size' => (int)$row->len
,
292 'content_sha1' => $row->sha1
,
293 'content_model' => $modelId,
294 'content_address' => $address,
299 // Step 2: Insert them, then read them back in for use in the next step.
300 if ( $contentRows ) {
301 $id = $this->dbw
->selectField( 'content', 'MAX(content_id)', '', __METHOD__
);
302 $this->dbw
->insert( 'content', $contentRows, __METHOD__
);
303 $res = $this->dbw
->select(
305 [ 'content_id', 'content_model', 'content_address' ],
306 'content_id > ' . (int)$id,
309 foreach ( $res as $row ) {
310 $key = $row->content_model
. ':' . $row->content_address
;
311 $map[$key] = $row->content_id
;
315 // Step 3: Insert the slot rows.
317 foreach ( $rows as $row ) {
318 $revisionId = $row->rev_id
;
319 $contentId = $map[$contentKeys[$revisionId]] ??
false;
320 if ( $contentId === false ) {
321 throw new \
RuntimeException( "Content row for $revisionId not found after content insert" );
324 'slot_revision_id' => $revisionId,
325 'slot_role_id' => $this->mainRoleId
,
326 'slot_content_id' => $contentId,
327 // There's no way to really know the previous revision, so assume no inheriting.
328 // rev_parent_id can get changed on undeletions, and deletions can screw up
329 // rev_timestamp ordering.
330 'slot_origin' => $revisionId,
333 $this->dbw
->insert( 'slots', $slotRows, __METHOD__
);
334 $this->count +
= count( $slotRows );
335 $this->totalCount +
= count( $slotRows );
336 } catch ( \Exception
$e ) {
337 $this->rollbackTransaction( $this->dbw
, __METHOD__
);
338 $this->fatalError( "Failed to populate content table $table row batch starting at $startId "
339 . "due to exception: " . $e->__toString() );
342 $this->commitTransaction( $this->dbw
, __METHOD__
);
346 * @param \stdClass $row
349 private function getContentModel( $row ) {
350 if ( isset( $row->content_model
) ) {
351 return $row->content_model
;
354 $title = Title
::makeTitle( $row->namespace, $row->title
);
356 return ContentHandler
::getDefaultModelFor( $title );
362 private function writeln( $msg ) {
363 $this->output( "$msg\n" );
367 * Compute any missing fields in $row.
368 * The way the missing values are computed must correspond to the way this is done in SlotRecord.
370 * @param object $row to be modified
371 * @param string $model
372 * @param string $address
374 private function fillMissingFields( $row, $model, $address ) {
375 if ( !isset( $row->content_model
) ) {
376 // just for completeness
377 $row->content_model
= $model;
380 if ( isset( $row->len
) && isset( $row->sha1
) && $row->sha1
!== '' ) {
381 // No need to load the content, quite now.
385 $blob = $this->blobStore
->getBlob( $address );
387 if ( !isset( $row->len
) ) {
388 // NOTE: The nominal size of the content may not be the length of the raw blob.
389 $handler = ContentHandler
::getForModelID( $model );
390 $content = $handler->unserializeContent( $blob );
392 $row->len
= $content->getSize();
395 if ( !isset( $row->sha1
) ||
$row->sha1
=== '' ) {
396 $row->sha1
= SlotRecord
::base36Sha1( $blob );
401 $maintClass = 'PopulateContentTables';
402 require_once RUN_MAINTENANCE_IF_MAIN
;