From: Brad Jorsch Date: Fri, 18 Nov 2016 20:42:11 +0000 (-0500) Subject: Populate externallinks.el_index_60 and drop default X-Git-Tag: 1.34.0-rc.0~5429^2 X-Git-Url: http://git.cyclocoop.org/%7D%7Cconcat%7B?a=commitdiff_plain;h=aef72c5cf62806a5146b9c41e36a0b070de1af46;p=lhc%2Fweb%2Fwiklou.git Populate externallinks.el_index_60 and drop default Adds a maintenance script to populate the field, has that be automatically run during update.php, and drops the no-longer-needed default value on the column (where possible: mssql has some sort of constraint thing going on that I have no idea how it works). Bug: T59176 Change-Id: I971edf013a1a39466aca3b6e34c915cb24fd3aa7 --- diff --git a/autoload.php b/autoload.php index 6e123a10b0..e0b810b15d 100644 --- a/autoload.php +++ b/autoload.php @@ -1161,6 +1161,7 @@ $wgAutoloadLocalClasses = [ 'PopulateBacklinkNamespace' => __DIR__ . '/maintenance/populateBacklinkNamespace.php', 'PopulateCategory' => __DIR__ . '/maintenance/populateCategory.php', 'PopulateContentModel' => __DIR__ . '/maintenance/populateContentModel.php', + 'PopulateExternallinksIndex60' => __DIR__ . '/maintenance/populateExternallinksIndex60.php', 'PopulateFilearchiveSha1' => __DIR__ . '/maintenance/populateFilearchiveSha1.php', 'PopulateImageSha1' => __DIR__ . '/maintenance/populateImageSha1.php', 'PopulateInterwiki' => __DIR__ . '/maintenance/populateInterwiki.php', diff --git a/includes/deferred/LinksUpdate.php b/includes/deferred/LinksUpdate.php index 4ddd15117b..398df01b9a 100644 --- a/includes/deferred/LinksUpdate.php +++ b/includes/deferred/LinksUpdate.php @@ -569,6 +569,7 @@ class LinksUpdate extends DataUpdate implements EnqueueableDataUpdate { 'el_from' => $this->mId, 'el_to' => $url, 'el_index' => $index, + 'el_index_60' => substr( $index, 0, 60 ), ]; } } diff --git a/includes/installer/DatabaseUpdater.php b/includes/installer/DatabaseUpdater.php index 1f6110bd44..e2deed10a2 100644 --- a/includes/installer/DatabaseUpdater.php +++ b/includes/installer/DatabaseUpdater.php @@ -1287,4 +1287,21 @@ abstract class DatabaseUpdater { } } + /** + * Populates the externallinks.el_index_60 field + * @since 1.32 + */ + protected function populateExternallinksIndex60() { + if ( !$this->updateRowExists( 'populate externallinks.el_index_60' ) ) { + $this->output( + "Populating el_index_60 field, printing progress markers. For large\n" . + "databases, you may want to hit Ctrl-C and do this manually with\n" . + "maintenance/populateExternallinksIndex60.php.\n" + ); + $task = $this->maintenance->runChild( 'PopulateExternallinksIndex60', + 'populateExternallinksIndex60.php' ); + $task->execute(); + $this->output( "done.\n" ); + } + } } diff --git a/includes/installer/MssqlUpdater.php b/includes/installer/MssqlUpdater.php index e389135be6..e04ad8ffb4 100644 --- a/includes/installer/MssqlUpdater.php +++ b/includes/installer/MssqlUpdater.php @@ -131,6 +131,9 @@ class MssqlUpdater extends DatabaseUpdater { // 1.32 [ 'addTable', 'change_tag_def', 'patch-change_tag_def.sql' ], + [ 'populateExternallinksIndex60' ], + [ 'modifyfield', 'externallinks', 'el_index_60', + 'patch-externallinks-el_index_60-drop-default.sql' ], ]; } diff --git a/includes/installer/MysqlUpdater.php b/includes/installer/MysqlUpdater.php index b4bb194195..1b9faf24ab 100644 --- a/includes/installer/MysqlUpdater.php +++ b/includes/installer/MysqlUpdater.php @@ -351,6 +351,9 @@ class MysqlUpdater extends DatabaseUpdater { // 1.32 [ 'addTable', 'change_tag_def', 'patch-change_tag_def.sql' ], + [ 'populateExternallinksIndex60' ], + [ 'modifyfield', 'externallinks', 'el_index_60', + 'patch-externallinks-el_index_60-drop-default.sql' ], ]; } diff --git a/includes/installer/OracleUpdater.php b/includes/installer/OracleUpdater.php index b08e26f0ab..d55b52089c 100644 --- a/includes/installer/OracleUpdater.php +++ b/includes/installer/OracleUpdater.php @@ -150,6 +150,7 @@ class OracleUpdater extends DatabaseUpdater { // 1.32 [ 'addTable', 'change_tag_def', 'patch-change_tag_def.sql' ], + [ 'populateExternallinksIndex60' ], // KEEP THIS AT THE BOTTOM!! [ 'doRebuildDuplicateFunction' ], diff --git a/includes/installer/PostgresUpdater.php b/includes/installer/PostgresUpdater.php index c72e206a55..70547b4231 100644 --- a/includes/installer/PostgresUpdater.php +++ b/includes/installer/PostgresUpdater.php @@ -572,6 +572,8 @@ class PostgresUpdater extends DatabaseUpdater { // 1.32 [ 'addTable', 'change_tag_def', 'patch-change_tag_def.sql' ], + [ 'populateExternallinksIndex60' ], + [ 'dropDefault', 'externallinks', 'el_index_60' ], ]; } @@ -907,6 +909,20 @@ END; } } + /** + * Drop a default value from a field + * @since 1.32 + * @param string $table + * @param string $field + */ + protected function dropDefault( $table, $field ) { + $info = $this->db->fieldInfo( $table, $field ); + if ( $info->defaultValue() !== false ) { + $this->output( "Removing '$table.$field' default value\n" ); + $this->db->query( "ALTER TABLE $table ALTER $field DROP DEFAULT" ); + } + } + protected function changeNullableField( $table, $field, $null, $update = false ) { $fi = $this->db->fieldInfo( $table, $field ); if ( is_null( $fi ) ) { diff --git a/includes/installer/SqliteUpdater.php b/includes/installer/SqliteUpdater.php index 76e1ca968f..c2f22133ff 100644 --- a/includes/installer/SqliteUpdater.php +++ b/includes/installer/SqliteUpdater.php @@ -215,6 +215,9 @@ class SqliteUpdater extends DatabaseUpdater { // 1.32 [ 'addTable', 'change_tag_def', 'patch-change_tag_def.sql' ], + [ 'populateExternallinksIndex60' ], + [ 'modifyfield', 'externallinks', 'el_index_60', + 'patch-externallinks-el_index_60-drop-default.sql' ], ]; } diff --git a/maintenance/archives/patch-externallinks-el_index_60-drop-default.sql b/maintenance/archives/patch-externallinks-el_index_60-drop-default.sql new file mode 100644 index 0000000000..f9242c52bb --- /dev/null +++ b/maintenance/archives/patch-externallinks-el_index_60-drop-default.sql @@ -0,0 +1,2 @@ +-- @since 1.32 +ALTER TABLE /*$wgDBprefix*/externallinks ALTER COLUMN el_index_60 DROP DEFAULT; diff --git a/maintenance/fixExtLinksProtocolRelative.php b/maintenance/fixExtLinksProtocolRelative.php index c70b8be7c9..f5861e5a0b 100644 --- a/maintenance/fixExtLinksProtocolRelative.php +++ b/maintenance/fixExtLinksProtocolRelative.php @@ -71,11 +71,13 @@ class FixExtLinksProtocolRelative extends LoggedUpdateMaintenance { 'el_from' => $row->el_from, 'el_to' => $row->el_to, 'el_index' => "http:{$row->el_index}", + 'el_index_60' => substr( "http:{$row->el_index}", 0, 60 ), ], [ 'el_from' => $row->el_from, 'el_to' => $row->el_to, 'el_index' => "https:{$row->el_index}", + 'el_index_60' => substr( "https:{$row->el_index}", 0, 60 ), ] ], __METHOD__, [ 'IGNORE' ] ); diff --git a/maintenance/mssql/archives/patch-externallinks-el_index_60-drop-default.sql b/maintenance/mssql/archives/patch-externallinks-el_index_60-drop-default.sql new file mode 100644 index 0000000000..7755e66fd0 --- /dev/null +++ b/maintenance/mssql/archives/patch-externallinks-el_index_60-drop-default.sql @@ -0,0 +1,13 @@ +DECLARE @sql nvarchar(max) +SET @sql='' + +SELECT @sql= @sql + 'ALTER TABLE /*_*/externallinks DROP CONSTRAINT ' + df.name + '; ' +FROM sys.default_constraints df +JOIN sys.columns c + ON c.object_id = df.parent_object_id + AND c.column_id = df.parent_column_id +WHERE + df.parent_object_id = OBJECT_ID('/*_*/externallinks') + AND c.name = 'el_index_60';-- + +EXEC sp_executesql @sql; diff --git a/maintenance/mssql/tables.sql b/maintenance/mssql/tables.sql index 858260b34b..315cb20ce7 100644 --- a/maintenance/mssql/tables.sql +++ b/maintenance/mssql/tables.sql @@ -549,8 +549,7 @@ CREATE TABLE /*_*/externallinks ( -- This is el_index truncated to 60 bytes to allow for sortable queries that -- aren't supported by a partial index. - -- @todo Drop the default once this is deployed everywhere and code is populating it. - el_index_60 varbinary(60) NOT NULL default '' + el_index_60 varbinary(60) NOT NULL ); CREATE INDEX /*i*/el_from ON /*_*/externallinks (el_from); diff --git a/maintenance/populateExternallinksIndex60.php b/maintenance/populateExternallinksIndex60.php new file mode 100644 index 0000000000..9b029fefaa --- /dev/null +++ b/maintenance/populateExternallinksIndex60.php @@ -0,0 +1,89 @@ +addDescription( + 'Populates the el_index_60 field in the externallinks table' ); + $this->setBatchSize( 200 ); + } + + protected function getUpdateKey() { + return 'populate externallinks.el_index_60'; + } + + protected function updateSkippedMessage() { + return 'externallinks.el_index_60 already populated.'; + } + + protected function doDBUpdates() { + $dbw = $this->getDB( DB_MASTER ); + $this->output( "Populating externallinks.el_index_60...\n" ); + + $count = 0; + $start = 0; + $last = $dbw->selectField( 'externallinks', 'MAX(el_id)', '', __METHOD__ ); + while ( $start <= $last ) { + $end = $start + $this->mBatchSize; + $this->output( "el_id $start - $end of $last\n" ); + $res = $dbw->select( 'externallinks', [ 'el_id', 'el_index' ], + [ + "el_id > $start", + "el_id <= $end", + 'el_index_60' => '', + ], + __METHOD__, + [ 'ORDER BY' => 'el_id' ] + ); + foreach ( $res as $row ) { + $count++; + $dbw->update( 'externallinks', + [ + 'el_index_60' => substr( $row->el_index, 0, 60 ), + ], + [ + 'el_id' => $row->el_id, + ], __METHOD__, [ 'IGNORE' ] + ); + } + wfWaitForSlaves(); + $start = $end; + } + $this->output( "Done, $count rows updated.\n" ); + + return true; + } +} + +$maintClass = "PopulateExternallinksIndex60"; +require_once RUN_MAINTENANCE_IF_MAIN; diff --git a/maintenance/postgres/tables.sql b/maintenance/postgres/tables.sql index 1a53359a99..492a0d5a4c 100644 --- a/maintenance/postgres/tables.sql +++ b/maintenance/postgres/tables.sql @@ -373,7 +373,7 @@ CREATE TABLE externallinks ( el_from INTEGER NOT NULL REFERENCES page(page_id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, el_to TEXT NOT NULL, el_index TEXT NOT NULL, - el_index_60 BYTEA NOT NULL DEFAULT '' + el_index_60 BYTEA NOT NULL ); ALTER SEQUENCE externallinks_el_id_seq OWNED BY externallinks.el_id; CREATE INDEX externallinks_from_to ON externallinks (el_from,el_to); diff --git a/maintenance/sqlite/archives/patch-externallinks-el_index_60-drop-default.sql b/maintenance/sqlite/archives/patch-externallinks-el_index_60-drop-default.sql new file mode 100644 index 0000000000..1d82f4d4d5 --- /dev/null +++ b/maintenance/sqlite/archives/patch-externallinks-el_index_60-drop-default.sql @@ -0,0 +1,21 @@ +-- To change the default on one column, sqlite requires we copy the whole table + +CREATE TABLE /*_*/externallinks_tmp ( + el_id int unsigned NOT NULL PRIMARY KEY AUTO_INCREMENT, + el_from int unsigned NOT NULL default 0, + el_to blob NOT NULL, + el_index blob NOT NULL, + el_index_60 varbinary(60) NOT NULL +) /*$wgDBTableOptions*/; + +INSERT INTO /*_*/externallinks_tmp + SELECT el_id, el_from, el_to, el_index, el_index_60 FROM /*_*/externallinks; + +DROP TABLE /*_*/externallinks; +ALTER TABLE /*_*/externallinks_tmp RENAME TO /*_*/externallinks; + +CREATE INDEX /*i*/el_from ON /*_*/externallinks (el_from, el_to(40)); +CREATE INDEX /*i*/el_to ON /*_*/externallinks (el_to(60), el_from); +CREATE INDEX /*i*/el_index ON /*_*/externallinks (el_index(60)); +CREATE INDEX /*i*/el_index_60 ON /*_*/externallinks (el_index_60, el_id); +CREATE INDEX /*i*/el_from_index_60 ON /*_*/externallinks (el_from, el_index_60, el_id); diff --git a/maintenance/tables.sql b/maintenance/tables.sql index 34c63ed72a..ed3bd43991 100644 --- a/maintenance/tables.sql +++ b/maintenance/tables.sql @@ -932,8 +932,7 @@ CREATE TABLE /*_*/externallinks ( -- This is el_index truncated to 60 bytes to allow for sortable queries that -- aren't supported by a partial index. - -- @todo Drop the default once this is deployed everywhere and code is populating it. - el_index_60 varbinary(60) NOT NULL default '' + el_index_60 varbinary(60) NOT NULL ) /*$wgDBTableOptions*/; -- Forward index, for page edit, save