External storage handling

diff --git a/maintenance/storage/blobs.sql b/maintenance/storage/blobs.sql
index 9bf4d14..3e72f77 100644 (file)
--- a/maintenance/storage/blobs.sql
+++ b/maintenance/storage/blobs.sql
@@ -1,7 +1,7 @@
 -- Blobs table for external storage
 
 CREATE TABLE /*$wgDBprefix*/blobs (
-       blob_id int(8) NOT NULL default '0',
+       blob_id int(8) NOT NULL default '0' AUTO_INCREMENT,
        blob_text mediumtext,
        PRIMARY KEY  (blob_id)
 ) TYPE=InnoDB;

diff --git a/maintenance/storage/moveToExternal.php b/maintenance/storage/moveToExternal.php
new file mode 100644 (file)
index 0000000..8b441d2
--- /dev/null
+++ b/maintenance/storage/moveToExternal.php
@@ -0,0 +1,80 @@
+<?php
+
+define( 'REPORTING_INTERVAL', 100 );
+define( 'STUB_HEADER', 'O:15:"historyblobstub"' );
+
+if ( !defined( 'MEDIAWIKI' ) ) {
+       $optionsWithArgs = array( 'm' );
+
+       require_once( '../commandLine.inc' );
+       require_once( 'ExternalStoreDB.php' );
+       require_once( 'resolveStubs.php' );
+
+       $fname = 'moveToExternal';
+
+       if ( !isset( $args[0] ) ) {
+               print "Usage: php moveToExternal.php [-m <maxid>] <cluster>\n";
+               exit;
+       }
+
+       $cluster = $args[0];
+       $dbw =& wfGetDB( DB_MASTER );
+
+       if ( isset( $options['m'] ) ) {
+               $maxID = $options['m'];
+       } else {
+               $maxID = $dbw->selectField( 'text', 'MAX(old_id)', false, $fname );
+       }
+
+       moveToExternal( $cluster, $maxID );
+}
+
+
+
+function moveToExternal( $cluster, $maxID ) {
+       $fname = 'moveToExternal';
+       $dbw =& wfGetDB( DB_MASTER );
+
+       print "Moving $maxID text rows to external storage\n";
+       $ext = new ExternalStoreDB;
+       for ( $id = 1; $id <= $maxID; $id++ ) {
+               if ( !($id % REPORTING_INTERVAL) ) {
+                       print "$id\n";
+                       wfWaitForSlaves( 5 );
+               }
+               $row = $dbw->selectRow( 'text', array( 'old_flags', 'old_text' ),
+                       array( 
+                               'old_id' => $id,
+                               "old_flags NOT LIKE '%external%'",
+                       ), $fname );
+               if ( !$row ) {
+                       # Non-existent or already done
+                       continue;
+               }
+
+               # Resolve stubs
+               $flags = explode( ',', $row->old_flags );
+               if ( in_array( 'object', $flags ) 
+                       && substr( $row->old_text, 0, strlen( STUB_HEADER ) ) === STUB_HEADER ) 
+               {
+                       resolveStub( $id, $row->old_text, $row->old_flags );
+                       continue;
+               }
+
+               $url = $ext->store( $cluster, $row->old_text );
+               if ( !$url ) {
+                       print "Error writing to external storage\n";
+                       exit;
+               }
+               if ( $row->old_flags === '' ) {
+                       $flags = 'external';
+               } else {
+                       $flags = "{$row->old_flags},external";
+               }
+               $dbw->update( 'text', 
+                       array( 'old_flags' => $flags, 'old_text' => $url ),
+                       array( 'old_id' => $id ), $fname );
+       }
+}
+
+?>

diff --git a/maintenance/storage/resolveStubs.php b/maintenance/storage/resolveStubs.php
index 840ae3f..d22976c 100644 (file)
--- a/maintenance/storage/resolveStubs.php
+++ b/maintenance/storage/resolveStubs.php
@@ -55,42 +55,54 @@ function resolveStubs() {
                        wfWaitForSlaves( 5 );
                }
 
-               $stub = unserialize( $stub );
-               if ( get_class( $stub ) !== 'historyblobstub' ) {
-                       print "Error, invalid stub object\n";
-                       return;
-               }
+               resolveStub( $id, $stub, $flagsArray[$id] );
+       }
+}
 
-               # Get the (maybe) external row
-               $externalRow = $dbr->selectRow( 'text', array( 'old_text' ), 
-                       array( 'old_id' => $stub->mOldId, "old_flags LIKE '%external%'" ),
-                       $fname 
-               );
+/**
+ * Resolve a history stub
+ */
+function resolveStub( $id, $stubText, $flags ) {
+       $fname = 'resolveStub';
 
-               if ( !$externalRow ) {
-                       # Object wasn't external
-                       continue;
-               }
+       $stub = unserialize( $stubText );
+       $flags = explode( ',', $flags );
 
-               # Preserve the legacy encoding flag, but switch from object to external
-               $flags = explode( ',', $flagsArray[$id] );
-               if ( in_array( 'utf-8', $flags ) ) {
-                       $newFlags = 'external,utf-8';
-               } else {
-                       $newFlags = 'external';
-               }
+       $dbr =& wfGetDB( DB_SLAVE );
+       $dbw =& wfGetDB( DB_MASTER );
+       
+       if ( get_class( $stub ) !== 'historyblobstub' ) {
+               print "Error, invalid stub object\n";
+               return;
+       }
 
-               # Update the row
-               $dbw->update( 'text',
-                       array( /* SET */
-                               'old_flags' => $newFlags, 
-                               'old_text' => $externalRow->old_text . '/' . $stub->mHash
-                       ), 
-                       array( /* WHERE */ 
-                               'old_id' => $id 
-                       ), $fname 
-               );
+       # Get the (maybe) external row
+       $externalRow = $dbr->selectRow( 'text', array( 'old_text' ), 
+               array( 'old_id' => $stub->mOldId, "old_flags LIKE '%external%'" ),
+               $fname 
+       );
+
+       if ( !$externalRow ) {
+               # Object wasn't external
+               continue;
        }
-}
 
+       # Preserve the legacy encoding flag, but switch from object to external
+       if ( in_array( 'utf-8', $flags ) ) {
+               $newFlags = 'external,utf-8';
+       } else {
+               $newFlags = 'external';
+       }
+
+       # Update the row
+       $dbw->update( 'text',
+               array( /* SET */
+                       'old_flags' => $newFlags, 
+                       'old_text' => $externalRow->old_text . '/' . $stub->mHash
+               ), 
+               array( /* WHERE */ 
+                       'old_id' => $id 
+               ), $fname 
+       );
+}
 ?>