3 * Copy all files in FileRepo to an originals container using SHA1 paths.
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License along
16 * with this program; if not, write to the Free Software Foundation, Inc.,
17 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18 * http://www.gnu.org/copyleft/gpl.html
21 * @ingroup Maintenance
24 require_once __DIR__
. '/Maintenance.php';
27 * Copy all files in FileRepo to an originals container using SHA1 paths.
29 * This script should be run while the repo is still set to the old layout.
31 * @ingroup Maintenance
33 class MigrateFileRepoLayout
extends Maintenance
{
34 public function __construct() {
35 parent
::__construct();
36 $this->addDescription( 'Copy files in repo to a different layout.' );
37 $this->addOption( 'oldlayout', "Old layout; one of 'name' or 'sha1'", true, true );
38 $this->addOption( 'newlayout', "New layout; one of 'name' or 'sha1'", true, true );
39 $this->addOption( 'since', "Copy only files from after this timestamp", false, true );
40 $this->setBatchSize( 50 );
43 public function execute() {
44 $oldLayout = $this->getOption( 'oldlayout' );
45 if ( !in_array( $oldLayout, [ 'name', 'sha1' ] ) ) {
46 $this->error( "Invalid old layout.", 1 );
48 $newLayout = $this->getOption( 'newlayout' );
49 if ( !in_array( $newLayout, [ 'name', 'sha1' ] ) ) {
50 $this->error( "Invalid new layout.", 1 );
52 $since = $this->getOption( 'since' );
54 $repo = $this->getRepo();
56 $be = $repo->getBackend();
57 if ( $be instanceof FileBackendDBRepoWrapper
) {
58 $be = $be->getInternalBackend(); // avoid path translations for this script
61 $dbw = $repo->getMasterDB();
63 $origBase = $be->getContainerStoragePath( "{$repo->getName()}-original" );
64 $startTime = wfTimestampNow();
66 // Do current and archived versions...
69 $conds[] = 'img_timestamp >= ' . $dbw->addQuotes( $dbw->timestamp( $since ) );
72 $batchSize = $this->getBatchSize();
76 $res = $dbw->select( 'image',
77 [ 'img_name', 'img_sha1' ],
78 array_merge( [ 'img_name > ' . $dbw->addQuotes( $lastName ) ], $conds ),
80 [ 'LIMIT' => $batchSize, 'ORDER BY' => 'img_name' ]
83 foreach ( $res as $row ) {
84 $lastName = $row->img_name
;
85 /** @var LocalFile $file */
86 $file = $repo->newFile( $row->img_name
);
87 // Check in case SHA1 rows are not populated for some files
88 $sha1 = strlen( $row->img_sha1
) ?
$row->img_sha1
: $file->getSha1();
90 if ( !strlen( $sha1 ) ) {
91 $this->error( "Image SHA-1 not known for {$row->img_name}." );
93 if ( $oldLayout === 'sha1' ) {
94 $spath = "{$origBase}/{$sha1[0]}/{$sha1[1]}/{$sha1[2]}/{$sha1}";
96 $spath = $file->getPath();
99 if ( $newLayout === 'sha1' ) {
100 $dpath = "{$origBase}/{$sha1[0]}/{$sha1[1]}/{$sha1[2]}/{$sha1}";
102 $dpath = $file->getPath();
105 $status = $be->prepare( [
106 'dir' => dirname( $dpath ), 'bypassReadOnly' => 1 ] );
107 if ( !$status->isOK() ) {
108 $this->error( print_r( $status->getErrors(), true ) );
111 $batch[] = [ 'op' => 'copy', 'overwrite' => true,
112 'src' => $spath, 'dst' => $dpath, 'img' => $row->img_name
];
115 foreach ( $file->getHistory() as $ofile ) {
116 $sha1 = $ofile->getSha1();
117 if ( !strlen( $sha1 ) ) {
118 $this->error( "Image SHA-1 not set for {$ofile->getArchiveName()}." );
122 if ( $oldLayout === 'sha1' ) {
123 $spath = "{$origBase}/{$sha1[0]}/{$sha1[1]}/{$sha1[2]}/{$sha1}";
124 } elseif ( $ofile->isDeleted( File
::DELETED_FILE
) ) {
125 $spath = $be->getContainerStoragePath( "{$repo->getName()}-deleted" ) .
126 '/' . $repo->getDeletedHashPath( $sha1 ) .
127 $sha1 . '.' . $ofile->getExtension();
129 $spath = $ofile->getPath();
132 if ( $newLayout === 'sha1' ) {
133 $dpath = "{$origBase}/{$sha1[0]}/{$sha1[1]}/{$sha1[2]}/{$sha1}";
135 $dpath = $ofile->getPath();
138 $status = $be->prepare( [
139 'dir' => dirname( $dpath ), 'bypassReadOnly' => 1 ] );
140 if ( !$status->isOK() ) {
141 $this->error( print_r( $status->getErrors(), true ) );
143 $batch[] = [ 'op' => 'copy', 'overwrite' => true,
144 'src' => $spath, 'dst' => $dpath, 'img' => $ofile->getArchiveName() ];
147 if ( count( $batch ) >= $batchSize ) {
148 $this->runBatch( $batch, $be );
152 } while ( $res->numRows() );
154 if ( count( $batch ) ) {
155 $this->runBatch( $batch, $be );
158 // Do deleted versions...
161 $conds[] = 'fa_deleted_timestamp >= ' . $dbw->addQuotes( $dbw->timestamp( $since ) );
167 $res = $dbw->select( 'filearchive', [ 'fa_storage_key', 'fa_id', 'fa_name' ],
168 array_merge( [ 'fa_id > ' . $dbw->addQuotes( $lastId ) ], $conds ),
170 [ 'LIMIT' => $batchSize, 'ORDER BY' => 'fa_id' ]
173 foreach ( $res as $row ) {
174 $lastId = $row->fa_id
;
175 $sha1Key = $row->fa_storage_key
;
176 if ( !strlen( $sha1Key ) ) {
177 $this->error( "Image SHA-1 not set for file #{$row->fa_id} (deleted)." );
180 $sha1 = substr( $sha1Key, 0, strpos( $sha1Key, '.' ) );
182 if ( $oldLayout === 'sha1' ) {
183 $spath = "{$origBase}/{$sha1[0]}/{$sha1[1]}/{$sha1[2]}/{$sha1}";
185 $spath = $be->getContainerStoragePath( "{$repo->getName()}-deleted" ) .
186 '/' . $repo->getDeletedHashPath( $sha1Key ) . $sha1Key;
189 if ( $newLayout === 'sha1' ) {
190 $dpath = "{$origBase}/{$sha1[0]}/{$sha1[1]}/{$sha1[2]}/{$sha1}";
192 $dpath = $be->getContainerStoragePath( "{$repo->getName()}-deleted" ) .
193 '/' . $repo->getDeletedHashPath( $sha1Key ) . $sha1Key;
196 $status = $be->prepare( [
197 'dir' => dirname( $dpath ), 'bypassReadOnly' => 1 ] );
198 if ( !$status->isOK() ) {
199 $this->error( print_r( $status->getErrors(), true ) );
202 $batch[] = [ 'op' => 'copy', 'src' => $spath, 'dst' => $dpath,
203 'overwriteSame' => true, 'img' => "(ID {$row->fa_id}) {$row->fa_name}" ];
205 if ( count( $batch ) >= $batchSize ) {
206 $this->runBatch( $batch, $be );
210 } while ( $res->numRows() );
212 if ( count( $batch ) ) {
213 $this->runBatch( $batch, $be );
216 $this->output( "Done (started $startTime)\n" );
219 protected function getRepo() {
220 return RepoGroup
::singleton()->getLocalRepo();
223 protected function runBatch( array $ops, FileBackend
$be ) {
224 $this->output( "Migrating file batch:\n" );
225 foreach ( $ops as $op ) {
226 $this->output( "\"{$op['img']}\" (dest: {$op['dst']})\n" );
229 $status = $be->doOperations( $ops, [ 'bypassReadOnly' => 1 ] );
230 if ( !$status->isOK() ) {
231 $this->output( print_r( $status->getErrors(), true ) );
234 $this->output( "Batch done\n\n" );
238 $maintClass = 'MigrateFileRepoLayout';
239 require_once RUN_MAINTENANCE_IF_MAIN
;