3 * Local repository that stores files in the local filesystem and registers them
4 * in the wiki's own database.
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License along
17 * with this program; if not, write to the Free Software Foundation, Inc.,
18 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
19 * http://www.gnu.org/copyleft/gpl.html
25 use MediaWiki\MediaWikiServices
;
26 use Wikimedia\Rdbms\IResultWrapper
;
27 use Wikimedia\Rdbms\Database
;
28 use Wikimedia\Rdbms\IDatabase
;
31 * A repository that stores files in the local filesystem and registers them
32 * in the wiki's own database. This is the most commonly used repository class.
35 * @method LocalFile|null newFile( $title, $time = false )
37 class LocalRepo
extends FileRepo
{
39 protected $fileFactory = [ LocalFile
::class, 'newFromTitle' ];
41 protected $fileFactoryKey = [ LocalFile
::class, 'newFromKey' ];
43 protected $fileFromRowFactory = [ LocalFile
::class, 'newFromRow' ];
45 protected $oldFileFromRowFactory = [ OldLocalFile
::class, 'newFromRow' ];
47 protected $oldFileFactory = [ OldLocalFile
::class, 'newFromTitle' ];
49 protected $oldFileFactoryKey = [ OldLocalFile
::class, 'newFromKey' ];
51 function __construct( array $info = null ) {
52 parent
::__construct( $info );
54 $this->hasSha1Storage
= isset( $info['storageLayout'] )
55 && $info['storageLayout'] === 'sha1';
57 if ( $this->hasSha1Storage() ) {
58 $this->backend
= new FileBackendDBRepoWrapper( [
59 'backend' => $this->backend
,
60 'repoName' => $this->name
,
61 'dbHandleFactory' => $this->getDBFactory()
68 * @param stdClass $row
71 function newFileFromRow( $row ) {
72 if ( isset( $row->img_name
) ) {
73 return call_user_func( $this->fileFromRowFactory
, $row, $this );
74 } elseif ( isset( $row->oi_name
) ) {
75 return call_user_func( $this->oldFileFromRowFactory
, $row, $this );
77 throw new MWException( __METHOD__
. ': invalid row' );
83 * @param string $archiveName
84 * @return OldLocalFile
86 function newFromArchiveName( $title, $archiveName ) {
87 return OldLocalFile
::newFromArchiveName( $title, $this, $archiveName );
91 * Delete files in the deleted directory if they are not referenced in the
92 * filearchive table. This needs to be done in the repo because it needs to
93 * interleave database locks with file operations, which is potentially a
96 * @param string[] $storageKeys
100 function cleanupDeletedBatch( array $storageKeys ) {
101 if ( $this->hasSha1Storage() ) {
102 wfDebug( __METHOD__
. ": skipped because storage uses sha1 paths\n" );
103 return Status
::newGood();
106 $backend = $this->backend
; // convenience
107 $root = $this->getZonePath( 'deleted' );
108 $dbw = $this->getMasterDB();
109 $status = $this->newGood();
110 $storageKeys = array_unique( $storageKeys );
111 foreach ( $storageKeys as $key ) {
112 $hashPath = $this->getDeletedHashPath( $key );
113 $path = "$root/$hashPath$key";
114 $dbw->startAtomic( __METHOD__
);
115 // Check for usage in deleted/hidden files and preemptively
116 // lock the key to avoid any future use until we are finished.
117 $deleted = $this->deletedFileHasKey( $key, 'lock' );
118 $hidden = $this->hiddenFileHasKey( $key, 'lock' );
119 if ( !$deleted && !$hidden ) { // not in use now
120 wfDebug( __METHOD__
. ": deleting $key\n" );
121 $op = [ 'op' => 'delete', 'src' => $path ];
122 if ( !$backend->doOperation( $op )->isOK() ) {
123 $status->error( 'undelete-cleanup-error', $path );
124 $status->failCount++
;
127 wfDebug( __METHOD__
. ": $key still in use\n" );
128 $status->successCount++
;
130 $dbw->endAtomic( __METHOD__
);
137 * Check if a deleted (filearchive) file has this sha1 key
139 * @param string $key File storage key (base-36 sha1 key with file extension)
140 * @param string|null $lock Use "lock" to lock the row via FOR UPDATE
141 * @return bool File with this key is in use
143 protected function deletedFileHasKey( $key, $lock = null ) {
144 $options = ( $lock === 'lock' ) ?
[ 'FOR UPDATE' ] : [];
146 $dbw = $this->getMasterDB();
148 return (bool)$dbw->selectField( 'filearchive', '1',
149 [ 'fa_storage_group' => 'deleted', 'fa_storage_key' => $key ],
155 * Check if a hidden (revision delete) file has this sha1 key
157 * @param string $key File storage key (base-36 sha1 key with file extension)
158 * @param string|null $lock Use "lock" to lock the row via FOR UPDATE
159 * @return bool File with this key is in use
161 protected function hiddenFileHasKey( $key, $lock = null ) {
162 $options = ( $lock === 'lock' ) ?
[ 'FOR UPDATE' ] : [];
164 $sha1 = self
::getHashFromKey( $key );
165 $ext = File
::normalizeExtension( substr( $key, strcspn( $key, '.' ) +
1 ) );
167 $dbw = $this->getMasterDB();
169 return (bool)$dbw->selectField( 'oldimage', '1',
170 [ 'oi_sha1' => $sha1,
171 'oi_archive_name ' . $dbw->buildLike( $dbw->anyString(), ".$ext" ),
172 $dbw->bitAnd( 'oi_deleted', File
::DELETED_FILE
) => File
::DELETED_FILE
],
178 * Gets the SHA1 hash from a storage key
183 public static function getHashFromKey( $key ) {
184 $sha1 = strtok( $key, '.' );
185 if ( is_string( $sha1 ) && strlen( $sha1 ) === 32 && $sha1[0] === '0' ) {
186 $sha1 = substr( $sha1, 1 );
192 * Checks if there is a redirect named as $title
194 * @param Title $title Title of file
197 function checkRedirect( Title
$title ) {
198 $title = File
::normalizeTitle( $title, 'exception' );
200 $memcKey = $this->getSharedCacheKey( 'image_redirect', md5( $title->getDBkey() ) );
201 if ( $memcKey === false ) {
202 $memcKey = $this->getLocalCacheKey( 'image_redirect', md5( $title->getDBkey() ) );
203 $expiry = 300; // no invalidation, 5 minutes
205 $expiry = 86400; // has invalidation, 1 day
208 $method = __METHOD__
;
209 $redirDbKey = $this->wanCache
->getWithSetCallback(
212 function ( $oldValue, &$ttl, array &$setOpts ) use ( $method, $title ) {
213 $dbr = $this->getReplicaDB(); // possibly remote DB
215 $setOpts +
= Database
::getCacheSetOptions( $dbr );
217 $row = $dbr->selectRow(
218 [ 'page', 'redirect' ],
219 [ 'rd_namespace', 'rd_title' ],
221 'page_namespace' => $title->getNamespace(),
222 'page_title' => $title->getDBkey(),
228 return ( $row && $row->rd_namespace
== NS_FILE
)
229 ? Title
::makeTitle( $row->rd_namespace
, $row->rd_title
)->getDBkey()
230 : ''; // negative cache
232 [ 'pcTTL' => WANObjectCache
::TTL_PROC_LONG
]
235 // @note: also checks " " for b/c
236 if ( $redirDbKey !== ' ' && strval( $redirDbKey ) !== '' ) {
237 // Page is a redirect to another file
238 return Title
::newFromText( $redirDbKey, NS_FILE
);
241 return false; // no redirect
244 public function findFiles( array $items, $flags = 0 ) {
245 $finalFiles = []; // map of (DB key => corresponding File) for matches
247 $searchSet = []; // map of (normalized DB key => search params)
248 foreach ( $items as $item ) {
249 if ( is_array( $item ) ) {
250 $title = File
::normalizeTitle( $item['title'] );
252 $searchSet[$title->getDBkey()] = $item;
255 $title = File
::normalizeTitle( $item );
257 $searchSet[$title->getDBkey()] = [];
262 $fileMatchesSearch = function ( File
$file, array $search ) {
263 // Note: file name comparison done elsewhere (to handle redirects)
264 $user = ( !empty( $search['private'] ) && $search['private'] instanceof User
)
271 ( empty( $search['time'] ) && !$file->isOld() ) ||
272 ( !empty( $search['time'] ) && $search['time'] === $file->getTimestamp() )
274 ( !empty( $search['private'] ) ||
!$file->isDeleted( File
::DELETED_FILE
) ) &&
275 $file->userCan( File
::DELETED_FILE
, $user )
279 $applyMatchingFiles = function ( IResultWrapper
$res, &$searchSet, &$finalFiles )
280 use ( $fileMatchesSearch, $flags )
282 $contLang = MediaWikiServices
::getInstance()->getContentLanguage();
283 $info = $this->getInfo();
284 foreach ( $res as $row ) {
285 $file = $this->newFileFromRow( $row );
286 // There must have been a search for this DB key, but this has to handle the
287 // cases were title capitalization is different on the client and repo wikis.
288 $dbKeysLook = [ strtr( $file->getName(), ' ', '_' ) ];
289 if ( !empty( $info['initialCapital'] ) ) {
290 // Search keys for "hi.png" and "Hi.png" should use the "Hi.png file"
291 $dbKeysLook[] = $contLang->lcfirst( $file->getName() );
293 foreach ( $dbKeysLook as $dbKey ) {
294 if ( isset( $searchSet[$dbKey] )
295 && $fileMatchesSearch( $file, $searchSet[$dbKey] )
297 $finalFiles[$dbKey] = ( $flags & FileRepo
::NAME_AND_TIME_ONLY
)
298 ?
[ 'title' => $dbKey, 'timestamp' => $file->getTimestamp() ]
300 unset( $searchSet[$dbKey] );
306 $dbr = $this->getReplicaDB();
310 foreach ( array_keys( $searchSet ) as $dbKey ) {
311 $imgNames[] = $this->getNameFromTitle( File
::normalizeTitle( $dbKey ) );
314 if ( count( $imgNames ) ) {
315 $fileQuery = LocalFile
::getQueryInfo();
316 $res = $dbr->select( $fileQuery['tables'], $fileQuery['fields'], [ 'img_name' => $imgNames ],
317 __METHOD__
, [], $fileQuery['joins'] );
318 $applyMatchingFiles( $res, $searchSet, $finalFiles );
321 // Query old image table
322 $oiConds = []; // WHERE clause array for each file
323 foreach ( $searchSet as $dbKey => $search ) {
324 if ( isset( $search['time'] ) ) {
325 $oiConds[] = $dbr->makeList(
327 'oi_name' => $this->getNameFromTitle( File
::normalizeTitle( $dbKey ) ),
328 'oi_timestamp' => $dbr->timestamp( $search['time'] )
335 if ( count( $oiConds ) ) {
336 $fileQuery = OldLocalFile
::getQueryInfo();
337 $res = $dbr->select( $fileQuery['tables'], $fileQuery['fields'],
338 $dbr->makeList( $oiConds, LIST_OR
),
339 __METHOD__
, [], $fileQuery['joins'] );
340 $applyMatchingFiles( $res, $searchSet, $finalFiles );
343 // Check for redirects...
344 foreach ( $searchSet as $dbKey => $search ) {
345 if ( !empty( $search['ignoreRedirect'] ) ) {
349 $title = File
::normalizeTitle( $dbKey );
350 $redir = $this->checkRedirect( $title ); // hopefully hits memcached
352 if ( $redir && $redir->getNamespace() == NS_FILE
) {
353 $file = $this->newFile( $redir );
354 if ( $file && $fileMatchesSearch( $file, $search ) ) {
355 $file->redirectedFrom( $title->getDBkey() );
356 if ( $flags & FileRepo
::NAME_AND_TIME_ONLY
) {
357 $finalFiles[$dbKey] = [
358 'title' => $file->getTitle()->getDBkey(),
359 'timestamp' => $file->getTimestamp()
362 $finalFiles[$dbKey] = $file;
372 * Get an array or iterator of file objects for files that have a given
373 * SHA-1 content hash.
375 * @param string $hash A sha1 hash to look for
376 * @return LocalFile[]
378 function findBySha1( $hash ) {
379 $dbr = $this->getReplicaDB();
380 $fileQuery = LocalFile
::getQueryInfo();
382 $fileQuery['tables'],
383 $fileQuery['fields'],
384 [ 'img_sha1' => $hash ],
386 [ 'ORDER BY' => 'img_name' ],
391 foreach ( $res as $row ) {
392 $result[] = $this->newFileFromRow( $row );
400 * Get an array of arrays or iterators of file objects for files that
401 * have the given SHA-1 content hashes.
403 * Overrides generic implementation in FileRepo for performance reason
405 * @param string[] $hashes An array of hashes
406 * @return array[] An Array of arrays or iterators of file objects and the hash as key
408 function findBySha1s( array $hashes ) {
409 if ( $hashes === [] ) {
410 return []; // empty parameter
413 $dbr = $this->getReplicaDB();
414 $fileQuery = LocalFile
::getQueryInfo();
416 $fileQuery['tables'],
417 $fileQuery['fields'],
418 [ 'img_sha1' => $hashes ],
420 [ 'ORDER BY' => 'img_name' ],
425 foreach ( $res as $row ) {
426 $file = $this->newFileFromRow( $row );
427 $result[$file->getSha1()][] = $file;
435 * Return an array of files where the name starts with $prefix.
437 * @param string $prefix The prefix to search for
438 * @param int $limit The maximum amount of files to return
439 * @return LocalFile[]
441 public function findFilesByPrefix( $prefix, $limit ) {
442 $selectOptions = [ 'ORDER BY' => 'img_name', 'LIMIT' => intval( $limit ) ];
445 $dbr = $this->getReplicaDB();
446 $fileQuery = LocalFile
::getQueryInfo();
448 $fileQuery['tables'],
449 $fileQuery['fields'],
450 'img_name ' . $dbr->buildLike( $prefix, $dbr->anyString() ),
456 // Build file objects
458 foreach ( $res as $row ) {
459 $files[] = $this->newFileFromRow( $row );
466 * Get a connection to the replica DB
469 function getReplicaDB() {
470 return wfGetDB( DB_REPLICA
);
474 * Alias for getReplicaDB()
477 * @deprecated Since 1.29
479 function getSlaveDB() {
480 return $this->getReplicaDB();
484 * Get a connection to the master DB
487 function getMasterDB() {
488 return wfGetDB( DB_MASTER
);
492 * Get a callback to get a DB handle given an index (DB_REPLICA/DB_MASTER)
495 protected function getDBFactory() {
496 return function ( $index ) {
497 return wfGetDB( $index );
502 * Get a key on the primary cache for this repository.
503 * Returns false if the repository's cache is not accessible at this site.
504 * The parameters are the parts of the key.
508 function getSharedCacheKey( /*...*/ ) {
509 $args = func_get_args();
511 return $this->wanCache
->makeKey( ...$args );
515 * Invalidates image redirect cache related to that image
517 * @param Title $title Title of page
520 function invalidateImageRedirect( Title
$title ) {
521 $key = $this->getSharedCacheKey( 'image_redirect', md5( $title->getDBkey() ) );
523 $this->getMasterDB()->onTransactionPreCommitOrIdle(
524 function () use ( $key ) {
525 $this->wanCache
->delete( $key );
533 * Return information about the repository.
541 return array_merge( parent
::getInfo(), [
542 'favicon' => wfExpandUrl( $wgFavicon ),
546 public function store( $srcPath, $dstZone, $dstRel, $flags = 0 ) {
547 return $this->skipWriteOperationIfSha1( __FUNCTION__
, func_get_args() );
550 public function storeBatch( array $triplets, $flags = 0 ) {
551 return $this->skipWriteOperationIfSha1( __FUNCTION__
, func_get_args() );
554 public function cleanupBatch( array $files, $flags = 0 ) {
555 return $this->skipWriteOperationIfSha1( __FUNCTION__
, func_get_args() );
558 public function publish(
565 return $this->skipWriteOperationIfSha1( __FUNCTION__
, func_get_args() );
568 public function publishBatch( array $ntuples, $flags = 0 ) {
569 return $this->skipWriteOperationIfSha1( __FUNCTION__
, func_get_args() );
572 public function delete( $srcRel, $archiveRel ) {
573 return $this->skipWriteOperationIfSha1( __FUNCTION__
, func_get_args() );
576 public function deleteBatch( array $sourceDestPairs ) {
577 return $this->skipWriteOperationIfSha1( __FUNCTION__
, func_get_args() );
581 * Skips the write operation if storage is sha1-based, executes it normally otherwise
583 * @param string $function
587 protected function skipWriteOperationIfSha1( $function, array $args ) {
588 $this->assertWritableRepo(); // fail out if read-only
590 if ( $this->hasSha1Storage() ) {
591 wfDebug( __METHOD__
. ": skipped because storage uses sha1 paths\n" );
592 return Status
::newGood();
594 return parent
::$function( ...$args );