3 * Base class for all backends using particular storage medium.
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License along
16 * with this program; if not, write to the Free Software Foundation, Inc.,
17 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18 * http://www.gnu.org/copyleft/gpl.html
21 * @ingroup FileBackend
22 * @author Aaron Schulz
26 * @brief Base class for all backends using particular storage medium.
28 * This class defines the methods as abstract that subclasses must implement.
29 * Outside callers should *not* use functions with "Internal" in the name.
31 * The FileBackend operations are implemented using basic functions
32 * such as storeInternal(), copyInternal(), deleteInternal() and the like.
33 * This class is also responsible for path resolution and sanitization.
35 * @ingroup FileBackend
38 abstract class FileBackendStore
extends FileBackend
{
41 /** @var ProcessCacheLRU */
42 protected $cheapCache; // Map of paths to small (RAM/disk) cache items
43 /** @var ProcessCacheLRU */
44 protected $expensiveCache; // Map of paths to large (RAM/disk) cache items
46 /** @var Array Map of container names to sharding settings */
47 protected $shardViaHashLevels = array(); // (container name => config array)
49 protected $maxFileSize = 4294967296; // integer bytes (4GiB)
51 const CACHE_TTL
= 10; // integer; TTL in seconds for process cache entries
54 * @see FileBackend::__construct()
56 * @param $config Array
58 public function __construct( array $config ) {
59 parent
::__construct( $config );
60 $this->memCache
= new EmptyBagOStuff(); // disabled by default
61 $this->cheapCache
= new ProcessCacheLRU( 300 );
62 $this->expensiveCache
= new ProcessCacheLRU( 5 );
66 * Get the maximum allowable file size given backend
67 * medium restrictions and basic performance constraints.
68 * Do not call this function from places outside FileBackend and FileOp.
70 * @return integer Bytes
72 final public function maxFileSizeInternal() {
73 return $this->maxFileSize
;
77 * Check if a file can be created or changed at a given storage path.
78 * FS backends should check if the parent directory exists, files can be
79 * written under it, and that any file already there is writable.
80 * Backends using key/value stores should check if the container exists.
82 * @param $storagePath string
85 abstract public function isPathUsableInternal( $storagePath );
88 * Create a file in the backend with the given contents.
89 * This will overwrite any file that exists at the destination.
90 * Do not call this function from places outside FileBackend and FileOp.
93 * - content : the raw file contents
94 * - dst : destination storage path
95 * - disposition : Content-Disposition header value for the destination
96 * - headers : HTTP header name/value map
97 * - async : Status will be returned immediately if supported.
98 * If the status is OK, then its value field will be
99 * set to a FileBackendStoreOpHandle object.
100 * - dstExists : Whether a file exists at the destination (optimization).
101 * Callers can use "false" if no existing file is being changed.
103 * @param $params Array
106 final public function createInternal( array $params ) {
107 wfProfileIn( __METHOD__
);
108 wfProfileIn( __METHOD__
. '-' . $this->name
);
109 if ( strlen( $params['content'] ) > $this->maxFileSizeInternal() ) {
110 $status = Status
::newFatal( 'backend-fail-maxsize',
111 $params['dst'], $this->maxFileSizeInternal() );
113 $status = $this->doCreateInternal( $params );
114 $this->clearCache( array( $params['dst'] ) );
115 if ( !isset( $params['dstExists'] ) ||
$params['dstExists'] ) {
116 $this->deleteFileCache( $params['dst'] ); // persistent cache
119 wfProfileOut( __METHOD__
. '-' . $this->name
);
120 wfProfileOut( __METHOD__
);
125 * @see FileBackendStore::createInternal()
128 abstract protected function doCreateInternal( array $params );
131 * Store a file into the backend from a file on disk.
132 * This will overwrite any file that exists at the destination.
133 * Do not call this function from places outside FileBackend and FileOp.
136 * - src : source path on disk
137 * - dst : destination storage path
138 * - disposition : Content-Disposition header value for the destination
139 * - headers : HTTP header name/value map
140 * - async : Status will be returned immediately if supported.
141 * If the status is OK, then its value field will be
142 * set to a FileBackendStoreOpHandle object.
143 * - dstExists : Whether a file exists at the destination (optimization).
144 * Callers can use "false" if no existing file is being changed.
146 * @param $params Array
149 final public function storeInternal( array $params ) {
150 wfProfileIn( __METHOD__
);
151 wfProfileIn( __METHOD__
. '-' . $this->name
);
152 if ( filesize( $params['src'] ) > $this->maxFileSizeInternal() ) {
153 $status = Status
::newFatal( 'backend-fail-maxsize',
154 $params['dst'], $this->maxFileSizeInternal() );
156 $status = $this->doStoreInternal( $params );
157 $this->clearCache( array( $params['dst'] ) );
158 if ( !isset( $params['dstExists'] ) ||
$params['dstExists'] ) {
159 $this->deleteFileCache( $params['dst'] ); // persistent cache
162 wfProfileOut( __METHOD__
. '-' . $this->name
);
163 wfProfileOut( __METHOD__
);
168 * @see FileBackendStore::storeInternal()
171 abstract protected function doStoreInternal( array $params );
174 * Copy a file from one storage path to another in the backend.
175 * This will overwrite any file that exists at the destination.
176 * Do not call this function from places outside FileBackend and FileOp.
179 * - src : source storage path
180 * - dst : destination storage path
181 * - ignoreMissingSource : do nothing if the source file does not exist
182 * - disposition : Content-Disposition header value for the destination
183 * - async : Status will be returned immediately if supported.
184 * If the status is OK, then its value field will be
185 * set to a FileBackendStoreOpHandle object.
186 * - dstExists : Whether a file exists at the destination (optimization).
187 * Callers can use "false" if no existing file is being changed.
189 * @param $params Array
192 final public function copyInternal( array $params ) {
193 wfProfileIn( __METHOD__
);
194 wfProfileIn( __METHOD__
. '-' . $this->name
);
195 $status = $this->doCopyInternal( $params );
196 $this->clearCache( array( $params['dst'] ) );
197 if ( !isset( $params['dstExists'] ) ||
$params['dstExists'] ) {
198 $this->deleteFileCache( $params['dst'] ); // persistent cache
200 wfProfileOut( __METHOD__
. '-' . $this->name
);
201 wfProfileOut( __METHOD__
);
206 * @see FileBackendStore::copyInternal()
209 abstract protected function doCopyInternal( array $params );
212 * Delete a file at the storage path.
213 * Do not call this function from places outside FileBackend and FileOp.
216 * - src : source storage path
217 * - ignoreMissingSource : do nothing if the source file does not exist
218 * - async : Status will be returned immediately if supported.
219 * If the status is OK, then its value field will be
220 * set to a FileBackendStoreOpHandle object.
222 * @param $params Array
225 final public function deleteInternal( array $params ) {
226 wfProfileIn( __METHOD__
);
227 wfProfileIn( __METHOD__
. '-' . $this->name
);
228 $status = $this->doDeleteInternal( $params );
229 $this->clearCache( array( $params['src'] ) );
230 $this->deleteFileCache( $params['src'] ); // persistent cache
231 wfProfileOut( __METHOD__
. '-' . $this->name
);
232 wfProfileOut( __METHOD__
);
237 * @see FileBackendStore::deleteInternal()
240 abstract protected function doDeleteInternal( array $params );
243 * Move a file from one storage path to another in the backend.
244 * This will overwrite any file that exists at the destination.
245 * Do not call this function from places outside FileBackend and FileOp.
248 * - src : source storage path
249 * - dst : destination storage path
250 * - ignoreMissingSource : do nothing if the source file does not exist
251 * - disposition : Content-Disposition header value for the destination
252 * - async : Status will be returned immediately if supported.
253 * If the status is OK, then its value field will be
254 * set to a FileBackendStoreOpHandle object.
255 * - dstExists : Whether a file exists at the destination (optimization).
256 * Callers can use "false" if no existing file is being changed.
258 * @param $params Array
261 final public function moveInternal( array $params ) {
262 wfProfileIn( __METHOD__
);
263 wfProfileIn( __METHOD__
. '-' . $this->name
);
264 $status = $this->doMoveInternal( $params );
265 $this->clearCache( array( $params['src'], $params['dst'] ) );
266 $this->deleteFileCache( $params['src'] ); // persistent cache
267 if ( !isset( $params['dstExists'] ) ||
$params['dstExists'] ) {
268 $this->deleteFileCache( $params['dst'] ); // persistent cache
270 wfProfileOut( __METHOD__
. '-' . $this->name
);
271 wfProfileOut( __METHOD__
);
276 * @see FileBackendStore::moveInternal()
279 protected function doMoveInternal( array $params ) {
280 unset( $params['async'] ); // two steps, won't work here :)
281 // Copy source to dest
282 $status = $this->copyInternal( $params );
283 if ( $status->isOK() ) {
284 // Delete source (only fails due to races or medium going down)
285 $status->merge( $this->deleteInternal( array( 'src' => $params['src'] ) ) );
286 $status->setResult( true, $status->value
); // ignore delete() errors
292 * Alter metadata for a file at the storage path.
293 * Do not call this function from places outside FileBackend and FileOp.
296 * - src : source storage path
297 * - disposition : Content-Disposition header value for the destination
298 * - headers : HTTP header name/value map
299 * - async : Status will be returned immediately if supported.
300 * If the status is OK, then its value field will be
301 * set to a FileBackendStoreOpHandle object.
303 * @param $params Array
306 final public function describeInternal( array $params ) {
307 wfProfileIn( __METHOD__
);
308 wfProfileIn( __METHOD__
. '-' . $this->name
);
309 $status = $this->doDescribeInternal( $params );
310 $this->clearCache( array( $params['src'] ) );
311 $this->deleteFileCache( $params['src'] ); // persistent cache
312 wfProfileOut( __METHOD__
. '-' . $this->name
);
313 wfProfileOut( __METHOD__
);
318 * @see FileBackendStore::describeInternal()
321 protected function doDescribeInternal( array $params ) {
322 return Status
::newGood();
326 * No-op file operation that does nothing.
327 * Do not call this function from places outside FileBackend and FileOp.
329 * @param $params Array
332 final public function nullInternal( array $params ) {
333 return Status
::newGood();
337 * @see FileBackend::concatenate()
340 final public function concatenate( array $params ) {
341 wfProfileIn( __METHOD__
);
342 wfProfileIn( __METHOD__
. '-' . $this->name
);
343 $status = Status
::newGood();
345 // Try to lock the source files for the scope of this function
346 $scopeLockS = $this->getScopedFileLocks( $params['srcs'], LockManager
::LOCK_UW
, $status );
347 if ( $status->isOK() ) {
348 // Actually do the file concatenation...
349 $start_time = microtime( true );
350 $status->merge( $this->doConcatenate( $params ) );
351 $sec = microtime( true ) - $start_time;
352 if ( !$status->isOK() ) {
353 wfDebugLog( 'FileOperation', get_class( $this ) . " failed to concatenate " .
354 count( $params['srcs'] ) . " file(s) [$sec sec]" );
358 wfProfileOut( __METHOD__
. '-' . $this->name
);
359 wfProfileOut( __METHOD__
);
364 * @see FileBackendStore::concatenate()
367 protected function doConcatenate( array $params ) {
368 $status = Status
::newGood();
369 $tmpPath = $params['dst']; // convenience
370 unset( $params['latest'] ); // sanity
372 // Check that the specified temp file is valid...
373 wfSuppressWarnings();
374 $ok = ( is_file( $tmpPath ) && filesize( $tmpPath ) == 0 );
376 if ( !$ok ) { // not present or not empty
377 $status->fatal( 'backend-fail-opentemp', $tmpPath );
381 // Get local FS versions of the chunks needed for the concatenation...
382 $fsFiles = $this->getLocalReferenceMulti( $params );
383 foreach ( $fsFiles as $path => &$fsFile ) {
384 if ( !$fsFile ) { // chunk failed to download?
385 $fsFile = $this->getLocalReference( array( 'src' => $path ) );
386 if ( !$fsFile ) { // retry failed?
387 $status->fatal( 'backend-fail-read', $path );
392 unset( $fsFile ); // unset reference so we can reuse $fsFile
394 // Get a handle for the destination temp file
395 $tmpHandle = fopen( $tmpPath, 'ab' );
396 if ( $tmpHandle === false ) {
397 $status->fatal( 'backend-fail-opentemp', $tmpPath );
401 // Build up the temp file using the source chunks (in order)...
402 foreach ( $fsFiles as $virtualSource => $fsFile ) {
403 // Get a handle to the local FS version
404 $sourceHandle = fopen( $fsFile->getPath(), 'rb' );
405 if ( $sourceHandle === false ) {
406 fclose( $tmpHandle );
407 $status->fatal( 'backend-fail-read', $virtualSource );
410 // Append chunk to file (pass chunk size to avoid magic quotes)
411 if ( !stream_copy_to_stream( $sourceHandle, $tmpHandle ) ) {
412 fclose( $sourceHandle );
413 fclose( $tmpHandle );
414 $status->fatal( 'backend-fail-writetemp', $tmpPath );
417 fclose( $sourceHandle );
419 if ( !fclose( $tmpHandle ) ) {
420 $status->fatal( 'backend-fail-closetemp', $tmpPath );
424 clearstatcache(); // temp file changed
430 * @see FileBackend::doPrepare()
433 final protected function doPrepare( array $params ) {
434 wfProfileIn( __METHOD__
);
435 wfProfileIn( __METHOD__
. '-' . $this->name
);
437 $status = Status
::newGood();
438 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
439 if ( $dir === null ) {
440 $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
441 wfProfileOut( __METHOD__
. '-' . $this->name
);
442 wfProfileOut( __METHOD__
);
443 return $status; // invalid storage path
446 if ( $shard !== null ) { // confined to a single container/shard
447 $status->merge( $this->doPrepareInternal( $fullCont, $dir, $params ) );
448 } else { // directory is on several shards
449 wfDebug( __METHOD__
. ": iterating over all container shards.\n" );
450 list( , $shortCont, ) = self
::splitStoragePath( $params['dir'] );
451 foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
452 $status->merge( $this->doPrepareInternal( "{$fullCont}{$suffix}", $dir, $params ) );
456 wfProfileOut( __METHOD__
. '-' . $this->name
);
457 wfProfileOut( __METHOD__
);
462 * @see FileBackendStore::doPrepare()
465 protected function doPrepareInternal( $container, $dir, array $params ) {
466 return Status
::newGood();
470 * @see FileBackend::doSecure()
473 final protected function doSecure( array $params ) {
474 wfProfileIn( __METHOD__
);
475 wfProfileIn( __METHOD__
. '-' . $this->name
);
476 $status = Status
::newGood();
478 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
479 if ( $dir === null ) {
480 $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
481 wfProfileOut( __METHOD__
. '-' . $this->name
);
482 wfProfileOut( __METHOD__
);
483 return $status; // invalid storage path
486 if ( $shard !== null ) { // confined to a single container/shard
487 $status->merge( $this->doSecureInternal( $fullCont, $dir, $params ) );
488 } else { // directory is on several shards
489 wfDebug( __METHOD__
. ": iterating over all container shards.\n" );
490 list( , $shortCont, ) = self
::splitStoragePath( $params['dir'] );
491 foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
492 $status->merge( $this->doSecureInternal( "{$fullCont}{$suffix}", $dir, $params ) );
496 wfProfileOut( __METHOD__
. '-' . $this->name
);
497 wfProfileOut( __METHOD__
);
502 * @see FileBackendStore::doSecure()
505 protected function doSecureInternal( $container, $dir, array $params ) {
506 return Status
::newGood();
510 * @see FileBackend::doPublish()
513 final protected function doPublish( array $params ) {
514 wfProfileIn( __METHOD__
);
515 wfProfileIn( __METHOD__
. '-' . $this->name
);
516 $status = Status
::newGood();
518 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
519 if ( $dir === null ) {
520 $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
521 wfProfileOut( __METHOD__
. '-' . $this->name
);
522 wfProfileOut( __METHOD__
);
523 return $status; // invalid storage path
526 if ( $shard !== null ) { // confined to a single container/shard
527 $status->merge( $this->doPublishInternal( $fullCont, $dir, $params ) );
528 } else { // directory is on several shards
529 wfDebug( __METHOD__
. ": iterating over all container shards.\n" );
530 list( , $shortCont, ) = self
::splitStoragePath( $params['dir'] );
531 foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
532 $status->merge( $this->doPublishInternal( "{$fullCont}{$suffix}", $dir, $params ) );
536 wfProfileOut( __METHOD__
. '-' . $this->name
);
537 wfProfileOut( __METHOD__
);
542 * @see FileBackendStore::doPublish()
545 protected function doPublishInternal( $container, $dir, array $params ) {
546 return Status
::newGood();
550 * @see FileBackend::doClean()
553 final protected function doClean( array $params ) {
554 wfProfileIn( __METHOD__
);
555 wfProfileIn( __METHOD__
. '-' . $this->name
);
556 $status = Status
::newGood();
558 // Recursive: first delete all empty subdirs recursively
559 if ( !empty( $params['recursive'] ) && !$this->directoriesAreVirtual() ) {
560 $subDirsRel = $this->getTopDirectoryList( array( 'dir' => $params['dir'] ) );
561 if ( $subDirsRel !== null ) { // no errors
562 foreach ( $subDirsRel as $subDirRel ) {
563 $subDir = $params['dir'] . "/{$subDirRel}"; // full path
564 $status->merge( $this->doClean( array( 'dir' => $subDir ) +
$params ) );
566 unset( $subDirsRel ); // free directory for rmdir() on Windows (for FS backends)
570 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
571 if ( $dir === null ) {
572 $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
573 wfProfileOut( __METHOD__
. '-' . $this->name
);
574 wfProfileOut( __METHOD__
);
575 return $status; // invalid storage path
578 // Attempt to lock this directory...
579 $filesLockEx = array( $params['dir'] );
580 $scopedLockE = $this->getScopedFileLocks( $filesLockEx, LockManager
::LOCK_EX
, $status );
581 if ( !$status->isOK() ) {
582 wfProfileOut( __METHOD__
. '-' . $this->name
);
583 wfProfileOut( __METHOD__
);
584 return $status; // abort
587 if ( $shard !== null ) { // confined to a single container/shard
588 $status->merge( $this->doCleanInternal( $fullCont, $dir, $params ) );
589 $this->deleteContainerCache( $fullCont ); // purge cache
590 } else { // directory is on several shards
591 wfDebug( __METHOD__
. ": iterating over all container shards.\n" );
592 list( , $shortCont, ) = self
::splitStoragePath( $params['dir'] );
593 foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
594 $status->merge( $this->doCleanInternal( "{$fullCont}{$suffix}", $dir, $params ) );
595 $this->deleteContainerCache( "{$fullCont}{$suffix}" ); // purge cache
599 wfProfileOut( __METHOD__
. '-' . $this->name
);
600 wfProfileOut( __METHOD__
);
605 * @see FileBackendStore::doClean()
608 protected function doCleanInternal( $container, $dir, array $params ) {
609 return Status
::newGood();
613 * @see FileBackend::fileExists()
616 final public function fileExists( array $params ) {
617 wfProfileIn( __METHOD__
);
618 wfProfileIn( __METHOD__
. '-' . $this->name
);
619 $stat = $this->getFileStat( $params );
620 wfProfileOut( __METHOD__
. '-' . $this->name
);
621 wfProfileOut( __METHOD__
);
622 return ( $stat === null ) ?
null : (bool)$stat; // null => failure
626 * @see FileBackend::getFileTimestamp()
629 final public function getFileTimestamp( array $params ) {
630 wfProfileIn( __METHOD__
);
631 wfProfileIn( __METHOD__
. '-' . $this->name
);
632 $stat = $this->getFileStat( $params );
633 wfProfileOut( __METHOD__
. '-' . $this->name
);
634 wfProfileOut( __METHOD__
);
635 return $stat ?
$stat['mtime'] : false;
639 * @see FileBackend::getFileSize()
642 final public function getFileSize( array $params ) {
643 wfProfileIn( __METHOD__
);
644 wfProfileIn( __METHOD__
. '-' . $this->name
);
645 $stat = $this->getFileStat( $params );
646 wfProfileOut( __METHOD__
. '-' . $this->name
);
647 wfProfileOut( __METHOD__
);
648 return $stat ?
$stat['size'] : false;
652 * @see FileBackend::getFileStat()
655 final public function getFileStat( array $params ) {
656 $path = self
::normalizeStoragePath( $params['src'] );
657 if ( $path === null ) {
658 return false; // invalid storage path
660 wfProfileIn( __METHOD__
);
661 wfProfileIn( __METHOD__
. '-' . $this->name
);
662 $latest = !empty( $params['latest'] ); // use latest data?
663 if ( !$this->cheapCache
->has( $path, 'stat', self
::CACHE_TTL
) ) {
664 $this->primeFileCache( array( $path ) ); // check persistent cache
666 if ( $this->cheapCache
->has( $path, 'stat', self
::CACHE_TTL
) ) {
667 $stat = $this->cheapCache
->get( $path, 'stat' );
668 // If we want the latest data, check that this cached
669 // value was in fact fetched with the latest available data.
670 if ( is_array( $stat ) ) {
671 if ( !$latest ||
$stat['latest'] ) {
672 wfProfileOut( __METHOD__
. '-' . $this->name
);
673 wfProfileOut( __METHOD__
);
676 } elseif ( in_array( $stat, array( 'NOT_EXIST', 'NOT_EXIST_LATEST' ) ) ) {
677 if ( !$latest ||
$stat === 'NOT_EXIST_LATEST' ) {
678 wfProfileOut( __METHOD__
. '-' . $this->name
);
679 wfProfileOut( __METHOD__
);
684 wfProfileIn( __METHOD__
. '-miss' );
685 wfProfileIn( __METHOD__
. '-miss-' . $this->name
);
686 $stat = $this->doGetFileStat( $params );
687 wfProfileOut( __METHOD__
. '-miss-' . $this->name
);
688 wfProfileOut( __METHOD__
. '-miss' );
689 if ( is_array( $stat ) ) { // file exists
690 $stat['latest'] = $latest;
691 $this->cheapCache
->set( $path, 'stat', $stat );
692 $this->setFileCache( $path, $stat ); // update persistent cache
693 if ( isset( $stat['sha1'] ) ) { // some backends store SHA-1 as metadata
694 $this->cheapCache
->set( $path, 'sha1',
695 array( 'hash' => $stat['sha1'], 'latest' => $latest ) );
697 } elseif ( $stat === false ) { // file does not exist
698 $this->cheapCache
->set( $path, 'stat', $latest ?
'NOT_EXIST_LATEST' : 'NOT_EXIST' );
699 wfDebug( __METHOD__
. ": File $path does not exist.\n" );
700 } else { // an error occurred
701 wfDebug( __METHOD__
. ": Could not stat file $path.\n" );
703 wfProfileOut( __METHOD__
. '-' . $this->name
);
704 wfProfileOut( __METHOD__
);
709 * @see FileBackendStore::getFileStat()
711 abstract protected function doGetFileStat( array $params );
714 * @see FileBackend::getFileContentsMulti()
717 public function getFileContentsMulti( array $params ) {
718 wfProfileIn( __METHOD__
);
719 wfProfileIn( __METHOD__
. '-' . $this->name
);
721 $params = $this->setConcurrencyFlags( $params );
722 $contents = $this->doGetFileContentsMulti( $params );
724 wfProfileOut( __METHOD__
. '-' . $this->name
);
725 wfProfileOut( __METHOD__
);
730 * @see FileBackendStore::getFileContentsMulti()
733 protected function doGetFileContentsMulti( array $params ) {
735 foreach ( $this->doGetLocalReferenceMulti( $params ) as $path => $fsFile ) {
736 wfSuppressWarnings();
737 $contents[$path] = $fsFile ?
file_get_contents( $fsFile->getPath() ) : false;
744 * @see FileBackend::getFileSha1Base36()
745 * @return bool|string
747 final public function getFileSha1Base36( array $params ) {
748 $path = self
::normalizeStoragePath( $params['src'] );
749 if ( $path === null ) {
750 return false; // invalid storage path
752 wfProfileIn( __METHOD__
);
753 wfProfileIn( __METHOD__
. '-' . $this->name
);
754 $latest = !empty( $params['latest'] ); // use latest data?
755 if ( $this->cheapCache
->has( $path, 'sha1', self
::CACHE_TTL
) ) {
756 $stat = $this->cheapCache
->get( $path, 'sha1' );
757 // If we want the latest data, check that this cached
758 // value was in fact fetched with the latest available data.
759 if ( !$latest ||
$stat['latest'] ) {
760 wfProfileOut( __METHOD__
. '-' . $this->name
);
761 wfProfileOut( __METHOD__
);
762 return $stat['hash'];
765 wfProfileIn( __METHOD__
. '-miss' );
766 wfProfileIn( __METHOD__
. '-miss-' . $this->name
);
767 $hash = $this->doGetFileSha1Base36( $params );
768 wfProfileOut( __METHOD__
. '-miss-' . $this->name
);
769 wfProfileOut( __METHOD__
. '-miss' );
770 $this->cheapCache
->set( $path, 'sha1', array( 'hash' => $hash, 'latest' => $latest ) );
771 wfProfileOut( __METHOD__
. '-' . $this->name
);
772 wfProfileOut( __METHOD__
);
777 * @see FileBackendStore::getFileSha1Base36()
778 * @return bool|string
780 protected function doGetFileSha1Base36( array $params ) {
781 $fsFile = $this->getLocalReference( $params );
785 return $fsFile->getSha1Base36();
790 * @see FileBackend::getFileProps()
793 final public function getFileProps( array $params ) {
794 wfProfileIn( __METHOD__
);
795 wfProfileIn( __METHOD__
. '-' . $this->name
);
796 $fsFile = $this->getLocalReference( $params );
797 $props = $fsFile ?
$fsFile->getProps() : FSFile
::placeholderProps();
798 wfProfileOut( __METHOD__
. '-' . $this->name
);
799 wfProfileOut( __METHOD__
);
804 * @see FileBackend::getLocalReferenceMulti()
807 final public function getLocalReferenceMulti( array $params ) {
808 wfProfileIn( __METHOD__
);
809 wfProfileIn( __METHOD__
. '-' . $this->name
);
811 $params = $this->setConcurrencyFlags( $params );
813 $fsFiles = array(); // (path => FSFile)
814 $latest = !empty( $params['latest'] ); // use latest data?
815 // Reuse any files already in process cache...
816 foreach ( $params['srcs'] as $src ) {
817 $path = self
::normalizeStoragePath( $src );
818 if ( $path === null ) {
819 $fsFiles[$src] = null; // invalid storage path
820 } elseif ( $this->expensiveCache
->has( $path, 'localRef' ) ) {
821 $val = $this->expensiveCache
->get( $path, 'localRef' );
822 // If we want the latest data, check that this cached
823 // value was in fact fetched with the latest available data.
824 if ( !$latest ||
$val['latest'] ) {
825 $fsFiles[$src] = $val['object'];
829 // Fetch local references of any remaning files...
830 $params['srcs'] = array_diff( $params['srcs'], array_keys( $fsFiles ) );
831 foreach ( $this->doGetLocalReferenceMulti( $params ) as $path => $fsFile ) {
832 $fsFiles[$path] = $fsFile;
833 if ( $fsFile ) { // update the process cache...
834 $this->expensiveCache
->set( $path, 'localRef',
835 array( 'object' => $fsFile, 'latest' => $latest ) );
839 wfProfileOut( __METHOD__
. '-' . $this->name
);
840 wfProfileOut( __METHOD__
);
845 * @see FileBackendStore::getLocalReferenceMulti()
848 protected function doGetLocalReferenceMulti( array $params ) {
849 return $this->doGetLocalCopyMulti( $params );
853 * @see FileBackend::getLocalCopyMulti()
856 final public function getLocalCopyMulti( array $params ) {
857 wfProfileIn( __METHOD__
);
858 wfProfileIn( __METHOD__
. '-' . $this->name
);
860 $params = $this->setConcurrencyFlags( $params );
861 $tmpFiles = $this->doGetLocalCopyMulti( $params );
863 wfProfileOut( __METHOD__
. '-' . $this->name
);
864 wfProfileOut( __METHOD__
);
869 * @see FileBackendStore::getLocalCopyMulti()
872 abstract protected function doGetLocalCopyMulti( array $params );
875 * @see FileBackend::getFileHttpUrl()
876 * @return string|null
878 public function getFileHttpUrl( array $params ) {
879 return null; // not supported
883 * @see FileBackend::streamFile()
886 final public function streamFile( array $params ) {
887 wfProfileIn( __METHOD__
);
888 wfProfileIn( __METHOD__
. '-' . $this->name
);
889 $status = Status
::newGood();
891 $info = $this->getFileStat( $params );
892 if ( !$info ) { // let StreamFile handle the 404
893 $status->fatal( 'backend-fail-notexists', $params['src'] );
896 // Set output buffer and HTTP headers for stream
897 $extraHeaders = isset( $params['headers'] ) ?
$params['headers'] : array();
898 $res = StreamFile
::prepareForStream( $params['src'], $info, $extraHeaders );
899 if ( $res == StreamFile
::NOT_MODIFIED
) {
900 // do nothing; client cache is up to date
901 } elseif ( $res == StreamFile
::READY_STREAM
) {
902 wfProfileIn( __METHOD__
. '-send' );
903 wfProfileIn( __METHOD__
. '-send-' . $this->name
);
904 $status = $this->doStreamFile( $params );
905 wfProfileOut( __METHOD__
. '-send-' . $this->name
);
906 wfProfileOut( __METHOD__
. '-send' );
907 if ( !$status->isOK() ) {
908 // Per bug 41113, nasty things can happen if bad cache entries get
909 // stuck in cache. It's also possible that this error can come up
910 // with simple race conditions. Clear out the stat cache to be safe.
911 $this->clearCache( array( $params['src'] ) );
912 $this->deleteFileCache( $params['src'] );
913 trigger_error( "Bad stat cache or race condition for file {$params['src']}." );
916 $status->fatal( 'backend-fail-stream', $params['src'] );
919 wfProfileOut( __METHOD__
. '-' . $this->name
);
920 wfProfileOut( __METHOD__
);
925 * @see FileBackendStore::streamFile()
928 protected function doStreamFile( array $params ) {
929 $status = Status
::newGood();
931 $fsFile = $this->getLocalReference( $params );
933 $status->fatal( 'backend-fail-stream', $params['src'] );
934 } elseif ( !readfile( $fsFile->getPath() ) ) {
935 $status->fatal( 'backend-fail-stream', $params['src'] );
942 * @see FileBackend::directoryExists()
945 final public function directoryExists( array $params ) {
946 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
947 if ( $dir === null ) {
948 return false; // invalid storage path
950 if ( $shard !== null ) { // confined to a single container/shard
951 return $this->doDirectoryExists( $fullCont, $dir, $params );
952 } else { // directory is on several shards
953 wfDebug( __METHOD__
. ": iterating over all container shards.\n" );
954 list( , $shortCont, ) = self
::splitStoragePath( $params['dir'] );
955 $res = false; // response
956 foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
957 $exists = $this->doDirectoryExists( "{$fullCont}{$suffix}", $dir, $params );
961 } elseif ( $exists === null ) { // error?
962 $res = null; // if we don't find anything, it is indeterminate
970 * @see FileBackendStore::directoryExists()
972 * @param string $container Resolved container name
973 * @param string $dir Resolved path relative to container
974 * @param $params Array
977 abstract protected function doDirectoryExists( $container, $dir, array $params );
980 * @see FileBackend::getDirectoryList()
981 * @return Traversable|Array|null Returns null on failure
983 final public function getDirectoryList( array $params ) {
984 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
985 if ( $dir === null ) { // invalid storage path
988 if ( $shard !== null ) {
989 // File listing is confined to a single container/shard
990 return $this->getDirectoryListInternal( $fullCont, $dir, $params );
992 wfDebug( __METHOD__
. ": iterating over all container shards.\n" );
993 // File listing spans multiple containers/shards
994 list( , $shortCont, ) = self
::splitStoragePath( $params['dir'] );
995 return new FileBackendStoreShardDirIterator( $this,
996 $fullCont, $dir, $this->getContainerSuffixes( $shortCont ), $params );
1001 * Do not call this function from places outside FileBackend
1003 * @see FileBackendStore::getDirectoryList()
1005 * @param string $container Resolved container name
1006 * @param string $dir Resolved path relative to container
1007 * @param $params Array
1008 * @return Traversable|Array|null Returns null on failure
1010 abstract public function getDirectoryListInternal( $container, $dir, array $params );
1013 * @see FileBackend::getFileList()
1014 * @return Traversable|Array|null Returns null on failure
1016 final public function getFileList( array $params ) {
1017 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
1018 if ( $dir === null ) { // invalid storage path
1021 if ( $shard !== null ) {
1022 // File listing is confined to a single container/shard
1023 return $this->getFileListInternal( $fullCont, $dir, $params );
1025 wfDebug( __METHOD__
. ": iterating over all container shards.\n" );
1026 // File listing spans multiple containers/shards
1027 list( , $shortCont, ) = self
::splitStoragePath( $params['dir'] );
1028 return new FileBackendStoreShardFileIterator( $this,
1029 $fullCont, $dir, $this->getContainerSuffixes( $shortCont ), $params );
1034 * Do not call this function from places outside FileBackend
1036 * @see FileBackendStore::getFileList()
1038 * @param string $container Resolved container name
1039 * @param string $dir Resolved path relative to container
1040 * @param $params Array
1041 * @return Traversable|Array|null Returns null on failure
1043 abstract public function getFileListInternal( $container, $dir, array $params );
1046 * Return a list of FileOp objects from a list of operations.
1047 * Do not call this function from places outside FileBackend.
1049 * The result must have the same number of items as the input.
1050 * An exception is thrown if an unsupported operation is requested.
1052 * @param array $ops Same format as doOperations()
1053 * @return Array List of FileOp objects
1054 * @throws MWException
1056 final public function getOperationsInternal( array $ops ) {
1057 $supportedOps = array(
1058 'store' => 'StoreFileOp',
1059 'copy' => 'CopyFileOp',
1060 'move' => 'MoveFileOp',
1061 'delete' => 'DeleteFileOp',
1062 'create' => 'CreateFileOp',
1063 'describe' => 'DescribeFileOp',
1064 'null' => 'NullFileOp'
1067 $performOps = array(); // array of FileOp objects
1068 // Build up ordered array of FileOps...
1069 foreach ( $ops as $operation ) {
1070 $opName = $operation['op'];
1071 if ( isset( $supportedOps[$opName] ) ) {
1072 $class = $supportedOps[$opName];
1073 // Get params for this operation
1074 $params = $operation;
1075 // Append the FileOp class
1076 $performOps[] = new $class( $this, $params );
1078 throw new MWException( "Operation '$opName' is not supported." );
1086 * Get a list of storage paths to lock for a list of operations
1087 * Returns an array with 'sh' (shared) and 'ex' (exclusive) keys,
1088 * each corresponding to a list of storage paths to be locked.
1089 * All returned paths are normalized.
1091 * @param array $performOps List of FileOp objects
1092 * @return Array ('sh' => list of paths, 'ex' => list of paths)
1094 final public function getPathsToLockForOpsInternal( array $performOps ) {
1095 // Build up a list of files to lock...
1096 $paths = array( 'sh' => array(), 'ex' => array() );
1097 foreach ( $performOps as $fileOp ) {
1098 $paths['sh'] = array_merge( $paths['sh'], $fileOp->storagePathsRead() );
1099 $paths['ex'] = array_merge( $paths['ex'], $fileOp->storagePathsChanged() );
1101 // Optimization: if doing an EX lock anyway, don't also set an SH one
1102 $paths['sh'] = array_diff( $paths['sh'], $paths['ex'] );
1103 // Get a shared lock on the parent directory of each path changed
1104 $paths['sh'] = array_merge( $paths['sh'], array_map( 'dirname', $paths['ex'] ) );
1110 * @see FileBackend::getScopedLocksForOps()
1113 public function getScopedLocksForOps( array $ops, Status
$status ) {
1114 $paths = $this->getPathsToLockForOpsInternal( $this->getOperationsInternal( $ops ) );
1116 $this->getScopedFileLocks( $paths['sh'], LockManager
::LOCK_UW
, $status ),
1117 $this->getScopedFileLocks( $paths['ex'], LockManager
::LOCK_EX
, $status )
1122 * @see FileBackend::doOperationsInternal()
1125 final protected function doOperationsInternal( array $ops, array $opts ) {
1126 wfProfileIn( __METHOD__
);
1127 wfProfileIn( __METHOD__
. '-' . $this->name
);
1128 $status = Status
::newGood();
1130 // Fix up custom header name/value pairs...
1131 $ops = array_map( array( $this, 'stripInvalidHeadersFromOp' ), $ops );
1133 // Build up a list of FileOps...
1134 $performOps = $this->getOperationsInternal( $ops );
1136 // Acquire any locks as needed...
1137 if ( empty( $opts['nonLocking'] ) ) {
1138 // Build up a list of files to lock...
1139 $paths = $this->getPathsToLockForOpsInternal( $performOps );
1140 // Try to lock those files for the scope of this function...
1141 $scopeLockS = $this->getScopedFileLocks( $paths['sh'], LockManager
::LOCK_UW
, $status );
1142 $scopeLockE = $this->getScopedFileLocks( $paths['ex'], LockManager
::LOCK_EX
, $status );
1143 if ( !$status->isOK() ) {
1144 wfProfileOut( __METHOD__
. '-' . $this->name
);
1145 wfProfileOut( __METHOD__
);
1146 return $status; // abort
1150 // Clear any file cache entries (after locks acquired)
1151 if ( empty( $opts['preserveCache'] ) ) {
1152 $this->clearCache();
1155 // Load from the persistent file and container caches
1156 $this->primeFileCache( $performOps );
1157 $this->primeContainerCache( $performOps );
1159 // Actually attempt the operation batch...
1160 $opts = $this->setConcurrencyFlags( $opts );
1161 $subStatus = FileOpBatch
::attempt( $performOps, $opts, $this->fileJournal
);
1163 // Merge errors into status fields
1164 $status->merge( $subStatus );
1165 $status->success
= $subStatus->success
; // not done in merge()
1167 wfProfileOut( __METHOD__
. '-' . $this->name
);
1168 wfProfileOut( __METHOD__
);
1173 * @see FileBackend::doQuickOperationsInternal()
1175 * @throws MWException
1177 final protected function doQuickOperationsInternal( array $ops ) {
1178 wfProfileIn( __METHOD__
);
1179 wfProfileIn( __METHOD__
. '-' . $this->name
);
1180 $status = Status
::newGood();
1182 // Fix up custom header name/value pairs...
1183 $ops = array_map( array( $this, 'stripInvalidHeadersFromOp' ), $ops );
1185 // Clear any file cache entries
1186 $this->clearCache();
1188 $supportedOps = array( 'create', 'store', 'copy', 'move', 'delete', 'null' );
1189 $async = ( $this->parallelize
=== 'implicit' );
1190 $maxConcurrency = $this->concurrency
; // throttle
1192 $statuses = array(); // array of (index => Status)
1193 $fileOpHandles = array(); // list of (index => handle) arrays
1194 $curFileOpHandles = array(); // current handle batch
1195 // Perform the sync-only ops and build up op handles for the async ops...
1196 foreach ( $ops as $index => $params ) {
1197 if ( !in_array( $params['op'], $supportedOps ) ) {
1198 wfProfileOut( __METHOD__
. '-' . $this->name
);
1199 wfProfileOut( __METHOD__
);
1200 throw new MWException( "Operation '{$params['op']}' is not supported." );
1202 $method = $params['op'] . 'Internal'; // e.g. "storeInternal"
1203 $subStatus = $this->$method( array( 'async' => $async ) +
$params );
1204 if ( $subStatus->value
instanceof FileBackendStoreOpHandle
) { // async
1205 if ( count( $curFileOpHandles ) >= $maxConcurrency ) {
1206 $fileOpHandles[] = $curFileOpHandles; // push this batch
1207 $curFileOpHandles = array();
1209 $curFileOpHandles[$index] = $subStatus->value
; // keep index
1210 } else { // error or completed
1211 $statuses[$index] = $subStatus; // keep index
1214 if ( count( $curFileOpHandles ) ) {
1215 $fileOpHandles[] = $curFileOpHandles; // last batch
1217 // Do all the async ops that can be done concurrently...
1218 foreach ( $fileOpHandles as $fileHandleBatch ) {
1219 $statuses = $statuses +
$this->executeOpHandlesInternal( $fileHandleBatch );
1221 // Marshall and merge all the responses...
1222 foreach ( $statuses as $index => $subStatus ) {
1223 $status->merge( $subStatus );
1224 if ( $subStatus->isOK() ) {
1225 $status->success
[$index] = true;
1226 ++
$status->successCount
;
1228 $status->success
[$index] = false;
1229 ++
$status->failCount
;
1233 wfProfileOut( __METHOD__
. '-' . $this->name
);
1234 wfProfileOut( __METHOD__
);
1239 * Execute a list of FileBackendStoreOpHandle handles in parallel.
1240 * The resulting Status object fields will correspond
1241 * to the order in which the handles where given.
1243 * @param array $handles List of FileBackendStoreOpHandle objects
1244 * @return Array Map of Status objects
1245 * @throws MWException
1247 final public function executeOpHandlesInternal( array $fileOpHandles ) {
1248 wfProfileIn( __METHOD__
);
1249 wfProfileIn( __METHOD__
. '-' . $this->name
);
1250 foreach ( $fileOpHandles as $fileOpHandle ) {
1251 if ( !( $fileOpHandle instanceof FileBackendStoreOpHandle
) ) {
1252 throw new MWException( "Given a non-FileBackendStoreOpHandle object." );
1253 } elseif ( $fileOpHandle->backend
->getName() !== $this->getName() ) {
1254 throw new MWException( "Given a FileBackendStoreOpHandle for the wrong backend." );
1257 $res = $this->doExecuteOpHandlesInternal( $fileOpHandles );
1258 foreach ( $fileOpHandles as $fileOpHandle ) {
1259 $fileOpHandle->closeResources();
1261 wfProfileOut( __METHOD__
. '-' . $this->name
);
1262 wfProfileOut( __METHOD__
);
1267 * @see FileBackendStore::executeOpHandlesInternal()
1268 * @param array $fileOpHandles
1269 * @throws MWException
1270 * @return Array List of corresponding Status objects
1272 protected function doExecuteOpHandlesInternal( array $fileOpHandles ) {
1273 foreach ( $fileOpHandles as $fileOpHandle ) { // OK if empty
1274 throw new MWException( "This backend supports no asynchronous operations." );
1280 * Strip long HTTP headers from a file operation
1282 * @param array $op Same format as doOperation()
1285 protected function stripInvalidHeadersFromOp( array $op ) {
1286 if ( isset( $op['headers'] ) ) {
1287 foreach ( $op['headers'] as $name => $value ) {
1288 if ( strlen( $name ) > 255 ||
strlen( $value ) > 255 ) {
1289 trigger_error( "Header '$name: $value' is too long." );
1290 unset( $op['headers'][$name] );
1291 } elseif ( !strlen( $value ) ) {
1292 $op['headers'][$name] = ''; // null/false => ""
1300 * @see FileBackend::preloadCache()
1302 final public function preloadCache( array $paths ) {
1303 $fullConts = array(); // full container names
1304 foreach ( $paths as $path ) {
1305 list( $fullCont, , ) = $this->resolveStoragePath( $path );
1306 $fullConts[] = $fullCont;
1308 // Load from the persistent file and container caches
1309 $this->primeContainerCache( $fullConts );
1310 $this->primeFileCache( $paths );
1314 * @see FileBackend::clearCache()
1316 final public function clearCache( array $paths = null ) {
1317 if ( is_array( $paths ) ) {
1318 $paths = array_map( 'FileBackend::normalizeStoragePath', $paths );
1319 $paths = array_filter( $paths, 'strlen' ); // remove nulls
1321 if ( $paths === null ) {
1322 $this->cheapCache
->clear();
1323 $this->expensiveCache
->clear();
1325 foreach ( $paths as $path ) {
1326 $this->cheapCache
->clear( $path );
1327 $this->expensiveCache
->clear( $path );
1330 $this->doClearCache( $paths );
1334 * Clears any additional stat caches for storage paths
1336 * @see FileBackend::clearCache()
1338 * @param array $paths Storage paths (optional)
1341 protected function doClearCache( array $paths = null ) {}
1344 * Is this a key/value store where directories are just virtual?
1345 * Virtual directories exists in so much as files exists that are
1346 * prefixed with the directory path followed by a forward slash.
1350 abstract protected function directoriesAreVirtual();
1353 * Check if a container name is valid.
1354 * This checks for for length and illegal characters.
1356 * @param $container string
1359 final protected static function isValidContainerName( $container ) {
1360 // This accounts for Swift and S3 restrictions while leaving room
1361 // for things like '.xxx' (hex shard chars) or '.seg' (segments).
1362 // This disallows directory separators or traversal characters.
1363 // Note that matching strings URL encode to the same string;
1364 // in Swift, the length restriction is *after* URL encoding.
1365 return preg_match( '/^[a-z0-9][a-z0-9-_]{0,199}$/i', $container );
1369 * Splits a storage path into an internal container name,
1370 * an internal relative file name, and a container shard suffix.
1371 * Any shard suffix is already appended to the internal container name.
1372 * This also checks that the storage path is valid and within this backend.
1374 * If the container is sharded but a suffix could not be determined,
1375 * this means that the path can only refer to a directory and can only
1376 * be scanned by looking in all the container shards.
1378 * @param $storagePath string
1379 * @return Array (container, path, container suffix) or (null, null, null) if invalid
1381 final protected function resolveStoragePath( $storagePath ) {
1382 list( $backend, $container, $relPath ) = self
::splitStoragePath( $storagePath );
1383 if ( $backend === $this->name
) { // must be for this backend
1384 $relPath = self
::normalizeContainerPath( $relPath );
1385 if ( $relPath !== null ) {
1386 // Get shard for the normalized path if this container is sharded
1387 $cShard = $this->getContainerShard( $container, $relPath );
1388 // Validate and sanitize the relative path (backend-specific)
1389 $relPath = $this->resolveContainerPath( $container, $relPath );
1390 if ( $relPath !== null ) {
1391 // Prepend any wiki ID prefix to the container name
1392 $container = $this->fullContainerName( $container );
1393 if ( self
::isValidContainerName( $container ) ) {
1394 // Validate and sanitize the container name (backend-specific)
1395 $container = $this->resolveContainerName( "{$container}{$cShard}" );
1396 if ( $container !== null ) {
1397 return array( $container, $relPath, $cShard );
1403 return array( null, null, null );
1407 * Like resolveStoragePath() except null values are returned if
1408 * the container is sharded and the shard could not be determined.
1410 * @see FileBackendStore::resolveStoragePath()
1412 * @param $storagePath string
1413 * @return Array (container, path) or (null, null) if invalid
1415 final protected function resolveStoragePathReal( $storagePath ) {
1416 list( $container, $relPath, $cShard ) = $this->resolveStoragePath( $storagePath );
1417 if ( $cShard !== null ) {
1418 return array( $container, $relPath );
1420 return array( null, null );
1424 * Get the container name shard suffix for a given path.
1425 * Any empty suffix means the container is not sharded.
1427 * @param string $container Container name
1428 * @param string $relPath Storage path relative to the container
1429 * @return string|null Returns null if shard could not be determined
1431 final protected function getContainerShard( $container, $relPath ) {
1432 list( $levels, $base, $repeat ) = $this->getContainerHashLevels( $container );
1433 if ( $levels == 1 ||
$levels == 2 ) {
1434 // Hash characters are either base 16 or 36
1435 $char = ( $base == 36 ) ?
'[0-9a-z]' : '[0-9a-f]';
1436 // Get a regex that represents the shard portion of paths.
1437 // The concatenation of the captures gives us the shard.
1438 if ( $levels === 1 ) { // 16 or 36 shards per container
1439 $hashDirRegex = '(' . $char . ')';
1440 } else { // 256 or 1296 shards per container
1441 if ( $repeat ) { // verbose hash dir format (e.g. "a/ab/abc")
1442 $hashDirRegex = $char . '/(' . $char . '{2})';
1443 } else { // short hash dir format (e.g. "a/b/c")
1444 $hashDirRegex = '(' . $char . ')/(' . $char . ')';
1447 // Allow certain directories to be above the hash dirs so as
1448 // to work with FileRepo (e.g. "archive/a/ab" or "temp/a/ab").
1449 // They must be 2+ chars to avoid any hash directory ambiguity.
1451 if ( preg_match( "!^(?:[^/]{2,}/)*$hashDirRegex(?:/|$)!", $relPath, $m ) ) {
1452 return '.' . implode( '', array_slice( $m, 1 ) );
1454 return null; // failed to match
1456 return ''; // no sharding
1460 * Check if a storage path maps to a single shard.
1461 * Container dirs like "a", where the container shards on "x/xy",
1462 * can reside on several shards. Such paths are tricky to handle.
1464 * @param string $storagePath Storage path
1467 final public function isSingleShardPathInternal( $storagePath ) {
1468 list( , , $shard ) = $this->resolveStoragePath( $storagePath );
1469 return ( $shard !== null );
1473 * Get the sharding config for a container.
1474 * If greater than 0, then all file storage paths within
1475 * the container are required to be hashed accordingly.
1477 * @param $container string
1478 * @return Array (integer levels, integer base, repeat flag) or (0, 0, false)
1480 final protected function getContainerHashLevels( $container ) {
1481 if ( isset( $this->shardViaHashLevels
[$container] ) ) {
1482 $config = $this->shardViaHashLevels
[$container];
1483 $hashLevels = (int)$config['levels'];
1484 if ( $hashLevels == 1 ||
$hashLevels == 2 ) {
1485 $hashBase = (int)$config['base'];
1486 if ( $hashBase == 16 ||
$hashBase == 36 ) {
1487 return array( $hashLevels, $hashBase, $config['repeat'] );
1491 return array( 0, 0, false ); // no sharding
1495 * Get a list of full container shard suffixes for a container
1497 * @param $container string
1500 final protected function getContainerSuffixes( $container ) {
1502 list( $digits, $base ) = $this->getContainerHashLevels( $container );
1503 if ( $digits > 0 ) {
1504 $numShards = pow( $base, $digits );
1505 for ( $index = 0; $index < $numShards; $index++
) {
1506 $shards[] = '.' . wfBaseConvert( $index, 10, $base, $digits );
1513 * Get the full container name, including the wiki ID prefix
1515 * @param $container string
1518 final protected function fullContainerName( $container ) {
1519 if ( $this->wikiId
!= '' ) {
1520 return "{$this->wikiId}-$container";
1527 * Resolve a container name, checking if it's allowed by the backend.
1528 * This is intended for internal use, such as encoding illegal chars.
1529 * Subclasses can override this to be more restrictive.
1531 * @param $container string
1532 * @return string|null
1534 protected function resolveContainerName( $container ) {
1539 * Resolve a relative storage path, checking if it's allowed by the backend.
1540 * This is intended for internal use, such as encoding illegal chars or perhaps
1541 * getting absolute paths (e.g. FS based backends). Note that the relative path
1542 * may be the empty string (e.g. the path is simply to the container).
1544 * @param string $container Container name
1545 * @param string $relStoragePath Storage path relative to the container
1546 * @return string|null Path or null if not valid
1548 protected function resolveContainerPath( $container, $relStoragePath ) {
1549 return $relStoragePath;
1553 * Get the cache key for a container
1555 * @param string $container Resolved container name
1558 private function containerCacheKey( $container ) {
1559 return wfMemcKey( 'backend', $this->getName(), 'container', $container );
1563 * Set the cached info for a container
1565 * @param string $container Resolved container name
1566 * @param $val mixed Information to cache
1568 final protected function setContainerCache( $container, $val ) {
1569 $this->memCache
->add( $this->containerCacheKey( $container ), $val, 14*86400 );
1573 * Delete the cached info for a container.
1574 * The cache key is salted for a while to prevent race conditions.
1576 * @param string $container Resolved container name
1578 final protected function deleteContainerCache( $container ) {
1579 if ( !$this->memCache
->set( $this->containerCacheKey( $container ), 'PURGED', 300 ) ) {
1580 trigger_error( "Unable to delete stat cache for container $container." );
1585 * Do a batch lookup from cache for container stats for all containers
1586 * used in a list of container names, storage paths, or FileOp objects.
1587 * This loads the persistent cache values into the process cache.
1589 * @param $items Array
1592 final protected function primeContainerCache( array $items ) {
1593 wfProfileIn( __METHOD__
);
1594 wfProfileIn( __METHOD__
. '-' . $this->name
);
1596 $paths = array(); // list of storage paths
1597 $contNames = array(); // (cache key => resolved container name)
1598 // Get all the paths/containers from the items...
1599 foreach ( $items as $item ) {
1600 if ( $item instanceof FileOp
) {
1601 $paths = array_merge( $paths, $item->storagePathsRead() );
1602 $paths = array_merge( $paths, $item->storagePathsChanged() );
1603 } elseif ( self
::isStoragePath( $item ) ) {
1605 } elseif ( is_string( $item ) ) { // full container name
1606 $contNames[$this->containerCacheKey( $item )] = $item;
1609 // Get all the corresponding cache keys for paths...
1610 foreach ( $paths as $path ) {
1611 list( $fullCont, , ) = $this->resolveStoragePath( $path );
1612 if ( $fullCont !== null ) { // valid path for this backend
1613 $contNames[$this->containerCacheKey( $fullCont )] = $fullCont;
1617 $contInfo = array(); // (resolved container name => cache value)
1618 // Get all cache entries for these container cache keys...
1619 $values = $this->memCache
->getMulti( array_keys( $contNames ) );
1620 foreach ( $values as $cacheKey => $val ) {
1621 $contInfo[$contNames[$cacheKey]] = $val;
1624 // Populate the container process cache for the backend...
1625 $this->doPrimeContainerCache( array_filter( $contInfo, 'is_array' ) );
1627 wfProfileOut( __METHOD__
. '-' . $this->name
);
1628 wfProfileOut( __METHOD__
);
1632 * Fill the backend-specific process cache given an array of
1633 * resolved container names and their corresponding cached info.
1634 * Only containers that actually exist should appear in the map.
1636 * @param array $containerInfo Map of resolved container names to cached info
1639 protected function doPrimeContainerCache( array $containerInfo ) {}
1642 * Get the cache key for a file path
1644 * @param string $path Normalized storage path
1647 private function fileCacheKey( $path ) {
1648 return wfMemcKey( 'backend', $this->getName(), 'file', sha1( $path ) );
1652 * Set the cached stat info for a file path.
1653 * Negatives (404s) are not cached. By not caching negatives, we can skip cache
1654 * salting for the case when a file is created at a path were there was none before.
1656 * @param string $path Storage path
1657 * @param $val mixed Information to cache
1659 final protected function setFileCache( $path, $val ) {
1660 $path = FileBackend
::normalizeStoragePath( $path );
1661 if ( $path === null ) {
1662 return; // invalid storage path
1664 $this->memCache
->add( $this->fileCacheKey( $path ), $val, 7*86400 );
1668 * Delete the cached stat info for a file path.
1669 * The cache key is salted for a while to prevent race conditions.
1670 * Since negatives (404s) are not cached, this does not need to be called when
1671 * a file is created at a path were there was none before.
1673 * @param string $path Storage path
1675 final protected function deleteFileCache( $path ) {
1676 $path = FileBackend
::normalizeStoragePath( $path );
1677 if ( $path === null ) {
1678 return; // invalid storage path
1680 if ( !$this->memCache
->set( $this->fileCacheKey( $path ), 'PURGED', 300 ) ) {
1681 trigger_error( "Unable to delete stat cache for file $path." );
1686 * Do a batch lookup from cache for file stats for all paths
1687 * used in a list of storage paths or FileOp objects.
1688 * This loads the persistent cache values into the process cache.
1690 * @param array $items List of storage paths or FileOps
1693 final protected function primeFileCache( array $items ) {
1694 wfProfileIn( __METHOD__
);
1695 wfProfileIn( __METHOD__
. '-' . $this->name
);
1697 $paths = array(); // list of storage paths
1698 $pathNames = array(); // (cache key => storage path)
1699 // Get all the paths/containers from the items...
1700 foreach ( $items as $item ) {
1701 if ( $item instanceof FileOp
) {
1702 $paths = array_merge( $paths, $item->storagePathsRead() );
1703 $paths = array_merge( $paths, $item->storagePathsChanged() );
1704 } elseif ( self
::isStoragePath( $item ) ) {
1705 $paths[] = FileBackend
::normalizeStoragePath( $item );
1708 // Get rid of any paths that failed normalization...
1709 $paths = array_filter( $paths, 'strlen' ); // remove nulls
1710 // Get all the corresponding cache keys for paths...
1711 foreach ( $paths as $path ) {
1712 list( , $rel, ) = $this->resolveStoragePath( $path );
1713 if ( $rel !== null ) { // valid path for this backend
1714 $pathNames[$this->fileCacheKey( $path )] = $path;
1717 // Get all cache entries for these container cache keys...
1718 $values = $this->memCache
->getMulti( array_keys( $pathNames ) );
1719 foreach ( $values as $cacheKey => $val ) {
1720 if ( is_array( $val ) ) {
1721 $path = $pathNames[$cacheKey];
1722 $this->cheapCache
->set( $path, 'stat', $val );
1723 if ( isset( $val['sha1'] ) ) { // some backends store SHA-1 as metadata
1724 $this->cheapCache
->set( $path, 'sha1',
1725 array( 'hash' => $val['sha1'], 'latest' => $val['latest'] ) );
1730 wfProfileOut( __METHOD__
. '-' . $this->name
);
1731 wfProfileOut( __METHOD__
);
1735 * Set the 'concurrency' option from a list of operation options
1737 * @param array $opts Map of operation options
1740 final protected function setConcurrencyFlags( array $opts ) {
1741 $opts['concurrency'] = 1; // off
1742 if ( $this->parallelize
=== 'implicit' ) {
1743 if ( !isset( $opts['parallelize'] ) ||
$opts['parallelize'] ) {
1744 $opts['concurrency'] = $this->concurrency
;
1746 } elseif ( $this->parallelize
=== 'explicit' ) {
1747 if ( !empty( $opts['parallelize'] ) ) {
1748 $opts['concurrency'] = $this->concurrency
;
1756 * FileBackendStore helper class for performing asynchronous file operations.
1758 * For example, calling FileBackendStore::createInternal() with the "async"
1759 * param flag may result in a Status that contains this object as a value.
1760 * This class is largely backend-specific and is mostly just "magic" to be
1761 * passed to FileBackendStore::executeOpHandlesInternal().
1763 abstract class FileBackendStoreOpHandle
{
1765 public $params = array(); // params to caller functions
1766 /** @var FileBackendStore */
1769 public $resourcesToClose = array();
1771 public $call; // string; name that identifies the function called
1774 * Close all open file handles
1778 public function closeResources() {
1779 array_map( 'fclose', $this->resourcesToClose
);
1784 * FileBackendStore helper function to handle listings that span container shards.
1785 * Do not use this class from places outside of FileBackendStore.
1787 * @ingroup FileBackend
1789 abstract class FileBackendStoreShardListIterator
implements Iterator
{
1790 /** @var FileBackendStore */
1795 protected $shardSuffixes;
1796 protected $container; // string; full container name
1797 protected $directory; // string; resolved relative path
1799 /** @var Traversable */
1801 protected $curShard = 0; // integer
1802 protected $pos = 0; // integer
1805 protected $multiShardPaths = array(); // (rel path => 1)
1808 * @param $backend FileBackendStore
1809 * @param string $container Full storage container name
1810 * @param string $dir Storage directory relative to container
1811 * @param array $suffixes List of container shard suffixes
1812 * @param $params Array
1814 public function __construct(
1815 FileBackendStore
$backend, $container, $dir, array $suffixes, array $params
1817 $this->backend
= $backend;
1818 $this->container
= $container;
1819 $this->directory
= $dir;
1820 $this->shardSuffixes
= $suffixes;
1821 $this->params
= $params;
1825 * @see Iterator::key()
1828 public function key() {
1833 * @see Iterator::valid()
1836 public function valid() {
1837 if ( $this->iter
instanceof Iterator
) {
1838 return $this->iter
->valid();
1839 } elseif ( is_array( $this->iter
) ) {
1840 return ( current( $this->iter
) !== false ); // no paths can have this value
1842 return false; // some failure?
1846 * @see Iterator::current()
1847 * @return string|bool String or false
1849 public function current() {
1850 return ( $this->iter
instanceof Iterator
)
1851 ?
$this->iter
->current()
1852 : current( $this->iter
);
1856 * @see Iterator::next()
1859 public function next() {
1861 ( $this->iter
instanceof Iterator
) ?
$this->iter
->next() : next( $this->iter
);
1863 $continue = false; // keep scanning shards?
1864 $this->filterViaNext(); // filter out duplicates
1865 // Find the next non-empty shard if no elements are left
1866 if ( !$this->valid() ) {
1867 $this->nextShardIteratorIfNotValid();
1868 $continue = $this->valid(); // re-filter unless we ran out of shards
1870 } while ( $continue );
1874 * @see Iterator::rewind()
1877 public function rewind() {
1879 $this->curShard
= 0;
1880 $this->setIteratorFromCurrentShard();
1882 $continue = false; // keep scanning shards?
1883 $this->filterViaNext(); // filter out duplicates
1884 // Find the next non-empty shard if no elements are left
1885 if ( !$this->valid() ) {
1886 $this->nextShardIteratorIfNotValid();
1887 $continue = $this->valid(); // re-filter unless we ran out of shards
1889 } while ( $continue );
1893 * Filter out duplicate items by advancing to the next ones
1895 protected function filterViaNext() {
1896 while ( $this->valid() ) {
1897 $rel = $this->iter
->current(); // path relative to given directory
1898 $path = $this->params
['dir'] . "/{$rel}"; // full storage path
1899 if ( $this->backend
->isSingleShardPathInternal( $path ) ) {
1900 break; // path is only on one shard; no issue with duplicates
1901 } elseif ( isset( $this->multiShardPaths
[$rel] ) ) {
1902 // Don't keep listing paths that are on multiple shards
1903 ( $this->iter
instanceof Iterator
) ?
$this->iter
->next() : next( $this->iter
);
1905 $this->multiShardPaths
[$rel] = 1;
1912 * If the list iterator for this container shard is out of items,
1913 * then move on to the next container that has items.
1914 * If there are none, then it advances to the last container.
1916 protected function nextShardIteratorIfNotValid() {
1917 while ( !$this->valid() && ++
$this->curShard
< count( $this->shardSuffixes
) ) {
1918 $this->setIteratorFromCurrentShard();
1923 * Set the list iterator to that of the current container shard
1925 protected function setIteratorFromCurrentShard() {
1926 $this->iter
= $this->listFromShard(
1927 $this->container
. $this->shardSuffixes
[$this->curShard
],
1928 $this->directory
, $this->params
);
1929 // Start loading results so that current() works
1930 if ( $this->iter
) {
1931 ( $this->iter
instanceof Iterator
) ?
$this->iter
->rewind() : reset( $this->iter
);
1936 * Get the list for a given container shard
1938 * @param string $container Resolved container name
1939 * @param string $dir Resolved path relative to container
1940 * @param $params Array
1941 * @return Traversable|Array|null
1943 abstract protected function listFromShard( $container, $dir, array $params );
1947 * Iterator for listing directories
1949 class FileBackendStoreShardDirIterator
extends FileBackendStoreShardListIterator
{
1951 * @see FileBackendStoreShardListIterator::listFromShard()
1952 * @return Array|null|Traversable
1954 protected function listFromShard( $container, $dir, array $params ) {
1955 return $this->backend
->getDirectoryListInternal( $container, $dir, $params );
1960 * Iterator for listing regular files
1962 class FileBackendStoreShardFileIterator
extends FileBackendStoreShardListIterator
{
1964 * @see FileBackendStoreShardListIterator::listFromShard()
1965 * @return Array|null|Traversable
1967 protected function listFromShard( $container, $dir, array $params ) {
1968 return $this->backend
->getFileListInternal( $container, $dir, $params );