dépôts / lhc / web / wiklou.git / blob
? search:


3e8ba88a4344099e541d328a83cb7e0116cd9994
[lhc/web/wiklou.git] / includes / filerepo / backend / FileBackend.php
1 <?php
2 /**
3 * @file
4 * @ingroup FileBackend
5 * @author Aaron Schulz
6 */
7
8 /**
9 * Base class for all file backend classes (including multi-write backends).
10 * This class defines the methods as abstract that subclasses must implement.
11 * Outside callers can assume that all backends will have these functions.
12 *
13 * All "storage paths" are of the format "mwstore://backend/container/path".
14 * The paths use UNIX file system (FS) notation, though any particular backend may
15 * not actually be using a local filesystem. Therefore, the paths are only virtual.
16 *
17 * Backend contents are stored under wiki-specific container names by default.
18 * For legacy reasons, this has no effect for the FS backend class, and per-wiki
19 * segregation must be done by setting the container paths appropriately.
20 *
21 * FS-based backends are somewhat more restrictive due to the existence of real
22 * directory files; a regular file cannot have the same name as a directory. Other
23 * backends with virtual directories may not have this limitation. Callers should
24 * store files in such a way that no files and directories are under the same path.
25 *
26 * Methods should avoid throwing exceptions at all costs.
27 * As a corollary, external dependencies should be kept to a minimum.
28 *
29 * @ingroup FileBackend
30 * @since 1.19
31 */
32 abstract class FileBackendBase {
33 protected $name; // unique backend name
34 protected $wikiId; // unique wiki name
35 protected $readOnly; // string
36 /** @var LockManager */
37 protected $lockManager;
38
39 /**
40 * Create a new backend instance from configuration.
41 * This should only be called from within FileBackendGroup.
42 *
43 * $config includes:
44 * 'name' : The unique name of this backend.
45 * 'wikiId' : Prefix to container names that is unique to this wiki.
46 * This should consist of alphanumberic, '-', and '_' chars.
47 * 'lockManager' : Registered name of a file lock manager to use.
48 * 'readOnly' : Write operations are disallowed if this is a non-empty string.
49 * It should be an explanation for the backend being read-only.
50 *
51 * @param $config Array
52 */
53 public function __construct( array $config ) {
54 $this->name = $config['name'];
55 $this->wikiId = isset( $config['wikiId'] )
56 ? $config['wikiId']
57 : wfWikiID(); // e.g. "my_wiki-en_"
58 $this->wikiId = $this->resolveWikiId( $this->wikiId );
59 $this->lockManager = LockManagerGroup::singleton()->get( $config['lockManager'] );
60 $this->readOnly = isset( $config['readOnly'] )
61 ? (string)$config['readOnly']
62 : '';
63 }
64
65 /**
66 * Normalize a wiki ID by replacing characters that are
67 * not supported by the backend as part of container names.
68 *
69 * @param $wikiId string
70 * @return string
71 */
72 protected function resolveWikiId( $wikiId ) {
73 return $wikiId;
74 }
75
76 /**
77 * Get the unique backend name.
78 * We may have multiple different backends of the same type.
79 * For example, we can have two Swift backends using different proxies.
80 *
81 * @return string
82 */
83 final public function getName() {
84 return $this->name;
85 }
86
87 /**
88 * This is the main entry point into the backend for write operations.
89 * Callers supply an ordered list of operations to perform as a transaction.
90 * Files will be locked, the stat cache cleared, and then the operations attempted.
91 * If any serious errors occur, all attempted operations will be rolled back.
92 *
93 * $ops is an array of arrays. The outer array holds a list of operations.
94 * Each inner array is a set of key value pairs that specify an operation.
95 *
96 * Supported operations and their parameters:
97 * a) Create a new file in storage with the contents of a string
98 * array(
99 * 'op' => 'create',
100 * 'dst' => <storage path>,
101 * 'content' => <string of new file contents>,
102 * 'overwrite' => <boolean>,
103 * 'overwriteSame' => <boolean>
104 * )
105 * b) Copy a file system file into storage
106 * array(
107 * 'op' => 'store',
108 * 'src' => <file system path>,
109 * 'dst' => <storage path>,
110 * 'overwrite' => <boolean>,
111 * 'overwriteSame' => <boolean>
112 * )
113 * c) Copy a file within storage
114 * array(
115 * 'op' => 'copy',
116 * 'src' => <storage path>,
117 * 'dst' => <storage path>,
118 * 'overwrite' => <boolean>,
119 * 'overwriteSame' => <boolean>
120 * )
121 * d) Move a file within storage
122 * array(
123 * 'op' => 'move',
124 * 'src' => <storage path>,
125 * 'dst' => <storage path>,
126 * 'overwrite' => <boolean>,
127 * 'overwriteSame' => <boolean>
128 * )
129 * e) Delete a file within storage
130 * array(
131 * 'op' => 'delete',
132 * 'src' => <storage path>,
133 * 'ignoreMissingSource' => <boolean>
134 * )
135 * f) Do nothing (no-op)
136 * array(
137 * 'op' => 'null',
138 * )
139 *
140 * Boolean flags for operations (operation-specific):
141 * 'ignoreMissingSource' : The operation will simply succeed and do
142 * nothing if the source file does not exist.
143 * 'overwrite' : Any destination file will be overwritten.
144 * 'overwriteSame' : An error will not be given if a file already
145 * exists at the destination that has the same
146 * contents as the new contents to be written there.
147 *
148 * $opts is an associative of boolean flags, including:
149 * 'force' : Errors that would normally cause a rollback do not.
150 * The remaining operations are still attempted if any fail.
151 * 'nonLocking' : No locks are acquired for the operations.
152 * This can increase performance for non-critical writes.
153 * This has no effect unless the 'force' flag is set.
154 * 'allowStale' : Don't require the latest available data.
155 * This can increase performance for non-critical writes.
156 * This has no effect unless the 'force' flag is set.
157 *
158 * Remarks on locking:
159 * File system paths given to operations should refer to files that are
160 * already locked or otherwise safe from modification from other processes.
161 * Normally these files will be new temp files, which should be adequate.
162 *
163 * Return value:
164 * This returns a Status, which contains all warnings and fatals that occured
165 * during the operation. The 'failCount', 'successCount', and 'success' members
166 * will reflect each operation attempted. The status will be "OK" unless any
167 * of the operations failed and the 'force' parameter was not set.
168 *
169 * @param $ops Array List of operations to execute in order
170 * @param $opts Array Batch operation options
171 * @return Status
172 */
173 final public function doOperations( array $ops, array $opts = array() ) {
174 if ( $this->readOnly != '' ) {
175 return Status::newFatal( 'backend-fail-readonly', $this->name, $this->readOnly );
176 }
177 if ( empty( $opts['force'] ) ) { // sanity
178 unset( $opts['nonLocking'] );
179 unset( $opts['allowStale'] );
180 }
181 return $this->doOperationsInternal( $ops, $opts );
182 }
183
184 /**
185 * @see FileBackendBase::doOperations()
186 */
187 abstract protected function doOperationsInternal( array $ops, array $opts );
188
189 /**
190 * Same as doOperations() except it takes a single operation.
191 * If you are doing a batch of operations that should either
192 * all succeed or all fail, then use that function instead.
193 *
194 * @see FileBackendBase::doOperations()
195 *
196 * @param $op Array Operation
197 * @param $opts Array Operation options
198 * @return Status
199 */
200 final public function doOperation( array $op, array $opts = array() ) {
201 return $this->doOperations( array( $op ), $opts );
202 }
203
204 /**
205 * Performs a single create operation.
206 * This sets $params['op'] to 'create' and passes it to doOperation().
207 *
208 * @see FileBackendBase::doOperation()
209 *
210 * @param $params Array Operation parameters
211 * @param $opts Array Operation options
212 * @return Status
213 */
214 final public function create( array $params, array $opts = array() ) {
215 $params['op'] = 'create';
216 return $this->doOperation( $params, $opts );
217 }
218
219 /**
220 * Performs a single store operation.
221 * This sets $params['op'] to 'store' and passes it to doOperation().
222 *
223 * @see FileBackendBase::doOperation()
224 *
225 * @param $params Array Operation parameters
226 * @param $opts Array Operation options
227 * @return Status
228 */
229 final public function store( array $params, array $opts = array() ) {
230 $params['op'] = 'store';
231 return $this->doOperation( $params, $opts );
232 }
233
234 /**
235 * Performs a single copy operation.
236 * This sets $params['op'] to 'copy' and passes it to doOperation().
237 *
238 * @see FileBackendBase::doOperation()
239 *
240 * @param $params Array Operation parameters
241 * @param $opts Array Operation options
242 * @return Status
243 */
244 final public function copy( array $params, array $opts = array() ) {
245 $params['op'] = 'copy';
246 return $this->doOperation( $params, $opts );
247 }
248
249 /**
250 * Performs a single move operation.
251 * This sets $params['op'] to 'move' and passes it to doOperation().
252 *
253 * @see FileBackendBase::doOperation()
254 *
255 * @param $params Array Operation parameters
256 * @param $opts Array Operation options
257 * @return Status
258 */
259 final public function move( array $params, array $opts = array() ) {
260 $params['op'] = 'move';
261 return $this->doOperation( $params, $opts );
262 }
263
264 /**
265 * Performs a single delete operation.
266 * This sets $params['op'] to 'delete' and passes it to doOperation().
267 *
268 * @see FileBackendBase::doOperation()
269 *
270 * @param $params Array Operation parameters
271 * @param $opts Array Operation options
272 * @return Status
273 */
274 final public function delete( array $params, array $opts = array() ) {
275 $params['op'] = 'delete';
276 return $this->doOperation( $params, $opts );
277 }
278
279 /**
280 * Concatenate a list of storage files into a single file on the file system
281 * $params include:
282 * srcs : ordered source storage paths (e.g. chunk1, chunk2, ...)
283 * dst : file system path to 0-byte temp file
284 *
285 * @param $params Array Operation parameters
286 * @return Status
287 */
288 abstract public function concatenate( array $params );
289
290 /**
291 * Prepare a storage directory for usage.
292 * This will create any required containers and parent directories.
293 * Backends using key/value stores only need to create the container.
294 *
295 * $params include:
296 * dir : storage directory
297 *
298 * @param $params Array
299 * @return Status
300 */
301 final public function prepare( array $params ) {
302 if ( $this->readOnly != '' ) {
303 return Status::newFatal( 'backend-fail-readonly', $this->name, $this->readOnly );
304 }
305 return $this->doPrepare( $params );
306 }
307
308 /**
309 * @see FileBackendBase::prepare()
310 */
311 abstract protected function doPrepare( array $params );
312
313 /**
314 * Take measures to block web access to a storage directory and
315 * the container it belongs to. FS backends might add .htaccess
316 * files whereas key/value store backends might restrict container
317 * access to the auth user that represents end-users in web request.
318 * This is not guaranteed to actually do anything.
319 *
320 * $params include:
321 * dir : storage directory
322 * noAccess : try to deny file access
323 * noListing : try to deny file listing
324 *
325 * @param $params Array
326 * @return Status
327 */
328 final public function secure( array $params ) {
329 if ( $this->readOnly != '' ) {
330 return Status::newFatal( 'backend-fail-readonly', $this->name, $this->readOnly );
331 }
332 $status = $this->doPrepare( $params ); // dir must exist to restrict it
333 if ( $status->isOK() ) {
334 $status->merge( $this->doSecure( $params ) );
335 }
336 return $status;
337 }
338
339 /**
340 * @see FileBackendBase::secure()
341 */
342 abstract protected function doSecure( array $params );
343
344 /**
345 * Delete a storage directory if it is empty.
346 * Backends using key/value stores may do nothing unless the directory
347 * is that of an empty container, in which case it should be deleted.
348 *
349 * $params include:
350 * dir : storage directory
351 *
352 * @param $params Array
353 * @return Status
354 */
355 final public function clean( array $params ) {
356 if ( $this->readOnly != '' ) {
357 return Status::newFatal( 'backend-fail-readonly', $this->name, $this->readOnly );
358 }
359 return $this->doClean( $params );
360 }
361
362 /**
363 * @see FileBackendBase::clean()
364 */
365 abstract protected function doClean( array $params );
366
367 /**
368 * Check if a file exists at a storage path in the backend.
369 * This returns false if only a directory exists at the path.
370 *
371 * $params include:
372 * src : source storage path
373 * latest : use the latest available data
374 *
375 * @param $params Array
376 * @return bool|null Returns null on failure
377 */
378 abstract public function fileExists( array $params );
379
380 /**
381 * Get the last-modified timestamp of the file at a storage path.
382 *
383 * $params include:
384 * src : source storage path
385 * latest : use the latest available data
386 *
387 * @param $params Array
388 * @return string|false TS_MW timestamp or false on failure
389 */
390 abstract public function getFileTimestamp( array $params );
391
392 /**
393 * Get the contents of a file at a storage path in the backend.
394 * This should be avoided for potentially large files.
395 *
396 * $params include:
397 * src : source storage path
398 * latest : use the latest available data
399 *
400 * @param $params Array
401 * @return string|false Returns false on failure
402 */
403 abstract public function getFileContents( array $params );
404
405 /**
406 * Get the size (bytes) of a file at a storage path in the backend.
407 *
408 * $params include:
409 * src : source storage path
410 * latest : use the latest available data
411 *
412 * @param $params Array
413 * @return integer|false Returns false on failure
414 */
415 abstract public function getFileSize( array $params );
416
417 /**
418 * Get quick information about a file at a storage path in the backend.
419 * If the file does not exist, then this returns false.
420 * Otherwise, the result is an associative array that includes:
421 * mtime : the last-modified timestamp (TS_MW)
422 * size : the file size (bytes)
423 * Additional values may be included for internal use only.
424 *
425 * $params include:
426 * src : source storage path
427 * latest : use the latest available data
428 *
429 * @param $params Array
430 * @return Array|false|null Returns null on failure
431 */
432 abstract public function getFileStat( array $params );
433
434 /**
435 * Get a SHA-1 hash of the file at a storage path in the backend.
436 *
437 * $params include:
438 * src : source storage path
439 * latest : use the latest available data
440 *
441 * @param $params Array
442 * @return string|false Hash string or false on failure
443 */
444 abstract public function getFileSha1Base36( array $params );
445
446 /**
447 * Get the properties of the file at a storage path in the backend.
448 * Returns FSFile::placeholderProps() on failure.
449 *
450 * $params include:
451 * src : source storage path
452 * latest : use the latest available data
453 *
454 * @param $params Array
455 * @return Array
456 */
457 abstract public function getFileProps( array $params );
458
459 /**
460 * Stream the file at a storage path in the backend.
461 * If the file does not exists, a 404 error will be given.
462 * Appropriate HTTP headers (Status, Content-Type, Content-Length)
463 * must be sent if streaming began, while none should be sent otherwise.
464 * Implementations should flush the output buffer before sending data.
465 *
466 * $params include:
467 * src : source storage path
468 * headers : additional HTTP headers to send on success
469 * latest : use the latest available data
470 *
471 * @param $params Array
472 * @return Status
473 */
474 abstract public function streamFile( array $params );
475
476 /**
477 * Returns a file system file, identical to the file at a storage path.
478 * The file returned is either:
479 * a) A local copy of the file at a storage path in the backend.
480 * The temporary copy will have the same extension as the source.
481 * b) An original of the file at a storage path in the backend.
482 * Temporary files may be purged when the file object falls out of scope.
483 *
484 * Write operations should *never* be done on this file as some backends
485 * may do internal tracking or may be instances of FileBackendMultiWrite.
486 * In that later case, there are copies of the file that must stay in sync.
487 *
488 * $params include:
489 * src : source storage path
490 * latest : use the latest available data
491 *
492 * @param $params Array
493 * @return FSFile|null Returns null on failure
494 */
495 abstract public function getLocalReference( array $params );
496
497 /**
498 * Get a local copy on disk of the file at a storage path in the backend.
499 * The temporary copy will have the same file extension as the source.
500 * Temporary files may be purged when the file object falls out of scope.
501 *
502 * $params include:
503 * src : source storage path
504 * latest : use the latest available data
505 *
506 * @param $params Array
507 * @return TempFSFile|null Returns null on failure
508 */
509 abstract public function getLocalCopy( array $params );
510
511 /**
512 * Get an iterator to list out all stored files under a storage directory.
513 * If the directory is of the form "mwstore://container", then all items in
514 * the container should be listed. If of the form "mwstore://container/dir",
515 * then all items under that container directory should be listed.
516 * Results should be storage paths relative to the given directory.
517 *
518 * $params include:
519 * dir : storage path directory
520 *
521 * @return Traversable|Array|null Returns null on failure
522 */
523 abstract public function getFileList( array $params );
524
525 /**
526 * Invalidate any in-process file existence and property cache.
527 * If $paths is given, then only the cache for those files will be cleared.
528 *
529 * @param $paths Array Storage paths (optional)
530 * @return void
531 */
532 public function clearCache( array $paths = null ) {}
533
534 /**
535 * Lock the files at the given storage paths in the backend.
536 * This will either lock all the files or none (on failure).
537 *
538 * Callers should consider using getScopedFileLocks() instead.
539 *
540 * @param $paths Array Storage paths
541 * @param $type integer LockManager::LOCK_* constant
542 * @return Status
543 */
544 final public function lockFiles( array $paths, $type ) {
545 return $this->lockManager->lock( $paths, $type );
546 }
547
548 /**
549 * Unlock the files at the given storage paths in the backend.
550 *
551 * @param $paths Array Storage paths
552 * @param $type integer LockManager::LOCK_* constant
553 * @return Status
554 */
555 final public function unlockFiles( array $paths, $type ) {
556 return $this->lockManager->unlock( $paths, $type );
557 }
558
559 /**
560 * Lock the files at the given storage paths in the backend.
561 * This will either lock all the files or none (on failure).
562 * On failure, the status object will be updated with errors.
563 *
564 * Once the return value goes out scope, the locks will be released and
565 * the status updated. Unlock fatals will not change the status "OK" value.
566 *
567 * @param $paths Array Storage paths
568 * @param $type integer LockManager::LOCK_* constant
569 * @param $status Status Status to update on lock/unlock
570 * @return ScopedLock|null Returns null on failure
571 */
572 final public function getScopedFileLocks( array $paths, $type, Status $status ) {
573 return ScopedLock::factory( $this->lockManager, $paths, $type, $status );
574 }
575 }
576
577 /**
578 * Base class for all single-write backends.
579 * This class defines the methods as abstract that subclasses must implement.
580 * Callers outside of FileBackend and its helper classes, such as FileOp,
581 * should only call functions that are present in FileBackendBase.
582 *
583 * The FileBackendBase operations are implemented using primitive functions
584 * such as storeInternal(), copyInternal(), deleteInternal() and the like.
585 * This class is also responsible for path resolution and sanitization.
586 *
587 * @ingroup FileBackend
588 * @since 1.19
589 */
590 abstract class FileBackend extends FileBackendBase {
591 /** @var Array */
592 protected $cache = array(); // (storage path => key => value)
593 protected $maxCacheSize = 75; // integer; max paths with entries
594 /** @var Array */
595 protected $shardViaHashLevels = array(); // (container name => integer)
596
597 protected $maxFileSize = 1000000000; // integer bytes (1GB)
598
599 /**
600 * Get the maximum allowable file size given backend
601 * medium restrictions and basic performance constraints.
602 * Do not call this function from places outside FileBackend and FileOp.
603 *
604 * @return integer Bytes
605 */
606 final public function maxFileSizeInternal() {
607 return $this->maxFileSize;
608 }
609
610 /**
611 * Check if a file can be created at a given storage path.
612 * FS backends should check if the parent directory exists and the file is writable.
613 * Backends using key/value stores should check if the container exists.
614 *
615 * @param $storagePath string
616 * @return bool
617 */
618 abstract public function isPathUsableInternal( $storagePath );
619
620 /**
621 * Create a file in the backend with the given contents.
622 * Do not call this function from places outside FileBackend and FileOp.
623 *
624 * $params include:
625 * content : the raw file contents
626 * dst : destination storage path
627 * overwrite : overwrite any file that exists at the destination
628 *
629 * @param $params Array
630 * @return Status
631 */
632 final public function createInternal( array $params ) {
633 wfProfileIn( __METHOD__ );
634 if ( strlen( $params['content'] ) > $this->maxFileSizeInternal() ) {
635 $status = Status::newFatal( 'backend-fail-create', $params['dst'] );
636 } else {
637 $status = $this->doCreateInternal( $params );
638 $this->clearCache( array( $params['dst'] ) );
639 }
640 wfProfileOut( __METHOD__ );
641 return $status;
642 }
643
644 /**
645 * @see FileBackend::createInternal()
646 */
647 abstract protected function doCreateInternal( array $params );
648
649 /**
650 * Store a file into the backend from a file on disk.
651 * Do not call this function from places outside FileBackend and FileOp.
652 *
653 * $params include:
654 * src : source path on disk
655 * dst : destination storage path
656 * overwrite : overwrite any file that exists at the destination
657 *
658 * @param $params Array
659 * @return Status
660 */
661 final public function storeInternal( array $params ) {
662 wfProfileIn( __METHOD__ );
663 if ( filesize( $params['src'] ) > $this->maxFileSizeInternal() ) {
664 $status = Status::newFatal( 'backend-fail-store', $params['dst'] );
665 } else {
666 $status = $this->doStoreInternal( $params );
667 $this->clearCache( array( $params['dst'] ) );
668 }
669 wfProfileOut( __METHOD__ );
670 return $status;
671 }
672
673 /**
674 * @see FileBackend::storeInternal()
675 */
676 abstract protected function doStoreInternal( array $params );
677
678 /**
679 * Copy a file from one storage path to another in the backend.
680 * Do not call this function from places outside FileBackend and FileOp.
681 *
682 * $params include:
683 * src : source storage path
684 * dst : destination storage path
685 * overwrite : overwrite any file that exists at the destination
686 *
687 * @param $params Array
688 * @return Status
689 */
690 final public function copyInternal( array $params ) {
691 wfProfileIn( __METHOD__ );
692 $status = $this->doCopyInternal( $params );
693 $this->clearCache( array( $params['dst'] ) );
694 wfProfileOut( __METHOD__ );
695 return $status;
696 }
697
698 /**
699 * @see FileBackend::copyInternal()
700 */
701 abstract protected function doCopyInternal( array $params );
702
703 /**
704 * Delete a file at the storage path.
705 * Do not call this function from places outside FileBackend and FileOp.
706 *
707 * $params include:
708 * src : source storage path
709 * ignoreMissingSource : do nothing if the source file does not exist
710 *
711 * @param $params Array
712 * @return Status
713 */
714 final public function deleteInternal( array $params ) {
715 wfProfileIn( __METHOD__ );
716 $status = $this->doDeleteInternal( $params );
717 $this->clearCache( array( $params['src'] ) );
718 wfProfileOut( __METHOD__ );
719 return $status;
720 }
721
722 /**
723 * @see FileBackend::deleteInternal()
724 */
725 abstract protected function doDeleteInternal( array $params );
726
727 /**
728 * Move a file from one storage path to another in the backend.
729 * Do not call this function from places outside FileBackend and FileOp.
730 *
731 * $params include:
732 * src : source storage path
733 * dst : destination storage path
734 * overwrite : overwrite any file that exists at the destination
735 *
736 * @param $params Array
737 * @return Status
738 */
739 final public function moveInternal( array $params ) {
740 wfProfileIn( __METHOD__ );
741 $status = $this->doMoveInternal( $params );
742 $this->clearCache( array( $params['src'], $params['dst'] ) );
743 wfProfileOut( __METHOD__ );
744 return $status;
745 }
746
747 /**
748 * @see FileBackend::moveInternal()
749 */
750 protected function doMoveInternal( array $params ) {
751 // Copy source to dest
752 $status = $this->copyInternal( $params );
753 if ( $status->isOK() ) {
754 // Delete source (only fails due to races or medium going down)
755 $status->merge( $this->deleteInternal( array( 'src' => $params['src'] ) ) );
756 $status->setResult( true, $status->value ); // ignore delete() errors
757 }
758 return $status;
759 }
760
761 /**
762 * @see FileBackendBase::concatenate()
763 */
764 final public function concatenate( array $params ) {
765 wfProfileIn( __METHOD__ );
766 $status = Status::newGood();
767
768 // Try to lock the source files for the scope of this function
769 $scopeLockS = $this->getScopedFileLocks( $params['srcs'], LockManager::LOCK_UW, $status );
770 if ( $status->isOK() ) {
771 // Actually do the concatenation
772 $status->merge( $this->doConcatenate( $params ) );
773 }
774
775 wfProfileOut( __METHOD__ );
776 return $status;
777 }
778
779 /**
780 * @see FileBackend::concatenate()
781 */
782 protected function doConcatenate( array $params ) {
783 $status = Status::newGood();
784 $tmpPath = $params['dst']; // convenience
785
786 // Check that the specified temp file is valid...
787 wfSuppressWarnings();
788 $ok = ( is_file( $tmpPath ) && !filesize( $tmpPath ) );
789 wfRestoreWarnings();
790 if ( !$ok ) { // not present or not empty
791 $status->fatal( 'backend-fail-opentemp', $tmpPath );
792 return $status;
793 }
794
795 // Build up the temp file using the source chunks (in order)...
796 $tmpHandle = fopen( $tmpPath, 'ab' );
797 if ( $tmpHandle === false ) {
798 $status->fatal( 'backend-fail-opentemp', $tmpPath );
799 return $status;
800 }
801 foreach ( $params['srcs'] as $virtualSource ) {
802 // Get a local FS version of the chunk
803 $tmpFile = $this->getLocalReference( array( 'src' => $virtualSource ) );
804 if ( !$tmpFile ) {
805 $status->fatal( 'backend-fail-read', $virtualSource );
806 return $status;
807 }
808 // Get a handle to the local FS version
809 $sourceHandle = fopen( $tmpFile->getPath(), 'r' );
810 if ( $sourceHandle === false ) {
811 fclose( $tmpHandle );
812 $status->fatal( 'backend-fail-read', $virtualSource );
813 return $status;
814 }
815 // Append chunk to file (pass chunk size to avoid magic quotes)
816 if ( !stream_copy_to_stream( $sourceHandle, $tmpHandle ) ) {
817 fclose( $sourceHandle );
818 fclose( $tmpHandle );
819 $status->fatal( 'backend-fail-writetemp', $tmpPath );
820 return $status;
821 }
822 fclose( $sourceHandle );
823 }
824 if ( !fclose( $tmpHandle ) ) {
825 $status->fatal( 'backend-fail-closetemp', $tmpPath );
826 return $status;
827 }
828
829 clearstatcache(); // temp file changed
830
831 return $status;
832 }
833
834 /**
835 * @see FileBackendBase::doPrepare()
836 */
837 final protected function doPrepare( array $params ) {
838 wfProfileIn( __METHOD__ );
839
840 $status = Status::newGood();
841 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
842 if ( $dir === null ) {
843 $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
844 wfProfileOut( __METHOD__ );
845 return $status; // invalid storage path
846 }
847
848 if ( $shard !== null ) { // confined to a single container/shard
849 $status->merge( $this->doPrepareInternal( $fullCont, $dir, $params ) );
850 } else { // directory is on several shards
851 wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
852 list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
853 foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
854 $status->merge( $this->doPrepareInternal( "{$fullCont}{$suffix}", $dir, $params ) );
855 }
856 }
857
858 wfProfileOut( __METHOD__ );
859 return $status;
860 }
861
862 /**
863 * @see FileBackend::doPrepare()
864 */
865 protected function doPrepareInternal( $container, $dir, array $params ) {
866 return Status::newGood();
867 }
868
869 /**
870 * @see FileBackendBase::doSecure()
871 */
872 final protected function doSecure( array $params ) {
873 wfProfileIn( __METHOD__ );
874 $status = Status::newGood();
875
876 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
877 if ( $dir === null ) {
878 $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
879 wfProfileOut( __METHOD__ );
880 return $status; // invalid storage path
881 }
882
883 if ( $shard !== null ) { // confined to a single container/shard
884 $status->merge( $this->doSecureInternal( $fullCont, $dir, $params ) );
885 } else { // directory is on several shards
886 wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
887 list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
888 foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
889 $status->merge( $this->doSecureInternal( "{$fullCont}{$suffix}", $dir, $params ) );
890 }
891 }
892
893 wfProfileOut( __METHOD__ );
894 return $status;
895 }
896
897 /**
898 * @see FileBackend::doSecure()
899 */
900 protected function doSecureInternal( $container, $dir, array $params ) {
901 return Status::newGood();
902 }
903
904 /**
905 * @see FileBackendBase::doClean()
906 */
907 final protected function doClean( array $params ) {
908 wfProfileIn( __METHOD__ );
909 $status = Status::newGood();
910
911 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
912 if ( $dir === null ) {
913 $status->fatal( 'backend-fail-invalidpath', $params['dir'] );
914 wfProfileOut( __METHOD__ );
915 return $status; // invalid storage path
916 }
917
918 // Attempt to lock this directory...
919 $filesLockEx = array( $params['dir'] );
920 $scopedLockE = $this->getScopedFileLocks( $filesLockEx, LockManager::LOCK_EX, $status );
921 if ( !$status->isOK() ) {
922 wfProfileOut( __METHOD__ );
923 return $status; // abort
924 }
925
926 if ( $shard !== null ) { // confined to a single container/shard
927 $status->merge( $this->doCleanInternal( $fullCont, $dir, $params ) );
928 } else { // directory is on several shards
929 wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
930 list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
931 foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) {
932 $status->merge( $this->doCleanInternal( "{$fullCont}{$suffix}", $dir, $params ) );
933 }
934 }
935
936 wfProfileOut( __METHOD__ );
937 return $status;
938 }
939
940 /**
941 * @see FileBackend::doClean()
942 */
943 protected function doCleanInternal( $container, $dir, array $params ) {
944 return Status::newGood();
945 }
946
947 /**
948 * @see FileBackendBase::fileExists()
949 */
950 final public function fileExists( array $params ) {
951 wfProfileIn( __METHOD__ );
952 $stat = $this->getFileStat( $params );
953 wfProfileOut( __METHOD__ );
954 return ( $stat === null ) ? null : (bool)$stat; // null => failure
955 }
956
957 /**
958 * @see FileBackendBase::getFileTimestamp()
959 */
960 final public function getFileTimestamp( array $params ) {
961 wfProfileIn( __METHOD__ );
962 $stat = $this->getFileStat( $params );
963 wfProfileOut( __METHOD__ );
964 return $stat ? $stat['mtime'] : false;
965 }
966
967 /**
968 * @see FileBackendBase::getFileSize()
969 */
970 final public function getFileSize( array $params ) {
971 wfProfileIn( __METHOD__ );
972 $stat = $this->getFileStat( $params );
973 wfProfileOut( __METHOD__ );
974 return $stat ? $stat['size'] : false;
975 }
976
977 /**
978 * @see FileBackendBase::getFileStat()
979 */
980 final public function getFileStat( array $params ) {
981 wfProfileIn( __METHOD__ );
982 $path = self::normalizeStoragePath( $params['src'] );
983 if ( $path === null ) {
984 return false; // invalid storage path
985 }
986 $latest = !empty( $params['latest'] );
987 if ( isset( $this->cache[$path]['stat'] ) ) {
988 // If we want the latest data, check that this cached
989 // value was in fact fetched with the latest available data.
990 if ( !$latest || $this->cache[$path]['stat']['latest'] ) {
991 wfProfileOut( __METHOD__ );
992 return $this->cache[$path]['stat'];
993 }
994 }
995 $stat = $this->doGetFileStat( $params );
996 if ( is_array( $stat ) ) { // don't cache negatives
997 $this->trimCache(); // limit memory
998 $this->cache[$path]['stat'] = $stat;
999 $this->cache[$path]['stat']['latest'] = $latest;
1000 }
1001 wfProfileOut( __METHOD__ );
1002 return $stat;
1003 }
1004
1005 /**
1006 * @see FileBackend::getFileStat()
1007 */
1008 abstract protected function doGetFileStat( array $params );
1009
1010 /**
1011 * @see FileBackendBase::getFileContents()
1012 */
1013 public function getFileContents( array $params ) {
1014 wfProfileIn( __METHOD__ );
1015 $tmpFile = $this->getLocalReference( $params );
1016 if ( !$tmpFile ) {
1017 wfProfileOut( __METHOD__ );
1018 return false;
1019 }
1020 wfSuppressWarnings();
1021 $data = file_get_contents( $tmpFile->getPath() );
1022 wfRestoreWarnings();
1023 wfProfileOut( __METHOD__ );
1024 return $data;
1025 }
1026
1027 /**
1028 * @see FileBackendBase::getFileSha1Base36()
1029 */
1030 final public function getFileSha1Base36( array $params ) {
1031 wfProfileIn( __METHOD__ );
1032 $path = $params['src'];
1033 if ( isset( $this->cache[$path]['sha1'] ) ) {
1034 wfProfileOut( __METHOD__ );
1035 return $this->cache[$path]['sha1'];
1036 }
1037 $hash = $this->doGetFileSha1Base36( $params );
1038 if ( $hash ) { // don't cache negatives
1039 $this->trimCache(); // limit memory
1040 $this->cache[$path]['sha1'] = $hash;
1041 }
1042 wfProfileOut( __METHOD__ );
1043 return $hash;
1044 }
1045
1046 /**
1047 * @see FileBackend::getFileSha1Base36()
1048 */
1049 protected function doGetFileSha1Base36( array $params ) {
1050 $fsFile = $this->getLocalReference( $params );
1051 if ( !$fsFile ) {
1052 return false;
1053 } else {
1054 return $fsFile->getSha1Base36();
1055 }
1056 }
1057
1058 /**
1059 * @see FileBackendBase::getFileProps()
1060 */
1061 final public function getFileProps( array $params ) {
1062 wfProfileIn( __METHOD__ );
1063 $fsFile = $this->getLocalReference( $params );
1064 $props = $fsFile ? $fsFile->getProps() : FSFile::placeholderProps();
1065 wfProfileOut( __METHOD__ );
1066 return $props;
1067 }
1068
1069 /**
1070 * @see FileBackendBase::getLocalReference()
1071 */
1072 public function getLocalReference( array $params ) {
1073 wfProfileIn( __METHOD__ );
1074 $path = $params['src'];
1075 if ( isset( $this->cache[$path]['localRef'] ) ) {
1076 wfProfileOut( __METHOD__ );
1077 return $this->cache[$path]['localRef'];
1078 }
1079 $tmpFile = $this->getLocalCopy( $params );
1080 if ( $tmpFile ) { // don't cache negatives
1081 $this->trimCache(); // limit memory
1082 $this->cache[$path]['localRef'] = $tmpFile;
1083 }
1084 wfProfileOut( __METHOD__ );
1085 return $tmpFile;
1086 }
1087
1088 /**
1089 * @see FileBackendBase::streamFile()
1090 */
1091 final public function streamFile( array $params ) {
1092 wfProfileIn( __METHOD__ );
1093 $status = Status::newGood();
1094
1095 $info = $this->getFileStat( $params );
1096 if ( !$info ) { // let StreamFile handle the 404
1097 $status->fatal( 'backend-fail-notexists', $params['src'] );
1098 }
1099
1100 // Set output buffer and HTTP headers for stream
1101 $extraHeaders = $params['headers'] ? $params['headers'] : array();
1102 $res = StreamFile::prepareForStream( $params['src'], $info, $extraHeaders );
1103 if ( $res == StreamFile::NOT_MODIFIED ) {
1104 // do nothing; client cache is up to date
1105 } elseif ( $res == StreamFile::READY_STREAM ) {
1106 $status = $this->doStreamFile( $params );
1107 } else {
1108 $status->fatal( 'backend-fail-stream', $params['src'] );
1109 }
1110
1111 wfProfileOut( __METHOD__ );
1112 return $status;
1113 }
1114
1115 /**
1116 * @see FileBackend::streamFile()
1117 */
1118 protected function doStreamFile( array $params ) {
1119 $status = Status::newGood();
1120
1121 $fsFile = $this->getLocalReference( $params );
1122 if ( !$fsFile ) {
1123 $status->fatal( 'backend-fail-stream', $params['src'] );
1124 } elseif ( !readfile( $fsFile->getPath() ) ) {
1125 $status->fatal( 'backend-fail-stream', $params['src'] );
1126 }
1127
1128 return $status;
1129 }
1130
1131 /**
1132 * @see FileBackendBase::getFileList()
1133 */
1134 final public function getFileList( array $params ) {
1135 list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
1136 if ( $dir === null ) { // invalid storage path
1137 return null;
1138 }
1139 if ( $shard !== null ) {
1140 // File listing is confined to a single container/shard
1141 return $this->getFileListInternal( $fullCont, $dir, $params );
1142 } else {
1143 wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
1144 // File listing spans multiple containers/shards
1145 list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] );
1146 return new FileBackendShardListIterator( $this,
1147 $fullCont, $this->getContainerSuffixes( $shortCont ), $params );
1148 }
1149 }
1150
1151 /**
1152 * Do not call this function from places outside FileBackend and ContainerFileListIterator
1153 *
1154 * @param $container string Resolved container name
1155 * @param $dir string Resolved path relative to container
1156 * @param $params Array
1157 * @see FileBackend::getFileList()
1158 */
1159 abstract public function getFileListInternal( $container, $dir, array $params );
1160
1161 /**
1162 * Get the list of supported operations and their corresponding FileOp classes.
1163 *
1164 * @return Array
1165 */
1166 protected function supportedOperations() {
1167 return array(
1168 'store' => 'StoreFileOp',
1169 'copy' => 'CopyFileOp',
1170 'move' => 'MoveFileOp',
1171 'delete' => 'DeleteFileOp',
1172 'create' => 'CreateFileOp',
1173 'null' => 'NullFileOp'
1174 );
1175 }
1176
1177 /**
1178 * Return a list of FileOp objects from a list of operations.
1179 * Do not call this function from places outside FileBackend.
1180 *
1181 * The result must have the same number of items as the input.
1182 * An exception is thrown if an unsupported operation is requested.
1183 *
1184 * @param $ops Array Same format as doOperations()
1185 * @return Array List of FileOp objects
1186 * @throws MWException
1187 */
1188 final public function getOperations( array $ops ) {
1189 $supportedOps = $this->supportedOperations();
1190
1191 $performOps = array(); // array of FileOp objects
1192 // Build up ordered array of FileOps...
1193 foreach ( $ops as $operation ) {
1194 $opName = $operation['op'];
1195 if ( isset( $supportedOps[$opName] ) ) {
1196 $class = $supportedOps[$opName];
1197 // Get params for this operation
1198 $params = $operation;
1199 // Append the FileOp class
1200 $performOps[] = new $class( $this, $params );
1201 } else {
1202 throw new MWException( "Operation `$opName` is not supported." );
1203 }
1204 }
1205
1206 return $performOps;
1207 }
1208
1209 /**
1210 * @see FileBackendBase::doOperationsInternal()
1211 */
1212 protected function doOperationsInternal( array $ops, array $opts ) {
1213 wfProfileIn( __METHOD__ );
1214 $status = Status::newGood();
1215
1216 // Build up a list of FileOps...
1217 $performOps = $this->getOperations( $ops );
1218
1219 // Acquire any locks as needed...
1220 if ( empty( $opts['nonLocking'] ) ) {
1221 // Build up a list of files to lock...
1222 $filesLockEx = $filesLockSh = array();
1223 foreach ( $performOps as $fileOp ) {
1224 $filesLockSh = array_merge( $filesLockSh, $fileOp->storagePathsRead() );
1225 $filesLockEx = array_merge( $filesLockEx, $fileOp->storagePathsChanged() );
1226 }
1227 // Optimization: if doing an EX lock anyway, don't also set an SH one
1228 $filesLockSh = array_diff( $filesLockSh, $filesLockEx );
1229 // Get a shared lock on the parent directory of each path changed
1230 $filesLockSh = array_merge( $filesLockSh, array_map( 'dirname', $filesLockEx ) );
1231 // Try to lock those files for the scope of this function...
1232 $scopeLockS = $this->getScopedFileLocks( $filesLockSh, LockManager::LOCK_UW, $status );
1233 $scopeLockE = $this->getScopedFileLocks( $filesLockEx, LockManager::LOCK_EX, $status );
1234 if ( !$status->isOK() ) {
1235 wfProfileOut( __METHOD__ );
1236 return $status; // abort
1237 }
1238 }
1239
1240 // Clear any cache entries (after locks acquired)
1241 $this->clearCache();
1242
1243 // Actually attempt the operation batch...
1244 $subStatus = FileOp::attemptBatch( $performOps, $opts );
1245
1246 // Merge errors into status fields
1247 $status->merge( $subStatus );
1248 $status->success = $subStatus->success; // not done in merge()
1249
1250 wfProfileOut( __METHOD__ );
1251 return $status;
1252 }
1253
1254 /**
1255 * @see FileBackendBase::clearCache()
1256 */
1257 final public function clearCache( array $paths = null ) {
1258 if ( is_array( $paths ) ) {
1259 $paths = array_map( 'FileBackend::normalizeStoragePath', $paths );
1260 $paths = array_filter( $paths, 'strlen' ); // remove nulls
1261 }
1262 if ( $paths === null ) {
1263 $this->cache = array();
1264 } else {
1265 foreach ( $paths as $path ) {
1266 unset( $this->cache[$path] );
1267 }
1268 }
1269 $this->doClearCache( $paths );
1270 }
1271
1272 /**
1273 * Clears any additional stat caches for storage paths
1274 *
1275 * @see FileBackendBase::clearCache()
1276 *
1277 * @param $paths Array Storage paths (optional)
1278 * @return void
1279 */
1280 protected function doClearCache( array $paths = null ) {}
1281
1282 /**
1283 * Prune the cache if it is too big to add an item
1284 *
1285 * @return void
1286 */
1287 protected function trimCache() {
1288 if ( count( $this->cache ) >= $this->maxCacheSize ) {
1289 reset( $this->cache );
1290 $key = key( $this->cache );
1291 unset( $this->cache[$key] );
1292 }
1293 }
1294
1295 /**
1296 * Get the parent storage directory of a storage path.
1297 * This returns a path like "mwstore://backend/container",
1298 * "mwstore://backend/container/...", or null if there is no parent.
1299 *
1300 * @param $storagePath string
1301 * @return string|null
1302 */
1303 final public static function parentStoragePath( $storagePath ) {
1304 $storagePath = dirname( $storagePath );
1305 list( $b, $cont, $rel ) = self::splitStoragePath( $storagePath );
1306 return ( $rel === null ) ? null : $storagePath;
1307 }
1308
1309 /**
1310 * Check if a given path is a mwstore:// path.
1311 * This does not do any actual validation or existence checks.
1312 *
1313 * @param $path string
1314 * @return bool
1315 */
1316 final public static function isStoragePath( $path ) {
1317 return ( strpos( $path, 'mwstore://' ) === 0 );
1318 }
1319
1320 /**
1321 * Normalize a storage path by cleaning up directory separators.
1322 * Returns null if the path is not of the format of a valid storage path.
1323 *
1324 * @param $storagePath string
1325 * @return string|null
1326 */
1327 final public static function normalizeStoragePath( $storagePath ) {
1328 list( $backend, $container, $relPath ) = self::splitStoragePath( $storagePath );
1329 if ( $relPath !== null ) { // must be for this backend
1330 $relPath = self::normalizeContainerPath( $relPath );
1331 if ( $relPath !== null ) {
1332 return ( $relPath != '' )
1333 ? "mwstore://{$backend}/{$container}/{$relPath}"
1334 : "mwstore://{$backend}/{$container}";
1335 }
1336 }
1337 return null;
1338 }
1339
1340 /**
1341 * Split a storage path into a backend name, a container name,
1342 * and a relative file path. The relative path may be the empty string.
1343 *
1344 * @param $storagePath string
1345 * @return Array (backend, container, rel object) or (null, null, null)
1346 */
1347 final public static function splitStoragePath( $storagePath ) {
1348 if ( self::isStoragePath( $storagePath ) ) {
1349 // Note: strlen( 'mwstore://' ) = 10
1350 $parts = explode( '/', substr( $storagePath, 10 ), 3 );
1351 if ( count( $parts ) == 3 ) {
1352 return $parts; // e.g. "backend/container/path"
1353 } elseif ( count( $parts ) == 2 ) {
1354 return array( $parts[0], $parts[1], '' ); // e.g. "backend/container"
1355 }
1356 }
1357 return array( null, null, null );
1358 }
1359
1360 /**
1361 * Check if a container name is valid.
1362 * This checks for for length and illegal characters.
1363 *
1364 * @param $container string
1365 * @return bool
1366 */
1367 final protected static function isValidContainerName( $container ) {
1368 // This accounts for Swift and S3 restrictions while leaving room
1369 // for things like '.xxx' (hex shard chars) or '.seg' (segments).
1370 // Note that matching strings URL encode to the same string;
1371 // in Swift, the length restriction is *after* URL encoding.
1372 return preg_match( '/^[a-z0-9][a-z0-9-_]{0,199}$/i', $container );
1373 }
1374
1375 /**
1376 * Validate and normalize a relative storage path.
1377 * Null is returned if the path involves directory traversal.
1378 * Traversal is insecure for FS backends and broken for others.
1379 *
1380 * @param $path string Storage path relative to a container
1381 * @return string|null
1382 */
1383 final protected static function normalizeContainerPath( $path ) {
1384 // Normalize directory separators
1385 $path = strtr( $path, '\\', '/' );
1386 // Collapse consecutive directory separators
1387 $path = preg_replace( '![/]{2,}!', '/', $path );
1388 // Use the same traversal protection as Title::secureAndSplit()
1389 if ( strpos( $path, '.' ) !== false ) {
1390 if (
1391 $path === '.' ||
1392 $path === '..' ||
1393 strpos( $path, './' ) === 0 ||
1394 strpos( $path, '../' ) === 0 ||
1395 strpos( $path, '/./' ) !== false ||
1396 strpos( $path, '/../' ) !== false
1397 ) {
1398 return null;
1399 }
1400 }
1401 return $path;
1402 }
1403
1404 /**
1405 * Splits a storage path into an internal container name,
1406 * an internal relative file name, and a container shard suffix.
1407 * Any shard suffix is already appended to the internal container name.
1408 * This also checks that the storage path is valid and within this backend.
1409 *
1410 * If the container is sharded but a suffix could not be determined,
1411 * this means that the path can only refer to a directory and can only
1412 * be scanned by looking in all the container shards.
1413 *
1414 * @param $storagePath string
1415 * @return Array (container, path, container suffix) or (null, null, null) if invalid
1416 */
1417 final protected function resolveStoragePath( $storagePath ) {
1418 list( $backend, $container, $relPath ) = self::splitStoragePath( $storagePath );
1419 if ( $backend === $this->name ) { // must be for this backend
1420 $relPath = self::normalizeContainerPath( $relPath );
1421 if ( $relPath !== null ) {
1422 // Get shard for the normalized path if this container is sharded
1423 $cShard = $this->getContainerShard( $container, $relPath );
1424 // Validate and sanitize the relative path (backend-specific)
1425 $relPath = $this->resolveContainerPath( $container, $relPath );
1426 if ( $relPath !== null ) {
1427 // Prepend any wiki ID prefix to the container name
1428 $container = $this->fullContainerName( $container );
1429 if ( self::isValidContainerName( $container ) ) {
1430 // Validate and sanitize the container name (backend-specific)
1431 $container = $this->resolveContainerName( "{$container}{$cShard}" );
1432 if ( $container !== null ) {
1433 return array( $container, $relPath, $cShard );
1434 }
1435 }
1436 }
1437 }
1438 }
1439 return array( null, null, null );
1440 }
1441
1442 /**
1443 * Like resolveStoragePath() except null values are returned if
1444 * the container is sharded and the shard could not be determined.
1445 *
1446 * @see FileBackend::resolveStoragePath()
1447 *
1448 * @param $storagePath string
1449 * @return Array (container, path) or (null, null) if invalid
1450 */
1451 final protected function resolveStoragePathReal( $storagePath ) {
1452 list( $container, $relPath, $cShard ) = $this->resolveStoragePath( $storagePath );
1453 if ( $cShard !== null ) {
1454 return array( $container, $relPath );
1455 }
1456 return array( null, null );
1457 }
1458
1459 /**
1460 * Get the container name shard suffix for a given path.
1461 * Any empty suffix means the container is not sharded.
1462 *
1463 * @param $container string Container name
1464 * @param $relStoragePath string Storage path relative to the container
1465 * @return string|null Returns null if shard could not be determined
1466 */
1467 final protected function getContainerShard( $container, $relPath ) {
1468 $hashLevels = $this->getContainerHashLevels( $container );
1469 if ( $hashLevels === 1 ) { // 16 shards per container
1470 $hashDirRegex = '(?P<shard>[0-9a-f])';
1471 } elseif ( $hashLevels === 2 ) { // 256 shards per container
1472 $hashDirRegex = '[0-9a-f]/(?P<shard>[0-9a-f]{2})';
1473 } else {
1474 return ''; // no sharding
1475 }
1476 // Allow certain directories to be above the hash dirs so as
1477 // to work with FileRepo (e.g. "archive/a/ab" or "temp/a/ab").
1478 // They must be 2+ chars to avoid any hash directory ambiguity.
1479 $m = array();
1480 if ( preg_match( "!^(?:[^/]{2,}/)*$hashDirRegex(?:/|$)!", $relPath, $m ) ) {
1481 return '.' . $m['shard'];
1482 }
1483 return null; // failed to match
1484 }
1485
1486 /**
1487 * Get the number of hash levels for a container.
1488 * If greater than 0, then all file storage paths within
1489 * the container are required to be hashed accordingly.
1490 *
1491 * @param $container string
1492 * @return integer
1493 */
1494 final protected function getContainerHashLevels( $container ) {
1495 if ( isset( $this->shardViaHashLevels[$container] ) ) {
1496 $hashLevels = (int)$this->shardViaHashLevels[$container];
1497 if ( $hashLevels >= 0 && $hashLevels <= 2 ) {
1498 return $hashLevels;
1499 }
1500 }
1501 return 0; // no sharding
1502 }
1503
1504 /**
1505 * Get a list of full container shard suffixes for a container
1506 *
1507 * @param $container string
1508 * @return Array
1509 */
1510 final protected function getContainerSuffixes( $container ) {
1511 $shards = array();
1512 $digits = $this->getContainerHashLevels( $container );
1513 if ( $digits > 0 ) {
1514 $numShards = 1 << ( $digits * 4 );
1515 for ( $index = 0; $index < $numShards; $index++ ) {
1516 $shards[] = '.' . str_pad( dechex( $index ), $digits, '0', STR_PAD_LEFT );
1517 }
1518 }
1519 return $shards;
1520 }
1521
1522 /**
1523 * Get the full container name, including the wiki ID prefix
1524 *
1525 * @param $container string
1526 * @return string
1527 */
1528 final protected function fullContainerName( $container ) {
1529 if ( $this->wikiId != '' ) {
1530 return "{$this->wikiId}-$container";
1531 } else {
1532 return $container;
1533 }
1534 }
1535
1536 /**
1537 * Resolve a container name, checking if it's allowed by the backend.
1538 * This is intended for internal use, such as encoding illegal chars.
1539 * Subclasses can override this to be more restrictive.
1540 *
1541 * @param $container string
1542 * @return string|null
1543 */
1544 protected function resolveContainerName( $container ) {
1545 return $container;
1546 }
1547
1548 /**
1549 * Resolve a relative storage path, checking if it's allowed by the backend.
1550 * This is intended for internal use, such as encoding illegal chars or perhaps
1551 * getting absolute paths (e.g. FS based backends). Note that the relative path
1552 * may be the empty string (e.g. the path is simply to the container).
1553 *
1554 * @param $container string Container name
1555 * @param $relStoragePath string Storage path relative to the container
1556 * @return string|null Path or null if not valid
1557 */
1558 protected function resolveContainerPath( $container, $relStoragePath ) {
1559 return $relStoragePath;
1560 }
1561
1562 /**
1563 * Get the final extension from a storage or FS path
1564 *
1565 * @param $path string
1566 * @return string
1567 */
1568 final public static function extensionFromPath( $path ) {
1569 $i = strrpos( $path, '.' );
1570 return strtolower( $i ? substr( $path, $i + 1 ) : '' );
1571 }
1572 }
1573
1574 /**
1575 * FileBackend helper function to handle file listings that span container shards.
1576 * Do not use this class from places outside of FileBackend.
1577 *
1578 * @ingroup FileBackend
1579 */
1580 class FileBackendShardListIterator implements Iterator {
1581 /* @var FileBackend */
1582 protected $backend;
1583 /* @var Array */
1584 protected $params;
1585 /* @var Array */
1586 protected $shardSuffixes;
1587 protected $container; // string
1588 protected $directory; // string
1589
1590 /* @var Traversable */
1591 protected $iter;
1592 protected $curShard = 0; // integer
1593 protected $pos = 0; // integer
1594
1595 /**
1596 * @param $backend FileBackend
1597 * @param $container string Full storage container name
1598 * @param $dir string Storage directory relative to container
1599 * @param $suffixes Array List of container shard suffixes
1600 * @param $params Array
1601 */
1602 public function __construct(
1603 FileBackend $backend, $container, $dir, array $suffixes, array $params
1604 ) {
1605 $this->backend = $backend;
1606 $this->container = $container;
1607 $this->directory = $dir;
1608 $this->shardSuffixes = $suffixes;
1609 $this->params = $params;
1610 }
1611
1612 public function current() {
1613 if ( is_array( $this->iter ) ) {
1614 return current( $this->iter );
1615 } else {
1616 return $this->iter->current();
1617 }
1618 }
1619
1620 public function key() {
1621 return $this->pos;
1622 }
1623
1624 public function next() {
1625 ++$this->pos;
1626 if ( is_array( $this->iter ) ) {
1627 next( $this->iter );
1628 } else {
1629 $this->iter->next();
1630 }
1631 // Find the next non-empty shard if no elements are left
1632 $this->nextShardIteratorIfNotValid();
1633 }
1634
1635 /**
1636 * If the iterator for this container shard is out of items,
1637 * then move on to the next container that has items.
1638 * If there are none, then it advances to the last container.
1639 */
1640 protected function nextShardIteratorIfNotValid() {
1641 while ( !$this->valid() ) {
1642 if ( ++$this->curShard >= count( $this->shardSuffixes ) ) {
1643 break; // no more container shards
1644 }
1645 $this->setIteratorFromCurrentShard();
1646 }
1647 }
1648
1649 protected function setIteratorFromCurrentShard() {
1650 $suffix = $this->shardSuffixes[$this->curShard];
1651 $this->iter = $this->backend->getFileListInternal(
1652 "{$this->container}{$suffix}", $this->directory, $this->params );
1653 }
1654
1655 public function rewind() {
1656 $this->pos = 0;
1657 $this->curShard = 0;
1658 $this->setIteratorFromCurrentShard();
1659 // Find the next non-empty shard if this one has no elements
1660 $this->nextShardIteratorIfNotValid();
1661 }
1662
1663 public function valid() {
1664 if ( $this->iter == null ) {
1665 return false; // some failure?
1666 } elseif ( is_array( $this->iter ) ) {
1667 return ( current( $this->iter ) !== false ); // no paths can have this value
1668 } else {
1669 return $this->iter->valid();
1670 }
1671 }
1672 }
Wiklou, le Wiki du Wiklou