3 * Backend for uploading files from chunks.
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License along
16 * with this program; if not, write to the Free Software Foundation, Inc.,
17 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18 * http://www.gnu.org/copyleft/gpl.html
25 * Implements uploading from chunks
28 * @author Michael Dale
30 class UploadFromChunks
extends UploadFromFile
{
33 /** @var UploadStash */
39 protected $mChunkIndex;
41 protected $mVirtualTempPath;
43 /** @noinspection PhpMissingParentConstructorInspection */
46 * Setup local pointers to stash, repo and user (similar to UploadFromStash)
49 * @param UploadStash|bool $stash Default: false
50 * @param FileRepo|bool $repo Default: false
52 public function __construct( User
$user, $stash = false, $repo = false ) {
58 $this->repo
= RepoGroup
::singleton()->getLocalRepo();
62 $this->stash
= $stash;
64 wfDebug( __METHOD__
. " creating new UploadFromChunks instance for " . $user->getId() . "\n" );
65 $this->stash
= new UploadStash( $this->repo
, $this->user
);
72 public function tryStashFile( User
$user, $isPartial = false ) {
75 } catch ( UploadChunkVerificationException
$e ) {
76 return Status
::newFatal( $e->msg
);
79 return parent
::tryStashFile( $user, $isPartial );
84 * @throws UploadChunkVerificationException
85 * @deprecated since 1.28 Use tryStashFile() instead
87 public function stashFile( User
$user = null ) {
88 wfDeprecated( __METHOD__
, '1.28' );
90 return parent
::stashFile( $user );
95 * @throws UploadChunkVerificationException
96 * @deprecated since 1.28
98 public function stashFileGetKey() {
99 wfDeprecated( __METHOD__
, '1.28' );
100 $this->verifyChunk();
101 return parent
::stashFileGetKey();
106 * @throws UploadChunkVerificationException
107 * @deprecated since 1.28
109 public function stashSession() {
110 wfDeprecated( __METHOD__
, '1.28' );
111 $this->verifyChunk();
112 return parent
::stashSession();
116 * Calls the parent doStashFile and updates the uploadsession table to handle "chunks"
118 * @param User|null $user
119 * @return UploadStashFile Stashed file
121 protected function doStashFile( User
$user = null ) {
122 // Stash file is the called on creating a new chunk session:
123 $this->mChunkIndex
= 0;
126 // Create a local stash target
127 $this->mStashFile
= parent
::doStashFile( $user );
128 // Update the initial file offset (based on file size)
129 $this->mOffset
= $this->mStashFile
->getSize();
130 $this->mFileKey
= $this->mStashFile
->getFileKey();
132 // Output a copy of this first to chunk 0 location:
133 $this->outputChunk( $this->mStashFile
->getPath() );
135 // Update db table to reflect initial "chunk" state
136 $this->updateChunkStatus();
138 return $this->mStashFile
;
142 * Continue chunk uploading
144 * @param string $name
146 * @param WebRequestUpload $webRequestUpload
148 public function continueChunks( $name, $key, $webRequestUpload ) {
149 $this->mFileKey
= $key;
150 $this->mUpload
= $webRequestUpload;
151 // Get the chunk status form the db:
152 $this->getChunkStatus();
154 $metadata = $this->stash
->getMetadata( $key );
155 $this->initializePathInfo( $name,
156 $this->getRealPath( $metadata['us_path'] ),
157 $metadata['us_size'],
163 * Append the final chunk and ready file for parent::performUpload()
166 public function concatenateChunks() {
167 $chunkIndex = $this->getChunkIndex();
168 wfDebug( __METHOD__
. " concatenate {$this->mChunkIndex} chunks:" .
169 $this->getOffset() . ' inx:' . $chunkIndex . "\n" );
171 // Concatenate all the chunks to mVirtualTempPath
173 // The first chunk is stored at the mVirtualTempPath path so we start on "chunk 1"
174 for ( $i = 0; $i <= $chunkIndex; $i++
) {
175 $fileList[] = $this->getVirtualChunkLocation( $i );
178 // Get the file extension from the last chunk
179 $ext = FileBackend
::extensionFromPath( $this->mVirtualTempPath
);
180 // Get a 0-byte temp file to perform the concatenation at
181 $tmpFile = TempFSFile
::factory( 'chunkedupload_', $ext, wfTempDir() );
182 $tmpPath = false; // fail in concatenate()
184 // keep alive with $this
185 $tmpPath = $tmpFile->bind( $this )->getPath();
188 // Concatenate the chunks at the temp file
189 $tStart = microtime( true );
190 $status = $this->repo
->concatenate( $fileList, $tmpPath, FileRepo
::DELETE_SOURCE
);
191 $tAmount = microtime( true ) - $tStart;
192 if ( !$status->isOK() ) {
196 wfDebugLog( 'fileconcatenate', "Combined $i chunks in $tAmount seconds." );
198 // File system path of the actual full temp file
199 $this->setTempFile( $tmpPath );
201 $ret = $this->verifyUpload();
202 if ( $ret['status'] !== UploadBase
::OK
) {
203 wfDebugLog( 'fileconcatenate', "Verification failed for chunked upload" );
204 $status->fatal( $this->getVerificationErrorCode( $ret['status'] ) );
209 // Update the mTempPath and mStashFile
210 // (for FileUpload or normal Stash to take over)
211 $tStart = microtime( true );
212 // This is a re-implementation of UploadBase::tryStashFile(), we can't call it because we
213 // override doStashFile() with completely different functionality in this class...
214 $error = $this->runUploadStashFileHook( $this->user
);
216 $status->fatal( ...$error );
220 $this->mStashFile
= parent
::doStashFile( $this->user
);
221 } catch ( UploadStashException
$e ) {
222 $status->fatal( 'uploadstash-exception', get_class( $e ), $e->getMessage() );
226 $tAmount = microtime( true ) - $tStart;
227 $this->mStashFile
->setLocalReference( $tmpFile ); // reuse (e.g. for getImageInfo())
228 wfDebugLog( 'fileconcatenate', "Stashed combined file ($i chunks) in $tAmount seconds." );
234 * Returns the virtual chunk location:
238 function getVirtualChunkLocation( $index ) {
239 return $this->repo
->getVirtualUrl( 'temp' ) .
241 $this->repo
->getHashPath(
242 $this->getChunkFileKey( $index )
244 $this->getChunkFileKey( $index );
248 * Add a chunk to the temporary directory
250 * @param string $chunkPath Path to temporary chunk file
251 * @param int $chunkSize Size of the current chunk
252 * @param int $offset Offset of current chunk ( mutch match database chunk offset )
255 public function addChunk( $chunkPath, $chunkSize, $offset ) {
256 // Get the offset before we add the chunk to the file system
257 $preAppendOffset = $this->getOffset();
259 if ( $preAppendOffset +
$chunkSize > $this->getMaxUploadSize() ) {
260 $status = Status
::newFatal( 'file-too-large' );
262 // Make sure the client is uploading the correct chunk with a matching offset.
263 if ( $preAppendOffset == $offset ) {
264 // Update local chunk index for the current chunk
265 $this->mChunkIndex++
;
267 # For some reason mTempPath is set to first part
268 $oldTemp = $this->mTempPath
;
269 $this->mTempPath
= $chunkPath;
270 $this->verifyChunk();
271 $this->mTempPath
= $oldTemp;
272 } catch ( UploadChunkVerificationException
$e ) {
273 return Status
::newFatal( $e->msg
);
275 $status = $this->outputChunk( $chunkPath );
276 if ( $status->isGood() ) {
277 // Update local offset:
278 $this->mOffset
= $preAppendOffset +
$chunkSize;
279 // Update chunk table status db
280 $this->updateChunkStatus();
283 $status = Status
::newFatal( 'invalid-chunk-offset' );
291 * Update the chunk db table with the current status:
293 private function updateChunkStatus() {
294 wfDebug( __METHOD__
. " update chunk status for {$this->mFileKey} offset:" .
295 $this->getOffset() . ' inx:' . $this->getChunkIndex() . "\n" );
297 $dbw = $this->repo
->getMasterDB();
301 'us_status' => 'chunks',
302 'us_chunk_inx' => $this->getChunkIndex(),
303 'us_size' => $this->getOffset()
305 [ 'us_key' => $this->mFileKey
],
311 * Get the chunk db state and populate update relevant local values
313 private function getChunkStatus() {
314 // get Master db to avoid race conditions.
315 // Otherwise, if chunk upload time < replag there will be spurious errors
316 $dbw = $this->repo
->getMasterDB();
317 $row = $dbw->selectRow(
324 [ 'us_key' => $this->mFileKey
],
329 $this->mChunkIndex
= $row->us_chunk_inx
;
330 $this->mOffset
= $row->us_size
;
331 $this->mVirtualTempPath
= $row->us_path
;
336 * Get the current Chunk index
337 * @return int Index of the current chunk
339 private function getChunkIndex() {
340 if ( $this->mChunkIndex
!== null ) {
341 return $this->mChunkIndex
;
348 * Get the offset at which the next uploaded chunk will be appended to
349 * @return int Current byte offset of the chunk file set
351 public function getOffset() {
352 if ( $this->mOffset
!== null ) {
353 return $this->mOffset
;
360 * Output the chunk to disk
362 * @param string $chunkPath
363 * @throws UploadChunkFileException
366 private function outputChunk( $chunkPath ) {
367 // Key is fileKey + chunk index
368 $fileKey = $this->getChunkFileKey();
370 // Store the chunk per its indexed fileKey:
371 $hashPath = $this->repo
->getHashPath( $fileKey );
372 $storeStatus = $this->repo
->quickImport( $chunkPath,
373 $this->repo
->getZonePath( 'temp' ) . "/{$hashPath}{$fileKey}" );
375 // Check for error in stashing the chunk:
376 if ( !$storeStatus->isOK() ) {
377 $error = $storeStatus->getErrorsArray();
378 $error = reset( $error );
379 if ( !count( $error ) ) {
380 $error = $storeStatus->getWarningsArray();
381 $error = reset( $error );
382 if ( !count( $error ) ) {
383 $error = [ 'unknown', 'no error recorded' ];
386 throw new UploadChunkFileException( "Error storing file in '$chunkPath': " .
387 implode( '; ', $error ) );
393 private function getChunkFileKey( $index = null ) {
394 if ( $index === null ) {
395 $index = $this->getChunkIndex();
398 return $this->mFileKey
. '.' . $index;
402 * Verify that the chunk isn't really an evil html file
404 * @throws UploadChunkVerificationException
406 private function verifyChunk() {
407 // Rest mDesiredDestName here so we verify the name as if it were mFileKey
408 $oldDesiredDestName = $this->mDesiredDestName
;
409 $this->mDesiredDestName
= $this->mFileKey
;
410 $this->mTitle
= false;
411 $res = $this->verifyPartialFile();
412 $this->mDesiredDestName
= $oldDesiredDestName;
413 $this->mTitle
= false;
414 if ( is_array( $res ) ) {
415 throw new UploadChunkVerificationException( $res );