3 * Backend for uploading files from chunks.
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License along
16 * with this program; if not, write to the Free Software Foundation, Inc.,
17 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18 * http://www.gnu.org/copyleft/gpl.html
25 * Implements uploading from chunks
28 * @author Michael Dale
30 class UploadFromChunks
extends UploadFromFile
{
32 protected $mChunkIndex;
34 protected $mVirtualTempPath;
39 * Setup local pointers to stash, repo and user (similar to UploadFromStash)
41 * @param User|null $user Default: null
42 * @param UploadStash|bool $stash Default: false
43 * @param FileRepo|bool $repo Default: false
45 public function __construct( $user = null, $stash = false, $repo = false ) {
46 // user object. sometimes this won't exist, as when running from cron.
52 $this->repo
= RepoGroup
::singleton()->getLocalRepo();
56 $this->stash
= $stash;
59 wfDebug( __METHOD__
. " creating new UploadFromChunks instance for " . $user->getId() . "\n" );
61 wfDebug( __METHOD__
. " creating new UploadFromChunks instance with no user\n" );
63 $this->stash
= new UploadStash( $this->repo
, $this->user
);
70 * Calls the parent stashFile and updates the uploadsession table to handle "chunks"
72 * @param User|null $user
73 * @return UploadStashFile Stashed file
75 public function stashFile( User
$user = null ) {
76 // Stash file is the called on creating a new chunk session:
77 $this->mChunkIndex
= 0;
81 // Create a local stash target
82 $this->mLocalFile
= parent
::stashFile();
83 // Update the initial file offset (based on file size)
84 $this->mOffset
= $this->mLocalFile
->getSize();
85 $this->mFileKey
= $this->mLocalFile
->getFileKey();
87 // Output a copy of this first to chunk 0 location:
88 $this->outputChunk( $this->mLocalFile
->getPath() );
90 // Update db table to reflect initial "chunk" state
91 $this->updateChunkStatus();
93 return $this->mLocalFile
;
97 * Continue chunk uploading
101 * @param WebRequestUpload $webRequestUpload
103 public function continueChunks( $name, $key, $webRequestUpload ) {
104 $this->mFileKey
= $key;
105 $this->mUpload
= $webRequestUpload;
106 // Get the chunk status form the db:
107 $this->getChunkStatus();
109 $metadata = $this->stash
->getMetadata( $key );
110 $this->initializePathInfo( $name,
111 $this->getRealPath( $metadata['us_path'] ),
112 $metadata['us_size'],
118 * Append the final chunk and ready file for parent::performUpload()
119 * @return FileRepoStatus
121 public function concatenateChunks() {
122 $chunkIndex = $this->getChunkIndex();
123 wfDebug( __METHOD__
. " concatenate {$this->mChunkIndex} chunks:" .
124 $this->getOffset() . ' inx:' . $chunkIndex . "\n" );
126 // Concatenate all the chunks to mVirtualTempPath
128 // The first chunk is stored at the mVirtualTempPath path so we start on "chunk 1"
129 for ( $i = 0; $i <= $chunkIndex; $i++
) {
130 $fileList[] = $this->getVirtualChunkLocation( $i );
133 // Get the file extension from the last chunk
134 $ext = FileBackend
::extensionFromPath( $this->mVirtualTempPath
);
135 // Get a 0-byte temp file to perform the concatenation at
136 $tmpFile = TempFSFile
::factory( 'chunkedupload_', $ext );
137 $tmpPath = false; // fail in concatenate()
139 // keep alive with $this
140 $tmpPath = $tmpFile->bind( $this )->getPath();
143 // Concatenate the chunks at the temp file
144 $tStart = microtime( true );
145 $status = $this->repo
->concatenate( $fileList, $tmpPath, FileRepo
::DELETE_SOURCE
);
146 $tAmount = microtime( true ) - $tStart;
147 if ( !$status->isOk() ) {
150 wfDebugLog( 'fileconcatenate', "Combined $i chunks in $tAmount seconds." );
153 $this->mTempPath
= $tmpPath;
154 // Since this was set for the last chunk previously
155 $this->mFileSize
= filesize( $this->mTempPath
);
156 $ret = $this->verifyUpload();
157 if ( $ret['status'] !== UploadBase
::OK
) {
158 wfDebugLog( 'fileconcatenate', "Verification failed for chunked upload" );
159 $status->fatal( $this->getVerificationErrorCode( $ret['status'] ) );
164 // Update the mTempPath and mLocalFile
165 // (for FileUpload or normal Stash to take over)
166 $tStart = microtime( true );
167 $this->mLocalFile
= parent
::stashFile( $this->user
);
168 $tAmount = microtime( true ) - $tStart;
169 $this->mLocalFile
->setLocalReference( $tmpFile ); // reuse (e.g. for getImageInfo())
170 wfDebugLog( 'fileconcatenate', "Stashed combined file ($i chunks) in $tAmount seconds." );
176 * Perform the upload, then remove the temp copy afterward
177 * @param string $comment
178 * @param string $pageText
183 public function performUpload( $comment, $pageText, $watch, $user ) {
184 $rv = parent
::performUpload( $comment, $pageText, $watch, $user );
190 * Returns the virtual chunk location:
194 function getVirtualChunkLocation( $index ) {
195 return $this->repo
->getVirtualUrl( 'temp' ) .
197 $this->repo
->getHashPath(
198 $this->getChunkFileKey( $index )
200 $this->getChunkFileKey( $index );
204 * Add a chunk to the temporary directory
206 * @param string $chunkPath Path to temporary chunk file
207 * @param int $chunkSize Size of the current chunk
208 * @param int $offset Offset of current chunk ( mutch match database chunk offset )
211 public function addChunk( $chunkPath, $chunkSize, $offset ) {
212 // Get the offset before we add the chunk to the file system
213 $preAppendOffset = $this->getOffset();
215 if ( $preAppendOffset +
$chunkSize > $this->getMaxUploadSize() ) {
216 $status = Status
::newFatal( 'file-too-large' );
218 // Make sure the client is uploading the correct chunk with a matching offset.
219 if ( $preAppendOffset == $offset ) {
220 // Update local chunk index for the current chunk
221 $this->mChunkIndex++
;
223 # For some reason mTempPath is set to first part
224 $oldTemp = $this->mTempPath
;
225 $this->mTempPath
= $chunkPath;
226 $this->verifyChunk();
227 $this->mTempPath
= $oldTemp;
228 } catch ( UploadChunkVerificationException
$e ) {
229 return Status
::newFatal( $e->getMessage() );
231 $status = $this->outputChunk( $chunkPath );
232 if ( $status->isGood() ) {
233 // Update local offset:
234 $this->mOffset
= $preAppendOffset +
$chunkSize;
235 // Update chunk table status db
236 $this->updateChunkStatus();
239 $status = Status
::newFatal( 'invalid-chunk-offset' );
247 * Update the chunk db table with the current status:
249 private function updateChunkStatus() {
250 wfDebug( __METHOD__
. " update chunk status for {$this->mFileKey} offset:" .
251 $this->getOffset() . ' inx:' . $this->getChunkIndex() . "\n" );
253 $dbw = $this->repo
->getMasterDb();
254 // Use a quick transaction since we will upload the full temp file into shared
255 // storage, which takes time for large files. We don't want to hold locks then.
256 $dbw->begin( __METHOD__
);
260 'us_status' => 'chunks',
261 'us_chunk_inx' => $this->getChunkIndex(),
262 'us_size' => $this->getOffset()
264 array( 'us_key' => $this->mFileKey
),
267 $dbw->commit( __METHOD__
);
271 * Get the chunk db state and populate update relevant local values
273 private function getChunkStatus() {
274 // get Master db to avoid race conditions.
275 // Otherwise, if chunk upload time < replag there will be spurious errors
276 $dbw = $this->repo
->getMasterDb();
277 $row = $dbw->selectRow(
284 array( 'us_key' => $this->mFileKey
),
289 $this->mChunkIndex
= $row->us_chunk_inx
;
290 $this->mOffset
= $row->us_size
;
291 $this->mVirtualTempPath
= $row->us_path
;
296 * Get the current Chunk index
297 * @return int Index of the current chunk
299 private function getChunkIndex() {
300 if ( $this->mChunkIndex
!== null ) {
301 return $this->mChunkIndex
;
308 * Gets the current offset in fromt the stashedupload table
309 * @return int Current byte offset of the chunk file set
311 private function getOffset() {
312 if ( $this->mOffset
!== null ) {
313 return $this->mOffset
;
320 * Output the chunk to disk
322 * @param string $chunkPath
323 * @throws UploadChunkFileException
324 * @return FileRepoStatus
326 private function outputChunk( $chunkPath ) {
327 // Key is fileKey + chunk index
328 $fileKey = $this->getChunkFileKey();
330 // Store the chunk per its indexed fileKey:
331 $hashPath = $this->repo
->getHashPath( $fileKey );
332 $storeStatus = $this->repo
->quickImport( $chunkPath,
333 $this->repo
->getZonePath( 'temp' ) . "/{$hashPath}{$fileKey}" );
335 // Check for error in stashing the chunk:
336 if ( !$storeStatus->isOK() ) {
337 $error = $storeStatus->getErrorsArray();
338 $error = reset( $error );
339 if ( !count( $error ) ) {
340 $error = $storeStatus->getWarningsArray();
341 $error = reset( $error );
342 if ( !count( $error ) ) {
343 $error = array( 'unknown', 'no error recorded' );
346 throw new UploadChunkFileException( "Error storing file in '$chunkPath': " .
347 implode( '; ', $error ) );
353 private function getChunkFileKey( $index = null ) {
354 if ( $index === null ) {
355 $index = $this->getChunkIndex();
358 return $this->mFileKey
. '.' . $index;
362 * Verify that the chunk isn't really an evil html file
364 * @throws UploadChunkVerificationException
366 private function verifyChunk() {
367 // Rest mDesiredDestName here so we verify the name as if it were mFileKey
368 $oldDesiredDestName = $this->mDesiredDestName
;
369 $this->mDesiredDestName
= $this->mFileKey
;
370 $this->mTitle
= false;
371 $res = $this->verifyPartialFile();
372 $this->mDesiredDestName
= $oldDesiredDestName;
373 $this->mTitle
= false;
374 if ( is_array( $res ) ) {
375 throw new UploadChunkVerificationException( $res[0] );
380 class UploadChunkZeroLengthFileException
extends MWException
{
383 class UploadChunkFileException
extends MWException
{
386 class UploadChunkVerificationException
extends MWException
{