From: Aaron Schulz Date: Mon, 19 Nov 2012 08:07:50 +0000 (-0800) Subject: [Upload] Added async upload concatenation support. X-Git-Tag: 1.31.0-rc.0~21411^2 X-Git-Url: http://git.cyclocoop.org//%27%40script%40/%27?a=commitdiff_plain;h=ba5e774de9f53863e3d7d2d1efc19811f9c9a849;p=lhc%2Fweb%2Fwiklou.git [Upload] Added async upload concatenation support. * Clients can send the last chunk with 'async' to get an immediate response and then check the status of the upload by polling the API using the 'checkstatus' parameter. * Pass the User object along to stash functions within UploadFromChunks. Change-Id: Ie2ad4c7e94862a728e8a687c3195306e16a5059e --- diff --git a/includes/api/ApiUpload.php b/includes/api/ApiUpload.php index 6b8639c8e6..f88332f903 100644 --- a/includes/api/ApiUpload.php +++ b/includes/api/ApiUpload.php @@ -114,6 +114,7 @@ class ApiUpload extends ApiBase { // Cleanup any temporary mess $this->mUpload->cleanupTempFile(); } + /** * Get an uplaod result based on upload context * @return array @@ -179,7 +180,9 @@ class ApiUpload extends ApiBase { * @param $warnings array Array of Api upload warnings * @return array */ - private function getChunkResult( $warnings ){ + private function getChunkResult( $warnings ) { + global $IP; + $result = array(); $result['result'] = 'Continue'; @@ -192,8 +195,8 @@ class ApiUpload extends ApiBase { if ($this->mParams['offset'] == 0) { $result['filekey'] = $this->performStash(); } else { - $status = $this->mUpload->addChunk($chunkPath, $chunkSize, - $this->mParams['offset']); + $status = $this->mUpload->addChunk( + $chunkPath, $chunkSize, $this->mParams['offset'] ); if ( !$status->isGood() ) { $this->dieUsage( $status->getWikiText(), 'stashfailed' ); return array(); @@ -201,23 +204,50 @@ class ApiUpload extends ApiBase { // Check we added the last chunk: if( $this->mParams['offset'] + $chunkSize == $this->mParams['filesize'] ) { - $status = $this->mUpload->concatenateChunks(); - - if ( !$status->isGood() ) { - $this->dieUsage( $status->getWikiText(), 'stashfailed' ); - return array(); - } - - // We have a new filekey for the fully concatenated file. - $result['filekey'] = $this->mUpload->getLocalFile()->getFileKey(); + if ( $this->mParams['async'] && !wfIsWindows() ) { + $progress = UploadBase::getSessionStatus( $this->mParams['filekey'] ); + if ( $progress && $progress['result'] !== 'Failed' ) { + $this->dieUsage( "Chunk assembly already in progress.", 'stashfailed' ); + } + UploadBase::setSessionStatus( + $this->mParams['filekey'], + array( 'result' => 'Poll', 'status' => Status::newGood() ) + ); + $retVal = 1; + $cmd = wfShellWikiCmd( + "$IP/includes/upload/AssembleUploadChunks.php", + array( + '--filename', $this->mParams['filename'], + '--filekey', $this->mParams['filekey'], + '--userid', $this->getUser()->getId(), + '--sessionid', session_id(), + '--quiet' + ) + ) . " < " . wfGetNull() . " > " . wfGetNull() . " 2>&1 &"; + wfShellExec( $cmd, $retVal ); // start a process in the background + if ( $retVal == 0 ) { + $result['result'] = 'Poll'; + } else { + UploadBase::setSessionStatus( $this->mParams['filekey'], false ); + $this->dieUsage( + "Failed to start AssembleUploadChunks.php", 'stashfailed' ); + } + } else { + $status = $this->mUpload->concatenateChunks(); + if ( !$status->isGood() ) { + $this->dieUsage( $status->getWikiText(), 'stashfailed' ); + return array(); + } - // Remove chunk from stash. (Checks against user ownership of chunks.) - $this->mUpload->stash->removeFile( $this->mParams['filekey'] ); + // We have a new filekey for the fully concatenated file. + $result['filekey'] = $this->mUpload->getLocalFile()->getFileKey(); - $result['result'] = 'Success'; + // Remove chunk from stash. (Checks against user ownership of chunks.) + $this->mUpload->stash->removeFile( $this->mParams['filekey'] ); + $result['result'] = 'Success'; + } } else { - // Continue passing through the filekey for adding further chunks. $result['filekey'] = $this->mParams['filekey']; } @@ -281,11 +311,23 @@ class ApiUpload extends ApiBase { $request = $this->getMain()->getRequest(); // chunk or one and only one of the following parameters is needed - if( !$this->mParams['chunk'] ) { + if ( !$this->mParams['chunk'] ) { $this->requireOnlyOneParameter( $this->mParams, 'filekey', 'file', 'url', 'statuskey' ); } + if ( $this->mParams['filekey'] && $this->mParams['checkstatus'] ) { + $progress = UploadBase::getSessionStatus( $this->mParams['filekey'] ); + if ( !$progress ) { + $this->dieUsage( 'No result in status data', 'missingresult' ); + } elseif ( !$progress['status']->isGood() ) { + $this->dieUsage( $progress['status']->getWikiText(), 'stashfailed' ); + } + unset( $progress['status'] ); // remove Status object + $this->getResult()->addValue( null, $this->getModuleName(), $progress ); + return false; + } + if ( $this->mParams['statuskey'] ) { $this->checkAsyncDownloadEnabled(); @@ -300,7 +342,6 @@ class ApiUpload extends ApiBase { } $this->getResult()->addValue( null, $this->getModuleName(), $sessionData ); return false; - } // The following modules all require the filename parameter to be set @@ -612,9 +653,11 @@ class ApiUpload extends ApiBase { 'offset' => null, 'chunk' => null, + 'async' => false, 'asyncdownload' => false, 'leavemessage' => false, 'statuskey' => null, + 'checkstatus' => false, ); return $params; @@ -639,9 +682,11 @@ class ApiUpload extends ApiBase { 'offset' => 'Offset of chunk in bytes', 'filesize' => 'Filesize of entire upload', + 'async', 'Make potentially large file operations asynchronous when possible', 'asyncdownload' => 'Make fetching a URL asynchronous', 'leavemessage' => 'If asyncdownload is used, leave a message on the user talk page if finished', - 'statuskey' => 'Fetch the upload status for this file key', + 'statuskey' => 'Fetch the upload status for this file key (upload by URL)', + 'checkstatus' => 'Only fetch the upload status for the given file key', ); return $params; diff --git a/includes/filerepo/FileRepo.php b/includes/filerepo/FileRepo.php index cbcc6c8dd1..82f7b490b5 100644 --- a/includes/filerepo/FileRepo.php +++ b/includes/filerepo/FileRepo.php @@ -1681,10 +1681,11 @@ class FileRepo { /** * Get an UploadStash associated with this repo. * + * @param $user User * @return UploadStash */ - public function getUploadStash() { - return new UploadStash( $this ); + public function getUploadStash( User $user = null ) { + return new UploadStash( $this, $user ); } /** diff --git a/includes/upload/AssembleUploadChunks.php b/includes/upload/AssembleUploadChunks.php new file mode 100644 index 0000000000..d5ce78d0f7 --- /dev/null +++ b/includes/upload/AssembleUploadChunks.php @@ -0,0 +1,103 @@ +mDescription = "Re-assemble the segments of a chunked upload into a single file"; + $this->addOption( 'filename', "Desired file name", true, true ); + $this->addOption( 'filekey', "Upload stash file key", true, true ); + $this->addOption( 'userid', "Upload owner user ID", true, true ); + $this->addOption( 'sessionid', "Upload owner session ID", true, true ); + } + + public function execute() { + wfSetupSession( $this->getOption( 'sessionid' ) ); + try { + $user = User::newFromId( $this->getOption( 'userid' ) ); + if ( !$user ) { + throw new MWException( "No user with ID " . $this->getOption( 'userid' ) . "." ); + } + + $upload = new UploadFromChunks( $user ); + $upload->continueChunks( + $this->getOption( 'filename' ), + $this->getOption( 'filekey' ), + RequestContext::getMain()->getRequest() // dummy request + ); + + // Combine all of the chunks into a local file and upload that to a new stash file + $status = $upload->concatenateChunks(); + if ( !$status->isGood() ) { + UploadBase::setSessionStatus( + $this->getOption( 'filekey' ), + array( 'result' => 'Failure', 'status' => $status ) + ); + $this->error( $status->getWikiText() . "\n", 1 ); // die + } + + // We have a new filekey for the fully concatenated file + $newFileKey = $upload->getLocalFile()->getFileKey(); + + // Remove the old stash file row and first chunk file + $upload->stash->removeFileNoAuth( $this->getOption( 'filekey' ) ); + + // Build the image info array while we have the local reference handy + $apiMain = new ApiMain(); // dummy object (XXX) + $imageInfo = $upload->getImageInfo( $apiMain->getResult() ); + + // Cleanup any temporary local file + $upload->cleanupTempFile(); + + // Cache the info so the user doesn't have to wait forever to get the final info + UploadBase::setSessionStatus( + $this->getOption( 'filekey' ), + array( + 'result' => 'Success', + 'filekey' => $newFileKey, + 'imageinfo' => $imageInfo, + 'status' => Status::newGood() + ) + ); + } catch ( MWException $e ) { + UploadBase::setSessionStatus( + $this->getOption( 'filekey' ), + array( + 'result' => 'Failure', + 'status' => Status::newFatal( 'api-error-stashfailed' ) + ) + ); + throw $e; + } + session_write_close(); + } +} + +$maintClass = "AssembleUploadChunks"; +require_once( RUN_MAINTENANCE_IF_MAIN ); diff --git a/includes/upload/UploadBase.php b/includes/upload/UploadBase.php index fa4931cfcf..02cf8fda24 100644 --- a/includes/upload/UploadBase.php +++ b/includes/upload/UploadBase.php @@ -63,6 +63,8 @@ abstract class UploadBase { const WINDOWS_NONASCII_FILENAME = 13; const FILENAME_TOO_LONG = 14; + const SESSION_STATUS_KEY = 'wsUploadStatusData'; + /** * @param $error int * @return string @@ -785,13 +787,14 @@ abstract class UploadBase { * This method returns the file object, which also has a 'fileKey' property which can be passed through a form or * API request to find this stashed file again. * + * @param $user User * @return UploadStashFile stashed file */ - public function stashFile() { + public function stashFile( User $user = null ) { // was stashSessionFile wfProfileIn( __METHOD__ ); - $stash = RepoGroup::singleton()->getLocalRepo()->getUploadStash(); + $stash = RepoGroup::singleton()->getLocalRepo()->getUploadStash( $user ); $file = $stash->stashFile( $this->mTempPath, $this->getSourceType() ); $this->mLocalFile = $file; @@ -1494,6 +1497,28 @@ abstract class UploadBase { } else { return intval( $wgMaxUploadSize ); } + } + /** + * Get the current status of a chunked upload (used for polling). + * The status will be read from the *current* user session. + * @param $statusKey string + * @return Array|bool + */ + public static function getSessionStatus( $statusKey ) { + return isset( $_SESSION[self::SESSION_STATUS_KEY][$statusKey] ) + ? $_SESSION[self::SESSION_STATUS_KEY][$statusKey] + : false; + } + + /** + * Set the current status of a chunked upload (used for polling). + * The status will be stored in the *current* user session. + * @param $statusKey string + * @param $value array|false + * @return void + */ + public static function setSessionStatus( $statusKey, $value ) { + $_SESSION[self::SESSION_STATUS_KEY][$statusKey] = $value; } } diff --git a/includes/upload/UploadFromChunks.php b/includes/upload/UploadFromChunks.php index 0a13683e8f..2b0128b376 100644 --- a/includes/upload/UploadFromChunks.php +++ b/includes/upload/UploadFromChunks.php @@ -37,7 +37,7 @@ class UploadFromChunks extends UploadFromFile { * @param $stash UploadStash * @param $repo FileRepo */ - public function __construct( $user = false, $stash = false, $repo = false ) { + public function __construct( $user = null, $stash = false, $repo = false ) { // user object. sometimes this won't exist, as when running from cron. $this->user = $user; @@ -60,6 +60,7 @@ class UploadFromChunks extends UploadFromFile { return true; } + /** * Calls the parent stashFile and updates the uploadsession table to handle "chunks" * @@ -134,7 +135,7 @@ class UploadFromChunks extends UploadFromFile { // ( for FileUpload or normal Stash to take over ) $this->mTempPath = $tmpPath; // file system path $tStart = microtime( true ); - $this->mLocalFile = parent::stashFile(); + $this->mLocalFile = parent::stashFile( $this->user ); $tAmount = microtime( true ) - $tStart; $this->mLocalFile->setLocalReference( $tmpFile ); // reuse (e.g. for getImageInfo()) wfDebugLog( 'fileconcatenate', "Stashed combined file ($i chunks) in $tAmount seconds.\n" ); diff --git a/includes/upload/UploadStash.php b/includes/upload/UploadStash.php index e608df23b5..d91649c93a 100644 --- a/includes/upload/UploadStash.php +++ b/includes/upload/UploadStash.php @@ -110,10 +110,9 @@ class UploadStash { throw new UploadStashBadPathException( "key '$key' is not in a proper format" ); } - if ( !$noAuth ) { - if ( !$this->isLoggedIn ) { - throw new UploadStashNotLoggedInException( __METHOD__ . ' No user is logged in, files must belong to users' ); - } + if ( !$noAuth && !$this->isLoggedIn ) { + throw new UploadStashNotLoggedInException( __METHOD__ . + ' No user is logged in, files must belong to users' ); } if ( !isset( $this->fileMetadata[$key] ) ) {