From a31d0f8edd167e9ee301b694a8092fe0cbcfdabf Mon Sep 17 00:00:00 2001 From: Aaron Schulz Date: Mon, 3 Dec 2012 18:18:48 -0800 Subject: [PATCH] Added async upload-from-stash support. * This works similarly to the async concatenation code. Change-Id: Iae38b93d65182158561b92168af51a1e9f50374c --- includes/api/ApiUpload.php | 120 +++++++++++++++++------- includes/upload/PublishStashedFile.php | 125 +++++++++++++++++++++++++ includes/upload/UploadBase.php | 14 ++- includes/upload/UploadFromStash.php | 12 ++- 4 files changed, 235 insertions(+), 36 deletions(-) create mode 100644 includes/upload/PublishStashedFile.php diff --git a/includes/api/ApiUpload.php b/includes/api/ApiUpload.php index cfa1683173..2e18e8ad68 100644 --- a/includes/api/ApiUpload.php +++ b/includes/api/ApiUpload.php @@ -51,6 +51,8 @@ class ApiUpload extends ApiBase { // Parameter handling $this->mParams = $this->extractRequestParams(); $request = $this->getMain()->getRequest(); + // Check if async mode is actually supported + $this->mParams['async'] = ( $this->mParams['async'] && !wfIsWindows() ); // Add the uploaded file to the params array $this->mParams['file'] = $request->getFileName( 'file' ); $this->mParams['chunk'] = $request->getFileName( 'chunk' ); @@ -62,17 +64,15 @@ class ApiUpload extends ApiBase { // Select an upload module if ( !$this->selectUploadModule() ) { - // This is not a true upload, but a status request or similar - return; - } - if ( !isset( $this->mUpload ) ) { + return; // not a true upload, but a status request or similar + } elseif ( !isset( $this->mUpload ) ) { $this->dieUsage( 'No upload module set', 'nomodule' ); } // First check permission to upload $this->checkPermissions( $user ); - // Fetch the file + // Fetch the file (usually a no-op) $status = $this->mUpload->fetchFile(); if ( !$status->isGood() ) { $errors = $status->getErrorsArray(); @@ -89,6 +89,8 @@ class ApiUpload extends ApiBase { if ( !$this->mUpload->getTitle() ) { $this->dieUsage( 'Invalid file title supplied', 'internal-error' ); } + } elseif ( $this->mParams['async'] ) { + // defer verification to background process } else { $this->verifyUpload(); } @@ -96,15 +98,15 @@ class ApiUpload extends ApiBase { // Check if the user has the rights to modify or overwrite the requested title // (This check is irrelevant if stashing is already requested, since the errors // can always be fixed by changing the title) - if ( ! $this->mParams['stash'] ) { + if ( !$this->mParams['stash'] ) { $permErrors = $this->mUpload->verifyTitlePermissions( $user ); if ( $permErrors !== true ) { $this->dieRecoverableError( $permErrors[0], 'filename' ); } } + // Get the result based on the current upload context: $result = $this->getContextResult(); - if ( $result['result'] === 'Success' ) { $result['imageinfo'] = $this->mUpload->getImageInfo( $this->getResult() ); } @@ -135,6 +137,7 @@ class ApiUpload extends ApiBase { // performUpload will return a formatted properly for the API with status return $this->performUpload( $warnings ); } + /** * Get Stash Result, throws an expetion if the file could not be stashed. * @param $warnings array Array of Api upload warnings @@ -156,6 +159,7 @@ class ApiUpload extends ApiBase { } return $result; } + /** * Get Warnings Result * @param $warnings array Array of Api upload warnings @@ -175,6 +179,7 @@ class ApiUpload extends ApiBase { } return $result; } + /** * Get the result of a chunk upload. * @param $warnings array Array of Api upload warnings @@ -206,7 +211,7 @@ class ApiUpload extends ApiBase { if( $this->mParams['offset'] + $chunkSize == $this->mParams['filesize'] ) { if ( $this->mParams['async'] && !wfIsWindows() ) { $progress = UploadBase::getSessionStatus( $this->mParams['filekey'] ); - if ( $progress && $progress['result'] !== 'Failed' ) { + if ( $progress && $progress['result'] === 'Poll' ) { $this->dieUsage( "Chunk assembly already in progress.", 'stashfailed' ); } UploadBase::setSessionStatus( @@ -266,7 +271,7 @@ class ApiUpload extends ApiBase { * @throws MWException * @return String file key */ - function performStash() { + private function performStash() { try { $stashFile = $this->mUpload->stashFile(); @@ -291,7 +296,7 @@ class ApiUpload extends ApiBase { * @param $data array Optional extra data to pass to the user * @throws UsageException */ - function dieRecoverableError( $error, $parameter, $data = array() ) { + private function dieRecoverableError( $error, $parameter, $data = array() ) { try { $data['filekey'] = $this->performStash(); $data['sessionkey'] = $data['filekey']; @@ -320,6 +325,7 @@ class ApiUpload extends ApiBase { 'filekey', 'file', 'url', 'statuskey' ); } + // Status report for "upload to stash"/"upload from stash" if ( $this->mParams['filekey'] && $this->mParams['checkstatus'] ) { $progress = UploadBase::getSessionStatus( $this->mParams['filekey'] ); if ( !$progress ) { @@ -327,6 +333,9 @@ class ApiUpload extends ApiBase { } elseif ( !$progress['status']->isGood() ) { $this->dieUsage( $progress['status']->getWikiText(), 'stashfailed' ); } + if ( isset( $progress['status']->value['verification'] ) ) { + $this->checkVerification( $progress['status']->value['verification'] ); + } unset( $progress['status'] ); // remove Status object $this->getResult()->addValue( null, $this->getModuleName(), $progress ); return false; @@ -377,8 +386,11 @@ class ApiUpload extends ApiBase { } $this->mUpload = new UploadFromStash( $this->getUser() ); - - $this->mUpload->initialize( $this->mParams['filekey'], $this->mParams['filename'] ); + // This will not download the temp file in initialize() in async mode. + // We still have enough information to call checkWarnings() and such. + $this->mUpload->initialize( + $this->mParams['filekey'], $this->mParams['filename'], !$this->mParams['async'] + ); } elseif ( isset( $this->mParams['file'] ) ) { $this->mUpload = new UploadFromFile(); $this->mUpload->initialize( @@ -440,12 +452,19 @@ class ApiUpload extends ApiBase { * Performs file verification, dies on error. */ protected function verifyUpload( ) { - global $wgFileExtensions; - $verification = $this->mUpload->verifyUpload( ); if ( $verification['status'] === UploadBase::OK ) { return; + } else { + return $this->checkVerification( $verification ); } + } + + /** + * Performs file verification, dies on error. + */ + protected function checkVerification( array $verification ) { + global $wgFileExtensions; // TODO: Move them to ApiBase's message map switch( $verification['status'] ) { @@ -555,6 +574,8 @@ class ApiUpload extends ApiBase { * @return array */ protected function performUpload( $warnings ) { + global $IP; + // Use comment as initial page text by default if ( is_null( $this->mParams['text'] ) ) { $this->mParams['text'] = $this->mParams['comment']; @@ -569,29 +590,63 @@ class ApiUpload extends ApiBase { } // No errors, no warnings: do the upload - $status = $this->mUpload->performUpload( $this->mParams['comment'], - $this->mParams['text'], $watch, $this->getUser() ); - - if ( !$status->isGood() ) { - $error = $status->getErrorsArray(); - - if ( count( $error ) == 1 && $error[0][0] == 'async' ) { - // The upload can not be performed right now, because the user - // requested so - return array( - 'result' => 'Queued', - 'statuskey' => $error[0][1], - ); + if ( $this->mParams['async'] ) { + $progress = UploadBase::getSessionStatus( $this->mParams['filekey'] ); + if ( $progress && $progress['result'] === 'Poll' ) { + $this->dieUsage( "Upload from stash already in progress.", 'publishfailed' ); + } + UploadBase::setSessionStatus( + $this->mParams['filekey'], + array( 'result' => 'Poll', 'stage' => 'queued', 'status' => Status::newGood() ) + ); + $retVal = 1; + $cmd = wfShellWikiCmd( + "$IP/includes/upload/PublishStashedFile.php", + array( + '--wiki', wfWikiID(), + '--filename', $this->mParams['filename'], + '--filekey', $this->mParams['filekey'], + '--userid', $this->getUser()->getId(), + '--comment', $this->mParams['comment'], + '--text', $this->mParams['text'], + '--watch', $watch, + '--sessionid', session_id(), + '--quiet' + ) + ) . " < " . wfGetNull() . " > " . wfGetNull() . " 2>&1 &"; + // Start a process in the background. Enforce the time limits via PHP + // since ulimit4.sh seems to often not work for this particular usage. + wfShellExec( $cmd, $retVal, array(), array( 'time' => 0, 'memory' => 0 ) ); + if ( $retVal == 0 ) { + $result['result'] = 'Poll'; } else { - $this->getResult()->setIndexedTagName( $error, 'error' ); + UploadBase::setSessionStatus( $this->mParams['filekey'], false ); + $this->dieUsage( + "Failed to start PublishStashedFile.php", 'publishfailed' ); + } + } else { + $status = $this->mUpload->performUpload( $this->mParams['comment'], + $this->mParams['text'], $watch, $this->getUser() ); + + if ( !$status->isGood() ) { + $error = $status->getErrorsArray(); + + if ( count( $error ) == 1 && $error[0][0] == 'async' ) { + // The upload can not be performed right now, because the user + // requested so + return array( + 'result' => 'Queued', + 'statuskey' => $error[0][1], + ); + } else { + $this->getResult()->setIndexedTagName( $error, 'error' ); - $this->dieUsage( 'An internal error occurred', 'internal-error', 0, $error ); + $this->dieUsage( 'An internal error occurred', 'internal-error', 0, $error ); + } } + $result['result'] = 'Success'; } - $file = $this->mUpload->getLocalFile(); - - $result['result'] = 'Success'; $result['filename'] = $file->getName(); if ( $warnings && count( $warnings ) > 0 ) { $result['warnings'] = $warnings; @@ -759,6 +814,7 @@ class ApiUpload extends ApiBase { array( 'code' => 'filename-tooshort', 'info' => 'The filename is too short' ), array( 'code' => 'overwrite', 'info' => 'Overwriting an existing file is not allowed' ), array( 'code' => 'stashfailed', 'info' => 'Stashing temporary file failed' ), + array( 'code' => 'publishfailed', 'info' => 'Publishing of stashed file failed' ), array( 'code' => 'internal-error', 'info' => 'An internal error occurred' ), array( 'code' => 'asynccopyuploaddisabled', 'info' => 'Asynchronous copy uploads disabled' ), array( 'fileexists-forbidden' ), diff --git a/includes/upload/PublishStashedFile.php b/includes/upload/PublishStashedFile.php new file mode 100644 index 0000000000..fec3c73269 --- /dev/null +++ b/includes/upload/PublishStashedFile.php @@ -0,0 +1,125 @@ +mDescription = "Upload stashed file into the local file repo"; + $this->addOption( 'filename', "Desired file name", true, true ); + $this->addOption( 'filekey', "Upload stash file key", true, true ); + $this->addOption( 'userid', "Upload owner user ID", true, true ); + $this->addOption( 'comment', "Upload comment", true, true ); + $this->addOption( 'text', "Upload description", true, true ); + $this->addOption( 'watch', "Whether the uploader should watch the page", true, true ); + $this->addOption( 'sessionid', "Upload owner session ID", true, true ); + } + + public function execute() { + wfSetupSession( $this->getOption( 'sessionid' ) ); + try { + $user = User::newFromId( $this->getOption( 'userid' ) ); + if ( !$user ) { + throw new MWException( "No user with ID " . $this->getOption( 'userid' ) . "." ); + } + + UploadBase::setSessionStatus( + $this->getOption( 'filekey' ), + array( 'result' => 'Poll', 'stage' => 'publish', 'status' => Status::newGood() ) + ); + + $upload = new UploadFromStash( $user ); + // @TODO: initialize() causes a GET, ideally we could frontload the antivirus + // checks and anything else to the stash stage (which includes concatenation and + // the local file is thus already there). That way, instead of GET+PUT, there could + // just be a COPY operation from the stash to the public zone. + $upload->initialize( $this->getOption( 'filekey' ), $this->getOption( 'filename' ) ); + + // Check if the local file checks out (this is generally a no-op) + $verification = $upload->verifyUpload(); + if ( $verification['status'] !== UploadBase::OK ) { + $status = Status::newFatal( 'verification-error' ); + $status->value = array( 'verification' => $verification ); + UploadBase::setSessionStatus( + $this->getOption( 'filekey' ), + array( 'result' => 'Failure', 'stage' => 'publish', 'status' => $status ) + ); + $this->error( "Could not verify upload.\n", 1 ); // die + } + + // Upload the stashed file to a permanent location + $status = $upload->performUpload( + $this->getOption( 'comment' ), + $this->getOption( 'text' ), + $this->getOption( 'watch' ), + $user + ); + if ( !$status->isGood() ) { + UploadBase::setSessionStatus( + $this->getOption( 'filekey' ), + array( 'result' => 'Failure', 'stage' => 'publish', 'status' => $status ) + ); + $this->error( $status->getWikiText() . "\n", 1 ); // die + } + + // Build the image info array while we have the local reference handy + $apiMain = new ApiMain(); // dummy object (XXX) + $imageInfo = $upload->getImageInfo( $apiMain->getResult() ); + + // Cleanup any temporary local file + $upload->cleanupTempFile(); + + // Cache the info so the user doesn't have to wait forever to get the final info + UploadBase::setSessionStatus( + $this->getOption( 'filekey' ), + array( + 'result' => 'Success', + 'stage' => 'publish', + 'filename' => $upload->getLocalFile()->getName(), + 'imageinfo' => $imageInfo, + 'status' => Status::newGood() + ) + ); + } catch ( MWException $e ) { + UploadBase::setSessionStatus( + $this->getOption( 'filekey' ), + array( + 'result' => 'Failure', + 'stage' => 'publish', + 'status' => Status::newFatal( 'api-error-stashfailed' ) + ) + ); + throw $e; + } + session_write_close(); + } +} + +$maintClass = "PublishStashedFile"; +require_once( RUN_MAINTENANCE_IF_MAIN ); diff --git a/includes/upload/UploadBase.php b/includes/upload/UploadBase.php index dc32a299ce..48ea584080 100644 --- a/includes/upload/UploadBase.php +++ b/includes/upload/UploadBase.php @@ -235,6 +235,14 @@ abstract class UploadBase { return $this->mFileSize; } + /** + * Get the base 36 SHA1 of the file + * @return string + */ + protected function getTempFileSha1Base36() { + return FSFile::getSha1Base36FromPath( $this->mTempPath ); + } + /** * @param $srcPath String: the source path * @return string the real path if it was a virtual URL @@ -546,7 +554,9 @@ abstract class UploadBase { } /** - * Check for non fatal problems with the file + * Check for non fatal problems with the file. + * + * This should not assume that mTempPath is set. * * @return Array of warnings */ @@ -594,7 +604,7 @@ abstract class UploadBase { } // Check dupes against existing files - $hash = FSFile::getSha1Base36FromPath( $this->mTempPath ); + $hash = $this->getTempFileSha1Base36(); $dupes = RepoGroup::singleton()->findBySha1( $hash ); $title = $this->getTitle(); // Remove all matches against self diff --git a/includes/upload/UploadFromStash.php b/includes/upload/UploadFromStash.php index c857f25466..71ee96b817 100644 --- a/includes/upload/UploadFromStash.php +++ b/includes/upload/UploadFromStash.php @@ -89,7 +89,7 @@ class UploadFromStash extends UploadBase { * @param $key string * @param $name string */ - public function initialize( $key, $name = 'upload_file' ) { + public function initialize( $key, $name = 'upload_file', $initTempFile = true ) { /** * Confirming a temporarily stashed upload. * We don't want path names to be forged, so we keep @@ -98,7 +98,7 @@ class UploadFromStash extends UploadBase { */ $metadata = $this->stash->getMetadata( $key ); $this->initializePathInfo( $name, - $this->getRealPath( $metadata['us_path'] ), + $initTempFile ? $this->getRealPath( $metadata['us_path'] ) : false, $metadata['us_size'], false ); @@ -129,6 +129,14 @@ class UploadFromStash extends UploadBase { return $this->mSourceType; } + /** + * Get the base 36 SHA1 of the file + * @return string + */ + protected function getTempFileSha1Base36() { + return $this->mFileProps['sha1']; + } + /** * File has been previously verified so no need to do so again. * -- 2.20.1