From c5d228be4caaa503b0210984612ce1b4a749d51e Mon Sep 17 00:00:00 2001 From: "Mark A. Hershberger" Date: Fri, 22 Jan 2010 04:37:23 +0000 Subject: [PATCH] UploadChunks added from js2 branch. Refactored. Still needs to be tested w/o the UI first. --- includes/AutoLoader.php | 7 +- includes/api/ApiUpload.php | 38 +++-- includes/upload/UploadFromChunks.php | 225 +++++++++++++++++++++++++++ 3 files changed, 256 insertions(+), 14 deletions(-) create mode 100644 includes/upload/UploadFromChunks.php diff --git a/includes/AutoLoader.php b/includes/AutoLoader.php index 851ea37d86..3837253f09 100644 --- a/includes/AutoLoader.php +++ b/includes/AutoLoader.php @@ -121,9 +121,6 @@ $wgAutoloadLocalClasses = array( 'HTMLInfoField' => 'includes/HTMLForm.php', 'Http' => 'includes/HttpFunctions.php', 'HttpRequest' => 'includes/HttpFunctions.php', - 'curlHttpRequest' => 'includes/HttpFunctions.php', - 'phpHttpRequest' => 'includes/HttpFunctions.php', - 'simpleFileWriter' => 'includes/HttpFunctions.php', 'IEContentAnalyzer' => 'includes/IEContentAnalyzer.php', 'ImageGallery' => 'includes/ImageGallery.php', 'ImageHistoryList' => 'includes/ImagePage.php', @@ -215,6 +212,9 @@ $wgAutoloadLocalClasses = array( 'SqlBagOStuff' => 'includes/BagOStuff.php', 'SquidUpdate' => 'includes/SquidUpdate.php', 'Status' => 'includes/Status.php', + 'StubUser' => 'includes/StubObject.php', + 'StubUserLang' => 'includes/StubObject.php', + 'StubObject' => 'includes/StubObject.php', 'StringUtils' => 'includes/StringUtils.php', 'TablePager' => 'includes/Pager.php', 'ThumbnailImage' => 'includes/MediaTransformOutput.php', @@ -231,6 +231,7 @@ $wgAutoloadLocalClasses = array( 'UploadFromStash' => 'includes/upload/UploadFromStash.php', 'UploadFromFile' => 'includes/upload/UploadFromFile.php', 'UploadFromUrl' => 'includes/upload/UploadFromUrl.php', + 'UploadFromChunks' => 'includes/upload/UploadFromChunks.php', 'User' => 'includes/User.php', 'UserArray' => 'includes/UserArray.php', 'UserArrayFromResult' => 'includes/UserArray.php', diff --git a/includes/api/ApiUpload.php b/includes/api/ApiUpload.php index 4ecefc8ec0..8a9c7c0c99 100644 --- a/includes/api/ApiUpload.php +++ b/includes/api/ApiUpload.php @@ -60,7 +60,7 @@ class ApiUpload extends ApiBase { // One and only one of the following parameters is needed $this->requireOnlyOneParameter( $this->mParams, - 'sessionkey', 'file', 'url' ); + 'sessionkey', 'file', 'url', 'enablechunks' ); if ( $this->mParams['sessionkey'] ) { /** @@ -69,20 +69,28 @@ class ApiUpload extends ApiBase { // Check the session key if ( !isset( $_SESSION['wsUploadData'][$this->mParams['sessionkey']] ) ) return $this->dieUsageMsg( array( 'invalid-session-key' ) ); - + $this->mUpload = new UploadFromStash(); $this->mUpload->initialize( $this->mParams['filename'], $_SESSION['wsUploadData'][$this->mParams['sessionkey']] ); } else { /** - * Upload from url or file + * Upload from url, etc * Parameter filename is required */ if ( !isset( $this->mParams['filename'] ) ) $this->dieUsageMsg( array( 'missingparam', 'filename' ) ); // Initialize $this->mUpload - if ( isset( $this->mParams['file'] ) ) { + if ( $this->mParams['enablechunks'] ) { + $this->mUpload = new UploadFromChunks(); + $this->mUpload->initialize( $request ); + + if ( !$this->mUpload->status->isOK() ) { + return $this->dieUsageMsg( $this->mUpload->status->getWikiText(), + 'chunked-error' ); + } + } elseif ( isset( $this->mParams['file'] ) ) { $this->mUpload = new UploadFromFile(); $this->mUpload->initialize( $this->mParams['filename'], @@ -90,15 +98,15 @@ class ApiUpload extends ApiBase { $request->getFileSize( 'file' ) ); } elseif ( isset( $this->mParams['url'] ) ) { - // make sure upload by url is enabled: + // make sure upload by url is enabled: if ( !$wgAllowCopyUploads ) $this->dieUsageMsg( array( 'uploaddisabled' ) ); - + // make sure the current user can upload if ( ! $wgUser->isAllowed( 'upload_by_url' ) ) $this->dieUsageMsg( array( 'badaccess-groups' ) ); - - + + $this->mUpload = new UploadFromUrl(); $this->mUpload->initialize( $this->mParams['filename'], $this->mParams['url'] ); @@ -116,7 +124,6 @@ class ApiUpload extends ApiBase { // Finish up the exec command: $this->doExecUpload(); - } protected function doExecUpload() { @@ -228,7 +235,7 @@ class ApiUpload extends ApiBase { // Use comment as initial page text by default if ( is_null( $this->mParams['text'] ) ) $this->mParams['text'] = $this->mParams['comment']; - + // No errors, no warnings: do the upload $status = $this->mUpload->performUpload( $this->mParams['comment'], $this->mParams['text'], $this->mParams['watch'], $wgUser ); @@ -271,6 +278,10 @@ class ApiUpload extends ApiBase { 'watch' => false, 'ignorewarnings' => false, 'file' => null, + 'enablechunks' => null, + 'chunksessionkey' => null, + 'chunk' => null, + 'done' => false, 'url' => null, 'sessionkey' => null, ); @@ -290,6 +301,7 @@ class ApiUpload extends ApiBase { 'watch' => 'Watch the page', 'ignorewarnings' => 'Ignore any warnings', 'file' => 'File contents', + 'enablechunks' => 'Set to use chunk mode; see http://firefogg.org/dev/chunk_post.html for protocol', 'url' => 'Url to fetch the file from', 'sessionkey' => array( 'Session key returned by a previous upload that failed due to warnings', @@ -301,9 +313,11 @@ class ApiUpload extends ApiBase { return array( 'Upload a file, or get the status of pending uploads. Several methods are available:', ' * Upload file contents directly, using the "file" parameter', + ' * Upload a file in chunks, using the "enablechunks",', + ' * Have the MediaWiki server fetch a file from a URL, using the "url" parameter', ' * Complete an earlier upload that failed due to warnings, using the "sessionkey" parameter', 'Note that the HTTP POST must be done as a file upload (i.e. using multipart/form-data) when', - 'sending the "file" parameter. Note also that queries using session keys must be', + 'sending the "file" or "chunk" parameters. Note also that queries using session keys must be', 'done in the same login session as the query that originally returned the key (i.e. do not', 'log out and then log back in). Also you must get and send an edit token before doing any upload stuff.' ); @@ -315,6 +329,8 @@ class ApiUpload extends ApiBase { ' api.php?action=upload&filename=Wiki.png&url=http%3A//upload.wikimedia.org/wikipedia/en/b/bc/Wiki.png', 'Complete an upload that failed due to warnings:', ' api.php?action=upload&filename=Wiki.png&sessionkey=sessionkey&ignorewarnings=1', + 'Begin a chunked upload:', + ' api.php?action=upload&filename=Wiki.png&enablechunks=1' ); } diff --git a/includes/upload/UploadFromChunks.php b/includes/upload/UploadFromChunks.php new file mode 100644 index 0000000000..945764d1c8 --- /dev/null +++ b/includes/upload/UploadFromChunks.php @@ -0,0 +1,225 @@ +getText( 'done' ); + $filename = $request->getText( 'filename' ); + $sessionKey = $request->getText( 'chunksessionkey' ); + + $this->initFromSessionKey( $sessionKey, $request ); + + if ( !$this->sessionKey && !$done ) { + // session key not set, init the chunk upload system: + $this->chunkMode = self::INIT; + $this->mDesiredDestName = $filename; + } else if ( $this->sessionKey && !$done ) { + $this->chunkMode = self::CHUNK; + } else if ( $this->sessionKey && $done ) { + $this->chunkMode = self::DONE; + } + + if ( $this->chunkMode == self::CHUNK || $this->chunkMode == self::DONE ) { + $this->mTempPath = $request->getFileTempName( 'chunk' ); + $this->fileSize += $request->getFileSize( 'chunk' ); + } + } + + /** + * Set session information for chunked uploads and allocate a unique key. + * @param $comment string + * @param $pageText string + * @param $watch boolean + * + * @returns string the session key for this chunked upload + */ + protected function setupChunkSession( $comment, $pageText, $watch ) { + $this->sessionKey = $this->getSessionKey(); + $_SESSION['wsUploadData'][$this->sessionKey] = array( + 'comment' => $comment, + 'pageText' => $pageText, + 'watch' => $watch, + 'mFilteredName' => $this->mFilteredName, + 'repoPath' => null, + 'mDesiredDestName' => $this->mDesiredDestName, + 'version' => self::SESSION_VERSION, + ); + return $this->sessionKey; + } + + /** + * Initialize a continuation of a chunked upload from a session key + * @param $sessionKey string + * @param $request WebRequest + * + * @returns void + */ + protected function initFromSessionKey( $sessionKey, $request ) { + if ( !$sessionKey || empty( $sessionKey ) ) { + $this->status = Status::newFromFatal( 'Missing session data.' ); + return; + } + $this->sessionKey = $sessionKey; + // load the sessionData array: + $sessionData = $request->getSessionData( 'wsUploadData' ); + + if ( isset( $sessionData[$this->sessionKey]['version'] ) + && $sessionData[$this->sessionKey]['version'] == self::SESSION_VERSION ) { + $this->comment = $sessionData[$this->sessionKey]['comment']; + $this->pageText = $sessionData[$this->sessionKey]['pageText']; + $this->watch = $sessionData[$this->sessionKey]['watch']; + $this->mFilteredName = $sessionData[$this->sessionKey]['mFilteredName']; + $this->repoPath = $sessionData[$this->sessionKey]['repoPath']; + $this->mDesiredDestName = $sessionData[$this->sessionKey]['mDesiredDestName']; + } else { + $this->status = Status::newFromFatal( 'Missing session data.' ); + } + } + + /** + * Handle a chunk of the upload. Overrides the parent method + * because Chunked Uploading clients (i.e. Firefogg) require + * specific API responses. + * @see UploadBase::performUpload + */ + public function performUpload( $comment, $pageText, $watch, $user ) { + wfDebug( "\n\n\performUpload(chunked): sum:" . $comment . ' c: ' . $pageText . ' w:' . $watch ); + global $wgUser; + + if ( $this->chunkMode == self::INIT ) { + // firefogg expects a specific result per: + // http://www.firefogg.org/dev/chunk_post.html + + // it's okay to return the token here because + // a) the user must have requested the token to get here and + // b) should only happen over POST + // c) we need the token to validate chunks are coming from a non-xss request + $token = urlencode( $wgUser->editToken() ); + ob_clean(); + echo FormatJson::encode( array( + 'uploadUrl' => wfExpandUrl( wfScript( 'api' ) ) . "?action=upload&" . + "token={$token}&format=json&enablechunks=true&chunksessionkey=" . + $this->setupChunkSession( $comment, $pageText, $watch ) ) ); + exit( 0 ); + } else if ( $this->chunkMode == self::CHUNK ) { + $status = $this->appendChunk(); + if ( !$status->isOK() ) { + return $status; + } + // return success: + // firefogg expects a specific result + // http://www.firefogg.org/dev/chunk_post.html + ob_clean(); + echo FormatJson::encode( + array( 'result' => 1, 'filesize' => $this->fileSize ) + ); + exit( 0 ); + } else if ( $this->chunkMode == self::DONE ) { + if ( $comment == '' ) + $comment = $this->comment; + + if ( $pageText == '' ) + $pageText = $this->pageText; + + if ( $watch == '' ) + $watch = $this->watch; + + $status = parent::performUpload( $comment, $pageText, $watch, $user ); + if ( !$status->isGood() ) { + return $status; + } + $file = $this->getLocalFile(); + + // firefogg expects a specific result + // http://www.firefogg.org/dev/chunk_post.html + ob_clean(); + echo FormatJson::encode( array( + 'result' => 1, + 'done' => 1, + 'resultUrl' => $file->getDescriptionUrl() ) + ); + exit( 0 ); + } + } + + /** + * Append a chunk to the Repo file + * + * @param string $srcPath Path to file to append from + * @param string $toAppendPath Path to file to append to + * @return Status Status + */ + protected function appendToUploadFile( $srcPath, $toAppendPath ) { + $repo = RepoGroup::singleton()->getLocalRepo(); + $status = $repo->append( $srcPath, $toAppendPath ); + return $status; + } + + /** + * Append a chunk to the temporary file. + * + * @return void + */ + protected function appendChunk() { + global $wgMaxUploadSize; + + if ( !$this->repoPath ) { + $this->status = $this->saveTempUploadedFile( $this->mDesiredDestName, $this->mTempPath ); + + if ( $status->isOK() ) { + $this->repoPath = $status->value; + $_SESSION['wsUploadData'][$this->sessionKey]['repoPath'] = $this->repoPath; + } + return $status; + } else { + if ( $this->getRealPath( $this->repoPath ) ) { + $this->status = $this->appendToUploadFile( $this->repoPath, $this->mTempPath ); + } else { + $this->status = Status::newFatal( 'filenotfound', $this->repoPath ); + } + + if ( $this->fileSize > $wgMaxUploadSize ) + $this->status = Status::newFatal( 'largefileserver' ); + } + } +} -- 2.20.1