UploadChunks added from js2 branch. Refactored. Still needs to be
authorMark A. Hershberger <mah@users.mediawiki.org>
Fri, 22 Jan 2010 04:37:23 +0000 (04:37 +0000)
committerMark A. Hershberger <mah@users.mediawiki.org>
Fri, 22 Jan 2010 04:37:23 +0000 (04:37 +0000)
tested w/o the UI first.

includes/AutoLoader.php
includes/api/ApiUpload.php
includes/upload/UploadFromChunks.php [new file with mode: 0644]

index 851ea37..3837253 100644 (file)
@@ -121,9 +121,6 @@ $wgAutoloadLocalClasses = array(
        'HTMLInfoField' => 'includes/HTMLForm.php',
        'Http' => 'includes/HttpFunctions.php',
        'HttpRequest' => 'includes/HttpFunctions.php',
-       'curlHttpRequest' => 'includes/HttpFunctions.php',
-       'phpHttpRequest' => 'includes/HttpFunctions.php',
-       'simpleFileWriter' => 'includes/HttpFunctions.php',
        'IEContentAnalyzer' => 'includes/IEContentAnalyzer.php',
        'ImageGallery' => 'includes/ImageGallery.php',
        'ImageHistoryList' => 'includes/ImagePage.php',
@@ -215,6 +212,9 @@ $wgAutoloadLocalClasses = array(
        'SqlBagOStuff' => 'includes/BagOStuff.php',
        'SquidUpdate' => 'includes/SquidUpdate.php',
        'Status' => 'includes/Status.php',
+       'StubUser' => 'includes/StubObject.php',
+       'StubUserLang' => 'includes/StubObject.php',
+       'StubObject' => 'includes/StubObject.php',
        'StringUtils' => 'includes/StringUtils.php',
        'TablePager' => 'includes/Pager.php',
        'ThumbnailImage' => 'includes/MediaTransformOutput.php',
@@ -231,6 +231,7 @@ $wgAutoloadLocalClasses = array(
        'UploadFromStash' => 'includes/upload/UploadFromStash.php',
        'UploadFromFile' => 'includes/upload/UploadFromFile.php',
        'UploadFromUrl' => 'includes/upload/UploadFromUrl.php',
+       'UploadFromChunks' => 'includes/upload/UploadFromChunks.php',
        'User' => 'includes/User.php',
        'UserArray' => 'includes/UserArray.php',
        'UserArrayFromResult' => 'includes/UserArray.php',
index 4ecefc8..8a9c7c0 100644 (file)
@@ -60,7 +60,7 @@ class ApiUpload extends ApiBase {
 
                // One and only one of the following parameters is needed
                $this->requireOnlyOneParameter( $this->mParams,
-                       'sessionkey', 'file', 'url' );
+                       'sessionkey', 'file', 'url', 'enablechunks' );
 
                if ( $this->mParams['sessionkey'] ) {
                        /**
@@ -69,20 +69,28 @@ class ApiUpload extends ApiBase {
                        // Check the session key
                        if ( !isset( $_SESSION['wsUploadData'][$this->mParams['sessionkey']] ) )
                                return $this->dieUsageMsg( array( 'invalid-session-key' ) );
-                       
+
                        $this->mUpload = new UploadFromStash();
                        $this->mUpload->initialize( $this->mParams['filename'],
                                        $_SESSION['wsUploadData'][$this->mParams['sessionkey']] );
                } else {
                        /**
-                        * Upload from url or file
+                        * Upload from url, etc
                         * Parameter filename is required
                         */
                        if ( !isset( $this->mParams['filename'] ) )
                                $this->dieUsageMsg( array( 'missingparam', 'filename' ) );
 
                        // Initialize $this->mUpload
-                       if ( isset( $this->mParams['file'] ) ) {
+                       if ( $this->mParams['enablechunks'] ) {
+                               $this->mUpload = new UploadFromChunks();
+                               $this->mUpload->initialize( $request );
+
+                               if ( !$this->mUpload->status->isOK() ) {
+                                       return $this->dieUsageMsg( $this->mUpload->status->getWikiText(),
+                                               'chunked-error' );
+                               }
+                       } elseif ( isset( $this->mParams['file'] ) ) {
                                $this->mUpload = new UploadFromFile();
                                $this->mUpload->initialize(
                                        $this->mParams['filename'],
@@ -90,15 +98,15 @@ class ApiUpload extends ApiBase {
                                        $request->getFileSize( 'file' )
                                );
                        } elseif ( isset( $this->mParams['url'] ) ) {
-                               // make sure upload by url is enabled: 
+                               // make sure upload by url is enabled:
                                if ( !$wgAllowCopyUploads )
                                        $this->dieUsageMsg( array( 'uploaddisabled' ) );
-                               
+
                                // make sure the current user can upload
                                if ( ! $wgUser->isAllowed( 'upload_by_url' ) )
                                        $this->dieUsageMsg( array( 'badaccess-groups' ) );
-                                       
-                               
+
+
                                $this->mUpload = new UploadFromUrl();
                                $this->mUpload->initialize( $this->mParams['filename'],
                                                $this->mParams['url'] );
@@ -116,7 +124,6 @@ class ApiUpload extends ApiBase {
 
                // Finish up the exec command:
                $this->doExecUpload();
-
        }
 
        protected function doExecUpload() {
@@ -228,7 +235,7 @@ class ApiUpload extends ApiBase {
                // Use comment as initial page text by default
                if ( is_null( $this->mParams['text'] ) )
                        $this->mParams['text'] = $this->mParams['comment'];
-               
+
                // No errors, no warnings: do the upload
                $status = $this->mUpload->performUpload( $this->mParams['comment'],
                        $this->mParams['text'], $this->mParams['watch'], $wgUser );
@@ -271,6 +278,10 @@ class ApiUpload extends ApiBase {
                        'watch' => false,
                        'ignorewarnings' => false,
                        'file' => null,
+                       'enablechunks' => null,
+                       'chunksessionkey' => null,
+                       'chunk' => null,
+                       'done' => false,
                        'url' => null,
                        'sessionkey' => null,
                );
@@ -290,6 +301,7 @@ class ApiUpload extends ApiBase {
                        'watch' => 'Watch the page',
                        'ignorewarnings' => 'Ignore any warnings',
                        'file' => 'File contents',
+                       'enablechunks' => 'Set to use chunk mode; see http://firefogg.org/dev/chunk_post.html for protocol',
                        'url' => 'Url to fetch the file from',
                        'sessionkey' => array(
                                'Session key returned by a previous upload that failed due to warnings',
@@ -301,9 +313,11 @@ class ApiUpload extends ApiBase {
                return array(
                        'Upload a file, or get the status of pending uploads. Several methods are available:',
                        ' * Upload file contents directly, using the "file" parameter',
+                       ' * Upload a file in chunks, using the "enablechunks",',
+                       ' * Have the MediaWiki server fetch a file from a URL, using the "url" parameter',
                        ' * Complete an earlier upload that failed due to warnings, using the "sessionkey" parameter',
                        'Note that the HTTP POST must be done as a file upload (i.e. using multipart/form-data) when',
-                       'sending the "file" parameter. Note also that queries using session keys must be',
+                       'sending the "file" or "chunk" parameters. Note also that queries using session keys must be',
                        'done in the same login session as the query that originally returned the key (i.e. do not',
                        'log out and then log back in). Also you must get and send an edit token before doing any upload stuff.'
                );
@@ -315,6 +329,8 @@ class ApiUpload extends ApiBase {
                        '    api.php?action=upload&filename=Wiki.png&url=http%3A//upload.wikimedia.org/wikipedia/en/b/bc/Wiki.png',
                        'Complete an upload that failed due to warnings:',
                        '    api.php?action=upload&filename=Wiki.png&sessionkey=sessionkey&ignorewarnings=1',
+                       'Begin a chunked upload:',
+                       '    api.php?action=upload&filename=Wiki.png&enablechunks=1'
                );
        }
 
diff --git a/includes/upload/UploadFromChunks.php b/includes/upload/UploadFromChunks.php
new file mode 100644 (file)
index 0000000..945764d
--- /dev/null
@@ -0,0 +1,225 @@
+<?php
+/**
+ * @file
+ * @ingroup upload
+ *
+ * First, destination checks are made, and, if ignorewarnings is not
+ * checked, errors / warning is returned.
+ *
+ * 1. We return the uploadUrl.
+ * 2. We then accept chunk uploads from the client.
+ * 3. Return chunk id on each POSTED chunk.
+ * 4. Once the client posts "done=1", the files are concatenated together.
+ *
+ * (More info at: http://firefogg.org/dev/chunk_post.html)
+ */
+class UploadFromChunks extends UploadBase {
+
+       const INIT = 1;
+       const CHUNK = 2;
+       const DONE = 3;
+
+       // STYLE NOTE: Coding guidelines says the 'm' prefix for object
+       // member variables is discouraged in new code but "stay
+       // consistent within a class". UploadFromChunks is new, but extends
+       // UploadBase which has the 'm' prefix.  I'm eschewing the prefix for
+       // member variables of this class.
+       protected $chunkMode; // INIT, CHUNK, DONE
+       protected $sessionKey;
+       protected $comment;
+       protected $fileSize = 0;
+       protected $repoPath;
+       protected $pageText;
+       protected $watch;
+
+       public $status;
+
+       // Parent class requires this function even though it is only
+       // used from SpecialUpload.php and we don't do chunked uploading
+       // from SpecialUpload -- best to raise an exception for
+       // now.
+       public function initializeFromRequest( &$request ) {
+               throw new MWException( 'not implemented' );
+       }
+
+       public function initialize( &$request ) {
+               $done = $request->getText( 'done' );
+               $filename = $request->getText( 'filename' );
+               $sessionKey = $request->getText( 'chunksessionkey' );
+
+               $this->initFromSessionKey( $sessionKey, $request );
+
+               if ( !$this->sessionKey && !$done ) {
+                       // session key not set, init the chunk upload system:
+                       $this->chunkMode = self::INIT;
+                       $this->mDesiredDestName = $filename;
+               } else if ( $this->sessionKey && !$done ) {
+                       $this->chunkMode = self::CHUNK;
+               } else if ( $this->sessionKey && $done ) {
+                       $this->chunkMode = self::DONE;
+               }
+
+               if ( $this->chunkMode == self::CHUNK || $this->chunkMode == self::DONE ) {
+                       $this->mTempPath = $request->getFileTempName( 'chunk' );
+                       $this->fileSize += $request->getFileSize( 'chunk' );
+               }
+       }
+
+       /**
+        * Set session information for chunked uploads and allocate a unique key.
+        * @param $comment string
+        * @param $pageText string
+        * @param $watch boolean
+        *
+        * @returns string the session key for this chunked upload
+        */
+       protected function setupChunkSession( $comment, $pageText, $watch ) {
+               $this->sessionKey = $this->getSessionKey();
+               $_SESSION['wsUploadData'][$this->sessionKey] = array(
+                       'comment' => $comment,
+                       'pageText' => $pageText,
+                       'watch' => $watch,
+                       'mFilteredName' => $this->mFilteredName,
+                       'repoPath' => null,
+                       'mDesiredDestName' => $this->mDesiredDestName,
+                       'version' => self::SESSION_VERSION,
+               );
+               return $this->sessionKey;
+       }
+
+       /**
+        * Initialize a continuation of a chunked upload from a session key
+        * @param $sessionKey string
+        * @param $request WebRequest
+        *
+        * @returns void
+        */
+       protected function initFromSessionKey( $sessionKey, $request ) {
+               if ( !$sessionKey || empty( $sessionKey ) ) {
+                       $this->status = Status::newFromFatal( 'Missing session data.' );
+                       return;
+               }
+               $this->sessionKey = $sessionKey;
+               // load the sessionData array:
+               $sessionData = $request->getSessionData( 'wsUploadData' );
+
+               if ( isset( $sessionData[$this->sessionKey]['version'] )
+                               && $sessionData[$this->sessionKey]['version'] == self::SESSION_VERSION ) {
+                       $this->comment = $sessionData[$this->sessionKey]['comment'];
+                       $this->pageText = $sessionData[$this->sessionKey]['pageText'];
+                       $this->watch = $sessionData[$this->sessionKey]['watch'];
+                       $this->mFilteredName = $sessionData[$this->sessionKey]['mFilteredName'];
+                       $this->repoPath = $sessionData[$this->sessionKey]['repoPath'];
+                       $this->mDesiredDestName = $sessionData[$this->sessionKey]['mDesiredDestName'];
+               } else {
+                       $this->status = Status::newFromFatal( 'Missing session data.' );
+               }
+       }
+
+       /**
+        * Handle a chunk of the upload.  Overrides the parent method
+        * because Chunked Uploading clients (i.e. Firefogg) require
+        * specific API responses.
+        * @see UploadBase::performUpload
+        */
+       public function performUpload( $comment, $pageText, $watch, $user ) {
+               wfDebug( "\n\n\performUpload(chunked): sum:" . $comment . ' c: ' . $pageText . ' w:' . $watch );
+               global $wgUser;
+
+               if ( $this->chunkMode == self::INIT ) {
+                       // firefogg expects a specific result per:
+                       // http://www.firefogg.org/dev/chunk_post.html
+
+                       // it's okay to return the token here because
+                       // a) the user must have requested the token to get here and
+                       // b) should only happen over POST
+                       // c) we need the token to validate chunks are coming from a non-xss request
+                       $token = urlencode( $wgUser->editToken() );
+                       ob_clean();
+                       echo FormatJson::encode( array(
+                               'uploadUrl' => wfExpandUrl( wfScript( 'api' ) ) . "?action=upload&" .
+                               "token={$token}&format=json&enablechunks=true&chunksessionkey=" .
+                               $this->setupChunkSession( $comment, $pageText, $watch ) ) );
+                       exit( 0 );
+               } else if ( $this->chunkMode == self::CHUNK ) {
+                       $status = $this->appendChunk();
+                       if ( !$status->isOK() ) {
+                               return $status;
+                       }
+                       // return success:
+                       // firefogg expects a specific result
+                       // http://www.firefogg.org/dev/chunk_post.html
+                       ob_clean();
+                       echo FormatJson::encode(
+                               array( 'result' => 1, 'filesize' => $this->fileSize )
+                       );
+                       exit( 0 );
+               } else if ( $this->chunkMode == self::DONE ) {
+                       if ( $comment == '' )
+                               $comment = $this->comment;
+
+                       if ( $pageText == '' )
+                               $pageText = $this->pageText;
+
+                       if ( $watch == '' )
+                               $watch = $this->watch;
+
+                       $status = parent::performUpload( $comment, $pageText, $watch, $user );
+                       if ( !$status->isGood() ) {
+                               return $status;
+                       }
+                       $file = $this->getLocalFile();
+
+                       // firefogg expects a specific result
+                       // http://www.firefogg.org/dev/chunk_post.html
+                       ob_clean();
+                       echo FormatJson::encode( array(
+                               'result' => 1,
+                               'done' => 1,
+                               'resultUrl' => $file->getDescriptionUrl() )
+                       );
+                       exit( 0 );
+               }
+       }
+
+       /**
+        * Append a chunk to the Repo file
+        *
+        * @param string $srcPath Path to file to append from
+        * @param string $toAppendPath Path to file to append to
+        * @return Status Status
+        */
+       protected function appendToUploadFile( $srcPath, $toAppendPath ) {
+               $repo = RepoGroup::singleton()->getLocalRepo();
+               $status = $repo->append( $srcPath, $toAppendPath );
+               return $status;
+       }
+
+       /**
+        * Append a chunk to the temporary file.
+        *
+        * @return void
+        */
+       protected function appendChunk() {
+               global $wgMaxUploadSize;
+
+               if ( !$this->repoPath ) {
+                       $this->status = $this->saveTempUploadedFile( $this->mDesiredDestName, $this->mTempPath );
+
+                       if ( $status->isOK() ) {
+                               $this->repoPath = $status->value;
+                               $_SESSION['wsUploadData'][$this->sessionKey]['repoPath'] = $this->repoPath;
+                       }
+                       return $status;
+               } else {
+                       if ( $this->getRealPath( $this->repoPath ) ) {
+                               $this->status = $this->appendToUploadFile( $this->repoPath, $this->mTempPath );
+                       } else {
+                               $this->status = Status::newFatal( 'filenotfound', $this->repoPath );
+                       }
+
+                       if ( $this->fileSize >  $wgMaxUploadSize )
+                               $this->status = Status::newFatal( 'largefileserver' );
+               }
+       }
+}