<?php
+/**
+ * Temporary storage for uploaded files.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @file
+ * @ingroup Upload
+ */
+
/**
* UploadStash is intended to accomplish a few things:
* - enable applications to temporarily stash files without publishing them to the wiki.
* Mostly all of them are the same except for storing some custom fields, which we subsume into the data array.
* - enable applications to find said files later, as long as the db table or temp files haven't been purged.
* - enable the uploading user (and *ONLY* the uploading user) to access said files, and thumbnails of said files, via a URL.
- * We accomplish this using a database table, with ownership checking as you might expect. See SpecialUploadStash, which
+ * We accomplish this using a database table, with ownership checking as you might expect. See SpecialUploadStash, which
* implements a web interface to some files stored this way.
*
+ * UploadStash right now is *mostly* intended to show you one user's slice of the entire stash. The user parameter is only optional
+ * because there are few cases where we clean out the stash from an automated script. In the future we might refactor this.
+ *
* UploadStash represents the entire stash of temporary files.
* UploadStashFile is a filestore for the actual physical disk files.
* UploadFromStash extends UploadBase, and represents a single stashed file as it is moved from the stash to the regular file repository
+ *
+ * @ingroup Upload
*/
class UploadStash {
// Format of the key for files -- has to be suitable as a filename itself (e.g. ab12cd34ef.jpg)
- const KEY_FORMAT_REGEX = '/^[\w-]+\.\w*$/';
-
- // When a given stashed file can't be loaded, wait for the slaves to catch up. If they're more than MAX_LAG
- // behind, throw an exception instead. (at what point is broken better than slow?)
- const MAX_LAG = 30;
-
- // Age of the repository in hours. That is, after how long will files be assumed abandoned and deleted?
- const REPO_AGE = 6;
+ const KEY_FORMAT_REGEX = '/^[\w-\.]+\.\w*$/';
/**
* repository that this uses to store temp files
// array of initialized repo objects
protected $files = array();
-
+
// cache of the file metadata that's stored in the database
protected $fileMetadata = array();
-
+
// fileprops cache
protected $fileProps = array();
-
+
// current user
protected $user, $userId, $isLoggedIn;
*
* @param $repo FileRepo
*/
- public function __construct( $repo, $user = null ) {
+ public function __construct( FileRepo $repo, $user = null ) {
// this might change based on wiki's configuration.
$this->repo = $repo;
-
+
// if a user was passed, use it. otherwise, attempt to use the global.
// this keeps FileRepo from breaking when it creates an UploadStash object
- if( $user ) {
+ if ( $user ) {
$this->user = $user;
} else {
global $wgUser;
$this->user = $wgUser;
}
-
- if( is_object($this->user) ) {
+
+ if ( is_object( $this->user ) ) {
$this->userId = $this->user->getId();
$this->isLoggedIn = $this->user->isLoggedIn();
}
/**
* Get a file and its metadata from the stash.
+ * The noAuth param is a bit janky but is required for automated scripts which clean out the stash.
*
* @param $key String: key under which file information is stored
* @param $noAuth Boolean (optional) Don't check authentication. Used by maintenance scripts.
* @return UploadStashFile
*/
public function getFile( $key, $noAuth = false ) {
-
+
if ( ! preg_match( self::KEY_FORMAT_REGEX, $key ) ) {
throw new UploadStashBadPathException( "key '$key' is not in a proper format" );
}
-
- if( !$noAuth ) {
- if( !$this->isLoggedIn ) {
+
+ if ( !$noAuth ) {
+ if ( !$this->isLoggedIn ) {
throw new UploadStashNotLoggedInException( __METHOD__ . ' No user is logged in, files must belong to users' );
}
}
-
- $dbr = $this->repo->getSlaveDb();
-
+
if ( !isset( $this->fileMetadata[$key] ) ) {
- // try this first. if it fails to find the row, check for lag, wait, try again. if its still missing, throw an exception.
- // this more complex solution keeps things moving for page loads with many requests
- // (ie. validating image ownership) when replag is high
- if( !$this->fetchFileMetadata($key) ) {
- $lag = $dbr->getLag();
- if( $lag > 0 && $lag <= self::MAX_LAG ) {
- // if there's not too much replication lag, just wait for the slave to catch up to our last insert.
- sleep( ceil( $lag ) );
- } elseif($lag > self::MAX_LAG ) {
- // that's a lot of lag to introduce into the middle of the UI.
- throw new UploadStashMaxLagExceededException(
- 'Couldn\'t load stashed file metadata, and replication lag is above threshold: (MAX_LAG=' . self::MAX_LAG . ')'
- );
- }
-
- // now that the waiting has happened, try again
- $this->fetchFileMetadata($key);
+ if ( !$this->fetchFileMetadata( $key ) ) {
+ // If nothing was received, it's likely due to replication lag. Check the master to see if the record is there.
+ $this->fetchFileMetadata( $key, DB_MASTER );
}
-
+
if ( !isset( $this->fileMetadata[$key] ) ) {
throw new UploadStashFileNotFoundException( "key '$key' not found in stash" );
}
// fetch fileprops
$path = $this->fileMetadata[$key]['us_path'];
- if ( $this->repo->isVirtualUrl( $path ) ) {
- $path = $this->repo->resolveVirtualUrl( $path );
- }
- $this->fileProps[$key] = File::getPropsFromPath( $path );
+ $this->fileProps[$key] = $this->repo->getFileProps( $path );
}
-
+
if ( ! $this->files[$key]->exists() ) {
wfDebug( __METHOD__ . " tried to get file at $key, but it doesn't exist\n" );
throw new UploadStashBadPathException( "path doesn't exist" );
}
-
- if( !$noAuth ) {
- if( $this->fileMetadata[$key]['us_user'] != $this->userId ) {
+
+ if ( !$noAuth ) {
+ if ( $this->fileMetadata[$key]['us_user'] != $this->userId ) {
throw new UploadStashWrongOwnerException( "This file ($key) doesn't belong to the current user." );
}
}
-
+
return $this->files[$key];
}
*/
public function getMetadata ( $key ) {
$this->getFile( $key );
- return $this->fileMetadata[$key];
+ return $this->fileMetadata[$key];
}
/**
*/
public function getFileProps ( $key ) {
$this->getFile( $key );
- return $this->fileProps[$key];
+ return $this->fileProps[$key];
}
/**
*
* @param $path String: path to file you want stashed
* @param $sourceType String: the type of upload that generated this file (currently, I believe, 'file' or null)
- * @param $key String: optional, unique key for this file. Used for directory hashing when storing, otherwise not important
* @throws UploadStashBadPathException
* @throws UploadStashFileException
* @throws UploadStashNotLoggedInException
* @return UploadStashFile: file, or null on failure
*/
- public function stashFile( $path, $sourceType = null, $key = null ) {
+ public function stashFile( $path, $sourceType = null ) {
if ( ! file_exists( $path ) ) {
wfDebug( __METHOD__ . " tried to stash file at '$path', but it doesn't exist\n" );
throw new UploadStashBadPathException( "path doesn't exist" );
}
- $fileProps = File::getPropsFromPath( $path );
-
+ $fileProps = FSFile::getPropsFromPath( $path );
wfDebug( __METHOD__ . " stashing file at '$path'\n" );
// we will be initializing from some tmpnam files that don't have extensions.
$path = $pathWithGoodExtension;
}
- // If no key was supplied, use content hash. Also has the nice property of collapsing multiple identical files
- // uploaded this session, which could happen if uploads had failed.
- if ( is_null( $key ) ) {
- $key = $fileProps['sha1'] . "." . $extension;
- }
+ // If no key was supplied, make one. a mysql insertid would be totally reasonable here, except
+ // that for historical reasons, the key is this random thing instead. At least it's not guessable.
+ //
+ // some things that when combined will make a suitably unique key.
+ // see: http://www.jwz.org/doc/mid.html
+ list ($usec, $sec) = explode( ' ', microtime() );
+ $usec = substr($usec, 2);
+ $key = wfBaseConvert( $sec . $usec, 10, 36 ) . '.' .
+ wfBaseConvert( mt_rand(), 10, 36 ) . '.'.
+ $this->userId . '.' .
+ $extension;
$this->fileProps[$key] = $fileProps;
-
+
if ( ! preg_match( self::KEY_FORMAT_REGEX, $key ) ) {
throw new UploadStashBadPathException( "key '$key' is not in a proper format" );
}
-
+
wfDebug( __METHOD__ . " key for '$path': $key\n" );
// if not already in a temporary area, put it there
- $storeResult = $this->repo->storeTemp( basename( $path ), $path );
+ $storeStatus = $this->repo->storeTemp( basename( $path ), $path );
- if( ! $storeResult->isOK() ) {
+ if ( ! $storeStatus->isOK() ) {
// It is a convention in MediaWiki to only return one error per API exception, even if multiple errors
// are available. We use reset() to pick the "first" thing that was wrong, preferring errors to warnings.
- // This is a bit lame, as we may have more info in the $storeResult and we're throwing it away, but to fix it means
+ // This is a bit lame, as we may have more info in the $storeStatus and we're throwing it away, but to fix it means
// redesigning API errors significantly.
- // $storeResult->value just contains the virtual URL (if anything) which is probably useless to the caller
- $error = $storeResult->getErrorsArray();
+ // $storeStatus->value just contains the virtual URL (if anything) which is probably useless to the caller
+ $error = $storeStatus->getErrorsArray();
$error = reset( $error );
if ( ! count( $error ) ) {
- $error = $storeResult->getWarningsArray();
+ $error = $storeStatus->getWarningsArray();
$error = reset( $error );
if ( ! count( $error ) ) {
$error = array( 'unknown', 'no error recorded' );
}
}
- throw new UploadStashFileException( "error storing file in '$path': " . implode( '; ', $error ) );
+ // at this point, $error should contain the single "most important" error, plus any parameters.
+ throw new UploadStashFileException( "Error storing file in '$path': " . wfMessage( $error )->text() );
}
- $stashPath = $storeResult->value;
-
+ $stashPath = $storeStatus->value;
+
// fetch the current user ID
- if( !$this->isLoggedIn ) {
- wfDebugCallstack();
+ if ( !$this->isLoggedIn ) {
throw new UploadStashNotLoggedInException( __METHOD__ . ' No user is logged in, files must belong to users' );
}
wfDebug( __METHOD__ . " inserting $stashPath under $key\n" );
$dbw = $this->repo->getMasterDb();
- // select happens on the master so this can all be in a transaction, which
- // avoids a race condition that's likely with multiple people uploading from the same
- // set of files
- $dbw->begin();
- // first, check to see if it's already there.
- $row = $dbw->selectRow(
- 'uploadstash',
- 'us_user, us_timestamp',
- array('us_key' => $key),
- __METHOD__
- );
-
- // The current user can't have this key if:
- // - the key is owned by someone else and
- // - the age of the key is less than REPO_AGE
- if( is_object( $row ) ) {
- if( $row->us_user != $this->userId &&
- $row->wfTimestamp( TS_UNIX, $row->us_timestamp ) > time() - UploadStash::REPO_AGE * 3600
- ) {
- $dbw->rollback();
- throw new UploadStashWrongOwnerException( "Attempting to upload a duplicate of a file that someone else has stashed" );
- }
- }
-
$this->fileMetadata[$key] = array(
+ 'us_id' => $dbw->nextSequenceValue( 'uploadstash_us_id_seq' ),
'us_user' => $this->userId,
'us_key' => $key,
'us_orig_path' => $path,
- 'us_path' => $stashPath,
+ 'us_path' => $stashPath, // virtual URL
'us_size' => $fileProps['size'],
'us_sha1' => $fileProps['sha1'],
'us_mime' => $fileProps['mime'],
'us_timestamp' => $dbw->timestamp(),
'us_status' => 'finished'
);
-
- // if a row exists but previous checks on it passed, let the current user take over this key.
- $dbw->replace(
+
+ $dbw->insert(
'uploadstash',
- 'us_key',
$this->fileMetadata[$key],
- __METHOD__
+ __METHOD__
);
- $dbw->commit();
// store the insertid in the class variable so immediate retrieval (possibly laggy) isn't necesary.
$this->fileMetadata[$key]['us_id'] = $dbw->insertId();
# create the UploadStashFile object for this file.
$this->initFile( $key );
-
+
return $this->getFile( $key );
}
* @return boolean: success
*/
public function clear() {
- if( !$this->isLoggedIn ) {
+ if ( !$this->isLoggedIn ) {
throw new UploadStashNotLoggedInException( __METHOD__ . ' No user is logged in, files must belong to users' );
}
-
- wfDebug( __METHOD__ . " clearing all rows for user $userId\n" );
+
+ wfDebug( __METHOD__ . ' clearing all rows for user ' . $this->userId . "\n" );
$dbw = $this->repo->getMasterDb();
$dbw->delete(
'uploadstash',
array( 'us_user' => $this->userId ),
- __METHOD__
+ __METHOD__
);
# destroy objects.
$this->files = array();
$this->fileMetadata = array();
-
+
return true;
}
* @throws UploadStashWrongOwnerException
* @return boolean: success
*/
- public function removeFile( $key ){
- if( !$this->isLoggedIn ) {
+ public function removeFile( $key ) {
+ if ( !$this->isLoggedIn ) {
throw new UploadStashNotLoggedInException( __METHOD__ . ' No user is logged in, files must belong to users' );
}
-
+
$dbw = $this->repo->getMasterDb();
-
+
// this is a cheap query. it runs on the master so that this function still works when there's lag.
// it won't be called all that often.
$row = $dbw->selectRow(
'uploadstash',
'us_user',
- array('us_key' => $key),
+ array( 'us_key' => $key ),
__METHOD__
);
-
- if( $row->us_user != $this->userId ) {
+
+ if( !$row ) {
+ throw new UploadStashNoSuchKeyException( "No such key ($key), cannot remove" );
+ }
+
+ if ( $row->us_user != $this->userId ) {
throw new UploadStashWrongOwnerException( "Can't delete: the file ($key) doesn't belong to this user." );
}
-
+
return $this->removeFileNoAuth( $key );
}
wfDebug( __METHOD__ . " clearing row $key\n" );
$dbw = $this->repo->getMasterDb();
-
+
// this gets its own transaction since it's called serially by the cleanupUploadStash maintenance script
- $dbw->begin();
+ $dbw->begin( __METHOD__ );
$dbw->delete(
'uploadstash',
- array( 'us_key' => $key),
- __METHOD__
+ array( 'us_key' => $key ),
+ __METHOD__
);
- $dbw->commit();
-
+ $dbw->commit( __METHOD__ );
+
// TODO: look into UnregisteredLocalFile and find out why the rv here is sometimes wrong (false when file was removed)
// for now, ignore.
$this->files[$key]->remove();
-
+
unset( $this->files[$key] );
unset( $this->fileMetadata[$key] );
-
+
return true;
}
* @return Array
*/
public function listFiles() {
- if( !$this->isLoggedIn ) {
+ if ( !$this->isLoggedIn ) {
throw new UploadStashNotLoggedInException( __METHOD__ . ' No user is logged in, files must belong to users' );
}
-
+
$dbr = $this->repo->getSlaveDb();
$res = $dbr->select(
'uploadstash',
'us_key',
- array('us_key' => $key),
+ array( 'us_user' => $this->userId ),
__METHOD__
);
- if( !is_object( $res ) || $res->numRows() == 0 ) {
+ if ( !is_object( $res ) || $res->numRows() == 0 ) {
// nothing to do.
return false;
}
// finish the read before starting writes.
$keys = array();
- foreach($res as $row) {
+ foreach ( $res as $row ) {
array_push( $keys, $row->us_key );
}
* with an extension.
* XXX this is somewhat redundant with the checks that ApiUpload.php does with incoming
* uploads versus the desired filename. Maybe we can get that passed to us...
+ * @return string
*/
public static function getExtensionForPath( $path ) {
// Does this have an extension?
* @param $key String: key
* @return boolean
*/
- protected function fetchFileMetadata( $key ) {
+ protected function fetchFileMetadata( $key, $readFromDB = DB_SLAVE ) {
// populate $fileMetadata[$key]
- $dbr = $this->repo->getSlaveDb();
+ $dbr = null;
+ if( $readFromDB === DB_MASTER ) {
+ // sometimes reading from the master is necessary, if there's replication lag.
+ $dbr = $this->repo->getMasterDb();
+ } else {
+ $dbr = $this->repo->getSlaveDb();
+ }
+
$row = $dbr->selectRow(
'uploadstash',
'*',
- array('us_key' => $key),
+ array( 'us_key' => $key ),
__METHOD__
);
-
- if( !is_object( $row ) ) {
+
+ if ( !is_object( $row ) ) {
// key wasn't present in the database. this will happen sometimes.
return false;
}
-
- $this->fileMetadata[$key] = array(
- 'us_user' => $row->us_user,
- 'us_key' => $row->us_key,
- 'us_orig_path' => $row->us_orig_path,
- 'us_path' => $row->us_path,
- 'us_size' => $row->us_size,
- 'us_sha1' => $row->us_sha1,
- 'us_mime' => $row->us_mime,
- 'us_media_type' => $row->us_media_type,
- 'us_image_width' => $row->us_image_width,
- 'us_image_height' => $row->us_image_height,
- 'us_image_bits' => $row->us_image_bits,
- 'us_source_type' => $row->us_source_type,
- 'us_timestamp' => $row->us_timestamp,
- 'us_status' => $row->us_status
- );
-
+
+ $this->fileMetadata[$key] = (array)$row;
+
return true;
}
-
+
/**
* Helper function: Initialize the UploadStashFile for a given file.
*
* A LocalFile wrapper around a file that has been temporarily stashed, so we can do things like create thumbnails for it
* Arguably UnregisteredLocalFile should be handling its own file repo but that class is a bit retarded currently
*
- * @param $repo FSRepo: repository where we should find the path
+ * @param $repo FileRepo: repository where we should find the path
* @param $path String: path to file
* @param $key String: key to store the path and any stashed data under
* @throws UploadStashBadPathException
}
// check if path exists! and is a plain file.
- if ( ! $repo->fileExists( $path, FileRepo::FILES_ONLY ) ) {
+ if ( ! $repo->fileExists( $path ) ) {
wfDebug( "UploadStash: tried to construct an UploadStashFile from a file that should already exist at '$path', but path is not found\n" );
throw new UploadStashFileNotFoundException( 'cannot find path, or not a plain file' );
}
* @return Status: success
*/
public function remove() {
- if( !$this->repo->fileExists( $this->path, FileRepo::FILES_ONLY ) ) {
+ if ( !$this->repo->fileExists( $this->path ) ) {
// Maybe the file's already been removed? This could totally happen in UploadBase.
return true;
}
-
+
return $this->repo->freeTemp( $this->path );
}
public function exists() {
- return $this->repo->fileExists( $this->path, FileRepo::FILES_ONLY );
+ return $this->repo->fileExists( $this->path );
}
}
class UploadStashZeroLengthFileException extends MWException {};
class UploadStashNotLoggedInException extends MWException {};
class UploadStashWrongOwnerException extends MWException {};
-class UploadStashMaxLagExceededException extends MWException {};
+class UploadStashNoSuchKeyException extends MWException {};
dépôts / lhc / web / wiklou.git / blobdiff