<?php
+/**
+ * Temporary storage for uploaded files.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @file
+ * @ingroup Upload
+ */
+
/**
* UploadStash is intended to accomplish a few things:
* - enable applications to temporarily stash files without publishing them to the wiki.
* We accomplish this using a database table, with ownership checking as you might expect. See SpecialUploadStash, which
* implements a web interface to some files stored this way.
*
+ * UploadStash right now is *mostly* intended to show you one user's slice of the entire stash. The user parameter is only optional
+ * because there are few cases where we clean out the stash from an automated script. In the future we might refactor this.
+ *
* UploadStash represents the entire stash of temporary files.
* UploadStashFile is a filestore for the actual physical disk files.
* UploadFromStash extends UploadBase, and represents a single stashed file as it is moved from the stash to the regular file repository
+ *
+ * @ingroup Upload
*/
class UploadStash {
// Format of the key for files -- has to be suitable as a filename itself (e.g. ab12cd34ef.jpg)
- const KEY_FORMAT_REGEX = '/^[\w-]+\.\w*$/';
-
- // When a given stashed file can't be loaded, wait for the slaves to catch up. If they're more than MAX_LAG
- // behind, throw an exception instead. (at what point is broken better than slow?)
- const MAX_LAG = 30;
-
- // Age of the repository in hours. That is, after how long will files be assumed abandoned and deleted?
- const REPO_AGE = 6;
+ const KEY_FORMAT_REGEX = '/^[\w-\.]+\.\w*$/';
/**
* repository that this uses to store temp files
*
* @param $repo FileRepo
*/
- public function __construct( $repo, $user = null ) {
+ public function __construct( FileRepo $repo, $user = null ) {
// this might change based on wiki's configuration.
$this->repo = $repo;
/**
* Get a file and its metadata from the stash.
+ * The noAuth param is a bit janky but is required for automated scripts which clean out the stash.
*
* @param $key String: key under which file information is stored
* @param $noAuth Boolean (optional) Don't check authentication. Used by maintenance scripts.
}
}
- $dbr = $this->repo->getSlaveDb();
-
if ( !isset( $this->fileMetadata[$key] ) ) {
- // try this first. if it fails to find the row, check for lag, wait, try again. if its still missing, throw an exception.
- // this more complex solution keeps things moving for page loads with many requests
- // (ie. validating image ownership) when replag is high
if ( !$this->fetchFileMetadata( $key ) ) {
- $lag = $dbr->getLag();
- if ( $lag > 0 && $lag <= self::MAX_LAG ) {
- // if there's not too much replication lag, just wait for the slave to catch up to our last insert.
- sleep( ceil( $lag ) );
- } elseif ( $lag > self::MAX_LAG ) {
- // that's a lot of lag to introduce into the middle of the UI.
- throw new UploadStashMaxLagExceededException(
- 'Couldn\'t load stashed file metadata, and replication lag is above threshold: (MAX_LAG=' . self::MAX_LAG . ')'
- );
- }
-
- // now that the waiting has happened, try again
- $this->fetchFileMetadata( $key );
+ // If nothing was received, it's likely due to replication lag. Check the master to see if the record is there.
+ $this->fetchFileMetadata( $key, DB_MASTER );
}
if ( !isset( $this->fileMetadata[$key] ) ) {
// fetch fileprops
$path = $this->fileMetadata[$key]['us_path'];
- if ( $this->repo->isVirtualUrl( $path ) ) {
- $path = $this->repo->resolveVirtualUrl( $path );
- }
- $this->fileProps[$key] = File::getPropsFromPath( $path );
+ $this->fileProps[$key] = $this->repo->getFileProps( $path );
}
if ( ! $this->files[$key]->exists() ) {
*
* @param $path String: path to file you want stashed
* @param $sourceType String: the type of upload that generated this file (currently, I believe, 'file' or null)
- * @param $key String: optional, unique key for this file. Used for directory hashing when storing, otherwise not important
* @throws UploadStashBadPathException
* @throws UploadStashFileException
* @throws UploadStashNotLoggedInException
* @return UploadStashFile: file, or null on failure
*/
- public function stashFile( $path, $sourceType = null, $key = null ) {
+ public function stashFile( $path, $sourceType = null ) {
if ( ! file_exists( $path ) ) {
wfDebug( __METHOD__ . " tried to stash file at '$path', but it doesn't exist\n" );
throw new UploadStashBadPathException( "path doesn't exist" );
}
- $fileProps = File::getPropsFromPath( $path );
-
+ $fileProps = FSFile::getPropsFromPath( $path );
wfDebug( __METHOD__ . " stashing file at '$path'\n" );
// we will be initializing from some tmpnam files that don't have extensions.
$path = $pathWithGoodExtension;
}
- // If no key was supplied, use content hash. Also has the nice property of collapsing multiple identical files
- // uploaded this session, which could happen if uploads had failed.
- if ( is_null( $key ) ) {
- $key = $fileProps['sha1'] . "." . $extension;
- }
+ // If no key was supplied, make one. a mysql insertid would be totally reasonable here, except
+ // that for historical reasons, the key is this random thing instead. At least it's not guessable.
+ //
+ // some things that when combined will make a suitably unique key.
+ // see: http://www.jwz.org/doc/mid.html
+ list ($usec, $sec) = explode( ' ', microtime() );
+ $usec = substr($usec, 2);
+ $key = wfBaseConvert( $sec . $usec, 10, 36 ) . '.' .
+ wfBaseConvert( mt_rand(), 10, 36 ) . '.'.
+ $this->userId . '.' .
+ $extension;
$this->fileProps[$key] = $fileProps;
wfDebug( __METHOD__ . " key for '$path': $key\n" );
// if not already in a temporary area, put it there
- $storeResult = $this->repo->storeTemp( basename( $path ), $path );
+ $storeStatus = $this->repo->storeTemp( basename( $path ), $path );
- if ( ! $storeResult->isOK() ) {
+ if ( ! $storeStatus->isOK() ) {
// It is a convention in MediaWiki to only return one error per API exception, even if multiple errors
// are available. We use reset() to pick the "first" thing that was wrong, preferring errors to warnings.
- // This is a bit lame, as we may have more info in the $storeResult and we're throwing it away, but to fix it means
+ // This is a bit lame, as we may have more info in the $storeStatus and we're throwing it away, but to fix it means
// redesigning API errors significantly.
- // $storeResult->value just contains the virtual URL (if anything) which is probably useless to the caller
- $error = $storeResult->getErrorsArray();
+ // $storeStatus->value just contains the virtual URL (if anything) which is probably useless to the caller
+ $error = $storeStatus->getErrorsArray();
$error = reset( $error );
if ( ! count( $error ) ) {
- $error = $storeResult->getWarningsArray();
+ $error = $storeStatus->getWarningsArray();
$error = reset( $error );
if ( ! count( $error ) ) {
$error = array( 'unknown', 'no error recorded' );
}
}
- throw new UploadStashFileException( "error storing file in '$path': " . implode( '; ', $error ) );
+ // at this point, $error should contain the single "most important" error, plus any parameters.
+ throw new UploadStashFileException( "Error storing file in '$path': " . wfMessage( $error )->text() );
}
- $stashPath = $storeResult->value;
+ $stashPath = $storeStatus->value;
// fetch the current user ID
if ( !$this->isLoggedIn ) {
- wfDebugCallstack();
throw new UploadStashNotLoggedInException( __METHOD__ . ' No user is logged in, files must belong to users' );
}
wfDebug( __METHOD__ . " inserting $stashPath under $key\n" );
$dbw = $this->repo->getMasterDb();
- // select happens on the master so this can all be in a transaction, which
- // avoids a race condition that's likely with multiple people uploading from the same
- // set of files
- $dbw->begin();
- // first, check to see if it's already there.
- $row = $dbw->selectRow(
- 'uploadstash',
- 'us_user, us_timestamp',
- array( 'us_key' => $key ),
- __METHOD__
- );
-
- // The current user can't have this key if:
- // - the key is owned by someone else and
- // - the age of the key is less than REPO_AGE
- if ( is_object( $row ) ) {
- if ( $row->us_user != $this->userId &&
- $row->wfTimestamp( TS_UNIX, $row->us_timestamp ) > time() - UploadStash::REPO_AGE * 3600
- ) {
- $dbw->rollback();
- throw new UploadStashWrongOwnerException( "Attempting to upload a duplicate of a file that someone else has stashed" );
- }
- }
-
$this->fileMetadata[$key] = array(
+ 'us_id' => $dbw->nextSequenceValue( 'uploadstash_us_id_seq' ),
'us_user' => $this->userId,
'us_key' => $key,
'us_orig_path' => $path,
- 'us_path' => $stashPath,
+ 'us_path' => $stashPath, // virtual URL
'us_size' => $fileProps['size'],
'us_sha1' => $fileProps['sha1'],
'us_mime' => $fileProps['mime'],
'us_status' => 'finished'
);
- // if a row exists but previous checks on it passed, let the current user take over this key.
- $dbw->replace(
+ $dbw->insert(
'uploadstash',
- 'us_key',
$this->fileMetadata[$key],
__METHOD__
);
- $dbw->commit();
// store the insertid in the class variable so immediate retrieval (possibly laggy) isn't necesary.
$this->fileMetadata[$key]['us_id'] = $dbw->insertId();
throw new UploadStashNotLoggedInException( __METHOD__ . ' No user is logged in, files must belong to users' );
}
- wfDebug( __METHOD__ . " clearing all rows for user $userId\n" );
+ wfDebug( __METHOD__ . ' clearing all rows for user ' . $this->userId . "\n" );
$dbw = $this->repo->getMasterDb();
$dbw->delete(
'uploadstash',
$dbw = $this->repo->getMasterDb();
// this gets its own transaction since it's called serially by the cleanupUploadStash maintenance script
- $dbw->begin();
+ $dbw->begin( __METHOD__ );
$dbw->delete(
'uploadstash',
array( 'us_key' => $key ),
__METHOD__
);
- $dbw->commit();
+ $dbw->commit( __METHOD__ );
// TODO: look into UnregisteredLocalFile and find out why the rv here is sometimes wrong (false when file was removed)
// for now, ignore.
$res = $dbr->select(
'uploadstash',
'us_key',
- array( 'us_key' => $key ),
+ array( 'us_user' => $this->userId ),
__METHOD__
);
* with an extension.
* XXX this is somewhat redundant with the checks that ApiUpload.php does with incoming
* uploads versus the desired filename. Maybe we can get that passed to us...
+ * @return string
*/
public static function getExtensionForPath( $path ) {
// Does this have an extension?
* @param $key String: key
* @return boolean
*/
- protected function fetchFileMetadata( $key ) {
+ protected function fetchFileMetadata( $key, $readFromDB = DB_SLAVE ) {
// populate $fileMetadata[$key]
- $dbr = $this->repo->getSlaveDb();
+ $dbr = null;
+ if( $readFromDB === DB_MASTER ) {
+ // sometimes reading from the master is necessary, if there's replication lag.
+ $dbr = $this->repo->getMasterDb();
+ } else {
+ $dbr = $this->repo->getSlaveDb();
+ }
+
$row = $dbr->selectRow(
'uploadstash',
'*',
return false;
}
- $this->fileMetadata[$key] = array(
- 'us_user' => $row->us_user,
- 'us_key' => $row->us_key,
- 'us_orig_path' => $row->us_orig_path,
- 'us_path' => $row->us_path,
- 'us_size' => $row->us_size,
- 'us_sha1' => $row->us_sha1,
- 'us_mime' => $row->us_mime,
- 'us_media_type' => $row->us_media_type,
- 'us_image_width' => $row->us_image_width,
- 'us_image_height' => $row->us_image_height,
- 'us_image_bits' => $row->us_image_bits,
- 'us_source_type' => $row->us_source_type,
- 'us_timestamp' => $row->us_timestamp,
- 'us_status' => $row->us_status
- );
+ $this->fileMetadata[$key] = (array)$row;
return true;
}
* A LocalFile wrapper around a file that has been temporarily stashed, so we can do things like create thumbnails for it
* Arguably UnregisteredLocalFile should be handling its own file repo but that class is a bit retarded currently
*
- * @param $repo FSRepo: repository where we should find the path
+ * @param $repo FileRepo: repository where we should find the path
* @param $path String: path to file
* @param $key String: key to store the path and any stashed data under
* @throws UploadStashBadPathException
}
// check if path exists! and is a plain file.
- if ( ! $repo->fileExists( $path, FileRepo::FILES_ONLY ) ) {
+ if ( ! $repo->fileExists( $path ) ) {
wfDebug( "UploadStash: tried to construct an UploadStashFile from a file that should already exist at '$path', but path is not found\n" );
throw new UploadStashFileNotFoundException( 'cannot find path, or not a plain file' );
}
* @return Status: success
*/
public function remove() {
- if ( !$this->repo->fileExists( $this->path, FileRepo::FILES_ONLY ) ) {
+ if ( !$this->repo->fileExists( $this->path ) ) {
// Maybe the file's already been removed? This could totally happen in UploadBase.
return true;
}
}
public function exists() {
- return $this->repo->fileExists( $this->path, FileRepo::FILES_ONLY );
+ return $this->repo->fileExists( $this->path );
}
}
class UploadStashZeroLengthFileException extends MWException {};
class UploadStashNotLoggedInException extends MWException {};
class UploadStashWrongOwnerException extends MWException {};
-class UploadStashMaxLagExceededException extends MWException {};
class UploadStashNoSuchKeyException extends MWException {};
dépôts / lhc / web / wiklou.git / blobdiff