From: jenkins-bot Date: Thu, 22 Sep 2016 03:30:06 +0000 (+0000) Subject: Merge "Move IP class to libs/" X-Git-Tag: 1.31.0-rc.0~5399 X-Git-Url: http://git.cyclocoop.org/%22%20.%20generer_url_ecrire%28%22articles%22%2C%22id_article=%24ze_article%22%29%20.%20%22?a=commitdiff_plain;h=e8985f2338469e5770f6a3f78ac7120b973b3284;hp=04dfd9b6bcfbcc23b2126e341155a961ac974d2b;p=lhc%2Fweb%2Fwiklou.git Merge "Move IP class to libs/" --- diff --git a/autoload.php b/autoload.php index 9e5bf04b78..efd34ead5c 100644 --- a/autoload.php +++ b/autoload.php @@ -432,7 +432,7 @@ $wgAutoloadLocalClasses = [ 'ExternalStoreHttp' => __DIR__ . '/includes/externalstore/ExternalStoreHttp.php', 'ExternalStoreMedium' => __DIR__ . '/includes/externalstore/ExternalStoreMedium.php', 'ExternalStoreMwstore' => __DIR__ . '/includes/externalstore/ExternalStoreMwstore.php', - 'FSFile' => __DIR__ . '/includes/filebackend/FSFile.php', + 'FSFile' => __DIR__ . '/includes/libs/filebackend/FSFile.php', 'FSFileBackend' => __DIR__ . '/includes/filebackend/FSFileBackend.php', 'FSFileBackendDirList' => __DIR__ . '/includes/filebackend/FSFileBackend.php', 'FSFileBackendFileList' => __DIR__ . '/includes/filebackend/FSFileBackend.php', @@ -1226,7 +1226,7 @@ $wgAutoloadLocalClasses = [ 'SQLiteField' => __DIR__ . '/includes/libs/rdbms/field/SQLiteField.php', 'SVGMetadataExtractor' => __DIR__ . '/includes/media/SVGMetadataExtractor.php', 'SVGReader' => __DIR__ . '/includes/media/SVGMetadataExtractor.php', - 'SamplingStatsdClient' => __DIR__ . '/includes/libs/SamplingStatsdClient.php', + 'SamplingStatsdClient' => __DIR__ . '/includes/libs/stats/SamplingStatsdClient.php', 'Sanitizer' => __DIR__ . '/includes/Sanitizer.php', 'SavepointPostgres' => __DIR__ . '/includes/libs/rdbms/database/utils/SavepointPostgres.php', 'ScopedCallback' => __DIR__ . '/includes/libs/ScopedCallback.php', @@ -1404,7 +1404,7 @@ $wgAutoloadLocalClasses = [ 'TableDiffFormatter' => __DIR__ . '/includes/diff/TableDiffFormatter.php', 'TablePager' => __DIR__ . '/includes/pager/TablePager.php', 'TagLogFormatter' => __DIR__ . '/includes/logging/TagLogFormatter.php', - 'TempFSFile' => __DIR__ . '/includes/filebackend/TempFSFile.php', + 'TempFSFile' => __DIR__ . '/includes/libs/filebackend/TempFSFile.php', 'TempFileRepo' => __DIR__ . '/includes/filerepo/FileRepo.php', 'TemplateParser' => __DIR__ . '/includes/TemplateParser.php', 'TemplatesOnThisPageFormatter' => __DIR__ . '/includes/TemplatesOnThisPageFormatter.php', diff --git a/includes/MimeMagic.php b/includes/MimeMagic.php index 5551865227..54d58d2d44 100644 --- a/includes/MimeMagic.php +++ b/includes/MimeMagic.php @@ -863,10 +863,8 @@ class MimeMagic { $mime = "application/x-opc+zip"; # TODO: remove the block below, as soon as improveTypeFromExtension is used everywhere if ( $ext !== true && $ext !== false ) { - /** This is the mode used by getPropsFromPath - * These MIME's are stored in the database, where we don't really want - * x-opc+zip, because we use it only for internal purposes - */ + // These MIME's are stored in the database, where we don't really want + // x-opc+zip, because we use it only for internal purposes if ( $this->isMatchingExtension( $ext, $mime ) ) { /* A known file extension for an OPC file, * find the proper mime type for that file extension diff --git a/includes/filebackend/FSFile.php b/includes/filebackend/FSFile.php deleted file mode 100644 index e7e2608ce8..0000000000 --- a/includes/filebackend/FSFile.php +++ /dev/null @@ -1,268 +0,0 @@ -path = $path; - } - - /** - * Returns the file system path - * - * @return string - */ - public function getPath() { - return $this->path; - } - - /** - * Checks if the file exists - * - * @return bool - */ - public function exists() { - return is_file( $this->path ); - } - - /** - * Get the file size in bytes - * - * @return int|bool - */ - public function getSize() { - return filesize( $this->path ); - } - - /** - * Get the file's last-modified timestamp - * - * @return string|bool TS_MW timestamp or false on failure - */ - public function getTimestamp() { - MediaWiki\suppressWarnings(); - $timestamp = filemtime( $this->path ); - MediaWiki\restoreWarnings(); - if ( $timestamp !== false ) { - $timestamp = wfTimestamp( TS_MW, $timestamp ); - } - - return $timestamp; - } - - /** - * Get an associative array containing information about - * a file with the given storage path. - * - * Resulting array fields include: - * - fileExists - * - size (filesize in bytes) - * - mime (as major/minor) - * - media_type (value to be used with the MEDIATYPE_xxx constants) - * - metadata (handler specific) - * - sha1 (in base 36) - * - width - * - height - * - bits (bitrate) - * - file-mime - * - major_mime - * - minor_mime - * - * @param string|bool $ext The file extension, or true to extract it from the filename. - * Set it to false to ignore the extension. - * @return array - */ - public function getProps( $ext = true ) { - $info = self::placeholderProps(); - $info['fileExists'] = $this->exists(); - - if ( $info['fileExists'] ) { - $info['size'] = $this->getSize(); // bytes - $info['sha1'] = $this->getSha1Base36(); - // @TODO: replace the code below with bare FileInfo use so this can go in /libs - $magic = MimeMagic::singleton(); - - # MIME type according to file contents - $info['file-mime'] = $magic->guessMimeType( $this->path, false ); - # Logical MIME type - $ext = ( $ext === true ) ? FileBackend::extensionFromPath( $this->path ) : $ext; - $info['mime'] = $magic->improveTypeFromExtension( $info['file-mime'], $ext ); - - list( $info['major_mime'], $info['minor_mime'] ) = File::splitMime( $info['mime'] ); - $info['media_type'] = $magic->getMediaType( $this->path, $info['mime'] ); - - # Height, width and metadata - $handler = MediaHandler::getHandler( $info['mime'] ); - if ( $handler ) { - $info['metadata'] = $handler->getMetadata( $this, $this->path ); - /** @noinspection PhpMethodParametersCountMismatchInspection */ - $gis = $handler->getImageSize( $this, $this->path, $info['metadata'] ); - if ( is_array( $gis ) ) { - $info = $this->extractImageSizeInfo( $gis ) + $info; - } - } - } - - return $info; - } - - /** - * Placeholder file properties to use for files that don't exist - * - * Resulting array fields include: - * - fileExists - * - size - * - file-mime (as major/minor) - * - mime (as major/minor) - * - major_mime - * - minor_mime - * - media_type (value to be used with the MEDIATYPE_xxx constants) - * - metadata (handler specific) - * - sha1 (in base 36) - * - width - * - height - * - bits (bitrate) - * - * @return array - */ - public static function placeholderProps() { - $info = []; - $info['fileExists'] = false; - $info['size'] = 0; - $info['file-mime'] = null; - $info['major_mime'] = null; - $info['minor_mime'] = null; - $info['mime'] = null; - $info['media_type'] = MEDIATYPE_UNKNOWN; - $info['metadata'] = ''; - $info['sha1'] = ''; - $info['width'] = 0; - $info['height'] = 0; - $info['bits'] = 0; - - return $info; - } - - /** - * Exract image size information - * - * @param array $gis - * @return array - */ - protected function extractImageSizeInfo( array $gis ) { - $info = []; - # NOTE: $gis[2] contains a code for the image type. This is no longer used. - $info['width'] = $gis[0]; - $info['height'] = $gis[1]; - if ( isset( $gis['bits'] ) ) { - $info['bits'] = $gis['bits']; - } else { - $info['bits'] = 0; - } - - return $info; - } - - /** - * Get a SHA-1 hash of a file in the local filesystem, in base-36 lower case - * encoding, zero padded to 31 digits. - * - * 160 log 2 / log 36 = 30.95, so the 160-bit hash fills 31 digits in base 36 - * fairly neatly. - * - * @param bool $recache - * @return bool|string False on failure - */ - public function getSha1Base36( $recache = false ) { - if ( $this->sha1Base36 !== null && !$recache ) { - return $this->sha1Base36; - } - - MediaWiki\suppressWarnings(); - $this->sha1Base36 = sha1_file( $this->path ); - MediaWiki\restoreWarnings(); - - if ( $this->sha1Base36 !== false ) { - $this->sha1Base36 = Wikimedia\base_convert( $this->sha1Base36, 16, 36, 31 ); - } - - return $this->sha1Base36; - } - - /** - * Get the final file extension from a file system path - * - * @param string $path - * @return string - */ - public static function extensionFromPath( $path ) { - $i = strrpos( $path, '.' ); - - return strtolower( $i ? substr( $path, $i + 1 ) : '' ); - } - - /** - * Get an associative array containing information about a file in the local filesystem. - * - * @param string $path Absolute local filesystem path - * @param string|bool $ext The file extension, or true to extract it from the filename. - * Set it to false to ignore the extension. - * @return array - */ - public static function getPropsFromPath( $path, $ext = true ) { - $fsFile = new self( $path ); - - return $fsFile->getProps( $ext ); - } - - /** - * Get a SHA-1 hash of a file in the local filesystem, in base-36 lower case - * encoding, zero padded to 31 digits. - * - * 160 log 2 / log 36 = 30.95, so the 160-bit hash fills 31 digits in base 36 - * fairly neatly. - * - * @param string $path - * @return bool|string False on failure - */ - public static function getSha1Base36FromPath( $path ) { - $fsFile = new self( $path ); - - return $fsFile->getSha1Base36(); - } -} diff --git a/includes/filebackend/TempFSFile.php b/includes/filebackend/TempFSFile.php deleted file mode 100644 index fed6812f5b..0000000000 --- a/includes/filebackend/TempFSFile.php +++ /dev/null @@ -1,196 +0,0 @@ - 1) for paths to delete on shutdown */ - protected static $pathsCollect = null; - - public function __construct( $path ) { - parent::__construct( $path ); - - if ( self::$pathsCollect === null ) { - self::$pathsCollect = []; - register_shutdown_function( [ __CLASS__, 'purgeAllOnShutdown' ] ); - } - } - - /** - * Make a new temporary file on the file system. - * Temporary files may be purged when the file object falls out of scope. - * - * @param string $prefix - * @param string $extension Optional file extension - * @param string|null $tmpDirectory Optional parent directory - * @return TempFSFile|null - */ - public static function factory( $prefix, $extension = '', $tmpDirectory = null ) { - $ext = ( $extension != '' ) ? ".{$extension}" : ''; - - $attempts = 5; - while ( $attempts-- ) { - $hex = sprintf( '%06x%06x', mt_rand( 0, 0xffffff ), mt_rand( 0, 0xffffff ) ); - if ( !is_string( $tmpDirectory ) ) { - $tmpDirectory = self::getUsableTempDirectory(); - } - $path = wfTempDir() . '/' . $prefix . $hex . $ext; - MediaWiki\suppressWarnings(); - $newFileHandle = fopen( $path, 'x' ); - MediaWiki\restoreWarnings(); - if ( $newFileHandle ) { - fclose( $newFileHandle ); - $tmpFile = new self( $path ); - $tmpFile->autocollect(); - // Safely instantiated, end loop. - return $tmpFile; - } - } - - // Give up - return null; - } - - /** - * @return string Filesystem path to a temporary directory - * @throws RuntimeException - */ - public static function getUsableTempDirectory() { - $tmpDir = array_map( 'getenv', [ 'TMPDIR', 'TMP', 'TEMP' ] ); - $tmpDir[] = sys_get_temp_dir(); - $tmpDir[] = ini_get( 'upload_tmp_dir' ); - foreach ( $tmpDir as $tmp ) { - if ( $tmp != '' && is_dir( $tmp ) && is_writable( $tmp ) ) { - return $tmp; - } - } - - // PHP on Windows will detect C:\Windows\Temp as not writable even though PHP can write to - // it so create a directory within that called 'mwtmp' with a suffix of the user running - // the current process. - // The user is included as if various scripts are run by different users they will likely - // not be able to access each others temporary files. - if ( strtoupper( substr( PHP_OS, 0, 3 ) ) === 'WIN' ) { - $tmp = sys_get_temp_dir() . DIRECTORY_SEPARATOR . 'mwtmp-' . get_current_user(); - if ( !file_exists( $tmp ) ) { - mkdir( $tmp ); - } - if ( is_dir( $tmp ) && is_writable( $tmp ) ) { - return $tmp; - } - } - - throw new RuntimeException( - 'No writable temporary directory could be found. ' . - 'Please explicitly specify a writable directory in configuration.' ); - } - - /** - * Purge this file off the file system - * - * @return bool Success - */ - public function purge() { - $this->canDelete = false; // done - MediaWiki\suppressWarnings(); - $ok = unlink( $this->path ); - MediaWiki\restoreWarnings(); - - unset( self::$pathsCollect[$this->path] ); - - return $ok; - } - - /** - * Clean up the temporary file only after an object goes out of scope - * - * @param object $object - * @return TempFSFile This object - */ - public function bind( $object ) { - if ( is_object( $object ) ) { - if ( !isset( $object->tempFSFileReferences ) ) { - // Init first since $object might use __get() and return only a copy variable - $object->tempFSFileReferences = []; - } - $object->tempFSFileReferences[] = $this; - } - - return $this; - } - - /** - * Set flag to not clean up after the temporary file - * - * @return TempFSFile This object - */ - public function preserve() { - $this->canDelete = false; - - unset( self::$pathsCollect[$this->path] ); - - return $this; - } - - /** - * Set flag clean up after the temporary file - * - * @return TempFSFile This object - */ - public function autocollect() { - $this->canDelete = true; - - self::$pathsCollect[$this->path] = 1; - - return $this; - } - - /** - * Try to make sure that all files are purged on error - * - * This method should only be called internally - */ - public static function purgeAllOnShutdown() { - foreach ( self::$pathsCollect as $path ) { - MediaWiki\suppressWarnings(); - unlink( $path ); - MediaWiki\restoreWarnings(); - } - } - - /** - * Cleans up after the temporary file by deleting it - */ - function __destruct() { - if ( $this->canDelete ) { - $this->purge(); - } - } -} diff --git a/includes/filerepo/FileRepo.php b/includes/filerepo/FileRepo.php index b5c5bf33fb..1a6c8180ef 100644 --- a/includes/filerepo/FileRepo.php +++ b/includes/filerepo/FileRepo.php @@ -1540,11 +1540,11 @@ class FileRepo { */ public function getFileProps( $virtualUrl ) { $fsFile = $this->getLocalReference( $virtualUrl ); + $mwProps = new MWFileProps( MimeMagic::singleton() ); if ( $fsFile ) { - $mwProps = new MWFileProps( MimeMagic::singleton() ); $props = $mwProps->getPropsFromPath( $fsFile->getPath(), true ); } else { - $props = FSFile::placeholderProps(); + $props = $mwProps->newPlaceholderProps(); } return $props; diff --git a/includes/libs/SamplingStatsdClient.php b/includes/libs/SamplingStatsdClient.php deleted file mode 100644 index dd1976c06e..0000000000 --- a/includes/libs/SamplingStatsdClient.php +++ /dev/null @@ -1,157 +0,0 @@ -samplingRates = $samplingRates; - } - - /** - * Sets sampling rate for all items in $data. - * The sample rate specified in a StatsdData entity overrides the sample rate specified here. - * - * {@inheritDoc} - */ - public function appendSampleRate( $data, $sampleRate = 1 ) { - $samplingRates = $this->samplingRates; - if ( !$samplingRates && $sampleRate !== 1 ) { - $samplingRates = [ '*' => $sampleRate ]; - } - if ( $samplingRates ) { - array_walk( $data, function( $item ) use ( $samplingRates ) { - /** @var $item StatsdData */ - foreach ( $samplingRates as $pattern => $rate ) { - if ( fnmatch( $pattern, $item->getKey(), FNM_NOESCAPE ) ) { - $item->setSampleRate( $item->getSampleRate() * $rate ); - break; - } - } - } ); - } - - return $data; - } - - /* - * Send the metrics over UDP - * Sample the metrics according to their sample rate and send the remaining ones. - * - * @param StatsdDataInterface|StatsdDataInterface[] $data message(s) to sent - * strings are not allowed here as sampleData requires a StatsdDataInterface - * @param int $sampleRate - * - * @return integer the data sent in bytes - */ - public function send( $data, $sampleRate = 1 ) { - if ( !is_array( $data ) ) { - $data = [ $data ]; - } - if ( !$data ) { - return; - } - foreach ( $data as $item ) { - if ( !( $item instanceof StatsdDataInterface ) ) { - throw new InvalidArgumentException( - 'SamplingStatsdClient does not accept stringified messages' ); - } - } - - // add sampling - $data = $this->appendSampleRate( $data, $sampleRate ); - $data = $this->sampleData( $data ); - - $data = array_map( 'strval', $data ); - - // reduce number of packets - if ( $this->getReducePacket() ) { - $data = $this->reduceCount( $data ); - } - - // failures in any of this should be silently ignored if .. - $written = 0; - try { - $fp = $this->getSender()->open(); - if ( !$fp ) { - return; - } - foreach ( $data as $message ) { - $written += $this->getSender()->write( $fp, $message ); - } - $this->getSender()->close( $fp ); - } catch ( Exception $e ) { - $this->throwException( $e ); - } - - return $written; - } - - /** - * Throw away some of the data according to the sample rate. - * @param StatsdDataInterface[] $data - * @return StatsdDataInterface[] - * @throws LogicException - */ - protected function sampleData( $data ) { - $newData = []; - $mt_rand_max = mt_getrandmax(); - foreach ( $data as $item ) { - $samplingRate = $item->getSampleRate(); - if ( $samplingRate <= 0.0 || $samplingRate > 1.0 ) { - throw new LogicException( 'Sampling rate shall be within ]0, 1]' ); - } - if ( - $samplingRate === 1 || - ( mt_rand() / $mt_rand_max <= $samplingRate ) - ) { - $newData[] = $item; - } - } - return $newData; - } - - /** - * {@inheritDoc} - */ - protected function throwException( Exception $exception ) { - if ( !$this->getFailSilently() ) { - throw $exception; - } - } -} diff --git a/includes/libs/filebackend/FSFile.php b/includes/libs/filebackend/FSFile.php new file mode 100644 index 0000000000..d0e93da7ba --- /dev/null +++ b/includes/libs/filebackend/FSFile.php @@ -0,0 +1,243 @@ +path = $path; + } + + /** + * Returns the file system path + * + * @return string + */ + public function getPath() { + return $this->path; + } + + /** + * Checks if the file exists + * + * @return bool + */ + public function exists() { + return is_file( $this->path ); + } + + /** + * Get the file size in bytes + * + * @return int|bool + */ + public function getSize() { + return filesize( $this->path ); + } + + /** + * Get the file's last-modified timestamp + * + * @return string|bool TS_MW timestamp or false on failure + */ + public function getTimestamp() { + MediaWiki\suppressWarnings(); + $timestamp = filemtime( $this->path ); + MediaWiki\restoreWarnings(); + if ( $timestamp !== false ) { + $timestamp = wfTimestamp( TS_MW, $timestamp ); + } + + return $timestamp; + } + + /** + * Get an associative array containing information about + * a file with the given storage path. + * + * Resulting array fields include: + * - fileExists + * - size (filesize in bytes) + * - mime (as major/minor) + * - file-mime (as major/minor) + * - sha1 (in base 36) + * - major_mime + * - minor_mime + * + * @param string|bool $ext The file extension, or true to extract it from the filename. + * Set it to false to ignore the extension. Currently unused. + * @return array + */ + public function getProps( $ext = true ) { + $info = self::placeholderProps(); + $info['fileExists'] = $this->exists(); + + if ( $info['fileExists'] ) { + $info['size'] = $this->getSize(); // bytes + $info['sha1'] = $this->getSha1Base36(); + + $mime = mime_content_type( $this->path ); + # MIME type according to file contents + $info['file-mime'] = ( $mime === false ) ? 'unknown/unknown' : $mime; + # logical MIME type + $info['mime'] = $mime; + + if ( strpos( $mime, '/' ) !== false ) { + list( $info['major_mime'], $info['minor_mime'] ) = explode( '/', $mime, 2 ); + } else { + list( $info['major_mime'], $info['minor_mime'] ) = [ $mime, 'unknown' ]; + } + } + + return $info; + } + + /** + * Placeholder file properties to use for files that don't exist + * + * Resulting array fields include: + * - fileExists + * - size (filesize in bytes) + * - mime (as major/minor) + * - file-mime (as major/minor) + * - sha1 (in base 36) + * - major_mime + * - minor_mime + * + * @return array + */ + public static function placeholderProps() { + $info = []; + $info['fileExists'] = false; + $info['size'] = 0; + $info['file-mime'] = null; + $info['major_mime'] = null; + $info['minor_mime'] = null; + $info['mime'] = null; + $info['sha1'] = ''; + + return $info; + } + + /** + * Exract image size information + * + * @param array $gis + * @return array + */ + protected function extractImageSizeInfo( array $gis ) { + $info = []; + # NOTE: $gis[2] contains a code for the image type. This is no longer used. + $info['width'] = $gis[0]; + $info['height'] = $gis[1]; + if ( isset( $gis['bits'] ) ) { + $info['bits'] = $gis['bits']; + } else { + $info['bits'] = 0; + } + + return $info; + } + + /** + * Get a SHA-1 hash of a file in the local filesystem, in base-36 lower case + * encoding, zero padded to 31 digits. + * + * 160 log 2 / log 36 = 30.95, so the 160-bit hash fills 31 digits in base 36 + * fairly neatly. + * + * @param bool $recache + * @return bool|string False on failure + */ + public function getSha1Base36( $recache = false ) { + if ( $this->sha1Base36 !== null && !$recache ) { + return $this->sha1Base36; + } + + MediaWiki\suppressWarnings(); + $this->sha1Base36 = sha1_file( $this->path ); + MediaWiki\restoreWarnings(); + + if ( $this->sha1Base36 !== false ) { + $this->sha1Base36 = Wikimedia\base_convert( $this->sha1Base36, 16, 36, 31 ); + } + + return $this->sha1Base36; + } + + /** + * Get the final file extension from a file system path + * + * @param string $path + * @return string + */ + public static function extensionFromPath( $path ) { + $i = strrpos( $path, '.' ); + + return strtolower( $i ? substr( $path, $i + 1 ) : '' ); + } + + /** + * Get an associative array containing information about a file in the local filesystem. + * + * @param string $path Absolute local filesystem path + * @param string|bool $ext The file extension, or true to extract it from the filename. + * Set it to false to ignore the extension. + * @return array + */ + public static function getPropsFromPath( $path, $ext = true ) { + $fsFile = new self( $path ); + + return $fsFile->getProps( $ext ); + } + + /** + * Get a SHA-1 hash of a file in the local filesystem, in base-36 lower case + * encoding, zero padded to 31 digits. + * + * 160 log 2 / log 36 = 30.95, so the 160-bit hash fills 31 digits in base 36 + * fairly neatly. + * + * @param string $path + * @return bool|string False on failure + */ + public static function getSha1Base36FromPath( $path ) { + $fsFile = new self( $path ); + + return $fsFile->getSha1Base36(); + } +} diff --git a/includes/libs/filebackend/TempFSFile.php b/includes/libs/filebackend/TempFSFile.php new file mode 100644 index 0000000000..fed6812f5b --- /dev/null +++ b/includes/libs/filebackend/TempFSFile.php @@ -0,0 +1,196 @@ + 1) for paths to delete on shutdown */ + protected static $pathsCollect = null; + + public function __construct( $path ) { + parent::__construct( $path ); + + if ( self::$pathsCollect === null ) { + self::$pathsCollect = []; + register_shutdown_function( [ __CLASS__, 'purgeAllOnShutdown' ] ); + } + } + + /** + * Make a new temporary file on the file system. + * Temporary files may be purged when the file object falls out of scope. + * + * @param string $prefix + * @param string $extension Optional file extension + * @param string|null $tmpDirectory Optional parent directory + * @return TempFSFile|null + */ + public static function factory( $prefix, $extension = '', $tmpDirectory = null ) { + $ext = ( $extension != '' ) ? ".{$extension}" : ''; + + $attempts = 5; + while ( $attempts-- ) { + $hex = sprintf( '%06x%06x', mt_rand( 0, 0xffffff ), mt_rand( 0, 0xffffff ) ); + if ( !is_string( $tmpDirectory ) ) { + $tmpDirectory = self::getUsableTempDirectory(); + } + $path = wfTempDir() . '/' . $prefix . $hex . $ext; + MediaWiki\suppressWarnings(); + $newFileHandle = fopen( $path, 'x' ); + MediaWiki\restoreWarnings(); + if ( $newFileHandle ) { + fclose( $newFileHandle ); + $tmpFile = new self( $path ); + $tmpFile->autocollect(); + // Safely instantiated, end loop. + return $tmpFile; + } + } + + // Give up + return null; + } + + /** + * @return string Filesystem path to a temporary directory + * @throws RuntimeException + */ + public static function getUsableTempDirectory() { + $tmpDir = array_map( 'getenv', [ 'TMPDIR', 'TMP', 'TEMP' ] ); + $tmpDir[] = sys_get_temp_dir(); + $tmpDir[] = ini_get( 'upload_tmp_dir' ); + foreach ( $tmpDir as $tmp ) { + if ( $tmp != '' && is_dir( $tmp ) && is_writable( $tmp ) ) { + return $tmp; + } + } + + // PHP on Windows will detect C:\Windows\Temp as not writable even though PHP can write to + // it so create a directory within that called 'mwtmp' with a suffix of the user running + // the current process. + // The user is included as if various scripts are run by different users they will likely + // not be able to access each others temporary files. + if ( strtoupper( substr( PHP_OS, 0, 3 ) ) === 'WIN' ) { + $tmp = sys_get_temp_dir() . DIRECTORY_SEPARATOR . 'mwtmp-' . get_current_user(); + if ( !file_exists( $tmp ) ) { + mkdir( $tmp ); + } + if ( is_dir( $tmp ) && is_writable( $tmp ) ) { + return $tmp; + } + } + + throw new RuntimeException( + 'No writable temporary directory could be found. ' . + 'Please explicitly specify a writable directory in configuration.' ); + } + + /** + * Purge this file off the file system + * + * @return bool Success + */ + public function purge() { + $this->canDelete = false; // done + MediaWiki\suppressWarnings(); + $ok = unlink( $this->path ); + MediaWiki\restoreWarnings(); + + unset( self::$pathsCollect[$this->path] ); + + return $ok; + } + + /** + * Clean up the temporary file only after an object goes out of scope + * + * @param object $object + * @return TempFSFile This object + */ + public function bind( $object ) { + if ( is_object( $object ) ) { + if ( !isset( $object->tempFSFileReferences ) ) { + // Init first since $object might use __get() and return only a copy variable + $object->tempFSFileReferences = []; + } + $object->tempFSFileReferences[] = $this; + } + + return $this; + } + + /** + * Set flag to not clean up after the temporary file + * + * @return TempFSFile This object + */ + public function preserve() { + $this->canDelete = false; + + unset( self::$pathsCollect[$this->path] ); + + return $this; + } + + /** + * Set flag clean up after the temporary file + * + * @return TempFSFile This object + */ + public function autocollect() { + $this->canDelete = true; + + self::$pathsCollect[$this->path] = 1; + + return $this; + } + + /** + * Try to make sure that all files are purged on error + * + * This method should only be called internally + */ + public static function purgeAllOnShutdown() { + foreach ( self::$pathsCollect as $path ) { + MediaWiki\suppressWarnings(); + unlink( $path ); + MediaWiki\restoreWarnings(); + } + } + + /** + * Cleans up after the temporary file by deleting it + */ + function __destruct() { + if ( $this->canDelete ) { + $this->purge(); + } + } +} diff --git a/includes/libs/stats/SamplingStatsdClient.php b/includes/libs/stats/SamplingStatsdClient.php new file mode 100644 index 0000000000..dd1976c06e --- /dev/null +++ b/includes/libs/stats/SamplingStatsdClient.php @@ -0,0 +1,157 @@ +samplingRates = $samplingRates; + } + + /** + * Sets sampling rate for all items in $data. + * The sample rate specified in a StatsdData entity overrides the sample rate specified here. + * + * {@inheritDoc} + */ + public function appendSampleRate( $data, $sampleRate = 1 ) { + $samplingRates = $this->samplingRates; + if ( !$samplingRates && $sampleRate !== 1 ) { + $samplingRates = [ '*' => $sampleRate ]; + } + if ( $samplingRates ) { + array_walk( $data, function( $item ) use ( $samplingRates ) { + /** @var $item StatsdData */ + foreach ( $samplingRates as $pattern => $rate ) { + if ( fnmatch( $pattern, $item->getKey(), FNM_NOESCAPE ) ) { + $item->setSampleRate( $item->getSampleRate() * $rate ); + break; + } + } + } ); + } + + return $data; + } + + /* + * Send the metrics over UDP + * Sample the metrics according to their sample rate and send the remaining ones. + * + * @param StatsdDataInterface|StatsdDataInterface[] $data message(s) to sent + * strings are not allowed here as sampleData requires a StatsdDataInterface + * @param int $sampleRate + * + * @return integer the data sent in bytes + */ + public function send( $data, $sampleRate = 1 ) { + if ( !is_array( $data ) ) { + $data = [ $data ]; + } + if ( !$data ) { + return; + } + foreach ( $data as $item ) { + if ( !( $item instanceof StatsdDataInterface ) ) { + throw new InvalidArgumentException( + 'SamplingStatsdClient does not accept stringified messages' ); + } + } + + // add sampling + $data = $this->appendSampleRate( $data, $sampleRate ); + $data = $this->sampleData( $data ); + + $data = array_map( 'strval', $data ); + + // reduce number of packets + if ( $this->getReducePacket() ) { + $data = $this->reduceCount( $data ); + } + + // failures in any of this should be silently ignored if .. + $written = 0; + try { + $fp = $this->getSender()->open(); + if ( !$fp ) { + return; + } + foreach ( $data as $message ) { + $written += $this->getSender()->write( $fp, $message ); + } + $this->getSender()->close( $fp ); + } catch ( Exception $e ) { + $this->throwException( $e ); + } + + return $written; + } + + /** + * Throw away some of the data according to the sample rate. + * @param StatsdDataInterface[] $data + * @return StatsdDataInterface[] + * @throws LogicException + */ + protected function sampleData( $data ) { + $newData = []; + $mt_rand_max = mt_getrandmax(); + foreach ( $data as $item ) { + $samplingRate = $item->getSampleRate(); + if ( $samplingRate <= 0.0 || $samplingRate > 1.0 ) { + throw new LogicException( 'Sampling rate shall be within ]0, 1]' ); + } + if ( + $samplingRate === 1 || + ( mt_rand() / $mt_rand_max <= $samplingRate ) + ) { + $newData[] = $item; + } + } + return $newData; + } + + /** + * {@inheritDoc} + */ + protected function throwException( Exception $exception ) { + if ( !$this->getFailSilently() ) { + throw $exception; + } + } +} diff --git a/includes/utils/MWFileProps.php b/includes/utils/MWFileProps.php index b8ecd4c0c3..e60b9ab762 100644 --- a/includes/utils/MWFileProps.php +++ b/includes/utils/MWFileProps.php @@ -63,7 +63,7 @@ class MWFileProps { public function getPropsFromPath( $path, $ext ) { $fsFile = new FSFile( $path ); - $info = FSFile::placeholderProps(); + $info = $this->newPlaceholderProps(); $info['fileExists'] = $fsFile->exists(); if ( $info['fileExists'] ) { $info['size'] = $fsFile->getSize(); // bytes @@ -112,4 +112,34 @@ class MWFileProps { return $info; } + + /** + * Empty place holder props for non-existing files + * + * Resulting array fields include: + * - fileExists + * - size (filesize in bytes) + * - mime (as major/minor) + * - media_type (value to be used with the MEDIATYPE_xxx constants) + * - metadata (handler specific) + * - sha1 (in base 36) + * - width + * - height + * - bits (bitrate) + * - file-mime + * - major_mime + * - minor_mime + * + * @return array + * @since 1.28 + */ + public function newPlaceholderProps() { + return FSFile::placeholderProps() + [ + 'metadata' => '', + 'width' => 0, + 'height' => 0, + 'bits' => 0, + 'media_type' => MEDIATYPE_UNKNOWN + ]; + } } diff --git a/maintenance/refreshLinks.php b/maintenance/refreshLinks.php index 106be1f132..e7a4d06902 100644 --- a/maintenance/refreshLinks.php +++ b/maintenance/refreshLinks.php @@ -90,7 +90,7 @@ class RefreshLinks extends Maintenance { $end = null, $redirectsOnly = false, $oldRedirectsOnly = false ) { $reportingInterval = 100; - $dbr = $this->getDB( DB_REPLICA ); + $dbr = $this->getDB( DB_REPLICA, [ 'vslow' ] ); if ( $start === null ) { $start = 1; @@ -282,7 +282,7 @@ class RefreshLinks extends Maintenance { ) { wfWaitForSlaves(); $this->output( "Deleting illegal entries from the links tables...\n" ); - $dbr = $this->getDB( DB_REPLICA ); + $dbr = $this->getDB( DB_REPLICA, [ 'vslow' ] ); do { // Find the start of the next chunk. This is based only // on existent page_ids. @@ -324,7 +324,7 @@ class RefreshLinks extends Maintenance { */ private function dfnCheckInterval( $start = null, $end = null, $batchSize = 100 ) { $dbw = $this->getDB( DB_MASTER ); - $dbr = $this->getDB( DB_REPLICA ); + $dbr = $this->getDB( DB_REPLICA, [ 'vslow' ] ); $linksTables = [ // table name => page_id field 'pagelinks' => 'pl_from',