From 623d7789025cd8ca3c1af43622e6443ba53c4465 Mon Sep 17 00:00:00 2001 From: Gilles Dubuc Date: Fri, 29 Aug 2014 16:50:18 +0200 Subject: [PATCH] Add ability to pre-render thumbnails at upload time This introduces an option that allows one to define a list of thumbnail sizes to be rendered by async jobs at upload time. Change-Id: Ida193699738c79aca333fa78b8b536d14a410841 Mingle: https://wikimedia.mingle.thoughtworks.com/projects/multimedia/cards/301 --- includes/AutoLoader.php | 1 + includes/DefaultSettings.php | 41 +++++++ includes/jobqueue/jobs/ThumbnailRenderJob.php | 101 ++++++++++++++++++ includes/upload/UploadBase.php | 22 ++++ 4 files changed, 165 insertions(+) create mode 100644 includes/jobqueue/jobs/ThumbnailRenderJob.php diff --git a/includes/AutoLoader.php b/includes/AutoLoader.php index 04802f9e97..8e08335d73 100644 --- a/includes/AutoLoader.php +++ b/includes/AutoLoader.php @@ -647,6 +647,7 @@ $wgAutoloadLocalClasses = array( 'UploadFromUrlJob' => 'includes/jobqueue/jobs/UploadFromUrlJob.php', 'AssembleUploadChunksJob' => 'includes/jobqueue/jobs/AssembleUploadChunksJob.php', 'PublishStashedFileJob' => 'includes/jobqueue/jobs/PublishStashedFileJob.php', + 'ThumbnailRenderJob' => 'includes/jobqueue/jobs/ThumbnailRenderJob.php', # includes/jobqueue/utils 'BacklinkJobUtils' => 'includes/jobqueue/utils/BacklinkJobUtils.php', diff --git a/includes/DefaultSettings.php b/includes/DefaultSettings.php index 5fc73776a9..245c87b826 100644 --- a/includes/DefaultSettings.php +++ b/includes/DefaultSettings.php @@ -1242,6 +1242,46 @@ $wgThumbnailBuckets = null; */ $wgThumbnailMinimumBucketDistance = 50; +/** + * When defined, is an array of thumbnail widths to be rendered at upload time. The idea is to + * prerender common thumbnail sizes, in order to avoid the necessity to render them on demand, which + * has a performance impact for the first client to view a certain size. + * + * This obviously means that more disk space is needed per upload upfront. + * + * @since 1.24 + */ + +$wgUploadThumbnailRenderMap = array(); + +/** + * The method through which the thumbnails will be prerendered for the entries in + * $wgUploadThumbnailRenderMap + * + * The method can be either "http" or "jobqueue". The former uses an http request to hit the + * thumbnail's URL. + * This method only works if thumbnails are configured to be rendered by a 404 handler. The latter + * option uses the job queue to render the thumbnail. + * + * @since 1.24 + */ +$wgUploadThumbnailRenderMethod = 'jobqueue'; + +/** + * When using the "http" wgUploadThumbnailRenderMethod, lets one specify a custom Host HTTP header. + * + * @since 1.24 + */ +$wgUploadThumbnailRenderHttpCustomHost = false; + +/** + * When using the "http" wgUploadThumbnailRenderMethod, lets one specify a custom domain to send the + * HTTP request to. + * + * @since 1.24 + */ +$wgUploadThumbnailRenderHttpCustomDomain = false; + /** * Default parameters for the "" tag */ @@ -6291,6 +6331,7 @@ $wgJobClasses = array( 'uploadFromUrl' => 'UploadFromUrlJob', 'AssembleUploadChunks' => 'AssembleUploadChunksJob', 'PublishStashedFile' => 'PublishStashedFileJob', + 'ThumbnailRender' => 'ThumbnailRenderJob', 'null' => 'NullJob' ); diff --git a/includes/jobqueue/jobs/ThumbnailRenderJob.php b/includes/jobqueue/jobs/ThumbnailRenderJob.php new file mode 100644 index 0000000000..ec68d7ef4e --- /dev/null +++ b/includes/jobqueue/jobs/ThumbnailRenderJob.php @@ -0,0 +1,101 @@ +params['transformParams']; + + $file = wfLocalFile( $this->title ); + + if ( $file && $file->exists() ) { + if ( $wgUploadThumbnailRenderMethod === 'jobqueue' ) { + $thumb = $file->transform( $transformParams, File::RENDER_NOW ); + + if ( $thumb && !$thumb->isError() ) { + return true; + } else { + $this->setLastError( __METHOD__ . ': thumbnail couln\'t be generated' ); + return false; + } + } elseif ( $wgUploadThumbnailRenderMethod === 'http' ) { + $status = $this->hitThumbUrl( $file, $transformParams ); + + wfDebug( __METHOD__ . ": received status {$status}\n" ); + + if ( $status === 200 || $status === 301 || $status === 302 ) { + return true; + } elseif ( $status ) { + // Note that this currently happens (500) when requesting sizes larger then or + // equal to the original, which is harmless. + $this->setLastError( __METHOD__ . ': incorrect HTTP status ' . $status ); + return false; + } else { + $this->setLastError( __METHOD__ . ': HTTP request failure' ); + return false; + } + } else { + $this->setLastError( __METHOD__ . ': unknown thumbnail render method ' . $wgUploadThumbnailRenderMethod ); + return false; + } + } else { + $this->setLastError( __METHOD__ . ': file doesn\'t exist' ); + return false; + } + } + + protected function hitThumbUrl( $file, $transformParams ) { + global $wgUploadThumbnailRenderHttpCustomHost, $wgUploadThumbnailRenderHttpCustomDomain; + + $thumbName = $file->thumbName( $transformParams ); + $thumbUrl = $file->getThumbUrl( $thumbName ); + + if ( $wgUploadThumbnailRenderHttpCustomDomain ) { + $thumbUrl = '//' . $wgUploadThumbnailRenderHttpCustomDomain . parse_url( $thumbUrl, PHP_URL_PATH ); + } + + wfDebug( __METHOD__ . ": hitting url {$thumbUrl}\n" ); + + $request = MWHttpRequest::factory( $thumbUrl, array( + 'method' => 'HEAD', + 'followRedirects' => true ) ); + + if ( $wgUploadThumbnailRenderHttpCustomHost ) { + $request->setHeader( 'Host', $wgUploadThumbnailRenderHttpCustomHost ); + } + + $status = $request->execute(); + + return $request->getStatus(); + } +} diff --git a/includes/upload/UploadBase.php b/includes/upload/UploadBase.php index 5de543e0ee..369fcd1cd4 100644 --- a/includes/upload/UploadBase.php +++ b/includes/upload/UploadBase.php @@ -746,6 +746,8 @@ abstract class UploadBase { ); } wfRunHooks( 'UploadComplete', array( &$this ) ); + + $this->postProcessUpload(); } wfProfileOut( __METHOD__ ); @@ -753,6 +755,26 @@ abstract class UploadBase { return $status; } + /** + * Perform extra steps after a successful upload. + */ + public function postProcessUpload() { + global $wgUploadThumbnailRenderMap; + + $jobs = array(); + + $sizes = $wgUploadThumbnailRenderMap; + rsort( $sizes ); + + foreach ( $sizes as $size ) { + $jobs []= new ThumbnailRenderJob( $this->getLocalFile()->getTitle(), array( + 'transformParams' => array( 'width' => $size ), + ) ); + } + + JobQueueGroup::singleton()->push( $jobs ); + } + /** * Returns the title of the file to be uploaded. Sets mTitleError in case * the name was illegal. -- 2.20.1