3 * Foreign repository accessible through api.php requests.
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License along
16 * with this program; if not, write to the Free Software Foundation, Inc.,
17 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18 * http://www.gnu.org/copyleft/gpl.html
24 use MediaWiki\Logger\LoggerFactory
;
27 * A foreign repository with a remote MediaWiki with an API thingy
31 * $wgForeignFileRepos[] = [
32 * 'class' => ForeignAPIRepo::class,
34 * 'apibase' => 'https://en.wikipedia.org/w/api.php',
35 * 'fetchDescription' => true, // Optional
36 * 'descriptionCacheExpiry' => 3600,
41 class ForeignAPIRepo
extends FileRepo
{
42 /* This version string is used in the user agent for requests and will help
43 * server maintainers in identify ForeignAPI usage.
44 * Update the version every time you make breaking or significant changes. */
45 const VERSION
= "2.1";
48 * List of iiprop values for the thumbnail fetch queries.
51 protected static $imageInfoProps = [
56 protected $fileFactory = [ ForeignAPIFile
::class, 'newFromTitle' ];
57 /** @var int Check back with Commons after this expiry */
58 protected $apiThumbCacheExpiry = 86400; // 1 day (24*3600)
60 /** @var int Redownload thumbnail files after this expiry */
61 protected $fileCacheExpiry = 2592000; // 1 month (30*24*3600)
64 protected $mFileExists = [];
70 * @param array|null $info
72 function __construct( $info ) {
73 global $wgLocalFileRepo;
74 parent
::__construct( $info );
76 // https://commons.wikimedia.org/w/api.php
77 $this->mApiBase
= $info['apibase'] ??
null;
79 if ( isset( $info['apiThumbCacheExpiry'] ) ) {
80 $this->apiThumbCacheExpiry
= $info['apiThumbCacheExpiry'];
82 if ( isset( $info['fileCacheExpiry'] ) ) {
83 $this->fileCacheExpiry
= $info['fileCacheExpiry'];
85 if ( !$this->scriptDirUrl
) {
86 // hack for description fetches
87 $this->scriptDirUrl
= dirname( $this->mApiBase
);
89 // If we can cache thumbs we can guess sane defaults for these
90 if ( $this->canCacheThumbs() && !$this->url
) {
91 $this->url
= $wgLocalFileRepo['url'];
93 if ( $this->canCacheThumbs() && !$this->thumbUrl
) {
94 $this->thumbUrl
= $this->url
. '/thumb';
102 function getApiUrl() {
103 return $this->mApiBase
;
107 * Per docs in FileRepo, this needs to return false if we don't support versioned
108 * files. Well, we don't.
110 * @param Title $title
111 * @param string|bool $time
114 function newFile( $title, $time = false ) {
119 return parent
::newFile( $title, $time );
123 * @param string[] $files
126 function fileExistsBatch( array $files ) {
128 foreach ( $files as $k => $f ) {
129 if ( isset( $this->mFileExists
[$f] ) ) {
130 $results[$k] = $this->mFileExists
[$f];
132 } elseif ( self
::isVirtualUrl( $f ) ) {
133 # @todo FIXME: We need to be able to handle virtual
134 # URLs better, at least when we know they refer to the
136 $results[$k] = false;
138 } elseif ( FileBackend
::isStoragePath( $f ) ) {
139 $results[$k] = false;
141 wfWarn( "Got mwstore:// path '$f'." );
145 $data = $this->fetchImageQuery( [
146 'titles' => implode( '|', $files ),
147 'prop' => 'imageinfo' ]
150 if ( isset( $data['query']['pages'] ) ) {
151 # First, get results from the query. Note we only care whether the image exists,
152 # not whether it has a description page.
153 foreach ( $data['query']['pages'] as $p ) {
154 $this->mFileExists
[$p['title']] = ( $p['imagerepository'] !== '' );
156 # Second, copy the results to any redirects that were queried
157 if ( isset( $data['query']['redirects'] ) ) {
158 foreach ( $data['query']['redirects'] as $r ) {
159 $this->mFileExists
[$r['from']] = $this->mFileExists
[$r['to']];
162 # Third, copy the results to any non-normalized titles that were queried
163 if ( isset( $data['query']['normalized'] ) ) {
164 foreach ( $data['query']['normalized'] as $n ) {
165 $this->mFileExists
[$n['from']] = $this->mFileExists
[$n['to']];
168 # Finally, copy the results to the output
169 foreach ( $files as $key => $file ) {
170 $results[$key] = $this->mFileExists
[$file];
178 * @param string $virtualUrl
181 function getFileProps( $virtualUrl ) {
186 * @param array $query
189 function fetchImageQuery( $query ) {
190 global $wgLanguageCode;
192 $query = array_merge( $query,
196 'redirects' => 'true'
199 if ( !isset( $query['uselang'] ) ) { // uselang is unset or null
200 $query['uselang'] = $wgLanguageCode;
203 $data = $this->httpGetCached( 'Metadata', $query );
206 return FormatJson
::decode( $data, true );
216 function getImageInfo( $data ) {
217 if ( $data && isset( $data['query']['pages'] ) ) {
218 foreach ( $data['query']['pages'] as $info ) {
219 if ( isset( $info['imageinfo'][0] ) ) {
220 $return = $info['imageinfo'][0];
221 if ( isset( $info['pageid'] ) ) {
222 $return['pageid'] = $info['pageid'];
233 * @param string $hash
234 * @return ForeignAPIFile[]
236 function findBySha1( $hash ) {
237 $results = $this->fetchImageQuery( [
238 'aisha1base36' => $hash,
239 'aiprop' => ForeignAPIFile
::getProps(),
240 'list' => 'allimages',
243 if ( isset( $results['query']['allimages'] ) ) {
244 foreach ( $results['query']['allimages'] as $img ) {
245 // 1.14 was broken, doesn't return name attribute
246 if ( !isset( $img['name'] ) ) {
249 $ret[] = new ForeignAPIFile( Title
::makeTitle( NS_FILE
, $img['name'] ), $this, $img );
257 * @param string $name
260 * @param array|null &$result Output-only parameter, guaranteed to become an array
261 * @param string $otherParams
263 * @return string|false
265 function getThumbUrl( $name, $width = -1, $height = -1, &$result = null, $otherParams = '' ) {
266 $data = $this->fetchImageQuery( [
267 'titles' => 'File:' . $name,
268 'iiprop' => self
::getIIProps(),
269 'iiurlwidth' => $width,
270 'iiurlheight' => $height,
271 'iiurlparam' => $otherParams,
272 'prop' => 'imageinfo' ] );
273 $info = $this->getImageInfo( $data );
275 if ( $data && $info && isset( $info['thumburl'] ) ) {
276 wfDebug( __METHOD__
. " got remote thumb " . $info['thumburl'] . "\n" );
279 return $info['thumburl'];
286 * @param string $name
289 * @param string $otherParams
290 * @param string|null $lang Language code for language of error
291 * @return bool|MediaTransformError
294 function getThumbError( $name, $width = -1, $height = -1, $otherParams = '', $lang = null ) {
295 $data = $this->fetchImageQuery( [
296 'titles' => 'File:' . $name,
297 'iiprop' => self
::getIIProps(),
298 'iiurlwidth' => $width,
299 'iiurlheight' => $height,
300 'iiurlparam' => $otherParams,
301 'prop' => 'imageinfo',
304 $info = $this->getImageInfo( $data );
306 if ( $data && $info && isset( $info['thumberror'] ) ) {
307 wfDebug( __METHOD__
. " got remote thumb error " . $info['thumberror'] . "\n" );
309 return new MediaTransformError(
310 'thumbnail_error_remote',
313 $this->getDisplayName(),
314 $info['thumberror'] // already parsed message from foreign repo
322 * Return the imageurl from cache if possible
324 * If the url has been requested today, get it from cache
325 * Otherwise retrieve remote thumb url, check for local file.
327 * @param string $name Is a dbkey form of a title
330 * @param string $params Other rendering parameters (page number, etc)
331 * from handler's makeParamString.
332 * @return bool|string
334 function getThumbUrlFromCache( $name, $width, $height, $params = "" ) {
335 // We can't check the local cache using FileRepo functions because
336 // we override fileExistsBatch(). We have to use the FileBackend directly.
337 $backend = $this->getBackend(); // convenience
339 if ( !$this->canCacheThumbs() ) {
340 $result = null; // can't pass "null" by reference, but it's ok as default value
341 return $this->getThumbUrl( $name, $width, $height, $result, $params );
343 $key = $this->getLocalCacheKey( 'ForeignAPIRepo', 'ThumbUrl', $name );
344 $sizekey = "$width:$height:$params";
346 /* Get the array of urls that we already know */
347 $knownThumbUrls = $this->wanCache
->get( $key );
348 if ( !$knownThumbUrls ) {
349 /* No knownThumbUrls for this file */
350 $knownThumbUrls = [];
351 } elseif ( isset( $knownThumbUrls[$sizekey] ) ) {
352 wfDebug( __METHOD__
. ': Got thumburl from local cache: ' .
353 "{$knownThumbUrls[$sizekey]} \n" );
355 return $knownThumbUrls[$sizekey];
359 $foreignUrl = $this->getThumbUrl( $name, $width, $height, $metadata, $params );
361 if ( !$foreignUrl ) {
362 wfDebug( __METHOD__
. " Could not find thumburl\n" );
367 // We need the same filename as the remote one :)
368 $fileName = rawurldecode( pathinfo( $foreignUrl, PATHINFO_BASENAME
) );
369 if ( !$this->validateFilename( $fileName ) ) {
370 wfDebug( __METHOD__
. " The deduced filename $fileName is not safe\n" );
374 $localPath = $this->getZonePath( 'thumb' ) . "/" . $this->getHashPath( $name ) . $name;
375 $localFilename = $localPath . "/" . $fileName;
376 $localUrl = $this->getZoneUrl( 'thumb' ) . "/" . $this->getHashPath( $name ) .
377 rawurlencode( $name ) . "/" . rawurlencode( $fileName );
379 if ( $backend->fileExists( [ 'src' => $localFilename ] )
380 && isset( $metadata['timestamp'] )
382 wfDebug( __METHOD__
. " Thumbnail was already downloaded before\n" );
383 $modified = $backend->getFileTimestamp( [ 'src' => $localFilename ] );
384 $remoteModified = strtotime( $metadata['timestamp'] );
386 $diff = abs( $modified - $current );
387 if ( $remoteModified < $modified && $diff < $this->fileCacheExpiry
) {
388 /* Use our current and already downloaded thumbnail */
389 $knownThumbUrls[$sizekey] = $localUrl;
390 $this->wanCache
->set( $key, $knownThumbUrls, $this->apiThumbCacheExpiry
);
394 /* There is a new Commons file, or existing thumbnail older than a month */
397 $thumb = self
::httpGet( $foreignUrl, 'default', [], $mtime );
399 wfDebug( __METHOD__
. " Could not download thumb\n" );
404 # @todo FIXME: Delete old thumbs that aren't being used. Maintenance script?
405 $backend->prepare( [ 'dir' => dirname( $localFilename ) ] );
406 $params = [ 'dst' => $localFilename, 'content' => $thumb ];
407 if ( !$backend->quickCreate( $params )->isOK() ) {
408 wfDebug( __METHOD__
. " could not write to thumb path '$localFilename'\n" );
412 $knownThumbUrls[$sizekey] = $localUrl;
415 ?
$this->wanCache
->adaptiveTTL( $mtime, $this->apiThumbCacheExpiry
)
416 : $this->apiThumbCacheExpiry
;
417 $this->wanCache
->set( $key, $knownThumbUrls, $ttl );
418 wfDebug( __METHOD__
. " got local thumb $localUrl, saving to cache \n" );
424 * @see FileRepo::getZoneUrl()
425 * @param string $zone
426 * @param string|null $ext Optional file extension
429 function getZoneUrl( $zone, $ext = null ) {
434 return $this->thumbUrl
;
436 return parent
::getZoneUrl( $zone, $ext );
441 * Get the local directory corresponding to one of the basic zones
442 * @param string $zone
443 * @return bool|null|string
445 function getZonePath( $zone ) {
446 $supported = [ 'public', 'thumb' ];
447 if ( in_array( $zone, $supported ) ) {
448 return parent
::getZonePath( $zone );
455 * Are we locally caching the thumbnails?
458 public function canCacheThumbs() {
459 return ( $this->apiThumbCacheExpiry
> 0 );
463 * The user agent the ForeignAPIRepo will use.
466 public static function getUserAgent() {
467 return Http
::userAgent() . " ForeignAPIRepo/" . self
::VERSION
;
471 * Get information about the repo - overrides/extends the parent
472 * class's information.
477 $info = parent
::getInfo();
478 $info['apiurl'] = $this->getApiUrl();
483 'meta' => 'siteinfo',
484 'siprop' => 'general',
487 $data = $this->httpGetCached( 'SiteInfo', $query, 7200 );
490 $siteInfo = FormatJson
::decode( $data, true );
491 $general = $siteInfo['query']['general'];
493 $info['articlepath'] = $general['articlepath'];
494 $info['server'] = $general['server'];
496 if ( isset( $general['favicon'] ) ) {
497 $info['favicon'] = $general['favicon'];
505 * Like a Http:get request, but with custom User-Agent.
508 * @param string $timeout
509 * @param array $options
510 * @param int|bool &$mtime Resulting Last-Modified UNIX timestamp if received
511 * @return bool|string
513 public static function httpGet(
514 $url, $timeout = 'default', $options = [], &$mtime = false
516 $options['timeout'] = $timeout;
518 $url = wfExpandUrl( $url, PROTO_HTTP
);
519 wfDebug( "ForeignAPIRepo: HTTP GET: $url\n" );
520 $options['method'] = "GET";
522 if ( !isset( $options['timeout'] ) ) {
523 $options['timeout'] = 'default';
526 $req = MWHttpRequest
::factory( $url, $options, __METHOD__
);
527 $req->setUserAgent( self
::getUserAgent() );
528 $status = $req->execute();
530 if ( $status->isOK() ) {
531 $lmod = $req->getResponseHeader( 'Last-Modified' );
532 $mtime = $lmod ?
wfTimestamp( TS_UNIX
, $lmod ) : false;
534 return $req->getContent();
536 $logger = LoggerFactory
::getInstance( 'http' );
538 $status->getWikiText( false, false, 'en' ),
539 [ 'caller' => 'ForeignAPIRepo::httpGet' ]
550 protected static function getIIProps() {
551 return implode( '|', self
::$imageInfoProps );
555 * HTTP GET request to a mediawiki API (with caching)
556 * @param string $target Used in cache key creation, mostly
557 * @param array $query The query parameters for the API request
558 * @param int $cacheTTL Time to live for the memcached caching
559 * @return string|null
561 public function httpGetCached( $target, $query, $cacheTTL = 3600 ) {
562 if ( $this->mApiBase
) {
563 $url = wfAppendQuery( $this->mApiBase
, $query );
565 $url = $this->makeUrl( $query, 'api' );
568 return $this->wanCache
->getWithSetCallback(
569 $this->getLocalCacheKey( static::class, $target, md5( $url ) ),
571 function ( $curValue, &$ttl ) use ( $url ) {
572 $html = self
::httpGet( $url, 'default', [], $mtime );
573 if ( $html !== false ) {
574 $ttl = $mtime ?
$this->wanCache
->adaptiveTTL( $mtime, $ttl ) : $ttl;
576 $ttl = $this->wanCache
->adaptiveTTL( $mtime, $ttl );
577 $html = null; // caches negatives
582 [ 'pcTTL' => WANObjectCache
::TTL_PROC_LONG
]
587 * @param callable $callback
588 * @throws MWException
590 function enumFiles( $callback ) {
591 throw new MWException( 'enumFiles is not supported by ' . static::class );
595 * @throws MWException
597 protected function assertWritableRepo() {
598 throw new MWException( static::class . ': write operations are not supported.' );