3 * Foreign repository accessible through api.php requests.
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License along
16 * with this program; if not, write to the Free Software Foundation, Inc.,
17 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18 * http://www.gnu.org/copyleft/gpl.html
24 use MediaWiki\Logger\LoggerFactory
;
25 use MediaWiki\MediaWikiServices
;
28 * A foreign repository with a remote MediaWiki with an API thingy
32 * $wgForeignFileRepos[] = [
33 * 'class' => ForeignAPIRepo::class,
35 * 'apibase' => 'https://en.wikipedia.org/w/api.php',
36 * 'fetchDescription' => true, // Optional
37 * 'descriptionCacheExpiry' => 3600,
42 class ForeignAPIRepo
extends FileRepo
{
43 /* This version string is used in the user agent for requests and will help
44 * server maintainers in identify ForeignAPI usage.
45 * Update the version every time you make breaking or significant changes. */
46 const VERSION
= "2.1";
49 * List of iiprop values for the thumbnail fetch queries.
52 protected static $imageInfoProps = [
57 protected $fileFactory = [ ForeignAPIFile
::class, 'newFromTitle' ];
58 /** @var int Check back with Commons after this expiry */
59 protected $apiThumbCacheExpiry = 86400; // 1 day (24*3600)
61 /** @var int Redownload thumbnail files after this expiry */
62 protected $fileCacheExpiry = 2592000; // 1 month (30*24*3600)
65 protected $mFileExists = [];
71 * @param array|null $info
73 function __construct( $info ) {
74 global $wgLocalFileRepo;
75 parent
::__construct( $info );
77 // https://commons.wikimedia.org/w/api.php
78 $this->mApiBase
= isset( $info['apibase'] ) ?
$info['apibase'] : null;
80 if ( isset( $info['apiThumbCacheExpiry'] ) ) {
81 $this->apiThumbCacheExpiry
= $info['apiThumbCacheExpiry'];
83 if ( isset( $info['fileCacheExpiry'] ) ) {
84 $this->fileCacheExpiry
= $info['fileCacheExpiry'];
86 if ( !$this->scriptDirUrl
) {
87 // hack for description fetches
88 $this->scriptDirUrl
= dirname( $this->mApiBase
);
90 // If we can cache thumbs we can guess sane defaults for these
91 if ( $this->canCacheThumbs() && !$this->url
) {
92 $this->url
= $wgLocalFileRepo['url'];
94 if ( $this->canCacheThumbs() && !$this->thumbUrl
) {
95 $this->thumbUrl
= $this->url
. '/thumb';
103 function getApiUrl() {
104 return $this->mApiBase
;
108 * Per docs in FileRepo, this needs to return false if we don't support versioned
109 * files. Well, we don't.
111 * @param Title $title
112 * @param string|bool $time
115 function newFile( $title, $time = false ) {
120 return parent
::newFile( $title, $time );
124 * @param string[] $files
127 function fileExistsBatch( array $files ) {
129 foreach ( $files as $k => $f ) {
130 if ( isset( $this->mFileExists
[$f] ) ) {
131 $results[$k] = $this->mFileExists
[$f];
133 } elseif ( self
::isVirtualUrl( $f ) ) {
134 # @todo FIXME: We need to be able to handle virtual
135 # URLs better, at least when we know they refer to the
137 $results[$k] = false;
139 } elseif ( FileBackend
::isStoragePath( $f ) ) {
140 $results[$k] = false;
142 wfWarn( "Got mwstore:// path '$f'." );
146 $data = $this->fetchImageQuery( [
147 'titles' => implode( '|', $files ),
148 'prop' => 'imageinfo' ]
151 if ( isset( $data['query']['pages'] ) ) {
152 # First, get results from the query. Note we only care whether the image exists,
153 # not whether it has a description page.
154 foreach ( $data['query']['pages'] as $p ) {
155 $this->mFileExists
[$p['title']] = ( $p['imagerepository'] !== '' );
157 # Second, copy the results to any redirects that were queried
158 if ( isset( $data['query']['redirects'] ) ) {
159 foreach ( $data['query']['redirects'] as $r ) {
160 $this->mFileExists
[$r['from']] = $this->mFileExists
[$r['to']];
163 # Third, copy the results to any non-normalized titles that were queried
164 if ( isset( $data['query']['normalized'] ) ) {
165 foreach ( $data['query']['normalized'] as $n ) {
166 $this->mFileExists
[$n['from']] = $this->mFileExists
[$n['to']];
169 # Finally, copy the results to the output
170 foreach ( $files as $key => $file ) {
171 $results[$key] = $this->mFileExists
[$file];
179 * @param string $virtualUrl
182 function getFileProps( $virtualUrl ) {
187 * @param array $query
190 function fetchImageQuery( $query ) {
191 global $wgLanguageCode;
193 $query = array_merge( $query,
197 'redirects' => 'true'
200 if ( !isset( $query['uselang'] ) ) { // uselang is unset or null
201 $query['uselang'] = $wgLanguageCode;
204 $data = $this->httpGetCached( 'Metadata', $query );
207 return FormatJson
::decode( $data, true );
217 function getImageInfo( $data ) {
218 if ( $data && isset( $data['query']['pages'] ) ) {
219 foreach ( $data['query']['pages'] as $info ) {
220 if ( isset( $info['imageinfo'][0] ) ) {
221 $return = $info['imageinfo'][0];
222 if ( isset( $info['pageid'] ) ) {
223 $return['pageid'] = $info['pageid'];
234 * @param string $hash
235 * @return ForeignAPIFile[]
237 function findBySha1( $hash ) {
238 $results = $this->fetchImageQuery( [
239 'aisha1base36' => $hash,
240 'aiprop' => ForeignAPIFile
::getProps(),
241 'list' => 'allimages',
244 if ( isset( $results['query']['allimages'] ) ) {
245 foreach ( $results['query']['allimages'] as $img ) {
246 // 1.14 was broken, doesn't return name attribute
247 if ( !isset( $img['name'] ) ) {
250 $ret[] = new ForeignAPIFile( Title
::makeTitle( NS_FILE
, $img['name'] ), $this, $img );
258 * @param string $name
261 * @param array|null &$result Output-only parameter, guaranteed to become an array
262 * @param string $otherParams
264 * @return string|false
266 function getThumbUrl( $name, $width = -1, $height = -1, &$result = null, $otherParams = '' ) {
267 $data = $this->fetchImageQuery( [
268 'titles' => 'File:' . $name,
269 'iiprop' => self
::getIIProps(),
270 'iiurlwidth' => $width,
271 'iiurlheight' => $height,
272 'iiurlparam' => $otherParams,
273 'prop' => 'imageinfo' ] );
274 $info = $this->getImageInfo( $data );
276 if ( $data && $info && isset( $info['thumburl'] ) ) {
277 wfDebug( __METHOD__
. " got remote thumb " . $info['thumburl'] . "\n" );
280 return $info['thumburl'];
287 * @param string $name
290 * @param string $otherParams
291 * @param string|null $lang Language code for language of error
292 * @return bool|MediaTransformError
295 function getThumbError( $name, $width = -1, $height = -1, $otherParams = '', $lang = null ) {
296 $data = $this->fetchImageQuery( [
297 'titles' => 'File:' . $name,
298 'iiprop' => self
::getIIProps(),
299 'iiurlwidth' => $width,
300 'iiurlheight' => $height,
301 'iiurlparam' => $otherParams,
302 'prop' => 'imageinfo',
305 $info = $this->getImageInfo( $data );
307 if ( $data && $info && isset( $info['thumberror'] ) ) {
308 wfDebug( __METHOD__
. " got remote thumb error " . $info['thumberror'] . "\n" );
310 return new MediaTransformError(
311 'thumbnail_error_remote',
314 $this->getDisplayName(),
315 $info['thumberror'] // already parsed message from foreign repo
323 * Return the imageurl from cache if possible
325 * If the url has been requested today, get it from cache
326 * Otherwise retrieve remote thumb url, check for local file.
328 * @param string $name Is a dbkey form of a title
331 * @param string $params Other rendering parameters (page number, etc)
332 * from handler's makeParamString.
333 * @return bool|string
335 function getThumbUrlFromCache( $name, $width, $height, $params = "" ) {
336 $cache = MediaWikiServices
::getInstance()->getMainWANObjectCache();
337 // We can't check the local cache using FileRepo functions because
338 // we override fileExistsBatch(). We have to use the FileBackend directly.
339 $backend = $this->getBackend(); // convenience
341 if ( !$this->canCacheThumbs() ) {
342 $result = null; // can't pass "null" by reference, but it's ok as default value
343 return $this->getThumbUrl( $name, $width, $height, $result, $params );
345 $key = $this->getLocalCacheKey( 'ForeignAPIRepo', 'ThumbUrl', $name );
346 $sizekey = "$width:$height:$params";
348 /* Get the array of urls that we already know */
349 $knownThumbUrls = $cache->get( $key );
350 if ( !$knownThumbUrls ) {
351 /* No knownThumbUrls for this file */
352 $knownThumbUrls = [];
354 if ( isset( $knownThumbUrls[$sizekey] ) ) {
355 wfDebug( __METHOD__
. ': Got thumburl from local cache: ' .
356 "{$knownThumbUrls[$sizekey]} \n" );
358 return $knownThumbUrls[$sizekey];
360 /* This size is not yet known */
364 $foreignUrl = $this->getThumbUrl( $name, $width, $height, $metadata, $params );
366 if ( !$foreignUrl ) {
367 wfDebug( __METHOD__
. " Could not find thumburl\n" );
372 // We need the same filename as the remote one :)
373 $fileName = rawurldecode( pathinfo( $foreignUrl, PATHINFO_BASENAME
) );
374 if ( !$this->validateFilename( $fileName ) ) {
375 wfDebug( __METHOD__
. " The deduced filename $fileName is not safe\n" );
379 $localPath = $this->getZonePath( 'thumb' ) . "/" . $this->getHashPath( $name ) . $name;
380 $localFilename = $localPath . "/" . $fileName;
381 $localUrl = $this->getZoneUrl( 'thumb' ) . "/" . $this->getHashPath( $name ) .
382 rawurlencode( $name ) . "/" . rawurlencode( $fileName );
384 if ( $backend->fileExists( [ 'src' => $localFilename ] )
385 && isset( $metadata['timestamp'] )
387 wfDebug( __METHOD__
. " Thumbnail was already downloaded before\n" );
388 $modified = $backend->getFileTimestamp( [ 'src' => $localFilename ] );
389 $remoteModified = strtotime( $metadata['timestamp'] );
391 $diff = abs( $modified - $current );
392 if ( $remoteModified < $modified && $diff < $this->fileCacheExpiry
) {
393 /* Use our current and already downloaded thumbnail */
394 $knownThumbUrls[$sizekey] = $localUrl;
395 $cache->set( $key, $knownThumbUrls, $this->apiThumbCacheExpiry
);
399 /* There is a new Commons file, or existing thumbnail older than a month */
402 $thumb = self
::httpGet( $foreignUrl, 'default', [], $mtime );
404 wfDebug( __METHOD__
. " Could not download thumb\n" );
409 # @todo FIXME: Delete old thumbs that aren't being used. Maintenance script?
410 $backend->prepare( [ 'dir' => dirname( $localFilename ) ] );
411 $params = [ 'dst' => $localFilename, 'content' => $thumb ];
412 if ( !$backend->quickCreate( $params )->isOK() ) {
413 wfDebug( __METHOD__
. " could not write to thumb path '$localFilename'\n" );
417 $knownThumbUrls[$sizekey] = $localUrl;
420 ?
$cache->adaptiveTTL( $mtime, $this->apiThumbCacheExpiry
)
421 : $this->apiThumbCacheExpiry
;
422 $cache->set( $key, $knownThumbUrls, $ttl );
423 wfDebug( __METHOD__
. " got local thumb $localUrl, saving to cache \n" );
429 * @see FileRepo::getZoneUrl()
430 * @param string $zone
431 * @param string|null $ext Optional file extension
434 function getZoneUrl( $zone, $ext = null ) {
439 return $this->thumbUrl
;
441 return parent
::getZoneUrl( $zone, $ext );
446 * Get the local directory corresponding to one of the basic zones
447 * @param string $zone
448 * @return bool|null|string
450 function getZonePath( $zone ) {
451 $supported = [ 'public', 'thumb' ];
452 if ( in_array( $zone, $supported ) ) {
453 return parent
::getZonePath( $zone );
460 * Are we locally caching the thumbnails?
463 public function canCacheThumbs() {
464 return ( $this->apiThumbCacheExpiry
> 0 );
468 * The user agent the ForeignAPIRepo will use.
471 public static function getUserAgent() {
472 return Http
::userAgent() . " ForeignAPIRepo/" . self
::VERSION
;
476 * Get information about the repo - overrides/extends the parent
477 * class's information.
482 $info = parent
::getInfo();
483 $info['apiurl'] = $this->getApiUrl();
488 'meta' => 'siteinfo',
489 'siprop' => 'general',
492 $data = $this->httpGetCached( 'SiteInfo', $query, 7200 );
495 $siteInfo = FormatJson
::decode( $data, true );
496 $general = $siteInfo['query']['general'];
498 $info['articlepath'] = $general['articlepath'];
499 $info['server'] = $general['server'];
501 if ( isset( $general['favicon'] ) ) {
502 $info['favicon'] = $general['favicon'];
510 * Like a Http:get request, but with custom User-Agent.
513 * @param string $timeout
514 * @param array $options
515 * @param int|bool &$mtime Resulting Last-Modified UNIX timestamp if received
516 * @return bool|string
518 public static function httpGet(
519 $url, $timeout = 'default', $options = [], &$mtime = false
521 $options['timeout'] = $timeout;
523 $url = wfExpandUrl( $url, PROTO_HTTP
);
524 wfDebug( "ForeignAPIRepo: HTTP GET: $url\n" );
525 $options['method'] = "GET";
527 if ( !isset( $options['timeout'] ) ) {
528 $options['timeout'] = 'default';
531 $req = MWHttpRequest
::factory( $url, $options, __METHOD__
);
532 $req->setUserAgent( self
::getUserAgent() );
533 $status = $req->execute();
535 if ( $status->isOK() ) {
536 $lmod = $req->getResponseHeader( 'Last-Modified' );
537 $mtime = $lmod ?
wfTimestamp( TS_UNIX
, $lmod ) : false;
539 return $req->getContent();
541 $logger = LoggerFactory
::getInstance( 'http' );
543 $status->getWikiText( false, false, 'en' ),
544 [ 'caller' => 'ForeignAPIRepo::httpGet' ]
555 protected static function getIIProps() {
556 return implode( '|', self
::$imageInfoProps );
560 * HTTP GET request to a mediawiki API (with caching)
561 * @param string $target Used in cache key creation, mostly
562 * @param array $query The query parameters for the API request
563 * @param int $cacheTTL Time to live for the memcached caching
564 * @return string|null
566 public function httpGetCached( $target, $query, $cacheTTL = 3600 ) {
567 if ( $this->mApiBase
) {
568 $url = wfAppendQuery( $this->mApiBase
, $query );
570 $url = $this->makeUrl( $query, 'api' );
573 $cache = MediaWikiServices
::getInstance()->getMainWANObjectCache();
574 return $cache->getWithSetCallback(
575 $this->getLocalCacheKey( static::class, $target, md5( $url ) ),
577 function ( $curValue, &$ttl ) use ( $url, $cache ) {
578 $html = self
::httpGet( $url, 'default', [], $mtime );
579 if ( $html !== false ) {
580 $ttl = $mtime ?
$cache->adaptiveTTL( $mtime, $ttl ) : $ttl;
582 $ttl = $cache->adaptiveTTL( $mtime, $ttl );
583 $html = null; // caches negatives
588 [ 'pcTTL' => $cache::TTL_PROC_LONG
]
593 * @param callable $callback
594 * @throws MWException
596 function enumFiles( $callback ) {
597 throw new MWException( 'enumFiles is not supported by ' . static::class );
601 * @throws MWException
603 protected function assertWritableRepo() {
604 throw new MWException( static::class . ': write operations are not supported.' );