* Introduced a new system for localisation caching. The system is based around fast...
[lhc/web/wiklou.git] / includes / MessageCache.php
1 <?php
2 /**
3 * @file
4 * @ingroup Cache
5 */
6
7 /**
8 *
9 */
10 define( 'MSG_LOAD_TIMEOUT', 60);
11 define( 'MSG_LOCK_TIMEOUT', 10);
12 define( 'MSG_WAIT_TIMEOUT', 10);
13 define( 'MSG_CACHE_VERSION', 1 );
14
15 /**
16 * Message cache
17 * Performs various MediaWiki namespace-related functions
18 * @ingroup Cache
19 */
20 class MessageCache {
21 // Holds loaded messages that are defined in MediaWiki namespace.
22 var $mCache;
23
24 var $mUseCache, $mDisable, $mExpiry;
25 var $mKeys, $mParserOptions, $mParser;
26
27 // Variable for tracking which variables are loaded
28 var $mLoadedLanguages = array();
29
30 function __construct( &$memCached, $useDB, $expiry, /*ignored*/ $memcPrefix ) {
31 $this->mUseCache = !is_null( $memCached );
32 $this->mMemc = &$memCached;
33 $this->mDisable = !$useDB;
34 $this->mExpiry = $expiry;
35 $this->mDisableTransform = false;
36 $this->mKeys = false; # initialised on demand
37 $this->mParser = null;
38 }
39
40
41 /**
42 * ParserOptions is lazy initialised.
43 */
44 function getParserOptions() {
45 if ( !$this->mParserOptions ) {
46 $this->mParserOptions = new ParserOptions;
47 }
48 return $this->mParserOptions;
49 }
50
51 /**
52 * Try to load the cache from a local file.
53 * Actual format of the file depends on the $wgLocalMessageCacheSerialized
54 * setting.
55 *
56 * @param $hash String: the hash of contents, to check validity.
57 * @param $code Mixed: Optional language code, see documenation of load().
58 * @return false on failure.
59 */
60 function loadFromLocal( $hash, $code ) {
61 global $wgCacheDirectory, $wgLocalMessageCacheSerialized;
62
63 $filename = "$wgCacheDirectory/messages-" . wfWikiID() . "-$code";
64
65 # Check file existence
66 wfSuppressWarnings();
67 $file = fopen( $filename, 'r' );
68 wfRestoreWarnings();
69 if ( !$file ) {
70 return false; // No cache file
71 }
72
73 if ( $wgLocalMessageCacheSerialized ) {
74 // Check to see if the file has the hash specified
75 $localHash = fread( $file, 32 );
76 if ( $hash === $localHash ) {
77 // All good, get the rest of it
78 $serialized = '';
79 while ( !feof( $file ) ) {
80 $serialized .= fread( $file, 100000 );
81 }
82 fclose( $file );
83 return $this->setCache( unserialize( $serialized ), $code );
84 } else {
85 fclose( $file );
86 return false; // Wrong hash
87 }
88 } else {
89 $localHash=substr(fread($file,40),8);
90 fclose($file);
91 if ($hash!=$localHash) {
92 return false; // Wrong hash
93 }
94
95 # Require overwrites the member variable or just shadows it?
96 require( $filename );
97 return $this->setCache( $this->mCache, $code );
98 }
99 }
100
101 /**
102 * Save the cache to a local file.
103 */
104 function saveToLocal( $serialized, $hash, $code ) {
105 global $wgCacheDirectory;
106
107 $filename = "$wgCacheDirectory/messages-" . wfWikiID() . "-$code";
108 wfMkdirParents( $wgCacheDirectory ); // might fail
109
110 wfSuppressWarnings();
111 $file = fopen( $filename, 'w' );
112 wfRestoreWarnings();
113
114 if ( !$file ) {
115 wfDebug( "Unable to open local cache file for writing\n" );
116 return;
117 }
118
119 fwrite( $file, $hash . $serialized );
120 fclose( $file );
121 @chmod( $filename, 0666 );
122 }
123
124 function saveToScript( $array, $hash, $code ) {
125 global $wgCacheDirectory;
126
127 $filename = "$wgCacheDirectory/messages-" . wfWikiID() . "-$code";
128 $tempFilename = $filename . '.tmp';
129 wfMkdirParents( $wgCacheDirectory ); // might fail
130
131 wfSuppressWarnings();
132 $file = fopen( $tempFilename, 'w');
133 wfRestoreWarnings();
134
135 if ( !$file ) {
136 wfDebug( "Unable to open local cache file for writing\n" );
137 return;
138 }
139
140 fwrite($file,"<?php\n//$hash\n\n \$this->mCache = array(");
141
142 foreach ($array as $key => $message) {
143 $key = $this->escapeForScript($key);
144 $messages = $this->escapeForScript($message);
145 fwrite($file, "'$key' => '$message',\n");
146 }
147
148 fwrite($file,");\n?>");
149 fclose($file);
150 rename($tempFilename, $filename);
151 }
152
153 function escapeForScript($string) {
154 $string = str_replace( '\\', '\\\\', $string );
155 $string = str_replace( '\'', '\\\'', $string );
156 return $string;
157 }
158
159 /**
160 * Set the cache to $cache, if it is valid. Otherwise set the cache to false.
161 */
162 function setCache( $cache, $code ) {
163 if ( isset( $cache['VERSION'] ) && $cache['VERSION'] == MSG_CACHE_VERSION ) {
164 $this->mCache[$code] = $cache;
165 return true;
166 } else {
167 return false;
168 }
169 }
170
171 /**
172 * Loads messages from caches or from database in this order:
173 * (1) local message cache (if $wgUseLocalMessageCache is enabled)
174 * (2) memcached
175 * (3) from the database.
176 *
177 * When succesfully loading from (2) or (3), all higher level caches are
178 * updated for the newest version.
179 *
180 * Nothing is loaded if member variable mDisabled is true, either manually
181 * set by calling code or if message loading fails (is this possible?).
182 *
183 * Returns true if cache is already populated or it was succesfully populated,
184 * or false if populating empty cache fails. Also returns true if MessageCache
185 * is disabled.
186 *
187 * @param $code String: language to which load messages
188 */
189 function load( $code = false ) {
190 global $wgUseLocalMessageCache;
191
192 if ( !$this->mUseCache ) {
193 return true;
194 }
195
196 if( !is_string( $code ) ) {
197 # This isn't really nice, so at least make a note about it and try to
198 # fall back
199 wfDebug( __METHOD__ . " called without providing a language code\n" );
200 $code = 'en';
201 }
202
203 # Don't do double loading...
204 if ( isset($this->mLoadedLanguages[$code]) ) return true;
205
206 # 8 lines of code just to say (once) that message cache is disabled
207 if ( $this->mDisable ) {
208 static $shownDisabled = false;
209 if ( !$shownDisabled ) {
210 wfDebug( __METHOD__ . ": disabled\n" );
211 $shownDisabled = true;
212 }
213 return true;
214 }
215
216 # Loading code starts
217 wfProfileIn( __METHOD__ );
218 $success = false; # Keep track of success
219 $where = array(); # Debug info, delayed to avoid spamming debug log too much
220 $cacheKey = wfMemcKey( 'messages', $code ); # Key in memc for messages
221
222
223 # (1) local cache
224 # Hash of the contents is stored in memcache, to detect if local cache goes
225 # out of date (due to update in other thread?)
226 if ( $wgUseLocalMessageCache ) {
227 wfProfileIn( __METHOD__ . '-fromlocal' );
228
229 $hash = $this->mMemc->get( wfMemcKey( 'messages', $code, 'hash' ) );
230 if ( $hash ) {
231 $success = $this->loadFromLocal( $hash, $code );
232 if ( $success ) $where[] = 'got from local cache';
233 }
234 wfProfileOut( __METHOD__ . '-fromlocal' );
235 }
236
237 # (2) memcache
238 # Fails if nothing in cache, or in the wrong version.
239 if ( !$success ) {
240 wfProfileIn( __METHOD__ . '-fromcache' );
241 $cache = $this->mMemc->get( $cacheKey );
242 $success = $this->setCache( $cache, $code );
243 if ( $success ) {
244 $where[] = 'got from global cache';
245 $this->saveToCaches( $cache, false, $code );
246 }
247 wfProfileOut( __METHOD__ . '-fromcache' );
248 }
249
250
251 # (3)
252 # Nothing in caches... so we need create one and store it in caches
253 if ( !$success ) {
254 $where[] = 'cache is empty';
255 $where[] = 'loading from database';
256
257 $this->lock($cacheKey);
258
259 # Limit the concurrency of loadFromDB to a single process
260 # This prevents the site from going down when the cache expires
261 $statusKey = wfMemcKey( 'messages', $code, 'status' );
262 $success = $this->mMemc->add( $statusKey, 'loading', MSG_LOAD_TIMEOUT );
263 if ( $success ) {
264 $cache = $this->loadFromDB( $code );
265 $success = $this->setCache( $cache, $code );
266 }
267 if ( $success ) {
268 $success = $this->saveToCaches( $cache, true, $code );
269 if ( $success ) {
270 $this->mMemc->delete( $statusKey );
271 } else {
272 $this->mMemc->set( $statusKey, 'error', 60*5 );
273 wfDebug( "MemCached set error in MessageCache: restart memcached server!\n" );
274 }
275 }
276 $this->unlock($cacheKey);
277 }
278
279 if ( !$success ) {
280 # Bad luck... this should not happen
281 $where[] = 'loading FAILED - cache is disabled';
282 $info = implode( ', ', $where );
283 wfDebug( __METHOD__ . ": Loading $code... $info\n" );
284 $this->mDisable = true;
285 $this->mCache = false;
286 } else {
287 # All good, just record the success
288 $info = implode( ', ', $where );
289 wfDebug( __METHOD__ . ": Loading $code... $info\n" );
290 $this->mLoadedLanguages[$code] = true;
291 }
292 wfProfileOut( __METHOD__ );
293 return $success;
294 }
295
296 /**
297 * Loads cacheable messages from the database. Messages bigger than
298 * $wgMaxMsgCacheEntrySize are assigned a special value, and are loaded
299 * on-demand from the database later.
300 *
301 * @param $code Optional language code, see documenation of load().
302 * @return Array: Loaded messages for storing in caches.
303 */
304 function loadFromDB( $code = false ) {
305 wfProfileIn( __METHOD__ );
306 global $wgMaxMsgCacheEntrySize, $wgContLanguageCode;
307 $dbr = wfGetDB( DB_SLAVE );
308 $cache = array();
309
310 # Common conditions
311 $conds = array(
312 'page_is_redirect' => 0,
313 'page_namespace' => NS_MEDIAWIKI,
314 );
315
316 if ( $code ) {
317 # Is this fast enough. Should not matter if the filtering is done in the
318 # database or in code.
319 if ( $code !== $wgContLanguageCode ) {
320 # Messages for particular language
321 $escapedCode = $dbr->escapeLike( $code );
322 $conds[] = "page_title like '%%/$escapedCode'";
323 } else {
324 # Effectively disallows use of '/' character in NS_MEDIAWIKI for uses
325 # other than language code.
326 $conds[] = "page_title not like '%%/%%'";
327 }
328 }
329
330 # Conditions to fetch oversized pages to ignore them
331 $bigConds = $conds;
332 $bigConds[] = 'page_len > ' . intval( $wgMaxMsgCacheEntrySize );
333
334 # Load titles for all oversized pages in the MediaWiki namespace
335 $res = $dbr->select( 'page', 'page_title', $bigConds, __METHOD__ );
336 while ( $row = $dbr->fetchObject( $res ) ) {
337 $cache[$row->page_title] = '!TOO BIG';
338 }
339 $dbr->freeResult( $res );
340
341 # Conditions to load the remaining pages with their contents
342 $smallConds = $conds;
343 $smallConds[] = 'page_latest=rev_id';
344 $smallConds[] = 'rev_text_id=old_id';
345 $smallConds[] = 'page_len <= ' . intval( $wgMaxMsgCacheEntrySize );
346
347 $res = $dbr->select( array( 'page', 'revision', 'text' ),
348 array( 'page_title', 'old_text', 'old_flags' ),
349 $smallConds, __METHOD__ );
350
351 for ( $row = $dbr->fetchObject( $res ); $row; $row = $dbr->fetchObject( $res ) ) {
352 $cache[$row->page_title] = ' ' . Revision::getRevisionText( $row );
353 }
354 $dbr->freeResult( $res );
355
356 $cache['VERSION'] = MSG_CACHE_VERSION;
357 wfProfileOut( __METHOD__ );
358 return $cache;
359 }
360
361 /**
362 * Updates cache as necessary when message page is changed
363 *
364 * @param $title String: name of the page changed.
365 * @param $text Mixed: new contents of the page.
366 */
367 public function replace( $title, $text ) {
368 global $wgMaxMsgCacheEntrySize;
369 wfProfileIn( __METHOD__ );
370
371
372 list( , $code ) = $this->figureMessage( $title );
373
374 $cacheKey = wfMemcKey( 'messages', $code );
375 $this->load($code);
376 $this->lock($cacheKey);
377
378 if ( is_array($this->mCache[$code]) ) {
379 $titleKey = wfMemcKey( 'messages', 'individual', $title );
380
381 if ( $text === false ) {
382 # Article was deleted
383 unset( $this->mCache[$code][$title] );
384 $this->mMemc->delete( $titleKey );
385
386 } elseif ( strlen( $text ) > $wgMaxMsgCacheEntrySize ) {
387 # Check for size
388 $this->mCache[$code][$title] = '!TOO BIG';
389 $this->mMemc->set( $titleKey, ' ' . $text, $this->mExpiry );
390
391 } else {
392 $this->mCache[$code][$title] = ' ' . $text;
393 $this->mMemc->delete( $titleKey );
394 }
395
396 # Update caches
397 $this->saveToCaches( $this->mCache[$code], true, $code );
398 }
399 $this->unlock($cacheKey);
400
401 // Also delete cached sidebar... just in case it is affected
402 global $parserMemc;
403 $sidebarKey = wfMemcKey( 'sidebar', $code );
404 $parserMemc->delete( $sidebarKey );
405
406 wfRunHooks( "MessageCacheReplace", array( $title, $text ) );
407
408 wfProfileOut( __METHOD__ );
409 }
410
411 /**
412 * Shortcut to update caches.
413 *
414 * @param $cache Array: cached messages with a version.
415 * @param $cacheKey String: Identifier for the cache.
416 * @param $memc Bool: Wether to update or not memcache.
417 * @param $code String: Language code.
418 * @return False on somekind of error.
419 */
420 protected function saveToCaches( $cache, $memc = true, $code = false ) {
421 wfProfileIn( __METHOD__ );
422 global $wgUseLocalMessageCache, $wgLocalMessageCacheSerialized;
423
424 $cacheKey = wfMemcKey( 'messages', $code );
425
426 $i = 0;
427 if ( $memc ) {
428 # Save in memcached
429 # Keep trying if it fails, this is kind of important
430
431 for ($i=0; $i<20 &&
432 !$this->mMemc->set( $cacheKey, $cache, $this->mExpiry );
433 $i++ ) {
434 usleep(mt_rand(500000,1500000));
435 }
436 }
437
438 # Save to local cache
439 if ( $wgUseLocalMessageCache ) {
440 $serialized = serialize( $cache );
441 $hash = md5( $serialized );
442 $this->mMemc->set( wfMemcKey( 'messages', $code, 'hash' ), $hash, $this->mExpiry );
443 if ($wgLocalMessageCacheSerialized) {
444 $this->saveToLocal( $serialized, $hash, $code );
445 } else {
446 $this->saveToScript( $cache, $hash, $code );
447 }
448 }
449
450 if ( $i == 20 ) {
451 $success = false;
452 } else {
453 $success = true;
454 }
455 wfProfileOut( __METHOD__ );
456 return $success;
457 }
458
459 /**
460 * Returns success
461 * Represents a write lock on the messages key
462 */
463 function lock($key) {
464 if ( !$this->mUseCache ) {
465 return true;
466 }
467
468 $lockKey = $key . ':lock';
469 for ($i=0; $i < MSG_WAIT_TIMEOUT && !$this->mMemc->add( $lockKey, 1, MSG_LOCK_TIMEOUT ); $i++ ) {
470 sleep(1);
471 }
472
473 return $i >= MSG_WAIT_TIMEOUT;
474 }
475
476 function unlock($key) {
477 if ( !$this->mUseCache ) {
478 return;
479 }
480
481 $lockKey = $key . ':lock';
482 $this->mMemc->delete( $lockKey );
483 }
484
485 /**
486 * Get a message from either the content language or the user language.
487 *
488 * @param string $key The message cache key
489 * @param bool $useDB Get the message from the DB, false to use only the localisation
490 * @param string $langcode Code of the language to get the message for, if
491 * it is a valid code create a language for that
492 * language, if it is a string but not a valid code
493 * then make a basic language object, if it is a
494 * false boolean then use the current users
495 * language (as a fallback for the old parameter
496 * functionality), or if it is a true boolean then
497 * use the wikis content language (also as a
498 * fallback).
499 * @param bool $isFullKey Specifies whether $key is a two part key "msg/lang".
500 */
501 function get( $key, $useDB = true, $langcode = true, $isFullKey = false ) {
502 global $wgContLanguageCode, $wgContLang;
503
504 $lang = wfGetLangObj( $langcode );
505 $langcode = $lang->getCode();
506
507 $message = false;
508
509 # Normalise title-case input
510 $lckey = str_replace( ' ', '_', $key );
511 $lckey[0] = strtolower( $lckey[0] );
512 $uckey = ucfirst( $lckey );
513
514 # Try the MediaWiki namespace
515 if( !$this->mDisable && $useDB ) {
516 $title = $uckey;
517 if(!$isFullKey && ( $langcode != $wgContLanguageCode ) ) {
518 $title .= '/' . $langcode;
519 }
520 $message = $this->getMsgFromNamespace( $title, $langcode );
521 }
522
523 # Try the array in the language object
524 if ( $message === false ) {
525 $message = $lang->getMessage( $lckey );
526 if ( is_null( $message ) ) {
527 $message = false;
528 }
529 }
530
531 # Try the array of another language
532 if( $message === false ) {
533 $parts = explode( '/', $lckey );
534 # We may get calls for things that are http-urls from sidebar
535 # Let's not load nonexistent languages for those
536 # They usually have more than one slash.
537 if ( count( $parts ) == 2 && $parts[1] !== '' ) {
538 $message = Language::getMessageFor( $parts[0], $parts[1] );
539 if ( is_null( $message ) ) {
540 $message = false;
541 }
542 }
543 }
544
545 # Is this a custom message? Try the default language in the db...
546 if( ($message === false || $message === '-' ) &&
547 !$this->mDisable && $useDB &&
548 !$isFullKey && ($langcode != $wgContLanguageCode) ) {
549 $message = $this->getMsgFromNamespace( $uckey, $wgContLanguageCode );
550 }
551
552 # Final fallback
553 if( $message === false ) {
554 return '&lt;' . htmlspecialchars($key) . '&gt;';
555 }
556
557 # Fix whitespace
558 $message = strtr( $message,
559 array(
560 # Fix for trailing whitespace, removed by textarea
561 '&#32;' => ' ',
562 # Fix for NBSP, converted to space by firefox
563 '&nbsp;' => "\xc2\xa0",
564 ) );
565
566 return $message;
567 }
568
569 /**
570 * Get a message from the MediaWiki namespace, with caching. The key must
571 * first be converted to two-part lang/msg form if necessary.
572 *
573 * @param $title String: Message cache key with initial uppercase letter.
574 * @param $code String: code denoting the language to try.
575 */
576 function getMsgFromNamespace( $title, $code ) {
577 $type = false;
578 $message = false;
579
580 if ( $this->mUseCache ) {
581 $this->load( $code );
582 if (isset( $this->mCache[$code][$title] ) ) {
583 $entry = $this->mCache[$code][$title];
584 $type = substr( $entry, 0, 1 );
585 if ( $type == ' ' ) {
586 return substr( $entry, 1 );
587 }
588 }
589 }
590
591 # Call message hooks, in case they are defined
592 wfRunHooks('MessagesPreLoad', array( $title, &$message ) );
593 if ( $message !== false ) {
594 return $message;
595 }
596
597 # If there is no cache entry and no placeholder, it doesn't exist
598 if ( $type !== '!' ) {
599 return false;
600 }
601
602 $titleKey = wfMemcKey( 'messages', 'individual', $title );
603
604 # Try the individual message cache
605 if ( $this->mUseCache ) {
606 $entry = $this->mMemc->get( $titleKey );
607 if ( $entry ) {
608 $type = substr( $entry, 0, 1 );
609
610 if ( $type === ' ' ) {
611 # Ok!
612 $message = substr( $entry, 1 );
613 $this->mCache[$code][$title] = $entry;
614 return $message;
615 } elseif ( $entry === '!NONEXISTENT' ) {
616 return false;
617 } else {
618 # Corrupt/obsolete entry, delete it
619 $this->mMemc->delete( $titleKey );
620 }
621
622 }
623 }
624
625 # Try loading it from the DB
626 $revision = Revision::newFromTitle( Title::makeTitle( NS_MEDIAWIKI, $title ) );
627 if( $revision ) {
628 $message = $revision->getText();
629 if ($this->mUseCache) {
630 $this->mCache[$code][$title] = ' ' . $message;
631 $this->mMemc->set( $titleKey, $message, $this->mExpiry );
632 }
633 } else {
634 # Negative caching
635 # Use some special text instead of false, because false gets converted to '' somewhere
636 $this->mMemc->set( $titleKey, '!NONEXISTENT', $this->mExpiry );
637 $this->mCache[$code][$title] = false;
638 }
639 return $message;
640 }
641
642 function transform( $message, $interface = false, $language = null ) {
643 // Avoid creating parser if nothing to transform
644 if( strpos( $message, '{{' ) === false ) {
645 return $message;
646 }
647
648 global $wgParser, $wgParserConf;
649 if ( !$this->mParser && isset( $wgParser ) ) {
650 # Do some initialisation so that we don't have to do it twice
651 $wgParser->firstCallInit();
652 # Clone it and store it
653 $class = $wgParserConf['class'];
654 if ( $class == 'Parser_DiffTest' ) {
655 # Uncloneable
656 $this->mParser = new $class( $wgParserConf );
657 } else {
658 $this->mParser = clone $wgParser;
659 }
660 #wfDebug( __METHOD__ . ": following contents triggered transform: $message\n" );
661 }
662 if ( $this->mParser ) {
663 $popts = $this->getParserOptions();
664 $popts->setInterfaceMessage( $interface );
665 $popts->setTargetLanguage( $language );
666 $message = $this->mParser->transformMsg( $message, $popts );
667 }
668 return $message;
669 }
670
671 function disable() { $this->mDisable = true; }
672 function enable() { $this->mDisable = false; }
673
674 /** @deprecated */
675 function disableTransform(){
676 wfDeprecated( __METHOD__ );
677 }
678 function enableTransform() {
679 wfDeprecated( __METHOD__ );
680 }
681 function setTransform( $x ) {
682 wfDeprecated( __METHOD__ );
683 }
684 function getTransform() {
685 wfDeprecated( __METHOD__ );
686 return false;
687 }
688
689 /**
690 * Clear all stored messages. Mainly used after a mass rebuild.
691 */
692 function clear() {
693 if( $this->mUseCache ) {
694 $langs = Language::getLanguageNames( false );
695 foreach ( array_keys($langs) as $code ) {
696 # Global cache
697 $this->mMemc->delete( wfMemcKey( 'messages', $code ) );
698 # Invalidate all local caches
699 $this->mMemc->delete( wfMemcKey( 'messages', $code, 'hash' ) );
700 }
701 }
702 }
703
704 /**
705 * @deprecated
706 */
707 function loadAllMessages( $lang = false ) {
708 }
709
710 /**
711 * @deprecated
712 */
713 function loadMessagesFile( $filename, $langcode = false ) {
714 }
715
716 public function figureMessage( $key ) {
717 global $wgContLanguageCode;
718 $pieces = explode( '/', $key );
719 if( count( $pieces ) < 2 )
720 return array( $key, $wgContLanguageCode );
721
722 $lang = array_pop( $pieces );
723 $validCodes = Language::getLanguageNames();
724 if( !array_key_exists( $lang, $validCodes ) )
725 return array( $key, $wgContLanguageCode );
726
727 $message = implode( '/', $pieces );
728 return array( $message, $lang );
729 }
730
731 }