includes/BagOStuff.php

   1 <?php
   2 #
   3 # Copyright (C) 2003-2004 Brion Vibber <brion@pobox.com>
   4 # http://www.mediawiki.org/
   5 #
   6 # This program is free software; you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation; either version 2 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License along
  17 # with this program; if not, write to the Free Software Foundation, Inc.,
  18 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  19 # http://www.gnu.org/copyleft/gpl.html
  20 /**
  21  *
  22  */
  23
  24 /**
  25  * Simple generic object store
  26  *
  27  * interface is intended to be more or less compatible with
  28  * the PHP memcached client.
  29  *
  30  * backends for local hash array and SQL table included:
  31  * <code>
  32  *   $bag = new HashBagOStuff();
  33  *   $bag = new MediaWikiBagOStuff($tablename); # connect to db first
  34  * </code>
  35  *
  36  * @atttogroup Cache
  37  */
  38 class BagOStuff {
  39         var $debugmode;
  40
  41         function __construct() {
  42                 $this->set_debug( false );
  43         }
  44
  45         function set_debug($bool) {
  46                 $this->debugmode = $bool;
  47         }
  48
  49         /* *** THE GUTS OF THE OPERATION *** */
  50         /* Override these with functional things in subclasses */
  51
  52         function get($key) {
  53                 /* stub */
  54                 return false;
  55         }
  56
  57         function set($key, $value, $exptime=0) {
  58                 /* stub */
  59                 return false;
  60         }
  61
  62         function delete($key, $time=0) {
  63                 /* stub */
  64                 return false;
  65         }
  66
  67         function lock($key, $timeout = 0) {
  68                 /* stub */
  69                 return true;
  70         }
  71
  72         function unlock($key) {
  73                 /* stub */
  74                 return true;
  75         }
  76
  77         function keys() {
  78                 /* stub */
  79                 return array();
  80         }
  81
  82         /* *** Emulated functions *** */
  83         /* Better performance can likely be got with custom written versions */
  84         function get_multi($keys) {
  85                 $out = array();
  86                 foreach($keys as $key)
  87                         $out[$key] = $this->get($key);
  88                 return $out;
  89         }
  90
  91         function set_multi($hash, $exptime=0) {
  92                 foreach($hash as $key => $value)
  93                         $this->set($key, $value, $exptime);
  94         }
  95
  96         function add($key, $value, $exptime=0) {
  97                 if( $this->get($key) == false ) {
  98                         $this->set($key, $value, $exptime);
  99                         return true;
 100                 }
 101         }
 102
 103         function add_multi($hash, $exptime=0) {
 104                 foreach($hash as $key => $value)
 105                         $this->add($key, $value, $exptime);
 106         }
 107
 108         function delete_multi($keys, $time=0) {
 109                 foreach($keys as $key)
 110                         $this->delete($key, $time);
 111         }
 112
 113         function replace($key, $value, $exptime=0) {
 114                 if( $this->get($key) !== false )
 115                         $this->set($key, $value, $exptime);
 116         }
 117
 118         function incr($key, $value=1) {
 119                 if ( !$this->lock($key) ) {
 120                         return false;
 121                 }
 122                 $value = intval($value);
 123                 if($value < 0) $value = 0;
 124
 125                 $n = false;
 126                 if( ($n = $this->get($key)) !== false ) {
 127                         $n += $value;
 128                         $this->set($key, $n); // exptime?
 129                 }
 130                 $this->unlock($key);
 131                 return $n;
 132         }
 133
 134         function decr($key, $value=1) {
 135                 if ( !$this->lock($key) ) {
 136                         return false;
 137                 }
 138                 $value = intval($value);
 139                 if($value < 0) $value = 0;
 140
 141                 $m = false;
 142                 if( ($n = $this->get($key)) !== false ) {
 143                         $m = $n - $value;
 144                         if($m < 0) $m = 0;
 145                         $this->set($key, $m); // exptime?
 146                 }
 147                 $this->unlock($key);
 148                 return $m;
 149         }
 150
 151         function _debug($text) {
 152                 if($this->debugmode)
 153                         wfDebug("BagOStuff debug: $text\n");
 154         }
 155
 156         /**
 157          * Convert an optionally relative time to an absolute time
 158          */
 159         static function convertExpiry( $exptime ) {
 160                 if(($exptime != 0) && ($exptime < 3600*24*30)) {
 161                         return time() + $exptime;
 162                 } else {
 163                         return $exptime;
 164                 }
 165         }
 166 }
 167
 168
 169 /**
 170  * Functional versions!
 171  * This is a test of the interface, mainly. It stores things in an associative
 172  * array, which is not going to persist between program runs.
 173  *
 174  * @atttogroup Cache
 175  */
 176 class HashBagOStuff extends BagOStuff {
 177         var $bag;
 178
 179         function __construct() {
 180                 $this->bag = array();
 181         }
 182
 183         function _expire($key) {
 184                 $et = $this->bag[$key][1];
 185                 if(($et == 0) || ($et > time()))
 186                         return false;
 187                 $this->delete($key);
 188                 return true;
 189         }
 190
 191         function get($key) {
 192                 if(!$this->bag[$key])
 193                         return false;
 194                 if($this->_expire($key))
 195                         return false;
 196                 return $this->bag[$key][0];
 197         }
 198
 199         function set($key,$value,$exptime=0) {
 200                 $this->bag[$key] = array( $value, BagOStuff::convertExpiry( $exptime ) );
 201         }
 202
 203         function delete($key,$time=0) {
 204                 if(!$this->bag[$key])
 205                         return false;
 206                 unset($this->bag[$key]);
 207                 return true;
 208         }
 209
 210         function keys() {
 211                 return array_keys( $this->bag );
 212         }
 213 }
 214
 215 /**
 216  * Generic class to store objects in a database
 217  *
 218  * @atttogroup Cache
 219  */
 220 abstract class SqlBagOStuff extends BagOStuff {
 221         var $table;
 222         var $lastexpireall = 0;
 223
 224         /**
 225          * Constructor
 226          *
 227          * @param string $tablename name of the table to use
 228          */
 229         function __construct($tablename = 'objectcache') {
 230                 $this->table = $tablename;
 231         }
 232
 233         function get($key) {
 234                 /* expire old entries if any */
 235                 $this->garbageCollect();
 236
 237                 $res = $this->_query(
 238                         "SELECT value,exptime FROM $0 WHERE keyname='$1'", $key);
 239                 if(!$res) {
 240                         $this->_debug("get: ** error: " . $this->_dberror($res) . " **");
 241                         return false;
 242                 }
 243                 if($row=$this->_fetchobject($res)) {
 244                         $this->_debug("get: retrieved data; exp time is " . $row->exptime);
 245                         if ( $row->exptime != $this->_maxdatetime() &&
 246                           wfTimestamp( TS_UNIX, $row->exptime ) < time() )
 247                         {
 248                                 $this->_debug("get: key has expired, deleting");
 249                                 $this->delete($key);
 250                                 return false;
 251                         }
 252                         return $this->_unserialize($this->_blobdecode($row->value));
 253                 } else {
 254                         $this->_debug('get: no matching rows');
 255                 }
 256                 return false;
 257         }
 258
 259         function set($key,$value,$exptime=0) {
 260                 if ( $this->_readonly() ) {
 261                         return false;
 262                 }
 263                 $exptime = intval($exptime);
 264                 if($exptime < 0) $exptime = 0;
 265                 if($exptime == 0) {
 266                         $exp = $this->_maxdatetime();
 267                 } else {
 268                         if($exptime < 3.16e8) # ~10 years
 269                                 $exptime += time();
 270                         $exp = $this->_fromunixtime($exptime);
 271                 }
 272                 $this->delete( $key );
 273                 $this->_doinsert($this->getTableName(), array(
 274                                         'keyname' => $key,
 275                                         'value' => $this->_blobencode($this->_serialize($value)),
 276                                         'exptime' => $exp
 277                                 ));
 278                 return true; /* ? */
 279         }
 280
 281         function delete($key,$time=0) {
 282                 if ( $this->_readonly() ) {
 283                         return false;
 284                 }
 285                 $this->_query(
 286                         "DELETE FROM $0 WHERE keyname='$1'", $key );
 287                 return true; /* ? */
 288         }
 289
 290         function keys() {
 291                 $res = $this->_query( "SELECT keyname FROM $0" );
 292                 if(!$res) {
 293                         $this->_debug("keys: ** error: " . $this->_dberror($res) . " **");
 294                         return array();
 295                 }
 296                 $result = array();
 297                 while( $row = $this->_fetchobject($res) ) {
 298                         $result[] = $row->keyname;
 299                 }
 300                 return $result;
 301         }
 302
 303         function getTableName() {
 304                 return $this->table;
 305         }
 306
 307         function _query($sql) {
 308                 $reps = func_get_args();
 309                 $reps[0] = $this->getTableName();
 310                 // ewwww
 311                 for($i=0;$i<count($reps);$i++) {
 312                         $sql = str_replace(
 313                                 '$' . $i,
 314                                 $i > 0 ? $this->_strencode($reps[$i]) : $reps[$i],
 315                                 $sql);
 316                 }
 317                 $res = $this->_doquery($sql);
 318                 if($res == false) {
 319                         $this->_debug('query failed: ' . $this->_dberror($res));
 320                 }
 321                 return $res;
 322         }
 323
 324         function _strencode($str) {
 325                 /* Protect strings in SQL */
 326                 return str_replace( "'", "''", $str );
 327         }
 328         function _blobencode($str) {
 329                 return $str;
 330         }
 331         function _blobdecode($str) {
 332                 return $str;
 333         }
 334
 335         abstract function _doinsert($table, $vals);
 336         abstract function _doquery($sql);
 337
 338         abstract function _readonly();
 339
 340         function _freeresult($result) {
 341                 /* stub */
 342                 return false;
 343         }
 344
 345         function _dberror($result) {
 346                 /* stub */
 347                 return 'unknown error';
 348         }
 349
 350         abstract function _maxdatetime();
 351         abstract function _fromunixtime($ts);
 352
 353         function garbageCollect() {
 354                 /* Ignore 99% of requests */
 355                 if ( !mt_rand( 0, 100 ) ) {
 356                         $nowtime = time();
 357                         /* Avoid repeating the delete within a few seconds */
 358                         if ( $nowtime > ($this->lastexpireall + 1) ) {
 359                                 $this->lastexpireall = $nowtime;
 360                                 $this->expireall();
 361                         }
 362                 }
 363         }
 364
 365         function expireall() {
 366                 /* Remove any items that have expired */
 367                 if ( $this->_readonly() ) {
 368                         return false;
 369                 }
 370                 $now = $this->_fromunixtime( time() );
 371                 $this->_query( "DELETE FROM $0 WHERE exptime < '$now'" );
 372         }
 373
 374         function deleteall() {
 375                 /* Clear *all* items from cache table */
 376                 if ( $this->_readonly() ) {
 377                         return false;
 378                 }
 379                 $this->_query( "DELETE FROM $0" );
 380         }
 381
 382         /**
 383          * Serialize an object and, if possible, compress the representation.
 384          * On typical message and page data, this can provide a 3X decrease
 385          * in storage requirements.
 386          *
 387          * @param mixed $data
 388          * @return string
 389          */
 390         function _serialize( &$data ) {
 391                 $serial = serialize( $data );
 392                 if( function_exists( 'gzdeflate' ) ) {
 393                         return gzdeflate( $serial );
 394                 } else {
 395                         return $serial;
 396                 }
 397         }
 398
 399         /**
 400          * Unserialize and, if necessary, decompress an object.
 401          * @param string $serial
 402          * @return mixed
 403          */
 404         function _unserialize( $serial ) {
 405                 if( function_exists( 'gzinflate' ) ) {
 406                         $decomp = @gzinflate( $serial );
 407                         if( false !== $decomp ) {
 408                                 $serial = $decomp;
 409                         }
 410                 }
 411                 $ret = unserialize( $serial );
 412                 return $ret;
 413         }
 414 }
 415
 416 /**
 417  * Stores objects in the main database of the wiki
 418  *
 419  * @atttogroup Cache
 420  */
 421 class MediaWikiBagOStuff extends SqlBagOStuff {
 422         var $tableInitialised = false;
 423
 424         function _getDB(){
 425                 static $db;
 426                 if( !isset( $db ) )
 427                         $db = wfGetDB( DB_MASTER );
 428                 return $db;
 429         }
 430         function _doquery($sql) {
 431                 return $this->_getDB()->query( $sql, __METHOD__ );
 432         }
 433         function _doinsert($t, $v) {
 434                 return $this->_getDB()->insert($t, $v, __METHOD__, array( 'IGNORE' ) );
 435         }
 436         function _fetchobject($result) {
 437                 return $this->_getDB()->fetchObject($result);
 438         }
 439         function _freeresult($result) {
 440                 return $this->_getDB()->freeResult($result);
 441         }
 442         function _dberror($result) {
 443                 return $this->_getDB()->lastError();
 444         }
 445         function _maxdatetime() {
 446                 if ( time() > 0x7fffffff ) {
 447                         return $this->_fromunixtime( 1<<62 );
 448                 } else {
 449                         return $this->_fromunixtime( 0x7fffffff );
 450                 }
 451         }
 452         function _fromunixtime($ts) {
 453                 return $this->_getDB()->timestamp($ts);
 454         }
 455         function _readonly(){
 456                 return wfReadOnly();
 457         }
 458         function _strencode($s) {
 459                 return $this->_getDB()->strencode($s);
 460         }
 461         function _blobencode($s) {
 462                 return $this->_getDB()->encodeBlob($s);
 463         }
 464         function _blobdecode($s) {
 465                 return $this->_getDB()->decodeBlob($s);
 466         }
 467         function getTableName() {
 468                 if ( !$this->tableInitialised ) {
 469                         $dbw = $this->_getDB();
 470                         /* This is actually a hack, we should be able
 471                            to use Language classes here... or not */
 472                         if (!$dbw)
 473                                 throw new MWException("Could not connect to database");
 474                         $this->table = $dbw->tableName( $this->table );
 475                         $this->tableInitialised = true;
 476                 }
 477                 return $this->table;
 478         }
 479 }
 480
 481 /**
 482  * This is a wrapper for Turck MMCache's shared memory functions.
 483  *
 484  * You can store objects with mmcache_put() and mmcache_get(), but Turck seems
 485  * to use a weird custom serializer that randomly segfaults. So we wrap calls
 486  * with serialize()/unserialize().
 487  *
 488  * The thing I noticed about the Turck serialized data was that unlike ordinary
 489  * serialize(), it contained the names of methods, and judging by the amount of
 490  * binary data, perhaps even the bytecode of the methods themselves. It may be
 491  * that Turck's serializer is faster, so a possible future extension would be
 492  * to use it for arrays but not for objects.
 493  *
 494  * @atttogroup Cache
 495  */
 496 class TurckBagOStuff extends BagOStuff {
 497         function get($key) {
 498                 $val = mmcache_get( $key );
 499                 if ( is_string( $val ) ) {
 500                         $val = unserialize( $val );
 501                 }
 502                 return $val;
 503         }
 504
 505         function set($key, $value, $exptime=0) {
 506                 mmcache_put( $key, serialize( $value ), $exptime );
 507                 return true;
 508         }
 509
 510         function delete($key, $time=0) {
 511                 mmcache_rm( $key );
 512                 return true;
 513         }
 514
 515         function lock($key, $waitTimeout = 0 ) {
 516                 mmcache_lock( $key );
 517                 return true;
 518         }
 519
 520         function unlock($key) {
 521                 mmcache_unlock( $key );
 522                 return true;
 523         }
 524 }
 525
 526 /**
 527  * This is a wrapper for APC's shared memory functions
 528  *
 529  * @atttogroup Cache
 530  */
 531 class APCBagOStuff extends BagOStuff {
 532         function get($key) {
 533                 $val = apc_fetch($key);
 534                 if ( is_string( $val ) ) {
 535                         $val = unserialize( $val );
 536                 }
 537                 return $val;
 538         }
 539
 540         function set($key, $value, $exptime=0) {
 541                 apc_store($key, serialize($value), $exptime);
 542                 return true;
 543         }
 544
 545         function delete($key, $time=0) {
 546                 apc_delete($key);
 547                 return true;
 548         }
 549 }
 550
 551
 552 /**
 553  * This is a wrapper for eAccelerator's shared memory functions.
 554  *
 555  * This is basically identical to the Turck MMCache version,
 556  * mostly because eAccelerator is based on Turck MMCache.
 557  *
 558  * @atttogroup Cache
 559  */
 560 class eAccelBagOStuff extends BagOStuff {
 561         function get($key) {
 562                 $val = eaccelerator_get( $key );
 563                 if ( is_string( $val ) ) {
 564                         $val = unserialize( $val );
 565                 }
 566                 return $val;
 567         }
 568
 569         function set($key, $value, $exptime=0) {
 570                 eaccelerator_put( $key, serialize( $value ), $exptime );
 571                 return true;
 572         }
 573
 574         function delete($key, $time=0) {
 575                 eaccelerator_rm( $key );
 576                 return true;
 577         }
 578
 579         function lock($key, $waitTimeout = 0 ) {
 580                 eaccelerator_lock( $key );
 581                 return true;
 582         }
 583
 584         function unlock($key) {
 585                 eaccelerator_unlock( $key );
 586                 return true;
 587         }
 588 }
 589
 590 /**
 591  * Wrapper for XCache object caching functions; identical interface
 592  * to the APC wrapper
 593  *
 594  * @atttogroup Cache
 595  */
 596 class XCacheBagOStuff extends BagOStuff {
 597
 598         /**
 599          * Get a value from the XCache object cache
 600          *
 601          * @param string $key Cache key
 602          * @return mixed
 603          */
 604         public function get( $key ) {
 605                 $val = xcache_get( $key );
 606                 if( is_string( $val ) )
 607                         $val = unserialize( $val );
 608                 return $val;
 609         }
 610
 611         /**
 612          * Store a value in the XCache object cache
 613          *
 614          * @param string $key Cache key
 615          * @param mixed $value Object to store
 616          * @param int $expire Expiration time
 617          * @return bool
 618          */
 619         public function set( $key, $value, $expire = 0 ) {
 620                 xcache_set( $key, serialize( $value ), $expire );
 621                 return true;
 622         }
 623
 624         /**
 625          * Remove a value from the XCache object cache
 626          *
 627          * @param string $key Cache key
 628          * @param int $time Not used in this implementation
 629          * @return bool
 630          */
 631         public function delete( $key, $time = 0 ) {
 632                 xcache_unset( $key );
 633                 return true;
 634         }
 635
 636 }
 637
 638 /**
 639  * @todo document
 640  * @atttogroup Cache
 641  */
 642 class DBABagOStuff extends BagOStuff {
 643         var $mHandler, $mFile, $mReader, $mWriter, $mDisabled;
 644
 645         function __construct( $handler = 'db3', $dir = false ) {
 646                 if ( $dir === false ) {
 647                         global $wgTmpDirectory;
 648                         $dir = $wgTmpDirectory;
 649                 }
 650                 $this->mFile = "$dir/mw-cache-" . wfWikiID();
 651                 $this->mFile .= '.db';
 652                 wfDebug( __CLASS__.": using cache file {$this->mFile}\n" );
 653                 $this->mHandler = $handler;
 654         }
 655
 656         /**
 657          * Encode value and expiry for storage
 658          */
 659         function encode( $value, $expiry ) {
 660                 # Convert to absolute time
 661                 $expiry = BagOStuff::convertExpiry( $expiry );
 662                 return sprintf( '%010u', intval( $expiry ) ) . ' ' . serialize( $value );
 663         }
 664
 665         /**
 666          * @return list containing value first and expiry second
 667          */
 668         function decode( $blob ) {
 669                 if ( !is_string( $blob ) ) {
 670                         return array( null, 0 );
 671                 } else {
 672                         return array(
 673                                 unserialize( substr( $blob, 11 ) ),
 674                                 intval( substr( $blob, 0, 10 ) )
 675                         );
 676                 }
 677         }
 678
 679         function getReader() {
 680                 if ( file_exists( $this->mFile ) ) {
 681                         $handle = dba_open( $this->mFile, 'rl', $this->mHandler );
 682                 } else {
 683                         $handle = $this->getWriter();
 684                 }
 685                 if ( !$handle ) {
 686                         wfDebug( "Unable to open DBA cache file {$this->mFile}\n" );
 687                 }
 688                 return $handle;
 689         }
 690
 691         function getWriter() {
 692                 $handle = dba_open( $this->mFile, 'cl', $this->mHandler );
 693                 if ( !$handle ) {
 694                         wfDebug( "Unable to open DBA cache file {$this->mFile}\n" );
 695                 }
 696                 return $handle;
 697         }
 698
 699         function get( $key ) {
 700                 wfProfileIn( __METHOD__ );
 701                 wfDebug( __METHOD__."($key)\n" );
 702                 $handle = $this->getReader();
 703                 if ( !$handle ) {
 704                         return null;
 705                 }
 706                 $val = dba_fetch( $key, $handle );
 707                 list( $val, $expiry ) = $this->decode( $val );
 708                 # Must close ASAP because locks are held
 709                 dba_close( $handle );
 710
 711                 if ( !is_null( $val ) && $expiry && $expiry < time() ) {
 712                         # Key is expired, delete it
 713                         $handle = $this->getWriter();
 714                         dba_delete( $key, $handle );
 715                         dba_close( $handle );
 716                         wfDebug( __METHOD__.": $key expired\n" );
 717                         $val = null;
 718                 }
 719                 wfProfileOut( __METHOD__ );
 720                 return $val;
 721         }
 722
 723         function set( $key, $value, $exptime=0 ) {
 724                 wfProfileIn( __METHOD__ );
 725                 wfDebug( __METHOD__."($key)\n" );
 726                 $blob = $this->encode( $value, $exptime );
 727                 $handle = $this->getWriter();
 728                 if ( !$handle ) {
 729                         return false;
 730                 }
 731                 $ret = dba_replace( $key, $blob, $handle );
 732                 dba_close( $handle );
 733                 wfProfileOut( __METHOD__ );
 734                 return $ret;
 735         }
 736
 737         function delete( $key, $time = 0 ) {
 738                 wfProfileIn( __METHOD__ );
 739                 wfDebug( __METHOD__."($key)\n" );
 740                 $handle = $this->getWriter();
 741                 if ( !$handle ) {
 742                         return false;
 743                 }
 744                 $ret = dba_delete( $key, $handle );
 745                 dba_close( $handle );
 746                 wfProfileOut( __METHOD__ );
 747                 return $ret;
 748         }
 749
 750         function add( $key, $value, $exptime = 0 ) {
 751                 wfProfileIn( __METHOD__ );
 752                 $blob = $this->encode( $value, $exptime );
 753                 $handle = $this->getWriter();
 754                 if ( !$handle ) {
 755                         return false;
 756                 }
 757                 $ret = dba_insert( $key, $blob, $handle );
 758                 # Insert failed, check to see if it failed due to an expired key
 759                 if ( !$ret ) {
 760                         list( $value, $expiry ) = $this->decode( dba_fetch( $key, $handle ) );
 761                         if ( $expiry < time() ) {
 762                                 # Yes expired, delete and try again
 763                                 dba_delete( $key, $handle );
 764                                 $ret = dba_insert( $key, $blob, $handle );
 765                                 # This time if it failed then it will be handled by the caller like any other race
 766                         }
 767                 }
 768
 769                 dba_close( $handle );
 770                 wfProfileOut( __METHOD__ );
 771                 return $ret;
 772         }
 773
 774         function keys() {
 775                 $reader = $this->getReader();
 776                 $k1 = dba_firstkey( $reader );
 777                 if( !$k1 ) {
 778                         return array();
 779                 }
 780                 $result[] = $k1;
 781                 while( $key = dba_nextkey( $reader ) ) {
 782                         $result[] = $key;
 783                 }
 784                 return $result;
 785         }
 786 }