Add an objectcache table for limited caching when memcached isn't
authorBrion Vibber <brion@users.mediawiki.org>
Sun, 9 May 2004 05:12:55 +0000 (05:12 +0000)
committerBrion Vibber <brion@users.mediawiki.org>
Sun, 9 May 2004 05:12:55 +0000 (05:12 +0000)
available. Currently using for the message cache to avoid reading
every message separately. This now is only slightly slower than
memcached in my tests when $wgUseDatabaseMessages is enabled, so
it's a bit of a speedup for common hosts.

config/index.php
includes/MessageCache.php
includes/ObjectCache.php [new file with mode: 0644]
includes/Setup.php
maintenance/archives/patch-list.txt
maintenance/archives/patch-objectcache.sql [new file with mode: 0644]
maintenance/tables.sql
maintenance/updaters.inc

index e2010a6..6382837 100644 (file)
@@ -398,6 +398,7 @@ if( $conf->posted && ( 0 == count( $errs ) ) ) {
                        echo "FIXME: need the link table change here\n";
                        do_user_real_name_update(); flush();
                        do_querycache_update(); flush();
+                       do_objectcache_update(); flush();
                        initialiseMessages(); flush();
                        chdir( "config" );
                        
index 9a4a18f..e3a44e9 100755 (executable)
@@ -14,8 +14,9 @@ class MessageCache
        
        var $mInitialised = false;
 
-       function initialise( $useMemCached, $useDB, $expiry, $memcPrefix ) {
-               $this->mUseCache = $useMemCached;
+       function initialise( &$memCached, $useDB, $expiry, $memcPrefix ) {
+               $this->mUseCache = !is_null( $memCached );
+               $this->mMemc = &$memCached;
                $this->mDisable = !$useDB;
                $this->mExpiry = $expiry;
                $this->mDisableTransform = false;
@@ -32,7 +33,7 @@ class MessageCache
        # On error, quietly switches to a fallback mode
        # Returns false for a reportable error, true otherwise
        function load() {
-               global $wgAllMessagesEn, $wgMemc;
+               global $wgAllMessagesEn;
                
                if ( $this->mDisable ) {
                        return true;
@@ -41,18 +42,18 @@ class MessageCache
                $success = true;
                
                if ( $this->mUseCache ) {
-                       $this->mCache = $wgMemc->get( $this->mMemcKey );
+                       $this->mCache = $this->mMemc->get( $this->mMemcKey );
                        
                        # If there's nothing in memcached, load all the messages from the database
                        if ( !$this->mCache ) {
                                $this->lock();
                                # Other threads don't need to load the messages if another thread is doing it.
-                               $wgMemc->set( $this->mMemcKey, "loading", MSG_LOAD_TIMEOUT );
+                               $this->mMemc->set( $this->mMemcKey, "loading", MSG_LOAD_TIMEOUT );
                                $this->loadFromDB();
                                # Save in memcached
-                               if ( !$wgMemc->set( $this->mMemcKey, $this->mCache, $this->mExpiry ) ) {
+                               if ( !$this->mMemc->set( $this->mMemcKey, $this->mCache, $this->mExpiry ) ) {
                                        # Hack for slabs reassignment problem
-                                       $wgMemc->set( $this->mMemcKey, "error" );
+                                       $this->mMemc->set( $this->mMemcKey, "error" );
                                        wfDebug( "MemCached set error in MessageCache: restart memcached server!\n" );
                                }
                                $this->unlock();
@@ -113,12 +114,11 @@ class MessageCache
        }
 
        function replace( $title, $text ) {
-               global $wgMemc;
                $this->lock();
                $this->load();
                if ( is_array( $this->mCache ) ) {
                        $this->mCache[$title] = $text;
-                       $wgMemc->set( $this->mMemcKey, $this->mCache, $this->mExpiry );
+                       $this->mMemc->set( $this->mMemcKey, $this->mCache, $this->mExpiry );
                }
                $this->unlock();
        }
@@ -126,14 +126,12 @@ class MessageCache
        # Returns success
        # Represents a write lock on the messages key
        function lock() {
-               global $wgMemc;
-
                if ( !$this->mUseCache ) {
                        return true;
                }
 
                $lockKey = $this->mMemcKey . "lock";
-               for ($i=0; $i < MSG_WAIT_TIMEOUT && !$wgMemc->add( $lockKey, 1, MSG_LOCK_TIMEOUT ); $i++ ) {
+               for ($i=0; $i < MSG_WAIT_TIMEOUT && !$this->mMemc->add( $lockKey, 1, MSG_LOCK_TIMEOUT ); $i++ ) {
                        sleep(1);
                }
                
@@ -141,14 +139,12 @@ class MessageCache
        }
        
        function unlock() {
-               global $wgMemc;
-               
                if ( !$this->mUseCache ) {
                        return;
                }
 
                $lockKey = $this->mMemcKey . "lock";
-               $wgMemc->delete( $lockKey );
+               $this->mMemc->delete( $lockKey );
        }
        
        function get( $key, $useDB ) {
diff --git a/includes/ObjectCache.php b/includes/ObjectCache.php
new file mode 100644 (file)
index 0000000..00864c6
--- /dev/null
@@ -0,0 +1,320 @@
+<?php
+# Copyright (C) 2003-2004 Brion Vibber <brion@pobox.com>
+# http://www.mediawiki.org/
+# 
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or 
+# (at your option) any later version.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+# http://www.gnu.org/copyleft/gpl.html
+
+# Simple generic object store
+# interface is intended to be more or less compatible with
+# the PHP memcached client.
+#
+# backends for local hash array and SQL table included:
+#  $bag = new HashBagOStuff();
+#  $bag = new MysqlBagOStuff($tablename); # connect to db first
+
+class /* abstract */ BagOStuff {
+       var $debugmode;
+       
+       function BagOStuff() {
+               set_debug( false );
+       }
+       
+       function set_debug($bool) {
+               $this->debugmode = $bool;
+       }
+       
+       /* *** THE GUTS OF THE OPERATION *** */
+       /* Override these with functional things in subclasses */
+       
+       function get($key) {
+               /* stub */
+               return false;
+       }
+
+       function set($key, $value, $exptime=0) {
+               /* stub */
+               return false;
+       }
+       
+       function delete($key, $time=0) {
+               /* stub */
+               return false;
+       }
+       
+       /* *** Emulated functions *** */
+       /* Better performance can likely be got with custom written versions */
+       function get_multi($keys) {
+               $out = array();
+               foreach($keys as $key)
+                       $out[$key] = $this->get($key);
+               return $out;
+       }
+       
+       function set_multi($hash, $exptime=0) {
+               foreach($hash as $key => $value)
+                       $this->set($key, $value, $exptime);
+       }
+       
+       function add($key, $value, $exptime=0) {
+               if( $this->get($key) === false )
+                       $this->set($key, $value, $exptime);
+       }
+       
+       function add_multi($hash, $exptime=0) {
+               foreach($hash as $key => $value)
+                       $this->add($key, $value, $exptime);
+       }
+
+       function delete_multi($keys, $time=0) {
+               foreach($keys as $key)
+                       $this->delete($key, $time);
+       }
+       
+       function replace($key, $value, $exptime=0) {
+               if( $this->get($key) !== false )
+                       $this->set($key, $value, $exptime);
+       }
+       
+       function incr($key, $value=1) {
+               $value = intval($value);
+               if($value < 0) $value = 0;
+               if( ($n = $this->get($key)) !== false ) {
+                       $this->set($key, $n+$value); // exptime?
+                       return $n+$value;
+               } else {
+                       return false;
+               }
+       }
+       
+       function decr($key, $value=1) {
+               $value = intval($value);
+               if($value < 0) $value = 0;
+               if( ($n = $this->get($key)) !== false ) {
+                       $m = $n - $value;
+                       if($m < 0) $m = 0;
+                       $this->set($key, $m); // exptime?
+                       return $m;
+               } else {
+                       return false;
+               }
+       }
+       
+       function _debug($text) {
+               if($this->debugmode)
+                       echo "\ndebug: $text\n";
+       }
+}
+
+
+/* Functional versions! */
+class HashBagOStuff extends BagOStuff {
+       /*
+          This is a test of the interface, mainly. It stores
+          things in an associative array, which is not going to
+          persist between program runs.
+       */
+       var $bag;
+       
+       function HashBagOStuff() {
+               $this->bag = array();
+       }
+       
+       function _expire($key) {
+               $et = $this->bag[$key][1];
+               if(($et == 0) || ($et > time()))
+                       return false;
+               $this->delete($key);
+               return true;
+       }
+       
+       function get($key) {
+               if(!$this->bag[$key])
+                       return false;
+               if($this->_expire($key))
+                       return false;
+               return $this->bag[$key][0];
+       }
+       
+       function set($key,$value,$exptime=0) {
+               if(($exptime != 0) && ($exptime < 3600*24*30))
+                       $exptime = time() + $exptime;
+               $this->bag[$key] = array( $value, $exptime );
+       }
+       
+       function delete($key,$time=0) {
+               if(!$this->bag[$key])
+                       return false;
+               unset($this->bag[$key]);
+               return true;
+       }
+}
+
+/*
+CREATE TABLE objectcache (
+  keyname char(255) binary not null default '',
+  value mediumblob,
+  exptime datetime,
+  unique key (keyname),
+  key (exptime)
+);
+*/
+class /* abstract */ SqlBagOStuff extends BagOStuff {
+       var $table;
+       function SqlBagOStuff($tablename = "objectcache") {
+               $this->table = $tablename;
+       }
+       
+       function get($key) {
+               /* expire old entries if any */
+               $this->expireall();
+               
+               $res = $this->_query(
+                       "SELECT value,exptime FROM $0 WHERE keyname='$1'", $key);
+               if(!$res) {
+                       $this->_debug("get: ** error: " . $this->_dberror($res) . " **");
+                       return false;
+               }
+               if($arr = $this->_fetchrow($res)) {
+                       $this->_debug("get: retrieved data; exp time is " . $arr['exptime']);
+                       return unserialize($arr['value']);
+               } else {
+                       $this->_debug("get: no matching rows");
+               }
+               return false;
+       }
+       
+       function set($key,$value,$exptime=0) {
+               $exptime = intval($exptime);
+               if($exptime < 0) $exptime = 0;
+               if($exptime == 0) {
+                       $exp = $this->_maxdatetime();
+               } else {
+                       if($exptime < 3600*24*30)
+                               $exptime += time();
+                       $exp = $this->_fromunixtime($exptime);
+               }
+               $this->delete( $key );
+               $this->_query(
+                       "INSERT INTO $0 (keyname,value,exptime) VALUES('$1','$2',$exp)",
+                       $key, serialize(&$value));
+               return true; /* ? */
+       }
+       
+       function delete($key,$time=0) {
+               $this->_query(
+                       "DELETE FROM $0 WHERE keyname='$1'", $key );
+               return true; /* ? */
+       }
+       
+       function _query($sql) {
+               $reps = func_get_args();
+               $reps[0] = $this->table;
+               // ewwww
+               for($i=0;$i<count($reps);$i++) {
+                       $sql = str_replace(
+                               "$" . $i,
+                               $this->_strencode($reps[$i]),
+                               $sql);
+               }
+               $res = $this->_doquery($sql);
+               if($res === false) {
+                       $this->_debug("query failed: " . $this->_dberror($res));
+               }
+               return $res;
+       }
+       
+       function _strencode($str) {
+               /* Protect strings in SQL */
+               return str_replace( "'", "''", $str );
+       }
+       
+       function _doquery($sql) {
+               die( "abstract function SqlBagOStuff::_doquery() must be defined" );
+       }
+       
+       function _fetchrow($res) {
+               die( "abstract function SqlBagOStuff::_fetchrow() must be defined" );
+       }
+       
+       function _freeresult($result) {
+               /* stub */
+               return false;
+       }
+       
+       function _dberror($result) {
+               /* stub */
+               return "unknown error";
+       }
+       
+       function _maxdatetime() {
+               die( "abstract function SqlBagOStuff::_maxdatetime() must be defined" );
+       }
+       
+       function _fromunixtime() {
+               die( "abstract function SqlBagOStuff::_fromunixtime() must be defined" );
+       }
+       
+       function expireall() {
+               /* Remove any items that have expired */
+               $this->_query( "DELETE FROM $0 WHERE exptime<=NOW()" );
+       }
+       
+       function deleteall() {
+               /* Clear *all* items from cache table */
+               $this->_query( "DELETE FROM $0" );
+       }
+}
+
+class MysqlBagOStuff extends SqlBagOStuff {
+       function _doquery($sql) {
+               return mysql_query($sql);
+       }
+       function _fetchrow($result) {
+               return mysql_fetch_array($result);
+       }
+       function _freeresult($result) {
+               return mysql_free_result($result);
+       }
+       function _dberror($result) {
+               if($result)
+                       return mysql_error($result);
+               else
+                       return mysql_error();
+       }
+       
+       function _maxdatetime() {
+               return "'9999-12-31 12:59:59'";
+       }
+       
+       function _fromunixtime($ts) {
+               return "FROM_UNIXTIME($ts)";
+       }
+       
+       function _strencode($s) {
+               return mysql_escape_string($s);
+       }
+}
+
+class MediaWikiBagOStuff extends MysqlBagOStuff {
+       function _doquery($sql) {
+               return wfQuery($sql, DB_READ, "MediaWikiBagOStuff:_doquery");
+       }
+       function _freeresult($result) {
+               return wfFreeResult($result);
+       }
+}
+
+?>
index 6a55446..065f7d7 100644 (file)
@@ -128,8 +128,14 @@ if( $wgUseMemCached ) {
                $wgUseMemCached = false;
                $wgMemc = new FakeMemCachedClient();
        }
+       $messageMemc = &$wgMemc;
 } else {
        $wgMemc = new FakeMemCachedClient();
+       
+       # Give the message cache a separate cache in the DB.
+       # This is a speedup over separately querying every message used
+       require_once( "ObjectCache.php" );
+       $messageMemc = new MediaWikiBagOStuff("objectcache");
 }
 
 wfProfileOut( "$fname-memcached" );
@@ -149,7 +155,7 @@ $wgLang = new $wgLangClass();
 if ( !is_object($wgLang) ) {
        print "No language class ($wgLang)\N";
 }
-$wgMessageCache->initialise( $wgUseMemCached, $wgUseDatabaseMessages, $wgMsgCacheExpiry, $wgDBname );
+$wgMessageCache->initialise( $messageMemc, $wgUseDatabaseMessages, $wgMsgCacheExpiry, $wgDBname );
 
 $wgOut = new OutputPage();
 wfDebug( "\n\n" );
index cae48cd..2585672 100644 (file)
@@ -173,5 +173,7 @@ patch-linktables.sql
 * 2004-04: Add user_real_name field
 patch-user-realname.sql
 
-* 2004-05-08: Add querycache table for caching special pages
+* 2004-05-08: Add querycache table for caching special pages and generic
+       object cache to cover some slow operations w/o memcached.
 patch-querycache.sql
+patch-objectcache.sql
diff --git a/maintenance/archives/patch-objectcache.sql b/maintenance/archives/patch-objectcache.sql
new file mode 100644 (file)
index 0000000..7d79c74
--- /dev/null
@@ -0,0 +1,8 @@
+-- For a few generic cache operations if not using Memcached
+CREATE TABLE objectcache (
+  keyname char(255) binary not null default '',
+  value mediumblob,
+  exptime datetime,
+  unique key (keyname),
+  key (exptime)
+);
index b98e225..3e15e1d 100644 (file)
@@ -217,3 +217,12 @@ CREATE TABLE querycache (
   qc_title char(255) binary NOT NULL default '',
   KEY (qc_type,qc_value)
 );
+
+-- For a few generic cache operations if not using Memcached
+CREATE TABLE objectcache (
+  keyname char(255) binary not null default '',
+  value mediumblob,
+  exptime datetime,
+  unique key (keyname),
+  key (exptime)
+);
index 7ecb39c..9509c0a 100644 (file)
@@ -145,4 +145,15 @@ function do_querycache_update() {
        }
 }
 
+function do_objectcache_update() {
+       global $wgDatabase;
+       if( $wgDatabase->tableExists( "objectcache" ) ) {
+               echo "...have objectcache table.\n";
+       } else {
+               echo "Adding objectcache table for message caching... ";
+               dbsource( "maintenance/archives/patch-objectcache.sql", $wgDatabase );
+               echo "ok\n";
+       }
+}
+
 ?>
\ No newline at end of file