From 0c2fba0ac401938729a14c3bda5e793a96295139 Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Sun, 9 May 2004 05:12:55 +0000 Subject: [PATCH] Add an objectcache table for limited caching when memcached isn't available. Currently using for the message cache to avoid reading every message separately. This now is only slightly slower than memcached in my tests when $wgUseDatabaseMessages is enabled, so it's a bit of a speedup for common hosts. --- config/index.php | 1 + includes/MessageCache.php | 26 +- includes/ObjectCache.php | 320 +++++++++++++++++++++ includes/Setup.php | 8 +- maintenance/archives/patch-list.txt | 4 +- maintenance/archives/patch-objectcache.sql | 8 + maintenance/tables.sql | 9 + maintenance/updaters.inc | 11 + 8 files changed, 370 insertions(+), 17 deletions(-) create mode 100644 includes/ObjectCache.php create mode 100644 maintenance/archives/patch-objectcache.sql diff --git a/config/index.php b/config/index.php index e2010a6011..6382837547 100644 --- a/config/index.php +++ b/config/index.php @@ -398,6 +398,7 @@ if( $conf->posted && ( 0 == count( $errs ) ) ) { echo "FIXME: need the link table change here\n"; do_user_real_name_update(); flush(); do_querycache_update(); flush(); + do_objectcache_update(); flush(); initialiseMessages(); flush(); chdir( "config" ); diff --git a/includes/MessageCache.php b/includes/MessageCache.php index 9a4a18f464..e3a44e928b 100755 --- a/includes/MessageCache.php +++ b/includes/MessageCache.php @@ -14,8 +14,9 @@ class MessageCache var $mInitialised = false; - function initialise( $useMemCached, $useDB, $expiry, $memcPrefix ) { - $this->mUseCache = $useMemCached; + function initialise( &$memCached, $useDB, $expiry, $memcPrefix ) { + $this->mUseCache = !is_null( $memCached ); + $this->mMemc = &$memCached; $this->mDisable = !$useDB; $this->mExpiry = $expiry; $this->mDisableTransform = false; @@ -32,7 +33,7 @@ class MessageCache # On error, quietly switches to a fallback mode # Returns false for a reportable error, true otherwise function load() { - global $wgAllMessagesEn, $wgMemc; + global $wgAllMessagesEn; if ( $this->mDisable ) { return true; @@ -41,18 +42,18 @@ class MessageCache $success = true; if ( $this->mUseCache ) { - $this->mCache = $wgMemc->get( $this->mMemcKey ); + $this->mCache = $this->mMemc->get( $this->mMemcKey ); # If there's nothing in memcached, load all the messages from the database if ( !$this->mCache ) { $this->lock(); # Other threads don't need to load the messages if another thread is doing it. - $wgMemc->set( $this->mMemcKey, "loading", MSG_LOAD_TIMEOUT ); + $this->mMemc->set( $this->mMemcKey, "loading", MSG_LOAD_TIMEOUT ); $this->loadFromDB(); # Save in memcached - if ( !$wgMemc->set( $this->mMemcKey, $this->mCache, $this->mExpiry ) ) { + if ( !$this->mMemc->set( $this->mMemcKey, $this->mCache, $this->mExpiry ) ) { # Hack for slabs reassignment problem - $wgMemc->set( $this->mMemcKey, "error" ); + $this->mMemc->set( $this->mMemcKey, "error" ); wfDebug( "MemCached set error in MessageCache: restart memcached server!\n" ); } $this->unlock(); @@ -113,12 +114,11 @@ class MessageCache } function replace( $title, $text ) { - global $wgMemc; $this->lock(); $this->load(); if ( is_array( $this->mCache ) ) { $this->mCache[$title] = $text; - $wgMemc->set( $this->mMemcKey, $this->mCache, $this->mExpiry ); + $this->mMemc->set( $this->mMemcKey, $this->mCache, $this->mExpiry ); } $this->unlock(); } @@ -126,14 +126,12 @@ class MessageCache # Returns success # Represents a write lock on the messages key function lock() { - global $wgMemc; - if ( !$this->mUseCache ) { return true; } $lockKey = $this->mMemcKey . "lock"; - for ($i=0; $i < MSG_WAIT_TIMEOUT && !$wgMemc->add( $lockKey, 1, MSG_LOCK_TIMEOUT ); $i++ ) { + for ($i=0; $i < MSG_WAIT_TIMEOUT && !$this->mMemc->add( $lockKey, 1, MSG_LOCK_TIMEOUT ); $i++ ) { sleep(1); } @@ -141,14 +139,12 @@ class MessageCache } function unlock() { - global $wgMemc; - if ( !$this->mUseCache ) { return; } $lockKey = $this->mMemcKey . "lock"; - $wgMemc->delete( $lockKey ); + $this->mMemc->delete( $lockKey ); } function get( $key, $useDB ) { diff --git a/includes/ObjectCache.php b/includes/ObjectCache.php new file mode 100644 index 0000000000..00864c6b16 --- /dev/null +++ b/includes/ObjectCache.php @@ -0,0 +1,320 @@ + +# http://www.mediawiki.org/ +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# http://www.gnu.org/copyleft/gpl.html + +# Simple generic object store +# interface is intended to be more or less compatible with +# the PHP memcached client. +# +# backends for local hash array and SQL table included: +# $bag = new HashBagOStuff(); +# $bag = new MysqlBagOStuff($tablename); # connect to db first + +class /* abstract */ BagOStuff { + var $debugmode; + + function BagOStuff() { + set_debug( false ); + } + + function set_debug($bool) { + $this->debugmode = $bool; + } + + /* *** THE GUTS OF THE OPERATION *** */ + /* Override these with functional things in subclasses */ + + function get($key) { + /* stub */ + return false; + } + + function set($key, $value, $exptime=0) { + /* stub */ + return false; + } + + function delete($key, $time=0) { + /* stub */ + return false; + } + + /* *** Emulated functions *** */ + /* Better performance can likely be got with custom written versions */ + function get_multi($keys) { + $out = array(); + foreach($keys as $key) + $out[$key] = $this->get($key); + return $out; + } + + function set_multi($hash, $exptime=0) { + foreach($hash as $key => $value) + $this->set($key, $value, $exptime); + } + + function add($key, $value, $exptime=0) { + if( $this->get($key) === false ) + $this->set($key, $value, $exptime); + } + + function add_multi($hash, $exptime=0) { + foreach($hash as $key => $value) + $this->add($key, $value, $exptime); + } + + function delete_multi($keys, $time=0) { + foreach($keys as $key) + $this->delete($key, $time); + } + + function replace($key, $value, $exptime=0) { + if( $this->get($key) !== false ) + $this->set($key, $value, $exptime); + } + + function incr($key, $value=1) { + $value = intval($value); + if($value < 0) $value = 0; + if( ($n = $this->get($key)) !== false ) { + $this->set($key, $n+$value); // exptime? + return $n+$value; + } else { + return false; + } + } + + function decr($key, $value=1) { + $value = intval($value); + if($value < 0) $value = 0; + if( ($n = $this->get($key)) !== false ) { + $m = $n - $value; + if($m < 0) $m = 0; + $this->set($key, $m); // exptime? + return $m; + } else { + return false; + } + } + + function _debug($text) { + if($this->debugmode) + echo "\ndebug: $text\n"; + } +} + + +/* Functional versions! */ +class HashBagOStuff extends BagOStuff { + /* + This is a test of the interface, mainly. It stores + things in an associative array, which is not going to + persist between program runs. + */ + var $bag; + + function HashBagOStuff() { + $this->bag = array(); + } + + function _expire($key) { + $et = $this->bag[$key][1]; + if(($et == 0) || ($et > time())) + return false; + $this->delete($key); + return true; + } + + function get($key) { + if(!$this->bag[$key]) + return false; + if($this->_expire($key)) + return false; + return $this->bag[$key][0]; + } + + function set($key,$value,$exptime=0) { + if(($exptime != 0) && ($exptime < 3600*24*30)) + $exptime = time() + $exptime; + $this->bag[$key] = array( $value, $exptime ); + } + + function delete($key,$time=0) { + if(!$this->bag[$key]) + return false; + unset($this->bag[$key]); + return true; + } +} + +/* +CREATE TABLE objectcache ( + keyname char(255) binary not null default '', + value mediumblob, + exptime datetime, + unique key (keyname), + key (exptime) +); +*/ +class /* abstract */ SqlBagOStuff extends BagOStuff { + var $table; + function SqlBagOStuff($tablename = "objectcache") { + $this->table = $tablename; + } + + function get($key) { + /* expire old entries if any */ + $this->expireall(); + + $res = $this->_query( + "SELECT value,exptime FROM $0 WHERE keyname='$1'", $key); + if(!$res) { + $this->_debug("get: ** error: " . $this->_dberror($res) . " **"); + return false; + } + if($arr = $this->_fetchrow($res)) { + $this->_debug("get: retrieved data; exp time is " . $arr['exptime']); + return unserialize($arr['value']); + } else { + $this->_debug("get: no matching rows"); + } + return false; + } + + function set($key,$value,$exptime=0) { + $exptime = intval($exptime); + if($exptime < 0) $exptime = 0; + if($exptime == 0) { + $exp = $this->_maxdatetime(); + } else { + if($exptime < 3600*24*30) + $exptime += time(); + $exp = $this->_fromunixtime($exptime); + } + $this->delete( $key ); + $this->_query( + "INSERT INTO $0 (keyname,value,exptime) VALUES('$1','$2',$exp)", + $key, serialize(&$value)); + return true; /* ? */ + } + + function delete($key,$time=0) { + $this->_query( + "DELETE FROM $0 WHERE keyname='$1'", $key ); + return true; /* ? */ + } + + function _query($sql) { + $reps = func_get_args(); + $reps[0] = $this->table; + // ewwww + for($i=0;$i_strencode($reps[$i]), + $sql); + } + $res = $this->_doquery($sql); + if($res === false) { + $this->_debug("query failed: " . $this->_dberror($res)); + } + return $res; + } + + function _strencode($str) { + /* Protect strings in SQL */ + return str_replace( "'", "''", $str ); + } + + function _doquery($sql) { + die( "abstract function SqlBagOStuff::_doquery() must be defined" ); + } + + function _fetchrow($res) { + die( "abstract function SqlBagOStuff::_fetchrow() must be defined" ); + } + + function _freeresult($result) { + /* stub */ + return false; + } + + function _dberror($result) { + /* stub */ + return "unknown error"; + } + + function _maxdatetime() { + die( "abstract function SqlBagOStuff::_maxdatetime() must be defined" ); + } + + function _fromunixtime() { + die( "abstract function SqlBagOStuff::_fromunixtime() must be defined" ); + } + + function expireall() { + /* Remove any items that have expired */ + $this->_query( "DELETE FROM $0 WHERE exptime<=NOW()" ); + } + + function deleteall() { + /* Clear *all* items from cache table */ + $this->_query( "DELETE FROM $0" ); + } +} + +class MysqlBagOStuff extends SqlBagOStuff { + function _doquery($sql) { + return mysql_query($sql); + } + function _fetchrow($result) { + return mysql_fetch_array($result); + } + function _freeresult($result) { + return mysql_free_result($result); + } + function _dberror($result) { + if($result) + return mysql_error($result); + else + return mysql_error(); + } + + function _maxdatetime() { + return "'9999-12-31 12:59:59'"; + } + + function _fromunixtime($ts) { + return "FROM_UNIXTIME($ts)"; + } + + function _strencode($s) { + return mysql_escape_string($s); + } +} + +class MediaWikiBagOStuff extends MysqlBagOStuff { + function _doquery($sql) { + return wfQuery($sql, DB_READ, "MediaWikiBagOStuff:_doquery"); + } + function _freeresult($result) { + return wfFreeResult($result); + } +} + +?> diff --git a/includes/Setup.php b/includes/Setup.php index 6a55446a91..065f7d7938 100644 --- a/includes/Setup.php +++ b/includes/Setup.php @@ -128,8 +128,14 @@ if( $wgUseMemCached ) { $wgUseMemCached = false; $wgMemc = new FakeMemCachedClient(); } + $messageMemc = &$wgMemc; } else { $wgMemc = new FakeMemCachedClient(); + + # Give the message cache a separate cache in the DB. + # This is a speedup over separately querying every message used + require_once( "ObjectCache.php" ); + $messageMemc = new MediaWikiBagOStuff("objectcache"); } wfProfileOut( "$fname-memcached" ); @@ -149,7 +155,7 @@ $wgLang = new $wgLangClass(); if ( !is_object($wgLang) ) { print "No language class ($wgLang)\N"; } -$wgMessageCache->initialise( $wgUseMemCached, $wgUseDatabaseMessages, $wgMsgCacheExpiry, $wgDBname ); +$wgMessageCache->initialise( $messageMemc, $wgUseDatabaseMessages, $wgMsgCacheExpiry, $wgDBname ); $wgOut = new OutputPage(); wfDebug( "\n\n" ); diff --git a/maintenance/archives/patch-list.txt b/maintenance/archives/patch-list.txt index cae48cddea..258567289a 100644 --- a/maintenance/archives/patch-list.txt +++ b/maintenance/archives/patch-list.txt @@ -173,5 +173,7 @@ patch-linktables.sql * 2004-04: Add user_real_name field patch-user-realname.sql -* 2004-05-08: Add querycache table for caching special pages +* 2004-05-08: Add querycache table for caching special pages and generic + object cache to cover some slow operations w/o memcached. patch-querycache.sql +patch-objectcache.sql diff --git a/maintenance/archives/patch-objectcache.sql b/maintenance/archives/patch-objectcache.sql new file mode 100644 index 0000000000..7d79c74613 --- /dev/null +++ b/maintenance/archives/patch-objectcache.sql @@ -0,0 +1,8 @@ +-- For a few generic cache operations if not using Memcached +CREATE TABLE objectcache ( + keyname char(255) binary not null default '', + value mediumblob, + exptime datetime, + unique key (keyname), + key (exptime) +); diff --git a/maintenance/tables.sql b/maintenance/tables.sql index b98e22591c..3e15e1d836 100644 --- a/maintenance/tables.sql +++ b/maintenance/tables.sql @@ -217,3 +217,12 @@ CREATE TABLE querycache ( qc_title char(255) binary NOT NULL default '', KEY (qc_type,qc_value) ); + +-- For a few generic cache operations if not using Memcached +CREATE TABLE objectcache ( + keyname char(255) binary not null default '', + value mediumblob, + exptime datetime, + unique key (keyname), + key (exptime) +); diff --git a/maintenance/updaters.inc b/maintenance/updaters.inc index 7ecb39c7b9..9509c0a8fb 100644 --- a/maintenance/updaters.inc +++ b/maintenance/updaters.inc @@ -145,4 +145,15 @@ function do_querycache_update() { } } +function do_objectcache_update() { + global $wgDatabase; + if( $wgDatabase->tableExists( "objectcache" ) ) { + echo "...have objectcache table.\n"; + } else { + echo "Adding objectcache table for message caching... "; + dbsource( "maintenance/archives/patch-objectcache.sql", $wgDatabase ); + echo "ok\n"; + } +} + ?> \ No newline at end of file -- 2.20.1