From c592011404221a439d7fd88a3811d7eb0a0db4b2 Mon Sep 17 00:00:00 2001 From: Kunal Mehta Date: Sat, 14 Dec 2013 23:08:29 -0800 Subject: [PATCH] Add BagOStuff::setMulti for batch insertions Includes implementions for Redis, Sql and MemcachedPecl, other types will fallback to using $this->set repeatedly. Change-Id: I0924a197b28ee69e883128ccd672343e5c041929 --- includes/objectcache/BagOStuff.php | 17 +++++ .../objectcache/MemcachedPeclBagOStuff.php | 21 ++++++ includes/objectcache/RedisBagOStuff.php | 53 ++++++++++++++ includes/objectcache/SqlBagOStuff.php | 70 +++++++++++++++++++ 4 files changed, 161 insertions(+) diff --git a/includes/objectcache/BagOStuff.php b/includes/objectcache/BagOStuff.php index 74af7a4f26..56f1be23b5 100644 --- a/includes/objectcache/BagOStuff.php +++ b/includes/objectcache/BagOStuff.php @@ -247,6 +247,23 @@ abstract class BagOStuff { return $res; } + /** + * Batch insertion + * @param array $data $key => $value assoc array + * @param int $exptime Either an interval in seconds or a unix timestamp for expiry + * @return bool success + * @since 1.24 + */ + public function setMulti( array $data, $exptime = 0 ) { + $res = true; + foreach ( $data as $key => $value ) { + if ( !$this->set( $key, $value, $exptime ) ) { + $res = false; + } + } + return $res; + } + /** * @param string $key * @param mixed $value diff --git a/includes/objectcache/MemcachedPeclBagOStuff.php b/includes/objectcache/MemcachedPeclBagOStuff.php index 5a96f7b48c..f7dfe46a90 100644 --- a/includes/objectcache/MemcachedPeclBagOStuff.php +++ b/includes/objectcache/MemcachedPeclBagOStuff.php @@ -250,6 +250,27 @@ class MemcachedPeclBagOStuff extends MemcachedBagOStuff { return $this->checkResult( false, $result ); } + /** + * @param array $data + * @param int $exptime + * @return bool + */ + public function setMulti( array $data, $exptime = 0 ) { + wfProfileIn( __METHOD__ ); + foreach ( $data as $key => $value ) { + $encKey = $this->encodeKey( $key ); + if ( $encKey !== $key ) { + $data[$encKey] = $value; + unset( $data[$key] ); + } + } + $this->debugLog( 'setMulti(' . implode( ', ', array_keys( $data ) ) . ')' ); + $result = $this->client->setMulti( $data, $this->fixExpiry( $exptime ) ); + wfProfileOut( __METHOD__ ); + return $this->checkResult( false, $result ); + } + + /* NOTE: there is no cas() method here because it is currently not supported * by the BagOStuff interface and other BagOStuff subclasses, such as * SqlBagOStuff. diff --git a/includes/objectcache/RedisBagOStuff.php b/includes/objectcache/RedisBagOStuff.php index d6d49ae582..e770b73886 100644 --- a/includes/objectcache/RedisBagOStuff.php +++ b/includes/objectcache/RedisBagOStuff.php @@ -211,6 +211,59 @@ class RedisBagOStuff extends BagOStuff { return $result; } + /** + * @param array $data + * @param int $expiry + * @return bool + */ + public function setMulti( array $data, $expiry = 0 ) { + $section = new ProfileSection( __METHOD__ ); + + $batches = array(); + $conns = array(); + foreach ( $data as $key => $value ) { + list( $server, $conn ) = $this->getConnection( $key ); + if ( !$conn ) { + continue; + } + $conns[$server] = $conn; + $batches[$server][] = $key; + } + + $expiry = $this->convertToRelative( $expiry ); + $result = true; + foreach ( $batches as $server => $batchKeys ) { + $conn = $conns[$server]; + try { + $conn->multi( Redis::PIPELINE ); + foreach ( $batchKeys as $key ) { + if ( $expiry ) { + $conn->setex( $key, $expiry, $this->serialize( $data[$key] ) ); + } else { + $conn->set( $key, $this->serialize( $data[$key] ) ); + } + } + $batchResult = $conn->exec(); + if ( $batchResult === false ) { + $this->debug( "setMulti request to $server failed" ); + continue; + } + foreach ( $batchResult as $value ) { + if ( $value === false ) { + $result = false; + } + } + } catch ( RedisException $e ) { + $this->handleException( $server, $conn, $e ); + $result = false; + } + } + + return $result; + } + + + public function add( $key, $value, $expiry = 0 ) { $section = new ProfileSection( __METHOD__ ); diff --git a/includes/objectcache/SqlBagOStuff.php b/includes/objectcache/SqlBagOStuff.php index 8bf0c8144e..e6a8c45ed9 100644 --- a/includes/objectcache/SqlBagOStuff.php +++ b/includes/objectcache/SqlBagOStuff.php @@ -271,6 +271,76 @@ class SqlBagOStuff extends BagOStuff { return $values; } + /** + * @param array $data + * @param int $expiry + * @return bool + */ + public function setMulti( array $data, $expiry = 0 ) { + $keysByTable = array(); + foreach ( $data as $key => $value ) { + list( $serverIndex, $tableName ) = $this->getTableByKey( $key ); + $keysByTable[$serverIndex][$tableName][] = $key; + } + + $this->garbageCollect(); // expire old entries if any + + $result = true; + $exptime = (int)$expiry; + foreach ( $keysByTable as $serverIndex => $serverKeys ) { + try { + $db = $this->getDB( $serverIndex ); + } catch ( DBError $e ) { + $this->handleWriteError( $e, $serverIndex ); + $result = false; + continue; + } + + if ( $exptime < 0 ) { + $exptime = 0; + } + + if ( $exptime == 0 ) { + $encExpiry = $this->getMaxDateTime( $db ); + } else { + if ( $exptime < 3.16e8 ) { # ~10 years + $exptime += time(); + } + $encExpiry = $db->timestamp( $exptime ); + } + foreach ( $serverKeys as $tableName => $tableKeys ) { + $rows = array(); + foreach ( $tableKeys as $key ) { + $rows[] = array( + 'keyname' => $key, + 'value' => $db->encodeBlob( $this->serialize( $data[$key] ) ), + 'exptime' => $encExpiry, + ); + } + + try { + $db->commit( __METHOD__, 'flush' ); + $db->replace( + $tableName, + array( 'keyname' ), + $rows, + __METHOD__ + ); + $db->commit( __METHOD__, 'flush' ); + } catch ( DBError $e ) { + $this->handleWriteError( $e, $serverIndex ); + $result = false; + } + + } + + } + + return $result; + } + + + /** * @param string $key * @param mixed $value -- 2.20.1