3 * Object caching using Redis (http://redis.io/).
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License along
16 * with this program; if not, write to the Free Software Foundation, Inc.,
17 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18 * http://www.gnu.org/copyleft/gpl.html
24 * Redis-based caching module for redis server >= 2.6.12 and phpredis >= 2.2.4
26 * @see https://github.com/phpredis/phpredis/blob/d310ed7c8/Changelog.md
27 * @note Avoid use of Redis::MULTI transactions for twemproxy support
32 class RedisBagOStuff
extends MediumSpecificBagOStuff
{
33 /** @var RedisConnectionPool */
35 /** @var array List of server names */
37 /** @var array Map of (tag => server name) */
38 protected $serverTagMap;
40 protected $automaticFailover;
43 * Construct a RedisBagOStuff object. Parameters are:
45 * - servers: An array of server names. A server name may be a hostname,
46 * a hostname/port combination or the absolute path of a UNIX socket.
47 * If a hostname is specified but no port, the standard port number
48 * 6379 will be used. Arrays keys can be used to specify the tag to
49 * hash on in place of the host/port. Required.
51 * - connectTimeout: The timeout for new connections, in seconds. Optional,
52 * default is 1 second.
54 * - persistent: Set this to true to allow connections to persist across
55 * multiple web requests. False by default.
57 * - password: The authentication password, will be sent to Redis in
58 * clear text. Optional, if it is unspecified, no AUTH command will be
61 * - automaticFailover: If this is false, then each key will be mapped to
62 * a single server, and if that server is down, any requests for that key
63 * will fail. If this is true, a connection failure will cause the client
64 * to immediately try the next server in the list (as determined by a
65 * consistent hashing algorithm). True by default. This has the
66 * potential to create consistency issues if a server is slow enough to
67 * flap, for example if it is in swap death.
68 * @param array $params
70 function __construct( $params ) {
71 parent
::__construct( $params );
72 $redisConf = [ 'serializer' => 'none' ]; // manage that in this class
73 foreach ( [ 'connectTimeout', 'persistent', 'password' ] as $opt ) {
74 if ( isset( $params[$opt] ) ) {
75 $redisConf[$opt] = $params[$opt];
78 $this->redisPool
= RedisConnectionPool
::singleton( $redisConf );
80 $this->servers
= $params['servers'];
81 foreach ( $this->servers
as $key => $server ) {
82 $this->serverTagMap
[is_int( $key ) ?
$server : $key] = $server;
85 $this->automaticFailover
= $params['automaticFailover'] ??
true;
87 $this->attrMap
[self
::ATTR_SYNCWRITES
] = self
::QOS_SYNCWRITES_NONE
;
90 protected function doGet( $key, $flags = 0, &$casToken = null ) {
93 $conn = $this->getConnection( $key );
100 $value = $conn->get( $key );
102 $result = $this->unserialize( $value );
103 } catch ( RedisException
$e ) {
105 $this->handleException( $conn, $e );
108 $this->logRequest( 'get', $key, $conn->getServer(), $e );
113 protected function doSet( $key, $value, $exptime = 0, $flags = 0 ) {
114 $conn = $this->getConnection( $key );
119 $ttl = $this->getExpirationAsTTL( $exptime );
124 $result = $conn->setex( $key, $ttl, $this->serialize( $value ) );
126 $result = $conn->set( $key, $this->serialize( $value ) );
128 } catch ( RedisException
$e ) {
130 $this->handleException( $conn, $e );
133 $this->logRequest( 'set', $key, $conn->getServer(), $e );
138 protected function doDelete( $key, $flags = 0 ) {
139 $conn = $this->getConnection( $key );
146 // Note that redis does not return false if the key was not there
147 $result = ( $conn->del( $key ) !== false );
148 } catch ( RedisException
$e ) {
150 $this->handleException( $conn, $e );
153 $this->logRequest( 'delete', $key, $conn->getServer(), $e );
158 protected function doGetMulti( array $keys, $flags = 0 ) {
159 /** @var RedisConnRef[]|Redis[] $conns */
162 foreach ( $keys as $key ) {
163 $conn = $this->getConnection( $key );
165 $server = $conn->getServer();
166 $conns[$server] = $conn;
167 $batches[$server][] = $key;
172 foreach ( $batches as $server => $batchKeys ) {
173 $conn = $conns[$server];
177 // Avoid mget() to reduce CPU hogging from a single request
178 $conn->multi( Redis
::PIPELINE
);
179 foreach ( $batchKeys as $key ) {
182 $batchResult = $conn->exec();
183 if ( $batchResult === false ) {
184 $this->logRequest( 'get', implode( ',', $batchKeys ), $server, true );
188 foreach ( $batchResult as $i => $value ) {
189 if ( $value !== false ) {
190 $result[$batchKeys[$i]] = $this->unserialize( $value );
193 } catch ( RedisException
$e ) {
194 $this->handleException( $conn, $e );
197 $this->logRequest( 'get', implode( ',', $batchKeys ), $server, $e );
203 protected function doSetMulti( array $data, $exptime = 0, $flags = 0 ) {
204 /** @var RedisConnRef[]|Redis[] $conns */
207 foreach ( $data as $key => $value ) {
208 $conn = $this->getConnection( $key );
210 $server = $conn->getServer();
211 $conns[$server] = $conn;
212 $batches[$server][] = $key;
216 $ttl = $this->getExpirationAsTTL( $exptime );
217 $op = $ttl ?
'setex' : 'set';
220 foreach ( $batches as $server => $batchKeys ) {
221 $conn = $conns[$server];
225 // Avoid mset() to reduce CPU hogging from a single request
226 $conn->multi( Redis
::PIPELINE
);
227 foreach ( $batchKeys as $key ) {
229 $conn->setex( $key, $ttl, $this->serialize( $data[$key] ) );
231 $conn->set( $key, $this->serialize( $data[$key] ) );
234 $batchResult = $conn->exec();
235 if ( $batchResult === false ) {
236 $this->logRequest( $op, implode( ',', $batchKeys ), $server, true );
239 $result = $result && !in_array( false, $batchResult, true );
240 } catch ( RedisException
$e ) {
241 $this->handleException( $conn, $e );
245 $this->logRequest( $op, implode( ',', $batchKeys ), $server, $e );
251 protected function doDeleteMulti( array $keys, $flags = 0 ) {
252 /** @var RedisConnRef[]|Redis[] $conns */
255 foreach ( $keys as $key ) {
256 $conn = $this->getConnection( $key );
258 $server = $conn->getServer();
259 $conns[$server] = $conn;
260 $batches[$server][] = $key;
265 foreach ( $batches as $server => $batchKeys ) {
266 $conn = $conns[$server];
270 // Avoid delete() with array to reduce CPU hogging from a single request
271 $conn->multi( Redis
::PIPELINE
);
272 foreach ( $batchKeys as $key ) {
275 $batchResult = $conn->exec();
276 if ( $batchResult === false ) {
277 $this->logRequest( 'delete', implode( ',', $batchKeys ), $server, true );
280 // Note that redis does not return false if the key was not there
281 $result = $result && !in_array( false, $batchResult, true );
282 } catch ( RedisException
$e ) {
283 $this->handleException( $conn, $e );
287 $this->logRequest( 'delete', implode( ',', $batchKeys ), $server, $e );
293 public function changeTTLMulti( array $keys, $exptime, $flags = 0 ) {
294 /** @var RedisConnRef[]|Redis[] $conns */
297 foreach ( $keys as $key ) {
298 $conn = $this->getConnection( $key );
300 $server = $conn->getServer();
301 $conns[$server] = $conn;
302 $batches[$server][] = $key;
306 $relative = $this->isRelativeExpiration( $exptime );
307 $op = ( $exptime == self
::TTL_INDEFINITE
)
309 : ( $relative ?
'expire' : 'expireAt' );
312 foreach ( $batches as $server => $batchKeys ) {
313 $conn = $conns[$server];
317 $conn->multi( Redis
::PIPELINE
);
318 foreach ( $batchKeys as $key ) {
319 if ( $exptime == self
::TTL_INDEFINITE
) {
320 $conn->persist( $key );
321 } elseif ( $relative ) {
322 $conn->expire( $key, $this->getExpirationAsTTL( $exptime ) );
324 $conn->expireAt( $key, $this->getExpirationAsTimestamp( $exptime ) );
327 $batchResult = $conn->exec();
328 if ( $batchResult === false ) {
329 $this->logRequest( $op, implode( ',', $batchKeys ), $server, true );
332 $result = in_array( false, $batchResult, true ) ?
false : $result;
333 } catch ( RedisException
$e ) {
334 $this->handleException( $conn, $e );
338 $this->logRequest( $op, implode( ',', $batchKeys ), $server, $e );
344 public function add( $key, $value, $expiry = 0, $flags = 0 ) {
345 $conn = $this->getConnection( $key );
350 $ttl = $this->getExpirationAsTTL( $expiry );
352 $result = $conn->set(
354 $this->serialize( $value ),
355 $ttl ?
[ 'nx', 'ex' => $ttl ] : [ 'nx' ]
357 } catch ( RedisException
$e ) {
359 $this->handleException( $conn, $e );
362 $this->logRequest( 'add', $key, $conn->getServer(), $result );
367 public function incr( $key, $value = 1 ) {
368 $conn = $this->getConnection( $key );
374 if ( !$conn->exists( $key ) ) {
377 // @FIXME: on races, the key may have a 0 TTL
378 $result = $conn->incrBy( $key, $value );
379 } catch ( RedisException
$e ) {
381 $this->handleException( $conn, $e );
384 $this->logRequest( 'incr', $key, $conn->getServer(), $result );
389 protected function doChangeTTL( $key, $exptime, $flags ) {
390 $conn = $this->getConnection( $key );
395 $relative = $this->isRelativeExpiration( $exptime );
397 if ( $exptime == self
::TTL_INDEFINITE
) {
398 $result = $conn->persist( $key );
399 $this->logRequest( 'persist', $key, $conn->getServer(), $result );
400 } elseif ( $relative ) {
401 $result = $conn->expire( $key, $this->getExpirationAsTTL( $exptime ) );
402 $this->logRequest( 'expire', $key, $conn->getServer(), $result );
404 $result = $conn->expireAt( $key, $this->getExpirationAsTimestamp( $exptime ) );
405 $this->logRequest( 'expireAt', $key, $conn->getServer(), $result );
407 } catch ( RedisException
$e ) {
409 $this->handleException( $conn, $e );
417 * @return RedisConnRef|Redis|null Redis handle wrapper for the key or null on failure
419 protected function getConnection( $key ) {
420 $candidates = array_keys( $this->serverTagMap
);
422 if ( count( $this->servers
) > 1 ) {
423 ArrayUtils
::consistentHashSort( $candidates, $key, '/' );
424 if ( !$this->automaticFailover
) {
425 $candidates = array_slice( $candidates, 0, 1 );
429 while ( ( $tag = array_shift( $candidates ) ) !== null ) {
430 $server = $this->serverTagMap
[$tag];
431 $conn = $this->redisPool
->getConnection( $server, $this->logger
);
436 // If automatic failover is enabled, check that the server's link
437 // to its master (if any) is up -- but only if there are other
438 // viable candidates left to consider. Also, getMasterLinkStatus()
439 // does not work with twemproxy, though $candidates will be empty
440 // by now in such cases.
441 if ( $this->automaticFailover
&& $candidates ) {
443 /** @var string[] $info */
444 $info = $conn->info();
445 if ( ( $info['master_link_status'] ??
null ) === 'down' ) {
446 // If the master cannot be reached, fail-over to the next server.
447 // If masters are in data-center A, and replica DBs in data-center B,
448 // this helps avoid the case were fail-over happens in A but not
449 // to the corresponding server in B (e.g. read/write mismatch).
452 } catch ( RedisException
$e ) {
453 // Server is not accepting commands
454 $this->redisPool
->handleError( $conn, $e );
462 $this->setLastError( BagOStuff
::ERR_UNREACHABLE
);
471 protected function logError( $msg ) {
472 $this->logger
->error( "Redis error: $msg" );
476 * The redis extension throws an exception in response to various read, write
477 * and protocol errors. Sometimes it also closes the connection, sometimes
478 * not. The safest response for us is to explicitly destroy the connection
479 * object and let it be reopened during the next request.
480 * @param RedisConnRef $conn
481 * @param RedisException $e
483 protected function handleException( RedisConnRef
$conn, RedisException
$e ) {
484 $this->setLastError( BagOStuff
::ERR_UNEXPECTED
);
485 $this->redisPool
->handleError( $conn, $e );
489 * Send information about a single request to the debug log
491 * @param string $keys
492 * @param string $server
493 * @param Exception|bool|null $e
495 public function logRequest( $op, $keys, $server, $e = null ) {
496 $this->debug( "$op($keys) on $server: " . ( $e ?
"failure" : "success" ) );