4 * Base class for general text storage via the "object" flag in old_flags, or
5 * two-part external storage URLs. Used for represent efficient concatenated
6 * storage, and migration-related pointer objects.
11 * Adds an item of text, returns a stub object which points to the item.
12 * You must call setLocation() on the stub object before storing it to the
14 * Returns the key for getItem()
16 public function addItem( $text );
19 * Get item by key, or false if the key is not present
21 public function getItem( $key );
24 * Set the "default text"
25 * This concept is an odd property of the current DB schema, whereby each text item has a revision
26 * associated with it. The default text is the text of the associated revision. There may, however,
27 * be other revisions in the same object.
29 * Default text is not required for two-part external storage URLs.
31 public function setText( $text );
34 * Get default text. This is called from Revision::getRevisionText()
40 * Concatenated gzip (CGZ) storage
41 * Improves compression ratio by concatenating like objects before gzipping
43 class ConcatenatedGzipHistoryBlob
implements HistoryBlob
45 public $mVersion = 0, $mCompressed = false, $mItems = array(), $mDefaultHash = '';
47 public $mMaxSize = 10000000;
48 public $mMaxCount = 100;
51 public function ConcatenatedGzipHistoryBlob() {
52 if ( !function_exists( 'gzdeflate' ) ) {
53 throw new MWException( "Need zlib support to read or write this kind of history object (ConcatenatedGzipHistoryBlob)\n" );
57 public function addItem( $text ) {
60 if ( !isset( $this->mItems
[$hash] ) ) {
61 $this->mItems
[$hash] = $text;
62 $this->mSize +
= strlen( $text );
67 public function getItem( $hash ) {
69 if ( array_key_exists( $hash, $this->mItems
) ) {
70 return $this->mItems
[$hash];
76 public function setText( $text ) {
78 $this->mDefaultHash
= $this->addItem( $text );
81 public function getText() {
83 return $this->getItem( $this->mDefaultHash
);
89 public function removeItem( $hash ) {
90 $this->mSize
-= strlen( $this->mItems
[$hash] );
91 unset( $this->mItems
[$hash] );
95 * Compress the bulk data in the object
97 public function compress() {
98 if ( !$this->mCompressed
) {
99 $this->mItems
= gzdeflate( serialize( $this->mItems
) );
100 $this->mCompressed
= true;
105 * Uncompress bulk data
107 public function uncompress() {
108 if ( $this->mCompressed
) {
109 $this->mItems
= unserialize( gzinflate( $this->mItems
) );
110 $this->mCompressed
= false;
117 return array( 'mVersion', 'mCompressed', 'mItems', 'mDefaultHash' );
120 function __wakeup() {
125 * Helper function for compression jobs
126 * Returns true until the object is "full" and ready to be committed
128 public function isHappy() {
129 return $this->mSize
< $this->mMaxSize
130 && count( $this->mItems
) < $this->mMaxCount
;
136 * One-step cache variable to hold base blobs; operations that
137 * pull multiple revisions may often pull multiple times from
138 * the same blob. By keeping the last-used one open, we avoid
139 * redundant unserialization and decompression overhead.
142 $wgBlobCache = array();
146 * Pointer object for an item within a CGZ blob stored in the text table.
148 class HistoryBlobStub
{
149 var $mOldId, $mHash, $mRef;
152 * @param string $hash The content hash of the text
153 * @param integer $oldid The old_id for the CGZ object
155 function HistoryBlobStub( $hash = '', $oldid = 0 ) {
156 $this->mHash
= $hash;
160 * Sets the location (old_id) of the main object to which this object
163 function setLocation( $id ) {
168 * Sets the location (old_id) of the referring object
170 function setReferrer( $id ) {
175 * Gets the location of the referring object
177 function getReferrer() {
182 $fname = 'HistoryBlobStub::getText';
184 if( isset( $wgBlobCache[$this->mOldId
] ) ) {
185 $obj = $wgBlobCache[$this->mOldId
];
187 $dbr = wfGetDB( DB_SLAVE
);
188 $row = $dbr->selectRow( 'text', array( 'old_flags', 'old_text' ), array( 'old_id' => $this->mOldId
) );
192 $flags = explode( ',', $row->old_flags
);
193 if( in_array( 'external', $flags ) ) {
195 @list
( /* $proto */ ,$path)=explode('://',$url,2);
197 wfProfileOut( $fname );
200 $row->old_text
=ExternalStore
::fetchFromUrl($url);
203 if( !in_array( 'object', $flags ) ) {
207 if( in_array( 'gzip', $flags ) ) {
208 // This shouldn't happen, but a bug in the compress script
209 // may at times gzip-compress a HistoryBlob object row.
210 $obj = unserialize( gzinflate( $row->old_text
) );
212 $obj = unserialize( $row->old_text
);
215 if( !is_object( $obj ) ) {
216 // Correct for old double-serialization bug.
217 $obj = unserialize( $obj );
220 // Save this item for reference; if pulling many
221 // items in a row we'll likely use it again.
223 $wgBlobCache = array( $this->mOldId
=> $obj );
225 return $obj->getItem( $this->mHash
);
229 * Get the content hash
238 * To speed up conversion from 1.4 to 1.5 schema, text rows can refer to the
239 * leftover cur table as the backend. This avoids expensively copying hundreds
240 * of megabytes of data during the conversion downtime.
242 * Serialized HistoryBlobCurStub objects will be inserted into the text table
243 * on conversion if $wgFastSchemaUpgrades is set to true.
245 class HistoryBlobCurStub
{
249 * @param integer $curid The cur_id pointed to
251 function HistoryBlobCurStub( $curid = 0 ) {
252 $this->mCurId
= $curid;
256 * Sets the location (cur_id) of the main object to which this object
259 function setLocation( $id ) {
264 $dbr = wfGetDB( DB_SLAVE
);
265 $row = $dbr->selectRow( 'cur', array( 'cur_text' ), array( 'cur_id' => $this->mCurId
) );
269 return $row->cur_text
;
274 * Diff-based history compression
275 * Requires xdiff 1.5+ and zlib
277 class DiffHistoryBlob
implements HistoryBlob
{
278 /** Uncompressed item cache */
279 var $mItems = array();
281 /** Total uncompressed size */
285 * Array of diffs. If a diff D from A to B is notated D = B - A, and Z is
288 * { item[map[i]] - item[map[i-1]] where i > 0
290 * { item[map[i]] - Z where i = 0
294 /** The diff map, see above */
298 * The key for getText()
308 * True if the object is locked against further writes
310 var $mFrozen = false;
313 * The maximum uncompressed size before the object becomes sad
314 * Should be less than max_allowed_packet
316 var $mMaxSize = 10000000;
319 * The maximum number of text items before the object becomes sad
321 var $mMaxCount = 100;
323 function __construct() {
324 if ( !function_exists( 'xdiff_string_bdiff' ) ){
325 throw new MWException( "Need xdiff 1.5+ support to read or write DiffHistoryBlob\n" );
327 if ( !function_exists( 'gzdeflate' ) ) {
328 throw new MWException( "Need zlib support to read or write DiffHistoryBlob\n" );
332 function addItem( $text ) {
333 if ( $this->mFrozen
) {
334 throw new MWException( __METHOD__
.": Cannot add more items after sleep/wakeup" );
337 $this->mItems
[] = $text;
338 $this->mSize +
= strlen( $text );
339 $this->mDiffs
= null; // later
340 return count( $this->mItems
) - 1;
343 function getItem( $key ) {
344 return $this->mItems
[$key];
347 function setText( $text ) {
348 $this->mDefaultKey
= $this->addItem( $text );
352 return $this->getItem( $this->mDefaultKey
);
355 function compress() {
356 if ( isset( $this->mDiffs
) ) {
357 // Already compressed
360 if ( !count( $this->mItems
) ) {
365 // Create two diff sequences: one for main text and one for small text
380 for ( $i = 0; $i < count( $this->mItems
); $i++
) {
381 $text = $this->mItems
[$i];
385 $mainTail = $sequences['main']['tail'];
386 if ( strlen( $text ) < strlen( $mainTail ) * $smallFactor ) {
392 $seq =& $sequences[$seqName];
393 $tail = $seq['tail'];
394 $diff = $this->diff( $tail, $text );
395 $seq['diffs'][] = $diff;
397 $seq['tail'] = $text;
399 unset( $seq ); // unlink dangerous alias
401 // Knit the sequences together
403 $this->mDiffs
= array();
404 $this->mDiffMap
= array();
405 foreach ( $sequences as $seq ) {
406 if ( !count( $seq['diffs'] ) ) {
409 if ( $tail === '' ) {
410 $this->mDiffs
[] = $seq['diffs'][0];
412 $head = $this->patch( '', $seq['diffs'][0] );
413 $this->mDiffs
[] = $this->diff( $tail, $head );
415 $this->mDiffMap
[] = $seq['map'][0];
416 for ( $i = 1; $i < count( $seq['diffs'] ); $i++
) {
417 $this->mDiffs
[] = $seq['diffs'][$i];
418 $this->mDiffMap
[] = $seq['map'][$i];
420 $tail = $seq['tail'];
424 function diff( $t1, $t2 ) {
425 # Need to do a null concatenation with warnings off, due to bugs in the current version of xdiff
426 # "String is not zero-terminated"
427 wfSuppressWarnings();
428 $diff = xdiff_string_bdiff( $t1, $t2 ) . '';
433 function patch( $base, $diff ) {
434 wfSuppressWarnings();
435 $text = xdiff_string_bpatch( $base, $diff ) . '';
440 function uncompress() {
441 if ( !$this->mDiffs
) {
445 for ( $diffKey = 0; $diffKey < count( $this->mDiffs
); $diffKey++
) {
446 $textKey = $this->mDiffMap
[$diffKey];
447 $text = $this->patch( $tail, $this->mDiffs
[$diffKey] );
448 $this->mItems
[$textKey] = $text;
455 if ( !count( $this->mItems
) ) {
459 // Take forward differences to improve the compression ratio for sequences
462 foreach ( $this->mDiffMap
as $i ) {
470 'diffs' => $this->mDiffs
,
474 if ( isset( $this->mDefaultKey
) ) {
475 $info['default'] = $this->mDefaultKey
;
477 $this->mCompressed
= gzdeflate( serialize( $info ) );
478 return array( 'mCompressed' );
481 function __wakeup() {
482 // addItem() doesn't work if mItems is partially filled from mDiffs
483 $this->mFrozen
= true;
484 $info = unserialize( gzinflate( $this->mCompressed
) );
485 unset( $this->mCompressed
);
492 if ( isset( $info['default'] ) ) {
493 $this->mDefaultKey
= $info['default'];
495 $map = explode( ',', $info['map'] );
497 $this->mDiffMap
= array();
498 foreach ( $map as $i ) {
500 $this->mDiffMap
[] = $cur;
502 $this->mDiffs
= $info['diffs'];
507 * Helper function for compression jobs
508 * Returns true until the object is "full" and ready to be committed
511 return $this->mSize
< $this->mMaxSize
512 && count( $this->mItems
) < $this->mMaxCount
;