maintenance/backupPrefetch.inc

   1 <?php
   2
   3 /**
   4  * Readahead helper for making large MediaWiki data dumps;
   5  * reads in a previous XML dump to sequentially prefetch text
   6  * records already normalized and decompressed.
   7  *
   8  * This can save load on the external database servers, hopefully.
   9  *
  10  * Assumes that dumps will be recorded in the canonical order:
  11  * - ascending by page_id
  12  * - ascending by rev_id within each page
  13  * - text contents are immutable and should not change once
  14  *   recorded, so the previous dump is a reliable source
  15  *
  16  * Requires PHP 5 and the XMLReader PECL extension.
  17  */
  18 class BaseDump {
  19         var $reader = null;
  20         var $atEnd = false;
  21         var $lastPage = 0;
  22         var $lastRev = 0;
  23
  24         function BaseDump( $infile ) {
  25                 $this->reader = new XMLReader();
  26                 $this->reader->open( $infile );
  27         }
  28
  29         /**
  30          * Attempts to fetch the text of a particular page revision
  31          * from the dump stream. May return null if the page is
  32          * unavailable.
  33          *
  34          * @param int $page ID number of page to read
  35          * @param int $rev ID number of revision to read
  36          * @return string or null
  37          */
  38         function prefetch( $page, $rev ) {
  39                 while( $this->lastPage < $page && !$this->atEnd ) {
  40                         $this->nextPage();
  41                 }
  42                 if( $this->lastPage > $page || $this->atEnd ) {
  43                         $this->debug( "BaseDump::prefetch already past page $page looking for rev $rev\n" );
  44                         return null;
  45                 }
  46                 while( $this->lastRev < $rev && !$this->atEnd ) {
  47                         $this->nextRev();
  48                 }
  49                 if( $this->lastRev == $rev ) {
  50                         $this->debug( "BaseDump::prefetch hit on $page, $rev\n" );
  51                         return $this->nextText();
  52                 } else {
  53                         $this->debug( "BaseDump::prefetch already past rev $rev on page $page\n" );
  54                         return null;
  55                 }
  56         }
  57
  58         function debug( $str ) {
  59                 wfDebug( $str );
  60                 //global $dumper;
  61                 //$dumper->progress( $str );
  62         }
  63
  64         /**
  65          * @access private
  66          */
  67         function nextPage() {
  68                 $this->skipTo( 'page' );
  69                 $this->skipTo( 'id' );
  70                 $this->lastPage = intval( $this->nodeContents() );
  71                 $this->lastRev = 0;
  72         }
  73
  74         /**
  75          * @access private
  76          */
  77         function nextRev() {
  78                 $this->skipTo( 'revision' );
  79                 $this->skipTo( 'id' );
  80                 $this->lastRev = intval( $this->nodeContents() );
  81         }
  82
  83         /**
  84          * @access private
  85          */
  86         function nextText() {
  87                 $this->skipTo( 'text' );
  88                 return strval( $this->nodeContents() );
  89         }
  90
  91         /**
  92          * @access private
  93          */
  94         function skipTo( $name ) {
  95                 while( $this->reader->read() ) {
  96                         if( $this->reader->nodeType == XMLREADER_ELEMENT &&
  97                                 $this->reader->name == $name ) {
  98                                 return true;
  99                         }
 100                 }
 101                 return $this->close();
 102         }
 103
 104         /**
 105          * Shouldn't something like this be built-in to XMLReader?
 106          * Fetches text contents of the current element, assuming
 107          * no sub-elements or such scary things.
 108          * @return string
 109          * @access private
 110          */
 111         function nodeContents() {
 112                 if( $this->reader->isEmptyElement ) {
 113                         return "";
 114                 }
 115                 $buffer = "";
 116                 while( $this->reader->read() ) {
 117                         switch( $this->reader->nodeType ) {
 118                         case XMLREADER_TEXT:
 119 //                      case XMLREADER_WHITESPACE:
 120                         case XMLREADER_SIGNIFICANT_WHITESPACE:
 121                                 $buffer .= $this->reader->value;
 122                                 break;
 123                         case XMLREADER_END_ELEMENT:
 124                                 return $buffer;
 125                         }
 126                 }
 127                 return $this->close();
 128         }
 129
 130         /**
 131          * @access private
 132          */
 133         function close() {
 134                 $this->reader->close();
 135                 $this->reader = null;
 136                 $this->atEnd = true;
 137                 return false;
 138         }
 139 }
 140
 141 ?>