maintenance/dumpHTML.inc

   1 <?php
   2 /**
   3  * @package MediaWiki
   4  * @subpackage Maintenance
   5  */
   6
   7 define( 'REPORTING_INTERVAL', 10 );
   8
   9 require_once( 'includes/ImagePage.php' );
  10 require_once( 'includes/CategoryPage.php' );
  11
  12 class DumpHTML {
  13         # Destination directory
  14         var $dest;
  15
  16         # Show interlanguage links?
  17         var $interwiki = true;
  18
  19         # Depth of HTML directory tree
  20         var $depth = 3;
  21
  22         # Directory that commons images are copied into
  23         var $sharedStaticPath;
  24
  25         # Relative path to image directory
  26         var $imageRel = 'upload';
  27
  28         # Copy commons images instead of symlinking
  29         var $forceCopy = false;
  30
  31         # Make links assuming the script path is in the same directory as
  32         # the destination
  33         var $alternateScriptPath = false;
  34
  35         function DumpHTML( $settings ) {
  36                 foreach ( $settings as $var => $value ) {
  37                         $this->$var = $value;
  38                 }
  39         }
  40
  41         /**
  42          * Write a set of articles specified by start and end page_id
  43          * Skip categories and images, they will be done separately
  44          */
  45         function doArticles( $start, $end = false ) {
  46                 $fname = 'DumpHTML::doArticles';
  47
  48                 $this->setupGlobals();
  49
  50                 if ( $end === false ) {
  51                         $dbr =& wfGetDB( DB_SLAVE );
  52                         $end = $dbr->selectField( 'page', 'max(page_id)', false, $fname );
  53                 }
  54
  55
  56                 for ($id = $start; $id <= $end; $id++) {
  57                         wfWaitForSlaves( 20 );
  58                         if ( !($id % REPORTING_INTERVAL) ) {
  59                                 print "Processing ID: $id\r";
  60                         }
  61                         if ( !($id % (REPORTING_INTERVAL*10) ) ) {
  62                                 print "\n";
  63                         }
  64                         $title = Title::newFromID( $id );
  65                         if ( $title ) {
  66                                 $ns = $title->getNamespace() ;
  67                                 if ( $ns != NS_CATEGORY ) {
  68                                         $this->doArticle( $title );
  69                                 }
  70                         }
  71                 }
  72                 print "\n";
  73         }
  74
  75         function doSpecials() {
  76                 $this->doMainPage();
  77
  78                 $this->setupGlobals();
  79                 print "Special:Categories...";
  80                 $this->doArticle( Title::makeTitle( NS_SPECIAL, 'Categories' ) );
  81                 print "\n";
  82         }
  83
  84         /** Write the main page as index.html */
  85         function doMainPage() {
  86                 global $wgMakeDumpLinks;
  87
  88                 print "Making index.html  ";
  89
  90                 // Set up globals with no ../../.. in the link URLs
  91                 $this->setupGlobals( 0 );
  92
  93                 // But still use that directory style
  94                 $wgMakeDumpLinks = 3;
  95
  96                 $title = Title::newMainPage();
  97                 $text = $this->getArticleHTML( $title );
  98                 $file = fopen( "{$this->dest}/index.html", "w" );
  99                 if ( !$file ) {
 100                         print "\nCan't open index.html for writing\n";
 101                         return false;
 102                 }
 103                 fwrite( $file, $text );
 104                 fclose( $file );
 105                 print "\n";
 106         }
 107
 108         function doImageDescriptions() {
 109                 global $wgSharedUploadDirectory;
 110
 111                 $fname = 'DumpHTML::doImageDescriptions';
 112
 113                 $this->setupGlobals( 3 );
 114
 115                 /**
 116                  * Dump image description pages that don't have an associated article, but do
 117                  * have a local image
 118                  */
 119                 $dbr =& wfGetDB( DB_SLAVE );
 120                 extract( $dbr->tableNames( 'image', 'page' ) );
 121                 $res = $dbr->select( 'image', array( 'img_name' ), false, $fname );
 122
 123                 $i = 0;
 124                 print "Writing image description pages for local images\n";
 125                 $num = $dbr->numRows( $res );
 126                 while ( $row = $dbr->fetchObject( $res ) ) {
 127                         wfWaitForSlaves( 10 );
 128                         if ( !( ++$i % REPORTING_INTERVAL ) ) {
 129                                 print "Done $i of $num\r";
 130                         }
 131                         $title = Title::makeTitle( NS_IMAGE, $row->img_name );
 132                         if ( $title->getArticleID() ) {
 133                                 // Already done by dumpHTML
 134                                 continue;
 135                         }
 136                         $this->doArticle( $title );
 137                 }
 138                 print "\n";
 139
 140                 /**
 141                  * Dump images which only have a real description page on commons
 142                  */
 143                 print "Writing description pages for commons images\n";
 144                 $i = 0;
 145                 for ( $hash = 0; $hash < 256; $hash++ ) {
 146                         $dir = sprintf( "%01x/%02x", intval( $hash / 16 ), $hash );
 147                         $paths = array_merge( glob( "{$this->sharedStaticPath}/$dir/*" ),
 148                                 glob( "{$this->sharedStaticPath}/thumb/$dir/*" ) );
 149
 150                         foreach ( $paths as $path ) {
 151                                 $file = basename( $path );
 152                                 if ( !(++$i % REPORTING_INTERVAL ) ) {
 153                                         print "$i\r";
 154                                 }
 155
 156                                 $title = Title::makeTitle( NS_IMAGE, $file );
 157                                 $this->doArticle( $title );
 158                         }
 159                 }
 160                 print "\n";
 161         }
 162
 163         function doCategories() {
 164                 $fname = 'DumpHTML::doCategories';
 165                 $this->setupGlobals();
 166
 167                 $dbr =& wfGetDB( DB_SLAVE );
 168                 $categorylinks = $dbr->tableName( 'categorylinks' );
 169                 print "Selecting categories...";
 170                 $sql = 'SELECT DISTINCT cl_to FROM categorylinks';
 171                 $res = $dbr->query( $sql, $fname );
 172
 173                 print "\nWriting " . $dbr->numRows( $res ).  " category pages\n";
 174                 $i = 0;
 175                 while ( $row = $dbr->fetchObject( $res ) ) {
 176                         wfWaitForSlaves( 10 );
 177                         if ( !(++$i % REPORTING_INTERVAL ) ) {
 178                                 print "$i\r";
 179                         }
 180                         $title = Title::makeTitle( NS_CATEGORY, $row->cl_to );
 181                         $this->doArticle( $title );
 182                 }
 183                 print "\n";
 184         }
 185
 186
 187         /** Write an article specified by title */
 188         function doArticle( $title ) {
 189                 global $wgTitle, $wgSharedUploadPath, $wgSharedUploadDirectory;
 190                 global $wgUploadDirectory;
 191
 192                 $text = $this->getArticleHTML( $title );
 193                 if ( $text === false ) {
 194                         return;
 195                 }
 196
 197                 # Parse the XHTML to find the images
 198                 $images = $this->findImages( $text );
 199                 $this->copyImages( $images );
 200
 201                 # Write to file
 202                 $this->writeArticle( $title, $text );
 203         }
 204
 205         /** Write the given text to the file identified by the given title object */
 206         function writeArticle( &$title, $text ) {
 207                 $filename = $title->getHashedFilename();
 208                 $fullName = "{$this->dest}/$filename";
 209                 $fullDir = dirname( $fullName );
 210
 211                 wfMkdirParents( $fullDir, 0755 );
 212
 213                 $file = fopen( $fullName, 'w' );
 214                 if ( !$file ) {
 215                         print("Can't open file $fullName for writing\n");
 216                         return;
 217                 }
 218
 219                 fwrite( $file, $text );
 220                 fclose( $file );
 221         }
 222
 223         /** Set up globals required for parsing */
 224         function setupGlobals( $depth = NULL ) {
 225                 global $wgUser, $wgTitle, $wgMakeDumpLinks, $wgStylePath, $wgArticlePath;
 226                 global $wgUploadPath, $wgLogo, $wgMaxCredits, $wgSharedUploadPath;
 227                 global $wgHideInterlanguageLinks, $wgUploadDirectory, $wgThumbnailScriptPath;
 228                 global $wgSharedThumbnailScriptPath, $wgEnableParserCache;
 229
 230                 static $oldLogo = NULL;
 231
 232                 if ( is_null( $depth ) ) {
 233                         $wgMakeDumpLinks = $this->depth;
 234                 } else {
 235                         $wgMakeDumpLinks = $depth;
 236                 }
 237
 238                 if ( $this->alternateScriptPath ) {
 239                         if ( $wgMakeDumpLinks == 0 ) {
 240                                 $wgScriptPath = '.';
 241                         } else {
 242                                 $wgScriptPath = '..' . str_repeat( '/..', $wgMakeDumpLinks - 1 );
 243                         }
 244                 } else {
 245                         $wgScriptPath = '..' . str_repeat( '/..', $wgMakeDumpLinks );
 246                 }
 247
 248                 $wgArticlePath = str_repeat( '../', $wgMakeDumpLinks ) . '$1';
 249
 250                 # Logo image
 251                 # Allow for repeated setup
 252                 if ( !is_null( $oldLogo ) ) {
 253                         $wgLogo = $oldLogo;
 254                 } else {
 255                         $oldLogo = $wgLogo;
 256                 }
 257
 258                 if ( strpos( $wgLogo, $wgUploadPath ) === 0 ) {
 259                         # If it's in the upload directory, rewrite it to the new upload directory
 260                         $wgLogo = "$wgScriptPath/{$this->imageRel}/" . substr( $wgLogo, strlen( $wgUploadPath ) + 1 );
 261                 } elseif ( $wgLogo{0} == '/' ) {
 262                         # This is basically heuristic
 263                         # Rewrite an absolute logo path to one relative to the the script path
 264                         $wgLogo = $wgScriptPath . $wgLogo;
 265                 }
 266
 267                 $wgStylePath = "$wgScriptPath/skins";
 268                 $wgUploadPath = "$wgScriptPath/{$this->imageRel}";
 269                 $wgSharedUploadPath = "$wgUploadPath/shared";
 270                 $wgMaxCredits = -1;
 271                 $wgHideInterlangageLinks = !$this->interwiki;
 272                 $wgThumbnailScriptPath = $wgSharedThumbnailScriptPath = false;
 273                 $wgEnableParserCache = false;
 274                 $wgMathPath = "$wgScriptPath/math";
 275
 276                 $wgUser = new User;
 277                 $wgUser->setOption( 'skin', 'htmldump' );
 278                 $wgUser->setOption( 'editsection', 0 );
 279
 280                 $this->sharedStaticPath = "$wgUploadDirectory/shared";
 281
 282         }
 283
 284         /** Reads the content of a title object, executes the skin and captures the result */
 285         function getArticleHTML( &$title ) {
 286                 global $wgOut, $wgTitle, $wgArticle, $wgUser, $wgUseCategoryMagic, $wgLinkCache;
 287
 288                 $wgOut = new OutputPage;
 289                 $wgOut->setParserOptions( new ParserOptions );
 290                 $wgLinkCache = new LinkCache;
 291
 292                 $wgTitle = $title;
 293                 if ( is_null( $wgTitle ) ) {
 294                         return false;
 295                 }
 296
 297                 $ns = $wgTitle->getNamespace();
 298                 if ( $ns == NS_SPECIAL ) {
 299                         SpecialPage::executePath( $wgTitle );
 300                 } else {
 301                         if ( $ns == NS_IMAGE ) {
 302                                 $wgArticle = new ImagePage( $wgTitle );
 303                         } elseif ( $wgUseCategoryMagic && $ns == NS_CATEGORY ) {
 304                                 $wgArticle = new CategoryPage( $wgTitle );
 305                         } else {
 306                                 $wgArticle = new Article( $wgTitle );
 307                         }
 308                         $rt = Title::newFromRedirect( $wgArticle->fetchContent() );
 309                         if ( $rt != NULL ) {
 310                                 $wgOut->addMeta( 'http:Refresh', '3;url=' . $rt->escapeLocalURL() );
 311                                 $wgOut->setPageTitle( $wgTitle->getPrefixedText() );
 312                                 $wgOut->addWikiText( wfMsg( 'redirectingto', $rt->getPrefixedText() ) );
 313                         } else {
 314                                 $wgArticle->view();
 315                         }
 316                 }
 317
 318                 $sk =& $wgUser->getSkin();
 319                 ob_start();
 320                 $sk->outputPage( $wgOut );
 321                 $text = ob_get_contents();
 322                 ob_end_clean();
 323
 324                 return $text;
 325         }
 326
 327         /** Returns image paths used in an XHTML document */
 328         function findImages( $text ) {
 329                 global $wgOutputEncoding, $wgDumpImages;
 330                 $parser = xml_parser_create( $wgOutputEncoding );
 331                 xml_set_element_handler( $parser, 'wfDumpStartTagHandler', 'wfDumpEndTagHandler' );
 332
 333                 $wgDumpImages = array();
 334                 xml_parse( $parser, $text );
 335                 xml_parser_free( $parser );
 336
 337                 return $wgDumpImages;
 338         }
 339
 340         /**
 341          * Copy images (or create symlinks) from commons to a static directory.
 342          * This is necessary even if you intend to distribute all of commons, because
 343          * the directory contents is used to work out which image description pages
 344          * are needed.
 345          *
 346          * Also copies math images
 347          *
 348          */
 349         function copyImages( $images ) {
 350                 global $wgSharedUploadPath, $wgSharedUploadDirectory, $wgMathPath, $wgMathDirectory;
 351                 # Find shared uploads and copy them into the static directory
 352                 $sharedPathLength = strlen( $wgSharedUploadPath );
 353                 $mathPathLength = strlen( $wgMathPath );
 354                 foreach ( $images as $escapedImage => $dummy ) {
 355                         $image = urldecode( $escapedImage );
 356
 357                         # Is it shared?
 358                         if ( substr( $image, 0, $sharedPathLength ) == $wgSharedUploadPath ) {
 359                                 # Reconstruct full filename
 360                                 $rel = substr( $image, $sharedPathLength + 1 ); // +1 for slash
 361                                 $sourceLoc = "$wgSharedUploadDirectory/$rel";
 362                                 $staticLoc = "{$this->sharedStaticPath}/$rel";
 363                                 #print "Copying $sourceLoc to $staticLoc\n";
 364                                 # Copy to static directory
 365                                 if ( !file_exists( $staticLoc ) ) {
 366                                         wfMkdirParents( dirname( $staticLoc ), 0755 );
 367                                         if ( function_exists( 'symlink' ) && !$this->forceCopy ) {
 368                                                 symlink( $sourceLoc, $staticLoc );
 369                                         } else {
 370                                                 copy( $sourceLoc, $staticLoc );
 371                                         }
 372                                 }
 373
 374                                 if ( substr( $rel, 0, 6 ) == 'thumb/' ) {
 375                                         # That was a thumbnail
 376                                         # We will also copy the real image
 377                                         $parts = explode( '/', $rel );
 378                                         $rel = "{$parts[1]}/{$parts[2]}/{$parts[3]}";
 379                                         $sourceLoc = "$wgSharedUploadDirectory/$rel";
 380                                         $staticLoc = "{$this->sharedStaticPath}/$rel";
 381                                         #print "Copying $sourceLoc to $staticLoc\n";
 382                                         if ( !file_exists( $staticLoc ) ) {
 383                                                 wfMkdirParents( dirname( $staticLoc ), 0755 );
 384                                                 if ( function_exists( 'symlink' ) && !$this->forceCopy ) {
 385                                                         symlink( $sourceLoc, $staticLoc );
 386                                                 } else {
 387                                                         copy( $sourceLoc, $staticLoc );
 388                                                 }
 389                                         }
 390                                 }
 391                         } else
 392                         # Is it math?
 393                         if ( substr( $image, 0, $mathPathLength ) == $wgMathPath ) {
 394                                 $rel = substr( $image, $mathPathLength + 1 ); // +1 for slash
 395                                 $source = "$wgMathDirectory/$rel";
 396                                 $dest = "{$this->dest}/math/$rel";
 397                                 if ( !file_exists( $dest ) ) {
 398                                         copy( $source, $dest );
 399                                 }
 400                         }
 401                 }
 402         }
 403 }
 404
 405 /** XML parser callback */
 406 function wfDumpStartTagHandler( $parser, $name, $attribs ) {
 407         global $wgDumpImages;
 408
 409         if ( $name == 'IMG' && isset( $attribs['SRC'] ) ) {
 410                 $wgDumpImages[$attribs['SRC']] = true;
 411         }
 412 }
 413
 414 /** XML parser callback */
 415 function wfDumpEndTagHandler( $parser, $name ) {}
 416
 417 # vim: syn=php
 418 ?>