4 * @subpackage Maintenance
7 define( 'REPORTING_INTERVAL', 10 );
9 require_once( 'includes/ImagePage.php' );
10 require_once( 'includes/CategoryPage.php' );
11 require_once( 'includes/RawPage.php' );
14 # Destination directory
17 # Show interlanguage links?
18 var $interwiki = true;
20 # Depth of HTML directory tree
23 # Directory that commons images are copied into
24 var $sharedStaticDirectory;
26 # Directory that the images are in, after copying
27 var $destUploadDirectory;
29 # Relative path to image directory
30 var $imageRel = 'upload';
32 # Copy commons images instead of symlinking
33 var $forceCopy = false;
35 # Make a copy of all images encountered
36 var $makeSnapshot = false;
38 # Make links assuming the script path is in the same directory as
40 var $alternateScriptPath = false;
42 # Original values of various globals
43 var $oldArticlePath = false, $oldCopyrightIcon = false;
45 # Has setupGlobals been called?
46 var $setupDone = false;
48 # List of raw pages used in the current article
52 var $skin = 'htmldump';
55 var $checkpointFile = false, $checkpoints = false;
57 function DumpHTML( $settings ) {
58 foreach ( $settings as $var => $value ) {
63 function loadCheckpoints() {
64 if ( $this->checkpoints !== false ) {
66 } elseif ( !$this->checkpointFile ) {
69 $lines = @file( $this->checkpointFile );
70 if ( $lines === false ) {
71 print "Starting new checkpoint file \"{$this->checkpointFile}\"\n";
72 $this->checkpoints = array();
74 $lines = array_map( 'trim', $lines );
75 $this->checkpoints = array();
76 foreach ( $lines as $line ) {
77 list( $name, $value ) = explode( '=', $line, 2 );
78 $this->checkpoints[$name] = $value;
85 function getCheckpoint( $type, $defValue = false ) {
86 if ( !$this->loadCheckpoints() ) {
89 if ( !isset( $this->checkpoints[$type] ) ) {
92 return $this->checkpoints[$type];
96 function setCheckpoint( $type, $value ) {
97 if ( !$this->checkpointFile ) {
100 $this->checkpoints[$type] = $value;
102 foreach ( $this->checkpoints as $type => $value ) {
103 $blob .= "$type=$value\n";
105 file_put_contents( $this->checkpointFile, $blob );
108 function doEverything() {
109 if ( $this->getCheckpoint( 'everything' ) == 'done' ) {
110 print "Checkpoint says everything is already done\n";
114 $this->doLocalImageDescriptions();
115 $this->doSharedImageDescriptions();
116 $this->doCategories();
117 $this->doRedirects();
120 $this->setCheckpoint( 'everything', 'done' );
124 * Write a set of articles specified by start and end page_id
125 * Skip categories and images, they will be done separately
127 function doArticles() {
128 $fname = 'DumpHTML::doArticles';
130 $cp = $this->getCheckpoint( 'article' );
131 if ( $cp == 'done' ) {
132 print "Articles already done\n";
134 } elseif ( $cp !== false ) {
135 print "Resuming article dump from checkpoint at page_id $cp of {$this->endID}\n";
138 print "Starting from page_id {$this->startID} of {$this->endID}\n";
139 $start = $this->startID;
142 $this->setupGlobals();
144 if ( $this->endID === false ) {
145 $dbr =& wfGetDB( DB_SLAVE );
146 $this->endID = $dbr->selectField( 'page', 'max(page_id)', false, $fname );
150 $mainPageObj = Title::newMainPage();
151 $mainPage = $mainPageObj->getPrefixedDBkey();
153 for ($id = $start; $id <= $this->endID; $id++) {
154 wfWaitForSlaves( 20 );
155 if ( !($id % REPORTING_INTERVAL) ) {
156 print "Processing ID: $id\r";
157 $this->setCheckpoint( 'article', $id );
159 if ( !($id % (REPORTING_INTERVAL*10) ) ) {
162 $title = Title::newFromID( $id );
164 $ns = $title->getNamespace() ;
165 if ( $ns != NS_CATEGORY && $title->getPrefixedDBkey() != $mainPage ) {
166 $this->doArticle( $title );
170 $this->setCheckpoint( 'article', 'done' );
174 function doSpecials() {
177 $this->setupGlobals();
178 print "Special:Categories...";
179 $this->doArticle( Title::makeTitle( NS_SPECIAL, 'Categories' ) );
183 /** Write the main page as index.html */
184 function doMainPage() {
186 print "Making index.html ";
188 // Set up globals with no ../../.. in the link URLs
189 $this->setupGlobals( 0 );
191 $title = Title::newMainPage();
192 $text = $this->getArticleHTML( $title );
194 # Parse the XHTML to find the images
195 $images = $this->findImages( $text );
196 $this->copyImages( $images );
198 $file = fopen( "{$this->dest}/index.html", "w" );
200 print "\nCan't open index.html for writing\n";
203 fwrite( $file, $text );
208 function doImageDescriptions() {
209 $this->doLocalImageDescriptions();
210 $this->doSharedImageDescriptions();
214 * Dump image description pages that don't have an associated article, but do
217 function doLocalImageDescriptions() {
218 global $wgSharedUploadDirectory;
220 $dbr =& wfGetDB( DB_SLAVE );
222 $cp = $this->getCheckpoint( 'local image' );
223 if ( $cp == 'done' ) {
224 print "Local image descriptions already done\n";
226 } elseif ( $cp !== false ) {
227 print "Writing image description pages starting from $cp\n";
228 $conds = array( 'img_name >= ' . $dbr->addQuotes( $cp ) );
230 print "Writing image description pages for local images\n";
234 $this->setupGlobals();
236 $res = $dbr->select( 'image', array( 'img_name' ), $conds, __METHOD__,
237 array( 'ORDER BY' => 'img_name' ) );
240 $num = $dbr->numRows( $res );
241 while ( $row = $dbr->fetchObject( $res ) ) {
242 wfWaitForSlaves( 10 );
243 if ( !( ++$i % REPORTING_INTERVAL ) ) {
244 print "Done $i of $num\r";
245 if ( $row->img_name !== 'done' ) {
246 $this->setCheckpoint( 'local image', $row->img_name );
249 $title = Title::makeTitle( NS_IMAGE, $row->img_name );
250 if ( $title->getArticleID() ) {
251 // Already done by dumpHTML
254 $this->doArticle( $title );
256 $this->setCheckpoint( 'local image', 'done' );
261 * Dump images which only have a real description page on commons
263 function doSharedImageDescriptions() {
264 $cp = $this->getCheckpoint( 'shared image' );
265 if ( $cp == 'done' ) {
266 print "Shared description pages already done\n";
268 } elseif ( $cp !== false ) {
269 print "Writing description pages for commons images starting from directory $cp/255\n";
272 print "Writing description pages for commons images\n";
276 $this->setupGlobals();
278 for ( $hash = $start; $hash < 256; $hash++ ) {
279 $this->setCheckpoint( 'shared image', $hash );
281 $dir = sprintf( "%01x/%02x", intval( $hash / 16 ), $hash );
282 $paths = array_merge( glob( "{$this->sharedStaticDirectory}/$dir/*" ),
283 glob( "{$this->sharedStaticDirectory}/thumb/$dir/*" ) );
285 foreach ( $paths as $path ) {
286 $file = basename( $path );
287 if ( !(++$i % REPORTING_INTERVAL ) ) {
291 $title = Title::makeTitle( NS_IMAGE, $file );
292 $this->doArticle( $title );
295 $this->setCheckpoint( 'shared image', 'done' );
299 function doCategories() {
300 $fname = 'DumpHTML::doCategories';
301 $this->setupGlobals();
302 $dbr =& wfGetDB( DB_SLAVE );
303 $sql = 'SELECT DISTINCT cl_to FROM ' . $dbr->tableName( 'categorylinks' );
305 $cp = $this->getCheckpoint( 'category' );
306 if ( $cp == 'done' ) {
307 print "Category pages already done\n";
309 } elseif ( $cp !== false ) {
310 print "Resuming category page dump from $cp";
311 $sql .= ' WHERE cl_to >= ' . $dbr->addQuotes( $cp );
314 $sql .= ' ORDER BY cl_to';
315 print "Selecting categories...";
316 $res = $dbr->query( $sql, $fname );
318 print "\nWriting " . $dbr->numRows( $res ). " category pages\n";
320 while ( $row = $dbr->fetchObject( $res ) ) {
321 wfWaitForSlaves( 10 );
322 if ( !(++$i % REPORTING_INTERVAL ) ) {
324 if ( $row->cl_to != 'done' ) {
325 $this->setCheckpoint( 'category', $row->cl_to );
328 $title = Title::makeTitle( NS_CATEGORY, $row->cl_to );
329 $this->doArticle( $title );
331 $this->setCheckpoint( 'category', 'done' );
335 function doRedirects() {
336 print "Doing redirects...\n";
337 $fname = 'DumpHTML::doRedirects';
338 $conds = array( 'page_is_redirect' => 1 );
340 $cp = $this->getCheckpoint( 'redirect' );
341 if ( $cp == 'done' ) {
342 print "Redirects already done\n";
344 } elseif ( $cp !== false ) {
345 print "Resuming redirect generation from page_id $cp\n";
346 $conds[] = 'page_id > ' . intval( $cp );
349 $this->setupGlobals();
350 $dbr =& wfGetDB( DB_SLAVE );
351 $res = $dbr->select( 'page', array( 'page_id', 'page_namespace', 'page_title' ),
353 $num = $dbr->numRows( $res );
354 print "$num redirects to do...\n";
356 while ( $row = $dbr->fetchObject( $res ) ) {
357 $title = Title::makeTitle( $row->page_namespace, $row->page_title );
358 if ( !(++$i % (REPORTING_INTERVAL*10) ) ) {
359 print "Done $i of $num\n";
360 $this->setCheckpoint( 'redirect', $row->page_id );
362 $this->doArticle( $title );
364 $this->setCheckpoint( 'redirect', 'done' );
367 /** Write an article specified by title */
368 function doArticle( $title ) {
369 global $wgTitle, $wgSharedUploadPath, $wgSharedUploadDirectory;
370 global $wgUploadDirectory;
372 $this->rawPages = array();
373 $text = $this->getArticleHTML( $title );
375 if ( $text === false ) {
379 # Parse the XHTML to find the images
380 $images = $this->findImages( $text );
381 $this->copyImages( $images );
384 $this->writeArticle( $title, $text );
387 wfMkdirParents( "{$this->dest}/raw", 0755 );
388 foreach( $this->rawPages as $record ) {
389 list( $file, $title, $params ) = $record;
391 $path = "{$this->dest}/raw/$file";
392 if ( !file_exists( $path ) ) {
393 $article = new Article( $title );
394 $request = new FauxRequest( $params );
395 $rp = new RawPage( $article, $request );
396 $text = $rp->getRawText();
398 print "Writing $file\n";
399 $file = fopen( $path, 'w' );
401 print("Can't open file $fullName for writing\n");
404 fwrite( $file, $text );
410 /** Write the given text to the file identified by the given title object */
411 function writeArticle( &$title, $text ) {
412 $filename = $this->getHashedFilename( $title );
413 $fullName = "{$this->dest}/$filename";
414 $fullDir = dirname( $fullName );
416 wfMkdirParents( $fullDir, 0755 );
418 wfSuppressWarnings();
419 $file = fopen( $fullName, 'w' );
423 die("Can't open file '$fullName' for writing.\nCheck permissions or use another destination (-d).\n");
427 fwrite( $file, $text );
431 /** Set up globals required for parsing */
432 function setupGlobals( $currentDepth = NULL ) {
433 global $wgUser, $wgTitle, $wgStylePath, $wgArticlePath, $wgMathPath;
434 global $wgUploadPath, $wgLogo, $wgMaxCredits, $wgSharedUploadPath;
435 global $wgHideInterlanguageLinks, $wgUploadDirectory, $wgThumbnailScriptPath;
436 global $wgSharedThumbnailScriptPath, $wgEnableParserCache, $wgHooks, $wgServer;
437 global $wgRightsUrl, $wgRightsText, $wgCopyrightIcon, $wgEnableSidebarCache;
438 global $wgGenerateThumbnailOnParse;
440 static $oldLogo = NULL;
442 if ( !$this->setupDone ) {
443 $wgHooks['GetLocalURL'][] =& $this;
444 $wgHooks['GetFullURL'][] =& $this;
445 $this->oldArticlePath = $wgServer . $wgArticlePath;
448 if ( is_null( $currentDepth ) ) {
449 $currentDepth = $this->depth;
452 if ( $this->alternateScriptPath ) {
453 if ( $currentDepth == 0 ) {
456 $wgScriptPath = '..' . str_repeat( '/..', $currentDepth - 1 );
459 $wgScriptPath = '..' . str_repeat( '/..', $currentDepth );
462 $wgArticlePath = str_repeat( '../', $currentDepth ) . '$1';
465 # Allow for repeated setup
466 if ( !is_null( $oldLogo ) ) {
472 if ( strpos( $wgLogo, $wgUploadPath ) === 0 ) {
473 # If it's in the upload directory, rewrite it to the new upload directory
474 $wgLogo = "$wgScriptPath/{$this->imageRel}/" . substr( $wgLogo, strlen( $wgUploadPath ) + 1 );
475 } elseif ( $wgLogo{0} == '/' ) {
476 # This is basically heuristic
477 # Rewrite an absolute logo path to one relative to the the script path
478 $wgLogo = $wgScriptPath . $wgLogo;
482 if ( !$this->setupDone ) {
483 $this->oldCopyrightIcon = $wgCopyrightIcon;
485 $wgCopyrightIcon = str_replace( 'src="/images',
486 'src="' . htmlspecialchars( $wgScriptPath ) . '/images', $this->oldCopyrightIcon );
488 $wgStylePath = "$wgScriptPath/skins";
489 $wgUploadPath = "$wgScriptPath/{$this->imageRel}";
490 $wgSharedUploadPath = "$wgUploadPath/shared";
492 $wgHideInterlanguageLinks = !$this->interwiki;
493 $wgThumbnailScriptPath = $wgSharedThumbnailScriptPath = false;
494 $wgEnableParserCache = false;
495 $wgMathPath = "$wgScriptPath/math";
496 $wgEnableSidebarCache = false;
497 $wgGenerateThumbnailOnParse = true;
499 if ( !empty( $wgRightsText ) ) {
500 $wgRightsUrl = "$wgScriptPath/COPYING.html";
504 $wgUser->setOption( 'skin', $this->skin );
505 $wgUser->setOption( 'editsection', 0 );
507 if ( $this->makeSnapshot ) {
508 $this->destUploadDirectory = "{$this->dest}/{$this->imageRel}";
509 if ( realpath( $this->destUploadDirectory == $wgUploadDirectory ) ) {
510 $this->makeSnapshot = false;
514 $this->sharedStaticDirectory = "{$this->destUploadDirectory}/shared";
516 $this->setupDone = true;
519 /** Reads the content of a title object, executes the skin and captures the result */
520 function getArticleHTML( &$title ) {
521 global $wgOut, $wgTitle, $wgArticle, $wgUser;
523 $linkCache =& LinkCache::singleton();
526 if ( is_null( $wgTitle ) ) {
530 $ns = $wgTitle->getNamespace();
531 if ( $ns == NS_SPECIAL ) {
532 $wgOut = new OutputPage;
533 $wgOut->setParserOptions( new ParserOptions );
534 SpecialPage::executePath( $wgTitle );
536 /** @todo merge with Wiki.php code */
537 if ( $ns == NS_IMAGE ) {
538 $wgArticle = new ImagePage( $wgTitle );
539 } elseif ( $ns == NS_CATEGORY ) {
540 $wgArticle = new CategoryPage( $wgTitle );
542 $wgArticle = new Article( $wgTitle );
544 $rt = Title::newFromRedirect( $wgArticle->fetchContent() );
546 return $this->getRedirect( $rt );
548 $wgOut = new OutputPage;
549 $wgOut->setParserOptions( new ParserOptions );
555 $sk =& $wgUser->getSkin();
557 $sk->outputPage( $wgOut );
558 $text = ob_get_contents();
564 function getRedirect( $rt ) {
565 $url = $rt->escapeLocalURL();
566 $text = $rt->getPrefixedText();
568 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
569 <html xmlns="http://www.w3.org/1999/xhtml">
571 <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
572 <meta http-equiv="Refresh" content="0;url=$url" />
575 <p>Redirecting to <a href="$url">$text</a></p>
581 /** Returns image paths used in an XHTML document */
582 function findImages( $text ) {
583 global $wgOutputEncoding, $wgDumpImages;
584 $parser = xml_parser_create( $wgOutputEncoding );
585 xml_set_element_handler( $parser, 'wfDumpStartTagHandler', 'wfDumpEndTagHandler' );
587 $wgDumpImages = array();
588 xml_parse( $parser, $text );
589 xml_parser_free( $parser );
591 return $wgDumpImages;
595 * Copy a file specified by a URL to a given directory
597 * @param string $srcPath The source URL
598 * @param string $srcPathBase The base directory of the source URL
599 * @param string $srcDirBase The base filesystem directory of the source URL
600 * @param string $destDirBase The base filesystem directory of the destination URL
602 function relativeCopy( $srcPath, $srcPathBase, $srcDirBase, $destDirBase ) {
603 $rel = substr( $srcPath, strlen( $srcPathBase ) + 1 ); // +1 for slash
604 $sourceLoc = "$srcDirBase/$rel";
605 $destLoc = "$destDirBase/$rel";
606 #print "Copying $sourceLoc to $destLoc\n";
607 if ( !file_exists( $destLoc ) ) {
608 wfMkdirParents( dirname( $destLoc ), 0755 );
609 if ( function_exists( 'symlink' ) && !$this->forceCopy ) {
610 symlink( $sourceLoc, $destLoc );
612 copy( $sourceLoc, $destLoc );
618 * Copy an image, and if it is a thumbnail, copy its parent image too
620 function copyImage( $srcPath, $srcPathBase, $srcDirBase, $destDirBase ) {
621 global $wgUploadPath, $wgUploadDirectory, $wgSharedUploadPath;
622 $this->relativeCopy( $srcPath, $srcPathBase, $srcDirBase, $destDirBase );
623 if ( substr( $srcPath, strlen( $srcPathBase ) + 1, 6 ) == 'thumb/' ) {
624 # The image was a thumbnail
625 # Copy the source image as well
626 $rel = substr( $srcPath, strlen( $srcPathBase ) + 1 );
627 $parts = explode( '/', $rel );
628 $rel = "{$parts[1]}/{$parts[2]}/{$parts[3]}";
629 $newSrc = "$srcPathBase/$rel";
630 $this->relativeCopy( $newSrc, $srcPathBase, $srcDirBase, $destDirBase );
635 * Copy images (or create symlinks) from commons to a static directory.
636 * This is necessary even if you intend to distribute all of commons, because
637 * the directory contents is used to work out which image description pages
640 * Also copies math images, and full-sized images if the makeSnapshot option
644 function copyImages( $images ) {
645 global $wgUploadPath, $wgUploadDirectory, $wgSharedUploadPath, $wgSharedUploadDirectory,
646 $wgMathPath, $wgMathDirectory;
647 # Find shared uploads and copy them into the static directory
648 $sharedPathLength = strlen( $wgSharedUploadPath );
649 $mathPathLength = strlen( $wgMathPath );
650 $uploadPathLength = strlen( $wgUploadPath );
651 foreach ( $images as $escapedImage => $dummy ) {
652 $image = urldecode( $escapedImage );
654 if ( substr( $image, 0, $sharedPathLength ) == $wgSharedUploadPath ) {
655 $this->copyImage( $image, $wgSharedUploadPath, $wgSharedUploadDirectory, $this->sharedStaticDirectory );
656 } elseif ( substr( $image, 0, $mathPathLength ) == $wgMathPath ) {
657 $this->relativeCopy( $image, $wgMathPath, $wgMathDirectory, "{$this->dest}/math" );
658 } elseif ( $this->makeSnapshot && substr( $image, 0, $uploadPathLength ) == $wgUploadPath ) {
659 $this->copyImage( $image, $wgUploadPath, $wgUploadDirectory, $this->destUploadDirectory );
664 function onGetFullURL( &$title, &$url, $query ) {
665 global $wgContLang, $wgArticlePath;
667 $iw = $title->getInterwiki();
668 if ( $title->isExternal() && $wgContLang->getLanguageName( $iw ) ) {
669 if ( $title->getDBkey() == '' ) {
670 $url = str_replace( '$1', "../$iw/index.html", $wgArticlePath );
672 $url = str_replace( '$1', "../$iw/" . wfUrlencode( $this->getHashedFilename( $title ) ),
681 function onGetLocalURL( &$title, &$url, $query ) {
682 global $wgArticlePath;
684 if ( $title->isExternal() ) {
685 # Default is fine for interwiki
690 if ( $query != '' ) {
691 parse_str( $query, $params );
692 if ( isset($params['action']) && $params['action'] == 'raw' ) {
693 if ( $params['gen'] == 'css' || $params['gen'] == 'js' ) {
694 $file = 'gen.' . $params['gen'];
696 $file = $this->getFriendlyName( $title->getPrefixedDBkey() );
697 // Clean up Monobook.css etc.
698 if ( preg_match( '/^(.*)\.(css|js)_[0-9a-f]{4}$/', $file, $matches ) ) {
699 $file = $matches[1] . '.' . $matches[2];
702 $this->rawPages[$file] = array( $file, $title, $params );
703 $url = str_replace( '$1', "raw/" . wfUrlencode( $file ), $wgArticlePath );
706 if ( $url === false ) {
707 $url = str_replace( '$1', wfUrlencode( $this->getHashedFilename( $title ) ), $wgArticlePath );
713 function getHashedFilename( &$title ) {
714 if ( '' != $title->mInterwiki ) {
715 $dbkey = $title->getDBkey();
717 $dbkey = $title->getPrefixedDBkey();
720 $mainPage = Title::newMainPage();
721 if ( $mainPage->getPrefixedDBkey() == $dbkey ) {
725 return $this->getHashedDirectory( $title ) . '/' .
726 $this->getFriendlyName( $dbkey ) . '.html';
729 function getFriendlyName( $name ) {
731 # Replace illegal characters for Windows paths with underscores
732 $friendlyName = strtr( $name, '/\\*?"<>|~', '_________' );
734 # Work out lower case form. We assume we're on a system with case-insensitive
735 # filenames, so unless the case is of a special form, we have to disambiguate
736 if ( function_exists( 'mb_strtolower' ) ) {
737 $lowerCase = $wgLang->ucfirst( mb_strtolower( $name ) );
739 $lowerCase = ucfirst( strtolower( $name ) );
742 # Make it mostly unique
743 if ( $lowerCase != $friendlyName ) {
744 $friendlyName .= '_' . substr(md5( $name ), 0, 4);
746 # Handle colon specially by replacing it with tilde
747 # Thus we reduce the number of paths with hashes appended
748 $friendlyName = str_replace( ':', '~', $friendlyName );
750 return $friendlyName;
754 * Get a relative directory for putting a title into
756 function getHashedDirectory( &$title ) {
757 if ( '' != $title->getInterwiki() ) {
758 $pdbk = $title->getDBkey();
760 $pdbk = $title->getPrefixedDBkey();
763 # Find the first colon if there is one, use characters after it
764 $p = strpos( $pdbk, ':' );
765 if ( $p !== false ) {
766 $dbk = substr( $pdbk, $p + 1 );
767 $dbk = substr( $dbk, strspn( $dbk, '_' ) );
772 # Split into characters
773 preg_match_all( '/./us', $dbk, $m );
776 $length = count( $chars );
779 for ( $i = 0; $i < $this->depth; $i++ ) {
783 if ( $i >= $length ) {
787 if ( ord( $c ) >= 128 || preg_match( '/[a-zA-Z0-9!#$%&()+,[\]^_`{}-]/', $c ) ) {
788 if ( function_exists( 'mb_strtolower' ) ) {
789 $dir .= mb_strtolower( $c );
791 $dir .= strtolower( $c );
794 $dir .= sprintf( "%02X", ord( $c ) );
803 /** XML parser callback */
804 function wfDumpStartTagHandler( $parser, $name, $attribs ) {
805 global $wgDumpImages;
807 if ( $name == 'IMG' && isset( $attribs['SRC'] ) ) {
808 $wgDumpImages[$attribs['SRC']] = true;
812 /** XML parser callback */
813 function wfDumpEndTagHandler( $parser, $name ) {}