X-Git-Url: https://git.cyclocoop.org/%27.WWW_URL.%27admin/?a=blobdiff_plain;f=maintenance%2FrefreshLinks.php;h=420f5553a413d63b44f4854bf3f521a092b284b7;hb=3112302c73dc1e30a114ee4ead4a6a4caab8b76a;hp=c444ab050a6fb6ae830df41e3d760ab299800a47;hpb=1d577369251870b761d08fbc8f1ff6a4d40e366d;p=lhc%2Fweb%2Fwiklou.git

diff --git a/maintenance/refreshLinks.php b/maintenance/refreshLinks.php
index c444ab050a..420f5553a4 100644
--- a/maintenance/refreshLinks.php
+++ b/maintenance/refreshLinks.php
@@ -1,43 +1,294 @@
 <?php
-define( "REPORTING_INTERVAL", 500 );
+/**
+ * Refresh link tables.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @ingroup Maintenance
+ */
 
-include_once( "commandLine.inc" );
-error_reporting( E_ALL & (~E_NOTICE) );
+require_once( dirname( __FILE__ ) . '/Maintenance.php' );
 
+class RefreshLinks extends Maintenance {
+	public function __construct() {
+		parent::__construct();
+		$this->mDescription = "Refresh link tables";
+		$this->addOption( 'dfn-only', 'Delete links from nonexistent articles only' );
+		$this->addOption( 'new-only', 'Only affect articles with just a single edit' );
+		$this->addOption( 'redirects-only', 'Only fix redirects, not all links' );
+		$this->addOption( 'old-redirects-only', 'Only fix redirects with no redirect table entry' );
+		$this->addOption( 'm', 'Maximum replication lag', false, true );
+		$this->addOption( 'e', 'Last page id to refresh', false, true );
+		$this->addArg( 'start', 'Page_id to start from, default 1', false );
+		$this->setBatchSize( 100 );
+	}
 
-if ($argv[2]) {
-	$start = (int)$argv[2];
-} else {
-	$start = 1;
-}
+	public function execute() {
+		$max = $this->getOption( 'm', 0 );
+		if ( !$this->hasOption( 'dfn-only' ) ) {
+			$start = $this->getArg( 0, 1 );
+			$new = $this->getOption( 'new-only', false );
+			$end = $this->getOption( 'e', 0 );
+			$redir = $this->getOption( 'redirects-only', false );
+			$oldRedir = $this->getOption( 'old-redirects-only', false );
+			$this->doRefreshLinks( $start, $new, $max, $end, $redir, $oldRedir );
+		}
+		$this->deleteLinksFromNonexistent( $max, $this->mBatchSize );
+	}
+
+	/**
+	 * Do the actual link refreshing.
+	 * @param $start int Page_id to start from
+	 * @param $newOnly bool Only do pages with 1 edit
+	 * @param $maxLag int Max DB replication lag
+	 * @param $end int Page_id to stop at
+	 * @param $redirectsOnly bool Only fix redirects
+	 * @param $oldRedirectsOnly bool Only fix redirects without redirect entries
+	 */
+	private function doRefreshLinks( $start, $newOnly = false, $maxLag = false,
+						$end = 0, $redirectsOnly = false, $oldRedirectsOnly = false ) {
+		global $wgParser, $wgUseTidy;
+
+		$reportingInterval = 100;
+		$dbr = wfGetDB( DB_SLAVE );
+		$start = intval( $start );
+
+		// Give extensions a chance to optimize settings
+		wfRunHooks( 'MaintenanceRefreshLinksInit', array( $this ) );
+
+		# Don't generate extension images (e.g. Timeline)
+		$wgParser->clearTagHooks();
+
+		# Don't use HTML tidy
+		$wgUseTidy = false;
+
+		$what = $redirectsOnly ? "redirects" : "links";
+
+		if ( $oldRedirectsOnly ) {
+			# This entire code path is cut-and-pasted from below.  Hurrah.
 
-$res = wfQuery("SELECT max(cur_id) as m FROM cur", DB_READ);
-$row = wfFetchObject( $res );
-$end = $row->m;
+			$conds = array(
+				"page_is_redirect=1",
+				"rd_from IS NULL"
+			);
 
-print("Refreshing link table. Starting from cur_id $start of $end.\n");
+			if ( $end == 0 ) {
+				$conds[] = "page_id >= $start";
+			} else {
+				$conds[] = "page_id BETWEEN $start AND $end";
+			}
 
-$wgUser->setOption("math", 3);
-for ($id = $start; $id <= $end; $id++) {
-	if ( !($id % REPORTING_INTERVAL) ) {
-		print "$id\n";
+			$res = $dbr->select(
+				array( 'page', 'redirect' ),
+				'page_id',
+				$conds,
+				__METHOD__,
+				array(),
+				array( 'redirect' => array( "LEFT JOIN", "page_id=rd_from" ) )
+			);
+			$num = $dbr->numRows( $res );
+			$this->output( "Refreshing $num old redirects from $start...\n" );
+
+			$i = 0;
+
+			foreach ( $res as $row ) {
+				if ( !( ++$i % $reportingInterval ) ) {
+					$this->output( "$i\n" );
+					wfWaitForSlaves( $maxLag );
+				}
+				$this->fixRedirect( $row->page_id );
+			}
+		} elseif ( $newOnly ) {
+			$this->output( "Refreshing $what from " );
+			$res = $dbr->select( 'page',
+				array( 'page_id' ),
+				array(
+					'page_is_new' => 1,
+					"page_id >= $start" ),
+				__METHOD__
+			);
+			$num = $dbr->numRows( $res );
+			$this->output( "$num new articles...\n" );
+
+			$i = 0;
+			foreach ( $res as $row ) {
+				if ( !( ++$i % $reportingInterval ) ) {
+					$this->output( "$i\n" );
+					wfWaitForSlaves( $maxLag );
+				}
+				if ( $redirectsOnly ) {
+					$this->fixRedirect( $row->page_id );
+				} else {
+					self::fixLinksFromArticle( $row->page_id );
+				}
+			}
+		} else {
+			if ( !$end ) {
+				$maxPage = $dbr->selectField( 'page', 'max(page_id)', false );
+				$maxRD = $dbr->selectField( 'redirect', 'max(rd_from)', false );
+				$end = max( $maxPage, $maxRD );
+			}
+			$this->output( "Refreshing redirects table.\n" );
+			$this->output( "Starting from page_id $start of $end.\n" );
+
+			for ( $id = $start; $id <= $end; $id++ ) {
+
+				if ( !( $id % $reportingInterval ) ) {
+					$this->output( "$id\n" );
+					wfWaitForSlaves( $maxLag );
+				}
+				$this->fixRedirect( $id );
+			}
+
+			if ( !$redirectsOnly ) {
+				$this->output( "Refreshing links table.\n" );
+				$this->output( "Starting from page_id $start of $end.\n" );
+
+				for ( $id = $start; $id <= $end; $id++ ) {
+
+					if ( !( $id % $reportingInterval ) ) {
+						$this->output( "$id\n" );
+						wfWaitForSlaves( $maxLag );
+					}
+					self::fixLinksFromArticle( $id );
+				}
+			}
+		}
 	}
-	
-	$wgTitle = Title::newFromID( $id );
-	if ( is_null( $wgTitle ) ) {
-		continue;
+
+	/**
+	 * Update the redirect entry for a given page
+	 * @param $id int The page_id of the redirect
+	 */
+	private function fixRedirect( $id ) {
+		$title = Title::newFromID( $id );
+		$dbw = wfGetDB( DB_MASTER );
+
+		if ( is_null( $title ) ) {
+			// This page doesn't exist (any more)
+			// Delete any redirect table entry for it
+			$dbw->delete( 'redirect', array( 'rd_from' => $id ),
+				__METHOD__ );
+			return;
+		}
+		$article = new Article( $title );
+
+		$rt = $article->followRedirect();
+
+		if ( !$rt || !is_object( $rt ) ) {
+			// $title is not a redirect
+			// Delete any redirect table entry for it
+			$dbw->delete( 'redirect', array( 'rd_from' => $id ),
+				__METHOD__ );
+		} else {
+			$article->updateRedirectOn( $dbw, $rt );
+		}
+	}
+
+	/**
+	 * Run LinksUpdate for all links on a given page_id
+	 * @param $id int The page_id
+	 */
+	public static function fixLinksFromArticle( $id ) {
+		global $wgParser;
+
+		$title = Title::newFromID( $id );
+		$dbw = wfGetDB( DB_MASTER );
+
+		LinkCache::singleton()->clear();
+
+		if ( is_null( $title ) ) {
+			return;
+		}
+		$dbw->begin();
+
+		$revision = Revision::newFromTitle( $title );
+		if ( !$revision ) {
+			return;
+		}
+
+		$options = new ParserOptions;
+		$parserOutput = $wgParser->parse( $revision->getText(), $title, $options, true, true, $revision->getId() );
+		$update = new LinksUpdate( $title, $parserOutput, false );
+		$update->doUpdate();
+		$dbw->commit();
 	}
-	
-	$wgLinkCache = new LinkCache;
-	$wgArticle = new Article( $wgTitle );
-	$text = $wgArticle->getContent( true );
-	@$wgOut->addWikiText( $text );
-	
-	$linksUpdate = new LinksUpdate( $id, $wgTitle );
-	$linksUpdate->doDumbUpdate();
-}
 
+	/*
+	 * Removes non-existing links from pages from pagelinks, imagelinks,
+	 * categorylinks, templatelinks and externallinks tables.
+	 *
+	 * @param $maxLag
+	 * @param $batchSize The size of deletion batches
+	 *
+	 * @author Merlijn van Deen <valhallasw@arctus.nl>
+	 */
+	private function deleteLinksFromNonexistent( $maxLag = 0, $batchSize = 100 ) {
+		wfWaitForSlaves( $maxLag );
 
-exit();
+		$dbw = wfGetDB( DB_MASTER );
+
+		$lb = wfGetLBFactory()->newMainLB();
+		$dbr = $lb->getConnection( DB_SLAVE );
+		$dbr->bufferResults( false );
+
+		$linksTables = array( // table name => page_id field
+			'pagelinks' => 'pl_from',
+			'imagelinks' => 'il_from',
+			'categorylinks' => 'cl_from',
+			'templatelinks' => 'tl_from',
+			'externallinks' => 'el_from',
+		);
+
+		foreach ( $linksTables as $table => $field ) {
+			$this->output( "Retrieving illegal entries from $table... " );
+
+			// SELECT DISTINCT( $field ) FROM $table LEFT JOIN page ON $field=page_id WHERE page_id IS NULL;
+			$results = $dbr->select( array( $table, 'page' ),
+						  $field,
+						  array( 'page_id' => null ),
+						  __METHOD__,
+						  'DISTINCT',
+						  array( 'page' => array( 'LEFT JOIN', "$field=page_id" ) )
+			);
+
+			$counter = 0;
+			$list = array();
+			$this->output( "0.." );
+
+			foreach ( $results as $row ) {
+				$counter++;
+				$list[] = $row->$field;
+				if ( ( $counter % $batchSize ) == 0 ) {
+					wfWaitForSlaves( 5 );
+					$dbw->delete( $table, array( $field => $list ), __METHOD__ );
+
+					$this->output( $counter . ".." );
+					$list = array();
+				}
+			}
+			$this->output( $counter );
+			if ( count( $list ) > 0 ) {
+				$dbw->delete( $table, array( $field => $list ), __METHOD__ );
+			}
+			$this->output( "\n" );
+		}
+		$lb->closeAll();
+	}
+}
 
-?>
+$maintClass = 'RefreshLinks';
+require_once( RUN_MAINTENANCE_IF_MAIN );