X-Git-Url: https://git.cyclocoop.org/%27.WWW_URL.%27admin/?a=blobdiff_plain;f=maintenance%2FrefreshLinks.php;h=6dccefb3358a8a07722e1502943699e695de2ca0;hb=291efd30f2173070df8c1939f659759521f8fae6;hp=923379e8708c1e30afe6b879687a66d6a75607bf;hpb=1d749bc52c4fc2f7acb8ba5dfa9ec508e6627fdb;p=lhc%2Fweb%2Fwiklou.git

diff --git a/maintenance/refreshLinks.php b/maintenance/refreshLinks.php
index 923379e870..6dccefb335 100644
--- a/maintenance/refreshLinks.php
+++ b/maintenance/refreshLinks.php
@@ -1,54 +1,273 @@
 <?php
 /**
- * @todo document
- * @addtogroup Maintenance
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @ingroup Maintenance
  */
 
-/** */
-$optionsWithArgs = array( 'm', 'e' );
+require_once( dirname(__FILE__) . '/Maintenance.php' );
 
-require_once( "commandLine.inc" );
-require_once( "refreshLinks.inc" );
+class RefreshLinks extends Maintenance {
+	public function __construct() {
+		parent::__construct();
+		$this->mDescription = "Refresh link tables";
+		$this->addOption( 'dfn-only', 'Delete links from nonexistent articles only' );
+		$this->addOption( 'new-only', 'Only affect articles with just a single edit' );
+		$this->addOption( 'redirects-only', 'Only fix redirects, not all links' );
+		$this->addOption( 'old-redirects-only', 'Only fix redirects with no redirect table entry' );
+		$this->addOption( 'm', 'Maximum replication lag', false, true );
+		$this->addOption( 'e', 'Last page id to refresh', false, true );
+		$this->addArg( 'start', 'Page_id to start from, default 1' );
+		$this->setBatchSize( 100 );
+	}
+
+	public function execute() {
+		if( !$this->hasOption( 'dfn-only' ) ) {
+			$start = $this->getArg( 0, 1 );
+			$new = $this->getOption( 'new-only', false );
+			$max = $this->getOption( 'm', false );
+			$end = $this->getOption( 'e', 0 );
+			$redir = $this->getOption( 'redirects-only', false );
+			$oldRedir = $this->getOption( 'old-redirects-only', false );
+			$this->doRefreshLinks( $start, $new, $max, $end, $redir, $oldRedir );
+		}
+		$this->deleteLinksFromNonexistent( $max, $this->mBatchSize );
+	}
 
-if( isset( $options['help'] ) ) {
-	echo <<<TEXT
-usage: php refreshLinks.php start [-e end] [-m maxlag] [--help] [possibly other
-    stuff]
+	/**
+	 * Do the actual link refreshing.
+	 * @param $start int Page_id to start from
+	 * @param $newOnly bool Only do pages with 1 edit
+	 * @param $maxLag int Max DB replication lag
+	 * @param $end int Page_id to stop at
+	 * @param $redirectsOnly bool Only fix redirects
+	 * @param $oldRedirectsOnly bool Only fix redirects without redirect entries
+	 */
+	private function doRefreshLinks( $start, $newOnly = false, $maxLag = false, 
+						$end = 0, $redirectsOnly = false, $oldRedirectsOnly = false ) {
+		global $wgUser, $wgParser, $wgUseTidy;
 
-    --help      : This help message
-    --dfn-only  : ???
-    -m <number> : Specifies max replication lag?  Does it abort or wait if this
-        is exceeded?
-    start       : First page id to refresh?  Doesn't work with --dfn-only set?
-    -e <number> : Last page id to refresh?
+		$reportingInterval = 100;
+		$dbr = wfGetDB( DB_SLAVE );
+		$start = intval( $start );
 
-This uses wfGetDB() to get the database, it seems not to accept a database ar-
-gument on the command line.  So I don't know if you can use it for non-default
-configuration.
+		# Don't generate TeX PNGs (lack of a sensible current directory causes errors anyway)
+		$wgUser->setOption('math', MW_MATH_SOURCE);
 
-Todo: Real documentation.
+		# Don't generate extension images (e.g. Timeline)
+		if( method_exists( $wgParser, "clearTagHooks" ) ) {
+			$wgParser->clearTagHooks();
+		}
 
-TEXT;
-	exit(0);
-}
+		# Don't use HTML tidy
+		$wgUseTidy = false;
+
+		$what = $redirectsOnly ? "redirects" : "links";
+
+		if( $oldRedirectsOnly ) {
+			# This entire code path is cut-and-pasted from below.  Hurrah.
+			$res = $dbr->query(
+				"SELECT page_id ".
+				"FROM page ".
+				"LEFT JOIN redirect ON page_id=rd_from ".
+				"WHERE page_is_redirect=1 AND rd_from IS NULL AND ".
+				($end == 0 ? "page_id >= $start"
+						   : "page_id BETWEEN $start AND $end"),
+				__METHOD__
+			);
+			$num = $dbr->numRows( $res );
+			$this->output( "Refreshing $num old redirects from $start...\n" );
+
+			foreach( $res as $row ) {
+				if ( !( ++$i % $reportingInterval ) ) {
+					$this->output( "$i\n" );
+					wfWaitForSlaves( $maxLag );
+				}
+				$this->fixRedirect( $row->page_id );
+			}
+		} elseif( $newOnly ) {
+			$this->output( "Refreshing $what from " );
+			$res = $dbr->select( 'page',
+				array( 'page_id' ),
+				array(
+					'page_is_new' => 1,
+					"page_id >= $start" ),
+				__METHOD__
+			);
+			$num = $dbr->numRows( $res );
+			$this->output( "$num new articles...\n" );
+	
+			$i = 0;
+			foreach ( $res as $row ) {
+				if ( !( ++$i % $reportingInterval ) ) {
+					$this->output( "$i\n" );
+					wfWaitForSlaves( $maxLag );
+				}
+				if($redirectsOnly)
+					$this->fixRedirect( $row->page_id );
+				else
+					$this->fixLinksFromArticle( $row->page_id );
+			}
+		} else {
+			if ( !$end ) {
+				$end = $dbr->selectField( 'page', 'max(page_id)', false );
+			}
+			$this->output( "Refreshing redirects table.\n" );
+			$this->output( "Starting from page_id $start of $end.\n" );
+	
+			for ($id = $start; $id <= $end; $id++) {
+	
+				if ( !($id % $reportingInterval) ) {
+					$this->output( "$id\n" );
+					wfWaitForSlaves( $maxLag );
+				}
+				$this->fixRedirect( $id );
+			}
 
-error_reporting( E_ALL & (~E_NOTICE) );
+			if(!$redirectsOnly) {
+				$this->output( "Refreshing links table.\n" );
+				$this->output( "Starting from page_id $start of $end.\n" );
 
-if ( !$options['dfn-only'] ) {
-	if ($args[0]) {
-		$start = (int)$args[0];
-	} else {
-		$start = 1;
+				for ($id = $start; $id <= $end; $id++) {
+	
+					if ( !($id % $reportingInterval) ) {
+						$this->output( "$id\n" );
+						wfWaitForSlaves( $maxLag );
+					}
+					$this->fixLinksFromArticle( $id );
+				}
+			}
+		}
 	}
 
-	refreshLinks( $start, $options['new-only'], $options['m'], $options['e'], $options['redirects-only'] );
-}
-// this bit's bad for replication: disabling temporarily
-// --brion 2005-07-16
-//deleteLinksFromNonexistent();
+	/**
+	 * Update the redirect entry for a given page
+	 * @param $id int The page_id of the redirect
+	 */
+	private function fixRedirect( $id ){
+		global $wgTitle, $wgArticle;
+	
+		$wgTitle = Title::newFromID( $id );
+		$dbw = wfGetDB( DB_MASTER );
+	
+		if ( is_null( $wgTitle ) ) {
+			return;
+		}
+		$wgArticle = new Article($wgTitle);
+	
+		$rt = $wgArticle->followRedirect();
+	
+		if($rt == false || !is_object($rt))
+			return;
+	
+		$wgArticle->updateRedirectOn($dbw,$rt);
+	}
 
-if ( $options['globals'] ) {
-	print_r( $GLOBALS );
-}
+	/**
+	 * Run LinksUpdate for all links on a given page_id
+	 * @param $id int The page_id
+	 */
+	private function fixLinksFromArticle( $id ) {
+		global $wgTitle, $wgParser;
+
+		$wgTitle = Title::newFromID( $id );
+		$dbw = wfGetDB( DB_MASTER );
+
+		$linkCache =& LinkCache::singleton();
+		$linkCache->clear();
+
+		if ( is_null( $wgTitle ) ) {
+			return;
+		}
+		$dbw->begin();
 
+		$revision = Revision::newFromTitle( $wgTitle );
+		if ( !$revision ) {
+			return;
+		}
+
+		$options = new ParserOptions;
+		$parserOutput = $wgParser->parse( $revision->getText(), $wgTitle, $options, true, true, $revision->getId() );
+		$update = new LinksUpdate( $wgTitle, $parserOutput, false );
+		$update->doUpdate();
+		$dbw->commit();
+	}
+
+	/*
+	 * Removes non-existing links from pages from pagelinks, imagelinks,
+	 * categorylinks, templatelinks and externallinks tables.
+	 *
+	 * @param $maxLag
+	 * @param $batchSize The size of deletion batches
+	 *
+	 * @author Merlijn van Deen <valhallasw@arctus.nl>
+	 */
+	private function deleteLinksFromNonexistent( $maxLag = 0, $batchSize = 100 ) {
+		wfWaitForSlaves( $maxLag );
+
+		$dbw = wfGetDB( DB_MASTER );
+
+		$lb = wfGetLBFactory()->newMainLB();
+		$dbr = $lb->getConnection( DB_SLAVE );
+		$dbr->bufferResults( false );
+
+		$linksTables = array( // table name => page_id field
+			'pagelinks' => 'pl_from',
+			'imagelinks' => 'il_from',
+			'categorylinks' => 'cl_from',
+			'templatelinks' => 'tl_from',
+			'externallinks' => 'el_from',
+		);
+
+		foreach ( $linksTables as $table => $field ) {
+			$this->output( "Retrieving illegal entries from $table... " );
+
+			// SELECT DISTINCT( $field ) FROM $table LEFT JOIN page ON $field=page_id WHERE page_id IS NULL;
+			$results = $dbr->select( array( $table, 'page' ),
+						  $field,
+						  array('page_id' => null ),
+						  __METHOD__,
+						  'DISTINCT',
+						  array( 'page' => array( 'LEFT JOIN', "$field=page_id"))
+			);
+
+			$counter = 0;
+			$list = array();
+			$this->output( "0.." );
+
+			foreach( $results as $row ) {
+				$counter++;
+				$list[] = $row->$field;
+				if ( ( $counter % $batchSize ) == 0 ) {
+					wfWaitForSlaves(5);
+					$dbw->delete( $table, array( $field => $list ), __METHOD__ );
+
+					$this->output( $counter . ".." );
+					$list = array();
+				}
+			}
+			$this->output( $counter );
+			if (count($list) > 0) {
+				$dbw->delete( $table, array( $field => $list ), __METHOD__ );
+			}
+			$this->output( "\n" );
+		}
+		$lb->closeAll();
+	}
+}
 
+$maintClass = 'RefreshLinks';
+require_once( DO_MAINTENANCE );