X-Git-Url: https://git.cyclocoop.org/%27.WWW_URL.%27admin/?a=blobdiff_plain;f=maintenance%2FrefreshLinks.php;h=6dccefb3358a8a07722e1502943699e695de2ca0;hb=291efd30f2173070df8c1939f659759521f8fae6;hp=aa91e1b510a7096f9a53dd45574ea7406bf104fd;hpb=05c146c191bfa58eb2b530a52c57962f94e864a7;p=lhc%2Fweb%2Fwiklou.git diff --git a/maintenance/refreshLinks.php b/maintenance/refreshLinks.php index aa91e1b510..6dccefb335 100644 --- a/maintenance/refreshLinks.php +++ b/maintenance/refreshLinks.php @@ -1,52 +1,273 @@ ] [-e ] [-m ] [--dfn-only] - [--new-only] [--redirects-only] - php refreshLinks.php [] [-e ] [-m ] --old-redirects-only - - --help : This help message - --dfn-only : Delete links from nonexistent articles only - --batch-size : The delete batch size when removing links from - nonexistent articles (default 100) - --new-only : Only affect articles with just a single edit - --redirects-only : Only fix redirects, not all links - --old-redirects-only : Only fix redirects with no redirect table entry - -m : Maximum replication lag - : First page id to refresh - -e : Last page id to refresh - -TEXT; - exit(0); -} +require_once( dirname(__FILE__) . '/Maintenance.php' ); -error_reporting( E_ALL & (~E_NOTICE) ); +class RefreshLinks extends Maintenance { + public function __construct() { + parent::__construct(); + $this->mDescription = "Refresh link tables"; + $this->addOption( 'dfn-only', 'Delete links from nonexistent articles only' ); + $this->addOption( 'new-only', 'Only affect articles with just a single edit' ); + $this->addOption( 'redirects-only', 'Only fix redirects, not all links' ); + $this->addOption( 'old-redirects-only', 'Only fix redirects with no redirect table entry' ); + $this->addOption( 'm', 'Maximum replication lag', false, true ); + $this->addOption( 'e', 'Last page id to refresh', false, true ); + $this->addArg( 'start', 'Page_id to start from, default 1' ); + $this->setBatchSize( 100 ); + } -if ( !$options['dfn-only'] ) { - if ( isset( $args[0] ) ) { - $start = (int)$args[0]; - } else { - $start = 1; + public function execute() { + if( !$this->hasOption( 'dfn-only' ) ) { + $start = $this->getArg( 0, 1 ); + $new = $this->getOption( 'new-only', false ); + $max = $this->getOption( 'm', false ); + $end = $this->getOption( 'e', 0 ); + $redir = $this->getOption( 'redirects-only', false ); + $oldRedir = $this->getOption( 'old-redirects-only', false ); + $this->doRefreshLinks( $start, $new, $max, $end, $redir, $oldRedir ); + } + $this->deleteLinksFromNonexistent( $max, $this->mBatchSize ); } - refreshLinks( $start, $options['new-only'], $options['m'], $options['e'], $options['redirects-only'], $options['old-redirects-only'] ); -} + /** + * Do the actual link refreshing. + * @param $start int Page_id to start from + * @param $newOnly bool Only do pages with 1 edit + * @param $maxLag int Max DB replication lag + * @param $end int Page_id to stop at + * @param $redirectsOnly bool Only fix redirects + * @param $oldRedirectsOnly bool Only fix redirects without redirect entries + */ + private function doRefreshLinks( $start, $newOnly = false, $maxLag = false, + $end = 0, $redirectsOnly = false, $oldRedirectsOnly = false ) { + global $wgUser, $wgParser, $wgUseTidy; + + $reportingInterval = 100; + $dbr = wfGetDB( DB_SLAVE ); + $start = intval( $start ); + + # Don't generate TeX PNGs (lack of a sensible current directory causes errors anyway) + $wgUser->setOption('math', MW_MATH_SOURCE); + + # Don't generate extension images (e.g. Timeline) + if( method_exists( $wgParser, "clearTagHooks" ) ) { + $wgParser->clearTagHooks(); + } + + # Don't use HTML tidy + $wgUseTidy = false; + + $what = $redirectsOnly ? "redirects" : "links"; + + if( $oldRedirectsOnly ) { + # This entire code path is cut-and-pasted from below. Hurrah. + $res = $dbr->query( + "SELECT page_id ". + "FROM page ". + "LEFT JOIN redirect ON page_id=rd_from ". + "WHERE page_is_redirect=1 AND rd_from IS NULL AND ". + ($end == 0 ? "page_id >= $start" + : "page_id BETWEEN $start AND $end"), + __METHOD__ + ); + $num = $dbr->numRows( $res ); + $this->output( "Refreshing $num old redirects from $start...\n" ); + + foreach( $res as $row ) { + if ( !( ++$i % $reportingInterval ) ) { + $this->output( "$i\n" ); + wfWaitForSlaves( $maxLag ); + } + $this->fixRedirect( $row->page_id ); + } + } elseif( $newOnly ) { + $this->output( "Refreshing $what from " ); + $res = $dbr->select( 'page', + array( 'page_id' ), + array( + 'page_is_new' => 1, + "page_id >= $start" ), + __METHOD__ + ); + $num = $dbr->numRows( $res ); + $this->output( "$num new articles...\n" ); + + $i = 0; + foreach ( $res as $row ) { + if ( !( ++$i % $reportingInterval ) ) { + $this->output( "$i\n" ); + wfWaitForSlaves( $maxLag ); + } + if($redirectsOnly) + $this->fixRedirect( $row->page_id ); + else + $this->fixLinksFromArticle( $row->page_id ); + } + } else { + if ( !$end ) { + $end = $dbr->selectField( 'page', 'max(page_id)', false ); + } + $this->output( "Refreshing redirects table.\n" ); + $this->output( "Starting from page_id $start of $end.\n" ); + + for ($id = $start; $id <= $end; $id++) { + + if ( !($id % $reportingInterval) ) { + $this->output( "$id\n" ); + wfWaitForSlaves( $maxLag ); + } + $this->fixRedirect( $id ); + } + + if(!$redirectsOnly) { + $this->output( "Refreshing links table.\n" ); + $this->output( "Starting from page_id $start of $end.\n" ); + + for ($id = $start; $id <= $end; $id++) { + + if ( !($id % $reportingInterval) ) { + $this->output( "$id\n" ); + wfWaitForSlaves( $maxLag ); + } + $this->fixLinksFromArticle( $id ); + } + } + } + } + + /** + * Update the redirect entry for a given page + * @param $id int The page_id of the redirect + */ + private function fixRedirect( $id ){ + global $wgTitle, $wgArticle; + + $wgTitle = Title::newFromID( $id ); + $dbw = wfGetDB( DB_MASTER ); + + if ( is_null( $wgTitle ) ) { + return; + } + $wgArticle = new Article($wgTitle); + + $rt = $wgArticle->followRedirect(); + + if($rt == false || !is_object($rt)) + return; + + $wgArticle->updateRedirectOn($dbw,$rt); + } + + /** + * Run LinksUpdate for all links on a given page_id + * @param $id int The page_id + */ + private function fixLinksFromArticle( $id ) { + global $wgTitle, $wgParser; -deleteLinksFromNonexistent($options['m'], $options['batch-size']); + $wgTitle = Title::newFromID( $id ); + $dbw = wfGetDB( DB_MASTER ); -if ( $options['globals'] ) { - print_r( $GLOBALS ); + $linkCache =& LinkCache::singleton(); + $linkCache->clear(); + + if ( is_null( $wgTitle ) ) { + return; + } + $dbw->begin(); + + $revision = Revision::newFromTitle( $wgTitle ); + if ( !$revision ) { + return; + } + + $options = new ParserOptions; + $parserOutput = $wgParser->parse( $revision->getText(), $wgTitle, $options, true, true, $revision->getId() ); + $update = new LinksUpdate( $wgTitle, $parserOutput, false ); + $update->doUpdate(); + $dbw->commit(); + } + + /* + * Removes non-existing links from pages from pagelinks, imagelinks, + * categorylinks, templatelinks and externallinks tables. + * + * @param $maxLag + * @param $batchSize The size of deletion batches + * + * @author Merlijn van Deen + */ + private function deleteLinksFromNonexistent( $maxLag = 0, $batchSize = 100 ) { + wfWaitForSlaves( $maxLag ); + + $dbw = wfGetDB( DB_MASTER ); + + $lb = wfGetLBFactory()->newMainLB(); + $dbr = $lb->getConnection( DB_SLAVE ); + $dbr->bufferResults( false ); + + $linksTables = array( // table name => page_id field + 'pagelinks' => 'pl_from', + 'imagelinks' => 'il_from', + 'categorylinks' => 'cl_from', + 'templatelinks' => 'tl_from', + 'externallinks' => 'el_from', + ); + + foreach ( $linksTables as $table => $field ) { + $this->output( "Retrieving illegal entries from $table... " ); + + // SELECT DISTINCT( $field ) FROM $table LEFT JOIN page ON $field=page_id WHERE page_id IS NULL; + $results = $dbr->select( array( $table, 'page' ), + $field, + array('page_id' => null ), + __METHOD__, + 'DISTINCT', + array( 'page' => array( 'LEFT JOIN', "$field=page_id")) + ); + + $counter = 0; + $list = array(); + $this->output( "0.." ); + + foreach( $results as $row ) { + $counter++; + $list[] = $row->$field; + if ( ( $counter % $batchSize ) == 0 ) { + wfWaitForSlaves(5); + $dbw->delete( $table, array( $field => $list ), __METHOD__ ); + + $this->output( $counter . ".." ); + $list = array(); + } + } + $this->output( $counter ); + if (count($list) > 0) { + $dbw->delete( $table, array( $field => $list ), __METHOD__ ); + } + $this->output( "\n" ); + } + $lb->closeAll(); + } } + +$maintClass = 'RefreshLinks'; +require_once( DO_MAINTENANCE );