<?php

die( "rebuildLinks.inc needs to be updated for the new schema\n" );

# Functions for rebuilding the link tracking tables; must
# be included within a script that also includes the Setup.
# See rebuildlinks.php, for example.
#

# Buffer this many rows before inserting them all in one sweep. More
# than about 1000 will probably not increase speed significantly on
# most setups.
/* private */ $rowbuf_size = 1000; // 1000 rows ~40 kB

function rebuildLinkTables()
{
	error_reporting (E_ALL);
	global $wgLang, $wgLinkCache, $rowbuf_size;

	print "This script may take several hours to complete. If you abort during that time,\n";
	print "your wiki will be in an inconsistent state. If you are going to abort, this is\n";
	print "the time to do it.\n\n";
	print "Press control-c to abort (will proceed automatically in 15 seconds)\n";
	sleep(15);

	$count = 0;
	print "Rebuilding link tables.\n";

	print "Setting AUTOCOMMIT=1\n";
	wfQuery("SET SESSION AUTOCOMMIT=1", DB_WRITE);

	print "Extracting often used data from cur (may take a few minutes)\n";
	$sql = "CREATE TEMPORARY TABLE cur_fast SELECT cur_namespace, cur_title, cur_id FROM cur";
	wfQuery( $sql, DB_WRITE );
	$sql = "ALTER TABLE cur_fast ADD INDEX(cur_namespace, cur_title)";
	wfQuery( $sql, DB_WRITE );

	print "Locking tables\n";
	$sql = "LOCK TABLES cur READ, cur_fast READ, interwiki READ, user_newtalk READ, " .
		"links WRITE, brokenlinks WRITE, imagelinks WRITE";
	wfQuery( $sql, DB_WRITE );


	print "Deleting old data in links table.\n";
	$sql = "DELETE FROM links";
	wfQuery( $sql, DB_WRITE );

	print "Deleting old data in brokenlinks table.\n";
	$sql = "DELETE FROM brokenlinks";
	wfQuery( $sql, DB_WRITE );

	print "Deleting old data in imagelinks table.\n";
	$sql = "DELETE FROM imagelinks";
	wfQuery( $sql, DB_WRITE );

	print "Finding number of articles to process... ";
	$sql = "SELECT COUNT(*) as count FROM cur";
	$res = wfQuery( $sql, DB_READ );
	$obj = wfFetchObject( $res );
	$total = $obj->count;
	print "$total\n";

	print "Finding highest article id\n";
	$sql = "SELECT MIN(cur_id) AS min, MAX(cur_id) AS max FROM cur";
	$res = wfQuery( $sql, DB_READ );
	$obj = wfFetchObject( $res );
 
	$cur_pulser = new SelectPulser("SELECT cur_id,cur_namespace,cur_title,cur_text " .
		"FROM cur WHERE cur_id ", 
		$obj->min, $obj->max, 100);

	$brokenlinks_inserter = new InsertBuffer(
		"INSERT IGNORE INTO brokenlinks (bl_from,bl_to) VALUES " , $rowbuf_size);

	$links_inserter = new InsertBuffer(
		"INSERT IGNORE INTO links (l_from,l_to) VALUES ", $rowbuf_size);

	$imagelinks_inserter = new InsertBuffer("INSERT IGNORE INTO imagelinks ".
		"(il_from,il_to) VALUES ", $rowbuf_size);

	print "Starting processing\n";

	$ins = $wgLang->getNsText( Namespace::getImage() );
	$inslen = strlen($ins)+1;

	$tc = Title::legalChars();
	
	$titleCache = new MRUCache( 10000 );
	$titlecount = 0;
	$start_time = time();

	while ( $row = $cur_pulser->next() ) {

		$from_id = intval($row->cur_id);
		$ns = $wgLang->getNsText( $row->cur_namespace );	
		$from_full_title = $row->cur_title;		
		if ( "" != $ns ) {
			$from_full_title = "$ns:{$from_full_title}";
		}
		$from_full_title_with_slashes = addslashes( $from_full_title );
		$text = $row->cur_text;

		$numlinks = preg_match_all( "/\\[\\[([{$tc}]+)(]|\\|)/", $text,
		  $m, PREG_PATTERN_ORDER );

		$seen_dbtitles = array(); // seen links (normalized and with ns, see below) 
		$titles_ready_for_insertion = array();
		$titles_needing_curdata = array();
		$titles_needing_curdata_pos = array();
		$links_corresponding_to_titles = array();

		for ( $i = 0 ; $i < $numlinks; ++$i ) {
			$link = $m[1][$i];
			if( preg_match( '/^(http|https|ftp|mailto|news):/', $m[1][$i] ) ) {
				# an URL link; not for us!
				continue;
			}
			
			# FIXME: Handle subpage links
			$nt = $titleCache->get( $link );
			if( $nt != false ){
				// Only process each unique link once per page
				$nt_key = $nt->getDBkey() . $nt->getNamespace();
				if( isset( $seen_dbtitles[$nt_key] ) )
					continue;
				$seen_dbtitles[$nt_key] = 1;

				$titles_ready_for_insertion[] = $nt;
			} else {
				$nt = Title::newFromText( $link );
				if (! $nt) {
					// Invalid link, probably something like "[[  ]]"
					continue;
				}
				
				// Only process each unique link once per page
				$nt_key = $nt->getDBkey() . $nt->getNamespace();
				if( isset( $seen_dbtitles[$nt_key] ) )
					continue;
				$seen_dbtitles[$nt_key] = 1;

				if( $nt->getInterwiki() != "" ) {
					# Interwiki links are not stored in the link tables
					continue;
				}
				if( $nt->getNamespace() == Namespace::getSpecial() ) {
					# Special links not stored in link tables
					continue;
				}
				if( $nt->getNamespace() == Namespace::getMedia() ) {
					# treat media: links as image: links
					$nt = Title::makeTitle( Namespace::getImage(), $nt->getDBkey() );
				}
				$nt->mArticleID = 0; // assume broken link until proven otherwise

				$pos = array_push($titles_needing_curdata, $nt) - 1;
				$titles_needing_curdata_pos[$nt->getDBkey() . $nt->getNamespace()] = $pos;
				$links_corresponding_to_titles[] = $link;
				unset( $link ); // useless outside this loop, but tempting 
			}
		}


		if ( count( $titles_needing_curdata ) > 0 ){
			$parts = array();
			foreach ($titles_needing_curdata as $nt ) {
				$parts[] = " (cur_namespace = " . $nt->getNamespace() . " AND " .
					"cur_title='" . wfStrencode( $nt->getDBkey() ) . "')";
			}
			$sql = "SELECT cur_namespace, cur_title, cur_id FROM cur_fast WHERE " . 
				implode(" OR ", $parts);
			$res = wfQuery( $sql, DB_WRITE );
			while($row = wfFetchObject( $res ) ){
				$pos = $titles_needing_curdata_pos[$row->cur_title . $row->cur_namespace];
				$titles_needing_curdata[$pos]->mArticleID = intval($row->cur_id);
			}
			for( $k = 0; $k < count( $titles_needing_curdata ) ; $k++) {
				$tmplink = $links_corresponding_to_titles[$k];
				$titleCache->set( $tmplink, $titles_needing_curdata[$k] );
				$titles_ready_for_insertion[] = $titles_needing_curdata[$k];
			}
		}

		foreach ( $titles_ready_for_insertion as $nt ) {
			$dest_noslashes = $nt->getPrefixedDBkey();
			$dest = addslashes( $dest_noslashes ); 
			$dest_id = $nt->getArticleID();
			$from = $from_full_title_with_slashes;

			# print "\nLINK '$from_full_title' ($from_id) -> '$dest' ($dest_id)\n";

			if ( 0 == strncmp( "$ins:", $dest_noslashes, $inslen ) ) { 
				$iname = addslashes( substr( $dest_noslashes, $inslen ) );
				$imagelinks_inserter->insert( "('{$from}','{$iname}')" );
			} else if ( 0 == $dest_id ) {
				$brokenlinks_inserter->insert( "({$from_id},'{$dest}')" );
			} else {
				$links_inserter->insert( "('{$from}',{$dest_id})" );
			}
			$titlecount++;
		}

		if ( ( $count % 20 ) == 0 )
			print "."; 

		if ( ( ++$count % 1000 ) == 0 ) {
			$dt = time() - $start_time;
			$start_time = time();
			$rps = persec(1000, $dt);
			$tps = persec($titlecount, $dt);
			$titlecount = 0;
			print "\n$count of $total articles scanned ({$rps} articles ".
				"and {$tps} titles per second)\n";
			print "Title cache hits: " . $titleCache->getPerformance() . "%\n";

		}

	}

	print "\nFlushing insertion buffers...";
	$imagelinks_inserter->flush();
	$links_inserter->flush();
	$brokenlinks_inserter->flush();
	print "ok\n";

	print "$count articles scanned.\n";

	$sql = "UNLOCK TABLES";
	wfQuery( $sql, DB_WRITE );
	print "Done\n";
}

/* private */ function persec($n, $t){
	if($n == 0)
		return "zero";
	if($t == 0)
		return "lots of";
	return intval($n/$t);
}

# InsertBuffer increases performance slightly by inserting many rows
# at once. The gain is small (<5%) when running against a local, idle
# database, but may be significant in other circumstances. It also
# limits the number of inserted rows uppwards, which should avoid
# problems with huge articles and certain mysql settings that limits
# the size of queries. It's also convenient.

class InsertBuffer {
	/* private */ var $mBuf, $mSql, $mBufcount, $mMaxsize;
	
	function InsertBuffer( $sql, $bufsize ){
		$this->mSql = $sql;
		$this->mBuf = array();
		$this->mBufcount = 0;
		$this->mMaxsize = $bufsize;
	}

	function insert( $value ){
		// print $this->mSql . " -> " . $value . "\n";
		$this->mBuf[] = $value;
		$this->mBufcount++;
		if($this->mBufcount > $this->mMaxsize){
			$this->flush();
		}
	}

	function flush(){
		if( $this->mBufcount > 0 ){
			$sql = $this->mSql . implode(",", $this->mBuf);
			wfQuery( $sql, DB_WRITE );
			$this->mBuf = array();
			$this->mBufcount = 0;
			// print "Wrote query of size " . strlen( $sql ) . "\n";
		}
	}
	
}

# Select parts from a large table by using the "BETWEEN X AND Y"
# operator on the id column. Avoids buffering the whole thing in
# RAM. It's also convenient.  

class SelectPulser {
	/* private */ var $mSql, $mSetsize, $mPos, $mMax, $mSet;

	function SelectPulser( $sql, $min, $max, $setsize) {
		$this->mSql = $sql;
		$this->mSet = array();
		$this->mPos = $min;
		$this->mMax = $max;
		$this->mSetsize = $setsize;
	}
	
	function next(){
		$result = current( $this->mSet );
		next( $this->mSet );
		if( false !== $result ){
			return $result;
		}
		while( $this->mPos <= $this->mMax ){
			$this->mSet = array();
			$sql = $this->mSql . " BETWEEN " . $this->mPos .
				" AND " . ($this->mPos + $this->mSetsize - 1);
			$this->mPos += $this->mSetsize;

			$res = wfQuery( $sql, DB_READ );
			while ( $row = wfFetchObject( $res ) ) {
				$this->mSet[] = $row;
			}
			wfFreeResult( $res );
			if( count( $this->mSet ) > 0 ){
				return $this->next();
			} 
		}
		return false;
	}
}

# A simple MRU for general cacheing.

class MRUCache {
	/* private */ var $mMru, $mCache, $mSize, $mPurgefreq, $nexti;
	/* private */ var $hits, $misses;

	function MRUCache( $size, $purgefreq = -1 ) {
		// purgefreq is 1/10 of $size if not stated		
		$purgefreq = ($purgefreq == -1 ? intval($size/10) : $purgefreq);
		$purgefreq = ($purgefreq <= 0 ? 1 : $purgefreq);

		$this->mSize = $size;
		$this->mMru = array();
		$this->mCache = array();
		$this->mPurgefreq = $purgefreq;
		$this->nexti = 1;
		print "purgefreq = " . $this->mPurgefreq . "\n";
	}

	function get( $key ){
		if ( ! array_key_exists( $key, $this->mCache) ){
			$this->misses++;
			return false;
		}
		$this->hits++;
		$this->mMru[$key] = $this->nexti++;
		return $this->mCache[$key];
	}
	
	function set( $key, $value ){	
		$this->mMru[$key] = $this->nexti++;
		$this->mCache[$key] = $value;

		if($this->nexti % $this->mPurgefreq == 0)
			$this->purge();
	}

	function purge(){
		$to_remove = count( $this->mMru ) - $this->mSize;
		if( $to_remove <= 0 ){
			return;
		}		
		asort( $this->mMru );
		$removed = array_splice( $this->mMru, 0, $to_remove );
		foreach( array_keys( $removed ) as $key ){
			unset( $this->mCache[$key] );
		}
	}
	
	function getPerformance(){
		$tot = $this->hits + $this->misses;
		if($tot > 0)
			return intval(100.0 * $this->hits / $tot);
		else
			return 0;
	}
}	

?>