From: Lee Daniel Crocker Date: Fri, 2 May 2003 22:55:37 +0000 (+0000) Subject: Completed several maintenance scripts for index rebuilding. X-Git-Tag: 1.1.0~571 X-Git-Url: http://git.cyclocoop.org/%7B%24www_url%7Dadmin/compta/exercices/?a=commitdiff_plain;h=f3a4fd56c020aa717dfd6eb69bdfe6a10ea35393;p=lhc%2Fweb%2Fwiklou.git Completed several maintenance scripts for index rebuilding. --- diff --git a/maintenance/rebuildIndex.php b/maintenance/rebuildIndex.php deleted file mode 100644 index 1bd085cff4..0000000000 --- a/maintenance/rebuildIndex.php +++ /dev/null @@ -1,46 +0,0 @@ -count} pages...\n"; -$n = 0; - -$sql = "SELECT cur_id, cur_namespace, cur_title, cur_text FROM cur"; -$res = wfQuery($sql); -while( $s = wfFetchObject($res)) { - $u = new SearchUpdate( $s->cur_id, $s->cur_title, $s->cur_text ); - $u->doUpdate(); - if ( ( (++$n) % 500) == 0) { - echo "$n\n"; - } -} -wfFreeResult( $res ); - -#echo "Rebuild the index...\n"; -##$sql = "ALTER TABLE searchindex ADD FULLTEXT si_title (si_title), -## ADD FULLTEXT si_text (si_text)"; -#$res = wfQuery($sql); - -print "Done.\n"; -exit(); - -?> diff --git a/maintenance/rebuildLinks.inc b/maintenance/rebuildLinks.inc deleted file mode 100644 index e98d6d7635..0000000000 --- a/maintenance/rebuildLinks.inc +++ /dev/null @@ -1,166 +0,0 @@ -cur_id; - $ns = $wgLang->getNsText( $row->cur_namespace ); - if ( "" == $ns ) { - $title = addslashes( $row->cur_title ); - } else { - $title = addslashes( "$ns:{$row->cur_title}" ); - } - $text = $row->cur_text; - $numlinks = preg_match_all( "/\\[\\[([{$tc}]+)(]|\\|)/", $text, - $m, PREG_PATTERN_ORDER ); - - if ( 0 != $numlinks ) { - $sql = "INSERT INTO rebuildlinks (rl_f_id,rl_f_title,rl_to) VALUES "; - for ( $i = 0; $i < $numlinks; ++$i ) { - $nt = Title::newFromText( $m[1][$i] ); - $dest = addslashes( $nt->getPrefixedDBkey() ); - - if ( 0 != $i ) { $sql .= ","; } - $sql .= "({$id},'{$title}','{$dest}')"; - } - wfQuery( $sql ); - } - if ( ( ++$count % 1000 ) == 0 ) { - print "$count of $total articles scanned.\n"; - } - } - print "$count articles scanned.\n"; - mysql_free_result( $res ); - - $sql = "UNLOCK TABLES"; - wfQuery( $sql ); -} - -function rebuildLinkTablesPass2() -{ - global $wgLang; - $count = 0; - print "Rebuilding link tables (pass 2).\n"; - - $sql = "LOCK TABLES cur READ, rebuildlinks READ, " . - "links WRITE, brokenlinks WRITE, imagelinks WRITE"; - wfQuery( $sql ); - - $sql = "DELETE FROM links"; - wfQuery( $sql ); - - $sql = "DELETE FROM brokenlinks"; - wfQuery( $sql ); - - $sql = "DELETE FROM links"; - wfQuery( $sql ); - - $ins = $wgLang->getNsText( Namespace::getImage() ); - $inslen = strlen($ins)+1; - $sql = "SELECT rl_f_title,rl_to FROM rebuildlinks " . - "WHERE rl_to LIKE '$ins:%'"; - $res = wfQuery( $sql ); - - $sql = "INSERT INTO imagelinks (il_from,il_to) VALUES "; - $sqlX = ""; - $first = true; - while ( $row = wfFetchObject( $res ) ) { - $iname = addslashes( substr( $row->rl_to, $inslen ) ); - $pname = addslashes( $row->rl_f_title ); - - if ( ! $first ) { $sqlX .= ","; } - $first = false; - - $sqlX .= "('{$pname}','{$iname}')"; - } - if ($sqlX != "") { - $sql .= $sqlX; - wfFreeResult( $res ); - wfQuery( $sql ); - } - - $sql = "SELECT DISTINCT rl_to FROM rebuildlinks " . - "ORDER BY rl_to"; - $res = wfQuery( $sql ); - $count = 0; - $total = wfNumRows( $res ); - - while ( $row = wfFetchObject( $res ) ) { - if ( 0 == strncmp( "$ins:", $row->rl_to, $inslen ) ) { continue; } - - $nt = Title::newFromDBkey( $row->rl_to ); - $id = $nt->getArticleID(); - $to = addslashes( $row->rl_to ); - - if ( 0 == $id ) { - $sql = "SELECT rl_f_id FROM rebuildlinks WHERE rl_to='{$to}'"; - $res2 = wfQuery( $sql ); - - $sql = "INSERT INTO brokenlinks (bl_from,bl_to) VALUES "; - $first = true; - while ( $row2 = wfFetchObject( $res2 ) ) { - $from = $row2->rl_f_id; - if ( ! $first ) { $sql .= ","; } - $first = false; - $sql .= "({$from},'{$to}')"; - } - wfFreeResult( $res2 ); - if ( ! $first ) { wfQuery( $sql ); } - } else { - $sql = "SELECT rl_f_title FROM rebuildlinks WHERE rl_to='{$to}'"; - $res2 = wfQuery( $sql ); - - $sql = "INSERT INTO links (l_from,l_to) VALUES "; - $first = true; - while ( $row2 = wfFetchObject( $res2 ) ) { - $from = addslashes( $row2->rl_f_title ); - if ( ! $first ) { $sql .= ","; } - $first = false; - $sql .= "('{$from}',{$id})"; - } - wfFreeResult( $res2 ); - if ( ! $first ) { wfQuery( $sql ); } - } - if ( ( ++$count % 1000 ) == 0 ) { - print "$count of $total titles processed.\n"; - } - } - wfFreeResult( $res ); - - $sql = "UNLOCK TABLES"; - wfQuery( $sql ); - - $sql = "DROP TABLE rebuildlinks"; - wfQuery( $sql ); -} -?> diff --git a/maintenance/rebuildLinks.php b/maintenance/rebuildLinks.php deleted file mode 100644 index 2890507dfe..0000000000 --- a/maintenance/rebuildLinks.php +++ /dev/null @@ -1,25 +0,0 @@ - diff --git a/maintenance/rebuildall.php b/maintenance/rebuildall.php new file mode 100644 index 0000000000..9e64d193c3 --- /dev/null +++ b/maintenance/rebuildall.php @@ -0,0 +1,39 @@ + diff --git a/maintenance/rebuildlinks.inc b/maintenance/rebuildlinks.inc new file mode 100644 index 0000000000..7dd3ddee50 --- /dev/null +++ b/maintenance/rebuildlinks.inc @@ -0,0 +1,166 @@ +cur_id; + $ns = $wgLang->getNsText( $row->cur_namespace ); + if ( "" == $ns ) { + $title = addslashes( $row->cur_title ); + } else { + $title = addslashes( "$ns:{$row->cur_title}" ); + } + $text = $row->cur_text; + $numlinks = preg_match_all( "/\\[\\[([{$tc}]+)(]|\\|)/", $text, + $m, PREG_PATTERN_ORDER ); + + if ( 0 != $numlinks ) { + $sql = "INSERT INTO rebuildlinks (rl_f_id,rl_f_title,rl_to) VALUES "; + for ( $i = 0; $i < $numlinks; ++$i ) { + $nt = Title::newFromText( $m[1][$i] ); + $dest = addslashes( $nt->getPrefixedDBkey() ); + + if ( 0 != $i ) { $sql .= ","; } + $sql .= "({$id},'{$title}','{$dest}')"; + } + wfQuery( $sql ); + } + if ( ( ++$count % 1000 ) == 0 ) { + print "$count of $total articles scanned.\n"; + } + } + print "$count articles scanned.\n"; + mysql_free_result( $res ); + + $sql = "UNLOCK TABLES"; + wfQuery( $sql ); +} + +function rebuildLinkTablesPass2() +{ + global $wgLang; + $count = 0; + print "Rebuilding link tables (pass 2).\n"; + + $sql = "LOCK TABLES cur READ, rebuildlinks READ, " . + "links WRITE, brokenlinks WRITE, imagelinks WRITE"; + wfQuery( $sql ); + + $sql = "DELETE FROM links"; + wfQuery( $sql ); + + $sql = "DELETE FROM brokenlinks"; + wfQuery( $sql ); + + $sql = "DELETE FROM links"; + wfQuery( $sql ); + + $ins = $wgLang->getNsText( Namespace::getImage() ); + $inslen = strlen($ins)+1; + $sql = "SELECT rl_f_title,rl_to FROM rebuildlinks " . + "WHERE rl_to LIKE '$ins:%'"; + $res = wfQuery( $sql ); + + $sql = "INSERT INTO imagelinks (il_from,il_to) VALUES "; + $sqlX = ""; + $first = true; + while ( $row = wfFetchObject( $res ) ) { + $iname = addslashes( substr( $row->rl_to, $inslen ) ); + $pname = addslashes( $row->rl_f_title ); + + if ( ! $first ) { $sqlX .= ","; } + $first = false; + + $sqlX .= "('{$pname}','{$iname}')"; + } + if ($sqlX != "") { + $sql .= $sqlX; + wfFreeResult( $res ); + wfQuery( $sql ); + } + + $sql = "SELECT DISTINCT rl_to FROM rebuildlinks " . + "ORDER BY rl_to"; + $res = wfQuery( $sql ); + $count = 0; + $total = wfNumRows( $res ); + + while ( $row = wfFetchObject( $res ) ) { + if ( 0 == strncmp( "$ins:", $row->rl_to, $inslen ) ) { continue; } + + $nt = Title::newFromDBkey( $row->rl_to ); + $id = $nt->getArticleID(); + $to = addslashes( $row->rl_to ); + + if ( 0 == $id ) { + $sql = "SELECT rl_f_id FROM rebuildlinks WHERE rl_to='{$to}'"; + $res2 = wfQuery( $sql ); + + $sql = "INSERT INTO brokenlinks (bl_from,bl_to) VALUES "; + $first = true; + while ( $row2 = wfFetchObject( $res2 ) ) { + $from = $row2->rl_f_id; + if ( ! $first ) { $sql .= ","; } + $first = false; + $sql .= "({$from},'{$to}')"; + } + wfFreeResult( $res2 ); + if ( ! $first ) { wfQuery( $sql ); } + } else { + $sql = "SELECT rl_f_title FROM rebuildlinks WHERE rl_to='{$to}'"; + $res2 = wfQuery( $sql ); + + $sql = "INSERT INTO links (l_from,l_to) VALUES "; + $first = true; + while ( $row2 = wfFetchObject( $res2 ) ) { + $from = addslashes( $row2->rl_f_title ); + if ( ! $first ) { $sql .= ","; } + $first = false; + $sql .= "('{$from}',{$id})"; + } + wfFreeResult( $res2 ); + if ( ! $first ) { wfQuery( $sql ); } + } + if ( ( ++$count % 1000 ) == 0 ) { + print "$count of $total titles processed.\n"; + } + } + wfFreeResult( $res ); + + $sql = "UNLOCK TABLES"; + wfQuery( $sql ); + + $sql = "DROP TABLE rebuildlinks"; + wfQuery( $sql ); +} +?> diff --git a/maintenance/rebuildlinks.php b/maintenance/rebuildlinks.php new file mode 100644 index 0000000000..de756c1911 --- /dev/null +++ b/maintenance/rebuildlinks.php @@ -0,0 +1,30 @@ + diff --git a/maintenance/rebuildrecentchanges.inc b/maintenance/rebuildrecentchanges.inc new file mode 100644 index 0000000000..2d9c00933a --- /dev/null +++ b/maintenance/rebuildrecentchanges.inc @@ -0,0 +1,115 @@ +rc_timestamp; + + $sql = "DELETE FROM recentchanges WHERE rc_timestamp < '{$ts}'"; + wfQuery( $sql ); +} + +function rebuildRecentChangesTablePass2() +{ + $ns = $id = $count = 0; + $title = $ct = ""; + + print( "Updating links...\n" ); + + $sql = "SELECT rc_namespace,rc_title,rc_timestamp FROM recentchanges " . + "ORDER BY rc_namespace,rc_title,rc_timestamp DESC"; + $res = wfQuery( $sql ); + + while ( $obj = wfFetchObject( $res ) ) { + if ( ! ( $ns == $obj->rc_namespace && + 0 == strcmp( $title, wfStrencode( $obj->rc_title ) ) ) ) { + + $ns = $obj->rc_namespace; + $title = wfStrencode( $obj->rc_title ); + + $sql = "SELECT cur_id,cur_timestamp FROM cur WHERE " . + "cur_namespace={$ns} AND cur_title='{$title}'"; + $res2 = wfQuery( $sql ); + $obj2 = wfFetchObject( $res2 ); + + $id = $obj2->cur_id; + $ct = $obj2->cur_timestamp; + } + $sql = "SELECT old_id FROM old WHERE old_namespace={$ns} " . + "AND old_title='{$title}' AND old_timestamp < '" . + "{$obj->rc_timestamp}' ORDER BY old_timestamp DESC LIMIT 1"; + $res2 = wfQuery( $sql ); + + if ( 0 != wfNumRows( $res2 ) ) { + $obj2 = wfFetchObject( $res2 ); + + $sql = "UPDATE recentchanges SET rc_cur_id={$id},rc_cur_time=" . + "'{$ct}',rc_last_oldid={$obj2->old_id} WHERE " . + "rc_namespace={$ns} AND rc_title='{$title}' AND " . + "rc_timestamp='{$obj->rc_timestamp}'"; + wfQuery( $sql ); + } else { + $sql = "UPDATE recentchanges SET rc_cur_id={$id},rc_cur_time=" . + "'{$ct}' WHERE rc_namespace={$ns} AND rc_title='{$title}' " . + "AND rc_timestamp='{$obj->rc_timestamp}'"; + wfQuery( $sql ); + } + + if ( 0 == ( ++$count % 500 ) ) { + printf( "%d records processed.\n", $count ); + } + } +} + +?> diff --git a/maintenance/rebuildrecentchanges.php b/maintenance/rebuildrecentchanges.php new file mode 100644 index 0000000000..894353f179 --- /dev/null +++ b/maintenance/rebuildrecentchanges.php @@ -0,0 +1,30 @@ + diff --git a/maintenance/rebuildtextindex.inc b/maintenance/rebuildtextindex.inc new file mode 100644 index 0000000000..8d681c8fdf --- /dev/null +++ b/maintenance/rebuildtextindex.inc @@ -0,0 +1,43 @@ +count} pages...\n"; + $n = 0; + + $sql = "SELECT cur_id, cur_namespace, cur_title, cur_text FROM cur"; + $res = wfQuery($sql); + + while( $s = wfFetchObject($res) ) { + $u = new SearchUpdate( $s->cur_id, $s->cur_title, $s->cur_text ); + $u->doUpdate(); + if ( ( (++$n) % 500) == 0) { echo "$n\n"; } + } + wfFreeResult( $res ); +} + +?> diff --git a/maintenance/rebuildtextindex.php b/maintenance/rebuildtextindex.php new file mode 100644 index 0000000000..bbc5327d3b --- /dev/null +++ b/maintenance/rebuildtextindex.php @@ -0,0 +1,31 @@ + diff --git a/maintenance/tables.sql b/maintenance/tables.sql index 09556af74d..9ca9f5e334 100644 --- a/maintenance/tables.sql +++ b/maintenance/tables.sql @@ -165,12 +165,12 @@ CREATE TABLE watchlist ( DROP TABLE IF EXISTS math; CREATE TABLE math ( - math_inputhash varchar(16) NOT NULL, - math_outputhash varchar(16) NOT NULL, - math_html_conservativeness tinyint(1) NOT NULL, - math_html text, - math_mathml text, - UNIQUE KEY math_inputhash (math_inputhash) + math_inputhash varchar(16) NOT NULL, + math_outputhash varchar(16) NOT NULL, + math_html_conservativeness tinyint(1) NOT NULL, + math_html text, + math_mathml text, + UNIQUE KEY math_inputhash (math_inputhash) ) TYPE=MyISAM; -- Table searchindex must be MyISAM for fulltext support