From: Brion Vibber Date: Mon, 18 Jul 2005 22:15:05 +0000 (+0000) Subject: * Cache the top-level index list in memcached X-Git-Tag: 1.5.0beta4~97 X-Git-Url: http://git.cyclocoop.org/%22.%28%24lien.?a=commitdiff_plain;h=1a9a48502fff9785e818d633102749a5ee9b98cb;p=lhc%2Fweb%2Fwiklou.git * Cache the top-level index list in memcached * Removed inverse checkbox that looked index-unfriendly * Use relative offsets in toplevel index generation to avoid retracing all previous steps on each line * Bump up page size from 480 to 960 to make the toplevels more manageable --- diff --git a/RELEASE-NOTES b/RELEASE-NOTES index 1df50f608b..034eb72749 100644 --- a/RELEASE-NOTES +++ b/RELEASE-NOTES @@ -580,6 +580,9 @@ of MediaWiki:Newpagetext) to &action=edit, if page is new. * (bug 2866) Revert experimental, non-cross-platform sortable table hack * PHP 4.1.2 compatibility fix: define floatval() equivalent if missing * (bug 2901) Number format for Catalan +* Special:Allpages performance hacks: index memcached caching, removed + inverse checkbox, use friendlier relative offsets in index build + === Caveats === diff --git a/includes/SpecialAllpages.php b/includes/SpecialAllpages.php index 4c704e3741..079cb0e8c5 100644 --- a/includes/SpecialAllpages.php +++ b/includes/SpecialAllpages.php @@ -11,36 +11,28 @@ function wfSpecialAllpages( $par=NULL, $specialPage ) { global $indexMaxperpage, $toplevelMaxperpage, $wgRequest, $wgOut, $wgContLang; # Config - $indexMaxperpage = 480; + $indexMaxperpage = 960; $toplevelMaxperpage = 50; # GET values $from = $wgRequest->getVal( 'from' ); $namespace = $wgRequest->getInt( 'namespace' ); - $invert = $wgRequest->getBool( 'invert' ); $namespaces = $wgContLang->getNamespaces(); if( !in_array($namespace, array_keys($namespaces)) ) $namespace = 0; - if ($invert) { - $wgOut->setPagetitle( $namespace > 0 ? - wfMsg( 'allnotinnamespace', $namespaces[$namespace] ) : - wfMsg( 'allnonarticles' ) - ); - } else { - $wgOut->setPagetitle( $namespace > 0 ? - wfMsg( 'allinnamespace', $namespaces[$namespace] ) : - wfMsg( 'allarticles' ) - ); - } + $wgOut->setPagetitle( $namespace > 0 ? + wfMsg( 'allinnamespace', $namespaces[$namespace] ) : + wfMsg( 'allarticles' ) + ); if ( isset($par) ) { - indexShowChunk( $namespace, $par, $invert, $specialPage->including() ); + indexShowChunk( $namespace, $par, $specialPage->including() ); } elseif ( isset($from) ) { - indexShowChunk( $namespace, $from, $invert, $specialPage->including() ); + indexShowChunk( $namespace, $from, $specialPage->including() ); } else { - indexShowToplevel ( $namespace, $invert, $specialPage->including() ); + indexShowToplevel ( $namespace, $specialPage->including() ); } } @@ -48,9 +40,8 @@ function wfSpecialAllpages( $par=NULL, $specialPage ) { * HTML for the top form * @param integer $namespace A namespace constant (default NS_MAIN). * @param string $from Article name we are starting listing at. - * @param bool $invert true if we want the namespaces inverted (default false) */ -function indexNamespaceForm ( $namespace = NS_MAIN, $from = '', $invert = false ) { +function indexNamespaceForm ( $namespace = NS_MAIN, $from = '' ) { global $wgContLang, $wgScript; $t = Title::makeTitle( NS_SPECIAL, "Allpages" ); @@ -69,8 +60,6 @@ function indexNamespaceForm ( $namespace = NS_MAIN, $from = '', $invert = false . htmlspecialchars ( $from ) . '"/>'; $submitbutton = ''; - $invertbox = "'; - $out = "
"; $out .= ''; $out .= " @@ -82,8 +71,7 @@ function indexNamespaceForm ( $namespace = NS_MAIN, $from = '', $invert = false - $namespaceselect $submitbutton $invertbox - + $namespaceselect $submitbutton @@ -94,9 +82,8 @@ function indexNamespaceForm ( $namespace = NS_MAIN, $from = '', $invert = false /** * @param integer $namespace (default NS_MAIN) - * @param bool $invert true if we want the namespaces inverted (default false) */ -function indexShowToplevel ( $namespace = NS_MAIN, $invert = false, $including = false ) { +function indexShowToplevel ( $namespace = NS_MAIN, $including = false ) { global $wgOut, $indexMaxperpage, $toplevelMaxperpage, $wgContLang, $wgRequest, $wgUser; $sk = $wgUser->getSkin(); $fname = "indexShowToplevel"; @@ -106,55 +93,63 @@ function indexShowToplevel ( $namespace = NS_MAIN, $invert = false, $including $dbr =& wfGetDB( DB_SLAVE ); $page = $dbr->tableName( 'page' ); - $fromwhere = "FROM $page WHERE page_namespace" . - ($invert ? '!' : '') . "=$namespace"; + $fromwhere = "FROM $page WHERE page_namespace=$namespace"; $order_arr = array ( 'ORDER BY' => 'page_title' ); $order_str = 'ORDER BY page_title'; $out = ""; $where = array( 'page_namespace' => $namespace ); - $count = $dbr->selectField( 'page', 'COUNT(*)', $where, $fname ); - $sections = ceil( $count / $indexMaxperpage ); + global $wgMemc, $wgDBname; + $key = "$wgDBname:allpages:ns:$namespace"; + $lines = $wgMemc->get( $key ); - if ( $sections < 3 ) { - # If there are only two or less sections, don't even display them. - # Instead, display the first section directly. - indexShowChunk( $namespace, '', $invert, $including ); - return; - } - - # We want to display $toplevelMaxperpage lines starting at $offset. - # NOTICE: $offset starts at 0 - $offset = intval ( $wgRequest->getVal( 'offset' ) ); - if ( $offset < 0 ) { $offset = 0; } - if ( $offset >= $sections ) { $offset = $sections - 1; } - - # Where to stop? Notice that this can take the value of $sections, but $offset can't, because if - # we're displaying only the very last section, we still need two DB queries to find the titles - $stopat = ( $offset + $toplevelMaxperpage < $sections ) - ? $offset + $toplevelMaxperpage : $sections ; - - # This array is going to hold the page_titles in order. - $lines = array(); - - # If we are going to show n rows, we need n+1 queries to find the relevant titles. - for ( $i = $offset; $i <= $stopat; ++$i ) { - if ( $i == $sections ) # if we're displaying the last section, we need to - $from = $count-1; # find the last page_title in the DB - else if ( $i > $offset ) - $from = $i * $indexMaxperpage - 1; - else - $from = $i * $indexMaxperpage; - $limit = ( $i == $offset || $i == $stopat ) ? 1 : 2; - $sql = "SELECT page_title $fromwhere $order_str " . $dbr->limitResult ( $limit, $from ); - $res = $dbr->query( $sql, $fname ); - if ( $s = $dbr->fetchObject( $res ) ) { - array_push ( $lines, $s->page_title ); + if( !is_array( $lines ) ) { + $firstTitle = $dbr->selectField( 'page', 'page_title', $where, $fname, array( 'LIMIT' => 1 ) ); + $lastTitle = $firstTitle; + + # This array is going to hold the page_titles in order. + $lines = array( $firstTitle ); + + # If we are going to show n rows, we need n+1 queries to find the relevant titles. + $done = false; + for( $i = 0; !$done; ++$i ) { + // Fetch the last title of this chunk and the first of the next + $chunk = is_null( $lastTitle ) + ? '1=1' + : 'page_title >= ' . $dbr->addQuotes( $lastTitle ); + $sql = "SELECT page_title $fromwhere AND $chunk $order_str " . + $dbr->limitResult( 2, $indexMaxperpage - 1 ); + $res = $dbr->query( $sql, $fname ); if ( $s = $dbr->fetchObject( $res ) ) { - array_push ( $lines, $s->page_title ); + array_push( $lines, $s->page_title ); + } else { + // Final chunk, but ended prematurely. Go back and find the end. + $endTitle = $dbr->selectField( 'page', 'MAX(page_title)', + array( + 'page_namespace' => $namespace, + $chunk + ), $fname ); + array_push( $lines, $endTitle ); + $done = true; + } + if( $s = $dbr->fetchObject( $res ) ) { + array_push( $lines, $s->page_title ); + $lastTitle = $s->page_title; + } else { + // This was a final chunk and ended exactly at the limit. + // Rare but convenient! + $done = true; } + $dbr->freeResult( $res ); } - $dbr->freeResult( $res ); + $wgMemc->add( $key, $lines, 3600 ); + } + + // If there are only two or less sections, don't even display them. + // Instead, display the first section directly. + if( count( $lines ) <= 2 ) { + indexShowChunk( $namespace, '', false, $including ); + return; } # At this point, $lines should contain an even number of elements. @@ -162,33 +157,17 @@ function indexShowToplevel ( $namespace = NS_MAIN, $invert = false, $including while ( count ( $lines ) > 0 ) { $inpoint = array_shift ( $lines ); $outpoint = array_shift ( $lines ); - $out .= indexShowline ( $inpoint, $outpoint, $namespace, $invert ); + $out .= indexShowline ( $inpoint, $outpoint, $namespace, false ); } $out .= ''; - $nsForm = indexNamespaceForm ( $namespace, '', $invert ); + $nsForm = indexNamespaceForm ( $namespace, '', false ); # Is there more? if ( $including ) { $out2 = ''; } else { $morelinks = ''; - if ( $offset > 0 ) { - $morelinks = $sk->makeKnownLink ( - $wgContLang->specialPage ( 'Allpages' ), - wfMsg ( 'allpagesprev' ), - ( $offset > $toplevelMaxperpage ) ? 'offset='.($offset-$toplevelMaxperpage) : '' - ); - } - if ( $stopat < $sections-1 ) { - if ( $morelinks != '' ) { $morelinks .= " | "; } - $morelinks .= $sk->makeKnownLink ( - $wgContLang->specialPage ( 'Allpages' ), - wfMsg ( 'allpagesnext' ), - 'offset=' . ($offset + $toplevelMaxperpage) - ); - } - if ( $morelinks != '' ) { $out2 = ''; $out2 .= '
' . $nsForm; @@ -206,16 +185,15 @@ function indexShowToplevel ( $namespace = NS_MAIN, $invert = false, $including * @todo Document * @param string $from * @param integer $namespace (Default NS_MAIN) - * @param bool $invert true if we want the namespaces inverted (default false) */ -function indexShowline( $inpoint, $outpoint, $namespace = NS_MAIN, $invert ) { +function indexShowline( $inpoint, $outpoint, $namespace = NS_MAIN ) { global $wgOut, $wgLang, $wgUser; $sk = $wgUser->getSkin(); $dbr =& wfGetDB( DB_SLAVE ); $inpointf = htmlspecialchars( str_replace( '_', ' ', $inpoint ) ); $outpointf = htmlspecialchars( str_replace( '_', ' ', $outpoint ) ); - $queryparams = ($namespace ? "namespace=$namespace" : '') . ($invert ? "&invert=$invert" : ''); + $queryparams = ($namespace ? "namespace=$namespace" : ''); $special = Title::makeTitle( NS_SPECIAL, 'Allpages/' . $inpoint ); $link = $special->escapeLocalUrl( $queryparams ); @@ -230,9 +208,8 @@ function indexShowline( $inpoint, $outpoint, $namespace = NS_MAIN, $invert ) { /** * @param integer $namespace (Default NS_MAIN) * @param string $from list all pages from this name (default FALSE) - * @param bool $invert true if we want the namespaces inverted (default false) */ -function indexShowChunk( $namespace = NS_MAIN, $from, $invert = false, $including = false ) { +function indexShowChunk( $namespace = NS_MAIN, $from, $including = false ) { global $wgOut, $wgUser, $indexMaxperpage, $wgContLang; $sk = $wgUser->getSkin(); $maxPlusOne = $indexMaxperpage + 1; @@ -244,8 +221,8 @@ function indexShowChunk( $namespace = NS_MAIN, $from, $invert = false, $includin $fromTitle = Title::newFromURL( $from ); $fromKey = is_null( $fromTitle ) ? '' : $fromTitle->getDBkey(); - $sql = "SELECT page_namespace,page_title FROM $page WHERE page_namespace" . - ($invert ? '!' : '') . "=$namespace" . + $sql = "SELECT page_namespace,page_title FROM $page" . + " WHERE page_namespace=$namespace" . " AND page_title >= ". $dbr->addQuotes( $fromKey ) . " ORDER BY page_title LIMIT " . $maxPlusOne; $res = $dbr->query( $sql, 'indexShowChunk' ); @@ -259,10 +236,7 @@ function indexShowChunk( $namespace = NS_MAIN, $from, $invert = false, $includin while( ($n < $indexMaxperpage) && ($s = $dbr->fetchObject( $res )) ) { $t = Title::makeTitle( $s->page_namespace, $s->page_title ); if( $t ) { - $ns = $s->page_namespace; - $prefix = $invert ? $namespaces[$ns] : ''; - $prefix .= $invert && $namespaces[$ns] != $wgContLang->getNsText(NS_MAIN) ? ':' : ''; - $link = $sk->makeKnownLinkObj( $t, $t->getText(), false, false, $prefix ); + $link = $sk->makeKnownLinkObj( $t, htmlspecialchars( $t->getText() ), false, false ); } else { $link = '[[' . htmlspecialchars( $s->page_title ) . ']]'; } @@ -283,7 +257,7 @@ function indexShowChunk( $namespace = NS_MAIN, $from, $invert = false, $includin if ( $including ) { $out2 = ''; } else { - $nsForm = indexNamespaceForm ( $namespace, $from, $invert ); + $nsForm = indexNamespaceForm ( $namespace, $from ); $out2 = ''; $out2 .= '
' . $nsForm; $out2 .= '' . @@ -291,11 +265,10 @@ function indexShowChunk( $namespace = NS_MAIN, $from, $invert = false, $includin wfMsg ( 'allpages' ) ); if ( ($n == $indexMaxperpage) && ($s = $dbr->fetchObject( $res )) ) { $namespaceparam = $namespace ? "&namespace=$namespace" : ""; - $invertparam = $invert ? "&invert=$invert" : ''; $out2 .= " | " . $sk->makeKnownLink( $wgContLang->specialPage( "Allpages" ), wfMsg ( 'nextpage', $s->page_title ), - "from=" . wfUrlEncode ( $s->page_title ) . $namespaceparam . $invertparam ); + "from=" . wfUrlEncode ( $s->page_title ) . $namespaceparam ); } $out2 .= "
"; }