From afedf99ebe40f69090602612fc1c0933d21041c6 Mon Sep 17 00:00:00 2001 From: Roan Kattouw Date: Mon, 7 Jul 2008 17:32:22 +0000 Subject: [PATCH] API: Add paging (i.e. limit and continue) parameters to prop={links,templatelinks,langlinks,extlinks,categories,images}. This means that these modules will no longer request data from the database without a LIMIT, and that clients will have to use the query-continue method to get all the results. --- RELEASE-NOTES | 2 + includes/api/ApiQueryCategories.php | 43 +++++++++++++++++++- includes/api/ApiQueryExternalLinks.php | 34 ++++++++++++++++ includes/api/ApiQueryImages.php | 55 +++++++++++++++++++++++++- includes/api/ApiQueryLangLinks.php | 49 +++++++++++++++++++++-- includes/api/ApiQueryLinks.php | 52 +++++++++++++++++++++--- 6 files changed, 223 insertions(+), 12 deletions(-) diff --git a/RELEASE-NOTES b/RELEASE-NOTES index 1d43372134..1a2c5f0a58 100644 --- a/RELEASE-NOTES +++ b/RELEASE-NOTES @@ -507,6 +507,8 @@ it from source control: http://www.mediawiki.org/wiki/Download_from_SVN * Added APIQueryInfoTokens and APIQueryRevisionsTokens hooks so extensions can add their own tokens * Added block and unblock tokens to prop=info as well +* Added paging (limit and continue parameters) to + prop={links,templatelinks,langlinks,extlinks,categories,images} === Languages updated in 1.13 === diff --git a/includes/api/ApiQueryCategories.php b/includes/api/ApiQueryCategories.php index e45e0f68ab..5e1f983f5d 100644 --- a/includes/api/ApiQueryCategories.php +++ b/includes/api/ApiQueryCategories.php @@ -80,7 +80,22 @@ class ApiQueryCategories extends ApiQueryGeneratorBase { $this->addTables('categorylinks'); $this->addWhereFld('cl_from', array_keys($this->getPageSet()->getGoodTitles())); - $this->addOption('ORDER BY', "cl_from, cl_to"); + if(!is_null($params['continue'])) { + $cont = explode('|', $params['continue']); + if(count($cont) != 2) + $this->dieUsage("Invalid continue param. You should pass the " . + "original value returned by the previous query", "_badcontinue"); + $clfrom = intval($cont[0]); + $clto = $this->getDb()->strencode($cont[1]); + $this->addWhere("cl_from > $clfrom OR ". + "(cl_from = $clfrom AND ". + "cl_to >= '$clto')"); + } + # Don't order by cl_from if it's constant in the WHERE clause + if(count($this->getPageSet()->getGoodTitles()) == 1) + $this->addOption('ORDER BY', 'cl_to'); + else + $this->addOption('ORDER BY', "cl_from, cl_to"); $db = $this->getDB(); $res = $this->select(__METHOD__); @@ -89,7 +104,14 @@ class ApiQueryCategories extends ApiQueryGeneratorBase { $data = array(); $lastId = 0; // database has no ID 0 + $count = 0; while ($row = $db->fetchObject($res)) { + if (++$count > $params['limit']) { + // We've reached the one extra which shows that + // there are additional pages to be had. Stop here... + $this->setContinueEnumParameter('continue', "{$row->cl_from}|{$row->cl_to}"); + break; + } if ($lastId != $row->cl_from) { if($lastId != 0) { $this->addPageSubItems($lastId, $data); @@ -118,6 +140,13 @@ class ApiQueryCategories extends ApiQueryGeneratorBase { $titles = array(); while ($row = $db->fetchObject($res)) { + if (++$count > $params['limit']) { + // We've reached the one extra which shows that + // there are additional pages to be had. Stop here... + $this->setContinueEnumParameter('continue', "{$row->il_from}|{$row->il_to}"); + break; + } + $titles[] = Title :: makeTitle(NS_CATEGORY, $row->cl_to); } $resultPageSet->populateFromTitles($titles); @@ -134,13 +163,23 @@ class ApiQueryCategories extends ApiQueryGeneratorBase { 'sortkey', 'timestamp', ) - ) + ), + 'limit' => array( + ApiBase :: PARAM_DFLT => 10, + ApiBase :: PARAM_TYPE => 'limit', + ApiBase :: PARAM_MIN => 1, + ApiBase :: PARAM_MAX => ApiBase :: LIMIT_BIG1, + ApiBase :: PARAM_MAX2 => ApiBase :: LIMIT_BIG2 + ), + 'continue' => null, ); } public function getParamDescription() { return array ( 'prop' => 'Which additional properties to get for each category.', + 'limit' => 'How many langlinks to return', + 'continue' => 'When more results are available, use this to continue', ); } diff --git a/includes/api/ApiQueryExternalLinks.php b/includes/api/ApiQueryExternalLinks.php index 2c41313ed4..0757405540 100644 --- a/includes/api/ApiQueryExternalLinks.php +++ b/includes/api/ApiQueryExternalLinks.php @@ -43,6 +43,7 @@ class ApiQueryExternalLinks extends ApiQueryBase { if ( $this->getPageSet()->getGoodTitleCount() == 0 ) return; + $params = $this->extractRequestParams(); $this->addFields(array ( 'el_from', 'el_to' @@ -50,13 +51,26 @@ class ApiQueryExternalLinks extends ApiQueryBase { $this->addTables('externallinks'); $this->addWhereFld('el_from', array_keys($this->getPageSet()->getGoodTitles())); + # Don't order by el_from if it's constant in the WHERE clause + if(count($this->getPageSet()->getGoodTitles()) != 1) + $this->addOption('ORDER BY', 'el_from'); + $this->addOption('LIMIT', $params['limit'] + 1); + if(!is_null($params['offset'])) + $this->addOption('OFFSET', $params['offset']); $db = $this->getDB(); $res = $this->select(__METHOD__); $data = array(); $lastId = 0; // database has no ID 0 + $count = 0; while ($row = $db->fetchObject($res)) { + if (++$count > $params['limit']) { + // We've reached the one extra which shows that + // there are additional pages to be had. Stop here... + $this->setContinueEnumParameter('offset', @$params['offset'] + $params['limit']); + break; + } if ($lastId != $row->el_from) { if($lastId != 0) { $this->addPageSubItems($lastId, $data); @@ -77,6 +91,26 @@ class ApiQueryExternalLinks extends ApiQueryBase { $db->freeResult($res); } + public function getAllowedParams() { + return array( + 'limit' => array( + ApiBase :: PARAM_DFLT => 10, + ApiBase :: PARAM_TYPE => 'limit', + ApiBase :: PARAM_MIN => 1, + ApiBase :: PARAM_MAX => ApiBase :: LIMIT_BIG1, + ApiBase :: PARAM_MAX2 => ApiBase :: LIMIT_BIG2 + ), + 'offset' => null, + ); + } + + public function getParamDescription () { + return array( + 'limit' => 'How many links to return', + 'offset' => 'When more results are available, use this to continue', + ); + } + public function getDescription() { return 'Returns all external urls (not interwikies) from the given page(s)'; } diff --git a/includes/api/ApiQueryImages.php b/includes/api/ApiQueryImages.php index 74411f5c53..7aed6eb5ca 100644 --- a/includes/api/ApiQueryImages.php +++ b/includes/api/ApiQueryImages.php @@ -29,7 +29,7 @@ if (!defined('MEDIAWIKI')) { } /** - * This query adds subelement to all pages with the list of images embedded into those pages. + * This query adds an subelement to all pages with the list of images embedded into those pages. * * @ingroup API */ @@ -52,6 +52,7 @@ class ApiQueryImages extends ApiQueryGeneratorBase { if ($this->getPageSet()->getGoodTitleCount() == 0) return; // nothing to do + $params = $this->extractRequestParams(); $this->addFields(array ( 'il_from', 'il_to' @@ -59,7 +60,23 @@ class ApiQueryImages extends ApiQueryGeneratorBase { $this->addTables('imagelinks'); $this->addWhereFld('il_from', array_keys($this->getPageSet()->getGoodTitles())); - $this->addOption('ORDER BY', "il_from, il_to"); + if(!is_null($params['continue'])) { + $cont = explode('|', $params['continue']); + if(count($cont) != 2) + $this->dieUsage("Invalid continue param. You should pass the " . + "original value returned by the previous query", "_badcontinue"); + $ilfrom = intval($cont[0]); + $ilto = $this->getDb()->strencode($cont[1]); + $this->addWhere("il_from > $ilfrom OR ". + "(il_from = $ilfrom AND ". + "il_to >= '$ilto')"); + } + # Don't order by il_from if it's constant in the WHERE clause + if(count($this->getPageSet()->getGoodTitles()) == 1) + $this->addOption('ORDER BY', 'il_to'); + else + $this->addOption('ORDER BY', 'il_from, il_to'); + $this->addOption('LIMIT', $params['limit'] + 1); $db = $this->getDB(); $res = $this->select(__METHOD__); @@ -68,7 +85,14 @@ class ApiQueryImages extends ApiQueryGeneratorBase { $data = array(); $lastId = 0; // database has no ID 0 + $count = 0; while ($row = $db->fetchObject($res)) { + if (++$count > $params['limit']) { + // We've reached the one extra which shows that + // there are additional pages to be had. Stop here... + $this->setContinueEnumParameter('continue', "{$row->il_from}|{$row->il_to}"); + break; + } if ($lastId != $row->il_from) { if($lastId != 0) { $this->addPageSubItems($lastId, $data); @@ -89,7 +113,14 @@ class ApiQueryImages extends ApiQueryGeneratorBase { } else { $titles = array(); + $count = 0; while ($row = $db->fetchObject($res)) { + if (++$count > $params['limit']) { + // We've reached the one extra which shows that + // there are additional pages to be had. Stop here... + $this->setContinueEnumParameter('continue', "{$row->il_from}|{$row->il_to}"); + break; + } $titles[] = Title :: makeTitle(NS_IMAGE, $row->il_to); } $resultPageSet->populateFromTitles($titles); @@ -98,6 +129,26 @@ class ApiQueryImages extends ApiQueryGeneratorBase { $db->freeResult($res); } + public function getAllowedParams() { + return array( + 'limit' => array( + ApiBase :: PARAM_DFLT => 10, + ApiBase :: PARAM_TYPE => 'limit', + ApiBase :: PARAM_MIN => 1, + ApiBase :: PARAM_MAX => ApiBase :: LIMIT_BIG1, + ApiBase :: PARAM_MAX2 => ApiBase :: LIMIT_BIG2 + ), + 'continue' => null, + ); + } + + public function getParamDescription () { + return array( + 'limit' => 'How many images to return', + 'continue' => 'When more results are available, use this to continue', + ); + } + public function getDescription() { return 'Returns all images contained on the given page(s)'; } diff --git a/includes/api/ApiQueryLangLinks.php b/includes/api/ApiQueryLangLinks.php index 1f0ab813d5..735f67c0c6 100644 --- a/includes/api/ApiQueryLangLinks.php +++ b/includes/api/ApiQueryLangLinks.php @@ -43,6 +43,7 @@ class ApiQueryLangLinks extends ApiQueryBase { if ( $this->getPageSet()->getGoodTitleCount() == 0 ) return; + $params = $this->extractRequestParams(); $this->addFields(array ( 'll_from', 'll_lang', @@ -51,14 +52,36 @@ class ApiQueryLangLinks extends ApiQueryBase { $this->addTables('langlinks'); $this->addWhereFld('ll_from', array_keys($this->getPageSet()->getGoodTitles())); - $this->addOption('ORDER BY', "ll_from, ll_lang"); + if(!is_null($params['continue'])) { + $cont = explode('|', $params['continue']); + if(count($cont) != 2) + $this->dieUsage("Invalid continue param. You should pass the " . + "original value returned by the previous query", "_badcontinue"); + $llfrom = intval($cont[0]); + $lllang = $this->getDb()->strencode($cont[1]); + $this->addWhere("ll_from > $llfrom OR ". + "(ll_from = $llfrom AND ". + "ll_lang >= '$lllang')"); + } + # Don't order by ll_from if it's constant in the WHERE clause + if(count($this->getPageSet()->getGoodTitles()) == 1) + $this->addOption('ORDER BY', 'll_lang'); + else + $this->addOption('ORDER BY', 'll_from, ll_lang'); + $this->addOption('LIMIT', $params['limit'] + 1); $res = $this->select(__METHOD__); $data = array(); $lastId = 0; // database has no ID 0 + $count = 0; $db = $this->getDB(); while ($row = $db->fetchObject($res)) { - + if (++$count > $params['limit']) { + // We've reached the one extra which shows that + // there are additional pages to be had. Stop here... + $this->setContinueEnumParameter('continue', "{$row->ll_from}|{$row->ll_lang}"); + break; + } if ($lastId != $row->ll_from) { if($lastId != 0) { $this->addPageSubItems($lastId, $data); @@ -67,7 +90,7 @@ class ApiQueryLangLinks extends ApiQueryBase { $lastId = $row->ll_from; } - $entry = array('lang'=>$row->ll_lang); + $entry = array('lang' => $row->ll_lang); ApiResult :: setContent($entry, $row->ll_title); $data[] = $entry; } @@ -79,6 +102,26 @@ class ApiQueryLangLinks extends ApiQueryBase { $db->freeResult($res); } + public function getAllowedParams() { + return array( + 'limit' => array( + ApiBase :: PARAM_DFLT => 10, + ApiBase :: PARAM_TYPE => 'limit', + ApiBase :: PARAM_MIN => 1, + ApiBase :: PARAM_MAX => ApiBase :: LIMIT_BIG1, + ApiBase :: PARAM_MAX2 => ApiBase :: LIMIT_BIG2 + ), + 'continue' => null, + ); + } + + public function getParamDescription () { + return array( + 'limit' => 'How many langlinks to return', + 'continue' => 'When more results are available, use this to continue', + ); + } + public function getDescription() { return 'Returns all interlanguage links from the given page(s)'; } diff --git a/includes/api/ApiQueryLinks.php b/includes/api/ApiQueryLinks.php index 73a2ad8a79..0761d913b6 100644 --- a/includes/api/ApiQueryLinks.php +++ b/includes/api/ApiQueryLinks.php @@ -84,7 +84,22 @@ class ApiQueryLinks extends ApiQueryGeneratorBase { $this->addTables($this->table); $this->addWhereFld($this->prefix . '_from', array_keys($this->getPageSet()->getGoodTitles())); $this->addWhereFld($this->prefix . '_namespace', $params['namespace']); - + + if(!is_null($params['continue'])) { + $cont = explode('|', $params['continue']); + if(count($cont) != 3) + $this->dieUsage("Invalid continue param. You should pass the " . + "original value returned by the previous query", "_badcontinue"); + $plfrom = intval($cont[0]); + $plns = intval($cont[1]); + $pltitle = $this->getDb()->strencode($cont[2]); + $this->addWhere("{$this->prefix}_from > $plfrom OR ". + "({$this->prefix}_from = $plfrom AND ". + "({$this->prefix}_namespace > $plns OR ". + "({$this->prefix}_namespace = $plns AND ". + "{$this->prefix}_title >= '$pltitle')))"); + } + # Here's some MySQL craziness going on: if you use WHERE foo='bar' # and later ORDER BY foo MySQL doesn't notice the ORDER BY is pointless # but instead goes and filesorts, because the index for foo was used @@ -93,11 +108,12 @@ class ApiQueryLinks extends ApiQueryGeneratorBase { $order = array(); if(count($this->getPageSet()->getGoodTitles()) != 1) $order[] = "{$this->prefix}_from"; - if(!isset($params['namespace'])) + if(count($params['namespace']) != 1) $order[] = "{$this->prefix}_namespace"; $order[] = "{$this->prefix}_title"; $this->addOption('ORDER BY', implode(", ", $order)); $this->addOption('USE INDEX', "{$this->prefix}_from"); + $this->addOption('LIMIT', $params['limit'] + 1); $db = $this->getDB(); $res = $this->select(__METHOD__); @@ -106,7 +122,15 @@ class ApiQueryLinks extends ApiQueryGeneratorBase { $data = array(); $lastId = 0; // database has no ID 0 + $count = 0; while ($row = $db->fetchObject($res)) { + if(++$count > $params['limit']) { + // We've reached the one extra which shows that + // there are additional pages to be had. Stop here... + $this->setContinueEnumParameter('continue', + "{$row->pl_from}|{$row->pl_namespace}|{$row->pl_title}"); + break; + } if ($lastId != $row->pl_from) { if($lastId != 0) { $this->addPageSubItems($lastId, $data); @@ -127,7 +151,15 @@ class ApiQueryLinks extends ApiQueryGeneratorBase { } else { $titles = array(); + $count = 0; while ($row = $db->fetchObject($res)) { + if(++$count > $params['limit']) { + // We've reached the one extra which shows that + // there are additional pages to be had. Stop here... + $this->setContinueEnumParameter('continue', + "{$row->pl_from}|{$row->pl_namespace}|{$row->pl_title}"); + break; + } $titles[] = Title :: makeTitle($row->pl_namespace, $row->pl_title); } $resultPageSet->populateFromTitles($titles); @@ -142,15 +174,25 @@ class ApiQueryLinks extends ApiQueryGeneratorBase { 'namespace' => array( ApiBase :: PARAM_TYPE => 'namespace', ApiBase :: PARAM_ISMULTI => true - ) + ), + 'limit' => array( + ApiBase :: PARAM_DFLT => 10, + ApiBase :: PARAM_TYPE => 'limit', + ApiBase :: PARAM_MIN => 1, + ApiBase :: PARAM_MAX => ApiBase :: LIMIT_BIG1, + ApiBase :: PARAM_MAX2 => ApiBase :: LIMIT_BIG2 + ), + 'continue' => null, ); } public function getParamDescription() { return array( - 'namespace' => "Show {$this->description}s in this namespace(s) only" - ); + 'namespace' => "Show {$this->description}s in this namespace(s) only", + 'limit' => 'How many links to return', + 'continue' => 'When more results are available, use this to continue', + ); } public function getDescription() { -- 2.20.1