From b7f60b9ab81224e4b52551d979d7db7bdb124517 Mon Sep 17 00:00:00 2001 From: Brad Jorsch Date: Tue, 20 Sep 2016 17:26:48 -0400 Subject: [PATCH] API: Add generator non-continuation data Some generators (e.g. generator=recentchanges) in their default mode of operation are particularly prone to not generating the same result set on subsequent requests due to intervening activity on the wiki adding new entries to the start of the list. We can mitigate this effect by allowing such generators to provide "non-continuation" data to be used if the generator isn't being continued. ApiQueryRecentChanges and ApiQueryAllRevisions are updated to set this new property. Other generators can easily be updated in the same way as needed. There isn't anything we can do about generators prone to having entries added at random positions in the list rather than the beginning, unfortunately. Bug: T146176 Change-Id: I8308d6aa2c89fd2a85e74c7dd8a0a2a9ec927490 --- includes/api/ApiContinuationManager.php | 39 +++++++++++++++++++++++-- includes/api/ApiQuery.php | 5 ++++ includes/api/ApiQueryAllRevisions.php | 7 +++++ includes/api/ApiQueryRecentChanges.php | 7 +++++ 4 files changed, 56 insertions(+), 2 deletions(-) diff --git a/includes/api/ApiContinuationManager.php b/includes/api/ApiContinuationManager.php index 6601fb7463..19e2453944 100644 --- a/includes/api/ApiContinuationManager.php +++ b/includes/api/ApiContinuationManager.php @@ -31,6 +31,7 @@ class ApiContinuationManager { private $continuationData = []; private $generatorContinuationData = []; + private $generatorNonContinuationData = []; private $generatorParams = []; private $generatorDone = false; @@ -142,6 +143,26 @@ class ApiContinuationManager { $this->continuationData[$name][$paramName] = $paramValue; } + /** + * Set the non-continuation parameter for the generator module + * + * In case the generator isn't going to be continued, this sets the fields + * to return. + * + * @since 1.28 + * @param ApiBase $module + * @param string $paramName + * @param string|array $paramValue + */ + public function addGeneratorNonContinueParam( ApiBase $module, $paramName, $paramValue ) { + $name = $module->getModuleName(); + $paramName = $module->encodeParamName( $paramName ); + if ( is_array( $paramValue ) ) { + $paramValue = implode( '|', $paramValue ); + } + $this->generatorNonContinuationData[$name][$paramName] = $paramValue; + } + /** * Set the continuation parameter for the generator module * @param ApiBase $module @@ -165,6 +186,15 @@ class ApiContinuationManager { return array_merge_recursive( $this->continuationData, $this->generatorContinuationData ); } + /** + * Fetch raw non-continuation data + * @since 1.28 + * @return array + */ + public function getRawNonContinuation() { + return $this->generatorNonContinuationData; + } + /** * Fetch continuation result data * @return array [ (array)$data, (bool)$batchcomplete ] @@ -192,8 +222,13 @@ class ApiContinuationManager { foreach ( $continuationData as $module => $kvp ) { $data += $kvp; } - $data += $this->generatorParams; - $generatorKeys = implode( '|', array_keys( $this->generatorParams ) ); + $generatorParams = []; + foreach ( $this->generatorNonContinuationData as $kvp ) { + $generatorParams += $kvp; + } + $generatorParams += $this->generatorParams; + $data += $generatorParams; + $generatorKeys = implode( '|', array_keys( $generatorParams ) ); } elseif ( $this->generatorContinuationData ) { // All the generator-using modules are complete, but the // generator isn't. Continue the generator and restart the diff --git a/includes/api/ApiQuery.php b/includes/api/ApiQuery.php index 5e3c7090cd..e8aa655aba 100644 --- a/includes/api/ApiQuery.php +++ b/includes/api/ApiQuery.php @@ -258,6 +258,11 @@ class ApiQuery extends ApiBase { // Write the continuation data into the result $this->setContinuationManager( null ); if ( $this->mParams['rawcontinue'] ) { + $data = $continuationManager->getRawNonContinuation(); + if ( $data ) { + $this->getResult()->addValue( null, 'query-noncontinue', $data, + ApiResult::ADD_ON_TOP | ApiResult::NO_SIZE_CHECK ); + } $data = $continuationManager->getRawContinuation(); if ( $data ) { $this->getResult()->addValue( null, 'query-continue', $data, diff --git a/includes/api/ApiQueryAllRevisions.php b/includes/api/ApiQueryAllRevisions.php index 060e3e10dc..d548c46ce0 100644 --- a/includes/api/ApiQueryAllRevisions.php +++ b/includes/api/ApiQueryAllRevisions.php @@ -172,6 +172,13 @@ class ApiQueryAllRevisions extends ApiQueryRevisionsBase { $nextIndex = 0; $generated = []; foreach ( $res as $row ) { + if ( $count === 0 && $resultPageSet !== null ) { + // Set the non-continue since the list of all revisions is + // prone to having entries added at the start frequently. + $this->getContinuationManager()->addGeneratorNonContinueParam( + $this, 'continue', "$row->rev_timestamp|$row->rev_id" + ); + } if ( ++$count > $this->limit ) { // We've had enough $this->setContinueEnumParameter( 'continue', "$row->rev_timestamp|$row->rev_id" ); diff --git a/includes/api/ApiQueryRecentChanges.php b/includes/api/ApiQueryRecentChanges.php index cc3ca60f3d..c4c8afbdbf 100644 --- a/includes/api/ApiQueryRecentChanges.php +++ b/includes/api/ApiQueryRecentChanges.php @@ -372,6 +372,13 @@ class ApiQueryRecentChanges extends ApiQueryGeneratorBase { /* Iterate through the rows, adding data extracted from them to our query result. */ foreach ( $res as $row ) { + if ( $count === 0 && $resultPageSet !== null ) { + // Set the non-continue since the list of recentchanges is + // prone to having entries added at the start frequently. + $this->getContinuationManager()->addGeneratorNonContinueParam( + $this, 'continue', "$row->rc_timestamp|$row->rc_id" + ); + } if ( ++$count > $params['limit'] ) { // We've reached the one extra which shows that there are // additional pages to be had. Stop here... -- 2.20.1