findHooks.php, importSiteScripts.php: Use format=json

* Also added query continuation support to findHooks.php.
* Also fixed query continuation support in importSiteScripts.php
  (broken by 2b3f4d821cd4).

Change-Id: I7ef62d370f5e2f598ac4c5857ac0dbf3ee4c8fa2

diff --git a/maintenance/findHooks.php b/maintenance/findHooks.php
index 36760d7..66e8da0 100644 (file)
--- a/maintenance/findHooks.php
+++ b/maintenance/findHooks.php
@@ -163,36 +163,39 @@ class FindHooks extends Maintenance {
         * @return array Array of documented hooks
         */
        private function getHooksFromOnlineDoc() {
-               // All hooks
-               $allhookdata = Http::get(
-                       'http://www.mediawiki.org/w/api.php?action=query&list=categorymembers&'
-                       . 'cmtitle=Category:MediaWiki_hooks&cmlimit=500&format=php'
+               $allhooks = $this->getHooksFromOnlineDocCategory( 'MediaWiki_hooks' );
+               $removed = $this->getHooksFromOnlineDocCategory( 'Removed_hooks' );
+               return array_diff( $allhooks, $removed );
+       }
+
+       /**
+        * @param string $title
+        * @return array
+        */
+       private function getHooksFromOnlineDocCategory( $title ) {
+               $params = array(
+                       'action' => 'query',
+                       'list' => 'categorymembers',
+                       'cmtitle' => "Category:$title",
+                       'cmlimit' => 500,
+                       'format' => 'json',
+                       'continue' => '',
                );
-               $allhookdata = unserialize( $allhookdata );
-               $allhooks = array();
-               foreach ( $allhookdata['query']['categorymembers'] as $page ) {
-                       $found = preg_match( '/Manual\:Hooks\/([a-zA-Z0-9- :]+)/', $page['title'], $matches );
-                       if ( $found ) {
-                               $hook = str_replace( ' ', '_', $matches[1] );
-                               $allhooks[] = $hook;
+
+               $retval = array();
+               while ( true ) {
+                       $json = Http::get( wfAppendQuery( 'http://www.mediawiki.org/w/api.php', $params ) );
+                       $data = FormatJson::decode( $json, true );
+                       foreach ( $data['query']['categorymembers'] as $page ) {
+                               if ( preg_match( '/Manual\:Hooks\/([a-zA-Z0-9- :]+)/', $page['title'], $m ) ) {
+                                       $retval[] = str_replace( ' ', '_', $m[1] );
+                               }
                        }
-               }
-               // Removed hooks
-               $oldhookdata = Http::get(
-                       'http://www.mediawiki.org/w/api.php?action=query&list=categorymembers&'
-                       . 'cmtitle=Category:Removed_hooks&cmlimit=500&format=php'
-               );
-               $oldhookdata = unserialize( $oldhookdata );
-               $removed = array();
-               foreach ( $oldhookdata['query']['categorymembers'] as $page ) {
-                       $found = preg_match( '/Manual\:Hooks\/([a-zA-Z0-9- :]+)/', $page['title'], $matches );
-                       if ( $found ) {
-                               $hook = str_replace( ' ', '_', $matches[1] );
-                               $removed[] = $hook;
+                       if ( !isset( $data['continue'] ) ) {
+                               return $retval;
                        }
+                       $params = array_replace( $params, $data['continue'] );
                }
-
-               return array_diff( $allhooks, $removed );
        }
 
        /**

diff --git a/maintenance/importSiteScripts.php b/maintenance/importSiteScripts.php
index 7705ec9..e67d077 100644 (file)
--- a/maintenance/importSiteScripts.php
+++ b/maintenance/importSiteScripts.php
@@ -70,33 +70,40 @@ class ImportSiteScripts extends Maintenance {
        protected function fetchScriptList() {
                $data = array(
                        'action' => 'query',
-                       'format' => 'php', //'json',
+                       'format' => 'json',
                        'list' => 'allpages',
                        'apnamespace' => '8',
                        'aplimit' => '500',
+                       'continue' => '',
                );
                $baseUrl = $this->getArg( 0 );
                $pages = array();
 
-               do {
+               while ( true ) {
                        $url = wfAppendQuery( $baseUrl, $data );
                        $strResult = Http::get( $url );
-                       //$result = FormatJson::decode( $strResult ); // Still broken
-                       $result = unserialize( $strResult );
+                       $result = FormatJson::decode( $strResult, true );
 
-                       if ( !empty( $result['query']['allpages'] ) ) {
-                               foreach ( $result['query']['allpages'] as $page ) {
-                                       if ( substr( $page['title'], -3 ) === '.js' ) {
-                                               strtok( $page['title'], ':' );
-                                               $pages[] = strtok( '' );
-                                       }
+                       $page = null;
+                       foreach ( $result['query']['allpages'] as $page ) {
+                               if ( substr( $page['title'], -3 ) === '.js' ) {
+                                       strtok( $page['title'], ':' );
+                                       $pages[] = strtok( '' );
                                }
                        }
-                       if ( !empty( $result['query-continue'] ) ) {
-                               $data['apfrom'] = $result['query-continue']['allpages']['apfrom'];
-                               $this->output( "Fetching new batch from {$data['apfrom']}\n" );
+
+                       if ( $page !== null ) {
+                               $this->output( "Fetched list up to {$page['title']}\n" );
+                       }
+
+                       if ( isset( $result['continue'] ) ) { // >= 1.21
+                               $data = array_replace( $data, $result['continue'] );
+                       } elseif ( isset( $result['query-continue']['allpages'] ) ) { // <= 1.20
+                               $data = array_replace( $data, $result['query-continue']['allpages'] );
+                       } else {
+                               break;
                        }
-               } while ( isset( $result['query-continue'] ) );
+               }
 
                return $pages;
        }