From f5c874cbaf2c1408342305fbc87dc6db36728e07 Mon Sep 17 00:00:00 2001 From: "C. Scott Ananian" Date: Wed, 27 May 2015 17:46:45 -0400 Subject: [PATCH] Use Parsoid v3 API; emulate the RESTBase v1 API Update the ParsoidVirtualRESTService and the RestbaseVirtualRESTService to use Parsoid's v3 API, instead of the deprecated v1/v2 APIs. Since Visual Editor still issues requests using the Parsoid v1 API, convert Parsoid v1 API requests into Parsoid v3 API requests when needed for a smooth transition. We also add support for converting RESTBase v1 API requests to Parsoid v3 API requests. The next step will be to convert Visual Editor to issue RESTBase v1 API requests (https://gerrit.wikimedia.org/r/217995), and then the Parsoid v1 conversion code added here can be removed (T100681). Tested Parsoid v1->v3 conversion, Parsoid v1->RESTBase conversion, plus Parsoid v3 and RESTBase v1->Parsoid v3 conversion using VE patched to issue RESTBase v1 API requests. Bug: T100681 Change-Id: I07ac60cdec7a52ef93187d40099325a069e3239a --- includes/DefaultSettings.php | 2 + includes/Setup.php | 2 + .../virtualrest/ParsoidVirtualRESTService.php | 220 +++++++++++----- .../RestbaseVirtualRESTService.php | 241 ++++++++++++------ 4 files changed, 329 insertions(+), 136 deletions(-) diff --git a/includes/DefaultSettings.php b/includes/DefaultSettings.php index bcf7e14c1f..c0aad5df04 100644 --- a/includes/DefaultSettings.php +++ b/includes/DefaultSettings.php @@ -7693,6 +7693,7 @@ $wgUseLinkNamespaceDBFields = true; * $wgVirtualRestConfig['modules']['parsoid'] = array( * 'url' => 'http://localhost:8000', * 'prefix' => 'enwiki', + * 'domain' => 'en.wikipedia.org', * ); * * @var array @@ -7703,6 +7704,7 @@ $wgVirtualRestConfig = array( 'global' => array( # Timeout in seconds 'timeout' => 360, + # 'domain' is set to $wgCanonicalServer in Setup.php 'forwardCookies' => false, 'HTTPProxy' => null ) diff --git a/includes/Setup.php b/includes/Setup.php index 4d7428a2dd..86df4b8667 100644 --- a/includes/Setup.php +++ b/includes/Setup.php @@ -535,6 +535,8 @@ if ( $wgSecureLogin && substr( $wgServer, 0, 2 ) !== '//' ) { . 'HTTP or HTTPS. Disabling secure login.' ); } +$wgVirtualRestConfig['global']['domain'] = $wgCanonicalServer; + // Now that GlobalFunctions is loaded, set defaults that depend on it. if ( $wgTmpDirectory === false ) { $wgTmpDirectory = wfTempDir(); diff --git a/includes/libs/virtualrest/ParsoidVirtualRESTService.php b/includes/libs/virtualrest/ParsoidVirtualRESTService.php index 32a27f7956..4c96e297f5 100644 --- a/includes/libs/virtualrest/ParsoidVirtualRESTService.php +++ b/includes/libs/virtualrest/ParsoidVirtualRESTService.php @@ -24,21 +24,27 @@ */ class ParsoidVirtualRESTService extends VirtualRESTService { /** - * Example requests: - * GET /local/v1/page/$title/html/$oldid - * * $oldid is optional - * POST /local/v1/transform/html/to/wikitext/$title/$oldid + * Example Parsoid v3 requests: + * GET /local/v3/page/html/$title/{$revision} + * * $revision is optional + * POST /local/v3/transform/html/to/wikitext/{$title}{/$revision} * * body: array( 'html' => ... ) - * * $title and $oldid are optional - * POST /local/v1/transform/wikitext/to/html/$title - * * body: array( 'wikitext' => ... ) or array( 'wikitext' => ..., 'body' => true/false ) + * * $title and $revision are optional + * POST /local/v3/transform/wikitext/to/html/{$title}{/$revision} + * * body: array( 'wikitext' => ... ) or array( 'wikitext' => ..., 'bodyOnly' => true/false ) * * $title is optional + * * $revision is optional + * + * There are also deprecated "v1" requests; see onParsoid1Request + * for details. * @param array $params Key/value map * - url : Parsoid server URL - * - prefix : Parsoid prefix for this wiki + * - domain : Wiki domain to use * - timeout : Parsoid timeout (optional) * - forwardCookies : Cookies to forward to Parsoid, or false. (optional) * - HTTPProxy : Parsoid HTTP proxy (optional) + * - restbaseCompat : whether to parse URL as if they were meant for RESTBase + * boolean (optional) */ public function __construct( array $params ) { // for backwards compatibility: @@ -46,7 +52,29 @@ class ParsoidVirtualRESTService extends VirtualRESTService { $params['url'] = $params['URL']; unset( $params['URL'] ); } - parent::__construct( $params ); + // set up defaults and merge them with the given params + $mparams = array_merge( array( + 'url' => 'http://localhost:8000/', + 'prefix' => 'localhost', + 'domain' => 'localhost', + 'forwardCookies' => false, + 'HTTPProxy' => null, + ), $params ); + // Ensure that the url parameter has a trailing slash. + $mparams['url'] = preg_replace( + '#/?$#', + '/', + $mparams['url'] + ); + // Ensure the correct domain format: strip protocol, port, + // and trailing slash if present. This lets us use + // $wgCanonicalServer as a default value, which is very convenient. + $mparams['domain'] = preg_replace( + '/^(https?:\/\/)?([^\/:]+?)(:\d+)?\/?$/', + '$2', + $mparams['domain'] + ); + parent::__construct( $mparams ); } public function onRequests( array $reqs, Closure $idGeneratorFunc ) { @@ -56,71 +84,143 @@ class ParsoidVirtualRESTService extends VirtualRESTService { list( $targetWiki, // 'local' - $version, // 'v1' - $reqType // 'page' or 'transform' + $version, // 'v3' ('v1' for restbase compatibility) + $reqType, // 'page' or 'transform' + $format, // 'html' or 'wikitext' + // $title (optional) + // $revision (optional) ) = $parts; + if ( $this->params['restbaseCompat'] ) { + if ( $version !== 'v1' ) { + throw new Exception( "Only RESTBase v1 API is supported." ); + } + # Map RESTBase v1 API to Parsoid v3 API (pretty easy) + $req['url'] = preg_replace( '#^local/v1/#', 'local/v3/', $req['url'] ); + } elseif ( $version !== 'v3' ) { + $result[$key] = $this->onParsoid1Request( $req, $idGeneratorFunc ); + continue; + } if ( $targetWiki !== 'local' ) { + throw new Exception( "Only 'local' target wiki is currently supported" ); - } elseif ( $version !== 'v1' ) { - throw new Exception( "Only version 1 exists" ); - } elseif ( $reqType !== 'page' && $reqType !== 'transform' ) { - throw new Exception( "Request type must be either 'page' or 'transform'" ); } - - $req['url'] = $this->params['url'] . '/' . urlencode( $this->params['prefix'] ) . '/'; - - if ( $reqType === 'page' ) { - $title = $parts[3]; - if ( $parts[4] !== 'html' ) { - throw new Exception( "Only 'html' output format is currently supported" ); - } - if ( isset( $parts[5] ) ) { - $req['url'] .= $title . '?oldid=' . $parts[5]; - } else { - $req['url'] .= $title; - } - } elseif ( $reqType === 'transform' ) { - if ( $parts[4] !== 'to' ) { - throw new Exception( "Part index 4 is not 'to'" ); - } - - if ( isset( $parts[6] ) ) { - $req['url'] .= $parts[6]; - } - - if ( $parts[3] === 'html' & $parts[5] === 'wikitext' ) { - if ( !isset( $req['body']['html'] ) ) { - throw new Exception( "You must set an 'html' body key for this request" ); - } - if ( isset( $parts[7] ) ) { - $req['body']['oldid'] = $parts[7]; - } - } elseif ( $parts[3] == 'wikitext' && $parts[5] == 'html' ) { - if ( !isset( $req['body']['wikitext'] ) ) { - throw new Exception( "You must set a 'wikitext' body key for this request" ); - } - $req['body']['wt'] = $req['body']['wikitext']; - unset( $req['body']['wikitext'] ); - } else { - throw new Exception( "Transformation unsupported" ); - } + if ( $reqType !== 'page' && $reqType !== 'transform' ) { + throw new Exception( "Request action must be either 'page' or 'transform'" ); } - - if ( isset( $this->params['HTTPProxy'] ) && $this->params['HTTPProxy'] ) { + if ( $format !== 'html' && $format !== 'wikitext' ) { + throw new Exception( "Request format must be either 'html' or 'wt'" ); + } + // replace /local/ with the current domain + $req['url'] = preg_replace( '#^local/#', $this->params['domain'] . '/', $req['url'] ); + // and prefix it with the service URL + $req['url'] = $this->params['url'] . $req['url']; + // set the appropriate proxy, timeout and headers + if ( $this->params['HTTPProxy'] ) { $req['proxy'] = $this->params['HTTPProxy']; } - if ( isset( $this->params['timeout'] ) ) { + if ( $this->params['timeout'] != null ) { $req['reqTimeout'] = $this->params['timeout']; } - - // Forward cookies - if ( isset( $this->params['forwardCookies'] ) ) { + if ( $this->params['forwardCookies'] ) { $req['headers']['Cookie'] = $this->params['forwardCookies']; } - $result[$key] = $req; } return $result; } + + /** + * Remap a Parsoid v1 request to a Parsoid v3 request. + * + * Example Parsoid v1 requests: + * GET /local/v1/page/$title/html/$oldid + * * $oldid is optional + * POST /local/v1/transform/html/to/wikitext/$title/$oldid + * * body: array( 'html' => ... ) + * * $title and $oldid are optional + * POST /local/v1/transform/wikitext/to/html/$title + * * body: array( 'wikitext' => ... ) or array( 'wikitext' => ..., 'body' => true/false ) + * * $title is optional + * + * NOTE: the POST APIs aren't "real" Parsoid v1 APIs, they are just what + * Visual Editor "pretends" the V1 API is like. A previous version of + * ParsoidVirtualRESTService translated these to the "real" Parsoid v1 + * API. We now translate these to the "real" Parsoid v3 API. + */ + public function onParsoid1Request( array $req, Closure $idGeneratorFunc ) { + + $parts = explode( '/', $req['url'] ); + list( + $targetWiki, // 'local' + $version, // 'v1' + $reqType // 'page' or 'transform' + ) = $parts; + if ( $targetWiki !== 'local' ) { + throw new Exception( "Only 'local' target wiki is currently supported" ); + } elseif ( $version !== 'v1' ) { + throw new Exception( "Only v1 and v3 are supported." ); + } elseif ( $reqType !== 'page' && $reqType !== 'transform' ) { + throw new Exception( "Request type must be either 'page' or 'transform'" ); + } + $req['url'] = $this->params['url'] . $this->params['domain'] . '/v3/'; + if ( $reqType === 'page' ) { + $title = $parts[3]; + if ( $parts[4] !== 'html' ) { + throw new Exception( "Only 'html' output format is currently supported" ); + } + $req['url'] .= 'page/html/' . $title; + if ( isset( $parts[5] ) ) { + $req['url'] .= '/' . $parts[5]; + } elseif ( isset( $req['query']['oldid'] ) && $req['query']['oldid'] ) { + $req['url'] .= '/' . $req['query']['oldid']; + unset( $req['query']['oldid'] ); + } + } elseif ( $reqType === 'transform' ) { + $req['url'] .= 'transform/'. $parts[3] . '/to/' . $parts[5]; + // the title + if ( isset( $parts[6] ) ) { + $req['url'] .= '/' . $parts[6]; + } + // revision id + if ( isset( $parts[7] ) ) { + $req['url'] .= '/' . $parts[7]; + } elseif ( isset( $req['body']['oldid'] ) && $req['body']['oldid'] ) { + $req['url'] .= '/' . $req['body']['oldid']; + unset( $req['body']['oldid'] ); + } + if ( $parts[4] !== 'to' ) { + throw new Exception( "Part index 4 is not 'to'" ); + } + if ( $parts[3] === 'html' && $parts[5] === 'wikitext' ) { + if ( !isset( $req['body']['html'] ) ) { + throw new Exception( "You must set an 'html' body key for this request" ); + } + } elseif ( $parts[3] == 'wikitext' && $parts[5] == 'html' ) { + if ( !isset( $req['body']['wikitext'] ) ) { + throw new Exception( "You must set a 'wikitext' body key for this request" ); + } + if ( isset( $req['body']['body'] ) ) { + $req['body']['bodyOnly'] = $req['body']['body']; + unset( $req['body']['body'] ); + } + } else { + throw new Exception( "Transformation unsupported" ); + } + } + // set the appropriate proxy, timeout and headers + if ( $this->params['HTTPProxy'] ) { + $req['proxy'] = $this->params['HTTPProxy']; + } + if ( $this->params['timeout'] != null ) { + $req['reqTimeout'] = $this->params['timeout']; + } + if ( $this->params['forwardCookies'] ) { + $req['headers']['Cookie'] = $this->params['forwardCookies']; + } + + return $req; + + } + } diff --git a/includes/libs/virtualrest/RestbaseVirtualRESTService.php b/includes/libs/virtualrest/RestbaseVirtualRESTService.php index 8fe5b9214b..bc520aa106 100644 --- a/includes/libs/virtualrest/RestbaseVirtualRESTService.php +++ b/includes/libs/virtualrest/RestbaseVirtualRESTService.php @@ -1,6 +1,6 @@ ... ) * POST /local/v1/transform/wikitext/to/html{/title}{/revision} * * body: array( 'wikitext' => ... ) or array( 'wikitext' => ..., 'bodyOnly' => true/false ) * * @param array $params Key/value map - * - url : Restbase server URL + * - url : RESTBase server URL * - domain : Wiki domain to use * - timeout : request timeout in seconds (optional) - * - forwardCookies : cookies to forward to Restbase/Parsoid (as a Cookie + * - forwardCookies : cookies to forward to RESTBase/Parsoid (as a Cookie * header string) or false (optional) * Note: forwardCookies will in the future be a boolean * only, signifing request cookies should be forwarded @@ -48,18 +48,26 @@ class RestbaseVirtualRESTService extends VirtualRESTService { public function __construct( array $params ) { // set up defaults and merge them with the given params $mparams = array_merge( array( - 'url' => 'http://localhost:7231', + 'url' => 'http://localhost:7231/', 'domain' => 'localhost', 'timeout' => 100, 'forwardCookies' => false, 'HTTPProxy' => null, 'parsoidCompat' => false ), $params ); - // ensure the correct domain format + // Ensure that the url parameter has a trailing slash. + $mparams['url'] = preg_replace( + '#/?$#', + '/', + $mparams['url'] + ); + // Ensure the correct domain format: strip protocol, port, + // and trailing slash if present. This lets us use + // $wgCanonicalServer as a default value, which is very convenient. $mparams['domain'] = preg_replace( - '/^(https?:\/\/)?([^\/:]+?)(\/|:\d+\/?)?$/', - '$2', - $mparams['domain'] + '/^(https?:\/\/)?([^\/:]+?)(:\d+)?\/?$/', + '$2', + $mparams['domain'] ); parent::__construct( $mparams ); } @@ -73,7 +81,7 @@ class RestbaseVirtualRESTService extends VirtualRESTService { $result = array(); foreach ( $reqs as $key => $req ) { // replace /local/ with the current domain - $req['url'] = preg_replace( '/^\/local\//', '/' . $this->params['domain'] . '/', $req['url'] ); + $req['url'] = preg_replace( '#^local/#', $this->params['domain'] . '/', $req['url'] ); // and prefix it with the service URL $req['url'] = $this->params['url'] . $req['url']; // set the appropriate proxy, timeout and headers @@ -94,83 +102,164 @@ class RestbaseVirtualRESTService extends VirtualRESTService { } /** - * Remaps Parsoid requests to Restbase paths + * Remaps Parsoid v1/v3 requests to RESTBase v1 requests. */ public function onParsoidRequests( array $reqs, Closure $idGeneratorFunc ) { $result = array(); foreach ( $reqs as $key => $req ) { $parts = explode( '/', $req['url'] ); - list( - $targetWiki, // 'local' - $version, // 'v1' - $reqType // 'page' or 'transform' - ) = $parts; - if ( $targetWiki !== 'local' ) { - throw new Exception( "Only 'local' target wiki is currently supported" ); - } elseif ( $reqType !== 'page' && $reqType !== 'transform' ) { - throw new Exception( "Request type must be either 'page' or 'transform'" ); + if ( $parts[1] === 'v3' ) { + $result[$key] = $this->onParsoid3Request( $req, $idGeneratorFunc ); + } elseif ( $parts[1] === 'v1' ) { + $result[$key] = $this->onParsoid1Request( $req, $idGeneratorFunc ); + } else { + throw new Exception( "Only v1 and v3 are supported." ); } - $req['url'] = $this->params['url'] . '/' . $this->params['domain'] . '/v1/' . $reqType . '/'; - if ( $reqType === 'page' ) { - $title = $parts[3]; - if ( $parts[4] !== 'html' ) { - throw new Exception( "Only 'html' output format is currently supported" ); - } - $req['url'] .= 'html/' . $title; - if ( isset( $parts[5] ) ) { - $req['url'] .= '/' . $parts[5]; - } elseif ( isset( $req['query']['oldid'] ) && $req['query']['oldid'] ) { - $req['url'] .= '/' . $req['query']['oldid']; - unset( $req['query']['oldid'] ); - } - } elseif ( $reqType === 'transform' ) { - // from / to transform - $req['url'] .= $parts[3] . '/to/' . $parts[5]; - // the title - if ( isset( $parts[6] ) ) { - $req['url'] .= '/' . $parts[6]; - } - // revision id - if ( isset( $parts[7] ) ) { - $req['url'] .= '/' . $parts[7]; - } elseif ( isset( $req['body']['oldid'] ) && $req['body']['oldid'] ) { - $req['url'] .= '/' . $req['body']['oldid']; - unset( $req['body']['oldid'] ); - } - if ( $parts[4] !== 'to' ) { - throw new Exception( "Part index 4 is not 'to'" ); - } - if ( $parts[3] === 'html' & $parts[5] === 'wikitext' ) { - if ( !isset( $req['body']['html'] ) ) { - throw new Exception( "You must set an 'html' body key for this request" ); - } - } elseif ( $parts[3] == 'wikitext' && $parts[5] == 'html' ) { - if ( !isset( $req['body']['wikitext'] ) ) { - throw new Exception( "You must set a 'wikitext' body key for this request" ); - } - if ( isset( $req['body']['body'] ) ) { - $req['body']['bodyOnly'] = $req['body']['body']; - unset( $req['body']['body'] ); - } - } else { - throw new Exception( "Transformation unsupported" ); - } + } + + return $result; + + } + + /** + * Remap a Parsoid v1 request to a RESTBase v1 request. + * + * Example Parsoid v1 requests: + * GET /local/v1/page/$title/html/$oldid + * * $oldid is optional + * POST /local/v1/transform/html/to/wikitext/$title/$oldid + * * body: array( 'html' => ... ) + * * $title and $oldid are optional + * POST /local/v1/transform/wikitext/to/html/$title + * * body: array( 'wikitext' => ... ) or array( 'wikitext' => ..., 'body' => true/false ) + * * $title is optional + * + * NOTE: the POST APIs aren't "real" Parsoid v1 APIs, they are just what + * Visual Editor "pretends" the V1 API is like. (See + * ParsoidVirtualRESTService.) + */ + public function onParsoid1Request( array $req, Closure $idGeneratorFunc ) { + $parts = explode( '/', $req['url'] ); + list( + $targetWiki, // 'local' + $version, // 'v1' + $reqType // 'page' or 'transform' + ) = $parts; + if ( $targetWiki !== 'local' ) { + throw new Exception( "Only 'local' target wiki is currently supported" ); + } elseif ( $version !== 'v1' ) { + throw new Exception( "Version mismatch: should not happen." ); + } elseif ( $reqType !== 'page' && $reqType !== 'transform' ) { + throw new Exception( "Request type must be either 'page' or 'transform'" ); + } + $req['url'] = $this->params['url'] . $this->params['domain'] . '/v1/' . $reqType . '/'; + if ( $reqType === 'page' ) { + $title = $parts[3]; + if ( $parts[4] !== 'html' ) { + throw new Exception( "Only 'html' output format is currently supported" ); } - // set the appropriate proxy, timeout and headers - if ( $this->params['HTTPProxy'] ) { - $req['proxy'] = $this->params['HTTPProxy']; + $req['url'] .= 'html/' . $title; + if ( isset( $parts[5] ) ) { + $req['url'] .= '/' . $parts[5]; + } elseif ( isset( $req['query']['oldid'] ) && $req['query']['oldid'] ) { + $req['url'] .= '/' . $req['query']['oldid']; + unset( $req['query']['oldid'] ); } - if ( $this->params['timeout'] != null ) { - $req['reqTimeout'] = $this->params['timeout']; + } elseif ( $reqType === 'transform' ) { + // from / to transform + $req['url'] .= $parts[3] . '/to/' . $parts[5]; + // the title + if ( isset( $parts[6] ) ) { + $req['url'] .= '/' . $parts[6]; } - if ( $this->params['forwardCookies'] ) { - $req['headers']['Cookie'] = $this->params['forwardCookies']; + // revision id + if ( isset( $parts[7] ) ) { + $req['url'] .= '/' . $parts[7]; + } elseif ( isset( $req['body']['oldid'] ) && $req['body']['oldid'] ) { + $req['url'] .= '/' . $req['body']['oldid']; + unset( $req['body']['oldid'] ); + } + if ( $parts[4] !== 'to' ) { + throw new Exception( "Part index 4 is not 'to'" ); + } + if ( $parts[3] === 'html' && $parts[5] === 'wikitext' ) { + if ( !isset( $req['body']['html'] ) ) { + throw new Exception( "You must set an 'html' body key for this request" ); + } + } elseif ( $parts[3] == 'wikitext' && $parts[5] == 'html' ) { + if ( !isset( $req['body']['wikitext'] ) ) { + throw new Exception( "You must set a 'wikitext' body key for this request" ); + } + if ( isset( $req['body']['body'] ) ) { + $req['body']['bodyOnly'] = $req['body']['body']; + unset( $req['body']['body'] ); + } + } else { + throw new Exception( "Transformation unsupported" ); } - $result[$key] = $req; + } + // set the appropriate proxy, timeout and headers + if ( $this->params['HTTPProxy'] ) { + $req['proxy'] = $this->params['HTTPProxy']; + } + if ( $this->params['timeout'] != null ) { + $req['reqTimeout'] = $this->params['timeout']; + } + if ( $this->params['forwardCookies'] ) { + $req['headers']['Cookie'] = $this->params['forwardCookies']; } - return $result; + return $req; + + } + + /** + * Remap a Parsoid v3 request to a RESTBase v1 request. + * + * Example Parsoid v3 requests: + * GET /local/v3/page/html/$title/{$revision} + * * $revision is optional + * POST /local/v3/transform/html/to/wikitext/{$title}{/$revision} + * * body: array( 'html' => ... ) + * * $title and $revision are optional + * POST /local/v3/transform/wikitext/to/html/{$title}{/$revision} + * * body: array( 'wikitext' => ... ) or array( 'wikitext' => ..., 'bodyOnly' => true/false ) + * * $title is optional + * * $revision is optional + */ + public function onParsoid3Request( array $req, Closure $idGeneratorFunc ) { + + $parts = explode( '/', $req['url'] ); + list( + $targetWiki, // 'local' + $version, // 'v3' + $action, // 'transform' or 'page' + $format, // 'html' or 'wikitext' + // $title, // optional + // $revision, // optional + ) = $parts; + if ( $targetWiki !== 'local' ) { + throw new Exception( "Only 'local' target wiki is currently supported" ); + } elseif ( $version !== 'v3' ) { + throw new Exception( "Version mismatch: should not happen." ); + } + // replace /local/ with the current domain, change v3 to v1, + $req['url'] = preg_replace( '#^local/v3/#', $this->params['domain'] . '/v1/', $req['url'] ); + // and prefix it with the service URL + $req['url'] = $this->params['url'] . $req['url']; + // set the appropriate proxy, timeout and headers + if ( $this->params['HTTPProxy'] ) { + $req['proxy'] = $this->params['HTTPProxy']; + } + if ( $this->params['timeout'] != null ) { + $req['reqTimeout'] = $this->params['timeout']; + } + if ( $this->params['forwardCookies'] ) { + $req['headers']['Cookie'] = $this->params['forwardCookies']; + } + + return $req; } -- 2.20.1