From: Aaron Schulz Date: Tue, 28 Jan 2014 20:21:29 +0000 (-0800) Subject: Added VirtualRESTServiceClient/VirtualRESTService classes X-Git-Tag: 1.31.0-rc.0~15955^2 X-Git-Url: http://git.cyclocoop.org/%7D%7Cconcat%7B?a=commitdiff_plain;h=4d6d11d64ba80fe508f6c1fa095a7b987e367b5b;p=lhc%2Fweb%2Fwiklou.git Added VirtualRESTServiceClient/VirtualRESTService classes * Reference handling cleanup for loops in MultiHttpClient * Support a slightly more convenient request array format for MultiHttpClient * Added missing license header to MultiHttpClient Change-Id: Icc86ed9e4c2ac78363aa7b6f3147253bb59e2afb --- diff --git a/includes/AutoLoader.php b/includes/AutoLoader.php index 077c72cee0..1f05ef6031 100644 --- a/includes/AutoLoader.php +++ b/includes/AutoLoader.php @@ -697,6 +697,9 @@ $wgAutoloadLocalClasses = array( 'RunningStat' => 'includes/libs/RunningStat.php', 'ScopedCallback' => 'includes/libs/ScopedCallback.php', 'ScopedPHPTimeout' => 'includes/libs/ScopedPHPTimeout.php', + 'SwiftVirtualRESTService' => 'includes/libs/virtualrest/SwiftVirtualRESTService.php', + 'VirtualRESTService' => 'includes/libs/virtualrest/VirtualRESTService.php', + 'VirtualRESTServiceClient' => 'includes/libs/virtualrest/VirtualRESTServiceClient.php', 'XmlTypeCheck' => 'includes/libs/XmlTypeCheck.php', # includes/libs/lessphp diff --git a/includes/libs/MultiHttpClient.php b/includes/libs/MultiHttpClient.php index 340e7f44e0..671e812a47 100644 --- a/includes/libs/MultiHttpClient.php +++ b/includes/libs/MultiHttpClient.php @@ -86,7 +86,7 @@ class MultiHttpClient { * - err : Any cURL error string * The map also stores integer-indexed copies of these values. This lets callers do: * - * list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $req; + * list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $http->run( $req ); * * @param array $req HTTP request array * @param array $opts @@ -110,7 +110,7 @@ class MultiHttpClient { * - err : Any cURL error string * The map also stores integer-indexed copies of these values. This lets callers do: * - * list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $req; + * list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $req['response']; * * All headers in the 'headers' field are normalized to use lower case names. * This is true for the request headers and the response headers. Integer-indexed diff --git a/includes/libs/virtualrest/SwiftVirtualRESTService.php b/includes/libs/virtualrest/SwiftVirtualRESTService.php new file mode 100644 index 0000000000..011dabe080 --- /dev/null +++ b/includes/libs/virtualrest/SwiftVirtualRESTService.php @@ -0,0 +1,175 @@ +authCreds ) { + return true; + } + if ( $this->authErrorTimestamp !== null ) { + if ( ( time() - $this->authErrorTimestamp ) < 60 ) { + return $this->authCachedStatus; // failed last attempt; don't bother + } else { // actually retry this time + $this->authErrorTimestamp = null; + } + } + // Session keys expire after a while, so we renew them periodically + return ( ( time() - $this->authSessionTimestamp ) > $this->params['swiftAuthTTL'] ); + } + + protected function applyAuthResponse( array $req ) { + $this->authSessionTimestamp = 0; + list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $req['response']; + if ( $rcode >= 200 && $rcode <= 299 ) { // OK + $this->authCreds = array( + 'auth_token' => $rhdrs['x-auth-token'], + 'storage_url' => $rhdrs['x-storage-url'] + ); + $this->authSessionTimestamp = time(); + return true; + } elseif ( $rcode === 403 ) { + $this->authCachedStatus = 401; + $this->authCachedReason = 'Authorization Required'; + $this->authErrorTimestamp = time(); + return false; + } else { + $this->authCachedStatus = $rcode; + $this->authCachedReason = $rdesc; + $this->authErrorTimestamp = time(); + return null; + } + } + + public function onRequests( array $reqs, Closure $idGeneratorFunc ) { + $result = array(); + $firstReq = reset( $reqs ); + if ( $firstReq && count( $reqs ) == 1 && isset( $firstReq['isAuth'] ) ) { + // This was an authentication request for work requests... + $result = $reqs; // no change + } else { + // These are actual work requests... + $needsAuth = $this->needsAuthRequest(); + if ( $needsAuth === true ) { + // These are work requests and we don't have any token to use. + // Replace the work requests with an authentication request. + $result = array( + $idGeneratorFunc() => array( + 'method' => 'GET', + 'url' => $this->params['swiftAuthUrl'] . "/v1.0", + 'headers' => array( + 'x-auth-user' => $this->params['swiftUser'], + 'x-auth-key' => $this->params['swiftKey'] ), + 'isAuth' => true, + 'chain' => $reqs + ) + ); + } elseif ( $needsAuth !== false ) { + // These are work requests and authentication has previously failed. + // It is most efficient to just give failed pseudo responses back for + // the original work requests. + foreach ( $reqs as $key => $req ) { + $req['response'] = array( + 'code' => $this->authCachedStatus, + 'reason' => $this->authCachedReason, + 'headers' => array(), + 'body' => '', + 'error' => '' + ); + $result[$key] = $req; + } + } else { + // These are work requests and we have a token already. + // Go through and mangle each request to include a token. + foreach ( $reqs as $key => $req ) { + // The default encoding treats the URL as a REST style path that uses + // forward slash as a hierarchical delimiter (and never otherwise). + // Subclasses can override this, and should be documented in any case. + $parts = array_map( 'rawurlencode', explode( '/', $req['url'] ) ); + $req['url'] = $this->authCreds['storage_url'] . '/' . implode( '/', $parts ); + $req['headers']['x-auth-token'] = $this->authCreds['auth_token']; + $result[$key] = $req; + // @TODO: add ETag/Content-Length and such as needed + } + } + } + return $result; + } + + public function onResponses( array $reqs, Closure $idGeneratorFunc ) { + $firstReq = reset( $reqs ); + if ( $firstReq && count( $reqs ) == 1 && isset( $firstReq['isAuth'] ) ) { + $result = array(); + // This was an authentication request for work requests... + if ( $this->applyAuthResponse( $firstReq ) ) { + // If it succeeded, we can subsitute the work requests back. + // Call this recursively in order to munge and add headers. + $result = $this->onRequests( $firstReq['chain'], $idGeneratorFunc ); + } else { + // If it failed, it is most efficient to just give failing + // pseudo-responses back for the actual work requests. + foreach ( $firstReq['chain'] as $key => $req ) { + $req['response'] = array( + 'code' => $this->authCachedStatus, + 'reason' => $this->authCachedReason, + 'headers' => array(), + 'body' => '', + 'error' => '' + ); + $result[$key] = $req; + } + } + } else { + $result = $reqs; // no change + } + return $result; + } +} diff --git a/includes/libs/virtualrest/VirtualRESTService.php b/includes/libs/virtualrest/VirtualRESTService.php new file mode 100644 index 0000000000..05c2afc162 --- /dev/null +++ b/includes/libs/virtualrest/VirtualRESTService.php @@ -0,0 +1,107 @@ +params = $params; + } + + /** + * Prepare virtual HTTP(S) requests (for this service) for execution + * + * This method should mangle any of the $reqs entry fields as needed: + * - url : munge the URL to have an absolute URL with a protocol + * and encode path components as needed by the backend [required] + * - query : include any authentication signatures/parameters [as needed] + * - headers : include any authentication tokens/headers [as needed] + * + * The incoming URL parameter will be relative to the service mount point. + * + * This method can also remove some of the requests as well as add new ones + * (using $idGenerator to set each of the entries' array keys). For any existing + * or added request, the 'response' array can be filled in, which will prevent the + * client from executing it. If an original request is removed, at some point it + * must be added back (with the same key) in onRequests() or onResponses(); + * it's reponse may be filled in as with other requests. + * + * @param array $reqs Map of Virtual HTTP request arrays + * @param Closure $idGeneratorFunc Method to generate unique keys for new requests + * @return array Modified HTTP request array map + */ + public function onRequests( array $reqs, Closure $idGeneratorFunc ) { + $result = array(); + foreach ( $reqs as $key => $req ) { + // The default encoding treats the URL as a REST style path that uses + // forward slash as a hierarchical delimiter (and never otherwise). + // Subclasses can override this, and should be documented in any case. + $parts = array_map( 'rawurlencode', explode( '/', $req['url'] ) ); + $req['url'] = $this->params['baseUrl'] . '/' . implode( '/', $parts ); + $result[$key] = $req; + } + return $result; + } + + /** + * Mangle or replace virtual HTTP(S) requests which have been responded to + * + * This method may mangle any of the $reqs entry 'response' fields as needed: + * - code : perform any code normalization [as needed] + * - reason : perform any reason normalization [as needed] + * - headers : perform any header normalization [as needed] + * + * This method can also remove some of the requests as well as add new ones + * (using $idGenerator to set each of the entries' array keys). For any existing + * or added request, the 'response' array can be filled in, which will prevent the + * client from executing it. If an original request is removed, at some point it + * must be added back (with the same key) in onRequests() or onResponses(); + * it's reponse may be filled in as with other requests. All requests added to $reqs + * will be passed through onRequests() to handle any munging required as normal. + * + * The incoming URL parameter will be relative to the service mount point. + * + * @param array $reqs Map of Virtual HTTP request arrays with 'response' set + * @param Closure $idGeneratorFunc Method to generate unique keys for new requests + * @return array Modified HTTP request array map + */ + public function onResponses( array $reqs, Closure $idGeneratorFunc ) { + return $reqs; + } +} diff --git a/includes/libs/virtualrest/VirtualRESTServiceClient.php b/includes/libs/virtualrest/VirtualRESTServiceClient.php new file mode 100644 index 0000000000..2d21d3cfac --- /dev/null +++ b/includes/libs/virtualrest/VirtualRESTServiceClient.php @@ -0,0 +1,289 @@ + (uses RFC 3986) + * - headers :
+ * - body : source to get the HTTP request body from; + * this can simply be a string (always), a resource for + * PUT requests, and a field/value array for POST request; + * array bodies are encoded as multipart/form-data and strings + * use application/x-www-form-urlencoded (headers sent automatically) + * - stream : resource to stream the HTTP response body to + * Request maps can use integer index 0 instead of 'method' and 1 instead of 'url'. + * + * @author Aaron Schulz + * @since 1.23 + */ +class VirtualRESTServiceClient { + /** @var MultiHttpClient */ + protected $http; + /** @var Array Map of (prefix => VirtualRESTService) */ + protected $instances = array(); + + const VALID_MOUNT_REGEX = '#^/[0-9a-z]+/([0-9a-z]+/)*$#'; + + /** + * @param MultiHttpClient $http + */ + public function __construct( MultiHttpClient $http ) { + $this->http = $http; + } + + /** + * Map a prefix to service handler + * + * @param string $prefix Virtual path + * @param VirtualRESTService $instance + */ + public function mount( $prefix, VirtualRESTService $instance ) { + if ( !preg_match( self::VALID_MOUNT_REGEX, $prefix ) ) { + throw new UnexpectedValueException( "Invalid service mount point '$prefix'." ); + } elseif ( isset( $this->instances[$prefix] ) ) { + throw new UnexpectedValueException( "A service is already mounted on '$prefix'." ); + } + $this->instances[$prefix] = $instance; + } + + /** + * Unmap a prefix to service handler + * + * @param string $prefix Virtual path + */ + public function unmount( $prefix ) { + if ( !preg_match( self::VALID_MOUNT_REGEX, $prefix ) ) { + throw new UnexpectedValueException( "Invalid service mount point '$prefix'." ); + } elseif ( !isset( $this->instances[$prefix] ) ) { + throw new UnexpectedValueException( "No service is mounted on '$prefix'." ); + } + unset( $this->instances[$prefix] ); + } + + /** + * Get the prefix and service that a virtual path is serviced by + * + * @param string $path + * @return array (prefix,VirtualRESTService) or (null,null) if none found + */ + public function getMountAndService( $path ) { + $cmpFunc = function( $a, $b ) { + $al = substr_count( $a, '/' ); + $bl = substr_count( $b, '/' ); + if ( $al === $bl ) { + return 0; // should not actually happen + } + return ( $al < $bl ) ? 1 : -1; // largest prefix first + }; + + $matches = array(); // matching prefixes (mount points) + foreach ( $this->instances as $prefix => $service ) { + if ( strpos( $path, $prefix ) === 0 ) { + $matches[] = $prefix; + } + } + usort( $matches, $cmpFunc ); + + // Return the most specific prefix and corresponding service + return isset( $matches[0] ) + ? array( $matches[0], $this->instances[$matches[0]] ) + : array( null, null ); + } + + /** + * Execute a virtual HTTP(S) request + * + * This method returns a response map of: + * - code : HTTP response code or 0 if there was a serious cURL error + * - reason : HTTP response reason (empty if there was a serious cURL error) + * - headers :
+ * - body : HTTP response body or resource (if "stream" was set) + * - err : Any cURL error string + * The map also stores integer-indexed copies of these values. This lets callers do: + * + * list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $client->run( $req ); + * + * @param array $req Virtual HTTP request array + * @return array Response array for request + */ + public function run( array $req ) { + $req = $this->runMulti( array( $req ) ); + return $req[0]['response']; + } + + /** + * Execute a set of virtual HTTP(S) requests concurrently + * + * A map of requests keys to response maps is returned. Each response map has: + * - code : HTTP response code or 0 if there was a serious cURL error + * - reason : HTTP response reason (empty if there was a serious cURL error) + * - headers :
+ * - body : HTTP response body or resource (if "stream" was set) + * - err : Any cURL error string + * The map also stores integer-indexed copies of these values. This lets callers do: + * + * list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $responses[0]; + * + * + * @param array $req Map of Virtual HTTP request arrays + * @return array $reqs Map of corresponding response values with the same keys/order + */ + public function runMulti( array $reqs ) { + foreach ( $reqs as $index => &$req ) { + if ( isset( $req[0] ) ) { + $req['method'] = $req[0]; // short-form + unset( $req[0] ); + } + if ( isset( $req[1] ) ) { + $req['url'] = $req[1]; // short-form + unset( $req[1] ); + } + $req['chain'] = array(); // chain or list of replaced requests + } + unset( $req ); // don't assign over this by accident + + $curUniqueId = 0; + $armoredIndexMap = array(); // (original index => new index) + + $doneReqs = array(); // (index => request) + $executeReqs = array(); // (index => request) + $replaceReqsByService = array(); // (prefix => index => request) + $origPending = array(); // (index => 1) for original requests + + foreach ( $reqs as $origIndex => $req ) { + // Re-index keys to consecutive integers (they will be swapped back later) + $index = $curUniqueId++; + $armoredIndexMap[$origIndex] = $index; + $origPending[$index] = 1; + if ( preg_match( '#^(http|ftp)s?://#', $req['url'] ) ) { + // Absolute FTP/HTTP(S) URL, run it as normal + $executeReqs[$index] = $req; + } else { + // Must be a virtual HTTP URL; resolve it + list( $prefix, $service ) = $this->getMountAndService( $req['url'] ); + if ( !$service ) { + throw new UnexpectedValueException( "Path '{$req['url']}' has no service." ); + } + // Set the URL to the mount-relative portion + $req['url'] = substr( $req['url'], strlen( $prefix ) ); + $replaceReqsByService[$prefix][$index] = $req; + } + } + + // Function to get IDs that won't collide with keys in $armoredIndexMap + $idFunc = function() use ( &$curUniqueId ) { + return $curUniqueId++; + }; + + $rounds = 0; + do { + if ( ++$rounds > 5 ) { // sanity + throw new Exception( "Too many replacement rounds detected. Aborting." ); + } + // Resolve the virtual URLs valid and qualified HTTP(S) URLs + // and add any required authentication headers for the backend. + // Services can also replace requests with new ones, either to + // defer the original or to set a proxy response to the original. + $newReplaceReqsByService = array(); + foreach ( $replaceReqsByService as $prefix => $servReqs ) { + $service = $this->instances[$prefix]; + foreach ( $service->onRequests( $servReqs, $idFunc ) as $index => $req ) { + // Services use unique IDs for replacement requests + if ( isset( $servReqs[$index] ) || isset( $origPending[$index] ) ) { + // A current or original request which was not modified + } else { + // Replacement requests with pre-set responses should not execute + $newReplaceReqsByService[$prefix][$index] = $req; + } + if ( isset( $req['response'] ) ) { + // Replacement requests with pre-set responses should not execute + unset( $executeReqs[$index] ); + unset( $origPending[$index] ); + $doneReqs[$index] = $req; + } else { + // Original or mangled request included + $executeReqs[$index] = $req; + } + } + } + // Update index of requests to inspect for replacement + $replaceReqsByService = $newReplaceReqsByService; + // Run the actual work HTTP requests + foreach ( $this->http->runMulti( $executeReqs ) as $index => $ranReq ) { + $doneReqs[$index] = $ranReq; + unset( $origPending[$index] ); + } + $executeReqs = array(); + // Services can also replace requests with new ones, either to + // defer the original or to set a proxy response to the original. + // Any replacement requests executed above will need to be replaced + // with new requests (eventually the original). The responses can be + // forced instead of having the request sent over the wire. + $newReplaceReqsByService = array(); + foreach ( $replaceReqsByService as $prefix => $servReqs ) { + $service = $this->instances[$prefix]; + // Only the request copies stored in $doneReqs actually have the response + $servReqs = array_intersect_key( $doneReqs, $servReqs ); + foreach ( $service->onResponses( $servReqs, $idFunc ) as $index => $req ) { + // Services use unique IDs for replacement requests + if ( isset( $servReqs[$index] ) || isset( $origPending[$index] ) ) { + // A current or original request which was not modified + } else { + // Replacement requests with pre-set responses should not execute + $newReplaceReqsByService[$prefix][$index] = $req; + } + if ( isset( $req['response'] ) ) { + // Replacement requests with pre-set responses should not execute + unset( $origPending[$index] ); + $doneReqs[$index] = $req; + } else { + // Update the request in case it was mangled + $executeReqs[$index] = $req; + } + } + } + // Update index of requests to inspect for replacement + $replaceReqsByService = $newReplaceReqsByService; + } while ( count( $origPending ) ); + + $responses = array(); + // Update $reqs to include 'response' and normalized request 'headers'. + // This maintains the original order of $reqs. + foreach ( $reqs as $origIndex => $req ) { + $index = $armoredIndexMap[$origIndex]; + if ( !isset( $doneReqs[$index] ) ) { + throw new UnexpectedValueException( "Response for request '$index' is NULL." ); + } + $responses[$origIndex] = $doneReqs[$index]['response']; + } + + return $responses; + } +}