From 54bd63946780e34894a50707f8378b76a2208096 Mon Sep 17 00:00:00 2001 From: Timo Tijhof Date: Mon, 15 Oct 2018 02:56:46 +0100 Subject: [PATCH] resources: Extract ForeignResourceManager from manageForeignResources.php This should make it easier to test, and would also allow other repositories to create an instance of it in a maintenance script of their own. Change-Id: I6a28e184f80eb38b5c25a0be5a9041f0c587c852 --- autoload.php | 1 + includes/ForeignResourceManager.php | 327 ++++++++++++++++++ .../resources/manageForeignResources.php | 280 ++------------- 3 files changed, 348 insertions(+), 260 deletions(-) create mode 100644 includes/ForeignResourceManager.php diff --git a/autoload.php b/autoload.php index 0f92ccbde0..a57c9964e1 100644 --- a/autoload.php +++ b/autoload.php @@ -543,6 +543,7 @@ $wgAutoloadLocalClasses = [ 'ForeignDBFile' => __DIR__ . '/includes/filerepo/file/ForeignDBFile.php', 'ForeignDBRepo' => __DIR__ . '/includes/filerepo/ForeignDBRepo.php', 'ForeignDBViaLBRepo' => __DIR__ . '/includes/filerepo/ForeignDBViaLBRepo.php', + 'ForeignResourceManager' => __DIR__ . '/includes/ForeignResourceManager.php', 'ForeignTitle' => __DIR__ . '/includes/title/ForeignTitle.php', 'ForeignTitleFactory' => __DIR__ . '/includes/title/ForeignTitleFactory.php', 'ForkController' => __DIR__ . '/includes/ForkController.php', diff --git a/includes/ForeignResourceManager.php b/includes/ForeignResourceManager.php new file mode 100644 index 0000000000..d6175f6dba --- /dev/null +++ b/includes/ForeignResourceManager.php @@ -0,0 +1,327 @@ +registryFile = $registryFile; + $this->libDir = $libDir; + $this->infoPrinter = $infoPrinter ?? function () { + }; + $this->errorPrinter = $errorPrinter ?? $this->infoPrinter; + $this->verbosePrinter = $verbosePrinter ?? function () { + }; + + // Use a temporary directory under the destination directory instead + // of wfTempDir() because PHP's rename() does not work across file + // systems, as the user's /tmp and $IP may be on different filesystems. + $this->tmpParentDir = "{$this->libDir}/.tmp"; + } + + /** + * @return bool + * @throws Exception + */ + public function run( $action, $module ) { + if ( !in_array( $action, [ 'update', 'verify', 'make-sri' ] ) ) { + throw new Exception( 'Invalid action parameter.' ); + } + $this->action = $action; + + $registry = $this->parseBasicYaml( file_get_contents( $this->registryFile ) ); + if ( $module === 'all' ) { + $modules = $registry; + } elseif ( isset( $registry[ $module ] ) ) { + $modules = [ $module => $registry[ $module ] ]; + } else { + throw new Exception( 'Unknown module name.' ); + } + + foreach ( $modules as $moduleName => $info ) { + $this->verbose( "\n### {$moduleName}\n\n" ); + $destDir = "{$this->libDir}/$moduleName"; + + if ( $this->action === 'update' ) { + $this->output( "... updating '{$moduleName}'\n" ); + $this->verbose( "... emptying directory for $moduleName\n" ); + wfRecursiveRemoveDir( $destDir ); + } elseif ( $this->action === 'verify' ) { + $this->output( "... verifying '{$moduleName}'\n" ); + } else { + $this->output( "... checking '{$moduleName}'\n" ); + } + + $this->verbose( "... preparing {$this->tmpParentDir}\n" ); + wfRecursiveRemoveDir( $this->tmpParentDir ); + if ( !wfMkdirParents( $this->tmpParentDir ) ) { + throw new Exception( "Unable to create {$this->tmpParentDir}" ); + } + + if ( !isset( $info['type'] ) ) { + throw new Exception( "Module '$moduleName' must have a 'type' key." ); + } + switch ( $info['type'] ) { + case 'tar': + $this->handleTypeTar( $moduleName, $destDir, $info ); + break; + case 'file': + $this->handleTypeFile( $moduleName, $destDir, $info ); + break; + case 'multi-file': + $this->handleTypeMultiFile( $moduleName, $destDir, $info ); + break; + default: + throw new Exception( "Unknown type '{$info['type']}' for '$moduleName'" ); + } + } + + $this->cleanUp(); + $this->output( "\nDone!\n" ); + if ( $this->hasErrors ) { + // The verify mode should check all modules/files and fail after, not during. + return false; + } + + return true; + } + + private function fetch( $src, $integrity ) { + $data = Http::get( $src, [ 'followRedirects' => false ] ); + if ( $data === false ) { + throw new Exception( "Failed to download resource at {$src}" ); + } + $algo = $integrity === null ? $this->defaultAlgo : explode( '-', $integrity )[0]; + $actualIntegrity = $algo . '-' . base64_encode( hash( $algo, $data, true ) ); + if ( $integrity === $actualIntegrity ) { + $this->verbose( "... passed integrity check for {$src}\n" ); + } else { + if ( $this->action === 'make-sri' ) { + $this->output( "Integrity for {$src}\n\tintegrity: ${actualIntegrity}\n" ); + } else { + throw new Exception( "Integrity check failed for {$src}\n" . + "\tExpected: {$integrity}\n" . + "\tActual: {$actualIntegrity}" + ); + } + } + return $data; + } + + private function handleTypeFile( $moduleName, $destDir, array $info ) { + if ( !isset( $info['src'] ) ) { + throw new Exception( "Module '$moduleName' must have a 'src' key." ); + } + $data = $this->fetch( $info['src'], $info['integrity'] ?? null ); + $dest = $info['dest'] ?? basename( $info['src'] ); + $path = "$destDir/$dest"; + if ( $this->action === 'verify' && sha1_file( $path ) !== sha1( $data ) ) { + throw new Exception( "File for '$moduleName' is different." ); + } + if ( $this->action === 'update' ) { + wfMkdirParents( $destDir ); + file_put_contents( "$destDir/$dest", $data ); + } + } + + private function handleTypeMultiFile( $moduleName, $destDir, array $info ) { + if ( !isset( $info['files'] ) ) { + throw new Exception( "Module '$moduleName' must have a 'files' key." ); + } + foreach ( $info['files'] as $dest => $file ) { + if ( !isset( $file['src'] ) ) { + throw new Exception( "Module '$moduleName' file '$dest' must have a 'src' key." ); + } + $data = $this->fetch( $file['src'], $file['integrity'] ?? null ); + $path = "$destDir/$dest"; + if ( $this->action === 'verify' && sha1_file( $path ) !== sha1( $data ) ) { + throw new Exception( "File '$dest' for '$moduleName' is different." ); + } elseif ( $this->action === 'update' ) { + wfMkdirParents( $destDir ); + file_put_contents( "$destDir/$dest", $data ); + } + } + } + + private function handleTypeTar( $moduleName, $destDir, array $info ) { + $info += [ 'src' => null, 'integrity' => null, 'dest' => null ]; + if ( $info['src'] === null ) { + throw new Exception( "Module '$moduleName' must have a 'src' key." ); + } + // Download the resource to a temporary file and open it + $data = $this->fetch( $info['src'], $info['integrity' ] ); + $tmpFile = "{$this->tmpParentDir}/$moduleName.tar"; + $this->verbose( "... writing '$moduleName' src to $tmpFile\n" ); + file_put_contents( $tmpFile, $data ); + $p = new PharData( $tmpFile ); + $tmpDir = "{$this->tmpParentDir}/$moduleName"; + $p->extractTo( $tmpDir ); + unset( $data, $p ); + + if ( $info['dest'] === null ) { + // Default: Replace the entire directory + $toCopy = [ $tmpDir => $destDir ]; + } else { + // Expand and normalise the 'dest' entries + $toCopy = []; + foreach ( $info['dest'] as $fromSubPath => $toSubPath ) { + // Use glob() to expand wildcards and check existence + $fromPaths = glob( "{$tmpDir}/{$fromSubPath}", GLOB_BRACE ); + if ( !$fromPaths ) { + throw new Exception( "Path '$fromSubPath' of '$moduleName' not found." ); + } + foreach ( $fromPaths as $fromPath ) { + $toCopy[$fromPath] = $toSubPath === null + ? "$destDir/" . basename( $fromPath ) + : "$destDir/$toSubPath/" . basename( $fromPath ); + } + } + } + foreach ( $toCopy as $from => $to ) { + if ( $this->action === 'verify' ) { + $this->verbose( "... verifying $to\n" ); + if ( is_dir( $from ) ) { + $rii = new RecursiveIteratorIterator( new RecursiveDirectoryIterator( + $from, + RecursiveDirectoryIterator::SKIP_DOTS + ) ); + foreach ( $rii as $file ) { + $remote = $file->getPathname(); + $local = strtr( $remote, [ $from => $to ] ); + if ( sha1_file( $remote ) !== sha1_file( $local ) ) { + $this->error( "File '$local' is different." ); + $this->hasErrors = true; + } + } + } elseif ( sha1_file( $from ) !== sha1_file( $to ) ) { + $this->error( "File '$to' is different." ); + $this->hasErrors = true; + } + } elseif ( $this->action === 'update' ) { + $this->verbose( "... moving $from to $to\n" ); + wfMkdirParents( dirname( $to ) ); + if ( !rename( $from, $to ) ) { + throw new Exception( "Could not move $from to $to." ); + } + } + } + } + + private function verbose( $text ) { + ( $this->verbosePrinter )( $text ); + } + + private function output( $text ) { + ( $this->infoPrinter )( $text ); + } + + private function error( $text ) { + ( $this->errorPrinter )( $text ); + } + + private function cleanUp() { + wfRecursiveRemoveDir( $this->tmpParentDir ); + } + + /** + * Basic YAML parser. + * + * Supports only string or object values, and 2 spaces indentation. + * + * @todo Just ship symfony/yaml. + * @param string $input + * @return array + */ + private function parseBasicYaml( $input ) { + $lines = explode( "\n", $input ); + $root = []; + $stack = [ &$root ]; + $prev = 0; + foreach ( $lines as $i => $text ) { + $line = $i + 1; + $trimmed = ltrim( $text, ' ' ); + if ( $trimmed === '' || $trimmed[0] === '#' ) { + continue; + } + $indent = strlen( $text ) - strlen( $trimmed ); + if ( $indent % 2 !== 0 ) { + throw new Exception( __METHOD__ . ": Odd indentation on line $line." ); + } + $depth = $indent === 0 ? 0 : ( $indent / 2 ); + if ( $depth < $prev ) { + // Close previous branches we can't re-enter + array_splice( $stack, $depth + 1 ); + } + if ( !array_key_exists( $depth, $stack ) ) { + throw new Exception( __METHOD__ . ": Too much indentation on line $line." ); + } + if ( strpos( $trimmed, ':' ) === false ) { + throw new Exception( __METHOD__ . ": Missing colon on line $line." ); + } + $dest =& $stack[ $depth ]; + if ( $dest === null ) { + // Promote from null to object + $dest = []; + } + list( $key, $val ) = explode( ':', $trimmed, 2 ); + $val = ltrim( $val, ' ' ); + if ( $val !== '' ) { + // Add string + $dest[ $key ] = $val; + } else { + // Add null (may become an object later) + $val = null; + $stack[] = &$val; + $dest[ $key ] = &$val; + } + $prev = $depth; + unset( $dest, $val ); + } + return $root; + } +} diff --git a/maintenance/resources/manageForeignResources.php b/maintenance/resources/manageForeignResources.php index 5317b28c4e..6de82c0912 100644 --- a/maintenance/resources/manageForeignResources.php +++ b/maintenance/resources/manageForeignResources.php @@ -28,13 +28,7 @@ require_once __DIR__ . '/../Maintenance.php'; * @since 1.32 */ class ManageForeignResources extends Maintenance { - private $defaultAlgo = 'sha384'; - private $tmpParentDir; - private $action; - private $failAfterOutput = false; - public function __construct() { - global $IP; parent::__construct(); $this->addDescription( <<addArg( 'action', 'One of "update", "verify" or "make-sri"', true ); $this->addArg( 'module', 'Name of a single module (Default: all)', false ); $this->addOption( 'verbose', 'Be verbose', false, false, 'v' ); - - // Use a directory in $IP instead of wfTempDir() because - // PHP's rename() does not work across file systems. - $this->tmpParentDir = "{$IP}/resources/tmp"; } + /** + * @return bool + * @throws Exception + */ public function execute() { global $IP; - $this->action = $this->getArg( 0 ); - if ( !in_array( $this->action, [ 'update', 'verify', 'make-sri' ] ) ) { - $this->fatalError( "Invalid action argument." ); - } - - $registry = $this->parseBasicYaml( - file_get_contents( __DIR__ . '/foreign-resources.yaml' ) - ); - $module = $this->getArg( 1, 'all' ); - if ( $module === 'all' ) { - $modules = $registry; - } elseif ( isset( $registry[ $module ] ) ) { - $modules = [ $module => $registry[ $module ] ]; - } else { - $this->fatalError( 'Unknown module name.' ); - } - - foreach ( $modules as $moduleName => $info ) { - $this->verbose( "\n### {$moduleName}\n\n" ); - $destDir = "{$IP}/resources/lib/$moduleName"; - - if ( $this->action === 'update' ) { - $this->output( "... updating '{$moduleName}'\n" ); - $this->verbose( "... emptying /resources/lib/$moduleName\n" ); - wfRecursiveRemoveDir( $destDir ); - } elseif ( $this->action === 'verify' ) { - $this->output( "... verifying '{$moduleName}'\n" ); - } else { - $this->output( "... checking '{$moduleName}'\n" ); - } - - $this->verbose( "... preparing {$this->tmpParentDir}\n" ); - wfRecursiveRemoveDir( $this->tmpParentDir ); - if ( !wfMkdirParents( $this->tmpParentDir ) ) { - $this->fatalError( "Unable to create {$this->tmpParentDir}" ); - } - - if ( !isset( $info['type'] ) ) { - $this->fatalError( "Module '$moduleName' must have a 'type' key." ); - } - switch ( $info['type'] ) { - case 'tar': - $this->handleTypeTar( $moduleName, $destDir, $info ); - break; - case 'file': - $this->handleTypeFile( $moduleName, $destDir, $info ); - break; - case 'multi-file': - $this->handleTypeMultiFile( $moduleName, $destDir, $info ); - break; - default: - $this->fatalError( "Unknown type '{$info['type']}' for '$moduleName'" ); - } - } - - $this->cleanUp(); - $this->output( "\nDone!\n" ); - if ( $this->failAfterOutput ) { - // The verify mode should check all modules/files and fail after, not during. - return false; - } - } - - private function fetch( $src, $integrity ) { - $data = Http::get( $src, [ 'followRedirects' => false ] ); - if ( $data === false ) { - $this->fatalError( "Failed to download resource at {$src}" ); - } - $algo = $integrity === null ? $this->defaultAlgo : explode( '-', $integrity )[0]; - $actualIntegrity = $algo . '-' . base64_encode( hash( $algo, $data, true ) ); - if ( $integrity === $actualIntegrity ) { - $this->verbose( "... passed integrity check for {$src}\n" ); - } else { - if ( $this->action === 'make-sri' ) { - $this->output( "Integrity for {$src}\n\tintegrity: ${actualIntegrity}\n" ); - } else { - $this->fatalError( "Integrity check failed for {$src}\n" . - "\tExpected: {$integrity}\n" . - "\tActual: {$actualIntegrity}" - ); - } - } - return $data; - } - - private function handleTypeFile( $moduleName, $destDir, array $info ) { - if ( !isset( $info['src'] ) ) { - $this->fatalError( "Module '$moduleName' must have a 'src' key." ); - } - $data = $this->fetch( $info['src'], $info['integrity'] ?? null ); - $dest = $info['dest'] ?? basename( $info['src'] ); - $path = "$destDir/$dest"; - if ( $this->action === 'verify' && sha1_file( $path ) !== sha1( $data ) ) { - $this->fatalError( "File for '$moduleName' is different." ); - } elseif ( $this->action === 'update' ) { - wfMkdirParents( $destDir ); - file_put_contents( "$destDir/$dest", $data ); - } - } - - private function handleTypeMultiFile( $moduleName, $destDir, array $info ) { - if ( !isset( $info['files'] ) ) { - $this->fatalError( "Module '$moduleName' must have a 'files' key." ); - } - foreach ( $info['files'] as $dest => $file ) { - if ( !isset( $file['src'] ) ) { - $this->fatalError( "Module '$moduleName' file '$dest' must have a 'src' key." ); - } - $data = $this->fetch( $file['src'], $file['integrity'] ?? null ); - $path = "$destDir/$dest"; - if ( $this->action === 'verify' && sha1_file( $path ) !== sha1( $data ) ) { - $this->fatalError( "File '$dest' for '$moduleName' is different." ); - } elseif ( $this->action === 'update' ) { - wfMkdirParents( $destDir ); - file_put_contents( "$destDir/$dest", $data ); - } - } - } - - private function handleTypeTar( $moduleName, $destDir, array $info ) { - $info += [ 'src' => null, 'integrity' => null, 'dest' => null ]; - if ( $info['src'] === null ) { - $this->fatalError( "Module '$moduleName' must have a 'src' key." ); - } - // Download the resource to a temporary file and open it - $data = $this->fetch( $info['src'], $info['integrity' ] ); - $tmpFile = "{$this->tmpParentDir}/$moduleName.tar"; - $this->verbose( "... writing '$moduleName' src to $tmpFile\n" ); - file_put_contents( $tmpFile, $data ); - $p = new PharData( $tmpFile ); - $tmpDir = "{$this->tmpParentDir}/$moduleName"; - $p->extractTo( $tmpDir ); - unset( $data, $p ); - - if ( $info['dest'] === null ) { - // Default: Replace the entire directory - $toCopy = [ $tmpDir => $destDir ]; - } else { - // Expand and normalise the 'dest' entries - $toCopy = []; - foreach ( $info['dest'] as $fromSubPath => $toSubPath ) { - // Use glob() to expand wildcards and check existence - $fromPaths = glob( "{$tmpDir}/{$fromSubPath}", GLOB_BRACE ); - if ( !$fromPaths ) { - $this->fatalError( "Path '$fromSubPath' of '$moduleName' not found." ); - } - foreach ( $fromPaths as $fromPath ) { - $toCopy[$fromPath] = $toSubPath === null - ? "$destDir/" . basename( $fromPath ) - : "$destDir/$toSubPath/" . basename( $fromPath ); - } - } - } - foreach ( $toCopy as $from => $to ) { - if ( $this->action === 'verify' ) { - $this->verbose( "... verifying $to\n" ); - if ( is_dir( $from ) ) { - $rii = new RecursiveIteratorIterator( new RecursiveDirectoryIterator( - $from, - RecursiveDirectoryIterator::SKIP_DOTS - ) ); - foreach ( $rii as $file ) { - $remote = $file->getPathname(); - $local = strtr( $remote, [ $from => $to ] ); - if ( sha1_file( $remote ) !== sha1_file( $local ) ) { - $this->error( "File '$local' is different." ); - $this->failAfterOutput = true; - } - } - } elseif ( sha1_file( $from ) !== sha1_file( $to ) ) { - $this->error( "File '$to' is different." ); - $this->failAfterOutput = true; - } - } elseif ( $this->action === 'update' ) { - $this->verbose( "... moving $from to $to\n" ); - wfMkdirParents( dirname( $to ) ); - if ( !rename( $from, $to ) ) { - $this->fatalError( "Could not move $from to $to." ); + $frm = new ForeignResourceManager( + __DIR__ . '/foreign-resources.yaml', + "{$IP}/resources/lib", + function ( $text ) { + $this->output( $text ); + }, + function ( $text ) { + $this->error( $text ); + }, + function ( $text ) { + if ( $this->hasOption( 'verbose' ) ) { + $this->output( $text ); } } - } - } - - private function verbose( $text ) { - if ( $this->hasOption( 'verbose' ) ) { - $this->output( $text ); - } - } - - private function cleanUp() { - wfRecursiveRemoveDir( $this->tmpParentDir ); - } - - protected function fatalError( $msg, $exitCode = 1 ) { - $this->cleanUp(); - parent::fatalError( $msg, $exitCode ); - } + ); - /** - * Basic YAML parser. - * - * Supports only string or object values, and 2 spaces indentation. - * - * @todo Just ship symfony/yaml. - * @param string $input - * @return array - */ - private function parseBasicYaml( $input ) { - $lines = explode( "\n", $input ); - $root = []; - $stack = [ &$root ]; - $prev = 0; - foreach ( $lines as $i => $text ) { - $line = $i + 1; - $trimmed = ltrim( $text, ' ' ); - if ( $trimmed === '' || $trimmed[0] === '#' ) { - continue; - } - $indent = strlen( $text ) - strlen( $trimmed ); - if ( $indent % 2 !== 0 ) { - throw new Exception( __METHOD__ . ": Odd indentation on line $line." ); - } - $depth = $indent === 0 ? 0 : ( $indent / 2 ); - if ( $depth < $prev ) { - // Close previous branches we can't re-enter - array_splice( $stack, $depth + 1 ); - } - if ( !array_key_exists( $depth, $stack ) ) { - throw new Exception( __METHOD__ . ": Too much indentation on line $line." ); - } - if ( strpos( $trimmed, ':' ) === false ) { - throw new Exception( __METHOD__ . ": Missing colon on line $line." ); - } - $dest =& $stack[ $depth ]; - if ( $dest === null ) { - // Promote from null to object - $dest = []; - } - list( $key, $val ) = explode( ':', $trimmed, 2 ); - $val = ltrim( $val, ' ' ); - if ( $val !== '' ) { - // Add string - $dest[ $key ] = $val; - } else { - // Add null (may become an object later) - $val = null; - $stack[] = &$val; - $dest[ $key ] = &$val; - } - $prev = $depth; - unset( $dest, $val ); - } - return $root; + $action = $this->getArg( 0 ); + $module = $this->getArg( 1, 'all' ); + return $frm->run( $action, $module ); } } -- 2.20.1