From c5c21bcfa315ad42c9853d92ef08dc0ed895f19b Mon Sep 17 00:00:00 2001 From: Timo Tijhof Date: Mon, 20 Aug 2018 00:02:29 +0100 Subject: [PATCH] maintenance: Add 'verify' action to manageForeignResources.php * Instead of having --update and --make-sri which oddly overlap in less-than-intuitive ways (with dry-run in there as well), refactor with three simple modes: - make-sri: Only validates 'integrity' attribute with remote resource and prints it. - verify: Extract files temporarily and compare with current /resources/lib/. Fail if diffferent. - update: Extract files temporarily and move to /resources/lib/. * Abtract code for 'tar' handling in preparation for addition of a 'file' and 'multi-file' type in later commits. * Fix bug where tmp dir was not cleaned up by fatalError(). Change-Id: I5ca3749533496824d2c40ebb5ebe9e570da7ac8a --- maintenance/resources/foreign-resources.yaml | 22 +- .../resources/manageForeignResources.php | 191 ++++++++++-------- 2 files changed, 124 insertions(+), 89 deletions(-) diff --git a/maintenance/resources/foreign-resources.yaml b/maintenance/resources/foreign-resources.yaml index b8d9848358..8a21d40d1d 100644 --- a/maintenance/resources/foreign-resources.yaml +++ b/maintenance/resources/foreign-resources.yaml @@ -1,17 +1,22 @@ ### Format of this file # # The top-level keys are module names (as registered in Resources.php). -# The values of these keys are resource descriptors. +# Each top-level key holds a resource descriptor that must have +# the following `type` value: # -# In each resource descriptor object, the `src` and `integrity` keys are required. +# - `tar`: For tarball archive (may be gzip-compressed). # -# * `src`: Full URL to a remote resource. -# * `integrity`: Cryptographic hash used to verify the remote content. -# Uses the "integrity metadata" format defined at . -# * `dest`: An object mapping paths from the remote resource to a destination in -# `/resources/lib/$module/`. The value may be omitted to indicate that -# paths should be extracted to the destination directory itself. +### Type tar +# +# The `src` and `integrity` keys are quired. +# +# * `src`: Full URL to thes remote resource. +# * `integrity`: Cryptographic hash (integrity metadata format per ). +# * `dest`: An object mapping paths to files or directory from the remote resource to a destination +# in the module directory. The value of key in dest may be omitted, which will extract the key +# directly to the module directory. oojs: + type: tar src: https://registry.npmjs.org/oojs/-/oojs-2.2.2.tgz integrity: sha256-ebgQW2EGrSkBCnDJBGqDpsBDjA3PMN/M8U5DyLHt9mw= dest: @@ -20,6 +25,7 @@ oojs: package/LICENSE-MIT: package/README.md: oojs-ui: + type: tar src: https://registry.npmjs.org/oojs-ui/-/oojs-ui-0.28.0.tgz integrity: sha384-j8bzlCPrfS4sca+U9JO9tdcewDlLlDlOVOsLn+Vqlcg5GU59vLSd7TVm4FiuTowy dest: diff --git a/maintenance/resources/manageForeignResources.php b/maintenance/resources/manageForeignResources.php index 528d6e7c4c..6065401f24 100644 --- a/maintenance/resources/manageForeignResources.php +++ b/maintenance/resources/manageForeignResources.php @@ -30,6 +30,8 @@ require_once __DIR__ . '/../Maintenance.php'; class ManageForeignResources extends Maintenance { private $defaultAlgo = 'sha384'; private $tmpParentDir; + private $action; + private $failAfterOutput = false; public function __construct() { global $IP; @@ -40,17 +42,16 @@ Manage foreign resources registered with ResourceLoader. This helps developers to download, verify and update local copies of upstream libraries registered as ResourceLoader modules. See also foreign-resources.yaml. -For sources that don't publish an integrity hash, leave the value empty at -first, and run this script with --make-sri to compute the hashes. +For sources that don't publish an integrity hash, omit "integrity" (or leave empty) +and run the "make-sri" action to compute the missing hashes. This script runs in dry mode by default. Use --update to actually change, remove, or add files to /resources/lib/. TEXT ); + $this->addArg( 'action', 'One of "update", "verify" or "make-sri"', true ); $this->addArg( 'module', 'Name of a single module (Default: all)', false ); - $this->addOption( 'update', ' resources/lib/ missing integrity metadata' ); - $this->addOption( 'make-sri', 'Compute missing integrity metadata' ); - $this->addOption( 'verbose', 'Be verbose' ); + $this->addOption( 'verbose', 'Be verbose', false, false, 'v' ); // Use a directory in $IP instead of wfTempDir() because // PHP's rename() does not work across file systems. @@ -59,67 +60,87 @@ TEXT public function execute() { global $IP; - $module = $this->getArg(); - $makeSRI = $this->hasOption( 'make-sri' ); + $this->action = $this->getArg( 0 ); + if ( !in_array( $this->action, [ 'update', 'verify', 'make-sri' ] ) ) { + $this->fatalError( "Invalid action argument." ); + } $registry = $this->parseBasicYaml( file_get_contents( __DIR__ . '/foreign-resources.yaml' ) ); + $module = $this->getArg( 1, 'all' ); foreach ( $registry as $moduleName => $info ) { - if ( $module !== null && $moduleName !== $module ) { + if ( $module !== 'all' && $moduleName !== $module ) { continue; } $this->verbose( "\n### {$moduleName}\n\n" ); + $destDir = "{$IP}/resources/lib/$moduleName"; - // Validate required keys - $info += [ 'src' => null, 'integrity' => null, 'dest' => null ]; - if ( $info['src'] === null ) { - $this->fatalError( "Module '$moduleName' must have a 'src' key." ); - } - $integrity = is_string( $info['integrity'] ) ? $info['integrity'] : $makeSRI; - if ( $integrity === false ) { - $this->fatalError( "Module '$moduleName' must have an 'integrity' key." ); + if ( $this->action === 'update' ) { + $this->output( "... updating '{$moduleName}'\n" ); + $this->verbose( "... emptying /resources/lib/$moduleName\n" ); + wfRecursiveRemoveDir( $destDir ); + } elseif ( $this->action === 'verify' ) { + $this->output( "... verifying '{$moduleName}'\n" ); + } else { + $this->output( "... checking '{$moduleName}'\n" ); } - // Download the resource - $data = Http::get( $info['src'], [ 'followRedirects' => false ] ); - if ( $data === false ) { - $this->fatalError( "Failed to download resource for '$moduleName'." ); + $this->verbose( "... preparing {$this->tmpParentDir}\n" ); + wfRecursiveRemoveDir( $this->tmpParentDir ); + if ( !wfMkdirParents( $this->tmpParentDir ) ) { + $this->fatalError( "Unable to create {$this->tmpParentDir}" ); } - // Validate integrity metadata - $this->output( "... checking integrity of '{$moduleName}'\n" ); - $algo = $integrity === true ? $this->defaultAlgo : explode( '-', $integrity )[0]; - $actualIntegrity = $algo . '-' . base64_encode( hash( $algo, $data, true ) ); - if ( $integrity === true ) { - $this->output( "Integrity for '{$moduleName}':\n\t${actualIntegrity}\n" ); - continue; - } elseif ( $integrity !== $actualIntegrity ) { - $this->fatalError( "Integrity check failed for '{$moduleName}:\n" . - "Expected: {$integrity}\n" . - "Actual: {$actualIntegrity}" - ); + if ( !isset( $info['type'] ) ) { + $this->fatalError( "Module '$moduleName' must have a 'type' key." ); + } + switch ( $info['type'] ) { + case 'tar': + $this->handleTypeTar( $moduleName, $destDir, $info ); + break; + default: + $this->fatalError( "Unknown type '{$info['type']}' for '$moduleName'" ); } - - // Determine destination - $destDir = "{$IP}/resources/lib/$moduleName"; - $this->output( "... extracting files for '{$moduleName}'\n" ); - $this->handleTypeTar( $moduleName, $data, $destDir, $info ); } - // Clean up - wfRecursiveRemoveDir( $this->tmpParentDir ); + $this->cleanUp(); $this->output( "\nDone!\n" ); + if ( $this->failAfterOutput ) { + // The verify mode should check all modules/files and fail after, not during. + return false; + } } - private function handleTypeTar( $moduleName, $data, $destDir, array $info ) { - global $IP; - wfRecursiveRemoveDir( $this->tmpParentDir ); - if ( !wfMkdirParents( $this->tmpParentDir ) ) { - $this->fatalError( "Unable to create {$this->tmpParentDir}" ); + private function fetch( $src, $integrity ) { + $data = Http::get( $src, [ 'followRedirects' => false ] ); + if ( $data === false ) { + $this->fatalError( "Failed to download resource at {$src}" ); + } + $algo = $integrity === null ? $this->defaultAlgo : explode( '-', $integrity )[0]; + $actualIntegrity = $algo . '-' . base64_encode( hash( $algo, $data, true ) ); + if ( $integrity === $actualIntegrity ) { + $this->verbose( "... passed integrity check for {$src}\n" ); + } else { + if ( $this->action === 'make-sri' ) { + $this->output( "Integrity for {$src}\n\tintegrity: ${actualIntegrity}\n" ); + } else { + $this->fatalError( "Integrity check failed for {$src}\n" . + "\tExpected: {$integrity}\n" . + "\tActual: {$actualIntegrity}" + ); + } } + return $data; + } - // Write resource to temporary file and open it + private function handleTypeTar( $moduleName, $destDir, array $info ) { + $info += [ 'src' => null, 'integrity' => null, 'dest' => null ]; + if ( $info['src'] === null ) { + $this->fatalError( "Module '$moduleName' must have a 'src' key." ); + } + // Download the resource to a temporary file and open it + $data = $this->fetch( $info['src'], $info['integrity' ] ); $tmpFile = "{$this->tmpParentDir}/$moduleName.tar"; $this->verbose( "... writing '$moduleName' src to $tmpFile\n" ); file_put_contents( $tmpFile, $data ); @@ -129,46 +150,45 @@ TEXT unset( $data, $p ); if ( $info['dest'] === null ) { - // Replace the entire directory as-is - if ( !$this->hasOption( 'update' ) ) { - $this->output( "[dry run] Would replace /resources/lib/$moduleName\n" ); - } else { - wfRecursiveRemoveDir( $destDir ); - if ( !rename( $tmpDir, $destDir ) ) { - $this->fatalError( "Could not move $destDir to $tmpDir." ); - } - } - return; - } - - // Create and/or empty the destination - if ( !$this->hasOption( 'update' ) ) { - $this->output( "... [dry run] would empty /resources/lib/$moduleName\n" ); + // Default: Replace the entire directory + $toCopy = [ $tmpDir => $destDir ]; } else { - wfRecursiveRemoveDir( $destDir ); - wfMkdirParents( $destDir ); - } - - // Expand and normalise the 'dest' entries - $toCopy = []; - foreach ( $info['dest'] as $fromSubPath => $toSubPath ) { - // Use glob() to expand wildcards and check existence - $fromPaths = glob( "{$tmpDir}/{$fromSubPath}", GLOB_BRACE ); - if ( !$fromPaths ) { - $this->fatalError( "Path '$fromSubPath' of '$moduleName' not found." ); - } - foreach ( $fromPaths as $fromPath ) { - $toCopy[$fromPath] = $toSubPath === null - ? "$destDir/" . basename( $fromPath ) - : "$destDir/$toSubPath/" . basename( $fromPath ); + // Expand and normalise the 'dest' entries + $toCopy = []; + foreach ( $info['dest'] as $fromSubPath => $toSubPath ) { + // Use glob() to expand wildcards and check existence + $fromPaths = glob( "{$tmpDir}/{$fromSubPath}", GLOB_BRACE ); + if ( !$fromPaths ) { + $this->fatalError( "Path '$fromSubPath' of '$moduleName' not found." ); + } + foreach ( $fromPaths as $fromPath ) { + $toCopy[$fromPath] = $toSubPath === null + ? "$destDir/" . basename( $fromPath ) + : "$destDir/$toSubPath/" . basename( $fromPath ); + } } } foreach ( $toCopy as $from => $to ) { - if ( !$this->hasOption( 'update' ) ) { - $shortFrom = strtr( $from, [ "$tmpDir/" => '' ] ); - $shortTo = strtr( $to, [ "$IP/" => '' ] ); - $this->output( "... [dry run] would move $shortFrom to $shortTo\n" ); - } else { + if ( $this->action === 'verify' ) { + $this->verbose( "... verifying $to\n" ); + if ( is_dir( $from ) ) { + $rii = new RecursiveIteratorIterator( new RecursiveDirectoryIterator( + $from, + RecursiveDirectoryIterator::SKIP_DOTS + ) ); + foreach ( $rii as $file ) { + $remote = $file->getPathname(); + $local = strtr( $remote, [ $from => $to ] ); + if ( sha1_file( $remote ) !== sha1_file( $local ) ) { + $this->error( "File '$local' is different." ); + $this->failAfterOutput = true; + } + } + } elseif ( sha1_file( $from ) !== sha1_file( $to ) ) { + $this->error( "File '$to' is different." ); + $this->failAfterOutput = true; + } + } elseif ( $this->action === 'update' ) { $this->verbose( "... moving $from to $to\n" ); wfMkdirParents( dirname( $to ) ); if ( !rename( $from, $to ) ) { @@ -184,6 +204,15 @@ TEXT } } + private function cleanUp() { + wfRecursiveRemoveDir( $this->tmpParentDir ); + } + + protected function fatalError( $msg, $exitCode = 1 ) { + $this->cleanUp(); + parent::fatalError( $msg, $exitCode ); + } + /** * Basic YAML parser. * -- 2.20.1