maintenance/resources/manageForeignResources.php

   1 <?php
   2 /**
   3  * This program is free software; you can redistribute it and/or modify
   4  * it under the terms of the GNU General Public License as published by
   5  * the Free Software Foundation; either version 2 of the License, or
   6  * (at your option) any later version.
   7  *
   8  * This program is distributed in the hope that it will be useful,
   9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  11  * GNU General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public License along
  14  * with this program; if not, write to the Free Software Foundation, Inc.,
  15  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  16  * http://www.gnu.org/copyleft/gpl.html
  17  *
  18  * @file
  19  * @ingroup Maintenance
  20  */
  21
  22 require_once __DIR__ . '/../Maintenance.php';
  23
  24 /**
  25  * Manage foreign resources registered with ResourceLoader.
  26  *
  27  * @ingroup Maintenance
  28  * @since 1.32
  29  */
  30 class ManageForeignResources extends Maintenance {
  31         private $defaultAlgo = 'sha384';
  32         private $tmpParentDir;
  33         private $action;
  34         private $failAfterOutput = false;
  35
  36         public function __construct() {
  37                 global $IP;
  38                 parent::__construct();
  39                 $this->addDescription( <<<TEXT
  40 Manage foreign resources registered with ResourceLoader.
  41
  42 This helps developers to download, verify and update local copies of upstream
  43 libraries registered as ResourceLoader modules. See also foreign-resources.yaml.
  44
  45 For sources that don't publish an integrity hash, omit "integrity" (or leave empty)
  46 and run the "make-sri" action to compute the missing hashes.
  47
  48 This script runs in dry mode by default. Use --update to actually change, remove,
  49 or add files to /resources/lib/.
  50 TEXT
  51                 );
  52                 $this->addArg( 'action', 'One of "update", "verify" or "make-sri"', true );
  53                 $this->addArg( 'module', 'Name of a single module (Default: all)', false );
  54                 $this->addOption( 'verbose', 'Be verbose', false, false, 'v' );
  55
  56                 // Use a directory in $IP instead of wfTempDir() because
  57                 // PHP's rename() does not work across file systems.
  58                 $this->tmpParentDir = "{$IP}/resources/tmp";
  59         }
  60
  61         public function execute() {
  62                 global $IP;
  63                 $this->action = $this->getArg( 0 );
  64                 if ( !in_array( $this->action, [ 'update', 'verify', 'make-sri' ] ) ) {
  65                         $this->fatalError( "Invalid action argument." );
  66                 }
  67
  68                 $registry = $this->parseBasicYaml(
  69                         file_get_contents( __DIR__ . '/foreign-resources.yaml' )
  70                 );
  71                 $module = $this->getArg( 1, 'all' );
  72                 foreach ( $registry as $moduleName => $info ) {
  73                         if ( $module !== 'all' && $moduleName !== $module ) {
  74                                 continue;
  75                         }
  76                         $this->verbose( "\n### {$moduleName}\n\n" );
  77                         $destDir = "{$IP}/resources/lib/$moduleName";
  78
  79                         if ( $this->action === 'update' ) {
  80                                 $this->output( "... updating '{$moduleName}'\n" );
  81                                 $this->verbose( "... emptying /resources/lib/$moduleName\n" );
  82                                 wfRecursiveRemoveDir( $destDir );
  83                         } elseif ( $this->action === 'verify' ) {
  84                                 $this->output( "... verifying '{$moduleName}'\n" );
  85                         } else {
  86                                 $this->output( "... checking '{$moduleName}'\n" );
  87                         }
  88
  89                         $this->verbose( "... preparing {$this->tmpParentDir}\n" );
  90                         wfRecursiveRemoveDir( $this->tmpParentDir );
  91                         if ( !wfMkdirParents( $this->tmpParentDir ) ) {
  92                                 $this->fatalError( "Unable to create {$this->tmpParentDir}" );
  93                         }
  94
  95                         if ( !isset( $info['type'] ) ) {
  96                                 $this->fatalError( "Module '$moduleName' must have a 'type' key." );
  97                         }
  98                         switch ( $info['type'] ) {
  99                                 case 'tar':
 100                                         $this->handleTypeTar( $moduleName, $destDir, $info );
 101                                         break;
 102                                 default:
 103                                         $this->fatalError( "Unknown type '{$info['type']}' for '$moduleName'" );
 104                         }
 105                 }
 106
 107                 $this->cleanUp();
 108                 $this->output( "\nDone!\n" );
 109                 if ( $this->failAfterOutput ) {
 110                         // The verify mode should check all modules/files and fail after, not during.
 111                         return false;
 112                 }
 113         }
 114
 115         private function fetch( $src, $integrity ) {
 116                 $data = Http::get( $src, [ 'followRedirects' => false ] );
 117                 if ( $data === false ) {
 118                         $this->fatalError( "Failed to download resource at {$src}" );
 119                 }
 120                 $algo = $integrity === null ? $this->defaultAlgo : explode( '-', $integrity )[0];
 121                 $actualIntegrity = $algo . '-' . base64_encode( hash( $algo, $data, true ) );
 122                 if ( $integrity === $actualIntegrity ) {
 123                         $this->verbose( "... passed integrity check for {$src}\n" );
 124                 } else {
 125                         if ( $this->action === 'make-sri' ) {
 126                                 $this->output( "Integrity for {$src}\n\tintegrity: ${actualIntegrity}\n" );
 127                         } else {
 128                                 $this->fatalError( "Integrity check failed for {$src}\n" .
 129                                         "\tExpected: {$integrity}\n" .
 130                                         "\tActual: {$actualIntegrity}"
 131                                 );
 132                         }
 133                 }
 134                 return $data;
 135         }
 136
 137         private function handleTypeTar( $moduleName, $destDir, array $info ) {
 138                 $info += [ 'src' => null, 'integrity' => null, 'dest' => null ];
 139                 if ( $info['src'] === null ) {
 140                         $this->fatalError( "Module '$moduleName' must have a 'src' key." );
 141                 }
 142                 // Download the resource to a temporary file and open it
 143                 $data = $this->fetch( $info['src'], $info['integrity' ] );
 144                 $tmpFile = "{$this->tmpParentDir}/$moduleName.tar";
 145                 $this->verbose( "... writing '$moduleName' src to $tmpFile\n" );
 146                 file_put_contents( $tmpFile, $data );
 147                 $p = new PharData( $tmpFile );
 148                 $tmpDir = "{$this->tmpParentDir}/$moduleName";
 149                 $p->extractTo( $tmpDir );
 150                 unset( $data, $p );
 151
 152                 if ( $info['dest'] === null ) {
 153                         // Default: Replace the entire directory
 154                         $toCopy = [ $tmpDir => $destDir ];
 155                 } else {
 156                         // Expand and normalise the 'dest' entries
 157                         $toCopy = [];
 158                         foreach ( $info['dest'] as $fromSubPath => $toSubPath ) {
 159                                 // Use glob() to expand wildcards and check existence
 160                                 $fromPaths = glob( "{$tmpDir}/{$fromSubPath}", GLOB_BRACE );
 161                                 if ( !$fromPaths ) {
 162                                         $this->fatalError( "Path '$fromSubPath' of '$moduleName' not found." );
 163                                 }
 164                                 foreach ( $fromPaths as $fromPath ) {
 165                                         $toCopy[$fromPath] = $toSubPath === null
 166                                                 ? "$destDir/" . basename( $fromPath )
 167                                                 : "$destDir/$toSubPath/" . basename( $fromPath );
 168                                 }
 169                         }
 170                 }
 171                 foreach ( $toCopy as $from => $to ) {
 172                         if ( $this->action === 'verify' ) {
 173                                 $this->verbose( "... verifying $to\n" );
 174                                 if ( is_dir( $from ) ) {
 175                                         $rii = new RecursiveIteratorIterator( new RecursiveDirectoryIterator(
 176                                                 $from,
 177                                                 RecursiveDirectoryIterator::SKIP_DOTS
 178                                         ) );
 179                                         foreach ( $rii as $file ) {
 180                                                 $remote = $file->getPathname();
 181                                                 $local = strtr( $remote, [ $from => $to ] );
 182                                                 if ( sha1_file( $remote ) !== sha1_file( $local ) ) {
 183                                                         $this->error( "File '$local' is different." );
 184                                                         $this->failAfterOutput = true;
 185                                                 }
 186                                         }
 187                                 } elseif ( sha1_file( $from ) !== sha1_file( $to ) ) {
 188                                         $this->error( "File '$to' is different." );
 189                                         $this->failAfterOutput = true;
 190                                 }
 191                         } elseif ( $this->action === 'update' ) {
 192                                 $this->verbose( "... moving $from to $to\n" );
 193                                 wfMkdirParents( dirname( $to ) );
 194                                 if ( !rename( $from, $to ) ) {
 195                                         $this->fatalError( "Could not move $from to $to." );
 196                                 }
 197                         }
 198                 }
 199         }
 200
 201         private function verbose( $text ) {
 202                 if ( $this->hasOption( 'verbose' ) ) {
 203                         $this->output( $text );
 204                 }
 205         }
 206
 207         private function cleanUp() {
 208                 wfRecursiveRemoveDir( $this->tmpParentDir );
 209         }
 210
 211         protected function fatalError( $msg, $exitCode = 1 ) {
 212                 $this->cleanUp();
 213                 parent::fatalError( $msg, $exitCode );
 214         }
 215
 216         /**
 217          * Basic YAML parser.
 218          *
 219          * Supports only string or object values, and 2 spaces indentation.
 220          *
 221          * @todo Just ship symfony/yaml.
 222          * @param string $input
 223          * @return array
 224          */
 225         private function parseBasicYaml( $input ) {
 226                 $lines = explode( "\n", $input );
 227                 $root = [];
 228                 $stack = [ &$root ];
 229                 $prev = 0;
 230                 foreach ( $lines as $i => $text ) {
 231                         $line = $i + 1;
 232                         $trimmed = ltrim( $text, ' ' );
 233                         if ( $trimmed === '' || $trimmed[0] === '#' ) {
 234                                 continue;
 235                         }
 236                         $indent = strlen( $text ) - strlen( $trimmed );
 237                         if ( $indent % 2 !== 0 ) {
 238                                 throw new Exception( __METHOD__ . ": Odd indentation on line $line." );
 239                         }
 240                         $depth = $indent === 0 ? 0 : ( $indent / 2 );
 241                         if ( $depth < $prev ) {
 242                                 // Close previous branches we can't re-enter
 243                                 array_splice( $stack, $depth + 1 );
 244                         }
 245                         if ( !array_key_exists( $depth, $stack ) ) {
 246                                 throw new Exception( __METHOD__ . ": Too much indentation on line $line." );
 247                         }
 248                         if ( strpos( $trimmed, ':' ) === false ) {
 249                                 throw new Exception( __METHOD__ . ": Missing colon on line $line." );
 250                         }
 251                         $dest =& $stack[ $depth ];
 252                         if ( $dest === null ) {
 253                                 // Promote from null to object
 254                                 $dest = [];
 255                         }
 256                         list( $key, $val ) = explode( ':', $trimmed, 2 );
 257                         $val = ltrim( $val, ' ' );
 258                         if ( $val !== '' ) {
 259                                 // Add string
 260                                 $dest[ $key ] = $val;
 261                         } else {
 262                                 // Add null (may become an object later)
 263                                 $val = null;
 264                                 $stack[] = &$val;
 265                                 $dest[ $key ] = &$val;
 266                         }
 267                         $prev = $depth;
 268                         unset( $dest, $val );
 269                 }
 270                 return $root;
 271         }
 272 }
 273
 274 $maintClass = ManageForeignResources::class;
 275 require_once RUN_MAINTENANCE_IF_MAIN;