3 * This program is free software; you can redistribute it and/or modify
4 * it under the terms of the GNU General Public License as published by
5 * the Free Software Foundation; either version 2 of the License, or
6 * (at your option) any later version.
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
13 * You should have received a copy of the GNU General Public License along
14 * with this program; if not, write to the Free Software Foundation, Inc.,
15 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
16 * http://www.gnu.org/copyleft/gpl.html
19 * @ingroup Maintenance
23 * Manage foreign resources registered with ResourceLoader.
27 class ForeignResourceManager
{
28 private $defaultAlgo = 'sha384';
29 private $hasErrors = false;
30 private $registryFile;
32 private $tmpParentDir;
35 private $errorPrinter;
36 private $verbosePrinter;
41 * @param string $registryFile Path to YAML file
42 * @param string $libDir Path to a modules directory
43 * @param callable|null $infoPrinter Callback for printing info about the run.
44 * @param callable|null $errorPrinter Callback for printing errors from the run.
45 * @param callable|null $verbosePrinter Callback for printing extra verbose
46 * progress information from the run.
48 public function __construct(
51 callable
$infoPrinter = null,
52 callable
$errorPrinter = null,
53 callable
$verbosePrinter = null
55 $this->registryFile
= $registryFile;
56 $this->libDir
= $libDir;
57 $this->infoPrinter
= $infoPrinter ??
function () {
59 $this->errorPrinter
= $errorPrinter ??
$this->infoPrinter
;
60 $this->verbosePrinter
= $verbosePrinter ??
function () {
63 // Use a temporary directory under the destination directory instead
64 // of wfTempDir() because PHP's rename() does not work across file
65 // systems, and the user's /tmp and $IP may be on different filesystems.
66 $this->tmpParentDir
= "{$this->libDir}/.foreign/tmp";
68 $cacheHome = getenv( 'XDG_CACHE_HOME' ) ?
realpath( getenv( 'XDG_CACHE_HOME' ) ) : false;
69 $this->cacheDir
= $cacheHome ?
"$cacheHome/mw-foreign" : "{$this->libDir}/.foreign/cache";
76 public function run( $action, $module ) {
77 $actions = [ 'update', 'verify', 'make-sri' ];
78 if ( !in_array( $action, $actions ) ) {
79 $this->error( "Invalid action.\n\nMust be one of " . implode( ', ', $actions ) . '.' );
82 $this->action
= $action;
84 $this->registry
= $this->parseBasicYaml( file_get_contents( $this->registryFile
) );
85 if ( $module === 'all' ) {
86 $modules = $this->registry
;
87 } elseif ( isset( $this->registry
[ $module ] ) ) {
88 $modules = [ $module => $this->registry
[ $module ] ];
90 $this->error( "Unknown module name.\n\nMust be one of:\n" .
91 wordwrap( implode( ', ', array_keys( $this->registry
) ), 80 ) .
97 foreach ( $modules as $moduleName => $info ) {
98 $this->verbose( "\n### {$moduleName}\n\n" );
99 $destDir = "{$this->libDir}/$moduleName";
101 if ( $this->action
=== 'update' ) {
102 $this->output( "... updating '{$moduleName}'\n" );
103 $this->verbose( "... emptying directory for $moduleName\n" );
104 wfRecursiveRemoveDir( $destDir );
105 } elseif ( $this->action
=== 'verify' ) {
106 $this->output( "... verifying '{$moduleName}'\n" );
108 $this->output( "... checking '{$moduleName}'\n" );
111 $this->verbose( "... preparing {$this->tmpParentDir}\n" );
112 wfRecursiveRemoveDir( $this->tmpParentDir
);
113 if ( !wfMkdirParents( $this->tmpParentDir
) ) {
114 throw new Exception( "Unable to create {$this->tmpParentDir}" );
117 if ( !isset( $info['type'] ) ) {
118 throw new Exception( "Module '$moduleName' must have a 'type' key." );
120 switch ( $info['type'] ) {
122 $this->handleTypeTar( $moduleName, $destDir, $info );
125 $this->handleTypeFile( $moduleName, $destDir, $info );
128 $this->handleTypeMultiFile( $moduleName, $destDir, $info );
131 throw new Exception( "Unknown type '{$info['type']}' for '$moduleName'" );
135 $this->output( "\nDone!\n" );
137 if ( $this->hasErrors
) {
138 // The verify mode should check all modules/files and fail after, not during.
145 private function cacheKey( $src, $integrity ) {
146 $key = basename( $src ) . '_' . substr( $integrity, -12 );
147 $key = preg_replace( '/[.\/+?=_-]+/', '_', $key );
148 return rtrim( $key, '_' );
151 /** @return string|false */
152 private function cacheGet( $key ) {
153 return Wikimedia\
quietCall( 'file_get_contents', "{$this->cacheDir}/$key.data" );
156 private function cacheSet( $key, $data ) {
157 wfMkdirParents( $this->cacheDir
);
158 file_put_contents( "{$this->cacheDir}/$key.data", $data, LOCK_EX
);
161 private function fetch( $src, $integrity ) {
162 $key = $this->cacheKey( $src, $integrity );
163 $data = $this->cacheGet( $key );
168 $req = MWHttpRequest
::factory( $src, [ 'method' => 'GET', 'followRedirects' => false ] );
169 if ( !$req->execute()->isOK() ) {
170 throw new Exception( "Failed to download resource at {$src}" );
172 if ( $req->getStatus() !== 200 ) {
173 throw new Exception( "Unexpected HTTP {$req->getStatus()} response from {$src}" );
175 $data = $req->getContent();
176 $algo = $integrity === null ?
$this->defaultAlgo
: explode( '-', $integrity )[0];
177 $actualIntegrity = $algo . '-' . base64_encode( hash( $algo, $data, true ) );
178 if ( $integrity === $actualIntegrity ) {
179 $this->verbose( "... passed integrity check for {$src}\n" );
180 $this->cacheSet( $key, $data );
181 } elseif ( $this->action
=== 'make-sri' ) {
182 $this->output( "Integrity for {$src}\n\tintegrity: ${actualIntegrity}\n" );
184 throw new Exception( "Integrity check failed for {$src}\n" .
185 "\tExpected: {$integrity}\n" .
186 "\tActual: {$actualIntegrity}"
192 private function handleTypeFile( $moduleName, $destDir, array $info ) {
193 if ( !isset( $info['src'] ) ) {
194 throw new Exception( "Module '$moduleName' must have a 'src' key." );
196 $data = $this->fetch( $info['src'], $info['integrity'] ??
null );
197 $dest = $info['dest'] ??
basename( $info['src'] );
198 $path = "$destDir/$dest";
199 if ( $this->action
=== 'verify' && sha1_file( $path ) !== sha1( $data ) ) {
200 throw new Exception( "File for '$moduleName' is different." );
202 if ( $this->action
=== 'update' ) {
203 wfMkdirParents( $destDir );
204 file_put_contents( "$destDir/$dest", $data );
208 private function handleTypeMultiFile( $moduleName, $destDir, array $info ) {
209 if ( !isset( $info['files'] ) ) {
210 throw new Exception( "Module '$moduleName' must have a 'files' key." );
212 foreach ( $info['files'] as $dest => $file ) {
213 if ( !isset( $file['src'] ) ) {
214 throw new Exception( "Module '$moduleName' file '$dest' must have a 'src' key." );
216 $data = $this->fetch( $file['src'], $file['integrity'] ??
null );
217 $path = "$destDir/$dest";
218 if ( $this->action
=== 'verify' && sha1_file( $path ) !== sha1( $data ) ) {
219 throw new Exception( "File '$dest' for '$moduleName' is different." );
220 } elseif ( $this->action
=== 'update' ) {
221 wfMkdirParents( $destDir );
222 file_put_contents( "$destDir/$dest", $data );
227 private function handleTypeTar( $moduleName, $destDir, array $info ) {
228 $info +
= [ 'src' => null, 'integrity' => null, 'dest' => null ];
229 if ( $info['src'] === null ) {
230 throw new Exception( "Module '$moduleName' must have a 'src' key." );
232 // Download the resource to a temporary file and open it
233 $data = $this->fetch( $info['src'], $info['integrity' ] );
234 $tmpFile = "{$this->tmpParentDir}/$moduleName.tar";
235 $this->verbose( "... writing '$moduleName' src to $tmpFile\n" );
236 file_put_contents( $tmpFile, $data );
237 $p = new PharData( $tmpFile );
238 $tmpDir = "{$this->tmpParentDir}/$moduleName";
239 $p->extractTo( $tmpDir );
242 if ( $info['dest'] === null ) {
243 // Default: Replace the entire directory
244 $toCopy = [ $tmpDir => $destDir ];
246 // Expand and normalise the 'dest' entries
248 foreach ( $info['dest'] as $fromSubPath => $toSubPath ) {
249 // Use glob() to expand wildcards and check existence
250 $fromPaths = glob( "{$tmpDir}/{$fromSubPath}", GLOB_BRACE
);
252 throw new Exception( "Path '$fromSubPath' of '$moduleName' not found." );
254 foreach ( $fromPaths as $fromPath ) {
255 $toCopy[$fromPath] = $toSubPath === null
256 ?
"$destDir/" . basename( $fromPath )
257 : "$destDir/$toSubPath/" . basename( $fromPath );
261 foreach ( $toCopy as $from => $to ) {
262 if ( $this->action
=== 'verify' ) {
263 $this->verbose( "... verifying $to\n" );
264 if ( is_dir( $from ) ) {
265 $rii = new RecursiveIteratorIterator( new RecursiveDirectoryIterator(
267 RecursiveDirectoryIterator
::SKIP_DOTS
269 /** @var SplFileInfo $file */
270 foreach ( $rii as $file ) {
271 $remote = $file->getPathname();
272 $local = strtr( $remote, [ $from => $to ] );
273 if ( sha1_file( $remote ) !== sha1_file( $local ) ) {
274 $this->error( "File '$local' is different." );
275 $this->hasErrors
= true;
278 } elseif ( sha1_file( $from ) !== sha1_file( $to ) ) {
279 $this->error( "File '$to' is different." );
280 $this->hasErrors
= true;
282 } elseif ( $this->action
=== 'update' ) {
283 $this->verbose( "... moving $from to $to\n" );
284 wfMkdirParents( dirname( $to ) );
285 if ( !rename( $from, $to ) ) {
286 throw new Exception( "Could not move $from to $to." );
292 private function verbose( $text ) {
293 ( $this->verbosePrinter
)( $text );
296 private function output( $text ) {
297 ( $this->infoPrinter
)( $text );
300 private function error( $text ) {
301 ( $this->errorPrinter
)( $text );
304 private function cleanUp() {
305 wfRecursiveRemoveDir( $this->tmpParentDir
);
307 // Prune the cache of files we don't recognise.
309 foreach ( $this->registry
as $info ) {
310 if ( $info['type'] === 'file' ||
$info['type'] === 'tar' ) {
311 $knownKeys[] = $this->cacheKey( $info['src'], $info['integrity'] );
312 } elseif ( $info['type'] === 'multi-file' ) {
313 foreach ( $info['files'] as $file ) {
314 $knownKeys[] = $this->cacheKey( $file['src'], $file['integrity'] );
318 foreach ( glob( "{$this->cacheDir}/*" ) as $cacheFile ) {
319 if ( !in_array( basename( $cacheFile, '.data' ), $knownKeys ) ) {
320 unlink( $cacheFile );
328 * Supports only string or object values, and 2 spaces indentation.
330 * @todo Just ship symfony/yaml.
331 * @param string $input
334 private function parseBasicYaml( $input ) {
335 $lines = explode( "\n", $input );
339 foreach ( $lines as $i => $text ) {
341 $trimmed = ltrim( $text, ' ' );
342 if ( $trimmed === '' ||
$trimmed[0] === '#' ) {
345 $indent = strlen( $text ) - strlen( $trimmed );
346 if ( $indent %
2 !== 0 ) {
347 throw new Exception( __METHOD__
. ": Odd indentation on line $line." );
349 $depth = $indent === 0 ?
0 : ( $indent / 2 );
350 if ( $depth < $prev ) {
351 // Close previous branches we can't re-enter
352 array_splice( $stack, $depth +
1 );
354 if ( !array_key_exists( $depth, $stack ) ) {
355 throw new Exception( __METHOD__
. ": Too much indentation on line $line." );
357 if ( strpos( $trimmed, ':' ) === false ) {
358 throw new Exception( __METHOD__
. ": Missing colon on line $line." );
360 $dest =& $stack[ $depth ];
361 if ( $dest === null ) {
362 // Promote from null to object
365 list( $key, $val ) = explode( ':', $trimmed, 2 );
366 $val = ltrim( $val, ' ' );
369 $dest[ $key ] = $val;
371 // Add null (may become an object later)
374 $dest[ $key ] = &$val;
377 unset( $dest, $val );