3 * ZIP file directories reader, for the purposes of upload verification.
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License along
16 * with this program; if not, write to the Free Software Foundation, Inc.,
17 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18 * http://www.gnu.org/copyleft/gpl.html
24 * A class for reading ZIP file directories, for the purposes of upload
27 * Only a functional interface is provided: ZipFileReader::read(). No access is
28 * given to object instances.
30 class ZipDirectoryReader
{
32 * Read a ZIP file and call a function for each file discovered in it.
34 * Because this class is aimed at verification, an error is raised on
35 * suspicious or ambiguous input, instead of emulating some standard
38 * @param string $fileName The archive file name
39 * @param array $callback The callback function. It will be called for each file
40 * with a single associative array each time, with members:
42 * - name: The file name. Directories conventionally have a trailing
45 * - mtime: The file modification time, in MediaWiki 14-char format
47 * - size: The uncompressed file size
49 * @param array $options An associative array of read options, with the option
50 * name in the key. This may currently contain:
52 * - zip64: If this is set to true, then we will emulate a
53 * library with ZIP64 support, like OpenJDK 7. If it is set to
54 * false, then we will emulate a library with no knowledge of
57 * NOTE: The ZIP64 code is untested and probably doesn't work. It
58 * turned out to be easier to just reject ZIP64 archive uploads,
59 * since they are likely to be very rare. Confirming safety of a
60 * ZIP64 file is fairly complex. What do you do with a file that is
61 * ambiguous and broken when read with a non-ZIP64 reader, but valid
62 * when read with a ZIP64 reader? This situation is normal for a
63 * valid ZIP64 file, and working out what non-ZIP64 readers will make
64 * of such a file is not trivial.
66 * @return Status A Status object. The following fatal errors are defined:
68 * - zip-file-open-error: The file could not be opened.
70 * - zip-wrong-format: The file does not appear to be a ZIP file.
72 * - zip-bad: There was something wrong or ambiguous about the file
75 * - zip-unsupported: The ZIP file uses features which
76 * ZipDirectoryReader does not support.
78 * The default messages for those fatal errors are written in a way that
79 * makes sense for upload verification.
81 * If a fatal error is returned, more information about the error will be
82 * available in the debug log.
84 * Note that the callback function may be called any number of times before
85 * a fatal error is returned. If this occurs, the data sent to the callback
86 * function should be discarded.
88 public static function read( $fileName, $callback, $options = [] ) {
89 $zdr = new self( $fileName, $callback, $options );
91 return $zdr->execute();
97 /** The opened file resource */
100 /** The cached length of the file, or null if it has not been loaded yet. */
101 protected $fileLength;
103 /** A segmented cache of the file contents */
106 /** The file data callback */
109 /** The ZIP64 mode */
110 protected $zip64 = false;
112 /** Stored headers */
113 protected $eocdr, $eocdr64, $eocdr64Locator;
117 /** The "extra field" ID for ZIP64 central directory entries */
118 const ZIP64_EXTRA_HEADER
= 0x0001;
120 /** The segment size for the file contents cache */
121 const SEGSIZE
= 16384;
123 /** The index of the "general field" bit for UTF-8 file names */
124 const GENERAL_UTF8
= 11;
126 /** The index of the "general field" bit for central directory encryption */
127 const GENERAL_CD_ENCRYPTED
= 13;
130 * @param string $fileName
131 * @param callable $callback
132 * @param array $options
134 protected function __construct( $fileName, $callback, $options ) {
135 $this->fileName
= $fileName;
136 $this->callback
= $callback;
138 if ( isset( $options['zip64'] ) ) {
139 $this->zip64
= $options['zip64'];
144 * Read the directory according to settings in $this.
149 $this->file
= fopen( $this->fileName
, 'r' );
151 if ( !$this->file
) {
152 return Status
::newFatal( 'zip-file-open-error' );
155 $status = Status
::newGood();
157 $this->readEndOfCentralDirectoryRecord();
158 if ( $this->zip64
) {
159 list( $offset, $size ) = $this->findZip64CentralDirectory();
160 $this->readCentralDirectory( $offset, $size );
162 if ( $this->eocdr
['CD size'] == 0xffffffff
163 ||
$this->eocdr
['CD offset'] == 0xffffffff
164 ||
$this->eocdr
['CD entries total'] == 0xffff
166 $this->error( 'zip-unsupported', 'Central directory header indicates ZIP64, ' .
167 'but we are in legacy mode. Rejecting this upload is necessary to avoid ' .
168 'opening vulnerabilities on clients using OpenJDK 7 or later.' );
171 list( $offset, $size ) = $this->findOldCentralDirectory();
172 $this->readCentralDirectory( $offset, $size );
174 } catch ( ZipDirectoryReaderError
$e ) {
175 $status->fatal( $e->getErrorCode() );
178 fclose( $this->file
);
184 * Throw an error, and log a debug message
186 * @param string $debugMessage
187 * @throws ZipDirectoryReaderError
189 function error( $code, $debugMessage ) {
190 wfDebug( __CLASS__
. ": Fatal error: $debugMessage\n" );
191 throw new ZipDirectoryReaderError( $code );
195 * Read the header which is at the end of the central directory,
196 * unimaginatively called the "end of central directory record" by the ZIP
199 function readEndOfCentralDirectoryRecord() {
203 'CD start disk' => 2,
204 'CD entries this disk' => 2,
205 'CD entries total' => 2,
208 'file comment length' => 2,
210 $structSize = $this->getStructSize( $info );
211 $startPos = $this->getFileLength() - 65536 - $structSize;
212 if ( $startPos < 0 ) {
216 if ( $this->getFileLength() === 0 ) {
217 $this->error( 'zip-wrong-format', "The file is empty." );
220 $block = $this->getBlock( $startPos );
221 $sigPos = strrpos( $block, "PK\x05\x06" );
222 if ( $sigPos === false ) {
223 $this->error( 'zip-wrong-format',
224 "zip file lacks EOCDR signature. It probably isn't a zip file." );
227 $this->eocdr
= $this->unpack( substr( $block, $sigPos ), $info );
228 $this->eocdr
['EOCDR size'] = $structSize +
$this->eocdr
['file comment length'];
230 if ( $structSize +
$this->eocdr
['file comment length'] != strlen( $block ) - $sigPos ) {
231 $this->error( 'zip-bad', 'trailing bytes after the end of the file comment' );
233 if ( $this->eocdr
['disk'] !== 0
234 ||
$this->eocdr
['CD start disk'] !== 0
236 $this->error( 'zip-unsupported', 'more than one disk (in EOCDR)' );
238 $this->eocdr +
= $this->unpack(
240 [ 'file comment' => [ 'string', $this->eocdr
['file comment length'] ] ],
241 $sigPos +
$structSize );
242 $this->eocdr
['position'] = $startPos +
$sigPos;
246 * Read the header called the "ZIP64 end of central directory locator". An
247 * error will be raised if it does not exist.
249 function readZip64EndOfCentralDirectoryLocator() {
251 'signature' => [ 'string', 4 ],
252 'eocdr64 start disk' => 4,
253 'eocdr64 offset' => 8,
254 'number of disks' => 4,
256 $structSize = $this->getStructSize( $info );
258 $start = $this->getFileLength() - $this->eocdr
['EOCDR size'] - $structSize;
259 $block = $this->getBlock( $start, $structSize );
260 $this->eocdr64Locator
= $data = $this->unpack( $block, $info );
262 if ( $data['signature'] !== "PK\x06\x07" ) {
263 // Note: Java will allow this and continue to read the
264 // EOCDR64, so we have to reject the upload, we can't
265 // just use the EOCDR header instead.
266 $this->error( 'zip-bad', 'wrong signature on Zip64 end of central directory locator' );
271 * Read the header called the "ZIP64 end of central directory record". It
272 * may replace the regular "end of central directory record" in ZIP64 files.
274 function readZip64EndOfCentralDirectoryRecord() {
275 if ( $this->eocdr64Locator
['eocdr64 start disk'] != 0
276 ||
$this->eocdr64Locator
['number of disks'] != 0
278 $this->error( 'zip-unsupported', 'more than one disk (in EOCDR64 locator)' );
282 'signature' => [ 'string', 4 ],
284 'version made by' => 2,
285 'version needed' => 2,
287 'CD start disk' => 4,
288 'CD entries this disk' => 8,
289 'CD entries total' => 8,
293 $structSize = $this->getStructSize( $info );
294 $block = $this->getBlock( $this->eocdr64Locator
['eocdr64 offset'], $structSize );
295 $this->eocdr64
= $data = $this->unpack( $block, $info );
296 if ( $data['signature'] !== "PK\x06\x06" ) {
297 $this->error( 'zip-bad', 'wrong signature on Zip64 end of central directory record' );
299 if ( $data['disk'] !== 0
300 ||
$data['CD start disk'] !== 0
302 $this->error( 'zip-unsupported', 'more than one disk (in EOCDR64)' );
307 * Find the location of the central directory, as would be seen by a
310 * @return array List containing offset, size and end position.
312 function findOldCentralDirectory() {
313 $size = $this->eocdr
['CD size'];
314 $offset = $this->eocdr
['CD offset'];
315 $endPos = $this->eocdr
['position'];
317 // Some readers use the EOCDR position instead of the offset field
318 // to find the directory, so to be safe, we check if they both agree.
319 if ( $offset +
$size != $endPos ) {
320 $this->error( 'zip-bad', 'the central directory does not immediately precede the end ' .
321 'of central directory record' );
324 return [ $offset, $size ];
328 * Find the location of the central directory, as would be seen by a
329 * ZIP64-compliant reader.
331 * @return array List containing offset, size and end position.
333 function findZip64CentralDirectory() {
334 // The spec is ambiguous about the exact rules of precedence between the
335 // ZIP64 headers and the original headers. Here we follow zip_util.c
337 $size = $this->eocdr
['CD size'];
338 $offset = $this->eocdr
['CD offset'];
339 $numEntries = $this->eocdr
['CD entries total'];
340 $endPos = $this->eocdr
['position'];
341 if ( $size == 0xffffffff
342 ||
$offset == 0xffffffff
343 ||
$numEntries == 0xffff
345 $this->readZip64EndOfCentralDirectoryLocator();
347 if ( isset( $this->eocdr64Locator
['eocdr64 offset'] ) ) {
348 $this->readZip64EndOfCentralDirectoryRecord();
349 if ( isset( $this->eocdr64
['CD offset'] ) ) {
350 $size = $this->eocdr64
['CD size'];
351 $offset = $this->eocdr64
['CD offset'];
352 $endPos = $this->eocdr64Locator
['eocdr64 offset'];
356 // Some readers use the EOCDR position instead of the offset field
357 // to find the directory, so to be safe, we check if they both agree.
358 if ( $offset +
$size != $endPos ) {
359 $this->error( 'zip-bad', 'the central directory does not immediately precede the end ' .
360 'of central directory record' );
363 return [ $offset, $size ];
367 * Read the central directory at the given location
371 function readCentralDirectory( $offset, $size ) {
372 $block = $this->getBlock( $offset, $size );
375 'signature' => [ 'string', 4 ],
376 'version made by' => 2,
377 'version needed' => 2,
379 'compression method' => 2,
383 'compressed size' => 4,
384 'uncompressed size' => 4,
386 'extra field length' => 2,
387 'comment length' => 2,
388 'disk number start' => 2,
389 'internal attrs' => 2,
390 'external attrs' => 4,
391 'local header offset' => 4,
393 $fixedSize = $this->getStructSize( $fixedInfo );
396 while ( $pos < $size ) {
397 $data = $this->unpack( $block, $fixedInfo, $pos );
400 if ( $data['signature'] !== "PK\x01\x02" ) {
401 $this->error( 'zip-bad', 'Invalid signature found in directory entry' );
405 'name' => [ 'string', $data['name length'] ],
406 'extra field' => [ 'string', $data['extra field length'] ],
407 'comment' => [ 'string', $data['comment length'] ],
409 $data +
= $this->unpack( $block, $variableInfo, $pos );
410 $pos +
= $this->getStructSize( $variableInfo );
412 if ( $this->zip64
&& (
413 $data['compressed size'] == 0xffffffff
414 ||
$data['uncompressed size'] == 0xffffffff
415 ||
$data['local header offset'] == 0xffffffff )
417 $zip64Data = $this->unpackZip64Extra( $data['extra field'] );
419 $data = $zip64Data +
$data;
423 if ( $this->testBit( $data['general bits'], self
::GENERAL_CD_ENCRYPTED
) ) {
424 $this->error( 'zip-unsupported', 'central directory encryption is not supported' );
427 // Convert the timestamp into MediaWiki format
428 // For the format, please see the MS-DOS 2.0 Programmer's Reference,
429 // pages 3-5 and 3-6.
430 $time = $data['mod time'];
431 $date = $data['mod date'];
433 $year = 1980 +
( $date >> 9 );
434 $month = ( $date >> 5 ) & 15;
436 $hour = ( $time >> 11 ) & 31;
437 $minute = ( $time >> 5 ) & 63;
438 $second = ( $time & 31 ) * 2;
439 $timestamp = sprintf( "%04d%02d%02d%02d%02d%02d",
440 $year, $month, $day, $hour, $minute, $second );
442 // Convert the character set in the file name
443 if ( $this->testBit( $data['general bits'], self
::GENERAL_UTF8
) ) {
444 $name = $data['name'];
446 $name = iconv( 'CP437', 'UTF-8', $data['name'] );
449 // Compile a data array for the user, with a sensible format
452 'mtime' => $timestamp,
453 'size' => $data['uncompressed size'],
455 call_user_func( $this->callback
, $userData );
460 * Interpret ZIP64 "extra field" data and return an associative array.
461 * @param string $extraField
464 function unpackZip64Extra( $extraField ) {
469 $extraHeaderSize = $this->getStructSize( $extraHeaderInfo );
472 'uncompressed size' => 8,
473 'compressed size' => 8,
474 'local header offset' => 8,
475 'disk number start' => 4,
479 while ( $extraPos < strlen( $extraField ) ) {
480 $extra = $this->unpack( $extraField, $extraHeaderInfo, $extraPos );
481 $extraPos +
= $extraHeaderSize;
482 $extra +
= $this->unpack( $extraField,
483 [ 'data' => [ 'string', $extra['size'] ] ],
485 $extraPos +
= $extra['size'];
487 if ( $extra['id'] == self
::ZIP64_EXTRA_HEADER
) {
488 return $this->unpack( $extra['data'], $zip64ExtraInfo );
496 * Get the length of the file.
499 function getFileLength() {
500 if ( $this->fileLength
=== null ) {
501 $stat = fstat( $this->file
);
502 $this->fileLength
= $stat['size'];
505 return $this->fileLength
;
509 * Get the file contents from a given offset. If there are not enough bytes
510 * in the file to satisfy the request, an exception will be thrown.
512 * @param int $start The byte offset of the start of the block.
513 * @param int|null $length The number of bytes to return. If omitted, the remainder
514 * of the file will be returned.
518 function getBlock( $start, $length = null ) {
519 $fileLength = $this->getFileLength();
520 if ( $start >= $fileLength ) {
521 $this->error( 'zip-bad', "getBlock() requested position $start, " .
522 "file length is $fileLength" );
524 if ( $length === null ) {
525 $length = $fileLength - $start;
527 $end = $start +
$length;
528 if ( $end > $fileLength ) {
529 $this->error( 'zip-bad', "getBlock() requested end position $end, " .
530 "file length is $fileLength" );
532 $startSeg = floor( $start / self
::SEGSIZE
);
533 $endSeg = ceil( $end / self
::SEGSIZE
);
536 for ( $segIndex = $startSeg; $segIndex <= $endSeg; $segIndex++
) {
537 $block .= $this->getSegment( $segIndex );
540 $block = substr( $block,
541 $start - $startSeg * self
::SEGSIZE
,
544 if ( strlen( $block ) < $length ) {
545 $this->error( 'zip-bad', 'getBlock() returned an unexpectedly small amount of data' );
552 * Get a section of the file starting at position $segIndex * self::SEGSIZE,
553 * of length self::SEGSIZE. The result is cached. This is a helper function
556 * If there are not enough bytes in the file to satisfy the request, the
557 * return value will be truncated. If a request is made for a segment beyond
558 * the end of the file, an empty string will be returned.
560 * @param int $segIndex
564 function getSegment( $segIndex ) {
565 if ( !isset( $this->buffer
[$segIndex] ) ) {
566 $bytePos = $segIndex * self
::SEGSIZE
;
567 if ( $bytePos >= $this->getFileLength() ) {
568 $this->buffer
[$segIndex] = '';
572 if ( fseek( $this->file
, $bytePos ) ) {
573 $this->error( 'zip-bad', "seek to $bytePos failed" );
575 $seg = fread( $this->file
, self
::SEGSIZE
);
576 if ( $seg === false ) {
577 $this->error( 'zip-bad', "read from $bytePos failed" );
579 $this->buffer
[$segIndex] = $seg;
582 return $this->buffer
[$segIndex];
586 * Get the size of a structure in bytes. See unpack() for the format of $struct.
587 * @param array $struct
590 function getStructSize( $struct ) {
592 foreach ( $struct as $type ) {
593 if ( is_array( $type ) ) {
594 list( , $fieldSize ) = $type;
605 * Unpack a binary structure. This is like the built-in unpack() function
608 * @param string $string The binary data input
610 * @param array $struct An associative array giving structure members and their
611 * types. In the key is the field name. The value may be either an
612 * integer, in which case the field is a little-endian unsigned integer
613 * encoded in the given number of bytes, or an array, in which case the
614 * first element of the array is the type name, and the subsequent
615 * elements are type-dependent parameters. Only one such type is defined:
616 * - "string": The second array element gives the length of string.
617 * Not null terminated.
619 * @param int $offset The offset into the string at which to start unpacking.
621 * @throws MWException
622 * @return array Unpacked associative array. Note that large integers in the input
623 * may be represented as floating point numbers in the return value, so
624 * the use of weak comparison is advised.
626 function unpack( $string, $struct, $offset = 0 ) {
627 $size = $this->getStructSize( $struct );
628 if ( $offset +
$size > strlen( $string ) ) {
629 $this->error( 'zip-bad', 'unpack() would run past the end of the supplied string' );
634 foreach ( $struct as $key => $type ) {
635 if ( is_array( $type ) ) {
636 list( $typeName, $fieldSize ) = $type;
637 switch ( $typeName ) {
639 $data[$key] = substr( $string, $pos, $fieldSize );
643 throw new MWException( __METHOD__
. ": invalid type \"$typeName\"" );
646 // Unsigned little-endian integer
647 $length = intval( $type );
649 // Calculate the value. Use an algorithm which automatically
650 // upgrades the value to floating point if necessary.
652 for ( $i = $length - 1; $i >= 0; $i-- ) {
654 $value +
= ord( $string[$pos +
$i] );
657 // Throw an exception if there was loss of precision
658 if ( $value > 2 ** 52 ) {
659 $this->error( 'zip-unsupported', 'number too large to be stored in a double. ' .
660 'This could happen if we tried to unpack a 64-bit structure ' .
661 'at an invalid location.' );
663 $data[$key] = $value;
672 * Returns a bit from a given position in an integer value, converted to
676 * @param int $bitIndex The index of the bit, where 0 is the LSB.
679 function testBit( $value, $bitIndex ) {
680 return (bool)( ( $value >> $bitIndex ) & 1 );
684 * Debugging helper function which dumps a string in hexdump -C format.
687 function hexDump( $s ) {
689 for ( $i = 0; $i < $n; $i +
= 16 ) {
690 printf( "%08X ", $i );
691 for ( $j = 0; $j < 16; $j++
) {
696 if ( $i +
$j >= $n ) {
699 printf( "%02X", ord( $s[$i +
$j] ) );
704 for ( $j = 0; $j < 16; $j++
) {
705 if ( $i +
$j >= $n ) {
707 } elseif ( ctype_print( $s[$i +
$j] ) ) {