From 9b3128eb2b654761f21fd4ca1d5a1a4b796dc912 Mon Sep 17 00:00:00 2001 From: Bryan Tong Minh Date: Tue, 12 Apr 2011 19:25:56 +0000 Subject: [PATCH] Add support for importing/exporting files. This can be done by embedding the image as base64 in the XML stream or by copying the images directory manually and pointing the importer to the base images directory. Currently only backend code available and a few member variables need to be modified to enable the functionality. Export.php: * Add and elememnts to the XML output * Add optional and elements to the XML output. contains an encoding attribute, which is currently only set to base64. Import.php: * Add Import::$mImageBasePath which should point to the images/ directory to import from * Add methods to WikiRevision (terrible name btw) to set the rel, hash, archivename and filesrc. * Cleanup and made WikiRevision::importUpload working. It's still quite a mess though OldLocalFiel.php: * Fix a few timestamp related things from r85635 --- includes/Export.php | 32 ++++-- includes/Import.php | 152 ++++++++++++++++++----------- includes/filerepo/OldLocalFile.php | 12 ++- 3 files changed, 130 insertions(+), 66 deletions(-) diff --git a/includes/Export.php b/includes/Export.php index 65743b2f06..3a9180a810 100644 --- a/includes/Export.php +++ b/includes/Export.php @@ -35,6 +35,7 @@ class WikiExporter { var $author_list = "" ; var $dumpUploads = false; + var $dumpUploadFileContents = false; const FULL = 1; const CURRENT = 2; @@ -318,7 +319,7 @@ class WikiExporter { if ( isset( $last ) ) { $output = ''; if ( $this->dumpUploads ) { - $output .= $this->writer->writeUploads( $last ); + $output .= $this->writer->writeUploads( $last, $this->dumpUploadFileContents ); } $output .= $this->writer->closePage(); $this->sink->writeClosePage( $output ); @@ -333,7 +334,7 @@ class WikiExporter { if ( isset( $last ) ) { $output = ''; if ( $this->dumpUploads ) { - $output .= $this->writer->writeUploads( $last ); + $output .= $this->writer->writeUploads( $last, $this->dumpUploadFileContents ); } $output .= $this->author_list; $output .= $this->writer->closePage(); @@ -600,29 +601,48 @@ class XmlDumpWriter { /** * Warning! This data is potentially inconsistent. :( */ - function writeUploads( $row ) { + function writeUploads( $row, $dumpContents = false ) { if ( $row->page_namespace == NS_IMAGE ) { $img = wfFindFile( $row->page_title ); if ( $img ) { $out = ''; foreach ( array_reverse( $img->getHistory() ) as $ver ) { - $out .= $this->writeUpload( $ver ); + $out .= $this->writeUpload( $ver, $dumpContents ); } - $out .= $this->writeUpload( $img ); + $out .= $this->writeUpload( $img, $dumpContents ); return $out; } } return ''; } - function writeUpload( $file ) { + function writeUpload( $file, $dumpContents = false ) { + if ( $file->isOld() ) { + $archiveName = " " . + Xml::element( 'archivename', null, $file->getArchiveName() ) . "\n"; + } else { + $archiveName = ''; + } + if ( $dumpContents ) { + # Dump file as base64 + # Uses only XML-safe characters, so does not need escaping + $contents = ' ' . + chunk_split( base64_encode( file_get_contents( $file->getPath() ) ) ) . + " \n"; + } else { + $contents = ''; + } return " \n" . $this->writeTimestamp( $file->getTimestamp() ) . $this->writeContributor( $file->getUser( 'id' ), $file->getUser( 'text' ) ) . " " . Xml::elementClean( 'comment', null, $file->getDescription() ) . "\n" . " " . Xml::element( 'filename', null, $file->getName() ) . "\n" . + $archiveName . " " . Xml::element( 'src', null, $file->getFullUrl() ) . "\n" . " " . Xml::element( 'size', null, $file->getSize() ) . "\n" . + " " . Xml::element( 'sha1base36', null, $file->getSha1() ) . "\n" . + " " . Xml::element( 'rel', null, $file->getRel() ) . "\n" . + $contents . " \n"; } diff --git a/includes/Import.php b/includes/Import.php index f027f1a79a..c7662419dc 100644 --- a/includes/Import.php +++ b/includes/Import.php @@ -35,6 +35,7 @@ class WikiImporter { private $mLogItemCallback, $mUploadCallback, $mRevisionCallback, $mPageCallback; private $mSiteInfoCallback, $mTargetNamespace, $mPageOutCallback; private $mDebug; + private $mImportUploads, $mImageBasePath; /** * Creates an ImportXMLReader drawing from the source provided @@ -169,6 +170,13 @@ class WikiImporter { return false; } } + + /** + * + */ + public function setImageBasePath( $dir ) { + $this->mImageBasePath = $dir; + } /** * Default per-revision callback, performs the import. @@ -192,9 +200,8 @@ class WikiImporter { * Dummy for now... */ public function importUpload( $revision ) { - $revision->importUpload(); - //$dbw = wfGetDB( DB_MASTER ); - //return $dbw->deadlockLoop( array( $revision, 'importUpload' ) ); + $dbw = wfGetDB( DB_MASTER ); + return $dbw->deadlockLoop( array( $revision, 'importUpload' ) ); return false; } @@ -582,7 +589,7 @@ class WikiImporter { $uploadInfo = array(); $normalFields = array( 'timestamp', 'comment', 'filename', 'text', - 'src', 'size' ); + 'src', 'size', 'sha1base36', 'archivename', 'rel' ); $skip = false; @@ -601,26 +608,53 @@ class WikiImporter { $uploadInfo[$tag] = $this->nodeContents(); } elseif ( $tag == 'contributor' ) { $uploadInfo['contributor'] = $this->handleContributor(); + } elseif ( $tag == 'contents' ) { + $contents = $this->nodeContents(); + $encoding = $this->reader->getAttribute( 'encoding' ); + if ( $encoding === 'base64' ) { + $uploadInfo['fileSrc'] = $this->dumpTemp( base64_decode( $contents ) ); + } } elseif ( $tag != '#text' ) { $this->warn( "Unhandled upload XML tag $tag" ); $skip = true; } } + + if ( $this->mImageBasePath && isset( $uploadInfo['rel'] ) ) { + $path = "{$this->mImageBasePath}/{$uploadInfo['rel']}"; + if ( file_exists( $path ) ) { + $uploadInfo['fileSrc'] = $path; + } + } - return $this->processUpload( $pageInfo, $uploadInfo ); + if ( $this->mImportUploads ) { + return $this->processUpload( $pageInfo, $uploadInfo ); + } + } + + private function dumpTemp( $contents ) { + $filename = tempnam( wfTempDir(), 'importupload' ); + file_put_contents( $filename, $contents ); + return $filename; } private function processUpload( $pageInfo, $uploadInfo ) { $revision = new WikiRevision; - $text = isset( $uploadInfo['text'] ) ? $uploadInfo['text'] : ''; + $text = isset( $uploadInfo['text'] ) ? $uploadInfo['text'] : ''; $revision->setTitle( $pageInfo['_title'] ); - $revision->setID( $pageInfo['id'] ); + $revision->setID( $pageInfo['id'] ); $revision->setTimestamp( $uploadInfo['timestamp'] ); - $revision->setText( $text ); + $revision->setText( $text ); $revision->setFilename( $uploadInfo['filename'] ); + if ( isset( $uploadInfo['archivename'] ) ) { + $revision->setArchiveName( $uploadInfo['archivename'] ); + } $revision->setSrc( $uploadInfo['src'] ); + if ( isset( $uploadInfo['fileSrc'] ) ) { + $revision->setFileSrc( $uploadInfo['fileSrc'] ); + } $revision->setSize( intval( $uploadInfo['size'] ) ); $revision->setComment( $uploadInfo['comment'] ); @@ -631,7 +665,7 @@ class WikiImporter { $revision->setUserName( $uploadInfo['contributor']['username'] ); } - return call_user_func( $this->mUploadCallback, $revision ); + return call_user_func( $this->mUploadCallback, $revision ); } private function handleContributor() { @@ -790,6 +824,7 @@ class XMLReader2 extends XMLReader { * @ingroup SpecialPage */ class WikiRevision { + var $importer = null; var $title = null; var $id = 0; var $timestamp = "20010115000000"; @@ -801,6 +836,8 @@ class WikiRevision { var $type = ""; var $action = ""; var $params = ""; + var $fileSrc = ''; + var $archiveName = ''; function setTitle( $title ) { if( is_object( $title ) ) { @@ -844,10 +881,16 @@ class WikiRevision { function setSrc( $src ) { $this->src = $src; } + function setFileSrc( $src ) { + $this->fileSrc = $src; + } function setFilename( $filename ) { $this->filename = $filename; } + function setArchiveName( $archiveName ) { + $this->archiveName = $archiveName; + } function setSize( $size ) { $this->size = intval( $size ); @@ -896,10 +939,16 @@ class WikiRevision { function getSrc() { return $this->src; } + function getFileSrc() { + return $this->fileSrc; + } function getFilename() { return $this->filename; } + function getArchiveName() { + return $this->archiveName; + } function getSize() { return $this->size; @@ -1044,62 +1093,55 @@ class WikiRevision { } function importUpload() { - wfDebug( __METHOD__ . ": STUB\n" ); - - /** - // from file revert... - $source = $this->file->getArchiveVirtualUrl( $this->oldimage ); - $comment = $wgRequest->getText( 'wpComment' ); - // TODO: Preserve file properties from database instead of reloading from file - $status = $this->file->upload( $source, $comment, $comment ); - if( $status->isGood() ) { - */ - - /** - // from file upload... - $this->mLocalFile = wfLocalFile( $nt ); - $this->mDestName = $this->mLocalFile->getName(); - //.... - $status = $this->mLocalFile->upload( $this->mTempPath, $this->mComment, $pageText, - File::DELETE_SOURCE, $this->mFileProps ); - if ( !$status->isGood() ) { - $resultDetails = array( 'internal' => $status->getWikiText() ); - */ - - // @todo Fixme: it may create a page without our desire, also wrong potentially. - // and, it will record a *current* upload, but we might want an archive version here - - $file = wfLocalFile( $this->getTitle() ); + # Construct a file + $archiveName = $this->getArchiveName(); + if ( $archiveName ) { + wfDebug( __METHOD__ . "Importing archived file as $archiveName\n" ); + $file = OldLocalFile::newFromArchiveName( $this->getTitle(), + RepoGroup::singleton()->getLocalRepo(), $archiveName ); + } else { + $file = wfLocalFile( $this->getTitle() ); + wfDebug( __METHOD__ . 'Importing new file as ' . $file->getName() . "\n" ); + if ( $file->exists() && $file->getTimestamp() > $this->getTimestamp() ) { + $archiveName = $file->getTimestamp() . '!' . $file->getName(); + $file = OldLocalFile::newFromArchiveName( $this->getTitle(), + RepoGroup::singleton()->getLocalRepo(), $archiveName ); + wfDebug( __METHOD__ . "File already exists; importing as $archiveName\n" ); + } + } if( !$file ) { - wfDebug( "IMPORT: Bad file. :(\n" ); + wfDebug( __METHOD__ . ': Bad file for ' . $this->getTitle() . "\n" ); return false; } - - $source = $this->downloadSource(); + + # Get the file source or download if necessary + $source = $this->getFileSrc(); + if ( !$source ) { + $source = $this->downloadSource(); + } if( !$source ) { - wfDebug( "IMPORT: Could not fetch remote file. :(\n" ); + wfDebug( __METHOD__ . ": Could not fetch remote file.\n" ); return false; } $user = User::newFromName( $this->user_text ); - - $status = $file->upload( $source, - $this->getComment(), - $this->getComment(), // Initial page, if none present... - File::DELETE_SOURCE, - false, // props... - $this->getTimestamp(), - is_object( $user ) ? ( $user->isLoggedIn() ? $user : null ) : null ); - - if( $status->isGood() ) { - // yay? - wfDebug( "IMPORT: is ok?\n" ); + + # Do the actual upload + if ( $archiveName ) { + $status = $file->uploadOld( $source, $archiveName, + $this->getTimestamp(), $this->getComment(), $user, File::DELETE_SOURCE ); + } else { + $status = $file->upload( $source, $this->getComment(), $this->getComment(), + File::DELETE_SOURCE, false, $this->getTimestamp(), $user ); + } + + if ( $status->isGood() ) { + wfDebug( __METHOD__ . ": Succesful\n" ); return true; + } else { + wfDebug( __METHOD__ . ': failed: ' . $status->getXml() . "\n" ); + return false; } - - wfDebug( "IMPORT: is bad? " . $status->getXml() . "\n" ); - return false; - } function downloadSource() { diff --git a/includes/filerepo/OldLocalFile.php b/includes/filerepo/OldLocalFile.php index 2b1cddda3e..6d025feac9 100644 --- a/includes/filerepo/OldLocalFile.php +++ b/includes/filerepo/OldLocalFile.php @@ -227,12 +227,14 @@ class OldLocalFile extends LocalFile { * @param $archiveName string Full archive name of the file, in the form * $timestamp!$filename, where $filename must match $this->getName() */ - function uploadOld( $srcPath, $archiveName, $comment, $user ) { + function uploadOld( $srcPath, $archiveName, $timestamp, $comment, $user, $flags = 0 ) { $this->lock(); - $status = $this->publish( $srcPath, $flags, $archiveName ); + $status = $this->publish( $srcPath, + $flags & File::DELETE_SOURCE ? FileRepo::DELETE_SOURCE : 0, + $archiveName ); if ( $status->isGood() ) { - if ( !$this->recordOldUpload( $srcPath, $archiveName, $comment, $user ) ) { + if ( !$this->recordOldUpload( $srcPath, $archiveName, $timestamp, $comment, $user ) ) { $status->fatal( 'filenotfound', $srcPath ); } } @@ -251,7 +253,7 @@ class OldLocalFile extends LocalFile { * @param $user User User who did this upload * @return bool */ - function recordOldUpload( $srcPath, $archiveName, $comment, $user ) { + function recordOldUpload( $srcPath, $archiveName, $timestamp, $comment, $user ) { $dbw = $this->repo->getMasterDB(); $dbw->begin(); @@ -269,7 +271,7 @@ class OldLocalFile extends LocalFile { 'oi_width' => intval( $props['width'] ), 'oi_height' => intval( $props['height'] ), 'oi_bits' => $props['bits'], - 'oi_timestamp' => $props['timestamp'], + 'oi_timestamp' => $dbw->timestamp( $timestamp ), 'oi_description' => $comment, 'oi_user' => $user->getId(), 'oi_user_text' => $user->getName(), -- 2.20.1