X-Git-Url: https://git.cyclocoop.org/%27.WWW_URL.%27admin/?a=blobdiff_plain;f=includes%2FExport.php;h=7773d03c29de548a9916599517ca6e117c4dc713;hb=22746a580721d659d8bdabe00ffa639670497524;hp=c3ef9451aa55bbf86cd108e3892e6996532a4945;hpb=a8f50b8fafcd27c86560c36b04ce42c6ed81b873;p=lhc%2Fweb%2Fwiklou.git diff --git a/includes/Export.php b/includes/Export.php index c3ef9451aa..7773d03c29 100644 --- a/includes/Export.php +++ b/includes/Export.php @@ -1,33 +1,47 @@ -# http://www.mediawiki.org/ -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -# http://www.gnu.org/copyleft/gpl.html +/** + * Base classes for dumps and export + * + * Copyright © 2003, 2005, 2006 Brion Vibber + * http://www.mediawiki.org/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + */ +/** + * @defgroup Dump Dump + */ /** - * - * @addtogroup SpecialPage + * @ingroup SpecialPage Dump */ class WikiExporter { var $list_authors = false ; # Return distinct author list (when not returning full history) var $author_list = "" ; - const FULL = 0; - const CURRENT = 1; + var $dumpUploads = false; + var $dumpUploadFileContents = false; + + const FULL = 1; + const CURRENT = 2; + const STABLE = 4; // extension defined + const LOGS = 8; + const RANGE = 16; const BUFFER = 0; const STREAM = 1; @@ -42,13 +56,15 @@ class WikiExporter { * make additional queries to pull source data while the * main query is still running. * - * @param Database $db - * @param mixed $history one of WikiExporter::FULL or WikiExporter::CURRENT, or an - * associative array: - * offset: non-inclusive offset at which to start the query - * limit: maximum number of rows to return - * dir: "asc" or "desc" timestamp order - * @param int $buffer one of WikiExporter::BUFFER or WikiExporter::STREAM + * @param $db DatabaseBase + * @param $history Mixed: one of WikiExporter::FULL, WikiExporter::CURRENT, + * WikiExporter::RANGE or WikiExporter::STABLE, + * or an associative array: + * offset: non-inclusive offset at which to start the query + * limit: maximum number of rows to return + * dir: "asc" or "desc" timestamp order + * @param $buffer Int: one of WikiExporter::BUFFER or WikiExporter::STREAM + * @param $text Int: one of WikiExporter::TEXT or WikiExporter::STUB */ function __construct( &$db, $history = WikiExporter::CURRENT, $buffer = WikiExporter::BUFFER, $text = WikiExporter::TEXT ) { @@ -65,18 +81,18 @@ class WikiExporter { * various row objects and XML output for filtering. Filters * can be chained or used as callbacks. * - * @param mixed $callback + * @param $sink mixed */ - function setOutputSink( &$sink ) { + public function setOutputSink( &$sink ) { $this->sink =& $sink; } - function openStream() { + public function openStream() { $output = $this->writer->openStream(); $this->sink->writeOpenStream( $output ); } - function closeStream() { + public function closeStream() { $output = $this->writer->closeStream(); $this->sink->writeCloseStream( $output ); } @@ -86,161 +102,221 @@ class WikiExporter { * in the database, either including complete history or only * the most recent version. */ - function allPages() { + public function allPages() { return $this->dumpFrom( '' ); } /** * Dumps a series of page and revision records for those pages * in the database falling within the page_id range given. - * @param int $start Inclusive lower limit (this id is included) - * @param int $end Exclusive upper limit (this id is not included) + * @param $start Int: inclusive lower limit (this id is included) + * @param $end Int: Exclusive upper limit (this id is not included) * If 0, no upper limit. */ - function pagesByRange( $start, $end ) { + public function pagesByRange( $start, $end ) { $condition = 'page_id >= ' . intval( $start ); - if( $end ) { + if ( $end ) { $condition .= ' AND page_id < ' . intval( $end ); } return $this->dumpFrom( $condition ); } /** - * @param Title $title + * Dumps a series of page and revision records for those pages + * in the database with revisions falling within the rev_id range given. + * @param $start Int: inclusive lower limit (this id is included) + * @param $end Int: Exclusive upper limit (this id is not included) + * If 0, no upper limit. + */ + public function revsByRange( $start, $end ) { + $condition = 'rev_id >= ' . intval( $start ); + if ( $end ) { + $condition .= ' AND rev_id < ' . intval( $end ); + } + return $this->dumpFrom( $condition ); + } + + /** + * @param $title Title */ - function pageByTitle( $title ) { + public function pageByTitle( $title ) { return $this->dumpFrom( 'page_namespace=' . $title->getNamespace() . - ' AND page_title=' . $this->db->addQuotes( $title->getDbKey() ) ); + ' AND page_title=' . $this->db->addQuotes( $title->getDBkey() ) ); } - function pageByName( $name ) { + public function pageByName( $name ) { $title = Title::newFromText( $name ); - if( is_null( $title ) ) { - return new WikiError( "Can't export invalid title" ); + if ( is_null( $title ) ) { + throw new MWException( "Can't export invalid title" ); } else { return $this->pageByTitle( $title ); } } - function pagesByName( $names ) { - foreach( $names as $name ) { + public function pagesByName( $names ) { + foreach ( $names as $name ) { $this->pageByName( $name ); } } + public function allLogs() { + return $this->dumpFrom( '' ); + } - // -------------------- private implementation below -------------------- + public function logsByRange( $start, $end ) { + $condition = 'log_id >= ' . intval( $start ); + if ( $end ) { + $condition .= ' AND log_id < ' . intval( $end ); + } + return $this->dumpFrom( $condition ); + } # Generates the distinct list of authors of an article # Not called by default (depends on $this->list_authors) # Can be set by Special:Export when not exporting whole history - function do_list_authors ( $page , $revision , $cond ) { - $fname = "do_list_authors" ; - wfProfileIn( $fname ); + protected function do_list_authors( $cond ) { + wfProfileIn( __METHOD__ ); $this->author_list = ""; - //rev_deleted - $nothidden = '(rev_deleted & '.Revision::DELETED_USER.') = 0'; - - $sql = "SELECT DISTINCT rev_user_text,rev_user FROM {$page},{$revision} WHERE page_id=rev_page AND $nothidden AND " . $cond ; - $result = $this->db->query( $sql, $fname ); - $resultset = $this->db->resultObject( $result ); - while( $row = $resultset->fetchObject() ) { - $this->author_list .= "" . - "" . - htmlentities( $row->rev_user_text ) . - "" . - "" . + // rev_deleted + + $res = $this->db->select( + array( 'page', 'revision' ), + array( 'DISTINCT rev_user_text', 'rev_user' ), + array( + $this->db->bitAnd( 'rev_deleted', Revision::DELETED_USER ) . ' = 0', + $cond, + 'page_id = rev_id', + ), + __METHOD__ + ); + + foreach ( $res as $row ) { + $this->author_list .= "" . + "" . + htmlentities( $row->rev_user_text ) . + "" . + "" . $row->rev_user . - "" . + "" . ""; } - wfProfileOut( $fname ); $this->author_list .= ""; + wfProfileOut( __METHOD__ ); } - function dumpFrom( $cond = '' ) { - $fname = 'WikiExporter::dumpFrom'; - wfProfileIn( $fname ); - - $page = $this->db->tableName( 'page' ); - $revision = $this->db->tableName( 'revision' ); - $text = $this->db->tableName( 'text' ); - - $order = 'ORDER BY page_id'; - $limit = ''; - - if( $this->history == WikiExporter::FULL ) { - $join = 'page_id=rev_page'; - } elseif( $this->history == WikiExporter::CURRENT ) { - if ( $this->list_authors && $cond != '' ) { // List authors, if so desired - $this->do_list_authors ( $page , $revision , $cond ); + protected function dumpFrom( $cond = '' ) { + wfProfileIn( __METHOD__ ); + # For logging dumps... + if ( $this->history & self::LOGS ) { + if ( $this->buffer == WikiExporter::STREAM ) { + $prev = $this->db->bufferResults( false ); + } + $where = array( 'user_id = log_user' ); + # Hide private logs + $hideLogs = LogEventsList::getExcludeClause( $this->db ); + if ( $hideLogs ) $where[] = $hideLogs; + # Add on any caller specified conditions + if ( $cond ) $where[] = $cond; + # Get logging table name for logging.* clause + $logging = $this->db->tableName( 'logging' ); + $result = $this->db->select( array( 'logging', 'user' ), + array( "{$logging}.*", 'user_name' ), // grab the user name + $where, + __METHOD__, + array( 'ORDER BY' => 'log_id', 'USE INDEX' => array( 'logging' => 'PRIMARY' ) ) + ); + $wrapper = $this->db->resultObject( $result ); + $this->outputLogStream( $wrapper ); + if ( $this->buffer == WikiExporter::STREAM ) { + $this->db->bufferResults( $prev ); } - $join = 'page_id=rev_page AND page_latest=rev_id'; - } elseif ( is_array( $this->history ) ) { - $join = 'page_id=rev_page'; - if ( $this->history['dir'] == 'asc' ) { - $op = '>'; - $order .= ', rev_timestamp'; + # For page dumps... + } else { + $tables = array( 'page', 'revision' ); + $opts = array( 'ORDER BY' => 'page_id ASC' ); + $opts['USE INDEX'] = array(); + $join = array(); + if ( is_array( $this->history ) ) { + # Time offset/limit for all pages/history... + $revJoin = 'page_id=rev_page'; + # Set time order + if ( $this->history['dir'] == 'asc' ) { + $op = '>'; + $opts['ORDER BY'] = 'rev_timestamp ASC'; + } else { + $op = '<'; + $opts['ORDER BY'] = 'rev_timestamp DESC'; + } + # Set offset + if ( !empty( $this->history['offset'] ) ) { + $revJoin .= " AND rev_timestamp $op " . + $this->db->addQuotes( $this->db->timestamp( $this->history['offset'] ) ); + } + $join['revision'] = array( 'INNER JOIN', $revJoin ); + # Set query limit + if ( !empty( $this->history['limit'] ) ) { + $opts['LIMIT'] = intval( $this->history['limit'] ); + } + } elseif ( $this->history & WikiExporter::FULL ) { + # Full history dumps... + $join['revision'] = array( 'INNER JOIN', 'page_id=rev_page' ); + } elseif ( $this->history & WikiExporter::CURRENT ) { + # Latest revision dumps... + if ( $this->list_authors && $cond != '' ) { // List authors, if so desired + $this->do_list_authors( $cond ); + } + $join['revision'] = array( 'INNER JOIN', 'page_id=rev_page AND page_latest=rev_id' ); + } elseif ( $this->history & WikiExporter::STABLE ) { + # "Stable" revision dumps... + # Default JOIN, to be overridden... + $join['revision'] = array( 'INNER JOIN', 'page_id=rev_page AND page_latest=rev_id' ); + # One, and only one hook should set this, and return false + if ( wfRunHooks( 'WikiExporter::dumpStableQuery', array( &$tables, &$opts, &$join ) ) ) { + wfProfileOut( __METHOD__ ); + throw new MWException( __METHOD__ . " given invalid history dump type." ); + } + } elseif ( $this->history & WikiExporter::RANGE ) { + # Dump of revisions within a specified range + $join['revision'] = array( 'INNER JOIN', 'page_id=rev_page' ); + $opts['ORDER BY'] = 'rev_page ASC, rev_id ASC'; } else { - $op = '<'; - $order .= ', rev_timestamp DESC'; + # Uknown history specification parameter? + wfProfileOut( __METHOD__ ); + throw new MWException( __METHOD__ . " given invalid history dump type." ); } - if ( !empty( $this->history['offset'] ) ) { - $join .= " AND rev_timestamp $op " . $this->db->addQuotes( - $this->db->timestamp( $this->history['offset'] ) ); + # Query optimization hacks + if ( $cond == '' ) { + $opts[] = 'STRAIGHT_JOIN'; + $opts['USE INDEX']['page'] = 'PRIMARY'; } - if ( !empty( $this->history['limit'] ) ) { - $limitNum = intval( $this->history['limit'] ); - if ( $limitNum > 0 ) { - $limit = "LIMIT $limitNum"; - } + # Build text join options + if ( $this->text != WikiExporter::STUB ) { // 1-pass + $tables[] = 'text'; + $join['text'] = array( 'INNER JOIN', 'rev_text_id=old_id' ); } - } else { - wfProfileOut( $fname ); - return new WikiError( "$fname given invalid history dump type." ); - } - $where = ( $cond == '' ) ? '' : "$cond AND"; - if( $this->buffer == WikiExporter::STREAM ) { - $prev = $this->db->bufferResults( false ); - } - if( $cond == '' ) { - // Optimization hack for full-database dump - $revindex = $pageindex = $this->db->useIndexClause("PRIMARY"); - $straight = ' /*! STRAIGHT_JOIN */ '; - } else { - $pageindex = ''; - $revindex = ''; - $straight = ''; - } - if( $this->text == WikiExporter::STUB ) { - $sql = "SELECT $straight * FROM - $page $pageindex, - $revision $revindex - WHERE $where $join - $order $limit"; - } else { - $sql = "SELECT $straight * FROM - $page $pageindex, - $revision $revindex, - $text - WHERE $where $join AND rev_text_id=old_id - $order $limit"; - } - $result = $this->db->query( $sql, $fname ); - $wrapper = $this->db->resultObject( $result ); - $this->outputStream( $wrapper ); + if ( $this->buffer == WikiExporter::STREAM ) { + $prev = $this->db->bufferResults( false ); + } - if ( $this->list_authors ) { - $this->outputStream( $wrapper ); - } + wfRunHooks( 'ModifyExportQuery', + array( $this->db, &$tables, &$cond, &$opts, &$join ) ); - if( $this->buffer == WikiExporter::STREAM ) { - $this->db->bufferResults( $prev ); - } + # Do the query! + $result = $this->db->select( $tables, '*', $cond, __METHOD__, $opts, $join ); + $wrapper = $this->db->resultObject( $result ); + # Output dump results + $this->outputPageStream( $wrapper ); + if ( $this->list_authors ) { + $this->outputPageStream( $wrapper ); + } - wfProfileOut( $fname ); + if ( $this->buffer == WikiExporter::STREAM ) { + $this->db->bufferResults( $prev ); + } + } + wfProfileOut( __METHOD__ ); } /** @@ -253,17 +329,20 @@ class WikiExporter { * separate database connection not managed by LoadBalancer; some * blob storage types will make queries to pull source data. * - * @param ResultWrapper $resultset - * @access private + * @param $resultset ResultWrapper */ - function outputStream( $resultset ) { + protected function outputPageStream( $resultset ) { $last = null; - while( $row = $resultset->fetchObject() ) { - if( is_null( $last ) || + foreach ( $resultset as $row ) { + if ( is_null( $last ) || $last->page_namespace != $row->page_namespace || $last->page_title != $row->page_title ) { - if( isset( $last ) ) { - $output = $this->writer->closePage(); + if ( isset( $last ) ) { + $output = ''; + if ( $this->dumpUploads ) { + $output .= $this->writer->writeUploads( $last, $this->dumpUploadFileContents ); + } + $output .= $this->writer->closePage(); $this->sink->writeClosePage( $output ); } $output = $this->writer->openPage( $row ); @@ -273,25 +352,35 @@ class WikiExporter { $output = $this->writer->writeRevision( $row ); $this->sink->writeRevision( $row, $output ); } - if( isset( $last ) ) { - $output = $this->author_list . $this->writer->closePage(); + if ( isset( $last ) ) { + $output = ''; + if ( $this->dumpUploads ) { + $output .= $this->writer->writeUploads( $last, $this->dumpUploadFileContents ); + } + $output .= $this->author_list; + $output .= $this->writer->closePage(); $this->sink->writeClosePage( $output ); } - $resultset->free(); + } + + protected function outputLogStream( $resultset ) { + foreach ( $resultset as $row ) { + $output = $this->writer->writeLogItem( $row ); + $this->sink->writeLogItem( $row, $output ); + } } } /** - * @addtogroup Dump + * @ingroup Dump */ class XmlDumpWriter { - /** * Returns the export schema version. * @return string */ function schemaVersion() { - return "0.3"; // FIXME: upgrade to 0.4 when updated XSD is ready, for the revision deletion bits + return "0.6"; } /** @@ -305,15 +394,15 @@ class XmlDumpWriter { * @return string */ function openStream() { - global $wgContLanguageCode; + global $wgLanguageCode; $ver = $this->schemaVersion(); - return wfElement( 'mediawiki', array( + return Xml::element( 'mediawiki', array( 'xmlns' => "http://www.mediawiki.org/xml/export-$ver/", 'xmlns:xsi' => "http://www.w3.org/2001/XMLSchema-instance", 'xsi:schemaLocation' => "http://www.mediawiki.org/xml/export-$ver/ " . "http://www.mediawiki.org/xml/export-$ver.xsd", 'version' => $ver, - 'xml:lang' => $wgContLanguageCode ), + 'xml:lang' => $wgLanguageCode ), null ) . "\n" . $this->siteInfo(); @@ -333,30 +422,34 @@ class XmlDumpWriter { function sitename() { global $wgSitename; - return wfElement( 'sitename', array(), $wgSitename ); + return Xml::element( 'sitename', array(), $wgSitename ); } function generator() { global $wgVersion; - return wfElement( 'generator', array(), "MediaWiki $wgVersion" ); + return Xml::element( 'generator', array(), "MediaWiki $wgVersion" ); } function homelink() { - return wfElement( 'base', array(), Title::newMainPage()->getFullUrl() ); + return Xml::element( 'base', array(), Title::newMainPage()->getCanonicalUrl() ); } function caseSetting() { global $wgCapitalLinks; // "case-insensitive" option is reserved for future $sensitivity = $wgCapitalLinks ? 'first-letter' : 'case-sensitive'; - return wfElement( 'case', array(), $sensitivity ); + return Xml::element( 'case', array(), $sensitivity ); } function namespaces() { global $wgContLang; - $spaces = " \n"; - foreach( $wgContLang->getFormattedNamespaces() as $ns => $title ) { - $spaces .= ' ' . wfElement( 'namespace', array( 'key' => $ns ), $title ) . "\n"; + $spaces = "\n"; + foreach ( $wgContLang->getFormattedNamespaces() as $ns => $title ) { + $spaces .= ' ' . + Xml::element( 'namespace', + array( 'key' => $ns, + 'case' => MWNamespace::isCapitalized( $ns ) ? 'first-letter' : 'case-sensitive', + ), $title ) . "\n"; } $spaces .= " "; return $spaces; @@ -365,29 +458,48 @@ class XmlDumpWriter { /** * Closes the output stream with the closing root element. * Call when finished dumping things. + * + * @return string */ function closeStream() { return "\n"; } - /** * Opens a section on the output stream, with data * from the given database row. * - * @param object $row + * @param $row object * @return string * @access private */ function openPage( $row ) { $out = " \n"; $title = Title::makeTitle( $row->page_namespace, $row->page_title ); - $out .= ' ' . wfElementClean( 'title', array(), $title->getPrefixedText() ) . "\n"; - $out .= ' ' . wfElement( 'id', array(), strval( $row->page_id ) ) . "\n"; - if( '' != $row->page_restrictions ) { - $out .= ' ' . wfElement( 'restrictions', array(), + $out .= ' ' . Xml::elementClean( 'title', array(), self::canonicalTitle( $title ) ) . "\n"; + $out .= ' ' . Xml::element( 'ns', array(), strval( $row->page_namespace) ) . "\n"; + $out .= ' ' . Xml::element( 'id', array(), strval( $row->page_id ) ) . "\n"; + if ( $row->page_is_redirect ) { + $page = WikiPage::factory( $title ); + $redirect = $page->getRedirectTarget(); + if ( $redirect instanceOf Title && $redirect->isValidRedirectTarget() ) { + $out .= ' ' . Xml::element( 'redirect', array( 'title' => self::canonicalTitle( $redirect ) ) ) . "\n"; + } + } + + if ( $row->rev_sha1 ) { + $out .= " " . Xml::element('sha1', null, strval($row->rev_sha1) ) . "\n"; + } else { + $out .= " \n"; + } + + if ( $row->page_restrictions != '' ) { + $out .= ' ' . Xml::element( 'restrictions', array(), strval( $row->page_restrictions ) ) . "\n"; } + + wfRunHooks( 'XmlDumpWriterOpenPage', array( $this, &$out, $row, $title ) ); + return $out; } @@ -404,69 +516,204 @@ class XmlDumpWriter { * Dumps a section on the output stream, with * data filled in from the given database row. * - * @param object $row + * @param $row object * @return string * @access private */ function writeRevision( $row ) { - $fname = 'WikiExporter::dumpRev'; - wfProfileIn( $fname ); + wfProfileIn( __METHOD__ ); $out = " \n"; - $out .= " " . wfElement( 'id', null, strval( $row->rev_id ) ) . "\n"; + $out .= " " . Xml::element( 'id', null, strval( $row->rev_id ) ) . "\n"; - $ts = wfTimestamp( TS_ISO_8601, $row->rev_timestamp ); - $out .= " " . wfElement( 'timestamp', null, $ts ) . "\n"; + $out .= $this->writeTimestamp( $row->rev_timestamp ); - if( $row->rev_deleted & Revision::DELETED_USER ) { - $out .= " " . wfElement( 'contributor', array( 'deleted' => 'deleted' ) ) . "\n"; + if ( $row->rev_deleted & Revision::DELETED_USER ) { + $out .= " " . Xml::element( 'contributor', array( 'deleted' => 'deleted' ) ) . "\n"; } else { - $out .= " \n"; - if( $row->rev_user ) { - $out .= " " . wfElementClean( 'username', null, strval( $row->rev_user_text ) ) . "\n"; - $out .= " " . wfElement( 'id', null, strval( $row->rev_user ) ) . "\n"; - } else { - $out .= " " . wfElementClean( 'ip', null, strval( $row->rev_user_text ) ) . "\n"; - } - $out .= " \n"; + $out .= $this->writeContributor( $row->rev_user, $row->rev_user_text ); } - if( $row->rev_minor_edit ) { + if ( $row->rev_minor_edit ) { $out .= " \n"; } - if( $row->rev_deleted & Revision::DELETED_COMMENT ) { - $out .= " " . wfElement( 'comment', array( 'deleted' => 'deleted' ) ) . "\n"; - } elseif( $row->rev_comment != '' ) { - $out .= " " . wfElementClean( 'comment', null, strval( $row->rev_comment ) ) . "\n"; + if ( $row->rev_deleted & Revision::DELETED_COMMENT ) { + $out .= " " . Xml::element( 'comment', array( 'deleted' => 'deleted' ) ) . "\n"; + } elseif ( $row->rev_comment != '' ) { + $out .= " " . Xml::elementClean( 'comment', null, strval( $row->rev_comment ) ) . "\n"; } - if( $row->rev_deleted & Revision::DELETED_TEXT ) { - $out .= " " . wfElement( 'text', array( 'deleted' => 'deleted' ) ) . "\n"; - } elseif( isset( $row->old_text ) ) { + $text = ''; + if ( $row->rev_deleted & Revision::DELETED_TEXT ) { + $out .= " " . Xml::element( 'text', array( 'deleted' => 'deleted' ) ) . "\n"; + } elseif ( isset( $row->old_text ) ) { // Raw text from the database may have invalid chars $text = strval( Revision::getRevisionText( $row ) ); - $out .= " " . wfElementClean( 'text', - array( 'xml:space' => 'preserve' ), + $out .= " " . Xml::elementClean( 'text', + array( 'xml:space' => 'preserve', 'bytes' => intval( $row->rev_len ) ), strval( $text ) ) . "\n"; } else { // Stub output - $out .= " " . wfElement( 'text', - array( 'id' => $row->rev_text_id ), + $out .= " " . Xml::element( 'text', + array( 'id' => $row->rev_text_id, 'bytes' => intval( $row->rev_len ) ), "" ) . "\n"; } + wfRunHooks( 'XmlDumpWriterWriteRevision', array( &$this, &$out, $row, $text ) ); + $out .= " \n"; - wfProfileOut( $fname ); + wfProfileOut( __METHOD__ ); return $out; } + /** + * Dumps a section on the output stream, with + * data filled in from the given database row. + * + * @param $row object + * @return string + * @access private + */ + function writeLogItem( $row ) { + wfProfileIn( __METHOD__ ); + + $out = " \n"; + $out .= " " . Xml::element( 'id', null, strval( $row->log_id ) ) . "\n"; + + $out .= $this->writeTimestamp( $row->log_timestamp ); + + if ( $row->log_deleted & LogPage::DELETED_USER ) { + $out .= " " . Xml::element( 'contributor', array( 'deleted' => 'deleted' ) ) . "\n"; + } else { + $out .= $this->writeContributor( $row->log_user, $row->user_name ); + } + + if ( $row->log_deleted & LogPage::DELETED_COMMENT ) { + $out .= " " . Xml::element( 'comment', array( 'deleted' => 'deleted' ) ) . "\n"; + } elseif ( $row->log_comment != '' ) { + $out .= " " . Xml::elementClean( 'comment', null, strval( $row->log_comment ) ) . "\n"; + } + + $out .= " " . Xml::element( 'type', null, strval( $row->log_type ) ) . "\n"; + $out .= " " . Xml::element( 'action', null, strval( $row->log_action ) ) . "\n"; + + if ( $row->log_deleted & LogPage::DELETED_ACTION ) { + $out .= " " . Xml::element( 'text', array( 'deleted' => 'deleted' ) ) . "\n"; + } else { + $title = Title::makeTitle( $row->log_namespace, $row->log_title ); + $out .= " " . Xml::elementClean( 'logtitle', null, self::canonicalTitle( $title ) ) . "\n"; + $out .= " " . Xml::elementClean( 'params', + array( 'xml:space' => 'preserve' ), + strval( $row->log_params ) ) . "\n"; + } + + $out .= " \n"; + + wfProfileOut( __METHOD__ ); + return $out; + } + + function writeTimestamp( $timestamp ) { + $ts = wfTimestamp( TS_ISO_8601, $timestamp ); + return " " . Xml::element( 'timestamp', null, $ts ) . "\n"; + } + + function writeContributor( $id, $text ) { + $out = " \n"; + if ( $id || !IP::isValid( $text ) ) { + $out .= " " . Xml::elementClean( 'username', null, strval( $text ) ) . "\n"; + $out .= " " . Xml::element( 'id', null, strval( $id ) ) . "\n"; + } else { + $out .= " " . Xml::elementClean( 'ip', null, strval( $text ) ) . "\n"; + } + $out .= " \n"; + return $out; + } + + /** + * Warning! This data is potentially inconsistent. :( + */ + function writeUploads( $row, $dumpContents = false ) { + if ( $row->page_namespace == NS_IMAGE ) { + $img = wfLocalFile( $row->page_title ); + if ( $img && $img->exists() ) { + $out = ''; + foreach ( array_reverse( $img->getHistory() ) as $ver ) { + $out .= $this->writeUpload( $ver, $dumpContents ); + } + $out .= $this->writeUpload( $img, $dumpContents ); + return $out; + } + } + return ''; + } + + /** + * @param $file File + * @param $dumpContents bool + * @return string + */ + function writeUpload( $file, $dumpContents = false ) { + if ( $file->isOld() ) { + $archiveName = " " . + Xml::element( 'archivename', null, $file->getArchiveName() ) . "\n"; + } else { + $archiveName = ''; + } + if ( $dumpContents ) { + # Dump file as base64 + # Uses only XML-safe characters, so does not need escaping + $contents = ' ' . + chunk_split( base64_encode( file_get_contents( $file->getPath() ) ) ) . + " \n"; + } else { + $contents = ''; + } + return " \n" . + $this->writeTimestamp( $file->getTimestamp() ) . + $this->writeContributor( $file->getUser( 'id' ), $file->getUser( 'text' ) ) . + " " . Xml::elementClean( 'comment', null, $file->getDescription() ) . "\n" . + " " . Xml::element( 'filename', null, $file->getName() ) . "\n" . + $archiveName . + " " . Xml::element( 'src', null, $file->getCanonicalUrl() ) . "\n" . + " " . Xml::element( 'size', null, $file->getSize() ) . "\n" . + " " . Xml::element( 'sha1base36', null, $file->getSha1() ) . "\n" . + " " . Xml::element( 'rel', null, $file->getRel() ) . "\n" . + $contents . + " \n"; + } + + /** + * Return prefixed text form of title, but using the content language's + * canonical namespace. This skips any special-casing such as gendered + * user namespaces -- which while useful, are not yet listed in the + * XML data so are unsafe in export. + * + * @param Title $title + * @return string + * @since 1.18 + */ + public static function canonicalTitle( Title $title ) { + if ( $title->getInterwiki() ) { + return $title->getPrefixedText(); + } + + global $wgContLang; + $prefix = str_replace( '_', ' ', $wgContLang->getNsText( $title->getNamespace() ) ); + + if ( $prefix !== '' ) { + $prefix .= ':'; + } + + return $prefix . $title->getText(); + } } /** * Base class for output stream; prints to stdout or buffer or whereever. - * @addtogroup Dump + * @ingroup Dump */ class DumpOutput { function writeOpenStream( $string ) { @@ -489,6 +736,10 @@ class DumpOutput { $this->write( $string ); } + function writeLogItem( $rev, $string ) { + $this->write( $string ); + } + /** * Override to write to a different stream type. * @return bool @@ -496,70 +747,189 @@ class DumpOutput { function write( $string ) { print $string; } + + /** + * Close the old file, move it to a specified name, + * and reopen new file with the old name. Use this + * for writing out a file in multiple pieces + * at specified checkpoints (e.g. every n hours). + * @param $newname mixed File name. May be a string or an array with one element + */ + function closeRenameAndReopen( $newname ) { + return; + } + + /** + * Close the old file, and move it to a specified name. + * Use this for the last piece of a file written out + * at specified checkpoints (e.g. every n hours). + * @param $newname mixed File name. May be a string or an array with one element + * @param $open bool If true, a new file with the old filename will be opened again for writing (default: false) + */ + function closeAndRename( $newname, $open = false ) { + return; + } + + /** + * Returns the name of the file or files which are + * being written to, if there are any. + */ + function getFilenames() { + return NULL; + } } /** * Stream outputter to send data to a file. - * @addtogroup Dump + * @ingroup Dump */ class DumpFileOutput extends DumpOutput { - var $handle; + protected $handle, $filename; - function DumpFileOutput( $file ) { + function __construct( $file ) { $this->handle = fopen( $file, "wt" ); + $this->filename = $file; } function write( $string ) { fputs( $this->handle, $string ); } + + function closeRenameAndReopen( $newname ) { + $this->closeAndRename( $newname, true ); + } + + function renameOrException( $newname ) { + if (! rename( $this->filename, $newname ) ) { + throw new MWException( __METHOD__ . ": rename of file {$this->filename} to $newname failed\n" ); + } + } + + function checkRenameArgCount( $newname ) { + if ( is_array( $newname ) ) { + if ( count( $newname ) > 1 ) { + throw new MWException( __METHOD__ . ": passed multiple arguments for rename of single file\n" ); + } else { + $newname = $newname[0]; + } + } + return $newname; + } + + function closeAndRename( $newname, $open = false ) { + $newname = $this->checkRenameArgCount( $newname ); + if ( $newname ) { + fclose( $this->handle ); + $this->renameOrException( $newname ); + if ( $open ) { + $this->handle = fopen( $this->filename, "wt" ); + } + } + } + + function getFilenames() { + return $this->filename; + } } /** * Stream outputter to send data to a file via some filter program. * Even if compression is available in a library, using a separate * program can allow us to make use of a multi-processor system. - * @addtogroup Dump + * @ingroup Dump */ class DumpPipeOutput extends DumpFileOutput { - function DumpPipeOutput( $command, $file = null ) { - if( !is_null( $file ) ) { + protected $command, $filename; + + function __construct( $command, $file = null ) { + if ( !is_null( $file ) ) { $command .= " > " . wfEscapeShellArg( $file ); } - $this->handle = popen( $command, "w" ); + + $this->startCommand( $command ); + $this->command = $command; + $this->filename = $file; + } + + function startCommand( $command ) { + $spec = array( + 0 => array( "pipe", "r" ), + ); + $pipes = array(); + $this->procOpenResource = proc_open( $command, $spec, $pipes ); + $this->handle = $pipes[0]; + } + + function closeRenameAndReopen( $newname ) { + $this->closeAndRename( $newname, true ); + } + + function closeAndRename( $newname, $open = false ) { + $newname = $this->checkRenameArgCount( $newname ); + if ( $newname ) { + fclose( $this->handle ); + proc_close( $this->procOpenResource ); + $this->renameOrException( $newname ); + if ( $open ) { + $command = $this->command; + $command .= " > " . wfEscapeShellArg( $this->filename ); + $this->startCommand( $command ); + } + } } + } /** * Sends dump output via the gzip compressor. - * @addtogroup Dump + * @ingroup Dump */ class DumpGZipOutput extends DumpPipeOutput { - function DumpGZipOutput( $file ) { - parent::DumpPipeOutput( "gzip", $file ); + function __construct( $file ) { + parent::__construct( "gzip", $file ); } } /** * Sends dump output via the bgzip2 compressor. - * @addtogroup Dump + * @ingroup Dump */ class DumpBZip2Output extends DumpPipeOutput { - function DumpBZip2Output( $file ) { - parent::DumpPipeOutput( "bzip2", $file ); + function __construct( $file ) { + parent::__construct( "bzip2", $file ); } } /** * Sends dump output via the p7zip compressor. - * @addtogroup Dump + * @ingroup Dump */ class Dump7ZipOutput extends DumpPipeOutput { - function Dump7ZipOutput( $file ) { + function __construct( $file ) { + $command = $this->setup7zCommand( $file ); + parent::__construct( $command ); + $this->filename = $file; + } + + function setup7zCommand( $file ) { $command = "7za a -bd -si " . wfEscapeShellArg( $file ); // Suppress annoying useless crap from p7zip // Unfortunately this could suppress real error messages too $command .= ' >' . wfGetNull() . ' 2>&1'; - parent::DumpPipeOutput( $command ); + return( $command ); + } + + function closeAndRename( $newname, $open = false ) { + $newname = $this->checkRenameArgCount( $newname ); + if ( $newname ) { + fclose( $this->handle ); + proc_close( $this->procOpenResource ); + $this->renameOrException( $newname ); + if ( $open ) { + $command = $this->setup7zCommand( $this->filename ); + $this->startCommand( $command ); + } + } } } @@ -569,10 +939,10 @@ class Dump7ZipOutput extends DumpPipeOutput { * Dump output filter class. * This just does output filtering and streaming; XML formatting is done * higher up, so be careful in what you do. - * @addtogroup Dump + * @ingroup Dump */ class DumpFilter { - function DumpFilter( &$sink ) { + function __construct( &$sink ) { $this->sink =& $sink; } @@ -586,24 +956,40 @@ class DumpFilter { function writeOpenPage( $page, $string ) { $this->sendingThisPage = $this->pass( $page, $string ); - if( $this->sendingThisPage ) { + if ( $this->sendingThisPage ) { $this->sink->writeOpenPage( $page, $string ); } } function writeClosePage( $string ) { - if( $this->sendingThisPage ) { + if ( $this->sendingThisPage ) { $this->sink->writeClosePage( $string ); $this->sendingThisPage = false; } } function writeRevision( $rev, $string ) { - if( $this->sendingThisPage ) { + if ( $this->sendingThisPage ) { $this->sink->writeRevision( $rev, $string ); } } + function writeLogItem( $rev, $string ) { + $this->sink->writeRevision( $rev, $string ); + } + + function closeRenameAndReopen( $newname ) { + $this->sink->closeRenameAndReopen( $newname ); + } + + function closeAndRename( $newname, $open = false ) { + $this->sink->closeAndRename( $newname, $open ); + } + + function getFilenames() { + return $this->sink->getFilenames(); + } + /** * Override for page-based filter types. * @return bool @@ -615,24 +1001,24 @@ class DumpFilter { /** * Simple dump output filter to exclude all talk pages. - * @addtogroup Dump + * @ingroup Dump */ class DumpNotalkFilter extends DumpFilter { function pass( $page ) { - return !Namespace::isTalk( $page->page_namespace ); + return !MWNamespace::isTalk( $page->page_namespace ); } } /** * Dump output filter to include or exclude pages in a given set of namespaces. - * @addtogroup Dump + * @ingroup Dump */ class DumpNamespaceFilter extends DumpFilter { var $invert = false; var $namespaces = array(); - function DumpNamespaceFilter( &$sink, $param ) { - parent::DumpFilter( $sink ); + function __construct( &$sink, $param ) { + parent::__construct( $sink ); $constants = array( "NS_MAIN" => NS_MAIN, @@ -641,7 +1027,9 @@ class DumpNamespaceFilter extends DumpFilter { "NS_USER_TALK" => NS_USER_TALK, "NS_PROJECT" => NS_PROJECT, "NS_PROJECT_TALK" => NS_PROJECT_TALK, - "NS_IMAGE" => NS_IMAGE, + "NS_FILE" => NS_FILE, + "NS_FILE_TALK" => NS_FILE_TALK, + "NS_IMAGE" => NS_IMAGE, // NS_IMAGE is an alias for NS_FILE "NS_IMAGE_TALK" => NS_IMAGE_TALK, "NS_MEDIAWIKI" => NS_MEDIAWIKI, "NS_MEDIAWIKI_TALK" => NS_MEDIAWIKI_TALK, @@ -652,17 +1040,17 @@ class DumpNamespaceFilter extends DumpFilter { "NS_CATEGORY" => NS_CATEGORY, "NS_CATEGORY_TALK" => NS_CATEGORY_TALK ); - if( $param{0} == '!' ) { + if ( $param { 0 } == '!' ) { $this->invert = true; $param = substr( $param, 1 ); } - foreach( explode( ',', $param ) as $key ) { + foreach ( explode( ',', $param ) as $key ) { $key = trim( $key ); - if( isset( $constants[$key] ) ) { + if ( isset( $constants[$key] ) ) { $ns = $constants[$key]; $this->namespaces[$ns] = true; - } elseif( is_numeric( $key ) ) { + } elseif ( is_numeric( $key ) ) { $ns = intval( $key ); $this->namespaces[$ns] = true; } else { @@ -680,7 +1068,7 @@ class DumpNamespaceFilter extends DumpFilter { /** * Dump output filter to include only the last revision in each page sequence. - * @addtogroup Dump + * @ingroup Dump */ class DumpLatestFilter extends DumpFilter { var $page, $pageString, $rev, $revString; @@ -691,7 +1079,7 @@ class DumpLatestFilter extends DumpFilter { } function writeClosePage( $string ) { - if( $this->rev ) { + if ( $this->rev ) { $this->sink->writeOpenPage( $this->page, $this->pageString ); $this->sink->writeRevision( $this->rev, $this->revString ); $this->sink->writeClosePage( $string ); @@ -703,7 +1091,7 @@ class DumpLatestFilter extends DumpFilter { } function writeRevision( $rev, $string ) { - if( $rev->rev_id == $this->page->page_latest ) { + if ( $rev->rev_id == $this->page->page_latest ) { $this->rev = $rev; $this->revString = $string; } @@ -712,48 +1100,66 @@ class DumpLatestFilter extends DumpFilter { /** * Base class for output stream; prints to stdout or buffer or whereever. - * @addtogroup Dump + * @ingroup Dump */ class DumpMultiWriter { - function DumpMultiWriter( $sinks ) { + function __construct( $sinks ) { $this->sinks = $sinks; $this->count = count( $sinks ); } function writeOpenStream( $string ) { - for( $i = 0; $i < $this->count; $i++ ) { + for ( $i = 0; $i < $this->count; $i++ ) { $this->sinks[$i]->writeOpenStream( $string ); } } function writeCloseStream( $string ) { - for( $i = 0; $i < $this->count; $i++ ) { + for ( $i = 0; $i < $this->count; $i++ ) { $this->sinks[$i]->writeCloseStream( $string ); } } function writeOpenPage( $page, $string ) { - for( $i = 0; $i < $this->count; $i++ ) { + for ( $i = 0; $i < $this->count; $i++ ) { $this->sinks[$i]->writeOpenPage( $page, $string ); } } function writeClosePage( $string ) { - for( $i = 0; $i < $this->count; $i++ ) { + for ( $i = 0; $i < $this->count; $i++ ) { $this->sinks[$i]->writeClosePage( $string ); } } function writeRevision( $rev, $string ) { - for( $i = 0; $i < $this->count; $i++ ) { + for ( $i = 0; $i < $this->count; $i++ ) { $this->sinks[$i]->writeRevision( $rev, $string ); } } + + function closeRenameAndReopen( $newnames ) { + $this->closeAndRename( $newnames, true ); + } + + function closeAndRename( $newnames, $open = false ) { + for ( $i = 0; $i < $this->count; $i++ ) { + $this->sinks[$i]->closeAndRename( $newnames[$i], $open ); + } + } + + function getFilenames() { + $filenames = array(); + for ( $i = 0; $i < $this->count; $i++ ) { + $filenames[] = $this->sinks[$i]->getFilenames(); + } + return $filenames; + } + } function xmlsafe( $string ) { - $fname = 'xmlsafe'; - wfProfileIn( $fname ); + wfProfileIn( __FUNCTION__ ); /** * The page may contain old data which has not been properly normalized. @@ -763,8 +1169,6 @@ function xmlsafe( $string ) { $string = UtfNormal::cleanUp( $string ); $string = htmlspecialchars( $string ); - wfProfileOut( $fname ); + wfProfileOut( __FUNCTION__ ); return $string; } - -