From 85e20863690bfee68048aac8472e9d585df78172 Mon Sep 17 00:00:00 2001 From: daniel Date: Mon, 22 Dec 2014 10:30:03 +0000 Subject: [PATCH] Maintenance script for importing site info. Bug: T87176 Bug: T87183 Change-Id: I3936417bc79e08cf3d04270158a6e483b5515246 --- autoload.php | 1 + docs/sitelist-1.0.xsd | 68 +++++ docs/sitelist.txt | 47 ++++ includes/site/SiteImporter.php | 257 ++++++++++++++++++ maintenance/importSites.php | 52 ++++ .../includes/site/SiteImporterTest.php | 201 ++++++++++++++ .../includes/site/SiteImporterTest.xml | 19 ++ 7 files changed, 645 insertions(+) create mode 100644 docs/sitelist-1.0.xsd create mode 100644 docs/sitelist.txt create mode 100644 includes/site/SiteImporter.php create mode 100644 maintenance/importSites.php create mode 100644 tests/phpunit/includes/site/SiteImporterTest.php create mode 100644 tests/phpunit/includes/site/SiteImporterTest.xml diff --git a/autoload.php b/autoload.php index 46c8b01cc8..552566a528 100644 --- a/autoload.php +++ b/autoload.php @@ -1046,6 +1046,7 @@ $wgAutoloadLocalClasses = array( 'Site' => __DIR__ . '/includes/site/Site.php', 'SiteArray' => __DIR__ . '/includes/site/SiteList.php', 'SiteConfiguration' => __DIR__ . '/includes/SiteConfiguration.php', + 'SiteImporter' => __DIR__ . '/includes/site/SiteImporter.php', 'SiteList' => __DIR__ . '/includes/site/SiteList.php', 'SiteListFileCache' => __DIR__ . '/includes/site/SiteListFileCache.php', 'SiteListFileCacheBuilder' => __DIR__ . '/includes/site/SiteListFileCacheBuilder.php', diff --git a/docs/sitelist-1.0.xsd b/docs/sitelist-1.0.xsd new file mode 100644 index 0000000000..126cd03996 --- /dev/null +++ b/docs/sitelist-1.0.xsd @@ -0,0 +1,68 @@ + + + + + + + MediaWiki's export format for site definitions. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/sitelist.txt b/docs/sitelist.txt new file mode 100644 index 0000000000..48c7ce522a --- /dev/null +++ b/docs/sitelist.txt @@ -0,0 +1,47 @@ +This document describes the XML format used to represent information about external sites known +to a MediaWiki installation. This information about external sites is used to allow "inter-wiki" +links, cross-language navigation, as well as close integration via direct access to the other +site's web API or even directly to their database. + +Lists of external sites can be imported and exported using the importSites.php and exportSites.php +scripts. In the database, external sites are described by the sites and site_ids tables. + +The formal specification of the format used by importSites.php and exportSites.php can be found in +the sitelist-1.0.xsd file. Below is an example and a brief description of what the individual XML +elements and attributes mean: + + + + + acme.com + acme + Vendor + http://acme.com/ + meta.wikimedia.org + + + de.wikidik.example + de + Dictionary + + http://acme.com/ + + + + +The XML elements are used as follows: + +* sites: The root element, containing a set of site tags. May have a version attribute with the value 1.0. +* site: A site entry, representing an external website. May have a type attribute with one of the following values: +** ''unknown'': (default) any website +** ''mediawiki'': A MediaWiki site +* globalid: A unique identifier for the site. For a given site, the same unique global ID must be used across all wikis in a wiki farm (aka wiki family). +* localid: An identifier for the site, for use on the local wiki. Multiple local IDs may be assigned to a given site. The same local ID can be used to refer to different sites by different wikis on the same farm/family. The localid element may have a type attribute with one of the following values: +** interwiki: Used as an "interwiki" link prefix, for creating cross-wiki links. +** equivalent: Used as a "language" link prefix, for cross-linking equivalent content in different languages. +* group: The site group (e.g. wiki family) the site belongs to. +* path: A URL template for accessing resources on the site. Several paths may be defined for a given site, for accessing different kinds of resources, identified by the type attribute, using one of the following values: +** link: Generic URL template, often the document root. +** page_path: (for mediawiki sites) URL template for wiki pages (corresponds to the target wiki's $wgArticlePath setting) +** file_path: (for mediawiki sites) URL pattern for application entry points and resources (corresponds to the target wiki's $wgScriptPath setting). +* forward: Whether using a prefix defined by a localid tag in the URL will cause the request to be redirected to the corresponding page on the target wiki (currently unused). E.g. whether http://wiki.acme.com/wiki/foo:Buzz should be forwarded to http://wiki.foo.com/read/Buzz. (CAVEAT: not yet implement, can be specified but has no effect) \ No newline at end of file diff --git a/includes/site/SiteImporter.php b/includes/site/SiteImporter.php new file mode 100644 index 0000000000..02c3ca4815 --- /dev/null +++ b/includes/site/SiteImporter.php @@ -0,0 +1,257 @@ +store = $store; + } + + /** + * @return callable + */ + public function getExceptionCallback() { + return $this->exceptionCallback; + } + + /** + * @param callable $exceptionCallback + */ + public function setExceptionCallback( $exceptionCallback ) { + $this->exceptionCallback = $exceptionCallback; + } + + /** + * @param string $file + */ + public function importFromFile( $file ) { + $xml = file_get_contents( $file ); + + if ( $xml === false ) { + throw new RuntimeException( 'Failed to read ' . $file . '!' ); + } + + $this->importFromXML( $xml ); + } + + /** + * @param string $xml + * + * @throws InvalidArgumentException + */ + public function importFromXML( $xml ) { + $document = new DOMDocument(); + + $oldLibXmlErrors = libxml_use_internal_errors( true ); + $ok = $document->loadXML( $xml, LIBXML_NONET ); + + if ( !$ok ) { + $errors = libxml_get_errors(); + libxml_use_internal_errors( $oldLibXmlErrors ); + + foreach ( $errors as $error ) { + /** @var LibXMLError $error */ + throw new InvalidArgumentException( 'Malformed XML: ' . $error->message . ' in line ' . $error->line ); + } + + throw new InvalidArgumentException( 'Malformed XML!' ); + } + + libxml_use_internal_errors( $oldLibXmlErrors ); + $this->importFromDOM( $document->documentElement ); + } + + /** + * @param DOMElement $root + */ + private function importFromDOM( DOMElement $root ) { + $sites = $this->makeSiteList( $root ); + $this->store->saveSites( $sites ); + } + + /** + * @param DOMElement $root + * + * @return Site[] + */ + private function makeSiteList( DOMElement $root ) { + $sites = array(); + + // Old sites, to get the row IDs that correspond to the global site IDs. + // TODO: Get rid of internal row IDs, they just get in the way. Get rid of ORMRow, too. + $oldSites = $this->store->getSites(); + + $current = $root->firstChild; + while ( $current ) { + if ( $current instanceof DOMElement && $current->tagName === 'site' ) { + try { + $site = $this->makeSite( $current ); + $key = $site->getGlobalId(); + + if ( $oldSites->hasSite( $key ) ) { + $oldSite = $oldSites->getSite( $key ); + $site->setInternalId( $oldSite->getInternalId() ); + } + + $sites[$key] = $site; + } catch ( Exception $ex ) { + $this->handleException( $ex ); + } + } + + $current = $current->nextSibling; + } + + return $sites; + } + + /** + * @param DOMElement $siteElement + * + * @return Site + * @throws InvalidArgumentException + */ + public function makeSite( DOMElement $siteElement ) { + if ( $siteElement->tagName !== 'site' ) { + throw new InvalidArgumentException( 'Expected tag, found ' . $siteElement->tagName ); + } + + $type = $this->getAttributeValue( $siteElement, 'type', Site::TYPE_UNKNOWN ); + $site = Site::newForType( $type ); + + $site->setForward( $this->hasChild( $siteElement, 'forward' ) ); + $site->setGlobalId( $this->getChildText( $siteElement, 'globalid' ) ); + $site->setGroup( $this->getChildText( $siteElement, 'group', Site::GROUP_NONE ) ); + $site->setSource( $this->getChildText( $siteElement, 'source', Site::SOURCE_LOCAL ) ); + + $pathTags = $siteElement->getElementsByTagName( 'path' ); + for ( $i = 0; $i < $pathTags->length; $i++ ) { + $pathElement = $pathTags->item( $i ); + $pathType = $this->getAttributeValue( $pathElement, 'type' ); + $path = $pathElement->textContent; + + $site->setPath( $pathType, $path ); + } + + $idTags = $siteElement->getElementsByTagName( 'localid' ); + for ( $i = 0; $i < $idTags->length; $i++ ) { + $idElement = $idTags->item( $i ); + $idType = $this->getAttributeValue( $idElement, 'type' ); + $id = $idElement->textContent; + + $site->addLocalId( $idType, $id ); + } + + //@todo: import + //@todo: import + + return $site; + } + + /** + * @param DOMElement $element + * @param $name + * @param string|null|bool $default + * + * @return null|string + * @throws MWException If the attribute is not found and no default is provided + */ + private function getAttributeValue( DOMElement $element, $name, $default = false ) { + $node = $element->getAttributeNode( $name ); + + if ( !$node ) { + if ( $default !== false ) { + return $default; + } else { + throw new MWException( 'Required ' . $name . ' attribute not found in <' . $element->tagName . '> tag' ); + } + } + + return $node->textContent; + } + + /** + * @param DOMElement $element + * @param string $name + * @param string|null|bool $default + * + * @return null|string + * @throws MWException If the child element is not found and no default is provided + */ + private function getChildText( DOMElement $element, $name, $default = false ) { + $elements = $element->getElementsByTagName( $name ); + + if ( $elements->length < 1 ) { + if ( $default !== false ) { + return $default; + } else { + throw new MWException( 'Required <' . $name . '> tag not found inside <' . $element->tagName . '> tag' ); + } + } + + $node = $elements->item( 0 ); + return $node->textContent; + } + + /** + * @param DOMElement $element + * @param string $name + * + * @return bool + * @throws MWException + */ + private function hasChild( DOMElement $element, $name ) { + return $this->getChildText( $element, $name, null ) !== null; + } + + /** + * @param Exception $ex + */ + private function handleException( Exception $ex ) { + if ( $this->exceptionCallback ) { + call_user_func( $this->exceptionCallback, $ex ); + } else { + wfLogWarning( $ex->getMessage() ); + } + } + +} diff --git a/maintenance/importSites.php b/maintenance/importSites.php new file mode 100644 index 0000000000..7abb8d72b8 --- /dev/null +++ b/maintenance/importSites.php @@ -0,0 +1,52 @@ +mDescription = 'Imports site definitions from XML into the sites table.'; + + $this->addArg( 'file', 'An XML file containing site definitions (see docs/sitelist.txt). Use "php://stdin" to read from stdin.', true ); + + parent::__construct(); + } + + + /** + * Do the import. + */ + public function execute() { + $file = $this->getArg( 0 ); + + $importer = new SiteImporter( SiteSQLStore::newInstance() ); + $importer->setExceptionCallback( array( $this, 'reportException' ) ); + + $importer->importFromFile( $file ); + + $this->output( "Done.\n" ); + } + + /** + * Outputs a message via the output() method. + * + * @param Exception $ex + */ + public function reportException( Exception $ex ) { + $msg = $ex->getMessage(); + $this->output( "$msg\n" ); + } +} + +$maintClass = 'ImportSites'; +require_once( RUN_MAINTENANCE_IF_MAIN ); diff --git a/tests/phpunit/includes/site/SiteImporterTest.php b/tests/phpunit/includes/site/SiteImporterTest.php new file mode 100644 index 0000000000..ceef1bfc35 --- /dev/null +++ b/tests/phpunit/includes/site/SiteImporterTest.php @@ -0,0 +1,201 @@ +getMock( 'SiteStore' ); + + $self = $this; + $store->expects( $this->once() ) + ->method( 'saveSites' ) + ->will( $this->returnCallback( function ( $sites ) use ( $expectedSites, $self ) { + $self->assertSitesEqual( $expectedSites, $sites ); + } ) ); + + $store->expects( $this->any() ) + ->method( 'getSites' ) + ->will( $this->returnValue( new SiteList() ) ); + + $errorHandler = $this->getMock( 'Psr\Log\LoggerInterface' ); + $errorHandler->expects( $this->exactly( $errorCount ) ) + ->method( 'error' ); + + $importer = new SiteImporter( $store ); + $importer->setExceptionCallback( array( $errorHandler, 'error' ) ); + + return $importer; + } + + public function assertSitesEqual( $expected, $actual, $message = '' ) { + $this->assertEquals( + $this->getSerializedSiteList( $expected ), + $this->getSerializedSiteList( $actual ), + $message + ); + } + + public function provideImportFromXML() { + $foo = Site::newForType( Site::TYPE_UNKNOWN ); + $foo->setGlobalId( 'Foo' ); + + $acme = Site::newForType( Site::TYPE_UNKNOWN ); + $acme->setGlobalId( 'acme.com' ); + $acme->setGroup( 'Test' ); + $acme->addLocalId( Site::ID_INTERWIKI, 'acme' ); + $acme->setPath( Site::PATH_LINK, 'http://acme.com/' ); + + $dewiki = Site::newForType( Site::TYPE_MEDIAWIKI ); + $dewiki->setGlobalId( 'dewiki' ); + $dewiki->setGroup( 'wikipedia' ); + $dewiki->setForward( true ); + $dewiki->addLocalId( Site::ID_INTERWIKI, 'wikipedia' ); + $dewiki->addLocalId( Site::ID_EQUIVALENT, 'de' ); + $dewiki->setPath( Site::PATH_LINK, 'http://de.wikipedia.org/w/' ); + $dewiki->setPath( MediaWikiSite::PATH_PAGE, 'http://de.wikipedia.org/wiki/' ); + $dewiki->setSource( 'meta.wikimedia.org' ); + + return array( + 'empty' => array( + '', + array(), + ), + 'no sites' => array( + 'FooBla', + array(), + ), + 'minimal' => array( + '' . + 'Foo' . + '', + array( $foo ), + ), + 'full' => array( + '' . + 'Foo' . + '' . + 'acme.com' . + 'acme' . + 'Test' . + 'http://acme.com/' . + '' . + '' . + 'meta.wikimedia.org' . + 'dewiki' . + 'wikipedia' . + 'de' . + 'wikipedia' . + '' . + 'http://de.wikipedia.org/w/' . + 'http://de.wikipedia.org/wiki/' . + '' . + '', + array( $foo, $acme, $dewiki ), + ), + 'skip' => array( + '' . + 'Foo' . + 'Foo' . + '' . + 'acme.com' . + 'acme' . + 'boop!' . + 'Test' . + 'http://acme.com/' . + '' . + '', + array( $foo, $acme ), + 1 + ), + ); + } + + /** + * @dataProvider provideImportFromXML + */ + public function testImportFromXML( $xml, array $expectedSites, $errorCount = 0 ) { + $importer = $this->newSiteImporter( $expectedSites, $errorCount ); + $importer->importFromXML( $xml ); + } + + public function testImportFromXML_malformed() { + $this->setExpectedException( 'Exception' ); + + $store = $this->getMock( 'SiteStore' ); + $importer = new SiteImporter( $store ); + $importer->importFromXML( 'THIS IS NOT XML' ); + } + + public function testImportFromFile() { + $foo = Site::newForType( Site::TYPE_UNKNOWN ); + $foo->setGlobalId( 'Foo' ); + + $acme = Site::newForType( Site::TYPE_UNKNOWN ); + $acme->setGlobalId( 'acme.com' ); + $acme->setGroup( 'Test' ); + $acme->addLocalId( Site::ID_INTERWIKI, 'acme' ); + $acme->setPath( Site::PATH_LINK, 'http://acme.com/' ); + + $dewiki = Site::newForType( Site::TYPE_MEDIAWIKI ); + $dewiki->setGlobalId( 'dewiki' ); + $dewiki->setGroup( 'wikipedia' ); + $dewiki->setForward( true ); + $dewiki->addLocalId( Site::ID_INTERWIKI, 'wikipedia' ); + $dewiki->addLocalId( Site::ID_EQUIVALENT, 'de' ); + $dewiki->setPath( Site::PATH_LINK, 'http://de.wikipedia.org/w/' ); + $dewiki->setPath( MediaWikiSite::PATH_PAGE, 'http://de.wikipedia.org/wiki/' ); + $dewiki->setSource( 'meta.wikimedia.org' ); + + $importer = $this->newSiteImporter( array( $foo, $acme, $dewiki ), 0 ); + + $file = __DIR__ . '/SiteImporterTest.xml'; + $importer->importFromFile( $file ); + } + + /** + * @param Site[] $sites + * + * @return array[] + */ + private function getSerializedSiteList( $sites ) { + $serialized = array(); + + foreach ( $sites as $site ) { + $key = $site->getGlobalId(); + $data = unserialize( $site->serialize() ); + + $serialized[$key] = $data; + } + + return $serialized; + } +} diff --git a/tests/phpunit/includes/site/SiteImporterTest.xml b/tests/phpunit/includes/site/SiteImporterTest.xml new file mode 100644 index 0000000000..720b1faf1a --- /dev/null +++ b/tests/phpunit/includes/site/SiteImporterTest.xml @@ -0,0 +1,19 @@ + + Foo + + acme.com + acme + Test + http://acme.com/ + + + meta.wikimedia.org + dewiki + wikipedia + de + wikipedia + + http://de.wikipedia.org/w/ + http://de.wikipedia.org/wiki/ + + -- 2.20.1