From dc2fd34eb1ee1eac053a668e5d2865e1d90b2ab5 Mon Sep 17 00:00:00 2001
From: Stanislav Malyshev <smalyshev@gmail.com>
Date: Fri, 17 Nov 2017 15:58:37 -0800
Subject: [PATCH] Add more data to categories dump

Hidden category: <cat> a mediawiki:HiddenCategory .
Pages count: <cat> mediawiki:pages 10 .
Subcats count: <cat> mediawiki:subcategories 5 .

Note that pages count includes only actual articles, unlike mediawiki
table.

Bug: T173980
Change-Id: I6d34c58f844411f891195776406e11acd2aef7b1
---
 docs/ontology.owl                             | 19 +++++++++++
 includes/CategoriesRdf.php                    | 11 ++++++-
 maintenance/dumpCategoriesAsRdf.php           | 32 +++++++++++++++++--
 .../data/categoriesrdf/categoriesRdf-out.nt   |  7 ++++
 .../phpunit/maintenance/categoriesRdfTest.php | 27 ++++++++++++++--
 5 files changed, 89 insertions(+), 7 deletions(-)

diff --git a/docs/ontology.owl b/docs/ontology.owl
index 6b2e0b7f44..19476a35f2 100644
--- a/docs/ontology.owl
+++ b/docs/ontology.owl
@@ -38,6 +38,11 @@
     <rdfs:comment>MediaWiki category.</rdfs:comment>
   </owl:Class>
 
+  <owl:Class rdf:about="&mediawiki;HiddenCategory">
+    <rdfs:label>HiddenCategory</rdfs:label>
+    <rdfs:comment>MediaWiki hidden category.</rdfs:comment>
+  </owl:Class>
+
   <!--
   ///////////////////////////////////////////////////////////////////////////////////////
   //
@@ -53,4 +58,18 @@
       <rdfs:domain rdf:resource="&mediawiki;Category"/>
   </owl:ObjectProperty>
 
+  <owl:DatatypeProperty rdf:about="&mediawiki;pages">
+      <rdfs:label>pages</rdfs:label>
+      <rdfs:comment>Number of articles belonging to this category.</rdfs:comment>
+      <rdfs:range rdf:resource="&mediawiki;Category"/>
+      <rdfs:range rdf:resource="&xsd;integer"/>
+  </owl:DatatypeProperty>
+
+  <owl:DatatypeProperty rdf:about="&mediawiki;subcategories">
+      <rdfs:label>subcategories</rdfs:label>
+      <rdfs:comment>Number of subcategories belonging to this category.</rdfs:comment>
+      <rdfs:range rdf:resource="&mediawiki;Category"/>
+      <rdfs:range rdf:resource="&xsd;integer"/>
+  </owl:DatatypeProperty>
+
 </rdf:RDF>
diff --git a/includes/CategoriesRdf.php b/includes/CategoriesRdf.php
index 463f6e846b..fc296d4c32 100644
--- a/includes/CategoriesRdf.php
+++ b/includes/CategoriesRdf.php
@@ -80,14 +80,23 @@ class CategoriesRdf {
 	/**
 	 * Write out the data for single category.
 	 * @param string $categoryName Category name
+	 * @param bool $isHidden Hidden category?
+	 * @param int $pages Page count (note this includes only Wiki articles, not subcats or files)
+	 * @param int $subcategories Subcategory count
 	 */
-	public function writeCategoryData( $categoryName ) {
+	public function writeCategoryData( $categoryName, $isHidden, $pages, $subcategories ) {
 		$title = Title::makeTitle( NS_CATEGORY, $categoryName );
 		$this->rdfWriter->about( $this->titleToUrl( $title ) )
 			->say( 'a' )
 			->is( self::ONTOLOGY_PREFIX, 'Category' );
+		if ( $isHidden ) {
+			$this->rdfWriter->is( self::ONTOLOGY_PREFIX, 'HiddenCategory' );
+		}
 		$titletext = $title->getText();
 		$this->rdfWriter->say( 'rdfs', 'label' )->value( $titletext );
+		$this->rdfWriter->say( self::ONTOLOGY_PREFIX, 'pages' )->value( $pages );
+		$this->rdfWriter->say( self::ONTOLOGY_PREFIX, 'subcategories' )->value( $subcategories );
+		// TODO: do we want files too here? Easy to add, but don't have use case so far.
 	}
 
 	/**
diff --git a/maintenance/dumpCategoriesAsRdf.php b/maintenance/dumpCategoriesAsRdf.php
index c1835d05d1..3467932940 100644
--- a/maintenance/dumpCategoriesAsRdf.php
+++ b/maintenance/dumpCategoriesAsRdf.php
@@ -58,14 +58,32 @@ class DumpCategoriesAsRdf extends Maintenance {
 	public function getCategoryIterator( IDatabase $dbr ) {
 		$it = new BatchRowIterator(
 			$dbr,
-			'page',
+			[ 'page', 'page_props', 'category' ],
 			[ 'page_title' ],
 			$this->getBatchSize()
 		);
 		$it->addConditions( [
 			'page_namespace' => NS_CATEGORY,
 		] );
-		$it->setFetchColumns( [ 'page_title', 'page_id' ] );
+		$it->setFetchColumns( [
+			'page_title',
+			'page_id',
+			'pp_propname',
+			'cat_pages',
+			'cat_subcats',
+			'cat_files'
+		] );
+		$it->addJoinConditions(
+			[
+				'page_props' => [
+					'LEFT JOIN', [ 'pp_propname' => 'hiddencat', 'pp_page = page_id' ]
+				],
+				'category' => [
+					'LEFT JOIN', [ 'cat_title = page_title' ]
+				]
+			]
+
+		);
 		return $it;
 	}
 
@@ -90,6 +108,9 @@ class DumpCategoriesAsRdf extends Maintenance {
 		return new RecursiveIteratorIterator( $it );
 	}
 
+	/**
+	 * @param int $timestamp
+	 */
 	public function addDumpHeader( $timestamp ) {
 		global $wgRightsUrl;
 		$licenseUrl = $wgRightsUrl;
@@ -129,7 +150,12 @@ class DumpCategoriesAsRdf extends Maintenance {
 		foreach ( $this->getCategoryIterator( $dbr ) as $batch ) {
 			$pages = [];
 			foreach ( $batch as $row ) {
-				$this->categoriesRdf->writeCategoryData( $row->page_title );
+				$this->categoriesRdf->writeCategoryData(
+					$row->page_title,
+					$row->pp_propname === 'hiddencat',
+					(int)$row->cat_pages - (int)$row->cat_subcats - (int)$row->cat_files,
+					(int)$row->cat_subcats
+				);
 				$pages[$row->page_id] = $row->page_title;
 			}
 
diff --git a/tests/phpunit/data/categoriesrdf/categoriesRdf-out.nt b/tests/phpunit/data/categoriesrdf/categoriesRdf-out.nt
index b8bd8e02e2..bbb3787021 100644
--- a/tests/phpunit/data/categoriesrdf/categoriesRdf-out.nt
+++ b/tests/phpunit/data/categoriesrdf/categoriesRdf-out.nt
@@ -7,10 +7,17 @@
 <http://acme.test/wiki/Special:CategoryDump> <http://www.w3.org/2002/07/owl#imports> <https://www.mediawiki.org/ontology/ontology.owl> .
 <http://acme.test/wiki/Category:Category_One> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.mediawiki.org/ontology#Category> .
 <http://acme.test/wiki/Category:Category_One> <http://www.w3.org/2000/01/rdf-schema#label> "Category One" .
+<http://acme.test/wiki/Category:Category_One> <https://www.mediawiki.org/ontology#pages> "7"^^<http://www.w3.org/2001/XMLSchema#integer> .
+<http://acme.test/wiki/Category:Category_One> <https://www.mediawiki.org/ontology#subcategories> "10"^^<http://www.w3.org/2001/XMLSchema#integer> .
 <http://acme.test/wiki/Category:2_Category_Two> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.mediawiki.org/ontology#Category> .
+<http://acme.test/wiki/Category:2_Category_Two> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.mediawiki.org/ontology#HiddenCategory> .
 <http://acme.test/wiki/Category:2_Category_Two> <http://www.w3.org/2000/01/rdf-schema#label> "2 Category Two" .
+<http://acme.test/wiki/Category:2_Category_Two> <https://www.mediawiki.org/ontology#pages> "17"^^<http://www.w3.org/2001/XMLSchema#integer> .
+<http://acme.test/wiki/Category:2_Category_Two> <https://www.mediawiki.org/ontology#subcategories> "0"^^<http://www.w3.org/2001/XMLSchema#integer> .
 <http://acme.test/wiki/Category:Category_One> <https://www.mediawiki.org/ontology#isInCategory> <http://acme.test/wiki/Category:Parent_of_1> .
 <http://acme.test/wiki/Category:2_Category_Two> <https://www.mediawiki.org/ontology#isInCategory> <http://acme.test/wiki/Category:Parent_of_2> .
 <http://acme.test/wiki/Category:%D0%A2%D1%80%D0%B5%D1%82%D1%8C%D1%8F_%D0%BA%D0%B0%D1%82%D0%B5%D0%B3%D0%BE%D1%80%D0%B8%D1%8F> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.mediawiki.org/ontology#Category> .
 <http://acme.test/wiki/Category:%D0%A2%D1%80%D0%B5%D1%82%D1%8C%D1%8F_%D0%BA%D0%B0%D1%82%D0%B5%D0%B3%D0%BE%D1%80%D0%B8%D1%8F> <http://www.w3.org/2000/01/rdf-schema#label> "\u0422\u0440\u0435\u0442\u044C\u044F \u043A\u0430\u0442\u0435\u0433\u043E\u0440\u0438\u044F" .
+<http://acme.test/wiki/Category:%D0%A2%D1%80%D0%B5%D1%82%D1%8C%D1%8F_%D0%BA%D0%B0%D1%82%D0%B5%D0%B3%D0%BE%D1%80%D0%B8%D1%8F> <https://www.mediawiki.org/ontology#pages> "0"^^<http://www.w3.org/2001/XMLSchema#integer> .
+<http://acme.test/wiki/Category:%D0%A2%D1%80%D0%B5%D1%82%D1%8C%D1%8F_%D0%BA%D0%B0%D1%82%D0%B5%D0%B3%D0%BE%D1%80%D0%B8%D1%8F> <https://www.mediawiki.org/ontology#subcategories> "0"^^<http://www.w3.org/2001/XMLSchema#integer> .
 <http://acme.test/wiki/Category:%D0%A2%D1%80%D0%B5%D1%82%D1%8C%D1%8F_%D0%BA%D0%B0%D1%82%D0%B5%D0%B3%D0%BE%D1%80%D0%B8%D1%8F> <https://www.mediawiki.org/ontology#isInCategory> <http://acme.test/wiki/Category:Parent_of_3> .
diff --git a/tests/phpunit/maintenance/categoriesRdfTest.php b/tests/phpunit/maintenance/categoriesRdfTest.php
index 2edbae1b7a..c0850ab949 100644
--- a/tests/phpunit/maintenance/categoriesRdfTest.php
+++ b/tests/phpunit/maintenance/categoriesRdfTest.php
@@ -9,12 +9,33 @@ class CategoriesRdfTest extends MediaWikiLangTestCase {
 		return [
 			// batch 1
 			[
-				(object)[ 'page_title' => 'Category One', 'page_id' => 1 ],
-				(object)[ 'page_title' => '2 Category Two', 'page_id' => 2 ],
+				(object)[
+					'page_title' => 'Category One',
+					'page_id' => 1,
+					'pp_propname' => null,
+					'cat_pages' => '20',
+					'cat_subcats' => '10',
+					'cat_files' => '3'
+				],
+				(object)[
+					'page_title' => '2 Category Two',
+					'page_id' => 2,
+					'pp_propname' => 'hiddencat',
+					'cat_pages' => 20,
+					'cat_subcats' => 0,
+					'cat_files' => 3
+				],
 			],
 			// batch 2
 			[
-				(object)[ 'page_title' => 'Третья категория', 'page_id' => 3 ],
+				(object)[
+					'page_title' => 'Третья категория',
+					'page_id' => 3,
+					'pp_propname' => null,
+					'cat_pages' => '0',
+					'cat_subcats' => '0',
+					'cat_files' => '0'
+				],
 			]
 		];
 	}
-- 
2.20.1