includes/specials/SpecialExport.php

   1 <?php
   2 /**
   3  * Implements Special:Export
   4  *
   5  * Copyright © 2003-2008 Brion Vibber <brion@pobox.com>
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License as published by
   9  * the Free Software Foundation; either version 2 of the License, or
  10  * (at your option) any later version.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  15  * GNU General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU General Public License along
  18  * with this program; if not, write to the Free Software Foundation, Inc.,
  19  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  20  * http://www.gnu.org/copyleft/gpl.html
  21  *
  22  * @file
  23  * @ingroup SpecialPage
  24  */
  25
  26 use MediaWiki\MediaWikiServices;
  27
  28 /**
  29  * A special page that allows users to export pages in a XML file
  30  *
  31  * @ingroup SpecialPage
  32  */
  33 class SpecialExport extends SpecialPage {
  34         private $curonly, $doExport, $pageLinkDepth, $templates;
  35
  36         public function __construct() {
  37                 parent::__construct( 'Export' );
  38         }
  39
  40         public function execute( $par ) {
  41                 $this->setHeaders();
  42                 $this->outputHeader();
  43                 $config = $this->getConfig();
  44
  45                 // Set some variables
  46                 $this->curonly = true;
  47                 $this->doExport = false;
  48                 $request = $this->getRequest();
  49                 $this->templates = $request->getCheck( 'templates' );
  50                 $this->pageLinkDepth = $this->validateLinkDepth(
  51                         $request->getIntOrNull( 'pagelink-depth' )
  52                 );
  53                 $nsindex = '';
  54                 $exportall = false;
  55
  56                 if ( $request->getCheck( 'addcat' ) ) {
  57                         $page = $request->getText( 'pages' );
  58                         $catname = $request->getText( 'catname' );
  59
  60                         if ( $catname !== '' && $catname !== null && $catname !== false ) {
  61                                 $t = Title::makeTitleSafe( NS_MAIN, $catname );
  62                                 if ( $t ) {
  63                                         /**
  64                                          * @todo FIXME: This can lead to hitting memory limit for very large
  65                                          * categories. Ideally we would do the lookup synchronously
  66                                          * during the export in a single query.
  67                                          */
  68                                         $catpages = $this->getPagesFromCategory( $t );
  69                                         if ( $catpages ) {
  70                                                 if ( $page !== '' ) {
  71                                                         $page .= "\n";
  72                                                 }
  73                                                 $page .= implode( "\n", $catpages );
  74                                         }
  75                                 }
  76                         }
  77                 } elseif ( $request->getCheck( 'addns' ) && $config->get( 'ExportFromNamespaces' ) ) {
  78                         $page = $request->getText( 'pages' );
  79                         $nsindex = $request->getText( 'nsindex', '' );
  80
  81                         if ( strval( $nsindex ) !== '' ) {
  82                                 /**
  83                                  * Same implementation as above, so same @todo
  84                                  */
  85                                 $nspages = $this->getPagesFromNamespace( $nsindex );
  86                                 if ( $nspages ) {
  87                                         $page .= "\n" . implode( "\n", $nspages );
  88                                 }
  89                         }
  90                 } elseif ( $request->getCheck( 'exportall' ) && $config->get( 'ExportAllowAll' ) ) {
  91                         $this->doExport = true;
  92                         $exportall = true;
  93
  94                         /* Although $page and $history are not used later on, we
  95                         nevertheless set them to avoid that PHP notices about using
  96                         undefined variables foul up our XML output (see call to
  97                         doExport(...) further down) */
  98                         $page = '';
  99                         $history = '';
 100                 } elseif ( $request->wasPosted() && $par == '' ) {
 101                         $page = $request->getText( 'pages' );
 102                         $this->curonly = $request->getCheck( 'curonly' );
 103                         $rawOffset = $request->getVal( 'offset' );
 104
 105                         if ( $rawOffset ) {
 106                                 $offset = wfTimestamp( TS_MW, $rawOffset );
 107                         } else {
 108                                 $offset = null;
 109                         }
 110
 111                         $maxHistory = $config->get( 'ExportMaxHistory' );
 112                         $limit = $request->getInt( 'limit' );
 113                         $dir = $request->getVal( 'dir' );
 114                         $history = [
 115                                 'dir' => 'asc',
 116                                 'offset' => false,
 117                                 'limit' => $maxHistory,
 118                         ];
 119                         $historyCheck = $request->getCheck( 'history' );
 120
 121                         if ( $this->curonly ) {
 122                                 $history = WikiExporter::CURRENT;
 123                         } elseif ( !$historyCheck ) {
 124                                 if ( $limit > 0 && ( $maxHistory == 0 || $limit < $maxHistory ) ) {
 125                                         $history['limit'] = $limit;
 126                                 }
 127
 128                                 if ( !is_null( $offset ) ) {
 129                                         $history['offset'] = $offset;
 130                                 }
 131
 132                                 if ( strtolower( $dir ) == 'desc' ) {
 133                                         $history['dir'] = 'desc';
 134                                 }
 135                         }
 136
 137                         if ( $page != '' ) {
 138                                 $this->doExport = true;
 139                         }
 140                 } else {
 141                         // Default to current-only for GET requests.
 142                         $page = $request->getText( 'pages', $par );
 143                         $historyCheck = $request->getCheck( 'history' );
 144
 145                         if ( $historyCheck ) {
 146                                 $history = WikiExporter::FULL;
 147                         } else {
 148                                 $history = WikiExporter::CURRENT;
 149                         }
 150
 151                         if ( $page != '' ) {
 152                                 $this->doExport = true;
 153                         }
 154                 }
 155
 156                 if ( !$config->get( 'ExportAllowHistory' ) ) {
 157                         // Override
 158                         $history = WikiExporter::CURRENT;
 159                 }
 160
 161                 $list_authors = $request->getCheck( 'listauthors' );
 162                 if ( !$this->curonly || !$config->get( 'ExportAllowListContributors' ) ) {
 163                         $list_authors = false;
 164                 }
 165
 166                 if ( $this->doExport ) {
 167                         $this->getOutput()->disable();
 168
 169                         // Cancel output buffering and gzipping if set
 170                         // This should provide safer streaming for pages with history
 171                         wfResetOutputBuffers();
 172                         $request->response()->header( "Content-type: application/xml; charset=utf-8" );
 173                         $request->response()->header( "X-Robots-Tag: noindex,nofollow" );
 174
 175                         if ( $request->getCheck( 'wpDownload' ) ) {
 176                                 // Provide a sane filename suggestion
 177                                 $filename = urlencode( $config->get( 'Sitename' ) . '-' . wfTimestampNow() . '.xml' );
 178                                 $request->response()->header( "Content-disposition: attachment;filename={$filename}" );
 179                         }
 180
 181                         $this->doExport( $page, $history, $list_authors, $exportall );
 182
 183                         return;
 184                 }
 185
 186                 $out = $this->getOutput();
 187                 $out->addWikiMsg( 'exporttext' );
 188
 189                 if ( $page == '' ) {
 190                         $categoryName = $request->getText( 'catname' );
 191                 } else {
 192                         $categoryName = '';
 193                 }
 194
 195                 $formDescriptor = [
 196                         'catname' => [
 197                                 'type' => 'textwithbutton',
 198                                 'name' => 'catname',
 199                                 'horizontal-label' => true,
 200                                 'label-message' => 'export-addcattext',
 201                                 'default' => $categoryName,
 202                                 'size' => 40,
 203                                 'buttontype' => 'submit',
 204                                 'buttonname' => 'addcat',
 205                                 'buttondefault' => $this->msg( 'export-addcat' )->text(),
 206                                 'hide-if' => [ '===', 'exportall', '1' ],
 207                         ],
 208                 ];
 209                 if ( $config->get( 'ExportFromNamespaces' ) ) {
 210                         $formDescriptor += [
 211                                 'nsindex' => [
 212                                         'type' => 'namespaceselectwithbutton',
 213                                         'default' => $nsindex,
 214                                         'label-message' => 'export-addnstext',
 215                                         'horizontal-label' => true,
 216                                         'name' => 'nsindex',
 217                                         'id' => 'namespace',
 218                                         'cssclass' => 'namespaceselector',
 219                                         'buttontype' => 'submit',
 220                                         'buttonname' => 'addns',
 221                                         'buttondefault' => $this->msg( 'export-addns' )->text(),
 222                                         'hide-if' => [ '===', 'exportall', '1' ],
 223                                 ],
 224                         ];
 225                 }
 226
 227                 if ( $config->get( 'ExportAllowAll' ) ) {
 228                         $formDescriptor += [
 229                                 'exportall' => [
 230                                         'type' => 'check',
 231                                         'label-message' => 'exportall',
 232                                         'name' => 'exportall',
 233                                         'id' => 'exportall',
 234                                         'default' => $request->wasPosted() ? $request->getCheck( 'exportall' ) : false,
 235                                 ],
 236                         ];
 237                 }
 238
 239                 $formDescriptor += [
 240                         'textarea' => [
 241                                 'class' => HTMLTextAreaField::class,
 242                                 'name' => 'pages',
 243                                 'label-message' => 'export-manual',
 244                                 'nodata' => true,
 245                                 'rows' => 10,
 246                                 'default' => $page,
 247                                 'hide-if' => [ '===', 'exportall', '1' ],
 248                         ],
 249                 ];
 250
 251                 if ( $config->get( 'ExportAllowHistory' ) ) {
 252                         $formDescriptor += [
 253                                 'curonly' => [
 254                                         'type' => 'check',
 255                                         'label-message' => 'exportcuronly',
 256                                         'name' => 'curonly',
 257                                         'id' => 'curonly',
 258                                         'default' => $request->wasPosted() ? $request->getCheck( 'curonly' ) : true,
 259                                 ],
 260                         ];
 261                 } else {
 262                         $out->addWikiMsg( 'exportnohistory' );
 263                 }
 264
 265                 $formDescriptor += [
 266                         'templates' => [
 267                                 'type' => 'check',
 268                                 'label-message' => 'export-templates',
 269                                 'name' => 'templates',
 270                                 'id' => 'wpExportTemplates',
 271                                 'default' => $request->wasPosted() ? $request->getCheck( 'templates' ) : false,
 272                         ],
 273                 ];
 274
 275                 if ( $config->get( 'ExportMaxLinkDepth' ) || $this->userCanOverrideExportDepth() ) {
 276                         $formDescriptor += [
 277                                 'pagelink-depth' => [
 278                                         'type' => 'text',
 279                                         'name' => 'pagelink-depth',
 280                                         'id' => 'pagelink-depth',
 281                                         'label-message' => 'export-pagelinks',
 282                                         'default' => '0',
 283                                         'size' => 20,
 284                                 ],
 285                         ];
 286                 }
 287
 288                 $formDescriptor += [
 289                         'wpDownload' => [
 290                                 'type' => 'check',
 291                                 'name' => 'wpDownload',
 292                                 'id' => 'wpDownload',
 293                                 'default' => $request->wasPosted() ? $request->getCheck( 'wpDownload' ) : true,
 294                                 'label-message' => 'export-download',
 295                         ],
 296                 ];
 297
 298                 if ( $config->get( 'ExportAllowListContributors' ) ) {
 299                         $formDescriptor += [
 300                                 'listauthors' => [
 301                                         'type' => 'check',
 302                                         'label-message' => 'exportlistauthors',
 303                                         'default' => $request->wasPosted() ? $request->getCheck( 'listauthors' ) : false,
 304                                         'name' => 'listauthors',
 305                                         'id' => 'listauthors',
 306                                 ],
 307                         ];
 308                 }
 309
 310                 $htmlForm = HTMLForm::factory( 'ooui', $formDescriptor, $this->getContext() );
 311                 $htmlForm->setSubmitTextMsg( 'export-submit' );
 312                 $htmlForm->prepareForm()->displayForm( false );
 313                 $this->addHelpLink( 'Help:Export' );
 314         }
 315
 316         /**
 317          * @return bool
 318          */
 319         private function userCanOverrideExportDepth() {
 320                 return $this->getUser()->isAllowed( 'override-export-depth' );
 321         }
 322
 323         /**
 324          * Do the actual page exporting
 325          *
 326          * @param string $page User input on what page(s) to export
 327          * @param int $history One of the WikiExporter history export constants
 328          * @param bool $list_authors Whether to add distinct author list (when
 329          *   not returning full history)
 330          * @param bool $exportall Whether to export everything
 331          */
 332         private function doExport( $page, $history, $list_authors, $exportall ) {
 333                 // If we are grabbing everything, enable full history and ignore the rest
 334                 if ( $exportall ) {
 335                         $history = WikiExporter::FULL;
 336                 } else {
 337                         $pageSet = []; // Inverted index of all pages to look up
 338
 339                         // Split up and normalize input
 340                         foreach ( explode( "\n", $page ) as $pageName ) {
 341                                 $pageName = trim( $pageName );
 342                                 $title = Title::newFromText( $pageName );
 343                                 if ( $title && !$title->isExternal() && $title->getText() !== '' ) {
 344                                         // Only record each page once!
 345                                         $pageSet[$title->getPrefixedText()] = true;
 346                                 }
 347                         }
 348
 349                         // Set of original pages to pass on to further manipulation...
 350                         $inputPages = array_keys( $pageSet );
 351
 352                         // Look up any linked pages if asked...
 353                         if ( $this->templates ) {
 354                                 $pageSet = $this->getTemplates( $inputPages, $pageSet );
 355                         }
 356                         $linkDepth = $this->pageLinkDepth;
 357                         if ( $linkDepth ) {
 358                                 $pageSet = $this->getPageLinks( $inputPages, $pageSet, $linkDepth );
 359                         }
 360
 361                         $pages = array_keys( $pageSet );
 362
 363                         // Normalize titles to the same format and remove dupes, see T19374
 364                         foreach ( $pages as $k => $v ) {
 365                                 $pages[$k] = str_replace( " ", "_", $v );
 366                         }
 367
 368                         $pages = array_unique( $pages );
 369                 }
 370
 371                 /* Ok, let's get to it... */
 372                 if ( $history == WikiExporter::CURRENT ) {
 373                         $lb = false;
 374                         $db = wfGetDB( DB_REPLICA );
 375                         $buffer = WikiExporter::BUFFER;
 376                 } else {
 377                         // Use an unbuffered query; histories may be very long!
 378                         $lb = MediaWikiServices::getInstance()->getDBLoadBalancerFactory()->newMainLB();
 379                         $db = $lb->getConnection( DB_REPLICA );
 380                         $buffer = WikiExporter::STREAM;
 381
 382                         // This might take a while... :D
 383                         Wikimedia\suppressWarnings();
 384                         set_time_limit( 0 );
 385                         Wikimedia\restoreWarnings();
 386                 }
 387
 388                 $exporter = new WikiExporter( $db, $history, $buffer );
 389                 $exporter->list_authors = $list_authors;
 390                 $exporter->openStream();
 391
 392                 if ( $exportall ) {
 393                         $exporter->allPages();
 394                 } else {
 395                         foreach ( $pages as $page ) {
 396                                 # T10824: Only export pages the user can read
 397                                 $title = Title::newFromText( $page );
 398                                 if ( is_null( $title ) ) {
 399                                         // @todo Perhaps output an <error> tag or something.
 400                                         continue;
 401                                 }
 402
 403                                 if ( !$title->userCan( 'read', $this->getUser() ) ) {
 404                                         // @todo Perhaps output an <error> tag or something.
 405                                         continue;
 406                                 }
 407
 408                                 $exporter->pageByTitle( $title );
 409                         }
 410                 }
 411
 412                 $exporter->closeStream();
 413
 414                 if ( $lb ) {
 415                         $lb->closeAll();
 416                 }
 417         }
 418
 419         /**
 420          * @param Title $title
 421          * @return array
 422          */
 423         private function getPagesFromCategory( $title ) {
 424                 $maxPages = $this->getConfig()->get( 'ExportPagelistLimit' );
 425
 426                 $name = $title->getDBkey();
 427
 428                 $dbr = wfGetDB( DB_REPLICA );
 429                 $res = $dbr->select(
 430                         [ 'page', 'categorylinks' ],
 431                         [ 'page_namespace', 'page_title' ],
 432                         [ 'cl_from=page_id', 'cl_to' => $name ],
 433                         __METHOD__,
 434                         [ 'LIMIT' => $maxPages ]
 435                 );
 436
 437                 $pages = [];
 438
 439                 foreach ( $res as $row ) {
 440                         $n = $row->page_title;
 441                         if ( $row->page_namespace ) {
 442                                 $ns = MediaWikiServices::getInstance()->getContentLanguage()->getNsText(
 443                                         $row->page_namespace );
 444                                 $n = $ns . ':' . $n;
 445                         }
 446
 447                         $pages[] = $n;
 448                 }
 449
 450                 return $pages;
 451         }
 452
 453         /**
 454          * @param int $nsindex
 455          * @return array
 456          */
 457         private function getPagesFromNamespace( $nsindex ) {
 458                 $maxPages = $this->getConfig()->get( 'ExportPagelistLimit' );
 459
 460                 $dbr = wfGetDB( DB_REPLICA );
 461                 $res = $dbr->select(
 462                         'page',
 463                         [ 'page_namespace', 'page_title' ],
 464                         [ 'page_namespace' => $nsindex ],
 465                         __METHOD__,
 466                         [ 'LIMIT' => $maxPages ]
 467                 );
 468
 469                 $pages = [];
 470
 471                 foreach ( $res as $row ) {
 472                         $n = $row->page_title;
 473
 474                         if ( $row->page_namespace ) {
 475                                 $ns = MediaWikiServices::getInstance()->getContentLanguage()->getNsText(
 476                                         $row->page_namespace );
 477                                 $n = $ns . ':' . $n;
 478                         }
 479
 480                         $pages[] = $n;
 481                 }
 482
 483                 return $pages;
 484         }
 485
 486         /**
 487          * Expand a list of pages to include templates used in those pages.
 488          * @param array $inputPages List of titles to look up
 489          * @param array $pageSet Associative array indexed by titles for output
 490          * @return array Associative array index by titles
 491          */
 492         private function getTemplates( $inputPages, $pageSet ) {
 493                 return $this->getLinks( $inputPages, $pageSet,
 494                         'templatelinks',
 495                         [ 'namespace' => 'tl_namespace', 'title' => 'tl_title' ],
 496                         [ 'page_id=tl_from' ]
 497                 );
 498         }
 499
 500         /**
 501          * Validate link depth setting, if available.
 502          * @param int $depth
 503          * @return int
 504          */
 505         private function validateLinkDepth( $depth ) {
 506                 if ( $depth < 0 ) {
 507                         return 0;
 508                 }
 509
 510                 if ( !$this->userCanOverrideExportDepth() ) {
 511                         $maxLinkDepth = $this->getConfig()->get( 'ExportMaxLinkDepth' );
 512                         if ( $depth > $maxLinkDepth ) {
 513                                 return $maxLinkDepth;
 514                         }
 515                 }
 516
 517                 /*
 518                  * There's a HARD CODED limit of 5 levels of recursion here to prevent a
 519                  * crazy-big export from being done by someone setting the depth
 520                  * number too high. In other words, last resort safety net.
 521                  */
 522
 523                 return intval( min( $depth, 5 ) );
 524         }
 525
 526         /**
 527          * Expand a list of pages to include pages linked to from that page.
 528          * @param array $inputPages
 529          * @param array $pageSet
 530          * @param int $depth
 531          * @return array
 532          */
 533         private function getPageLinks( $inputPages, $pageSet, $depth ) {
 534                 for ( ; $depth > 0; --$depth ) {
 535                         $pageSet = $this->getLinks(
 536                                 $inputPages, $pageSet, 'pagelinks',
 537                                 [ 'namespace' => 'pl_namespace', 'title' => 'pl_title' ],
 538                                 [ 'page_id=pl_from' ]
 539                         );
 540                         $inputPages = array_keys( $pageSet );
 541                 }
 542
 543                 return $pageSet;
 544         }
 545
 546         /**
 547          * Expand a list of pages to include items used in those pages.
 548          * @param array $inputPages Array of page titles
 549          * @param array $pageSet
 550          * @param string $table
 551          * @param array $fields Array of field names
 552          * @param array $join
 553          * @return array
 554          */
 555         private function getLinks( $inputPages, $pageSet, $table, $fields, $join ) {
 556                 $dbr = wfGetDB( DB_REPLICA );
 557
 558                 foreach ( $inputPages as $page ) {
 559                         $title = Title::newFromText( $page );
 560
 561                         if ( $title ) {
 562                                 $pageSet[$title->getPrefixedText()] = true;
 563                                 /// @todo FIXME: May or may not be more efficient to batch these
 564                                 ///        by namespace when given multiple input pages.
 565                                 $result = $dbr->select(
 566                                         [ 'page', $table ],
 567                                         $fields,
 568                                         array_merge(
 569                                                 $join,
 570                                                 [
 571                                                         'page_namespace' => $title->getNamespace(),
 572                                                         'page_title' => $title->getDBkey()
 573                                                 ]
 574                                         ),
 575                                         __METHOD__
 576                                 );
 577
 578                                 foreach ( $result as $row ) {
 579                                         $template = Title::makeTitle( $row->namespace, $row->title );
 580                                         $pageSet[$template->getPrefixedText()] = true;
 581                                 }
 582                         }
 583                 }
 584
 585                 return $pageSet;
 586         }
 587
 588         protected function getGroupName() {
 589                 return 'pagetools';
 590         }
 591 }