d13e0a04c078cafb73fb3fc6fc15fd82abc5937f
3 * Performs transformations of HTML by wrapping around libxml2 and working
4 * around its countless bugs.
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License along
17 * with this program; if not, write to the Free Software Foundation, Inc.,
18 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
19 * http://www.gnu.org/copyleft/gpl.html
30 private $itemsToRemove = array();
31 private $elementsToFlatten = array();
32 protected $removeMedia = false;
37 * @param string $html: Text to process
39 public function __construct( $html ) {
44 * Turns a chunk of HTML into a proper document
48 public static function wrapHTML( $html ) {
49 return '<!doctype html><html><head></head><body>' . $html . '</body></html>';
53 * Override this in descendant class to modify HTML after it has been converted from DOM tree
54 * @param string $html: HTML to process
55 * @return string: Processed HTML
57 protected function onHtmlReady( $html ) {
62 * @return DOMDocument: DOM to manipulate
64 public function getDoc() {
66 $html = mb_convert_encoding( $this->html
, 'HTML-ENTITIES', 'UTF-8' );
68 // Workaround for bug that caused spaces before references
69 // to disappear during processing:
70 // https://bugzilla.wikimedia.org/show_bug.cgi?id=53086
72 // Please replace with a better fix if one can be found.
73 $html = str_replace( ' <', ' <', $html );
75 libxml_use_internal_errors( true );
76 $loader = libxml_disable_entity_loader();
77 $this->doc
= new DOMDocument();
78 $this->doc
->strictErrorChecking
= false;
79 $this->doc
->loadHTML( $html );
80 libxml_disable_entity_loader( $loader );
81 libxml_use_internal_errors( false );
82 $this->doc
->encoding
= 'UTF-8';
88 * Sets whether images/videos/sounds should be removed from output
91 public function setRemoveMedia( $flag = true ) {
92 $this->removeMedia
= $flag;
96 * Adds one or more selector of content to remove. A subset of CSS selector
97 * syntax is supported:
104 * @param Array|string $selectors: Selector(s) of stuff to remove
106 public function remove( $selectors ) {
107 $this->itemsToRemove
= array_merge( $this->itemsToRemove
, (array)$selectors );
111 * Adds one or more element name to the list to flatten (remove tag, but not its content)
112 * Can accept undelimited regexes
114 * Note this interface may fail in surprising unexpected ways due to usage of regexes,
115 * so should not be relied on for HTML markup security measures.
117 * @param Array|string $elements: Name(s) of tag(s) to flatten
119 public function flatten( $elements ) {
120 $this->elementsToFlatten
= array_merge( $this->elementsToFlatten
, (array)$elements );
124 * Instructs the formatter to flatten all tags
126 public function flattenAllTags() {
127 $this->flatten( '[?!]?[a-z0-9]+' );
131 * Removes content we've chosen to remove
133 public function filterContent() {
134 wfProfileIn( __METHOD__
);
135 $removals = $this->parseItemsToRemove();
141 $doc = $this->getDoc();
145 // You can't remove DOMNodes from a DOMNodeList as you're iterating
146 // over them in a foreach loop. It will seemingly leave the internal
147 // iterator on the foreach out of wack and results will be quite
148 // strange. Though, making a queue of items to remove seems to work.
149 $domElemsToRemove = array();
150 foreach ( $removals['TAG'] as $tagToRemove ) {
151 $tagToRemoveNodes = $doc->getElementsByTagName( $tagToRemove );
152 foreach ( $tagToRemoveNodes as $tagToRemoveNode ) {
153 if ( $tagToRemoveNode ) {
154 $domElemsToRemove[] = $tagToRemoveNode;
159 $this->removeElements( $domElemsToRemove );
161 // Elements with named IDs
162 $domElemsToRemove = array();
163 foreach ( $removals['ID'] as $itemToRemove ) {
164 $itemToRemoveNode = $doc->getElementById( $itemToRemove );
165 if ( $itemToRemoveNode ) {
166 $domElemsToRemove[] = $itemToRemoveNode;
169 $this->removeElements( $domElemsToRemove );
172 $domElemsToRemove = array();
173 $xpath = new DOMXpath( $doc );
174 foreach ( $removals['CLASS'] as $classToRemove ) {
175 $elements = $xpath->query( '//*[contains(@class, "' . $classToRemove . '")]' );
177 /** @var $element DOMElement */
178 foreach ( $elements as $element ) {
179 $classes = $element->getAttribute( 'class' );
180 if ( preg_match( "/\b$classToRemove\b/", $classes ) && $element->parentNode
) {
181 $domElemsToRemove[] = $element;
185 $this->removeElements( $domElemsToRemove );
187 // Tags with CSS Classes
188 foreach ( $removals['TAG_CLASS'] as $classToRemove ) {
189 $parts = explode( '.', $classToRemove );
191 $elements = $xpath->query(
192 '//' . $parts[0] . '[@class="' . $parts[1] . '"]'
195 $this->removeElements( $elements );
198 wfProfileOut( __METHOD__
);
202 * Removes a list of elelments from DOMDocument
203 * @param array|DOMNodeList $elements
205 private function removeElements( $elements ) {
207 if ( $elements instanceof DOMNodeList
) {
209 foreach ( $elements as $element ) {
213 /** @var $element DOMElement */
214 foreach ( $list as $element ) {
215 if ( $element->parentNode
) {
216 $element->parentNode
->removeChild( $element );
222 * libxml in its usual pointlessness converts many chars to entities - this function
223 * perfoms a reverse conversion
224 * @param string $html
227 private function fixLibXML( $html ) {
228 wfProfileIn( __METHOD__
);
229 static $replacements;
230 if ( ! $replacements ) {
231 // We don't include rules like '"' => '&quot;' because entities had already been
232 // normalized by libxml. Using this function with input not sanitized by libxml is UNSAFE!
233 $replacements = new ReplacementArray( array(
234 '"' => '&quot;',
235 '&' => '&amp;',
236 '<' => '&lt;',
237 '>' => '&gt;',
240 $html = $replacements->replace( $html );
241 $html = mb_convert_encoding( $html, 'UTF-8', 'HTML-ENTITIES' );
242 wfProfileOut( __METHOD__
);
247 * Performs final transformations and returns resulting HTML
249 * @param DOMElement|string|null $element: ID of element to get HTML from or false to get it from the whole tree
250 * @return string: Processed HTML
252 public function getText( $element = null ) {
253 wfProfileIn( __METHOD__
);
256 if ( $element !== null && !( $element instanceof DOMElement
) ) {
257 $element = $this->doc
->getElementById( $element );
260 $body = $this->doc
->getElementsByTagName( 'body' )->item( 0 );
261 $nodesArray = array();
262 foreach ( $body->childNodes
as $node ) {
263 $nodesArray[] = $node;
265 foreach ( $nodesArray as $nodeArray ) {
266 $body->removeChild( $nodeArray );
268 $body->appendChild( $element );
270 $html = $this->doc
->saveHTML();
271 $html = $this->fixLibXml( $html );
275 if ( wfIsWindows() ) {
276 // Appears to be cleanup for CRLF misprocessing of unknown origin
277 // when running server on Windows platform.
279 // If this error continues in the future, please track it down in the
280 // XML code paths if possible and fix there.
281 $html = str_replace( ' ', '', $html );
283 $html = preg_replace( '/<!--.*?-->|^.*?<body>|<\/body>.*$/s', '', $html );
284 $html = $this->onHtmlReady( $html );
286 if ( $this->elementsToFlatten
) {
287 $elements = implode( '|', $this->elementsToFlatten
);
288 $html = preg_replace( "#</?($elements)\\b[^>]*>#is", '', $html );
291 wfProfileOut( __METHOD__
);
296 * @param $selector: CSS selector to parse
299 * @return bool: Whether the selector was successfully recognised
301 protected function parseSelector( $selector, &$type, &$rawName ) {
302 if ( strpos( $selector, '.' ) === 0 ) {
304 $rawName = substr( $selector, 1 );
305 } elseif ( strpos( $selector, '#' ) === 0 ) {
307 $rawName = substr( $selector, 1 );
308 } elseif ( strpos( $selector, '.' ) !== 0 &&
309 strpos( $selector, '.' ) !== false )
312 $rawName = $selector;
313 } elseif ( strpos( $selector, '[' ) === false
314 && strpos( $selector, ']' ) === false )
317 $rawName = $selector;
319 throw new MWException( __METHOD__
. "(): unrecognized selector '$selector'" );
326 * Transforms CSS selectors into an internal representation suitable for processing
329 protected function parseItemsToRemove() {
330 wfProfileIn( __METHOD__
);
335 'TAG_CLASS' => array(),
338 foreach ( $this->itemsToRemove
as $itemToRemove ) {
341 if ( $this->parseSelector( $itemToRemove, $type, $rawName ) ) {
342 $removals[$type][] = $rawName;
346 if ( $this->removeMedia
) {
347 $removals['TAG'][] = 'img';
348 $removals['TAG'][] = 'audio';
349 $removals['TAG'][] = 'video';
352 wfProfileOut( __METHOD__
);