Revert "Strip Unicode 6.3.0 directional formatting characters from title"

This reverts commit 7564624d1ca80a4d2f1cc2b4d3d32d5d2e0bca38.

Change-Id: I5d596f8f3c784920829de6ae50b270b0396369e0

diff --git a/RELEASE-NOTES-1.32 b/RELEASE-NOTES-1.32
index 46efbdf..59a5310 100644 (file)
--- a/RELEASE-NOTES-1.32
+++ b/RELEASE-NOTES-1.32
@@ -163,8 +163,6 @@ because of Phabricator reports.
 === Other changes in 1.32 ===
 * Soft hyphens (U+00AD) are now automatically removed from titles; these
   characters can accidentally end up in copy-and-pasted titles.
 === Other changes in 1.32 ===
 * Soft hyphens (U+00AD) are now automatically removed from titles; these
   characters can accidentally end up in copy-and-pasted titles.

-* Strip Unicode 6.3.0 directional formatting characters (U+061C, U+2066,
-  U+2067, U+2068, U+2069) from the title.

 * …
 
 == Compatibility ==
 * …
 
 == Compatibility ==

diff --git a/includes/title/MediaWikiTitleCodec.php b/includes/title/MediaWikiTitleCodec.php
index 7c2d393..655884b 100644 (file)
--- a/includes/title/MediaWikiTitleCodec.php
+++ b/includes/title/MediaWikiTitleCodec.php
@@ -275,15 +275,11 @@ class MediaWikiTitleCodec implements TitleFormatter, TitleParser {
                        'user_case_dbkey' => $dbkey,
                ];
 
                        'user_case_dbkey' => $dbkey,
                ];
 

-               # Strip soft hyphens (U+00AD) and Unicode directional formatting characters (U+061C, U+200E,
-               # U+200F, U+202A. U+202B, U+202C, U+202D, U+202E, U+2066, U+2067, U+2068, U+2069).
+               # Strip soft hyphens (U+00AD) and Unicode bidi override characters
+               # (U+200E, U+200F, U+202A. U+202B, U+202C, U+202D, U+202E).

                # Sometimes they slip into cut-n-pasted page titles, where the
                # soft hyphens or override chars get included in list displays.
                # Sometimes they slip into cut-n-pasted page titles, where the
                # soft hyphens or override chars get included in list displays.

-               $dbkey = preg_replace(
-                       '/\xC2\xAD|\xD8\x9C|\xE2\x80[\x8E\x8F\xAA-\xAE]|\xE2\x81[\xA6-\xA9]/S',
-                       '',
-                       $dbkey
-               );
+               $dbkey = preg_replace( '/\xC2\xAD|\xE2\x80[\x8E\x8F\xAA-\xAE]/S', '', $dbkey );

 
                # Clean up whitespace
                # Note: use of the /u option on preg_replace here will cause
 
                # Clean up whitespace
                # Note: use of the /u option on preg_replace here will cause

diff --git a/resources/src/mediawiki.Title/Title.js b/resources/src/mediawiki.Title/Title.js
index dcaae3e..b3542cd 100644 (file)
--- a/resources/src/mediawiki.Title/Title.js
+++ b/resources/src/mediawiki.Title/Title.js
@@ -149,7 +149,7 @@
                rWhitespace = /[ _\u00A0\u1680\u180E\u2000-\u200A\u2028\u2029\u202F\u205F\u3000]+/g,
 
                // From MediaWikiTitleCodec::splitTitleString() in PHP
                rWhitespace = /[ _\u00A0\u1680\u180E\u2000-\u200A\u2028\u2029\u202F\u205F\u3000]+/g,
 
                // From MediaWikiTitleCodec::splitTitleString() in PHP

-               rStripCharacters = /[\u00AD\u061C\u200E\u200F\u202A-\u202E\u2066-\u2069]/g,
+               rStripCharacters = /[\u00AD\u200E\u200F\u202A-\u202E]/g,

 
                /**
                 * Slightly modified from Flinfo. Credit goes to Lupo and Flominator.
 
                /**
                 * Slightly modified from Flinfo. Credit goes to Lupo and Flominator.


@@ -233,7 +233,7 @@
                        namespace = defaultNamespace === undefined ? NS_MAIN : defaultNamespace;
 
                        title = title
                        namespace = defaultNamespace === undefined ? NS_MAIN : defaultNamespace;
 
                        title = title

-                               // Strip soft hyphens and Unicode directional formatting characters
+                               // Strip soft hyphens and Unicode bidi override characters

                                .replace( rStripCharacters, '' )
                                // Normalise whitespace to underscores and remove duplicates
                                .replace( rWhitespace, '_' )
                                .replace( rStripCharacters, '' )
                                // Normalise whitespace to underscores and remove duplicates
                                .replace( rWhitespace, '_' )

diff --git a/tests/parser/parserTests.txt b/tests/parser/parserTests.txt
index 216d7e5..bc0c43d 100644 (file)
--- a/tests/parser/parserTests.txt
+++ b/tests/parser/parserTests.txt
@@ -28869,7 +28869,7 @@ foo {{echo|<span>bar</span> [[Category:baz]]}} bar
 # of the categories in wikitext
 # Do not remove these characters in edits.
 #
 # of the categories in wikitext
 # Do not remove these characters in edits.
 #

-# As part of the serialization, these Unicode directional formatting characters will get stripped.
+# As part of the serialization, these bidi characters will get stripped.

 !! test
 RTL (\u200f) and LTR (\u200e) markers around category tags should be stripped
 !! options
 !! test
 RTL (\u200f) and LTR (\u200e) markers around category tags should be stripped
 !! options

diff --git a/tests/phpunit/includes/title/MediaWikiTitleCodecTest.php b/tests/phpunit/includes/title/MediaWikiTitleCodecTest.php
index c272551..70aa071 100644 (file)
--- a/tests/phpunit/includes/title/MediaWikiTitleCodecTest.php
+++ b/tests/phpunit/includes/title/MediaWikiTitleCodecTest.php
@@ -104,13 +104,11 @@ class MediaWikiTitleCodecTest extends MediaWikiTestCase {
                        // names ending in "a" to be female.
                        [ NS_USER, 'Lisa_Müller', '', '', 'de', 'Benutzerin:Lisa Müller' ],
                        [ NS_MAIN, 'FooBar', '', 'remotetestiw', 'en', 'remotetestiw:FooBar' ],
                        // names ending in "a" to be female.
                        [ NS_USER, 'Lisa_Müller', '', '', 'de', 'Benutzerin:Lisa Müller' ],
                        [ NS_MAIN, 'FooBar', '', 'remotetestiw', 'en', 'remotetestiw:FooBar' ],

-                       // Strip soft hyphen and Unicode directional formatting characters
-                       [ NS_MAIN, "Foo\xC2\xAD\xD8\x9C\xE2\x80\x8E\xE2\x80\x8F\xE2\x80\xAA\xE2\x80\xAB" .
-                               "\xE2\x80\xAC\xE2\x80\xAD\xE2\x80\xAE\xE2\x81\xA6\xE2\x81\xA7" .
-                               "\xE2\x81\xA8\xE2\x81\xA9bar", '', '', 'en',
-                               "Foo\xC2\xAD\xD8\x9C\xE2\x80\x8E\xE2\x80\x8F\xE2\x80\xAA\xE2\x80\xAB" .
-                               "\xE2\x80\xAC\xE2\x80\xAD\xE2\x80\xAE\xE2\x81\xA6\xE2\x81\xA7" .
-                               "\xE2\x81\xA8\xE2\x81\xA9bar", 'Foobar' ],
+                       // Strip soft hyphen and Unicode bidi override characters
+                       [ NS_MAIN, "Foo\xC2\xAD\xE2\x80\x8E\xE2\x80\x8F\xE2\x80\xAA\xE2\x80\xAB" .
+                               "\xE2\x80\xAC\xE2\x80\xAD\xE2\x80\xAEbar", '', '', 'en',
+                               "Foo\xC2\xAD\xE2\x80\x8E\xE2\x80\x8F\xE2\x80\xAA\xE2\x80\xAB" .
+                               "\xE2\x80\xAC\xE2\x80\xAD\xE2\x80\xAEbar", 'Foobar' ],

                ];
        }
 
                ];
        }
 

diff --git a/tests/qunit/suites/resources/mediawiki/mediawiki.Title.test.js b/tests/qunit/suites/resources/mediawiki/mediawiki.Title.test.js
index e8db4e1..a775029 100644 (file)
--- a/tests/qunit/suites/resources/mediawiki/mediawiki.Title.test.js
+++ b/tests/qunit/suites/resources/mediawiki/mediawiki.Title.test.js
@@ -245,8 +245,8 @@
                title = new mw.Title( 'Foo \u00A0\u1680\u180E\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200A\u2028\u2029\u202F\u205F\u3000 bar' );
                assert.equal( title.getMain(), 'Foo_bar', 'Merge multiple types of whitespace/underscores into a single underscore' );
 
                title = new mw.Title( 'Foo \u00A0\u1680\u180E\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200A\u2028\u2029\u202F\u205F\u3000 bar' );
                assert.equal( title.getMain(), 'Foo_bar', 'Merge multiple types of whitespace/underscores into a single underscore' );
 

-               title = new mw.Title( 'Foo\u00AD\u061C\u200E\u200F\u202A\u202B\u202C\u202D\u202E\u2066\u2067\u2068\u2069bar' );
-               assert.equal( title.getMain(), 'Foobar', 'Strip soft hyphen and Unicode directional formatting characters' );
+               title = new mw.Title( 'Foo\u00AD\u200E\u200F\u202A\u202B\u202C\u202D\u202Ebar' );
+               assert.equal( title.getMain(), 'Foobar', 'Strip soft hyphen and Unicode bidi override characters' );

 
                // Regression test: Previously it would only detect an extension if there is no space after it
                title = new mw.Title( 'Example.js  ' );
 
                // Regression test: Previously it would only detect an extension if there is no space after it
                title = new mw.Title( 'Example.js  ' );


@@ -668,7 +668,7 @@
                                },
                                {
                                        fileName: 'BI\u200EDI.jpg',
                                },
                                {
                                        fileName: 'BI\u200EDI.jpg',

-                                       typeOfName: 'Name containing Unicode directional formatting characters',
+                                       typeOfName: 'Name containing BIDI overrides',

                                        nameText: 'BIDI',
                                        prefixedText: 'File:BIDI.jpg'
                                },
                                        nameText: 'BIDI',
                                        prefixedText: 'File:BIDI.jpg'
                                },