7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation; either version 2 of the License, or
10 * (at your option) any later version.
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
17 * You should have received a copy of the GNU General Public License along
18 * with this program; if not, write to the Free Software Foundation, Inc.,
19 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
20 * http://www.gnu.org/copyleft/gpl.html
26 use MediaWiki\MediaWikiServices
;
29 * Database independant search index updater
33 class SearchUpdate
implements DeferrableUpdate
{
34 /** @var int Page id being updated */
37 /** @var Title Title we're updating */
40 /** @var Content|bool Content of the page (not text) */
47 * @param int $id Page id to update
48 * @param Title|string $title Title of page to update
49 * @param Content|string|bool $c Content of the page to update. Default: false.
50 * If a Content object, text will be gotten from it. String is for back-compat.
51 * Passing false tells the backend to just update the title, not the content
53 public function __construct( $id, $title, $c = false ) {
54 if ( is_string( $title ) ) {
55 $nt = Title
::newFromText( $title );
62 // is_string() check is back-compat for ApprovedRevs
63 if ( is_string( $c ) ) {
64 $this->content
= new TextContent( $c );
66 $this->content
= $c ?
: false;
70 wfDebug( "SearchUpdate object created with invalid title '$title'\n" );
75 * Perform actual update for the entry
77 public function doUpdate() {
78 $config = MediaWikiServices
::getInstance()->getSearchEngineConfig();
80 if ( $config->getConfig()->get( 'DisableSearchUpdate' ) ||
!$this->id
) {
84 $seFactory = MediaWikiServices
::getInstance()->getSearchEngineFactory();
85 foreach ( $config->getSearchTypes() as $type ) {
86 $search = $seFactory->create( $type );
87 if ( !$search->supports( 'search-update' ) ) {
91 $normalTitle = $this->getNormalizedTitle( $search );
93 if ( $this->getLatestPage() === null ) {
94 $search->delete( $this->id
, $normalTitle );
96 } elseif ( $this->content
=== false ) {
97 $search->updateTitle( $this->id
, $normalTitle );
101 $text = $search->getTextFromContent( $this->title
, $this->content
);
102 if ( !$search->textAlreadyUpdatedForIndex() ) {
103 $text = $this->updateText( $text, $search );
106 # Perform the actual update
107 $search->update( $this->id
, $normalTitle, $search->normalizeText( $text ) );
112 * Clean text for indexing. Only really suitable for indexing in databases.
113 * If you're using a real search engine, you'll probably want to override
114 * this behavior and do something nicer with the original wikitext.
115 * @param string $text
116 * @param SearchEngine|null $se Search engine
119 public function updateText( $text, SearchEngine
$se = null ) {
120 # Language-specific strip/conversion
121 $text = MediaWikiServices
::getInstance()->getContentLanguage()->normalizeForSearch( $text );
122 $se = $se ?
: MediaWikiServices
::getInstance()->newSearchEngine();
123 $lc = $se->legalSearchChars() . '&#;';
126 $text = preg_replace( "/<\\/?\\s*[A-Za-z][^>]*?>/",
127 ' ', MediaWikiServices
::getInstance()->getContentLanguage()->lc( " " . $text . " " ) );
128 $text = preg_replace( "/(^|\\n)==\\s*([^\\n]+)\\s*==(\\s)/sD",
129 "\\1\\2 \\2 \\2\\3", $text ); # Emphasize headings
131 # Strip external URLs
132 $uc = "A-Za-z0-9_\\/:.,~%\\-+&;#?!=()@\\x80-\\xFF";
133 $protos = "http|https|ftp|mailto|news|gopher";
134 $pat = "/(^|[^\\[])({$protos}):[{$uc}]+([^{$uc}]|$)/";
135 $text = preg_replace( $pat, "\\1 \\3", $text );
137 $p1 = "/([^\\[])\\[({$protos}):[{$uc}]+]/";
138 $p2 = "/([^\\[])\\[({$protos}):[{$uc}]+\\s+([^\\]]+)]/";
139 $text = preg_replace( $p1, "\\1 ", $text );
140 $text = preg_replace( $p2, "\\1 \\3 ", $text );
142 # Internal image links
143 $pat2 = "/\\[\\[image:([{$uc}]+)\\.(gif|png|jpg|jpeg)([^{$uc}])/i";
144 $text = preg_replace( $pat2, " \\1 \\3", $text );
146 $text = preg_replace( "/([^{$lc}])([{$lc}]+)]]([a-z]+)/",
147 "\\1\\2 \\2\\3", $text ); # Handle [[game]]s
149 # Strip all remaining non-search characters
150 $text = preg_replace( "/[^{$lc}]+/", " ", $text );
155 * $text = preg_replace( "/([{$lc}]+)'s /", "\\1 \\1's ", $text );
156 * $text = preg_replace( "/([{$lc}]+)s' /", "\\1s ", $text );
158 * These tail-anchored regexps are insanely slow. The worst case comes
159 * when Japanese or Chinese text (ie, no word spacing) is written on
160 * a wiki configured for Western UTF-8 mode. The Unicode characters are
161 * expanded to hex codes and the "words" are very long paragraph-length
162 * monstrosities. On a large page the above regexps may take over 20
163 * seconds *each* on a 1GHz-level processor.
165 * Following are reversed versions which are consistently fast
166 * (about 3 milliseconds on 1GHz-level processor).
168 $text = strrev( preg_replace( "/ s'([{$lc}]+)/", " s'\\1 \\1", strrev( $text ) ) );
169 $text = strrev( preg_replace( "/ 's([{$lc}]+)/", " s\\1", strrev( $text ) ) );
171 # Strip wiki '' and '''
172 $text = preg_replace( "/''[']*/", " ", $text );
178 * Get WikiPage for the SearchUpdate $id using WikiPage::READ_LATEST
179 * and ensure using the same WikiPage object if there are multiple
180 * SearchEngine types.
182 * Returns null if a page has been deleted or is not found.
184 * @return WikiPage|null
186 private function getLatestPage() {
187 if ( !isset( $this->page
) ) {
188 $this->page
= WikiPage
::newFromID( $this->id
, WikiPage
::READ_LATEST
);
195 * Get a normalized string representation of a title suitable for
196 * including in a search index
198 * @param SearchEngine $search
199 * @return string A stripped-down title string ready for the search index
201 private function getNormalizedTitle( SearchEngine
$search ) {
202 $ns = $this->title
->getNamespace();
203 $title = $this->title
->getText();
205 $lc = $search->legalSearchChars() . '&#;';
206 $t = MediaWikiServices
::getInstance()->getContentLanguage()->normalizeForSearch( $title );
207 $t = preg_replace( "/[^{$lc}]+/", ' ', $t );
208 $t = MediaWikiServices
::getInstance()->getContentLanguage()->lc( $t );
211 $t = preg_replace( "/([{$lc}]+)'s( |$)/", "\\1 \\1's ", $t );
212 $t = preg_replace( "/([{$lc}]+)s'( |$)/", "\\1s ", $t );
214 $t = preg_replace( "/\\s+/", ' ', $t );
216 if ( $ns == NS_FILE
) {
217 $t = preg_replace( "/ (png|gif|jpg|jpeg|ogg)$/", "", $t );
220 return $search->normalizeText( trim( $t ) );