From 0cd425be04a0c65ff5a667c2c025b8d727bb7bcc Mon Sep 17 00:00:00 2001 From: Alexander Ebert Date: Tue, 9 Sep 2014 01:24:00 +0200 Subject: [PATCH] Search system overhaul --- .../acp/action/WorkerProxyAction.class.php | 2 + .../files/lib/form/SearchForm.class.php | 9 +- .../search/AbstractSearchEngine.class.php | 112 +++++++++- .../AbstractSearchIndexManager.class.php | 14 ++ .../AbstractSearchableObjectType.class.php | 4 +- .../lib/system/search/ISearchEngine.class.php | 26 +++ .../search/ISearchIndexManager.class.php | 16 +- .../search/ISearchableObjectType.class.php | 8 +- .../lib/system/search/SearchEngine.class.php | 20 ++ .../search/SearchIndexManager.class.php | 31 ++- .../search/mysql/MysqlSearchEngine.class.php | 207 +++++------------- .../mysql/MysqlSearchIndexManager.class.php | 6 +- .../AbstractRebuildDataWorker.class.php | 10 + .../system/worker/AbstractWorker.class.php | 7 + .../files/lib/system/worker/IWorker.class.php | 5 + wcfsetup/install/lang/de.xml | 4 + wcfsetup/install/lang/en.xml | 4 + 17 files changed, 306 insertions(+), 179 deletions(-) diff --git a/wcfsetup/install/files/lib/acp/action/WorkerProxyAction.class.php b/wcfsetup/install/files/lib/acp/action/WorkerProxyAction.class.php index fd18f0a5d6..7d8846dc36 100644 --- a/wcfsetup/install/files/lib/acp/action/WorkerProxyAction.class.php +++ b/wcfsetup/install/files/lib/acp/action/WorkerProxyAction.class.php @@ -92,6 +92,8 @@ class WorkerProxyAction extends AJAXInvokeAction { // execute worker $this->worker->execute(); + $this->worker->finalize(); + // send current state $this->sendResponse($progress, $this->worker->getParameters(), $this->worker->getProceedURL()); } diff --git a/wcfsetup/install/files/lib/form/SearchForm.class.php b/wcfsetup/install/files/lib/form/SearchForm.class.php index 2ba150d69c..eaece60230 100644 --- a/wcfsetup/install/files/lib/form/SearchForm.class.php +++ b/wcfsetup/install/files/lib/form/SearchForm.class.php @@ -114,12 +114,6 @@ class SearchForm extends AbstractCaptchaForm { */ public $searchIndexCondition = null; - /** - * class name for $searchIndexCondition object - * @var string - */ - public $searchIndexConditionClassName = 'wcf\system\database\util\PreparedStatementConditionBuilder'; - /** * search hash to modify existing search * @var string @@ -419,7 +413,8 @@ class SearchForm extends AbstractCaptchaForm { // default conditions $userIDs = $this->getUserIDs(); - $this->searchIndexCondition = new $this->searchIndexConditionClassName(false); + $conditionBuilderClassName = SearchEngine::getInstance()->getConditionBuilderClassName(); + $this->searchIndexCondition = new $conditionBuilderClassName(false); // user ids if (!empty($userIDs)) { diff --git a/wcfsetup/install/files/lib/system/search/AbstractSearchEngine.class.php b/wcfsetup/install/files/lib/system/search/AbstractSearchEngine.class.php index f1ba8fe598..214e2546c1 100644 --- a/wcfsetup/install/files/lib/system/search/AbstractSearchEngine.class.php +++ b/wcfsetup/install/files/lib/system/search/AbstractSearchEngine.class.php @@ -1,6 +1,7 @@ conditionBuilderClassName; + } + + /** + * Manipulates the search term (< and > used as quotation marks): + * + * - becomes <+test* +foo*> + * - becomes <+test* -foo* +bar*> + * - becomes <+test* +"foo bar"> + * + * @see http://dev.mysql.com/doc/refman/5.5/en/fulltext-boolean.html + * + * @param string $query + */ + protected function parseSearchQuery($query) { + $query = StringUtil::trim($query); + + // expand search terms with a * unless they're encapsulated with quotes + $inQuotes = false; + $previousChar = $tmp = ''; + $controlCharacterOrSpace = false; + $chars = array('+', '-', '*'); + $ftMinWordLen = $this->getFulltextMinimumWordLength(); + for ($i = 0, $length = mb_strlen($query); $i < $length; $i++) { + $char = mb_substr($query, $i, 1); + + if ($inQuotes) { + if ($char == '"') { + $inQuotes = false; + } + } + else { + if ($char == '"') { + $inQuotes = true; + } + else { + if ($char == ' ' && !$controlCharacterOrSpace) { + $controlCharacterOrSpace = true; + $tmp .= '*'; + } + else if (in_array($char, $chars)) { + $controlCharacterOrSpace = true; + } + else { + $controlCharacterOrSpace = false; + } + } + } + + /* + * prepend a plus sign (logical AND) if ALL these conditions are given: + * + * 1) previous character: + * - is empty (start of string) + * - is a space (MySQL uses spaces to separate words) + * + * 2) not within quotation marks + * + * 3) current char: + * - is NOT +, - or * + */ + if (($previousChar == '' || $previousChar == ' ') && !$inQuotes && !in_array($char, $chars)) { + // check if the term is shorter than the minimum fulltext word length + if ($i + $ftMinWordLen <= $length) { + $term = '';// $char; + for ($j = $i, $innerLength = $ftMinWordLen + $i; $j < $innerLength; $j++) { + $currentChar = mb_substr($query, $j, 1); + if ($currentChar == '"' || $currentChar == ' ' || in_array($currentChar, $chars)) { + break; + } + + $term .= $currentChar; + } + + if (mb_strlen($term) == $ftMinWordLen) { + $tmp .= '+'; + } + } + } + + $tmp .= $char; + $previousChar = $char; + } + + // handle last char + if (!$inQuotes && !$controlCharacterOrSpace) { + $tmp .= '*'; + } + + return $tmp; + } + + /** + * Returns minimum word length for fulltext indices. + * + * @return integer + */ + abstract protected function getFulltextMinimumWordLength(); +} diff --git a/wcfsetup/install/files/lib/system/search/AbstractSearchIndexManager.class.php b/wcfsetup/install/files/lib/system/search/AbstractSearchIndexManager.class.php index 3449d5d504..dc523b3d2d 100644 --- a/wcfsetup/install/files/lib/system/search/AbstractSearchIndexManager.class.php +++ b/wcfsetup/install/files/lib/system/search/AbstractSearchIndexManager.class.php @@ -46,4 +46,18 @@ abstract class AbstractSearchIndexManager extends SingletonFactory implements IS * @return boolean */ abstract protected function createSearchIndex(ObjectType $objectType); + + /** + * @see \wcf\system\search\ISearchIndexManager::beginBulkOperation() + */ + public function beginBulkOperation() { + // does nothing + } + + /** + * @see \wcf\system\search\ISearchIndexManager::commitBulkOperation() + */ + public function commitBulkOperation() { + // does nothing + } } diff --git a/wcfsetup/install/files/lib/system/search/AbstractSearchableObjectType.class.php b/wcfsetup/install/files/lib/system/search/AbstractSearchableObjectType.class.php index 4545ef8aea..5531ab77c7 100644 --- a/wcfsetup/install/files/lib/system/search/AbstractSearchableObjectType.class.php +++ b/wcfsetup/install/files/lib/system/search/AbstractSearchableObjectType.class.php @@ -91,9 +91,9 @@ abstract class AbstractSearchableObjectType extends AbstractObjectTypeProcessor } /** - * @see \wcf\system\search\ISearchableObjectType::getSpecialSQLQuery() + * @see \wcf\system\search\ISearchableObjectType::getOuterSQLQuery() */ - public function getSpecialSQLQuery(PreparedStatementConditionBuilder &$fulltextCondition = null, PreparedStatementConditionBuilder &$searchIndexConditions = null, PreparedStatementConditionBuilder &$additionalConditions = null, $orderBy = 'time DESC') { + public function getOuterSQLQuery($q, PreparedStatementConditionBuilder &$searchIndexConditions = null, PreparedStatementConditionBuilder &$additionalConditions = null) { return ''; } diff --git a/wcfsetup/install/files/lib/system/search/ISearchEngine.class.php b/wcfsetup/install/files/lib/system/search/ISearchEngine.class.php index e6398753ac..5dfbbf48ba 100644 --- a/wcfsetup/install/files/lib/system/search/ISearchEngine.class.php +++ b/wcfsetup/install/files/lib/system/search/ISearchEngine.class.php @@ -13,6 +13,32 @@ use wcf\system\database\util\PreparedStatementConditionBuilder; * @category Community Framework */ interface ISearchEngine { + /** + * Returns the condition builder class name required to provide conditions for getInnerJoin(). + * + * @return string + */ + public function getConditionBuilderClassName(); + + /** + * Returns the inner join query and the condition parameters. This method is allowed to return NULL + * for the 'fulltextCondition' index instead of a PreparedStatementConditionBuilder instance: + * + * array( + * 'fulltextCondition' => $fulltextCondition || null, + * 'sql' => $sql + * ); + * + * @param string $objectTypeName + * @param string $q + * @param boolean $subjectOnly + * @param \wcf\system\database\util\PreparedStatementConditionBuilder $searchIndexCondition + * @param string $orderBy + * @param integer $limit + * @return array + */ + public function getInnerJoin($objectTypeName, $q, $subjectOnly = false, PreparedStatementConditionBuilder $searchIndexCondition = null, $orderBy = 'time DESC', $limit = 1000); + /** * Searches for the given string and returns the data of the found messages. * diff --git a/wcfsetup/install/files/lib/system/search/ISearchIndexManager.class.php b/wcfsetup/install/files/lib/system/search/ISearchIndexManager.class.php index 0f26731630..06e82c36ab 100644 --- a/wcfsetup/install/files/lib/system/search/ISearchIndexManager.class.php +++ b/wcfsetup/install/files/lib/system/search/ISearchIndexManager.class.php @@ -24,9 +24,8 @@ interface ISearchIndexManager { * @param string $username * @param integer $languageID * @param string $metaData - * @param array $additionalData */ - public function add($objectType, $objectID, $message, $subject, $time, $userID, $username, $languageID = null, $metaData = '', array $additionalData = array()); + public function add($objectType, $objectID, $message, $subject, $time, $userID, $username, $languageID = null, $metaData = ''); /** * Updates the search index. @@ -40,9 +39,8 @@ interface ISearchIndexManager { * @param string $username * @param integer $languageID * @param string $metaData - * @param array $additionalData */ - public function update($objectType, $objectID, $message, $subject, $time, $userID, $username, $languageID = null, $metaData = '', array $additionalData = array()); + public function update($objectType, $objectID, $message, $subject, $time, $userID, $username, $languageID = null, $metaData = ''); /** * Deletes search index entries. @@ -63,4 +61,14 @@ interface ISearchIndexManager { * Creates the search index for all searchable objects. */ public function createSearchIndices(); + + /** + * Begins the bulk operation. + */ + public function beginBulkOperation(); + + /** + * Commits the bulk operation. + */ + public function commitBulkOperation(); } diff --git a/wcfsetup/install/files/lib/system/search/ISearchableObjectType.class.php b/wcfsetup/install/files/lib/system/search/ISearchableObjectType.class.php index 4fea25b9ec..80da3f67b8 100644 --- a/wcfsetup/install/files/lib/system/search/ISearchableObjectType.class.php +++ b/wcfsetup/install/files/lib/system/search/ISearchableObjectType.class.php @@ -116,15 +116,15 @@ interface ISearchableObjectType { public function getFormTemplateName(); /** - * Provides the option to replace the default search index SQL query by an own version. + * Replaces the outer SQL query with a custom version. Querying the search index requires the + * placeholder {WCF_SEARCH_INNER_JOIN} within an empty INNER JOIN() statement. * - * @param \wcf\system\database\util\PreparedStatementConditionBuilder $fulltextCondition + * @param string $q * @param \wcf\system\database\util\PreparedStatementConditionBuilder $searchIndexConditions * @param \wcf\system\database\util\PreparedStatementConditionBuilder $additionalConditions - * @param string $orderBy * @return string */ - public function getSpecialSQLQuery(PreparedStatementConditionBuilder &$fulltextCondition = null, PreparedStatementConditionBuilder &$searchIndexConditions = null, PreparedStatementConditionBuilder &$additionalConditions = null, $orderBy = 'time DESC'); + public function getOuterSQLQuery($q, PreparedStatementConditionBuilder &$searchIndexConditions = null, PreparedStatementConditionBuilder &$additionalConditions = null); /** * Returns the name of the active main menu item. diff --git a/wcfsetup/install/files/lib/system/search/SearchEngine.class.php b/wcfsetup/install/files/lib/system/search/SearchEngine.class.php index fafabced1b..2405166b10 100644 --- a/wcfsetup/install/files/lib/system/search/SearchEngine.class.php +++ b/wcfsetup/install/files/lib/system/search/SearchEngine.class.php @@ -2,6 +2,7 @@ namespace wcf\system\search; use wcf\data\object\type\ObjectTypeCache; use wcf\system\database\util\PreparedStatementConditionBuilder; +use wcf\system\exception\SystemException; use wcf\system\SingletonFactory; /** @@ -95,4 +96,23 @@ class SearchEngine extends SingletonFactory implements ISearchEngine { public function search($q, array $objectTypes, $subjectOnly = false, PreparedStatementConditionBuilder $searchIndexCondition = null, array $additionalConditions = array(), $orderBy = 'time DESC', $limit = 1000) { return $this->getSearchEngine()->search($q, $objectTypes, $subjectOnly, $searchIndexCondition, $additionalConditions, $orderBy, $limit); } + + /** + * @see \wcf\system\search\ISearchEngine::getInnerJoin() + */ + public function getInnerJoin($objectTypeName, $q, $subjectOnly = false, PreparedStatementConditionBuilder $searchIndexCondition = null, $orderBy = 'time DESC', $limit = 1000) { + $conditionBuilderClassName = $this->getConditionBuilderClassName(); + if ($searchIndexCondition !== null && !($searchIndexCondition instanceof $conditionBuilderClassName)) { + throw new SystemException("Search engine '" . SEARCH_ENGINE . "' requires a different condition builder, please use 'SearchEngine::getInstance()->getConditionBuilderClassName()'!"); + } + + return $this->getSearchEngine()->getInnerJoin($objectTypeName, $q, $subjectOnly, $searchIndexCondition, $orderBy, $limit); + } + + /** + * @see \wcf\system\search\ISearchEngine::getConditionBuilderClassName() + */ + public function getConditionBuilderClassName() { + return $this->getSearchEngine()->getConditionBuilderClassName(); + } } diff --git a/wcfsetup/install/files/lib/system/search/SearchIndexManager.class.php b/wcfsetup/install/files/lib/system/search/SearchIndexManager.class.php index 961ac96f25..6912833b12 100644 --- a/wcfsetup/install/files/lib/system/search/SearchIndexManager.class.php +++ b/wcfsetup/install/files/lib/system/search/SearchIndexManager.class.php @@ -80,7 +80,7 @@ class SearchIndexManager extends SingletonFactory implements ISearchIndexManager if ($this->searchIndexManager === null) { $className = ''; if (SEARCH_ENGINE != 'mysql') { - $className = 'wcf\system\search\\'.SEARCH_ENGINE.'\\'.ucfirst(SEARCH_ENGINE).'SearchEngine'; + $className = 'wcf\system\search\\'.SEARCH_ENGINE.'\\'.ucfirst(SEARCH_ENGINE).'SearchIndexManager'; if (!class_exists($className)) { $className = ''; } @@ -100,15 +100,15 @@ class SearchIndexManager extends SingletonFactory implements ISearchIndexManager /** * @see \wcf\system\search\ISearchIndexManager::add() */ - public function add($objectType, $objectID, $message, $subject, $time, $userID, $username, $languageID = null, $metaData = '', array $additionalData = array()) { - $this->getSearchIndexManager()->add($objectType, $objectID, $message, $subject, $time, $userID, $username, $languageID, $metaData, $additionalData); + public function add($objectType, $objectID, $message, $subject, $time, $userID, $username, $languageID = null, $metaData = '') { + $this->getSearchIndexManager()->add($objectType, $objectID, $message, $subject, $time, $userID, $username, $languageID, $metaData); } /** * @see \wcf\system\search\ISearchIndexManager::update() */ - public function update($objectType, $objectID, $message, $subject, $time, $userID, $username, $languageID = null, $metaData = '', array $additionalData = array()) { - $this->getSearchIndexManager()->update($objectType, $objectID, $message, $subject, $time, $userID, $username, $languageID, $metaData, $additionalData); + public function update($objectType, $objectID, $message, $subject, $time, $userID, $username, $languageID = null, $metaData = '') { + $this->getSearchIndexManager()->update($objectType, $objectID, $message, $subject, $time, $userID, $username, $languageID, $metaData); } /** @@ -132,6 +132,27 @@ class SearchIndexManager extends SingletonFactory implements ISearchIndexManager $this->getSearchIndexManager()->createSearchIndices(); } + /** + * @see \wcf\system\search\ISearchIndexManager::supportsBulkInsert() + */ + public function supportsBulkInsert() { + return $this->getSearchIndexManager()->supportsBulkInsert(); + } + + /** + * @see \wcf\system\search\ISearchIndexManager::beginBulkOperation() + */ + public function beginBulkOperation() { + $this->getSearchIndexManager()->beginBulkOperation(); + } + + /** + * @see \wcf\system\search\ISearchIndexManager::commitBulkOperation() + */ + public function commitBulkOperation() { + $this->getSearchIndexManager()->commitBulkOperation(); + } + /** * Returns the database table name for the object type's search index. * diff --git a/wcfsetup/install/files/lib/system/search/mysql/MysqlSearchEngine.class.php b/wcfsetup/install/files/lib/system/search/mysql/MysqlSearchEngine.class.php index 6c15b3e6dd..a8d4274d17 100644 --- a/wcfsetup/install/files/lib/system/search/mysql/MysqlSearchEngine.class.php +++ b/wcfsetup/install/files/lib/system/search/mysql/MysqlSearchEngine.class.php @@ -4,10 +4,9 @@ use wcf\system\database\util\PreparedStatementConditionBuilder; use wcf\system\database\DatabaseException; use wcf\system\exception\SystemException; use wcf\system\search\AbstractSearchEngine; +use wcf\system\search\SearchEngine; use wcf\system\search\SearchIndexManager; use wcf\system\WCF; -use wcf\util\StringUtil; -use wcf\system\search\SearchEngine; /** * Search engine using MySQL's FULLTEXT index. @@ -24,86 +23,49 @@ class MysqlSearchEngine extends AbstractSearchEngine { * MySQL's minimum word length for fulltext indices * @var integer */ - protected static $ftMinWordLen = null; + protected $ftMinWordLen = null; /** * @see \wcf\system\search\ISearchEngine::search() */ public function search($q, array $objectTypes, $subjectOnly = false, PreparedStatementConditionBuilder $searchIndexCondition = null, array $additionalConditions = array(), $orderBy = 'time DESC', $limit = 1000) { - // handle sql types - $fulltextCondition = null; - $relevanceCalc = ''; - if (!empty($q)) { - $q = $this->parseSearchQuery($q); - - $fulltextCondition = new PreparedStatementConditionBuilder(false); - switch (WCF::getDB()->getDBType()) { - case 'wcf\system\database\MySQLDatabase': - $fulltextCondition->add("MATCH (subject".(!$subjectOnly ? ', message, metaData' : '').") AGAINST (? IN BOOLEAN MODE)", array($q)); - break; - - case 'wcf\system\database\PostgreSQLDatabase': - // replace * with :* - $q = str_replace('*', ':*', $q); - - $fulltextCondition->add("fulltextIndex".($subjectOnly ? "SubjectOnly" : '')." @@ to_tsquery(?)", array($q)); - break; - - default: - throw new SystemException("your database type doesn't support fulltext search"); - break; - } - - if ($orderBy == 'relevance ASC' || $orderBy == 'relevance DESC') { - switch (WCF::getDB()->getDBType()) { - case 'wcf\system\database\MySQLDatabase': - $relevanceCalc = "MATCH (subject".(!$subjectOnly ? ', message, metaData' : '').") AGAINST ('".escapeString($q)."') + (5 / (1 + POW(LN(1 + (".TIME_NOW." - time) / 2592000), 2))) AS relevance"; - break; - - case 'wcf\system\database\PostgreSQLDatabase': - $relevanceCalc = "ts_rank_cd(fulltextIndex".($subjectOnly ? "SubjectOnly" : '').", '".escapeString($q)."') AS relevance"; - break; - } - } - } - // build search query $sql = ''; $parameters = array(); foreach ($objectTypes as $objectTypeName) { $objectType = SearchEngine::getInstance()->getObjectType($objectTypeName); - if (!empty($sql)) $sql .= "\nUNION\n"; + + if (!empty($sql)) $sql .= "\nUNION ALL\n"; $additionalConditionsConditionBuilder = (isset($additionalConditions[$objectTypeName]) ? $additionalConditions[$objectTypeName] : null); - if (($specialSQL = $objectType->getSpecialSQLQuery($fulltextCondition, $searchIndexCondition, $additionalConditionsConditionBuilder, $orderBy))) { - $sql .= "(".$specialSQL.")"; + + $query = $objectType->getOuterSQLQuery($q, $searchIndexCondition, $additionalConditionsConditionBuilder); + if (empty($query)) { + $query = "SELECT ".$objectType->getIDFieldName()." AS objectID, + ".$objectType->getSubjectFieldName()." AS subject, + ".$objectType->getTimeFieldName()." AS time, + ".$objectType->getUsernameFieldName()." AS username, + '".$objectTypeName."' AS objectType + ".($orderBy == 'relevance ASC' || $orderBy == 'relevance DESC' ? ',search_index.relevance' : '')." + FROM ".$objectType->getTableName()." + INNER JOIN ( + {WCF_SEARCH_INNER_JOIN} + ) search_index + ON (".$objectType->getIDFieldName()." = search_index.objectID) + ".$objectType->getJoins()." + ".(isset($additionalConditions[$objectTypeName]) ? $additionalConditions[$objectTypeName] : ''); } - else { - $sql .= "( - SELECT ".$objectType->getIDFieldName()." AS objectID, - ".$objectType->getSubjectFieldName()." AS subject, - ".$objectType->getTimeFieldName()." AS time, - ".$objectType->getUsernameFieldName()." AS username, - '".$objectTypeName."' AS objectType - ".($relevanceCalc ? ',search_index.relevance' : '')." - FROM ".$objectType->getTableName()." - INNER JOIN ( - SELECT objectID - ".($relevanceCalc ? ','.$relevanceCalc : '')." - FROM ".SearchIndexManager::getTableName($objectTypeName)." - WHERE ".($fulltextCondition !== null ? $fulltextCondition : '')." - ".(($searchIndexCondition !== null && $searchIndexCondition->__toString()) ? ($fulltextCondition !== null ? "AND " : '').$searchIndexCondition : '')." - ".(!empty($orderBy) && $fulltextCondition === null ? 'ORDER BY '.$orderBy : '')." - LIMIT 1000 - ) search_index - ON (".$objectType->getIDFieldName()." = search_index.objectID) - ".$objectType->getJoins()." - ".(isset($additionalConditions[$objectTypeName]) ? $additionalConditions[$objectTypeName] : '')." - )"; + + if (mb_strpos($query, '{WCF_SEARCH_INNER_JOIN}')) { + $innerJoin = $this->getInnerJoin($objectTypeName, $q, $subjectOnly, $searchIndexCondition, $orderBy, $limit); + + $query = str_replace('{WCF_SEARCH_INNER_JOIN}', $innerJoin['sql'], $query); + if ($innerJoin['fulltextCondition'] !== null) $parameters = array_merge($parameters, $innerJoin['fulltextCondition']->getParameters()); } - if ($fulltextCondition !== null) $parameters = array_merge($parameters, $fulltextCondition->getParameters()); if ($searchIndexCondition !== null) $parameters = array_merge($parameters, $searchIndexCondition->getParameters()); if (isset($additionalConditions[$objectTypeName])) $parameters = array_merge($parameters, $additionalConditions[$objectTypeName]->getParameters()); + + $sql .= $query; } if (empty($sql)) { throw new SystemException('no object types given'); @@ -128,102 +90,41 @@ class MysqlSearchEngine extends AbstractSearchEngine { } /** - * Manipulates the search term (< and > used as quotation marks): - * - * - becomes <+test* +foo*> - * - becomes <+test* -foo* +bar*> - * - becomes <+test* +"foo bar"> - * - * @see http://dev.mysql.com/doc/refman/5.5/en/fulltext-boolean.html - * - * @param string $query + * @see \wcf\system\search\ISearchEngine::getInnerJoin() */ - protected function parseSearchQuery($query) { - $query = StringUtil::trim($query); - - // expand search terms with a * unless they're encapsulated with quotes - $inQuotes = false; - $previousChar = $tmp = ''; - $controlCharacterOrSpace = false; - $chars = array('+', '-', '*'); - $ftMinWordLen = self::getFulltextMinimumWordLength(); - for ($i = 0, $length = mb_strlen($query); $i < $length; $i++) { - $char = mb_substr($query, $i, 1); + public function getInnerJoin($objectTypeName, $q, $subjectOnly = false, PreparedStatementConditionBuilder $searchIndexCondition = null, $orderBy = 'time DESC', $limit = 1000) { + $fulltextCondition = null; + $relevanceCalc = ''; + if (!empty($q)) { + $q = $this->parseSearchQuery($q); - if ($inQuotes) { - if ($char == '"') { - $inQuotes = false; - } - } - else { - if ($char == '"') { - $inQuotes = true; - } - else { - if ($char == ' ' && !$controlCharacterOrSpace) { - $controlCharacterOrSpace = true; - $tmp .= '*'; - } - else if (in_array($char, $chars)) { - $controlCharacterOrSpace = true; - } - else { - $controlCharacterOrSpace = false; - } - } - } + $fulltextCondition = new PreparedStatementConditionBuilder(false); + $fulltextCondition->add("MATCH (subject".(!$subjectOnly ? ', message, metaData' : '').") AGAINST (? IN BOOLEAN MODE)", array($q)); - /* - * prepend a plus sign (logical AND) if ALL these conditions are given: - * - * 1) previous character: - * - is empty (start of string) - * - is a space (MySQL uses spaces to separate words) - * - * 2) not within quotation marks - * - * 3) current char: - * - is NOT +, - or * - */ - if (($previousChar == '' || $previousChar == ' ') && !$inQuotes && !in_array($char, $chars)) { - // check if the term is shorter than MySQL's ft_min_word_len - - if ($i + $ftMinWordLen <= $length) { - $term = '';// $char; - for ($j = $i, $innerLength = $ftMinWordLen + $i; $j < $innerLength; $j++) { - $currentChar = mb_substr($query, $j, 1); - if ($currentChar == '"' || $currentChar == ' ' || in_array($currentChar, $chars)) { - break; - } - - $term .= $currentChar; - } - - if (mb_strlen($term) == $ftMinWordLen) { - $tmp .= '+'; - } - } + if ($orderBy == 'relevance ASC' || $orderBy == 'relevance DESC') { + $relevanceCalc = "MATCH (subject".(!$subjectOnly ? ', message, metaData' : '').") AGAINST ('".escapeString($q)."') + (5 / (1 + POW(LN(1 + (".TIME_NOW." - time) / 2592000), 2))) AS relevance"; } - - $tmp .= $char; - $previousChar = $char; } - // handle last char - if (!$inQuotes && !$controlCharacterOrSpace) { - $tmp .= '*'; - } + $sql = "SELECT objectID + ".($relevanceCalc ? ','.$relevanceCalc : '')." + FROM ".SearchIndexManager::getTableName($objectTypeName)." + WHERE ".($fulltextCondition !== null ? $fulltextCondition : '')." + ".(($searchIndexCondition !== null && $searchIndexCondition->__toString()) ? ($fulltextCondition !== null ? "AND " : '').$searchIndexCondition : '')." + ".(!empty($orderBy) && $fulltextCondition === null ? 'ORDER BY '.$orderBy : '')." + LIMIT 1000"; - return $tmp; + return array( + 'fulltextCondition' => $fulltextCondition, + 'sql' => $sql + ); } /** - * Returns MySQL's minimum word length for fulltext indices. - * - * @return integer + * @see \wcf\system\search\AbstractSearchEngine::getFulltextMinimumWordLength() */ - protected static function getFulltextMinimumWordLength() { - if (self::$ftMinWordLen === null) { + protected function getFulltextMinimumWordLength() { + if ($this->ftMinWordLen === null) { $sql = "SHOW VARIABLES LIKE ?"; $statement = WCF::getDB()->prepareStatement($sql); @@ -236,9 +137,9 @@ class MysqlSearchEngine extends AbstractSearchEngine { $row = array('Value' => 4); } - self::$ftMinWordLen = $row['Value']; + $this->ftMinWordLen = $row['Value']; } - return self::$ftMinWordLen; + return $this->ftMinWordLen; } } diff --git a/wcfsetup/install/files/lib/system/search/mysql/MysqlSearchIndexManager.class.php b/wcfsetup/install/files/lib/system/search/mysql/MysqlSearchIndexManager.class.php index 016ae2223f..a71e79f969 100644 --- a/wcfsetup/install/files/lib/system/search/mysql/MysqlSearchIndexManager.class.php +++ b/wcfsetup/install/files/lib/system/search/mysql/MysqlSearchIndexManager.class.php @@ -19,7 +19,7 @@ class MysqlSearchIndexManager extends AbstractSearchIndexManager { /** * @see \wcf\system\search\ISearchIndexManager::add() */ - public function add($objectType, $objectID, $message, $subject, $time, $userID, $username, $languageID = null, $metaData = '', array $additionalData = array()) { + public function add($objectType, $objectID, $message, $subject, $time, $userID, $username, $languageID = null, $metaData = '') { if ($languageID === null) $languageID = 0; // save new entry @@ -33,12 +33,12 @@ class MysqlSearchIndexManager extends AbstractSearchIndexManager { /** * @see \wcf\system\search\ISearchIndexManager::update() */ - public function update($objectType, $objectID, $message, $subject, $time, $userID, $username, $languageID = null, $metaData = '', array $additionalData = array()) { + public function update($objectType, $objectID, $message, $subject, $time, $userID, $username, $languageID = null, $metaData = '') { // delete existing entry $this->delete($objectType, array($objectID)); // save new entry - $this->add($objectType, $objectID, $message, $subject, $time, $userID, $username, $languageID, $metaData, $additionalData); + $this->add($objectType, $objectID, $message, $subject, $time, $userID, $username, $languageID, $metaData); } /** diff --git a/wcfsetup/install/files/lib/system/worker/AbstractRebuildDataWorker.class.php b/wcfsetup/install/files/lib/system/worker/AbstractRebuildDataWorker.class.php index f05572674d..071bc5a038 100644 --- a/wcfsetup/install/files/lib/system/worker/AbstractRebuildDataWorker.class.php +++ b/wcfsetup/install/files/lib/system/worker/AbstractRebuildDataWorker.class.php @@ -5,6 +5,7 @@ use wcf\system\exception\SystemException; use wcf\system\request\LinkHandler; use wcf\system\WCF; use wcf\util\ClassUtil; +use wcf\system\search\SearchIndexManager; /** * Abstract implementation of rebuild data worker. @@ -71,6 +72,8 @@ abstract class AbstractRebuildDataWorker extends AbstractWorker implements IRebu public function execute() { $this->objectList->readObjects(); + SearchIndexManager::getInstance()->beginBulkOperation(); + EventHandler::getInstance()->fireAction($this, 'execute'); } @@ -97,4 +100,11 @@ abstract class AbstractRebuildDataWorker extends AbstractWorker implements IRebu $this->objectList->sqlLimit = $this->limit; $this->objectList->sqlOffset = $this->limit * $this->loopCount; } + + /** + * @see \wcf\system\worker\IWorker::finalize() + */ + public function finalize() { + SearchIndexManager::getInstance()->commitBulkOperation(); + } } diff --git a/wcfsetup/install/files/lib/system/worker/AbstractWorker.class.php b/wcfsetup/install/files/lib/system/worker/AbstractWorker.class.php index 519f5acd33..6f702525e0 100644 --- a/wcfsetup/install/files/lib/system/worker/AbstractWorker.class.php +++ b/wcfsetup/install/files/lib/system/worker/AbstractWorker.class.php @@ -76,4 +76,11 @@ abstract class AbstractWorker implements IWorker { public function getParameters() { return $this->parameters; } + + /** + * @see \wcf\system\worker\IWorker::finalize() + */ + public function finalize() { + // does nothing + } } diff --git a/wcfsetup/install/files/lib/system/worker/IWorker.class.php b/wcfsetup/install/files/lib/system/worker/IWorker.class.php index f827bc2bf2..9195a720d5 100644 --- a/wcfsetup/install/files/lib/system/worker/IWorker.class.php +++ b/wcfsetup/install/files/lib/system/worker/IWorker.class.php @@ -57,4 +57,9 @@ interface IWorker { * @return string */ public function getProceedURL(); + + /** + * Executes actions after worker has been executed. + */ + public function finalize(); } diff --git a/wcfsetup/install/lang/de.xml b/wcfsetup/install/lang/de.xml index 3756f53a18..fa36f72110 100644 --- a/wcfsetup/install/lang/de.xml +++ b/wcfsetup/install/lang/de.xml @@ -700,6 +700,8 @@ + + @@ -1029,6 +1031,8 @@ GmbH=Gesellschaft mit beschränkter Haftung]]> + + diff --git a/wcfsetup/install/lang/en.xml b/wcfsetup/install/lang/en.xml index faaa5cb5bf..756a99a154 100644 --- a/wcfsetup/install/lang/en.xml +++ b/wcfsetup/install/lang/en.xml @@ -699,6 +699,8 @@ Examples for medium ID detection: + + @@ -1028,6 +1030,8 @@ GmbH=Gesellschaft mit beschränkter Haftung]]> + + -- 2.20.1