Search-System overhaul
authorAlexander Ebert <ebert@woltlab.com>
Sat, 30 Aug 2014 19:11:12 +0000 (21:11 +0200)
committerAlexander Ebert <ebert@woltlab.com>
Sat, 30 Aug 2014 19:11:12 +0000 (21:11 +0200)
Splitting search objects into dedicated tables, preventing some issues including potential performance bottlenecks if there is an uneven distribution of matching terms for various object types at once.

wcfsetup/install/files/lib/acp/action/InstallPackageAction.class.php
wcfsetup/install/files/lib/system/search/SearchEngine.class.php
wcfsetup/install/files/lib/system/search/SearchIndexManager.class.php
wcfsetup/setup/db/install.sql

index 69cf1bef3f1828288872bbb38b334ce4b88c6cef..2f498fdb933a2e8b6bee3a750d1aeb1ad797db30 100755 (executable)
@@ -5,6 +5,7 @@ use wcf\data\package\installation\queue\PackageInstallationQueue;
 use wcf\system\cache\CacheHandler;
 use wcf\system\exception\IllegalLinkException;
 use wcf\system\package\PackageInstallationDispatcher;
+use wcf\system\search\SearchIndexManager;
 use wcf\system\WCF;
 use wcf\util\StringUtil;
 
@@ -229,6 +230,9 @@ class InstallPackageAction extends AbstractDialogAction {
         * Clears resources after successful installation.
         */
        protected function finalize() {
+               // create search index tables
+               SearchIndexManager::createSearchIndexTables();
+               
                CacheHandler::getInstance()->flushAll();
        }
 }
index 45b5b46f101d451b2bea6ee12a73036f261c99bc..c6370063cba2a01e9a4459745d13f8ee8ab02594 100644 (file)
@@ -138,10 +138,9 @@ class SearchEngine extends SingletonFactory {
                                        INNER JOIN      (
                                                                SELECT          objectID
                                                                                ".($relevanceCalc ? ','.$relevanceCalc : '')."
-                                                               FROM            wcf".WCF_N."_search_index
+                                                               FROM            ".SearchIndexManager::getTableName($objectTypeName)."
                                                                WHERE           ".($fulltextCondition !== null ? $fulltextCondition : '')."
                                                                                ".(($searchIndexCondition !== null && $searchIndexCondition->__toString()) ? ($fulltextCondition !== null ? "AND " : '').$searchIndexCondition : '')."
-                                                                               AND objectTypeID = ".$objectType->objectTypeID."
                                                                ".(!empty($orderBy) && $fulltextCondition === null ? 'ORDER BY '.$orderBy : '')."
                                                                LIMIT           1000
                                                        ) search_index
index d22a69cb950a8582dbd930bab1716f4acdf37333..9dc6b92a6704734a977f8db0a50faa56cc86f204 100644 (file)
@@ -1,6 +1,8 @@
 <?php
 namespace wcf\system\search;
+use wcf\data\object\type\ObjectType;
 use wcf\data\object\type\ObjectTypeCache;
+use wcf\data\object\type\ObjectTypeList;
 use wcf\system\exception\SystemException;
 use wcf\system\SingletonFactory;
 use wcf\system\WCF;
@@ -44,6 +46,20 @@ class SearchIndexManager extends SingletonFactory {
                return $this->availableObjectTypes[$objectType]->objectTypeID;
        }
        
+       /**
+        * Returns the the object type with the given name.
+        *
+        * @param       string          $objectType
+        * @return      \wcf\data\object\type\ObjectType
+        */
+       public function getObjectType($objectType) {
+               if (!isset($this->availableObjectTypes[$objectType])) {
+                       throw new SystemException("unknown object type '".$objectType."'");
+               }
+               
+               return $this->availableObjectTypes[$objectType];
+       }
+       
        /**
         * Adds a new entry.
         * 
@@ -61,11 +77,11 @@ class SearchIndexManager extends SingletonFactory {
                if ($languageID === null) $languageID = 0;
                
                // save new entry
-               $sql = "REPLACE INTO    wcf".WCF_N."_search_index
-                                       (objectTypeID, objectID, subject, message, time, userID, username, languageID, metaData)
-                       VALUES          (?, ?, ?, ?, ?, ?, ?, ?, ?)";
+               $sql = "REPLACE INTO    " . self::getTableName($objectType) . "
+                                       (objectID, subject, message, time, userID, username, languageID, metaData)
+                       VALUES          (?, ?, ?, ?, ?, ?, ?, ?)";
                $statement = WCF::getDB()->prepareStatement($sql);
-               $statement->execute(array($this->getObjectTypeID($objectType), $objectID, $subject, $message, $time, $userID, $username, $languageID, $metaData));
+               $statement->execute(array($objectID, $subject, $message, $time, $userID, $username, $languageID, $metaData));
        }
        
        /**
@@ -96,17 +112,12 @@ class SearchIndexManager extends SingletonFactory {
         * @param       array<integer>  $objectIDs
         */
        public function delete($objectType, array $objectIDs) {
-               $objectTypeID = $this->getObjectTypeID($objectType);
-               
-               $sql = "DELETE FROM     wcf".WCF_N."_search_index
-                       WHERE           objectTypeID = ?
-                                       AND objectID = ?";
+               $sql = "DELETE FROM     " . self::getTableName($objectType) . "
+                       WHERE           objectID = ?";
                $statement = WCF::getDB()->prepareStatement($sql);
                WCF::getDB()->beginTransaction();
                foreach ($objectIDs as $objectID) {
-                       $parameters = array($objectTypeID, $objectID);
-                       
-                       $statement->execute($parameters);
+                       $statement->execute(array($objectID));
                }
                WCF::getDB()->commitTransaction();
        }
@@ -117,9 +128,101 @@ class SearchIndexManager extends SingletonFactory {
         * @param       string          $objectType
         */
        public function reset($objectType) {
-               $sql = "DELETE FROM     wcf".WCF_N."_search_index
-                       WHERE           objectTypeID = ?";
+               $sql = "TRUNCATE TABLE " . self::getTableName($objectType);
+               $statement = WCF::getDB()->prepareStatement($sql);
+               $statement->execute();
+       }
+       
+       /**
+        * Creates the search index tables for all registered, searchable object types.
+        */
+       public static function createSearchIndexTables() {
+               // get definition id
+               $sql = "SELECT  definitionID
+                       FROM    wcf".WCF_N."_object_type_definition
+                       WHERE   definitionName = ?";
+               $statement = WCF::getDB()->prepareStatement($sql);
+               $statement->execute(array('com.woltlab.wcf.searchableObjectType'));
+               $row = $statement->fetchArray();
+               
+               $objectTypeList = new ObjectTypeList();
+               $objectTypeList->getConditionBuilder()->add("object_type.definitionID = ?", array($row['definitionID']));
+               $objectTypeList->readObjects();
+               
+               foreach ($objectTypeList as $objectType) {
+                       self::createSearchIndexTable($objectType);
+               }
+       }
+       
+       /**
+        * Creates the search index table for given object type. Returns true if the
+        * table was created, otherwise false.
+        * 
+        * @param       \wcf\data\object\type\ObjectType        $objectType
+        * @return      boolean
+        */
+       protected static function createSearchIndexTable(ObjectType $objectType) {
+               $tableName = self::getTableName($objectType->objectType);
+               
+               // check if table already exists
+               $sql = "SELECT  COUNT(*) AS count
+                       FROM    wcf".WCF_N."_package_installation_sql_log
+                       WHERE   sqlTable = ?";
+               $statement = WCF::getDB()->prepareStatement($sql);
+               $statement->execute(array($tableName));
+               $row = $statement->fetchArray();
+               if ($row['count']) {
+                       // table already exists
+                       return false;
+               }
+               
+               $columns = array(
+                       array('name' => 'objectID', 'data' => array('length' => 10, 'notNull' => true, 'type' => 'int')),
+                       array('name' => 'subject', 'data' => array('default' => '', 'length' => 255, 'notNull' => true, 'type' => 'varchar')),
+                       array('name' => 'message', 'data' => array('type' => 'mediumtext')),
+                       array('name' => 'metaData', 'data' => array('type' => 'mediumtext')),
+                       array('name' => 'time', 'data' => array('default' => 0, 'length' => 10, 'notNull' => true, 'type' => 'int')),
+                       array('name' => 'userID', 'data' => array('default' => '', 'length' => 10, 'type' => 'int')),
+                       array('name' => 'username', 'data' => array('default' => '', 'length' => 255,'notNull' => true, 'type' => 'varchar')),
+                       array('name' => 'languageID', 'data' => array('default' => 0, 'length' => 10, 'notNull' => true, 'type' => 'int'))
+               );
+               
+               $indices = array(
+                       array('name' => 'objectAndLanguage', 'data' => array('columns' => 'objectID, languageID', 'type' => 'UNIQUE')),
+                       array('name' => 'fulltextIndex', 'data' => array('columns' => 'subject, message, metaData', 'type' => 'FULLTEXT')),
+                       array('name' => 'fulltextIndexSubjectOnly', 'data' => array('columns' => 'subject', 'type' => 'FULLTEXT')),
+                       array('name' => 'language', 'data' => array('columns' => 'languageID', 'type' => 'KEY')),
+                       array('name' => 'user', 'data' => array('columns' => 'userID, time', 'type'=> 'KEY'))
+               );
+               
+               WCF::getDB()->getEditor()->createTable($tableName, $columns, $indices);
+               
+               // add comment
+               $sql = "ALTER TABLE     ".$tableName."
+                       COMMENT         = ?";
+               $statement = WCF::getDB()->prepareStatement($sql);
+               $statement->execute(array(' Search index for ' . $objectType->objectType));
+               
+               // log table
+               $sql = "INSERT INTO     wcf".WCF_N."_package_installation_sql_log
+                                       (packageID, sqlTable)
+                       VALUES          (?, ?)";
                $statement = WCF::getDB()->prepareStatement($sql);
-               $statement->execute(array($this->getObjectTypeID($objectType)));
+               $statement->execute(array(
+                       $objectType->packageID,
+                       $tableName
+               ));
+               
+               return true;
+       }
+       
+       /**
+        * Returns the database table name for the object type's search index.
+        * 
+        * @param       string          $objectType
+        * @return      string
+        */
+       public static function getTableName($objectType) {
+               return 'wcf'.WCF_N.'_search_index_'.substr(sha1($objectType), 0, 8);
        }
 }
index 29a762637d0b928b3ac30d7ad71f4b4a895849c5..ed0409c5eccfbc598246f69e4bb4d2f53f2c988b 100644 (file)
@@ -840,24 +840,6 @@ CREATE TABLE wcf1_search (
        KEY searchHash (searchHash)
 );
 
-DROP TABLE IF EXISTS wcf1_search_index;
-CREATE TABLE wcf1_search_index (
-       objectTypeID INT(10) NOT NULL,
-       objectID INT(10) NOT NULL,
-       subject VARCHAR(255) NOT NULL DEFAULT '',
-       message MEDIUMTEXT,
-       metaData MEDIUMTEXT,
-       time INT(10) NOT NULL DEFAULT 0,
-       userID INT(10),
-       username VARCHAR(255) NOT NULL DEFAULT '',
-       languageID INT(10) NOT NULL DEFAULT 0,
-       UNIQUE KEY (objectTypeID, objectID, languageID),
-       FULLTEXT INDEX fulltextIndex (subject, message, metaData),
-       FULLTEXT INDEX fulltextIndexSubjectOnly (subject),
-       KEY (userID, objectTypeID, time),
-       KEY (objectTypeID)
-);
-
 DROP TABLE IF EXISTS wcf1_search_keyword;
 CREATE TABLE wcf1_search_keyword (
        keywordID INT(10) NOT NULL AUTO_INCREMENT PRIMARY KEY,
@@ -1693,9 +1675,6 @@ ALTER TABLE wcf1_tag_to_object ADD FOREIGN KEY (objectTypeID) REFERENCES wcf1_ob
 
 ALTER TABLE wcf1_stat_daily ADD FOREIGN KEY (objectTypeID) REFERENCES wcf1_object_type (objectTypeID) ON DELETE CASCADE;
 
-ALTER TABLE wcf1_search_index ADD FOREIGN KEY (objectTypeID) REFERENCES wcf1_object_type (objectTypeID) ON DELETE CASCADE;
-ALTER TABLE wcf1_search_index ADD FOREIGN KEY (languageID) REFERENCES wcf1_language (languageID) ON DELETE SET NULL;
-
 ALTER TABLE wcf1_poll ADD FOREIGN KEY (objectTypeID) REFERENCES wcf1_object_type (objectTypeID) ON DELETE CASCADE;
 
 ALTER TABLE wcf1_poll_option ADD FOREIGN KEY (pollID) REFERENCES wcf1_poll (pollID) ON DELETE CASCADE;