Adding in the machinery to do sphinx tests

2010-04-01 16:27:16 -07:00 · 2010-04-01 16:27:16 -07:00 · 8e39688483
--- a/apps/search/utils.py
+++ b/apps/search/utils.py
@ -1,3 +1,41 @@
+import subprocess
 import zlib

+from django.conf import settings
+
+
 crc32 = lambda x: zlib.crc32(x) & 0xffffffff
+
+
+call = lambda x: subprocess.Popen(x, stdout=subprocess.PIPE).communicate()
+
+
+def reindex(rotate=False):
+    """
+    Reindexes sphinx.  Note this is only to be used in dev and test
+    environments.
+    """
+    calls = [settings.SPHINX_INDEXER, '--all', '--config',
+             settings.SPHINX_CONFIG_PATH]
+    if rotate:
+        calls.append('--rotate')
+
+    call(calls)
+
+
+def start_sphinx():
+    """
+    Starts sphinx.  Note this is only to be used in dev and test environments.
+    """
+
+    call([settings.SPHINX_SEARCHD, '--config',
+        settings.SPHINX_CONFIG_PATH])
+
+
+def stop_sphinx():
+    """
+    Stops sphinx.  Note this is only to be used in dev and test environments.
+    """
+
+    call([settings.SPHINX_SEARCHD, '--stop', '--config',
+        settings.SPHINX_CONFIG_PATH])
--- a/apps/sumo/fixtures/categories.json
+++ b/apps/sumo/fixtures/categories.json
@ -0,0 +1,72 @@
+[
+    {
+        "pk": 25, 
+        "model": "sumo.category", 
+        "fields": {
+            "hits": 0, 
+            "parentId": 15, 
+            "name": "Firefox 3.5/3.6", 
+            "description": ""
+        }
+    }, 
+    {
+        "pk": 23, 
+        "model": "sumo.category", 
+        "fields": {
+            "hits": 0, 
+            "parentId": 0, 
+            "name": "How to Contribute", 
+            "description": ""
+        }
+    }, 
+    {
+        "pk": 19, 
+        "model": "sumo.category", 
+        "fields": {
+            "hits": 0, 
+            "parentId": 15, 
+            "name": "- This is a support/troubleshooting article -", 
+            "description": "http://wiki.mozilla.org/Support:PRD#Content_Types"
+        }
+    }, 
+    {
+        "pk": 14, 
+        "model": "sumo.category", 
+        "fields": {
+            "hits": 0, 
+            "parentId": 15, 
+            "name": "Firefox 3.0", 
+            "description": ""
+        }
+    }, 
+    {
+        "pk": 13, 
+        "model": "sumo.category", 
+        "fields": {
+            "hits": 0, 
+            "parentId": 15, 
+            "name": "Firefox 2", 
+            "description": ""
+        }
+    }, 
+    {
+        "pk": 8, 
+        "model": "sumo.category", 
+        "fields": {
+            "hits": 0, 
+            "parentId": 0, 
+            "name": "Administration", 
+            "description": ""
+        }
+    }, 
+    {
+        "pk": 1, 
+        "model": "sumo.category", 
+        "fields": {
+            "hits": 0, 
+            "parentId": 0, 
+            "name": "Knowledge Base", 
+            "description": ""
+        }
+    }
+]
--- a/apps/sumo/sql/category.sql
+++ b/apps/sumo/sql/category.sql
@ -0,0 +1,2 @@
+INSERT INTO `tiki_categories` VALUES (1,'Knowledge Base','',0,0),(8,'Administration','',0,0),(13,'Firefox 2','',15,0),(14,'Firefox 3.0','',15,0),(19,'- This is a support/troubleshooting article -','http://wiki.mozilla.org/Support:PRD#Content_Types',15,0),(23,'How to Contribute','',0,0),(25,'Firefox 3.5/3.6','',15,0);
+
--- a/apps/sumo/sql/wikipage.sql
+++ b/apps/sumo/sql/wikipage.sql
@ -0,0 +1,45 @@
+CREATE TABLE IF NOT EXISTS `tiki_freetags` (
+  `tagId` int(10) unsigned NOT NULL AUTO_INCREMENT,
+  `tag` varchar(30) NOT NULL DEFAULT '',
+  `raw_tag` varchar(50) NOT NULL DEFAULT '',
+  `lang` varchar(16) DEFAULT NULL,
+  PRIMARY KEY (`tagId`)
+) ENGINE=MyISAM AUTO_INCREMENT=12176 DEFAULT CHARSET=latin1;
+
+CREATE TABLE IF NOT EXISTS `tiki_freetagged_objects` (
+  `tagId` int(12) NOT NULL AUTO_INCREMENT,
+  `objectId` int(11) NOT NULL DEFAULT '0',
+  `user` varchar(200) NOT NULL DEFAULT '',
+  `created` int(14) NOT NULL DEFAULT '0',
+  PRIMARY KEY (`tagId`,`user`,`objectId`),
+  KEY `tagId` (`tagId`),
+  KEY `user` (`user`),
+  KEY `objectId` (`objectId`)
+) ENGINE=MyISAM AUTO_INCREMENT=12176 DEFAULT CHARSET=latin1;
+
+CREATE TABLE IF NOT EXISTS `tiki_objects` (
+  `objectId` int(12) NOT NULL AUTO_INCREMENT,
+  `type` varchar(50) DEFAULT NULL,
+  `itemId` varchar(255) DEFAULT NULL,
+  `description` text,
+  `created` int(14) DEFAULT NULL,
+  `name` varchar(200) DEFAULT NULL,
+  `href` varchar(200) DEFAULT NULL,
+  `hits` int(8) DEFAULT NULL,
+  PRIMARY KEY (`objectId`),
+  KEY `type` (`type`,`itemId`),
+  KEY `itemId` (`itemId`,`type`)
+) ENGINE=MyISAM AUTO_INCREMENT=35581 DEFAULT CHARSET=latin1;
+
+INSERT INTO tiki_objects (objectId, type, itemId, name) VALUES (79, 'wiki page', 'Firefox Support Home Page', 'Firefox Support Home Page');
+INSERT INTO tiki_objects (objectId, type, itemId, name) VALUES (84, 'wiki page', 'Style Guide', 'Style Guide');
+INSERT INTO tiki_objects (objectId, type, itemId, name) VALUES (62, 'wiki page', 'Video or audio does not play', 'Video or audio does not play');
+
+CREATE TABLE IF NOT EXISTS `tiki_category_objects` (
+  `catObjectId` int(12) NOT NULL DEFAULT '0',
+  `categId` int(12) NOT NULL DEFAULT '0',
+  PRIMARY KEY (`catObjectId`,`categId`),
+  KEY `categId` (`categId`)
+) ENGINE=MyISAM DEFAULT CHARSET=latin1;
+
+INSERT INTO tiki_category_objects (catObjectId, categId) VALUES (79, 8), (84, 23), (62, 1), (62, 13), (62, 14), (62, 19), (62, 25);
--- a/configs/sphinx/localsettings.py-dist
+++ b/configs/sphinx/localsettings.py-dist
@ -0,0 +1,32 @@
+
+
+## MySQL Connection Settings
+MYSQL_USER      = ''
+MYSQL_PASS      = ''
+MYSQL_HOST      = ''
+MYSQL_NAME      = ''
+
+## Data store settings
+# ROOT_PATH is prepended to all other paths
+ROOT_PATH       = '/opt/local'
+
+CATALOG_PATH    = '/data/sphinx'
+LOG_PATH        = '/log/searchd'
+ETC_PATH        = '/etc'
+
+# Listen on...?
+LISTEN_PORT     = 3312
+LISTEN_SQL_HOST = 'localhost'
+LISTEN_SQL_PORT = 3317
+
+## Age divisor
+# We convert the |age| attribute from the |forum_thread| index into a more
+# useful unit than seconds. In the list below, to convert to <unit> assign
+# this to <value>
+# <unit>    : <value>
+# second    : 1
+# minute    : 60
+# hour      : 3600
+# day       : 86400
+# week      : 604800
+AGE_DIVISOR = 86400
--- a/configs/sphinx/localsettings_django.py
+++ b/configs/sphinx/localsettings_django.py
@ -0,0 +1,32 @@
+import os
+import sys
+
+SETTINGS_DIR = os.path.realpath(
+        os.path.join(os.path.dirname(__file__), os.path.sep.join(('..',)*2)))
+
+sys.path.append(SETTINGS_DIR)
+sys.path.append(os.path.join(SETTINGS_DIR,'lib'))
+
+import settings_local as settings
+
+s = settings.DATABASES['default']
+MYSQL_PASS = s['PASSWORD']
+MYSQL_USER = s['USER']
+MYSQL_HOST = s.get('HOST', 'localhost')
+MYSQL_NAME = s['NAME']
+
+if MYSQL_HOST.endswith('.sock'):
+    MYSQL_HOST = 'localhost'
+
+if os.environ.get('DJANGO_ENVIRONMENT') == 'test':
+    MYSQL_NAME = 'test_' + MYSQL_NAME
+
+ROOT_PATH       = '/tmp'
+CATALOG_PATH    = '/data'
+LOG_PATH        = '/log'
+ETC_PATH        = '/etc'
+
+LISTEN_PORT     = 3312
+LISTEN_SQL_HOST = '127.0.0.1'
+LISTEN_SQL_PORT = 3317
+AGE_DIVISOR = 86400
--- a/configs/sphinx/sphinx.conf
+++ b/configs/sphinx/sphinx.conf
@ -0,0 +1,391 @@
+#!/usr/bin/env python2.6
+
+try:
+    from localsettings import *
+except ImportError:
+    from localsettings_django import *
+
+#############################################################################
+## data source definition
+#############################################################################
+
+config = """
+source forum_threads
+{{
+    type = mysql
+    sql_host    = {sql_host}
+    sql_user    = {sql_user}
+    sql_pass    = {sql_pass}
+    sql_db      = {sql_db}
+
+    sql_query_pre = SET SESSION query_cache_type = OFF
+
+    sql_query = \
+    SELECT \
+        threadId, \
+        object AS forumId, \
+        title, \
+        userName AS author, \
+        CRC32(userName) AS author_ord, \
+        CRC32(type) AS status, \
+        data AS content, \
+        commentDate AS created, \
+        ( \
+            SELECT \
+                COUNT(replies.threadId) \
+            FROM \
+                tiki_comments AS replies \
+            WHERE \
+                replies.parentId = threads.threadId \
+        ) AS replies, \
+        IF( \
+            ( \
+                SELECT \
+                    COUNT(replies0.threadId) \
+                FROM \
+                    tiki_comments AS replies0 \
+                WHERE \
+                    replies0.parentId = threads.threadId \
+            ), \
+            ( \
+                SELECT \
+                    MAX(replies2.commentDate) \
+                FROM \
+                    tiki_comments AS replies2 \
+                WHERE \
+                    replies2.parentId = threads.threadId \
+            ), \
+            threads.commentDate \
+        ) AS last_updated, \
+        IF( \
+            ( \
+                SELECT \
+                    COUNT(replies1.threadId) \
+                FROM \
+                    tiki_comments AS replies1 \
+                WHERE \
+                    replies1.parentId = threads.threadId \
+            ), \
+            (UNIX_TIMESTAMP() - ( \
+                SELECT \
+                    MAX(replies10.commentDate) \
+                FROM \
+                    tiki_comments AS replies10 \
+                WHERE \
+                    replies10.parentId = threads.threadId \
+            ))/{age_unit}, \
+            (UNIX_TIMESTAMP() - threads.commentDate)/{age_unit} \
+        ) AS age, \
+        (\
+            SELECT \
+                GROUP_CONCAT(replies3.data) \
+            FROM \
+                tiki_comments AS replies3 \
+            WHERE \
+                replies3.parentId = threads.threadId \
+        ) AS reply_content \
+    FROM \
+        tiki_comments AS threads\
+    WHERE \
+        threads.objectType = 'forum' \
+        AND threads.parentId = 0
+
+    sql_attr_uint       = forumId
+    sql_attr_uint       = author_ord
+    sql_attr_uint       = status
+    sql_attr_timestamp  = created
+    sql_attr_timestamp  = last_updated
+    sql_attr_uint       = age
+    sql_attr_uint       = replies
+
+    sql_attr_multi = uint authors from query; SELECT \
+        IF(parentId=0,threadId,parentId) AS threadid, \
+        CRC32(userName) AS poster \
+    FROM \
+        tiki_comments \
+    WHERE \
+        objectType = 'forum'
+
+}}
+""".format(sql_host = MYSQL_HOST,sql_user = MYSQL_USER,
+    sql_pass = MYSQL_PASS,sql_db=MYSQL_NAME,age_unit=AGE_DIVISOR)
+
+config = config + """
+index forum_threads
+{{
+	source      	= forum_threads
+	path            = {root_path}{catalog_path}/forum-thread-catalog
+	charset_type    = utf-8
+    morphology      = stem_en
+    min_stemming_len = 4
+    stopwords       = {root_path}/stopwords.txt
+    wordforms       = {root_path}/wordforms.txt
+	charset_table = U+FF10..U+FF19->0..9, 0..9, U+FF41..U+FF5A->a..z, U+FF21..U+FF3A->a..z,\
+        A..Z->a..z, a..z, U+0149, U+017F, U+0138, U+00DF, U+00FF, U+00C0..U+00D6->U+00E0..U+00F6,\
+        U+00E0..U+00F6, U+00D8..U+00DE->U+00F8..U+00FE, U+00F8..U+00FE, U+0100->U+0101, U+0101,\
+        U+0102->U+0103, U+0103, U+0104->U+0105, U+0105, U+0106->U+0107, U+0107, U+0108->U+0109,\
+        U+0109, U+010A->U+010B, U+010B, U+010C->U+010D, U+010D, U+010E->U+010F, U+010F,\
+        U+0110->U+0111, U+0111, U+0112->U+0113, U+0113, U+0114->U+0115, U+0115, U+0116->U+0117,\
+        U+0117, U+0118->U+0119, U+0119, U+011A->U+011B, U+011B, U+011C->U+011D, U+011D,\
+        U+011E->U+011F, U+011F, U+0130->U+0131, U+0131, U+0132->U+0133, U+0133, U+0134->U+0135,\
+        U+0135, U+0136->U+0137, U+0137, U+0139->U+013A, U+013A, U+013B->U+013C, U+013C,\
+        U+013D->U+013E, U+013E, U+013F->U+0140, U+0140, U+0141->U+0142, U+0142, U+0143->U+0144,\
+        U+0144, U+0145->U+0146, U+0146, U+0147->U+0148, U+0148, U+014A->U+014B, U+014B,\
+        U+014C->U+014D, U+014D, U+014E->U+014F, U+014F, U+0150->U+0151, U+0151, U+0152->U+0153,\
+        U+0153, U+0154->U+0155, U+0155, U+0156->U+0157, U+0157, U+0158->U+0159, U+0159,\
+        U+015A->U+015B, U+015B, U+015C->U+015D, U+015D, U+015E->U+015F, U+015F, U+0160->U+0161,\
+        U+0161, U+0162->U+0163, U+0163, U+0164->U+0165, U+0165, U+0166->U+0167, U+0167,\
+        U+0168->U+0169, U+0169, U+016A->U+016B, U+016B, U+016C->U+016D, U+016D, U+016E->U+016F,\
+        U+016F, U+0170->U+0171, U+0171, U+0172->U+0173, U+0173, U+0174->U+0175, U+0175,\
+        U+0176->U+0177, U+0177, U+0178->U+00FF, U+00FF, U+0179->U+017A, U+017A, U+017B->U+017C,\
+        U+017C, U+017D->U+017E, U+017E, U+0410..U+042F->U+0430..U+044F, U+0430..U+044F,\
+        U+05D0..U+05EA, U+0531..U+0556->U+0561..U+0586, U+0561..U+0587, U+0621..U+063A, U+01B9,\
+        U+01BF, U+0640..U+064A, U+0660..U+0669, U+066E, U+066F, U+0671..U+06D3, U+06F0..U+06FF,\
+        U+0904..U+0939, U+0958..U+095F, U+0960..U+0963, U+0966..U+096F, U+097B..U+097F,\
+        U+0985..U+09B9, U+09CE, U+09DC..U+09E3, U+09E6..U+09EF, U+0A05..U+0A39, U+0A59..U+0A5E,\
+        U+0A66..U+0A6F, U+0A85..U+0AB9, U+0AE0..U+0AE3, U+0AE6..U+0AEF, U+0B05..U+0B39,\
+        U+0B5C..U+0B61, U+0B66..U+0B6F, U+0B71, U+0B85..U+0BB9, U+0BE6..U+0BF2, U+0C05..U+0C39,\
+        U+0C66..U+0C6F, U+0C85..U+0CB9, U+0CDE..U+0CE3, U+0CE6..U+0CEF, U+0D05..U+0D39, U+0D60,\
+        U+0D61, U+0D66..U+0D6F, U+0D85..U+0DC6, U+1900..U+1938, U+1946..U+194F, U+A800..U+A805,\
+        U+A807..U+A822, U+0386->U+03B1, U+03AC->U+03B1, U+0388->U+03B5, U+03AD->U+03B5,\
+        U+0389->U+03B7, U+03AE->U+03B7, U+038A->U+03B9, U+0390->U+03B9, U+03AA->U+03B9,\
+        U+03AF->U+03B9, U+03CA->U+03B9, U+038C->U+03BF, U+03CC->U+03BF, U+038E->U+03C5,\
+        U+03AB->U+03C5, U+03B0->U+03C5, U+03CB->U+03C5, U+03CD->U+03C5, U+038F->U+03C9,\
+        U+03CE->U+03C9, U+03C2->U+03C3, U+0391..U+03A1->U+03B1..U+03C1,\
+        U+03A3..U+03A9->U+03C3..U+03C9, U+03B1..U+03C1, U+03C3..U+03C9, U+0E01..U+0E2E,\
+        U+0E30..U+0E3A, U+0E40..U+0E45, U+0E47, U+0E50..U+0E59, U+A000..U+A48F, U+4E00..U+9FBF,\
+        U+3400..U+4DBF, U+20000..U+2A6DF, U+F900..U+FAFF, U+2F800..U+2FA1F, U+2E80..U+2EFF,\
+        U+2F00..U+2FDF, U+3100..U+312F, U+31A0..U+31BF, U+3040..U+309F, U+30A0..U+30FF,\
+        U+31F0..U+31FF, U+AC00..U+D7AF, U+1100..U+11FF, U+3130..U+318F, U+A000..U+A48F,\
+        U+A490..U+A4CF
+}}
+""".format(root_path=ROOT_PATH, catalog_path=CATALOG_PATH)
+
+config = config + """
+source forum_posts
+{
+    type = mysql
+    sql_host    = %s
+    sql_user    = %s
+    sql_pass    = %s
+    sql_db      = %s
+
+    sql_query_pre = SET SESSION query_cache_type=OFF
+
+    sql_query = \
+    SELECT \
+        threadId AS postid, \
+        IF(parentId = 0, threadId, parentId) AS threadid, \
+        IF(parentId = 0, object, ( \
+            SELECT \
+                parents0.object \
+            FROM \
+                tiki_comments AS parents0 \
+            WHERE \
+                parents0.threadId = posts.parentId \
+        )) AS forumid, \
+        userName AS author, \
+        CRC32(userName) AS author_ord, \
+        CRC32(type) AS status, \
+        commentDate AS created, \
+        data AS content, \
+        IF(parentId = 0, title, ( \
+            SELECT \
+                title \
+            FROM \
+                tiki_comments AS parents \
+            WHERE \
+                parents.threadId = posts.parentId \
+        )) AS title \
+    FROM \
+        tiki_comments AS posts \
+    WHERE \
+        objectType = 'forum'
+    
+
+    sql_attr_uint = threadid
+    sql_attr_uint = author_ord
+    sql_attr_uint = status
+    sql_attr_uint = forumid
+    sql_attr_timestamp = created
+}
+
+""" % (MYSQL_HOST, MYSQL_USER, MYSQL_PASS, MYSQL_NAME)
+
+config = config + """
+index forum_posts
+{{
+    source          = forum_posts
+    path            = {root_path}{catalog_path}/forum-post-catalog
+    charset_type    = utf-8
+    morphology      = stem_en
+    min_stemming_len = 4
+    stopwords       = {root_path}/stopwords.txt
+    wordforms       = {root_path}/wordforms.txt
+    charset_table = U+FF10..U+FF19->0..9, 0..9, U+FF41..U+FF5A->a..z, U+FF21..U+FF3A->a..z,\
+        A..Z->a..z, a..z, U+0149, U+017F, U+0138, U+00DF, U+00FF, U+00C0..U+00D6->U+00E0..U+00F6,\
+        U+00E0..U+00F6, U+00D8..U+00DE->U+00F8..U+00FE, U+00F8..U+00FE, U+0100->U+0101, U+0101,\
+        U+0102->U+0103, U+0103, U+0104->U+0105, U+0105, U+0106->U+0107, U+0107, U+0108->U+0109,\
+        U+0109, U+010A->U+010B, U+010B, U+010C->U+010D, U+010D, U+010E->U+010F, U+010F,\
+        U+0110->U+0111, U+0111, U+0112->U+0113, U+0113, U+0114->U+0115, U+0115, U+0116->U+0117,\
+        U+0117, U+0118->U+0119, U+0119, U+011A->U+011B, U+011B, U+011C->U+011D, U+011D,\
+        U+011E->U+011F, U+011F, U+0130->U+0131, U+0131, U+0132->U+0133, U+0133, U+0134->U+0135,\
+        U+0135, U+0136->U+0137, U+0137, U+0139->U+013A, U+013A, U+013B->U+013C, U+013C,\
+        U+013D->U+013E, U+013E, U+013F->U+0140, U+0140, U+0141->U+0142, U+0142, U+0143->U+0144,\
+        U+0144, U+0145->U+0146, U+0146, U+0147->U+0148, U+0148, U+014A->U+014B, U+014B,\
+        U+014C->U+014D, U+014D, U+014E->U+014F, U+014F, U+0150->U+0151, U+0151, U+0152->U+0153,\
+        U+0153, U+0154->U+0155, U+0155, U+0156->U+0157, U+0157, U+0158->U+0159, U+0159,\
+        U+015A->U+015B, U+015B, U+015C->U+015D, U+015D, U+015E->U+015F, U+015F, U+0160->U+0161,\
+        U+0161, U+0162->U+0163, U+0163, U+0164->U+0165, U+0165, U+0166->U+0167, U+0167,\
+        U+0168->U+0169, U+0169, U+016A->U+016B, U+016B, U+016C->U+016D, U+016D, U+016E->U+016F,\
+        U+016F, U+0170->U+0171, U+0171, U+0172->U+0173, U+0173, U+0174->U+0175, U+0175,\
+        U+0176->U+0177, U+0177, U+0178->U+00FF, U+00FF, U+0179->U+017A, U+017A, U+017B->U+017C,\
+        U+017C, U+017D->U+017E, U+017E, U+0410..U+042F->U+0430..U+044F, U+0430..U+044F,\
+        U+05D0..U+05EA, U+0531..U+0556->U+0561..U+0586, U+0561..U+0587, U+0621..U+063A, U+01B9,\
+        U+01BF, U+0640..U+064A, U+0660..U+0669, U+066E, U+066F, U+0671..U+06D3, U+06F0..U+06FF,\
+        U+0904..U+0939, U+0958..U+095F, U+0960..U+0963, U+0966..U+096F, U+097B..U+097F,\
+        U+0985..U+09B9, U+09CE, U+09DC..U+09E3, U+09E6..U+09EF, U+0A05..U+0A39, U+0A59..U+0A5E,\
+        U+0A66..U+0A6F, U+0A85..U+0AB9, U+0AE0..U+0AE3, U+0AE6..U+0AEF, U+0B05..U+0B39,\
+        U+0B5C..U+0B61, U+0B66..U+0B6F, U+0B71, U+0B85..U+0BB9, U+0BE6..U+0BF2, U+0C05..U+0C39,\
+        U+0C66..U+0C6F, U+0C85..U+0CB9, U+0CDE..U+0CE3, U+0CE6..U+0CEF, U+0D05..U+0D39, U+0D60,\
+        U+0D61, U+0D66..U+0D6F, U+0D85..U+0DC6, U+1900..U+1938, U+1946..U+194F, U+A800..U+A805,\
+        U+A807..U+A822, U+0386->U+03B1, U+03AC->U+03B1, U+0388->U+03B5, U+03AD->U+03B5,\
+        U+0389->U+03B7, U+03AE->U+03B7, U+038A->U+03B9, U+0390->U+03B9, U+03AA->U+03B9,\
+        U+03AF->U+03B9, U+03CA->U+03B9, U+038C->U+03BF, U+03CC->U+03BF, U+038E->U+03C5,\
+        U+03AB->U+03C5, U+03B0->U+03C5, U+03CB->U+03C5, U+03CD->U+03C5, U+038F->U+03C9,\
+        U+03CE->U+03C9, U+03C2->U+03C3, U+0391..U+03A1->U+03B1..U+03C1,\
+        U+03A3..U+03A9->U+03C3..U+03C9, U+03B1..U+03C1, U+03C3..U+03C9, U+0E01..U+0E2E,\
+        U+0E30..U+0E3A, U+0E40..U+0E45, U+0E47, U+0E50..U+0E59, U+A000..U+A48F, U+4E00..U+9FBF,\
+        U+3400..U+4DBF, U+20000..U+2A6DF, U+F900..U+FAFF, U+2F800..U+2FA1F, U+2E80..U+2EFF,\
+        U+2F00..U+2FDF, U+3100..U+312F, U+31A0..U+31BF, U+3040..U+309F, U+30A0..U+30FF,\
+        U+31F0..U+31FF, U+AC00..U+D7AF, U+1100..U+11FF, U+3130..U+318F, U+A000..U+A48F,\
+        U+A490..U+A4CF
+}}
+""".format(root_path=ROOT_PATH,catalog_path=CATALOG_PATH)
+
+config = config + """
+source wiki_pages
+{
+    type = mysql
+    sql_host    = %s
+    sql_user    = %s
+    sql_pass    = %s
+    sql_db      = %s
+
+    sql_query_pre = SET SESSION query_cache_type=OFF
+
+    sql_query = \
+    SELECT \
+        page_id AS pageId, \
+        pageName, \
+        data AS content, \
+        lastModif, \
+        CRC32(lang) AS locale, \
+        keywords, \
+        ( \
+            SELECT \
+                GROUP_CONCAT(DISTINCT tag) \
+            FROM \
+                tiki_freetags t_f \
+            JOIN tiki_freetagged_objects t_f_o ON t_f.tagId = t_f_o.tagId \
+            JOIN tiki_objects t_o ON t_f_o.objectId = t_o.objectId \
+            WHERE \
+                t_o.itemId = pageName \
+        ) AS tags \
+    FROM \
+        tiki_pages \
+    WHERE \
+        data NOT LIKE '{REDIRECT%%'
+
+    sql_attr_timestamp = lastModif
+    sql_attr_uint = locale
+
+    sql_attr_multi = uint tag from query; SELECT \
+        page_id AS pageId, \
+        CRC32(tag) AS tag \
+    FROM \
+        tiki_freetags t_f \
+        INNER JOIN tiki_freetagged_objects t_f_o ON t_f.tagId = t_f_o.tagId \
+        INNER JOIN tiki_objects t_o ON t_f_o.objectId = t_o.objectId \
+        INNER JOIN tiki_pages t_p ON t_o.itemId = t_p.pageName \
+    WHERE \
+        page_id IS NOT NULL
+
+    sql_attr_multi = uint category from query; SELECT \
+        page_id AS pageId, \
+        t_c.categId \
+    FROM \
+        tiki_categories t_c \
+        INNER JOIN tiki_category_objects t_c_o ON t_c.categId = t_c_o.categId \
+        INNER JOIN tiki_objects t_o ON t_c_o.catObjectId = t_o.objectId \
+        INNER JOIN tiki_pages t_p ON t_o.itemId = t_p.pageName \
+    WHERE \
+        page_id IS NOT NULL
+}
+""" % (MYSQL_HOST, MYSQL_USER, MYSQL_PASS, MYSQL_NAME)
+
+config = config + """
+index wiki_pages
+{{
+    source          = wiki_pages
+    path            = {root_path}{catalog_path}/wiki-page-catalog
+    charset_type    = utf-8
+    morphology      = stem_en
+    min_stemming_len = 4
+    stopwords       = {root_path}/stopwords.txt
+    wordforms       = {root_path}/wordforms.txt
+    charset_table = U+FF10..U+FF19->0..9, 0..9, U+FF41..U+FF5A->a..z, U+FF21..U+FF3A->a..z,\
+    A..Z->a..z, a..z, U+0149, U+017F, U+0138, U+00DF, U+00FF, U+00C0..U+00D6->U+00E0..U+00F6,\
+    U+00E0..U+00F6, U+00D8..U+00DE->U+00F8..U+00FE, U+00F8..U+00FE, U+0100->U+0101, U+0101,\
+    U+0102->U+0103, U+0103, U+0104->U+0105, U+0105, U+0106->U+0107, U+0107, U+0108->U+0109,\
+    U+0109, U+010A->U+010B, U+010B, U+010C->U+010D, U+010D, U+010E->U+010F, U+010F,\
+    U+0110->U+0111, U+0111, U+0112->U+0113, U+0113, U+0114->U+0115, U+0115, U+0116->U+0117,\
+    U+0117, U+0118->U+0119, U+0119, U+011A->U+011B, U+011B, U+011C->U+011D, U+011D,\
+    U+011E->U+011F, U+011F, U+0130->U+0131, U+0131, U+0132->U+0133, U+0133, U+0134->U+0135,\
+    U+0135, U+0136->U+0137, U+0137, U+0139->U+013A, U+013A, U+013B->U+013C, U+013C,\
+    U+013D->U+013E, U+013E, U+013F->U+0140, U+0140, U+0141->U+0142, U+0142, U+0143->U+0144,\
+    U+0144, U+0145->U+0146, U+0146, U+0147->U+0148, U+0148, U+014A->U+014B, U+014B,\
+    U+014C->U+014D, U+014D, U+014E->U+014F, U+014F, U+0150->U+0151, U+0151, U+0152->U+0153,\
+    U+0153, U+0154->U+0155, U+0155, U+0156->U+0157, U+0157, U+0158->U+0159, U+0159,\
+    U+015A->U+015B, U+015B, U+015C->U+015D, U+015D, U+015E->U+015F, U+015F, U+0160->U+0161,\
+    U+0161, U+0162->U+0163, U+0163, U+0164->U+0165, U+0165, U+0166->U+0167, U+0167,\
+    U+0168->U+0169, U+0169, U+016A->U+016B, U+016B, U+016C->U+016D, U+016D, U+016E->U+016F,\
+    U+016F, U+0170->U+0171, U+0171, U+0172->U+0173, U+0173, U+0174->U+0175, U+0175,\
+    U+0176->U+0177, U+0177, U+0178->U+00FF, U+00FF, U+0179->U+017A, U+017A, U+017B->U+017C,\
+    U+017C, U+017D->U+017E, U+017E, U+0410..U+042F->U+0430..U+044F, U+0430..U+044F,\
+    U+05D0..U+05EA, U+0531..U+0556->U+0561..U+0586, U+0561..U+0587, U+0621..U+063A, U+01B9,\
+    U+01BF, U+0640..U+064A, U+0660..U+0669, U+066E, U+066F, U+0671..U+06D3, U+06F0..U+06FF,\
+    U+0904..U+0939, U+0958..U+095F, U+0960..U+0963, U+0966..U+096F, U+097B..U+097F,\
+    U+0985..U+09B9, U+09CE, U+09DC..U+09E3, U+09E6..U+09EF, U+0A05..U+0A39, U+0A59..U+0A5E,\
+    U+0A66..U+0A6F, U+0A85..U+0AB9, U+0AE0..U+0AE3, U+0AE6..U+0AEF, U+0B05..U+0B39,\
+    U+0B5C..U+0B61, U+0B66..U+0B6F, U+0B71, U+0B85..U+0BB9, U+0BE6..U+0BF2, U+0C05..U+0C39,\
+    U+0C66..U+0C6F, U+0C85..U+0CB9, U+0CDE..U+0CE3, U+0CE6..U+0CEF, U+0D05..U+0D39, U+0D60,\
+    U+0D61, U+0D66..U+0D6F, U+0D85..U+0DC6, U+1900..U+1938, U+1946..U+194F, U+A800..U+A805,\
+    U+A807..U+A822, U+0386->U+03B1, U+03AC->U+03B1, U+0388->U+03B5, U+03AD->U+03B5,\
+    U+0389->U+03B7, U+03AE->U+03B7, U+038A->U+03B9, U+0390->U+03B9, U+03AA->U+03B9,\
+    U+03AF->U+03B9, U+03CA->U+03B9, U+038C->U+03BF, U+03CC->U+03BF, U+038E->U+03C5,\
+    U+03AB->U+03C5, U+03B0->U+03C5, U+03CB->U+03C5, U+03CD->U+03C5, U+038F->U+03C9,\
+    U+03CE->U+03C9, U+03C2->U+03C3, U+0391..U+03A1->U+03B1..U+03C1,\
+    U+03A3..U+03A9->U+03C3..U+03C9, U+03B1..U+03C1, U+03C3..U+03C9, U+0E01..U+0E2E,\
+    U+0E30..U+0E3A, U+0E40..U+0E45, U+0E47, U+0E50..U+0E59, U+A000..U+A48F, U+4E00..U+9FBF,\
+    U+3400..U+4DBF, U+20000..U+2A6DF, U+F900..U+FAFF, U+2F800..U+2FA1F, U+2E80..U+2EFF,\
+    U+2F00..U+2FDF, U+3100..U+312F, U+31A0..U+31BF, U+3040..U+309F, U+30A0..U+30FF,\
+    U+31F0..U+31FF, U+AC00..U+D7AF, U+1100..U+11FF, U+3130..U+318F, U+A000..U+A48F,\
+    U+A490..U+A4CF
+}}
+""".format(root_path=ROOT_PATH, catalog_path=CATALOG_PATH)
+
+config = config + """
+searchd
+{{
+	listen				= {listen_port}
+    listen              = {listen_sql_host}:{listen_sql_port}:mysql41
+	log					= {root_path}{log_path}/searchd.log
+	pid_file			= {root_path}{etc_path}/searchd.pid
+}}
+""".format(listen_port=LISTEN_PORT,root_path=ROOT_PATH,log_path=LOG_PATH,
+           etc_path=ETC_PATH,listen_sql_host=LISTEN_SQL_HOST,
+           listen_sql_port=LISTEN_SQL_PORT,)
+
+print config
+
--- a/configs/sphinx/stopwords.txt
+++ b/configs/sphinx/stopwords.txt
@ -0,0 +1 @@
+a firefox he she the we I to it
--- a/configs/sphinx/wordforms.txt
+++ b/configs/sphinx/wordforms.txt
@ -0,0 +1,22 @@
+fire fox     > firefox
+thunder bird > thunderbird
+web site     > website
+web sites    > website
+book mark    > bookmark
+book marks   > bookmark
+home page    > homepage
+fire wall    > firewall
+fire walls   > firewall
+down load    > download
+down loads   > download
+up load      > upload
+up loads     > upload
+zone alarm   > zonealarm
+plug-in      > plugin
+plug-ins     > plugin
+plug ins     > plugin
+addon        > add-on
+addons       > add-on
+add on       > add-on
+add ons      > add-on
+face book    > facebook
--- a/settings.py
+++ b/settings.py
@ -156,6 +156,10 @@ WORD_LIST = path('configs/words.txt')
 SPHINX_HOST = '127.0.0.1'
 SPHINX_PORT = 3312

+SPHINX_INDEXER = '/usr/local/bin/indexer'
+SPHINX_SEARCHD = '/usr/local/bin/searchd'
+SPHINX_CONFIG_PATH = path('configs/sphinx/sphinx.conf')
+
 #
 # Sphinx results tweaking
 SEARCH_FORUM_MIN_AGE = 7 # age before which decay doesn't apply, in days
				`@ -0,0 +1,2 @@`
				INSERT INTO `tiki_categories` VALUES (1,'Knowledge Base','',0,0),(8,'Administration','',0,0),(13,'Firefox 2','',15,0),(14,'Firefox 3.0','',15,0),(19,'- This is a support/troubleshooting article -','http://wiki.mozilla.org/Support:PRD#Content_Types',15,0),(23,'How to Contribute','',0,0),(25,'Firefox 3.5/3.6','',15,0);