diff --git a/apps/search/utils.py b/apps/search/utils.py index fd0678bb3..91845a02b 100644 --- a/apps/search/utils.py +++ b/apps/search/utils.py @@ -1,3 +1,41 @@ +import subprocess import zlib +from django.conf import settings + + crc32 = lambda x: zlib.crc32(x) & 0xffffffff + + +call = lambda x: subprocess.Popen(x, stdout=subprocess.PIPE).communicate() + + +def reindex(rotate=False): + """ + Reindexes sphinx. Note this is only to be used in dev and test + environments. + """ + calls = [settings.SPHINX_INDEXER, '--all', '--config', + settings.SPHINX_CONFIG_PATH] + if rotate: + calls.append('--rotate') + + call(calls) + + +def start_sphinx(): + """ + Starts sphinx. Note this is only to be used in dev and test environments. + """ + + call([settings.SPHINX_SEARCHD, '--config', + settings.SPHINX_CONFIG_PATH]) + + +def stop_sphinx(): + """ + Stops sphinx. Note this is only to be used in dev and test environments. + """ + + call([settings.SPHINX_SEARCHD, '--stop', '--config', + settings.SPHINX_CONFIG_PATH]) diff --git a/apps/sumo/fixtures/categories.json b/apps/sumo/fixtures/categories.json new file mode 100644 index 000000000..d41b57574 --- /dev/null +++ b/apps/sumo/fixtures/categories.json @@ -0,0 +1,72 @@ +[ + { + "pk": 25, + "model": "sumo.category", + "fields": { + "hits": 0, + "parentId": 15, + "name": "Firefox 3.5/3.6", + "description": "" + } + }, + { + "pk": 23, + "model": "sumo.category", + "fields": { + "hits": 0, + "parentId": 0, + "name": "How to Contribute", + "description": "" + } + }, + { + "pk": 19, + "model": "sumo.category", + "fields": { + "hits": 0, + "parentId": 15, + "name": "- This is a support/troubleshooting article -", + "description": "http://wiki.mozilla.org/Support:PRD#Content_Types" + } + }, + { + "pk": 14, + "model": "sumo.category", + "fields": { + "hits": 0, + "parentId": 15, + "name": "Firefox 3.0", + "description": "" + } + }, + { + "pk": 13, + "model": "sumo.category", + "fields": { + "hits": 0, + "parentId": 15, + "name": "Firefox 2", + "description": "" + } + }, + { + "pk": 8, + "model": "sumo.category", + "fields": { + "hits": 0, + "parentId": 0, + "name": "Administration", + "description": "" + } + }, + { + "pk": 1, + "model": "sumo.category", + "fields": { + "hits": 0, + "parentId": 0, + "name": "Knowledge Base", + "description": "" + } + } +] diff --git a/apps/sumo/sql/category.sql b/apps/sumo/sql/category.sql new file mode 100644 index 000000000..6f427a637 --- /dev/null +++ b/apps/sumo/sql/category.sql @@ -0,0 +1,2 @@ +INSERT INTO `tiki_categories` VALUES (1,'Knowledge Base','',0,0),(8,'Administration','',0,0),(13,'Firefox 2','',15,0),(14,'Firefox 3.0','',15,0),(19,'- This is a support/troubleshooting article -','http://wiki.mozilla.org/Support:PRD#Content_Types',15,0),(23,'How to Contribute','',0,0),(25,'Firefox 3.5/3.6','',15,0); + diff --git a/apps/sumo/sql/wikipage.sql b/apps/sumo/sql/wikipage.sql new file mode 100644 index 000000000..26bf6964b --- /dev/null +++ b/apps/sumo/sql/wikipage.sql @@ -0,0 +1,45 @@ +CREATE TABLE IF NOT EXISTS `tiki_freetags` ( + `tagId` int(10) unsigned NOT NULL AUTO_INCREMENT, + `tag` varchar(30) NOT NULL DEFAULT '', + `raw_tag` varchar(50) NOT NULL DEFAULT '', + `lang` varchar(16) DEFAULT NULL, + PRIMARY KEY (`tagId`) +) ENGINE=MyISAM AUTO_INCREMENT=12176 DEFAULT CHARSET=latin1; + +CREATE TABLE IF NOT EXISTS `tiki_freetagged_objects` ( + `tagId` int(12) NOT NULL AUTO_INCREMENT, + `objectId` int(11) NOT NULL DEFAULT '0', + `user` varchar(200) NOT NULL DEFAULT '', + `created` int(14) NOT NULL DEFAULT '0', + PRIMARY KEY (`tagId`,`user`,`objectId`), + KEY `tagId` (`tagId`), + KEY `user` (`user`), + KEY `objectId` (`objectId`) +) ENGINE=MyISAM AUTO_INCREMENT=12176 DEFAULT CHARSET=latin1; + +CREATE TABLE IF NOT EXISTS `tiki_objects` ( + `objectId` int(12) NOT NULL AUTO_INCREMENT, + `type` varchar(50) DEFAULT NULL, + `itemId` varchar(255) DEFAULT NULL, + `description` text, + `created` int(14) DEFAULT NULL, + `name` varchar(200) DEFAULT NULL, + `href` varchar(200) DEFAULT NULL, + `hits` int(8) DEFAULT NULL, + PRIMARY KEY (`objectId`), + KEY `type` (`type`,`itemId`), + KEY `itemId` (`itemId`,`type`) +) ENGINE=MyISAM AUTO_INCREMENT=35581 DEFAULT CHARSET=latin1; + +INSERT INTO tiki_objects (objectId, type, itemId, name) VALUES (79, 'wiki page', 'Firefox Support Home Page', 'Firefox Support Home Page'); +INSERT INTO tiki_objects (objectId, type, itemId, name) VALUES (84, 'wiki page', 'Style Guide', 'Style Guide'); +INSERT INTO tiki_objects (objectId, type, itemId, name) VALUES (62, 'wiki page', 'Video or audio does not play', 'Video or audio does not play'); + +CREATE TABLE IF NOT EXISTS `tiki_category_objects` ( + `catObjectId` int(12) NOT NULL DEFAULT '0', + `categId` int(12) NOT NULL DEFAULT '0', + PRIMARY KEY (`catObjectId`,`categId`), + KEY `categId` (`categId`) +) ENGINE=MyISAM DEFAULT CHARSET=latin1; + +INSERT INTO tiki_category_objects (catObjectId, categId) VALUES (79, 8), (84, 23), (62, 1), (62, 13), (62, 14), (62, 19), (62, 25); diff --git a/configs/sphinx/localsettings.py-dist b/configs/sphinx/localsettings.py-dist new file mode 100644 index 000000000..10123480e --- /dev/null +++ b/configs/sphinx/localsettings.py-dist @@ -0,0 +1,32 @@ + + +## MySQL Connection Settings +MYSQL_USER = '' +MYSQL_PASS = '' +MYSQL_HOST = '' +MYSQL_NAME = '' + +## Data store settings +# ROOT_PATH is prepended to all other paths +ROOT_PATH = '/opt/local' + +CATALOG_PATH = '/data/sphinx' +LOG_PATH = '/log/searchd' +ETC_PATH = '/etc' + +# Listen on...? +LISTEN_PORT = 3312 +LISTEN_SQL_HOST = 'localhost' +LISTEN_SQL_PORT = 3317 + +## Age divisor +# We convert the |age| attribute from the |forum_thread| index into a more +# useful unit than seconds. In the list below, to convert to assign +# this to +# : +# second : 1 +# minute : 60 +# hour : 3600 +# day : 86400 +# week : 604800 +AGE_DIVISOR = 86400 diff --git a/configs/sphinx/localsettings_django.py b/configs/sphinx/localsettings_django.py new file mode 100644 index 000000000..3f5c16b62 --- /dev/null +++ b/configs/sphinx/localsettings_django.py @@ -0,0 +1,32 @@ +import os +import sys + +SETTINGS_DIR = os.path.realpath( + os.path.join(os.path.dirname(__file__), os.path.sep.join(('..',)*2))) + +sys.path.append(SETTINGS_DIR) +sys.path.append(os.path.join(SETTINGS_DIR,'lib')) + +import settings_local as settings + +s = settings.DATABASES['default'] +MYSQL_PASS = s['PASSWORD'] +MYSQL_USER = s['USER'] +MYSQL_HOST = s.get('HOST', 'localhost') +MYSQL_NAME = s['NAME'] + +if MYSQL_HOST.endswith('.sock'): + MYSQL_HOST = 'localhost' + +if os.environ.get('DJANGO_ENVIRONMENT') == 'test': + MYSQL_NAME = 'test_' + MYSQL_NAME + +ROOT_PATH = '/tmp' +CATALOG_PATH = '/data' +LOG_PATH = '/log' +ETC_PATH = '/etc' + +LISTEN_PORT = 3312 +LISTEN_SQL_HOST = '127.0.0.1' +LISTEN_SQL_PORT = 3317 +AGE_DIVISOR = 86400 diff --git a/configs/sphinx/sphinx.conf b/configs/sphinx/sphinx.conf new file mode 100755 index 000000000..865a0689c --- /dev/null +++ b/configs/sphinx/sphinx.conf @@ -0,0 +1,391 @@ +#!/usr/bin/env python2.6 + +try: + from localsettings import * +except ImportError: + from localsettings_django import * + +############################################################################# +## data source definition +############################################################################# + +config = """ +source forum_threads +{{ + type = mysql + sql_host = {sql_host} + sql_user = {sql_user} + sql_pass = {sql_pass} + sql_db = {sql_db} + + sql_query_pre = SET SESSION query_cache_type = OFF + + sql_query = \ + SELECT \ + threadId, \ + object AS forumId, \ + title, \ + userName AS author, \ + CRC32(userName) AS author_ord, \ + CRC32(type) AS status, \ + data AS content, \ + commentDate AS created, \ + ( \ + SELECT \ + COUNT(replies.threadId) \ + FROM \ + tiki_comments AS replies \ + WHERE \ + replies.parentId = threads.threadId \ + ) AS replies, \ + IF( \ + ( \ + SELECT \ + COUNT(replies0.threadId) \ + FROM \ + tiki_comments AS replies0 \ + WHERE \ + replies0.parentId = threads.threadId \ + ), \ + ( \ + SELECT \ + MAX(replies2.commentDate) \ + FROM \ + tiki_comments AS replies2 \ + WHERE \ + replies2.parentId = threads.threadId \ + ), \ + threads.commentDate \ + ) AS last_updated, \ + IF( \ + ( \ + SELECT \ + COUNT(replies1.threadId) \ + FROM \ + tiki_comments AS replies1 \ + WHERE \ + replies1.parentId = threads.threadId \ + ), \ + (UNIX_TIMESTAMP() - ( \ + SELECT \ + MAX(replies10.commentDate) \ + FROM \ + tiki_comments AS replies10 \ + WHERE \ + replies10.parentId = threads.threadId \ + ))/{age_unit}, \ + (UNIX_TIMESTAMP() - threads.commentDate)/{age_unit} \ + ) AS age, \ + (\ + SELECT \ + GROUP_CONCAT(replies3.data) \ + FROM \ + tiki_comments AS replies3 \ + WHERE \ + replies3.parentId = threads.threadId \ + ) AS reply_content \ + FROM \ + tiki_comments AS threads\ + WHERE \ + threads.objectType = 'forum' \ + AND threads.parentId = 0 + + sql_attr_uint = forumId + sql_attr_uint = author_ord + sql_attr_uint = status + sql_attr_timestamp = created + sql_attr_timestamp = last_updated + sql_attr_uint = age + sql_attr_uint = replies + + sql_attr_multi = uint authors from query; SELECT \ + IF(parentId=0,threadId,parentId) AS threadid, \ + CRC32(userName) AS poster \ + FROM \ + tiki_comments \ + WHERE \ + objectType = 'forum' + +}} +""".format(sql_host = MYSQL_HOST,sql_user = MYSQL_USER, + sql_pass = MYSQL_PASS,sql_db=MYSQL_NAME,age_unit=AGE_DIVISOR) + +config = config + """ +index forum_threads +{{ + source = forum_threads + path = {root_path}{catalog_path}/forum-thread-catalog + charset_type = utf-8 + morphology = stem_en + min_stemming_len = 4 + stopwords = {root_path}/stopwords.txt + wordforms = {root_path}/wordforms.txt + charset_table = U+FF10..U+FF19->0..9, 0..9, U+FF41..U+FF5A->a..z, U+FF21..U+FF3A->a..z,\ + A..Z->a..z, a..z, U+0149, U+017F, U+0138, U+00DF, U+00FF, U+00C0..U+00D6->U+00E0..U+00F6,\ + U+00E0..U+00F6, U+00D8..U+00DE->U+00F8..U+00FE, U+00F8..U+00FE, U+0100->U+0101, U+0101,\ + U+0102->U+0103, U+0103, U+0104->U+0105, U+0105, U+0106->U+0107, U+0107, U+0108->U+0109,\ + U+0109, U+010A->U+010B, U+010B, U+010C->U+010D, U+010D, U+010E->U+010F, U+010F,\ + U+0110->U+0111, U+0111, U+0112->U+0113, U+0113, U+0114->U+0115, U+0115, U+0116->U+0117,\ + U+0117, U+0118->U+0119, U+0119, U+011A->U+011B, U+011B, U+011C->U+011D, U+011D,\ + U+011E->U+011F, U+011F, U+0130->U+0131, U+0131, U+0132->U+0133, U+0133, U+0134->U+0135,\ + U+0135, U+0136->U+0137, U+0137, U+0139->U+013A, U+013A, U+013B->U+013C, U+013C,\ + U+013D->U+013E, U+013E, U+013F->U+0140, U+0140, U+0141->U+0142, U+0142, U+0143->U+0144,\ + U+0144, U+0145->U+0146, U+0146, U+0147->U+0148, U+0148, U+014A->U+014B, U+014B,\ + U+014C->U+014D, U+014D, U+014E->U+014F, U+014F, U+0150->U+0151, U+0151, U+0152->U+0153,\ + U+0153, U+0154->U+0155, U+0155, U+0156->U+0157, U+0157, U+0158->U+0159, U+0159,\ + U+015A->U+015B, U+015B, U+015C->U+015D, U+015D, U+015E->U+015F, U+015F, U+0160->U+0161,\ + U+0161, U+0162->U+0163, U+0163, U+0164->U+0165, U+0165, U+0166->U+0167, U+0167,\ + U+0168->U+0169, U+0169, U+016A->U+016B, U+016B, U+016C->U+016D, U+016D, U+016E->U+016F,\ + U+016F, U+0170->U+0171, U+0171, U+0172->U+0173, U+0173, U+0174->U+0175, U+0175,\ + U+0176->U+0177, U+0177, U+0178->U+00FF, U+00FF, U+0179->U+017A, U+017A, U+017B->U+017C,\ + U+017C, U+017D->U+017E, U+017E, U+0410..U+042F->U+0430..U+044F, U+0430..U+044F,\ + U+05D0..U+05EA, U+0531..U+0556->U+0561..U+0586, U+0561..U+0587, U+0621..U+063A, U+01B9,\ + U+01BF, U+0640..U+064A, U+0660..U+0669, U+066E, U+066F, U+0671..U+06D3, U+06F0..U+06FF,\ + U+0904..U+0939, U+0958..U+095F, U+0960..U+0963, U+0966..U+096F, U+097B..U+097F,\ + U+0985..U+09B9, U+09CE, U+09DC..U+09E3, U+09E6..U+09EF, U+0A05..U+0A39, U+0A59..U+0A5E,\ + U+0A66..U+0A6F, U+0A85..U+0AB9, U+0AE0..U+0AE3, U+0AE6..U+0AEF, U+0B05..U+0B39,\ + U+0B5C..U+0B61, U+0B66..U+0B6F, U+0B71, U+0B85..U+0BB9, U+0BE6..U+0BF2, U+0C05..U+0C39,\ + U+0C66..U+0C6F, U+0C85..U+0CB9, U+0CDE..U+0CE3, U+0CE6..U+0CEF, U+0D05..U+0D39, U+0D60,\ + U+0D61, U+0D66..U+0D6F, U+0D85..U+0DC6, U+1900..U+1938, U+1946..U+194F, U+A800..U+A805,\ + U+A807..U+A822, U+0386->U+03B1, U+03AC->U+03B1, U+0388->U+03B5, U+03AD->U+03B5,\ + U+0389->U+03B7, U+03AE->U+03B7, U+038A->U+03B9, U+0390->U+03B9, U+03AA->U+03B9,\ + U+03AF->U+03B9, U+03CA->U+03B9, U+038C->U+03BF, U+03CC->U+03BF, U+038E->U+03C5,\ + U+03AB->U+03C5, U+03B0->U+03C5, U+03CB->U+03C5, U+03CD->U+03C5, U+038F->U+03C9,\ + U+03CE->U+03C9, U+03C2->U+03C3, U+0391..U+03A1->U+03B1..U+03C1,\ + U+03A3..U+03A9->U+03C3..U+03C9, U+03B1..U+03C1, U+03C3..U+03C9, U+0E01..U+0E2E,\ + U+0E30..U+0E3A, U+0E40..U+0E45, U+0E47, U+0E50..U+0E59, U+A000..U+A48F, U+4E00..U+9FBF,\ + U+3400..U+4DBF, U+20000..U+2A6DF, U+F900..U+FAFF, U+2F800..U+2FA1F, U+2E80..U+2EFF,\ + U+2F00..U+2FDF, U+3100..U+312F, U+31A0..U+31BF, U+3040..U+309F, U+30A0..U+30FF,\ + U+31F0..U+31FF, U+AC00..U+D7AF, U+1100..U+11FF, U+3130..U+318F, U+A000..U+A48F,\ + U+A490..U+A4CF +}} +""".format(root_path=ROOT_PATH, catalog_path=CATALOG_PATH) + +config = config + """ +source forum_posts +{ + type = mysql + sql_host = %s + sql_user = %s + sql_pass = %s + sql_db = %s + + sql_query_pre = SET SESSION query_cache_type=OFF + + sql_query = \ + SELECT \ + threadId AS postid, \ + IF(parentId = 0, threadId, parentId) AS threadid, \ + IF(parentId = 0, object, ( \ + SELECT \ + parents0.object \ + FROM \ + tiki_comments AS parents0 \ + WHERE \ + parents0.threadId = posts.parentId \ + )) AS forumid, \ + userName AS author, \ + CRC32(userName) AS author_ord, \ + CRC32(type) AS status, \ + commentDate AS created, \ + data AS content, \ + IF(parentId = 0, title, ( \ + SELECT \ + title \ + FROM \ + tiki_comments AS parents \ + WHERE \ + parents.threadId = posts.parentId \ + )) AS title \ + FROM \ + tiki_comments AS posts \ + WHERE \ + objectType = 'forum' + + + sql_attr_uint = threadid + sql_attr_uint = author_ord + sql_attr_uint = status + sql_attr_uint = forumid + sql_attr_timestamp = created +} + +""" % (MYSQL_HOST, MYSQL_USER, MYSQL_PASS, MYSQL_NAME) + +config = config + """ +index forum_posts +{{ + source = forum_posts + path = {root_path}{catalog_path}/forum-post-catalog + charset_type = utf-8 + morphology = stem_en + min_stemming_len = 4 + stopwords = {root_path}/stopwords.txt + wordforms = {root_path}/wordforms.txt + charset_table = U+FF10..U+FF19->0..9, 0..9, U+FF41..U+FF5A->a..z, U+FF21..U+FF3A->a..z,\ + A..Z->a..z, a..z, U+0149, U+017F, U+0138, U+00DF, U+00FF, U+00C0..U+00D6->U+00E0..U+00F6,\ + U+00E0..U+00F6, U+00D8..U+00DE->U+00F8..U+00FE, U+00F8..U+00FE, U+0100->U+0101, U+0101,\ + U+0102->U+0103, U+0103, U+0104->U+0105, U+0105, U+0106->U+0107, U+0107, U+0108->U+0109,\ + U+0109, U+010A->U+010B, U+010B, U+010C->U+010D, U+010D, U+010E->U+010F, U+010F,\ + U+0110->U+0111, U+0111, U+0112->U+0113, U+0113, U+0114->U+0115, U+0115, U+0116->U+0117,\ + U+0117, U+0118->U+0119, U+0119, U+011A->U+011B, U+011B, U+011C->U+011D, U+011D,\ + U+011E->U+011F, U+011F, U+0130->U+0131, U+0131, U+0132->U+0133, U+0133, U+0134->U+0135,\ + U+0135, U+0136->U+0137, U+0137, U+0139->U+013A, U+013A, U+013B->U+013C, U+013C,\ + U+013D->U+013E, U+013E, U+013F->U+0140, U+0140, U+0141->U+0142, U+0142, U+0143->U+0144,\ + U+0144, U+0145->U+0146, U+0146, U+0147->U+0148, U+0148, U+014A->U+014B, U+014B,\ + U+014C->U+014D, U+014D, U+014E->U+014F, U+014F, U+0150->U+0151, U+0151, U+0152->U+0153,\ + U+0153, U+0154->U+0155, U+0155, U+0156->U+0157, U+0157, U+0158->U+0159, U+0159,\ + U+015A->U+015B, U+015B, U+015C->U+015D, U+015D, U+015E->U+015F, U+015F, U+0160->U+0161,\ + U+0161, U+0162->U+0163, U+0163, U+0164->U+0165, U+0165, U+0166->U+0167, U+0167,\ + U+0168->U+0169, U+0169, U+016A->U+016B, U+016B, U+016C->U+016D, U+016D, U+016E->U+016F,\ + U+016F, U+0170->U+0171, U+0171, U+0172->U+0173, U+0173, U+0174->U+0175, U+0175,\ + U+0176->U+0177, U+0177, U+0178->U+00FF, U+00FF, U+0179->U+017A, U+017A, U+017B->U+017C,\ + U+017C, U+017D->U+017E, U+017E, U+0410..U+042F->U+0430..U+044F, U+0430..U+044F,\ + U+05D0..U+05EA, U+0531..U+0556->U+0561..U+0586, U+0561..U+0587, U+0621..U+063A, U+01B9,\ + U+01BF, U+0640..U+064A, U+0660..U+0669, U+066E, U+066F, U+0671..U+06D3, U+06F0..U+06FF,\ + U+0904..U+0939, U+0958..U+095F, U+0960..U+0963, U+0966..U+096F, U+097B..U+097F,\ + U+0985..U+09B9, U+09CE, U+09DC..U+09E3, U+09E6..U+09EF, U+0A05..U+0A39, U+0A59..U+0A5E,\ + U+0A66..U+0A6F, U+0A85..U+0AB9, U+0AE0..U+0AE3, U+0AE6..U+0AEF, U+0B05..U+0B39,\ + U+0B5C..U+0B61, U+0B66..U+0B6F, U+0B71, U+0B85..U+0BB9, U+0BE6..U+0BF2, U+0C05..U+0C39,\ + U+0C66..U+0C6F, U+0C85..U+0CB9, U+0CDE..U+0CE3, U+0CE6..U+0CEF, U+0D05..U+0D39, U+0D60,\ + U+0D61, U+0D66..U+0D6F, U+0D85..U+0DC6, U+1900..U+1938, U+1946..U+194F, U+A800..U+A805,\ + U+A807..U+A822, U+0386->U+03B1, U+03AC->U+03B1, U+0388->U+03B5, U+03AD->U+03B5,\ + U+0389->U+03B7, U+03AE->U+03B7, U+038A->U+03B9, U+0390->U+03B9, U+03AA->U+03B9,\ + U+03AF->U+03B9, U+03CA->U+03B9, U+038C->U+03BF, U+03CC->U+03BF, U+038E->U+03C5,\ + U+03AB->U+03C5, U+03B0->U+03C5, U+03CB->U+03C5, U+03CD->U+03C5, U+038F->U+03C9,\ + U+03CE->U+03C9, U+03C2->U+03C3, U+0391..U+03A1->U+03B1..U+03C1,\ + U+03A3..U+03A9->U+03C3..U+03C9, U+03B1..U+03C1, U+03C3..U+03C9, U+0E01..U+0E2E,\ + U+0E30..U+0E3A, U+0E40..U+0E45, U+0E47, U+0E50..U+0E59, U+A000..U+A48F, U+4E00..U+9FBF,\ + U+3400..U+4DBF, U+20000..U+2A6DF, U+F900..U+FAFF, U+2F800..U+2FA1F, U+2E80..U+2EFF,\ + U+2F00..U+2FDF, U+3100..U+312F, U+31A0..U+31BF, U+3040..U+309F, U+30A0..U+30FF,\ + U+31F0..U+31FF, U+AC00..U+D7AF, U+1100..U+11FF, U+3130..U+318F, U+A000..U+A48F,\ + U+A490..U+A4CF +}} +""".format(root_path=ROOT_PATH,catalog_path=CATALOG_PATH) + +config = config + """ +source wiki_pages +{ + type = mysql + sql_host = %s + sql_user = %s + sql_pass = %s + sql_db = %s + + sql_query_pre = SET SESSION query_cache_type=OFF + + sql_query = \ + SELECT \ + page_id AS pageId, \ + pageName, \ + data AS content, \ + lastModif, \ + CRC32(lang) AS locale, \ + keywords, \ + ( \ + SELECT \ + GROUP_CONCAT(DISTINCT tag) \ + FROM \ + tiki_freetags t_f \ + JOIN tiki_freetagged_objects t_f_o ON t_f.tagId = t_f_o.tagId \ + JOIN tiki_objects t_o ON t_f_o.objectId = t_o.objectId \ + WHERE \ + t_o.itemId = pageName \ + ) AS tags \ + FROM \ + tiki_pages \ + WHERE \ + data NOT LIKE '{REDIRECT%%' + + sql_attr_timestamp = lastModif + sql_attr_uint = locale + + sql_attr_multi = uint tag from query; SELECT \ + page_id AS pageId, \ + CRC32(tag) AS tag \ + FROM \ + tiki_freetags t_f \ + INNER JOIN tiki_freetagged_objects t_f_o ON t_f.tagId = t_f_o.tagId \ + INNER JOIN tiki_objects t_o ON t_f_o.objectId = t_o.objectId \ + INNER JOIN tiki_pages t_p ON t_o.itemId = t_p.pageName \ + WHERE \ + page_id IS NOT NULL + + sql_attr_multi = uint category from query; SELECT \ + page_id AS pageId, \ + t_c.categId \ + FROM \ + tiki_categories t_c \ + INNER JOIN tiki_category_objects t_c_o ON t_c.categId = t_c_o.categId \ + INNER JOIN tiki_objects t_o ON t_c_o.catObjectId = t_o.objectId \ + INNER JOIN tiki_pages t_p ON t_o.itemId = t_p.pageName \ + WHERE \ + page_id IS NOT NULL +} +""" % (MYSQL_HOST, MYSQL_USER, MYSQL_PASS, MYSQL_NAME) + +config = config + """ +index wiki_pages +{{ + source = wiki_pages + path = {root_path}{catalog_path}/wiki-page-catalog + charset_type = utf-8 + morphology = stem_en + min_stemming_len = 4 + stopwords = {root_path}/stopwords.txt + wordforms = {root_path}/wordforms.txt + charset_table = U+FF10..U+FF19->0..9, 0..9, U+FF41..U+FF5A->a..z, U+FF21..U+FF3A->a..z,\ + A..Z->a..z, a..z, U+0149, U+017F, U+0138, U+00DF, U+00FF, U+00C0..U+00D6->U+00E0..U+00F6,\ + U+00E0..U+00F6, U+00D8..U+00DE->U+00F8..U+00FE, U+00F8..U+00FE, U+0100->U+0101, U+0101,\ + U+0102->U+0103, U+0103, U+0104->U+0105, U+0105, U+0106->U+0107, U+0107, U+0108->U+0109,\ + U+0109, U+010A->U+010B, U+010B, U+010C->U+010D, U+010D, U+010E->U+010F, U+010F,\ + U+0110->U+0111, U+0111, U+0112->U+0113, U+0113, U+0114->U+0115, U+0115, U+0116->U+0117,\ + U+0117, U+0118->U+0119, U+0119, U+011A->U+011B, U+011B, U+011C->U+011D, U+011D,\ + U+011E->U+011F, U+011F, U+0130->U+0131, U+0131, U+0132->U+0133, U+0133, U+0134->U+0135,\ + U+0135, U+0136->U+0137, U+0137, U+0139->U+013A, U+013A, U+013B->U+013C, U+013C,\ + U+013D->U+013E, U+013E, U+013F->U+0140, U+0140, U+0141->U+0142, U+0142, U+0143->U+0144,\ + U+0144, U+0145->U+0146, U+0146, U+0147->U+0148, U+0148, U+014A->U+014B, U+014B,\ + U+014C->U+014D, U+014D, U+014E->U+014F, U+014F, U+0150->U+0151, U+0151, U+0152->U+0153,\ + U+0153, U+0154->U+0155, U+0155, U+0156->U+0157, U+0157, U+0158->U+0159, U+0159,\ + U+015A->U+015B, U+015B, U+015C->U+015D, U+015D, U+015E->U+015F, U+015F, U+0160->U+0161,\ + U+0161, U+0162->U+0163, U+0163, U+0164->U+0165, U+0165, U+0166->U+0167, U+0167,\ + U+0168->U+0169, U+0169, U+016A->U+016B, U+016B, U+016C->U+016D, U+016D, U+016E->U+016F,\ + U+016F, U+0170->U+0171, U+0171, U+0172->U+0173, U+0173, U+0174->U+0175, U+0175,\ + U+0176->U+0177, U+0177, U+0178->U+00FF, U+00FF, U+0179->U+017A, U+017A, U+017B->U+017C,\ + U+017C, U+017D->U+017E, U+017E, U+0410..U+042F->U+0430..U+044F, U+0430..U+044F,\ + U+05D0..U+05EA, U+0531..U+0556->U+0561..U+0586, U+0561..U+0587, U+0621..U+063A, U+01B9,\ + U+01BF, U+0640..U+064A, U+0660..U+0669, U+066E, U+066F, U+0671..U+06D3, U+06F0..U+06FF,\ + U+0904..U+0939, U+0958..U+095F, U+0960..U+0963, U+0966..U+096F, U+097B..U+097F,\ + U+0985..U+09B9, U+09CE, U+09DC..U+09E3, U+09E6..U+09EF, U+0A05..U+0A39, U+0A59..U+0A5E,\ + U+0A66..U+0A6F, U+0A85..U+0AB9, U+0AE0..U+0AE3, U+0AE6..U+0AEF, U+0B05..U+0B39,\ + U+0B5C..U+0B61, U+0B66..U+0B6F, U+0B71, U+0B85..U+0BB9, U+0BE6..U+0BF2, U+0C05..U+0C39,\ + U+0C66..U+0C6F, U+0C85..U+0CB9, U+0CDE..U+0CE3, U+0CE6..U+0CEF, U+0D05..U+0D39, U+0D60,\ + U+0D61, U+0D66..U+0D6F, U+0D85..U+0DC6, U+1900..U+1938, U+1946..U+194F, U+A800..U+A805,\ + U+A807..U+A822, U+0386->U+03B1, U+03AC->U+03B1, U+0388->U+03B5, U+03AD->U+03B5,\ + U+0389->U+03B7, U+03AE->U+03B7, U+038A->U+03B9, U+0390->U+03B9, U+03AA->U+03B9,\ + U+03AF->U+03B9, U+03CA->U+03B9, U+038C->U+03BF, U+03CC->U+03BF, U+038E->U+03C5,\ + U+03AB->U+03C5, U+03B0->U+03C5, U+03CB->U+03C5, U+03CD->U+03C5, U+038F->U+03C9,\ + U+03CE->U+03C9, U+03C2->U+03C3, U+0391..U+03A1->U+03B1..U+03C1,\ + U+03A3..U+03A9->U+03C3..U+03C9, U+03B1..U+03C1, U+03C3..U+03C9, U+0E01..U+0E2E,\ + U+0E30..U+0E3A, U+0E40..U+0E45, U+0E47, U+0E50..U+0E59, U+A000..U+A48F, U+4E00..U+9FBF,\ + U+3400..U+4DBF, U+20000..U+2A6DF, U+F900..U+FAFF, U+2F800..U+2FA1F, U+2E80..U+2EFF,\ + U+2F00..U+2FDF, U+3100..U+312F, U+31A0..U+31BF, U+3040..U+309F, U+30A0..U+30FF,\ + U+31F0..U+31FF, U+AC00..U+D7AF, U+1100..U+11FF, U+3130..U+318F, U+A000..U+A48F,\ + U+A490..U+A4CF +}} +""".format(root_path=ROOT_PATH, catalog_path=CATALOG_PATH) + +config = config + """ +searchd +{{ + listen = {listen_port} + listen = {listen_sql_host}:{listen_sql_port}:mysql41 + log = {root_path}{log_path}/searchd.log + pid_file = {root_path}{etc_path}/searchd.pid +}} +""".format(listen_port=LISTEN_PORT,root_path=ROOT_PATH,log_path=LOG_PATH, + etc_path=ETC_PATH,listen_sql_host=LISTEN_SQL_HOST, + listen_sql_port=LISTEN_SQL_PORT,) + +print config + diff --git a/configs/sphinx/stopwords.txt b/configs/sphinx/stopwords.txt new file mode 100644 index 000000000..0209ecd98 --- /dev/null +++ b/configs/sphinx/stopwords.txt @@ -0,0 +1 @@ +a firefox he she the we I to it diff --git a/configs/sphinx/wordforms.txt b/configs/sphinx/wordforms.txt new file mode 100644 index 000000000..d111914e3 --- /dev/null +++ b/configs/sphinx/wordforms.txt @@ -0,0 +1,22 @@ +fire fox > firefox +thunder bird > thunderbird +web site > website +web sites > website +book mark > bookmark +book marks > bookmark +home page > homepage +fire wall > firewall +fire walls > firewall +down load > download +down loads > download +up load > upload +up loads > upload +zone alarm > zonealarm +plug-in > plugin +plug-ins > plugin +plug ins > plugin +addon > add-on +addons > add-on +add on > add-on +add ons > add-on +face book > facebook diff --git a/settings.py b/settings.py index 5382c9a85..c7f711496 100644 --- a/settings.py +++ b/settings.py @@ -156,6 +156,10 @@ WORD_LIST = path('configs/words.txt') SPHINX_HOST = '127.0.0.1' SPHINX_PORT = 3312 +SPHINX_INDEXER = '/usr/local/bin/indexer' +SPHINX_SEARCHD = '/usr/local/bin/searchd' +SPHINX_CONFIG_PATH = path('configs/sphinx/sphinx.conf') + # # Sphinx results tweaking SEARCH_FORUM_MIN_AGE = 7 # age before which decay doesn't apply, in days