addons-server/configs/sphinx/sphinx.conf

#!/usr/bin/env python

try:
    from localsettings import *
except ImportError:
    from localsettings_django import *

MYSQL_SOURCE_CONFIG = """
    type                    = mysql
    sql_host                = %s
    sql_user                = %s
    sql_pass                = %s
    sql_db                  = %s

    sql_query_pre           = SET NAMES utf8
    sql_query_pre           = SET SESSION query_cache_type=OFF
""" % (MYSQL_HOST, MYSQL_USER, MYSQL_PASS, MYSQL_NAME,)

CHARSET_DATA = """
    charset_type = utf-8
    charset_table = 0..9, U+41..U+5a->U+61..U+7a, U+61..U+7a, U+aa, U+b5, \
            U+ba, U+c0..U+d6->U+e0..U+f6, U+d8..U+de->U+f8..U+fe, U+df..U+f6, \
            U+f8..U+ff, U+100..U+12f/2, U+130->U+69, U+131, U+132..U+137/2, \
            U+138, U+139..U+148/2, U+149, U+14a..U+177/2, U+178->U+ff, \
            U+179..U+17e/2, U+17f..U+180, U+181->U+253, U+182..U+185/2, \
            U+186->U+254, U+187..U+188/2, U+189..U+18a->U+256..U+257, \
            U+18b..U+18c/2, U+18d, U+18e->U+1dd, U+18f->U+259, U+190->U+25b, \
            U+191..U+192/2, U+193->U+260, U+194->U+263, U+195, U+196->U+269, \
            U+197->U+268, U+198..U+199/2, U+19a..U+19b, U+19c->U+26f, \
            U+19d->U+272, U+19e,  U+19f->U+275, U+1a0..U+1a5/2, \
            U+1a6->U+280, U+1a7..U+1a8/2, U+1a9->U+283, U+1aa..U+1ab, \
            U+1ac..U+1ad/2, U+1ae->U+288, U+1af..U+1b0/2, \
            U+1b1..U+1b2->U+28a..U+28b, U+1b3..U+1b6/2, U+1b7->U+292, \
            U+1b8..U+1b9/2, U+1ba..U+1bb, U+1bc..U+1bd/2, U+1be..U+1c3, \
            U+1c4->U+1c6, U+1c5..U+1c6/2, U+1c7->U+1c9, U+1c8..U+1c9/2, \
            U+1ca->U+1cc, U+1cb..U+1dc/2, U+1dd, U+1de..U+1ef/2, U+1f0, \
            U+1f1->U+1f3, U+1f2..U+1f5/2, U+1f6->U+195, U+1f7->U+1bf, \
            U+1f8..U+21f/2, U+220->U+19e, U+221, U+222..U+233/2, \
            U+234..U+23a, U+23b..U+23c/2, U+23d->U+19a, U+23e..U+240, \
            U+241->U+294, U+250..U+2c1, U+2c6..U+2d1, U+2e0..U+2e4, U+2ee, \
            U+37a, U+386..U+389->U+3ac..U+3af, U+38c..U+38e->U+3cc..U+3ce, \
            U+390, U+391..U+3a1->U+3b1..U+3c1, U+3a3..U+3ab->U+3c3..U+3cb, \
            U+3ac..U+3ce, U+3d0..U+3d7, U+3d8..U+3ef/2, U+3f0..U+3f3, \
            U+3f4->U+3b8, U+3f5, U+3f7..U+3f8/2, U+3f9->U+3f2, \
            U+3fa..U+3fb/2, U+3fc..U+3ff, U+400..U+40f->U+450..U+45f, \
            U+410..U+42f->U+430..U+44f, U+430..U+45f, U+460..U+481/2, \
            U+48a..U+4bf/2, U+4c0, U+4c1..U+4ce/2, U+4d0..U+4f9/2, \
            U+500..U+50f/2, U+531..U+556->U+561..U+586, U+559, U+561..U+587, \
            U+5d0..U+5ea, U+5f0..U+5f2, U+621..U+63a, U+640..U+64a, \
            U+66e..U+66f, U+671..U+6d3, U+6d5, U+6e5..U+6e6, U+6ee..U+6ef, \
            U+6fa..U+6fc, U+6ff, U+e01..U+e30, U+e32..U+e33, U+e40..U+e46, \
            U+e81..U+e82, U+e84, U+e87..U+e88, U+e8a, U+e8d, U+e94..U+e97, \
            U+e99..U+e9f, U+ea1..U+ea3, U+ea5, U+ea7, U+eaa..U+eab, \
            U+ead..U+eb0, U+eb2..U+eb3, U+ebd, U+ec0..U+ec4,  U+ec6, \
            U+edc..U+edd, U+1000..U+1021, U+1023..U+1027, U+1029..U+102a, \
            U+1050..U+1055, U+10a0..U+10c5->U+2d00..U+2d25, U+10d0..U+10fa, \
            U+10fc, U+2d00..U+2d25, U+1d00..U+1dbf, U+1e00..U+1e95/2, \
            U+1e96..U+1e9b, U+1ea0..U+1ef9/2, U+3005..U+3006, U+3031..U+3035, \
            U+303b..U+303c, U+3041..U+3096, U+309d..U+309f, U+30a1..U+30fa, \
            U+30fc..U+30ff, U+31f0..U+31ff
    ngram_chars = U+3400..U+4DB5, U+4E00..U+9FA5, U+20000..U+2A6D6
    ngram_len = 1
    """

# TODO(davedash): When we stop serving featured/creatured via sphinx we can
# remove `recommended`

config = """
source addons
{
""" + MYSQL_SOURCE_CONFIG + """
    sql_query                = \
    SELECT \
        id, app, addon_id, type, status as addon_status, locale, \
        locale_ord, averagerating, weeklydownloads, totaldownloads, \
        inactive, guid_ord, name, UPPER(name) AS name_ord, \
        description, summary, developercomments, \
        ( \
            SELECT 1 FROM features f \
            WHERE f.addon_id=t.addon_id \
                AND f.start <= NOW() \
                AND f.end > NOW() \
                AND (f.locale IS NULL OR t.locale = f.locale) \
                AND t.app = f.application_id \
                LIMIT 1 \
        ) AS recommended, \
        IF (status=6 OR type=9, 1, \
          ( \
            SELECT count(*) \
            FROM files f \
            WHERE f.version_id=t.current_version AND f.status>0 \
          ) \
        ) AS num_files, \
        (\
            SELECT count(DISTINCT av.application_id) \
            FROM applications_versions av \
            WHERE av.version_id=t.current_version \
        ) AS num_apps, \
        ( \
            SELECT GROUP_CONCAT(username) \
            FROM addons_users au, users u \
            WHERE addon_id=t.addon_id AND au.user_id = u.id AND listed = 1 \
        ) AS authors, \
        ( \
             SELECT GROUP_CONCAT(tag_text) \
             FROM users_tags_addons, tags \
             WHERE tags.id = tag_id \
             AND tags.blacklisted = 0 \
             AND addon_id = t.addon_id \
        ) AS tags, \
        ( \
            SELECT IF(type=4,9999999999999, max(version_int)) \
            FROM applications_versions s_av, appversions max \
            WHERE s_av.application_id = t.app \
                AND s_av.version_id = t.current_version AND s_av.max = max.id \
        ) AS max_ver, \
        ( \
            SELECT IF(type=4,0, min(version_int)) \
            FROM applications_versions s_av, appversions min \
            WHERE s_av.application_id = t.app \
                AND s_av.version_id = t.current_version \
                AND s_av.min = min.id \
        ) AS min_ver, \
        created, \
        modified \
    FROM \
        ( \
            SELECT DISTINCT \
                (name.autoid*100+IFNULL(av.application_id,0)) AS id, \
                IF(a.addontype_id = 4, 99, \
                    IF(a.addontype_id = 9, 98, av.application_id)) AS app, \
                a.id AS addon_id, \
                a.addontype_id AS type, \
                a.status, \
                CRC32(LOWER(a.guid)) AS guid_ord, \
                name.locale, \
                CRC32(name.locale) AS locale_ord, \
                a.bayesianrating AS averagerating, \
                a.weeklydownloads, \
                a.totaldownloads, \
                a.inactive, \
                LTRIM(name.localized_string) AS name, \
                description.localized_string AS description, \
                summary.localized_string AS summary, \
                a.current_version, \
                developercomments.localized_string AS developercomments, \
                UNIX_TIMESTAMP(a.created) AS created, \
                UNIX_TIMESTAMP(a.last_updated) AS modified \
            FROM \
                ( \
                    translations name, \
                    addons a \
                    LEFT JOIN applications_versions av \
                        ON av.version_id = a.current_version \
                ) \
            LEFT JOIN translations description \
                ON a.description = description.id \
                AND description.locale = name.locale \
            LEFT JOIN translations summary \
                ON a.summary = summary.id \
                AND summary.locale = name.locale \
            LEFT JOIN translations developercomments \
                ON a.developercomments = developercomments.id \
                AND developercomments.locale = name.locale \
            WHERE \
                a.name = name.id \
                AND name.localized_string IS NOT NULL \
                AND \
                  ( \
                    description.localized_string IS NOT NULL \
                    OR summary.localized_string IS NOT NULL \
                    OR defaultlocale = name.locale \
                  ) \
        ) t

    sql_attr_uint        = addon_id
    sql_attr_uint        = app
    sql_attr_uint        = num_files
    sql_attr_uint        = num_apps
    sql_attr_str2ordinal = name_ord
    sql_attr_uint        = guid_ord
    sql_attr_uint        = type
    sql_attr_uint        = addon_status
    sql_attr_uint        = weeklydownloads
    sql_attr_uint        = totaldownloads
    sql_attr_uint        = locale_ord
    sql_attr_bigint      = max_ver
    sql_attr_bigint      = min_ver
    sql_attr_str2ordinal = locale
    sql_attr_float       = averagerating
    sql_attr_bool        = inactive
    sql_attr_bool        = recommended
    sql_attr_timestamp   = created
    sql_attr_timestamp   = modified

    sql_attr_multi = uint category from query; \
    SELECT \
      (name.autoid*100+IFNULL(av.application_id,0)) id, \
      ac.category_id AS category \
    FROM \
        (translations name, addons a, addons_categories ac) \
    LEFT JOIN applications_versions av ON av.version_id = a.current_version \
    WHERE \
        a.name = name.id AND \
        a.id = ac.addon_id

   sql_attr_multi = uint category_featured from query; \
    SELECT \
      (name.autoid*100+IFNULL(av.application_id,0)) id, \
            ac.category_id AS category \
    FROM \
        (translations name, addons a, addons_categories ac) \
    LEFT JOIN applications_versions av ON av.version_id = a.current_version \
    WHERE \
        ac.feature = 1 AND \
        a.name = name.id AND \
        a.id = ac.addon_id

    sql_attr_multi = uint tag from query; \
    SELECT DISTINCT\
        (name.autoid*100+IFNULL(av.application_id,0)) id, \
        t.tag_id AS tag \
    FROM \
        (translations name, addons a, users_tags_addons t, tags tt) \
    LEFT JOIN applications_versions av ON av.version_id = a.current_version \
    WHERE \
        a.name = name.id AND \
        tt.id = t.tag_id AND \
        tt.blacklisted = 0 AND \
        a.id = t.addon_id

    sql_attr_multi = uint platform from query; \
    SELECT \
        (name.autoid*100+IFNULL(av.application_id,0)) id, \
        IFNULL(platform_id,1) AS platform \
    FROM \
        (translations name, addons a) \
    LEFT JOIN applications_versions av ON av.version_id = a.current_version \
    LEFT JOIN files f ON f.version_id = a.current_version \
    WHERE \
        a.name = name.id


    sql_query_info       = SELECT a.* \
    FROM \
        translations name,\
        addons a\
    LEFT JOIN applications_versions av ON av.version_id = t.current_version \
    WHERE \
        a.name = name.id AND \
        (name.autoid*100+IFNULL(av.application_id,0)) = $id
}

"""

config = config + """
index addons
{
    source                  = addons
    path                    = %s/addons
    # morphology              = stem_en
    # symlink sphinx-wordforms.txt to this location:
    wordforms = %s/wordforms.txt

    # Firebug will get indexed as Firebug, Firebu, Fireb, Fire, Fir - equally
    # We can enable star if necessary.
    min_infix_len = 3
    infix_fields = name
    %s
}
""" % (CATALOG_PATH, ETC_PATH, CHARSET_DATA)

# Configuration for personas.

config = config + """
source personas
{
""" + MYSQL_SOURCE_CONFIG + """
    sql_query = \
        SELECT \
            name.autoid AS id, \
            a.id as addon_id, \
            name.localized_string AS name, \
            description.localized_string AS description, \
            name.locale AS locale, \
            CRC32(name.locale) AS locale_ord, \
            UNIX_TIMESTAMP(a.created) AS created \
            FROM \
                ( \
                    addons a, \
                    translations name \
                ) \
            LEFT JOIN translations description \
                ON a.description = description.id \
                AND description.locale = name.locale \
            WHERE \
                a.name = name.id \
                AND addontype_id = 9 \
                AND ( \
                    name.locale IS NOT NULL \
                    OR name.localized_string IS NOT NULL \
                    OR description.locale IS NOT NULL \
                    OR description.localized_string IS NOT NULL \
                    )\
                AND status IN (4, 8, 9) -- PUBLIC, LITE, LITE_AND_NOMINATED

    sql_attr_uint = addon_id
    sql_attr_uint = locale_ord
    sql_attr_str2ordinal = locale
    sql_attr_timestamp = created
}

"""

config = config + """
index personas
{
    source = personas
    path = %s/personas
    morphology = stem_en
    wordforms = %s/wordforms.txt
    %s
}
""" % (CATALOG_PATH, ETC_PATH, CHARSET_DATA)

# Configuration for collections.

config = config + """
source collections
{
""" + MYSQL_SOURCE_CONFIG + """
    sql_query = \
            SELECT \
                name.autoid AS id, \
                c.id AS collection_id, \
                name.localized_string AS name, \
                description.localized_string AS description, \
                name.locale AS locale, \
                CRC32(name.locale) AS locale_ord, \
                weekly_subscribers, \
                monthly_subscribers, \
                subscribers, \
                rating, \
                UNIX_TIMESTAMP(c.created) AS created \
            FROM \
                ( \
                    collections c, \
                    translations name \
                ) \
            LEFT JOIN translations description \
                ON c.description = description.id \
                AND description.locale = name.locale \
            WHERE \
                c.name = name.id \
                AND ( \
                    name.locale IS NOT NULL \
                    OR name.localized_string IS NOT NULL \
                    OR description.locale IS NOT NULL \
                    OR description.localized_string IS NOT NULL \
                    ) \
                AND c.author_id IS NOT NULL \
                AND c.uuid != '' \
                AND c.listed = 1
    sql_attr_uint = collection_id
    sql_attr_uint = locale_ord
    sql_attr_str2ordinal = locale
    sql_attr_uint = weekly_subscribers
    sql_attr_uint = monthly_subscribers
    sql_attr_uint = subscribers
    sql_attr_float = rating
    sql_attr_timestamp = created
}
"""

config = config + """
index collections
{
    source = collections
    path = %s/collections
    morphology = stem_en
    wordforms = %s/wordforms.txt
    %s
}
""" % (CATALOG_PATH, ETC_PATH, CHARSET_DATA)


config = config + """
searchd
{
    listen    = %d
    listen    = %s:%d:mysql41
    log       = %s/searchd.log
    query_log = %s/query.log
    pid_file  = %s/searchd.pid
}
""" % ((LISTEN_PORT, MYSQL_LISTEN_HOST, MYSQL_LISTEN_PORT) + (LOG_PATH,)*3)

print config