Search enhancements v2: better parsing of terms as regex can't do it all, improve highlighting of terms with symbols given sqlite's inexact matching of symbols.

2009-07-25 11:45:14 -06:00 · 2009-07-25 11:45:14 -06:00 · 7aaeb1cf64
--- a/content/collections.js
+++ b/content/collections.js
@ -354,7 +354,7 @@ this._log.info("onClick: START itemIds - " +this.itemIds.toSource());
  },

  onSearch: function(aValue) {
-    let term, terms = [];
+    let term, terms = [], filterTerms = [];
    let searchMsgs = this._searchFilter.getAttribute("messages") == "true";
    let searchCols = this._searchFilter.getAttribute("collections") == "true";
    let quotes = aValue.match("\"", "g");
@ -373,10 +373,6 @@ this._log.info("onClick: START itemIds - " +this.itemIds.toSource());
      return;
    }

-    if (aValue)
-      // Format | char spacing.
-      aValue = aValue.replace(/\s*\|+\s*/g, " | ");
-
    terms = aValue.match("[^\\s\"']+|\"[^\"]*\"*|'[^']*'*", "g");

    while (terms && (term = terms.shift())) {
@ -389,15 +385,16 @@ this._log.info("onClick: START itemIds - " +this.itemIds.toSource());
        }
      }
      else {
-        // Unquoted term: invalid if already have negation term; must start with
-        // valid chars, and cannot contain invalid chars.
-        if (oneNegation || term.match(invalidUnquoted)) {
+        // Unquoted term: invalid if already have negation term; | term cannot
+        // lead a term and must be standalone; must start with valid chars and
+        // cannot contain invalid chars.
+        if (oneNegation || term.match(/\|(?=\S)/)|| term.match(invalidUnquoted)) {
          this._searchFilter.setAttribute("invalid", true);
          return;
        }

        // Negation: can only have one negation term and it must be the last
-        // term (error on rest of search string ending in space indicates this),
+        // term (error on rest of search string ending in space indicates this)
        // and cannot contain invalid chars.
        if (term[0] == "-") {
          oneNegation = true;
@ -407,7 +404,17 @@ this._log.info("onClick: START itemIds - " +this.itemIds.toSource());
            return;
          }
        }
+
+        if (term == "|")
+          // Unquoted | means OR.  We do not use OR because it is 1)twice the
+          // typing, 2)english-centric.
+          term = "OR"
+        else
+          // Add asterisk to non quoted term for sqlite fts non-exact match.
+          term = SnowlUtils.appendAsterisks(term);
      }
+
+      filterTerms.push(term);
    }

    this._searchFilter.removeAttribute("invalid");
@ -420,7 +427,7 @@ this._log.info("onClick: START itemIds - " +this.itemIds.toSource());
      return;

    // Save string.
-    this.Filters["searchterms"] = aValue ? aValue : null;
+    this.Filters["searchterms"] = aValue ? filterTerms.join(" ") : null;

    if (aValue) {
      if (searchCols)
--- a/content/list.js
+++ b/content/list.js
@ -264,15 +264,11 @@ let SnowlMessageView = {
    // FIXME: use a left join here once the SQLite bug breaking left joins to
    // virtual tables has been fixed (i.e. after we upgrade to SQLite 3.5.7+).
    if (this.Filters["searchterms"]) {
-      // Add asterisk to non quoted terms for sqlite fts non-exact match.
-      let searchStr = SnowlUtils.appendAsterisks(this.Filters["searchterms"]);
-      // Replace | with OR for sqlite (but not in quoted string).
-      searchStr = searchStr.replace(/\|\*/g, "OR");
      filters.push({ expression: "messages.id IN " +
                                 "(SELECT messageID FROM parts" +
                                 " JOIN partsText ON parts.id = partsText.docid" +
                                 " WHERE partsText.content MATCH :filter)",
-                     parameters: { filter: searchStr } });
+                     parameters: { filter: this.Filters["searchterms"] } });
    }

    this._collection.filters = filters;
--- a/content/messagecontent.js
+++ b/content/messagecontent.js
@ -125,14 +125,14 @@ var messageContent = {
        subjectLink.target = "messageBody";
      }

-      if (message.author.person)
+      if (message.author && message.author.person)
        document.getElementById("briefAuthor").
                 setAttribute("value", message.author.person.name);
      document.getElementById("briefTimestamp").
               appendChild(document.createTextNode(SnowlDateUtils._formatDate(message.timestamp)));

      // Basic headers
-      if (message.author.person)
+      if (message.author && message.author.person)
        document.getElementById("author").
                 appendChild(document.createTextNode(message.author.person.name));
      document.getElementById("timestamp").
@ -366,7 +366,7 @@ var messageHeaderUtils = {

  // Highlight given phrase, skipping html tags & entities.
  highlight: function(aContent) {
-    var termsArray = [], hlindex = 0;
+    var term, terms = [], highlightTerms = [], hlindex = 0;
    var sidebarWin = gBrowserWindow.document.
                                    getElementById("sidebar").contentWindow;
    var collectionsView = sidebarWin.CollectionsView;
@ -380,23 +380,35 @@ var messageHeaderUtils = {
    if (!searchTerms || !searchMsgs)
      return aContent;

-    // Remove negations (quoted strings and words), OR |, wildcard *.
-    searchTerms = searchTerms.replace(/-[^".]*\s|-"[^".]*"|[\|\*]/g, "");
-    // Make lower case for highlight array.
-    // XXX: unicode? Bug 394604.  Result is that while sqlite may match the
-    // record, unless the user input is exactly what is on the page, it won't show.
-    searchTerms = searchTerms.toLowerCase();
-    // Create | delimited string of strings and words sans quotes for highligher.
-    searchTerms = searchTerms.match("[^\\s\"']+|\"[^\"]*\"|'[^']*'", "g").
-                              join("|").
-                              replace(/"/g, '');
-    // Array to match term for hilight classname index.
-    termsArray = searchTerms.split("|");
+    terms = searchTerms.match("[^\\s\"']+|\"[^\"]*\"|'[^']*'", "g");
+    while (terms && (term = terms.shift())) {
+      // Remove negation term, OR term, quotes ", last wildcard *.
+      term = term.replace(/^-.*|^OR|\"|\*\"$|\*$/g, "");
+      // Replace all non word symbols with . since sqlite does not match symbols
+      // exactly, ie for term of "one-off", "one---off", "one++off" sqlite returns
+      // a match for "one off"; for "one off" sqlite returns "one-off" etc. etc.
+      // and we need to highlight these.
+      term = term.replace(/[^\w\u0080-\uFFFFF]+/g, ".");
+      // Make lower case for highlight array.
+      // XXX: unicode? Bug 394604.  Result is that while sqlite may match the
+      // record, unless the user input is exactly what is on the page, it won't show.
+      term = term.toLowerCase();
+
+      if (term)
+        // Add term to array, term's index creates hilight classname.
+        highlightTerms.push(term);
+    }
+    
+    // Create | delimited string of strings and words for highligher.
+    searchTerms = highlightTerms.join("|");

    var regexp = new RegExp("(<[\\s\\S]*?>|&.*?;)|(" + searchTerms + ")", "gi");
    return aContent.replace(regexp, function($0, $1, $2) {
-      if ($2)
-        hlindex = termsArray.indexOf($2.toLowerCase());
+      if ($2) {
+        var hlterm = $2;
+        hlindex = highlightTerms.indexOf(hlterm.replace(/[^\w\u0080-\uFFFFF]+/g, ".").
+                                                toLowerCase());
+      }
      return $1 || '<span class="hldefault hl' + hlindex +'">' + $2 + "</span>";
    });
  }
--- a/modules/utils.js
+++ b/modules/utils.js
@ -520,8 +520,8 @@ let SnowlUtils = {
   */
  appendAsterisks: function(string) {
    // Support unicode word endings (which need to be delimted with a \s, as
-    // a \b only applies to a \w).  Include | for highlight processing.
-    let wordEnds = /\w\b(?!\*)|[\|\u0080-\uFFFF](?=\s+)(?!\*)/g;
+    // a \b only applies to a \w).
+    let wordEnds = /\w\b(?!\*)|[\u0080-\uFFFF](?!\*)(?=\s+)/g;
    let asterisk = "$&*";

    // This version appends asterisks to every word.