MozDef/examples/queries/ES Sample Queries.ipynb

{
 "metadata": {
  "name": ""
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import pyes\n",
      "from pyes.es import ES\n",
      "import pytz\n",
      "from datetime import datetime\n",
      "from dateutil.parser import parse\n",
      "from datetime import timedelta\n",
      "import json"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 1
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#change the default if you are not in Pacific time\n",
      "#and want to use dates like 'today 8am'\n",
      "def toUTC(suspectedDate,localTimeZone=\"US/Pacific\"):\n",
      "    '''make a UTC date out of almost anything'''\n",
      "    utc=pytz.UTC\n",
      "    objDate=None\n",
      "    if type(suspectedDate)==str:\n",
      "        objDate=parse(suspectedDate,fuzzy=True)\n",
      "    elif type(suspectedDate)==datetime:\n",
      "        objDate=suspectedDate\n",
      "    \n",
      "    if objDate.tzinfo is None:\n",
      "        objDate=pytz.timezone(localTimeZone).localize(objDate)\n",
      "        objDate=utc.normalize(objDate)\n",
      "    else:\n",
      "        objDate=utc.normalize(objDate)\n",
      "    if objDate is not None:\n",
      "        objDate=utc.normalize(objDate)\n",
      "        \n",
      "    return objDate"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 2
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#Set this to one of your ES servers:\n",
      "es=ES((\"http\", \"servername.goes.here\", 9200))\n",
      "\n",
      "#set a date range\n",
      "begindateUTC=toUTC(datetime.now() - timedelta(minutes=15))\n",
      "enddateUTC= toUTC(datetime.now())\n",
      "qDate = pyes.RangeQuery(qrange=pyes.ESRange('utctimestamp', from_value=begindateUTC, to_value=enddateUTC))\n",
      "\n",
      "#set up some criteria (Queries are less usefull than filters)\n",
      "q = pyes.ConstantScoreQuery(pyes.MatchAllQuery())\n",
      "\n",
      "#add as many 'must, must_not, should' criteria filters as you need\n",
      "#to get the data you want\n",
      "q = pyes.FilteredQuery(q,\n",
      "                       pyes.BoolFilter(\n",
      "                                       must=[qDate,\n",
      "                                             pyes.TermFilter('_type', 'mozdefstats')\n",
      "                                            ]\n",
      "                                       must_not=[],\n",
      "                                       should=[]\n",
      "                                       )\n",
      "                       )\n",
      "\n",
      "#in mozdef, events and events-previous\n",
      "#are aliases to the current day and previous day\n",
      "results=es.search(query=q,size=100,indices=['events','events-previous'])\n",
      "\n",
      "#how many docs were found? \n",
      "print(results.count())\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "15\n"
       ]
      }
     ],
     "prompt_number": 3
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#pyes has a iteration bug where \n",
      "#walking the results pops the results from the collection\n",
      "#so easiest way to capture results is _search_raw()\n",
      "#which gives you the raw ES json\n",
      "rawresults=results._search_raw()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 5
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}