Merge pull request #1223 from mozilla/create_statefile_index

Create mozdefstate index with specific mappings in docker environment

docker/compose/docker-compose-cloudy-mozdef.yml

@@ -40,7 +40,7 @@ services:
     image: mozdef/mozdef_bootstrap:latest
     env_file:
       - cloudy_mozdef.env
-    command: bash -c 'python initial_setup.py http://elasticsearch:9200 cron/defaultMappingTemplate.json cron/backup.conf http://kibana:5601'
+    command: bash -c 'python initial_setup.py http://elasticsearch:9200 cron/defaultMappingTemplate.json cron/mozdefStateDefaultMappingTemplate.json cron/backup.conf http://kibana:5601'
     depends_on:
       - base
     networks:

docker/compose/docker-compose.yml

@@ -103,7 +103,7 @@ services:
       cache_from:
         - mozdef/mozdef_bootstrap
         - mozdef_bootstrap:latest
-    command: bash -c 'while ! timeout 1 bash -c "echo > /dev/tcp/elasticsearch/9200";do sleep 1;done && python initial_setup.py http://elasticsearch:9200 cron/defaultMappingTemplate.json cron/backup.conf http://kibana:5601'
+    command: bash -c 'while ! timeout 1 bash -c "echo > /dev/tcp/elasticsearch/9200";do sleep 1;done && python initial_setup.py http://elasticsearch:9200 cron/defaultMappingTemplate.json cron/mozdefStateDefaultMappingTemplate.json cron/backup.conf http://kibana:5601'
     depends_on:
       - base
       - elasticsearch

docker/compose/mozdef_bootstrap/Dockerfile

@@ -4,6 +4,7 @@ LABEL maintainer="mozdef@mozilla.com"
 
 RUN mkdir -p /opt/mozdef/envs/mozdef/docker/conf
 
+COPY cron/mozdefStateDefaultMappingTemplate.json /opt/mozdef/envs/mozdef/cron/mozdefStateDefaultMappingTemplate.json
 COPY cron/defaultMappingTemplate.json /opt/mozdef/envs/mozdef/cron/defaultMappingTemplate.json
 COPY docker/compose/mozdef_cron/files/backup.conf /opt/mozdef/envs/mozdef/cron/backup.conf
 COPY docker/compose/mozdef_bootstrap/files/initial_setup.py /opt/mozdef/envs/mozdef/initial_setup.py

docker/compose/mozdef_bootstrap/files/initial_setup.py

@@ -25,6 +25,7 @@ from mozdef_util.query_models import SearchQuery, TermMatch
 parser = argparse.ArgumentParser(description='Create the correct indexes and aliases in elasticsearch')
 parser.add_argument('esserver', help='Elasticsearch server (ex: http://elasticsearch:9200)')
 parser.add_argument('default_mapping_file', help='The relative path to default mapping json file (ex: cron/defaultMappingTemplate.json)')
+parser.add_argument('state_mapping_file', help='The relative path to state mapping json file (ex: cron/mozdefStateDefaultMappingTemplate.json)')
 parser.add_argument('backup_conf_file', help='The relative path to backup.conf file (ex: cron/backup.conf)')
 parser.add_argument('kibana_url', help='The URL of the kibana endpoint (ex: http://kibana:5601)')
 args = parser.parse_args()
@@ -45,6 +46,7 @@ previous_event_index_name = (current_date - timedelta(days=1)).strftime("events-
 weekly_index_alias = 'events-weekly'
 alert_index_name = current_date.strftime("alerts-%Y%m")
 kibana_index_name = '.kibana'
+state_index_name = 'mozdefstate'
 
 index_settings_str = ''
 with open(args.default_mapping_file) as data_file:
@@ -52,6 +54,13 @@ with open(args.default_mapping_file) as data_file:
 
 index_settings = json.loads(index_settings_str)
 
+state_index_settings_str = ''
+with open(args.state_mapping_file) as data_file:
+    state_index_settings_str = data_file.read()
+
+state_index_settings = json.loads(state_index_settings_str)
+
+
 all_indices = []
 total_num_tries = 15
 for attempt in range(total_num_tries):
@@ -84,6 +93,7 @@ index_options = {
     }
 }
 index_settings['settings'] = index_options
+state_index_settings['settings'] = index_options
 
 # Create initial indices
 if event_index_name not in all_indices:
@@ -109,6 +119,10 @@ if kibana_index_name not in all_indices:
     print "Creating " + kibana_index_name
     client.create_index(kibana_index_name, index_config={"settings": index_options})
 
+if state_index_name not in all_indices:
+    print "Creating " + state_index_name
+    client.create_index(state_index_name, index_config=state_index_settings)
+
 # Wait for .kibana index to be ready
 num_times = 0
 while not client.index_exists(kibana_index_name):