From 4810796f72d7cd7d25414f8c520c124ec568af76 Mon Sep 17 00:00:00 2001 From: Brandon Myers Date: Wed, 28 Aug 2019 15:24:06 -0500 Subject: [PATCH 1/2] Move bootstrap setup files into scripts directory --- docker/compose/kibana/Dockerfile | 2 +- docker/compose/mozdef_bootstrap/Dockerfile | 7 +++---- .../setup}/index_mappings/alerts-star.json | 0 .../files => scripts/setup}/index_mappings/alerts.json | 0 .../setup}/index_mappings/events-weekly.json | 0 .../files => scripts/setup}/index_mappings/events.json | 0 .../files => scripts/setup}/initial_setup.py | 0 .../files => scripts/setup}/resources/all_events_area.json | 0 .../setup}/resources/all_events_count.json | 0 .../setup}/resources/category_pie_graph.json | 0 .../setup}/resources/cloudtrail_eventname_pie_graph.json | 0 .../setup}/resources/cloudtrail_eventname_table.json | 0 .../setup}/resources/cloudtrail_events_dashboard.json | 0 .../setup}/resources/cloudtrail_events_line_graph.json | 0 .../setup}/resources/cloudtrail_events_map.json | 0 .../setup}/resources/cloudtrail_total_event_count.json | 0 .../setup}/resources/cloudtrail_user_identity_table.json | 0 .../setup}/resources/destinationip_bar_graph.json | 0 .../setup}/resources/sample_dashboard.json | 0 .../setup}/resources/sourceip_bar_graph.json | 0 20 files changed, 4 insertions(+), 5 deletions(-) rename {docker/compose/mozdef_bootstrap/files => scripts/setup}/index_mappings/alerts-star.json (100%) rename {docker/compose/mozdef_bootstrap/files => scripts/setup}/index_mappings/alerts.json (100%) rename {docker/compose/mozdef_bootstrap/files => scripts/setup}/index_mappings/events-weekly.json (100%) rename {docker/compose/mozdef_bootstrap/files => scripts/setup}/index_mappings/events.json (100%) rename {docker/compose/mozdef_bootstrap/files => scripts/setup}/initial_setup.py (100%) rename {docker/compose/mozdef_bootstrap/files => scripts/setup}/resources/all_events_area.json (100%) rename {docker/compose/mozdef_bootstrap/files => scripts/setup}/resources/all_events_count.json (100%) rename {docker/compose/mozdef_bootstrap/files => scripts/setup}/resources/category_pie_graph.json (100%) rename {docker/compose/mozdef_bootstrap/files => scripts/setup}/resources/cloudtrail_eventname_pie_graph.json (100%) rename {docker/compose/mozdef_bootstrap/files => scripts/setup}/resources/cloudtrail_eventname_table.json (100%) rename {docker/compose/mozdef_bootstrap/files => scripts/setup}/resources/cloudtrail_events_dashboard.json (100%) rename {docker/compose/mozdef_bootstrap/files => scripts/setup}/resources/cloudtrail_events_line_graph.json (100%) rename {docker/compose/mozdef_bootstrap/files => scripts/setup}/resources/cloudtrail_events_map.json (100%) rename {docker/compose/mozdef_bootstrap/files => scripts/setup}/resources/cloudtrail_total_event_count.json (100%) rename {docker/compose/mozdef_bootstrap/files => scripts/setup}/resources/cloudtrail_user_identity_table.json (100%) rename {docker/compose/mozdef_bootstrap/files => scripts/setup}/resources/destinationip_bar_graph.json (100%) rename {docker/compose/mozdef_bootstrap/files => scripts/setup}/resources/sample_dashboard.json (100%) rename {docker/compose/mozdef_bootstrap/files => scripts/setup}/resources/sourceip_bar_graph.json (100%) diff --git a/docker/compose/kibana/Dockerfile b/docker/compose/kibana/Dockerfile index 238c3ab2..5d4f6506 100644 --- a/docker/compose/kibana/Dockerfile +++ b/docker/compose/kibana/Dockerfile @@ -2,7 +2,7 @@ FROM centos:7 LABEL maintainer="mozdef@mozilla.com" -# When changing the kibana version, we'll need to update https://github.com/mozilla/MozDef/blob/master/docker/compose/mozdef_bootstrap/files/initial_setup.py accordingly +# When changing the kibana version, we'll need to update https://github.com/mozilla/MozDef/blob/master/scripts/setup/initial_setup.py accordingly ENV KIBANA_VERSION 6.8.0 RUN \ diff --git a/docker/compose/mozdef_bootstrap/Dockerfile b/docker/compose/mozdef_bootstrap/Dockerfile index 8cbfaa9b..655515bb 100644 --- a/docker/compose/mozdef_bootstrap/Dockerfile +++ b/docker/compose/mozdef_bootstrap/Dockerfile @@ -7,10 +7,9 @@ RUN install --owner mozdef --group mozdef --directory /opt/mozdef/envs/mozdef/do COPY --chown=mozdef:mozdef cron/mozdefStateDefaultMappingTemplate.json /opt/mozdef/envs/mozdef/cron/mozdefStateDefaultMappingTemplate.json COPY --chown=mozdef:mozdef cron/defaultMappingTemplate.json /opt/mozdef/envs/mozdef/cron/defaultMappingTemplate.json COPY --chown=mozdef:mozdef docker/compose/mozdef_cron/files/backup.conf /opt/mozdef/envs/mozdef/cron/backup.conf -COPY --chown=mozdef:mozdef docker/compose/mozdef_bootstrap/files/initial_setup.py /opt/mozdef/envs/mozdef/initial_setup.py -COPY --chown=mozdef:mozdef docker/compose/mozdef_bootstrap/files/index_mappings /opt/mozdef/envs/mozdef/index_mappings -COPY --chown=mozdef:mozdef docker/compose/mozdef_bootstrap/files/resources /opt/mozdef/envs/mozdef/resources -WORKDIR /opt/mozdef/envs/mozdef +COPY --chown=mozdef:mozdef scripts/setup /opt/mozdef/envs/mozdef/scripts/setup + +WORKDIR /opt/mozdef/envs/mozdef/scripts USER mozdef diff --git a/docker/compose/mozdef_bootstrap/files/index_mappings/alerts-star.json b/scripts/setup/index_mappings/alerts-star.json similarity index 100% rename from docker/compose/mozdef_bootstrap/files/index_mappings/alerts-star.json rename to scripts/setup/index_mappings/alerts-star.json diff --git a/docker/compose/mozdef_bootstrap/files/index_mappings/alerts.json b/scripts/setup/index_mappings/alerts.json similarity index 100% rename from docker/compose/mozdef_bootstrap/files/index_mappings/alerts.json rename to scripts/setup/index_mappings/alerts.json diff --git a/docker/compose/mozdef_bootstrap/files/index_mappings/events-weekly.json b/scripts/setup/index_mappings/events-weekly.json similarity index 100% rename from docker/compose/mozdef_bootstrap/files/index_mappings/events-weekly.json rename to scripts/setup/index_mappings/events-weekly.json diff --git a/docker/compose/mozdef_bootstrap/files/index_mappings/events.json b/scripts/setup/index_mappings/events.json similarity index 100% rename from docker/compose/mozdef_bootstrap/files/index_mappings/events.json rename to scripts/setup/index_mappings/events.json diff --git a/docker/compose/mozdef_bootstrap/files/initial_setup.py b/scripts/setup/initial_setup.py similarity index 100% rename from docker/compose/mozdef_bootstrap/files/initial_setup.py rename to scripts/setup/initial_setup.py diff --git a/docker/compose/mozdef_bootstrap/files/resources/all_events_area.json b/scripts/setup/resources/all_events_area.json similarity index 100% rename from docker/compose/mozdef_bootstrap/files/resources/all_events_area.json rename to scripts/setup/resources/all_events_area.json diff --git a/docker/compose/mozdef_bootstrap/files/resources/all_events_count.json b/scripts/setup/resources/all_events_count.json similarity index 100% rename from docker/compose/mozdef_bootstrap/files/resources/all_events_count.json rename to scripts/setup/resources/all_events_count.json diff --git a/docker/compose/mozdef_bootstrap/files/resources/category_pie_graph.json b/scripts/setup/resources/category_pie_graph.json similarity index 100% rename from docker/compose/mozdef_bootstrap/files/resources/category_pie_graph.json rename to scripts/setup/resources/category_pie_graph.json diff --git a/docker/compose/mozdef_bootstrap/files/resources/cloudtrail_eventname_pie_graph.json b/scripts/setup/resources/cloudtrail_eventname_pie_graph.json similarity index 100% rename from docker/compose/mozdef_bootstrap/files/resources/cloudtrail_eventname_pie_graph.json rename to scripts/setup/resources/cloudtrail_eventname_pie_graph.json diff --git a/docker/compose/mozdef_bootstrap/files/resources/cloudtrail_eventname_table.json b/scripts/setup/resources/cloudtrail_eventname_table.json similarity index 100% rename from docker/compose/mozdef_bootstrap/files/resources/cloudtrail_eventname_table.json rename to scripts/setup/resources/cloudtrail_eventname_table.json diff --git a/docker/compose/mozdef_bootstrap/files/resources/cloudtrail_events_dashboard.json b/scripts/setup/resources/cloudtrail_events_dashboard.json similarity index 100% rename from docker/compose/mozdef_bootstrap/files/resources/cloudtrail_events_dashboard.json rename to scripts/setup/resources/cloudtrail_events_dashboard.json diff --git a/docker/compose/mozdef_bootstrap/files/resources/cloudtrail_events_line_graph.json b/scripts/setup/resources/cloudtrail_events_line_graph.json similarity index 100% rename from docker/compose/mozdef_bootstrap/files/resources/cloudtrail_events_line_graph.json rename to scripts/setup/resources/cloudtrail_events_line_graph.json diff --git a/docker/compose/mozdef_bootstrap/files/resources/cloudtrail_events_map.json b/scripts/setup/resources/cloudtrail_events_map.json similarity index 100% rename from docker/compose/mozdef_bootstrap/files/resources/cloudtrail_events_map.json rename to scripts/setup/resources/cloudtrail_events_map.json diff --git a/docker/compose/mozdef_bootstrap/files/resources/cloudtrail_total_event_count.json b/scripts/setup/resources/cloudtrail_total_event_count.json similarity index 100% rename from docker/compose/mozdef_bootstrap/files/resources/cloudtrail_total_event_count.json rename to scripts/setup/resources/cloudtrail_total_event_count.json diff --git a/docker/compose/mozdef_bootstrap/files/resources/cloudtrail_user_identity_table.json b/scripts/setup/resources/cloudtrail_user_identity_table.json similarity index 100% rename from docker/compose/mozdef_bootstrap/files/resources/cloudtrail_user_identity_table.json rename to scripts/setup/resources/cloudtrail_user_identity_table.json diff --git a/docker/compose/mozdef_bootstrap/files/resources/destinationip_bar_graph.json b/scripts/setup/resources/destinationip_bar_graph.json similarity index 100% rename from docker/compose/mozdef_bootstrap/files/resources/destinationip_bar_graph.json rename to scripts/setup/resources/destinationip_bar_graph.json diff --git a/docker/compose/mozdef_bootstrap/files/resources/sample_dashboard.json b/scripts/setup/resources/sample_dashboard.json similarity index 100% rename from docker/compose/mozdef_bootstrap/files/resources/sample_dashboard.json rename to scripts/setup/resources/sample_dashboard.json diff --git a/docker/compose/mozdef_bootstrap/files/resources/sourceip_bar_graph.json b/scripts/setup/resources/sourceip_bar_graph.json similarity index 100% rename from docker/compose/mozdef_bootstrap/files/resources/sourceip_bar_graph.json rename to scripts/setup/resources/sourceip_bar_graph.json From dd6dac9c3426ed85d538b7806003fe9194752c76 Mon Sep 17 00:00:00 2001 From: Brandon Myers Date: Wed, 28 Aug 2019 16:10:58 -0500 Subject: [PATCH 2/2] Fixup initial_setup script to have defaults --- .../compose/docker-compose-cloudy-mozdef.yml | 2 +- docker/compose/docker-compose.yml | 2 +- docker/compose/mozdef_bootstrap/Dockerfile | 2 +- .../all_events_area.json | 0 .../all_events_count.json | 0 .../category_pie_graph.json | 0 .../cloudtrail_eventname_pie_graph.json | 0 .../cloudtrail_eventname_table.json | 0 .../cloudtrail_events_dashboard.json | 0 .../cloudtrail_events_line_graph.json | 0 .../cloudtrail_events_map.json | 0 .../cloudtrail_total_event_count.json | 0 .../cloudtrail_user_identity_table.json | 0 .../destinationip_bar_graph.json | 0 .../sample_dashboard.json | 0 .../sourceip_bar_graph.json | 0 scripts/setup/initial_setup.py | 32 ++++++++++++++++--- 17 files changed, 30 insertions(+), 8 deletions(-) rename scripts/setup/{resources => example_resources}/all_events_area.json (100%) rename scripts/setup/{resources => example_resources}/all_events_count.json (100%) rename scripts/setup/{resources => example_resources}/category_pie_graph.json (100%) rename scripts/setup/{resources => example_resources}/cloudtrail_eventname_pie_graph.json (100%) rename scripts/setup/{resources => example_resources}/cloudtrail_eventname_table.json (100%) rename scripts/setup/{resources => example_resources}/cloudtrail_events_dashboard.json (100%) rename scripts/setup/{resources => example_resources}/cloudtrail_events_line_graph.json (100%) rename scripts/setup/{resources => example_resources}/cloudtrail_events_map.json (100%) rename scripts/setup/{resources => example_resources}/cloudtrail_total_event_count.json (100%) rename scripts/setup/{resources => example_resources}/cloudtrail_user_identity_table.json (100%) rename scripts/setup/{resources => example_resources}/destinationip_bar_graph.json (100%) rename scripts/setup/{resources => example_resources}/sample_dashboard.json (100%) rename scripts/setup/{resources => example_resources}/sourceip_bar_graph.json (100%) diff --git a/docker/compose/docker-compose-cloudy-mozdef.yml b/docker/compose/docker-compose-cloudy-mozdef.yml index 879d7a2f..6b7ca77f 100644 --- a/docker/compose/docker-compose-cloudy-mozdef.yml +++ b/docker/compose/docker-compose-cloudy-mozdef.yml @@ -49,7 +49,7 @@ services: max-size: "10m" env_file: - cloudy_mozdef.env - command: bash -c 'python initial_setup.py http://elasticsearch:9200 cron/defaultMappingTemplate.json cron/mozdefStateDefaultMappingTemplate.json cron/backup.conf http://kibana:5601' + command: bash -c 'python initial_setup.py http://elasticsearch:9200 http://kibana:5601' depends_on: - base networks: diff --git a/docker/compose/docker-compose.yml b/docker/compose/docker-compose.yml index 4e3be08f..a156ed53 100644 --- a/docker/compose/docker-compose.yml +++ b/docker/compose/docker-compose.yml @@ -130,7 +130,7 @@ services: cache_from: - mozdef/mozdef_bootstrap - mozdef_bootstrap:latest - command: bash -c 'while ! timeout 1 bash -c "echo > /dev/tcp/elasticsearch/9200";do sleep 1;done && python initial_setup.py http://elasticsearch:9200 cron/defaultMappingTemplate.json cron/mozdefStateDefaultMappingTemplate.json cron/backup.conf http://kibana:5601' + command: bash -c 'while ! timeout 1 bash -c "echo > /dev/tcp/elasticsearch/9200";do sleep 1;done && python initial_setup.py http://elasticsearch:9200 http://kibana:5601' depends_on: - base - elasticsearch diff --git a/docker/compose/mozdef_bootstrap/Dockerfile b/docker/compose/mozdef_bootstrap/Dockerfile index 655515bb..958872c3 100644 --- a/docker/compose/mozdef_bootstrap/Dockerfile +++ b/docker/compose/mozdef_bootstrap/Dockerfile @@ -10,6 +10,6 @@ COPY --chown=mozdef:mozdef docker/compose/mozdef_cron/files/backup.conf /opt/moz COPY --chown=mozdef:mozdef scripts/setup /opt/mozdef/envs/mozdef/scripts/setup -WORKDIR /opt/mozdef/envs/mozdef/scripts +WORKDIR /opt/mozdef/envs/mozdef/scripts/setup USER mozdef diff --git a/scripts/setup/resources/all_events_area.json b/scripts/setup/example_resources/all_events_area.json similarity index 100% rename from scripts/setup/resources/all_events_area.json rename to scripts/setup/example_resources/all_events_area.json diff --git a/scripts/setup/resources/all_events_count.json b/scripts/setup/example_resources/all_events_count.json similarity index 100% rename from scripts/setup/resources/all_events_count.json rename to scripts/setup/example_resources/all_events_count.json diff --git a/scripts/setup/resources/category_pie_graph.json b/scripts/setup/example_resources/category_pie_graph.json similarity index 100% rename from scripts/setup/resources/category_pie_graph.json rename to scripts/setup/example_resources/category_pie_graph.json diff --git a/scripts/setup/resources/cloudtrail_eventname_pie_graph.json b/scripts/setup/example_resources/cloudtrail_eventname_pie_graph.json similarity index 100% rename from scripts/setup/resources/cloudtrail_eventname_pie_graph.json rename to scripts/setup/example_resources/cloudtrail_eventname_pie_graph.json diff --git a/scripts/setup/resources/cloudtrail_eventname_table.json b/scripts/setup/example_resources/cloudtrail_eventname_table.json similarity index 100% rename from scripts/setup/resources/cloudtrail_eventname_table.json rename to scripts/setup/example_resources/cloudtrail_eventname_table.json diff --git a/scripts/setup/resources/cloudtrail_events_dashboard.json b/scripts/setup/example_resources/cloudtrail_events_dashboard.json similarity index 100% rename from scripts/setup/resources/cloudtrail_events_dashboard.json rename to scripts/setup/example_resources/cloudtrail_events_dashboard.json diff --git a/scripts/setup/resources/cloudtrail_events_line_graph.json b/scripts/setup/example_resources/cloudtrail_events_line_graph.json similarity index 100% rename from scripts/setup/resources/cloudtrail_events_line_graph.json rename to scripts/setup/example_resources/cloudtrail_events_line_graph.json diff --git a/scripts/setup/resources/cloudtrail_events_map.json b/scripts/setup/example_resources/cloudtrail_events_map.json similarity index 100% rename from scripts/setup/resources/cloudtrail_events_map.json rename to scripts/setup/example_resources/cloudtrail_events_map.json diff --git a/scripts/setup/resources/cloudtrail_total_event_count.json b/scripts/setup/example_resources/cloudtrail_total_event_count.json similarity index 100% rename from scripts/setup/resources/cloudtrail_total_event_count.json rename to scripts/setup/example_resources/cloudtrail_total_event_count.json diff --git a/scripts/setup/resources/cloudtrail_user_identity_table.json b/scripts/setup/example_resources/cloudtrail_user_identity_table.json similarity index 100% rename from scripts/setup/resources/cloudtrail_user_identity_table.json rename to scripts/setup/example_resources/cloudtrail_user_identity_table.json diff --git a/scripts/setup/resources/destinationip_bar_graph.json b/scripts/setup/example_resources/destinationip_bar_graph.json similarity index 100% rename from scripts/setup/resources/destinationip_bar_graph.json rename to scripts/setup/example_resources/destinationip_bar_graph.json diff --git a/scripts/setup/resources/sample_dashboard.json b/scripts/setup/example_resources/sample_dashboard.json similarity index 100% rename from scripts/setup/resources/sample_dashboard.json rename to scripts/setup/example_resources/sample_dashboard.json diff --git a/scripts/setup/resources/sourceip_bar_graph.json b/scripts/setup/example_resources/sourceip_bar_graph.json similarity index 100% rename from scripts/setup/resources/sourceip_bar_graph.json rename to scripts/setup/example_resources/sourceip_bar_graph.json diff --git a/scripts/setup/initial_setup.py b/scripts/setup/initial_setup.py index b832e547..65ce436f 100644 --- a/scripts/setup/initial_setup.py +++ b/scripts/setup/initial_setup.py @@ -11,7 +11,6 @@ from datetime import datetime, timedelta from time import sleep from configlib import getConfig import json -import time import os import sys @@ -20,12 +19,35 @@ import requests from mozdef_util.elasticsearch_client import ElasticsearchClient +cron_dir_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../cron') parser = argparse.ArgumentParser(description='Create the correct indexes and aliases in elasticsearch') parser.add_argument('esserver', help='Elasticsearch server (ex: http://elasticsearch:9200)') -parser.add_argument('default_mapping_file', help='The relative path to default mapping json file (ex: cron/defaultMappingTemplate.json)') -parser.add_argument('state_mapping_file', help='The relative path to state mapping json file (ex: cron/mozdefStateDefaultMappingTemplate.json)') -parser.add_argument('backup_conf_file', help='The relative path to backup.conf file (ex: cron/backup.conf)') + +default_file = os.path.realpath(cron_dir_path + '/defaultMappingTemplate.json') +parser.add_argument( + 'default_mapping_file', + help='The relative path to default mapping json file (default: {0})'.format(default_file), + default=default_file, + nargs='?' +) + +default_file = os.path.realpath(cron_dir_path + '/mozdefStateDefaultMappingTemplate.json') +parser.add_argument( + 'state_mapping_file', + help='The relative path to state mapping json file (default: {0})'.format(default_file), + default=default_file, + nargs='?' +) + +default_file = os.path.realpath(cron_dir_path + '/backup.json') +parser.add_argument( + 'backup_conf_file', + help='The relative path to backup.conf file (default: {0})'.format(default_file), + default=default_file, + nargs='?' +) + parser.add_argument('kibana_url', help='The URL of the kibana endpoint (ex: http://kibana:5601)') args = parser.parse_args() @@ -189,7 +211,7 @@ if kibana_index_name in client.get_indices(): sys.exit(0) # Create visualizations/dashboards -dashboards_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'resources') +dashboards_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'example_resources') listing = os.listdir(dashboards_path) for infile in listing: json_file_path = os.path.join(dashboards_path, infile)