Merge pull request #2508 from socprime/gcp_monitor_connector
GCP Monitoring Data Connector
This commit is contained in:
v-jayakal
GitHub
Коммит
c9b22fde50
|
|
@ -0,0 +1,105 @@
|
|||
{
|
||||
"Name": "GCP_MONITORING_CL",
|
||||
"Properties": [
|
||||
{
|
||||
"Name": "TenantId",
|
||||
"Type": "String"
|
||||
},
|
||||
{
|
||||
"Name": "SourceSystem",
|
||||
"Type": "String"
|
||||
},
|
||||
{
|
||||
"Name": "MG",
|
||||
"Type": "String"
|
||||
},
|
||||
{
|
||||
"Name": "ManagementGroupName",
|
||||
"Type": "String"
|
||||
},
|
||||
{
|
||||
"Name": "TimeGenerated",
|
||||
"Type": "String"
|
||||
},
|
||||
{
|
||||
"Name": "Computer",
|
||||
"Type": "String"
|
||||
},
|
||||
{
|
||||
"Name": "RawData",
|
||||
"Type": "String"
|
||||
},
|
||||
{
|
||||
"Name": "metric_labels_loadbalanced_s",
|
||||
"Type": "String"
|
||||
},
|
||||
{
|
||||
"Name": "value_doubleValue_d",
|
||||
"Type": "Double"
|
||||
},
|
||||
{
|
||||
"Name": "metric_labels_device_type_s",
|
||||
"Type": "String"
|
||||
},
|
||||
{
|
||||
"Name": "metric_labels_instance_name_s",
|
||||
"Type": "String"
|
||||
},
|
||||
{
|
||||
"Name": "metric_labels_device_name_s",
|
||||
"Type": "String"
|
||||
},
|
||||
{
|
||||
"Name": "metric_labels_storage_type_s",
|
||||
"Type": "String"
|
||||
},
|
||||
{
|
||||
"Name": "metric_type_s",
|
||||
"Type": "String"
|
||||
},
|
||||
{
|
||||
"Name": "resource_type_s",
|
||||
"Type": "String"
|
||||
},
|
||||
{
|
||||
"Name": "resource_labels_project_id_s",
|
||||
"Type": "String"
|
||||
},
|
||||
{
|
||||
"Name": "resource_labels_instance_id_s",
|
||||
"Type": "String"
|
||||
},
|
||||
{
|
||||
"Name": "resource_labels_zone_s",
|
||||
"Type": "String"
|
||||
},
|
||||
{
|
||||
"Name": "metricKind_s",
|
||||
"Type": "String"
|
||||
},
|
||||
{
|
||||
"Name": "valueType_s",
|
||||
"Type": "String"
|
||||
},
|
||||
{
|
||||
"Name": "interval_startTime_t",
|
||||
"Type": "DateTime"
|
||||
},
|
||||
{
|
||||
"Name": "interval_endTime_t",
|
||||
"Type": "DateTime"
|
||||
},
|
||||
{
|
||||
"Name": "value_int64Value_d",
|
||||
"Type": "Double"
|
||||
},
|
||||
{
|
||||
"Name": "Type",
|
||||
"Type": "String"
|
||||
},
|
||||
{
|
||||
"Name": "_ResourceId",
|
||||
"Type": "String"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,262 @@
|
|||
[
|
||||
{
|
||||
"TenantId": "00000000-0000-0000-0000-000000000000",
|
||||
"SourceSystem": "RestAPI",
|
||||
"MG": "",
|
||||
"ManagementGroupName": "",
|
||||
"Computer": "",
|
||||
"RawData": "",
|
||||
"metric_labels_loadbalanced_s": "",
|
||||
"value_doubleValue_d": "0.00152874941890104",
|
||||
"metric_labels_device_type_s": "",
|
||||
"metric_labels_instance_name_s": "centosps",
|
||||
"metric_labels_device_name_s": "",
|
||||
"metric_labels_storage_type_s": "",
|
||||
"metric_type_s": "compute.googleapis.com/instance/cpu/utilization",
|
||||
"resource_type_s": "gce_instance",
|
||||
"resource_labels_project_id_s": "sometestproject-111111",
|
||||
"resource_labels_instance_id_s": "1111111111111111111",
|
||||
"resource_labels_zone_s": "europe-west3-c",
|
||||
"metricKind_s": "GAUGE",
|
||||
"valueType_s": "DOUBLE",
|
||||
"interval_startTime_t": "2021-06-16T14:27:00.00000Z",
|
||||
"interval_endTime_t": "2021-06-16T14:27:00.00000Z",
|
||||
"value_int64Value_d": null,
|
||||
"Type": "GCP_MONITORING_CL",
|
||||
"_ResourceId": ""
|
||||
},
|
||||
{
|
||||
"TenantId": "00000000-0000-0000-0000-000000000000",
|
||||
"SourceSystem": "RestAPI",
|
||||
"MG": "",
|
||||
"ManagementGroupName": "",
|
||||
"Computer": "",
|
||||
"RawData": "",
|
||||
"metric_labels_loadbalanced_s": "",
|
||||
"value_doubleValue_d": "0.0032595385481045",
|
||||
"metric_labels_device_type_s": "",
|
||||
"metric_labels_instance_name_s": "centosps",
|
||||
"metric_labels_device_name_s": "",
|
||||
"metric_labels_storage_type_s": "",
|
||||
"metric_type_s": "compute.googleapis.com/instance/cpu/utilization",
|
||||
"resource_type_s": "gce_instance",
|
||||
"resource_labels_project_id_s": "sometestproject-111111",
|
||||
"resource_labels_instance_id_s": "1111111111111111111",
|
||||
"resource_labels_zone_s": "europe-west3-c",
|
||||
"metricKind_s": "GAUGE",
|
||||
"valueType_s": "DOUBLE",
|
||||
"interval_startTime_t": "2021-06-16T14:26:00.00000Z",
|
||||
"interval_endTime_t": "2021-06-16T14:26:00.00000Z",
|
||||
"value_int64Value_d": null,
|
||||
"Type": "GCP_MONITORING_CL",
|
||||
"_ResourceId": ""
|
||||
},
|
||||
{
|
||||
"TenantId": "00000000-0000-0000-0000-000000000000",
|
||||
"SourceSystem": "RestAPI",
|
||||
"MG": "",
|
||||
"ManagementGroupName": "",
|
||||
"Computer": "",
|
||||
"RawData": "",
|
||||
"metric_labels_loadbalanced_s": "",
|
||||
"value_doubleValue_d": "0.017032381528558",
|
||||
"metric_labels_device_type_s": "",
|
||||
"metric_labels_instance_name_s": "centosps",
|
||||
"metric_labels_device_name_s": "",
|
||||
"metric_labels_storage_type_s": "",
|
||||
"metric_type_s": "compute.googleapis.com/instance/cpu/utilization",
|
||||
"resource_type_s": "gce_instance",
|
||||
"resource_labels_project_id_s": "sometestproject-111111",
|
||||
"resource_labels_instance_id_s": "1111111111111111111",
|
||||
"resource_labels_zone_s": "europe-west3-c",
|
||||
"metricKind_s": "GAUGE",
|
||||
"valueType_s": "DOUBLE",
|
||||
"interval_startTime_t": "2021-06-16T14:25:00.00000Z",
|
||||
"interval_endTime_t": "2021-06-16T14:25:00.00000Z",
|
||||
"value_int64Value_d": null,
|
||||
"Type": "GCP_MONITORING_CL",
|
||||
"_ResourceId": ""
|
||||
},
|
||||
{
|
||||
"TenantId": "00000000-0000-0000-0000-000000000000",
|
||||
"SourceSystem": "RestAPI",
|
||||
"MG": "",
|
||||
"ManagementGroupName": "",
|
||||
"Computer": "",
|
||||
"RawData": "",
|
||||
"metric_labels_loadbalanced_s": "",
|
||||
"value_doubleValue_d": "",
|
||||
"metric_labels_device_type_s": "permanent",
|
||||
"metric_labels_instance_name_s": "centosps",
|
||||
"metric_labels_device_name_s": "centosps",
|
||||
"metric_labels_storage_type_s": "pd-balanced",
|
||||
"metric_type_s": "compute.googleapis.com/instance/disk/write_bytes_count",
|
||||
"resource_type_s": "gce_instance",
|
||||
"resource_labels_project_id_s": "sometestproject-111111",
|
||||
"resource_labels_instance_id_s": "1111111111111111111",
|
||||
"resource_labels_zone_s": "europe-west3-c",
|
||||
"metricKind_s": "DELTA",
|
||||
"valueType_s": "INT64",
|
||||
"interval_startTime_t": "2021-06-16T14:26:00.00000Z",
|
||||
"interval_endTime_t": "2021-06-16T14:27:00.00000Z",
|
||||
"value_int64Value_d": 47934,
|
||||
"Type": "GCP_MONITORING_CL",
|
||||
"_ResourceId": ""
|
||||
},
|
||||
{
|
||||
"TenantId": "00000000-0000-0000-0000-000000000000",
|
||||
"SourceSystem": "RestAPI",
|
||||
"MG": "",
|
||||
"ManagementGroupName": "",
|
||||
"Computer": "",
|
||||
"RawData": "",
|
||||
"metric_labels_loadbalanced_s": "",
|
||||
"value_doubleValue_d": "",
|
||||
"metric_labels_device_type_s": "permanent",
|
||||
"metric_labels_instance_name_s": "centosps",
|
||||
"metric_labels_device_name_s": "centosps",
|
||||
"metric_labels_storage_type_s": "pd-balanced",
|
||||
"metric_type_s": "compute.googleapis.com/instance/disk/write_bytes_count",
|
||||
"resource_type_s": "gce_instance",
|
||||
"resource_labels_project_id_s": "sometestproject-111111",
|
||||
"resource_labels_instance_id_s": "1111111111111111111",
|
||||
"resource_labels_zone_s": "europe-west3-c",
|
||||
"metricKind_s": "DELTA",
|
||||
"valueType_s": "INT64",
|
||||
"interval_startTime_t": "2021-06-16T14:25:00.00000Z",
|
||||
"interval_endTime_t": "2021-06-16T14:26:00.00000Z",
|
||||
"value_int64Value_d": 159286,
|
||||
"Type": "GCP_MONITORING_CL",
|
||||
"_ResourceId": ""
|
||||
},
|
||||
{
|
||||
"TenantId": "00000000-0000-0000-0000-000000000000",
|
||||
"SourceSystem": "RestAPI",
|
||||
"MG": "",
|
||||
"ManagementGroupName": "",
|
||||
"Computer": "",
|
||||
"RawData": "",
|
||||
"metric_labels_loadbalanced_s": "",
|
||||
"value_doubleValue_d": "",
|
||||
"metric_labels_device_type_s": "permanent",
|
||||
"metric_labels_instance_name_s": "centosps",
|
||||
"metric_labels_device_name_s": "centosps",
|
||||
"metric_labels_storage_type_s": "pd-balanced",
|
||||
"metric_type_s": "compute.googleapis.com/instance/disk/write_bytes_count",
|
||||
"resource_type_s": "gce_instance",
|
||||
"resource_labels_project_id_s": "sometestproject-111111",
|
||||
"resource_labels_instance_id_s": "1111111111111111111",
|
||||
"resource_labels_zone_s": "europe-west3-c",
|
||||
"metricKind_s": "DELTA",
|
||||
"valueType_s": "INT64",
|
||||
"interval_startTime_t": "2021-06-16T14:24:00.00000Z",
|
||||
"interval_endTime_t": "2021-06-16T14:25:00.00000Z",
|
||||
"value_int64Value_d": 208141,
|
||||
"Type": "GCP_MONITORING_CL",
|
||||
"_ResourceId": ""
|
||||
},
|
||||
{
|
||||
"TenantId": "00000000-0000-0000-0000-000000000000",
|
||||
"SourceSystem": "RestAPI",
|
||||
"MG": "",
|
||||
"ManagementGroupName": "",
|
||||
"Computer": "",
|
||||
"RawData": "",
|
||||
"metric_labels_loadbalanced_s": "",
|
||||
"value_doubleValue_d": "",
|
||||
"metric_labels_device_type_s": "permanent",
|
||||
"metric_labels_instance_name_s": "centosps",
|
||||
"metric_labels_device_name_s": "disk-1",
|
||||
"metric_labels_storage_type_s": "pd-balanced",
|
||||
"metric_type_s": "compute.googleapis.com/instance/disk/write_bytes_count",
|
||||
"resource_type_s": "gce_instance",
|
||||
"resource_labels_project_id_s": "sometestproject-111111",
|
||||
"resource_labels_instance_id_s": "1111111111111111111",
|
||||
"resource_labels_zone_s": "europe-west3-c",
|
||||
"metricKind_s": "DELTA",
|
||||
"valueType_s": "INT64",
|
||||
"interval_startTime_t": "2021-06-16T14:26:00.00000Z",
|
||||
"interval_endTime_t": "2021-06-16T14:27:00.00000Z",
|
||||
"value_int64Value_d": 0,
|
||||
"Type": "GCP_MONITORING_CL",
|
||||
"_ResourceId": ""
|
||||
},
|
||||
{
|
||||
"TenantId": "00000000-0000-0000-0000-000000000000",
|
||||
"SourceSystem": "RestAPI",
|
||||
"MG": "",
|
||||
"ManagementGroupName": "",
|
||||
"Computer": "",
|
||||
"RawData": "",
|
||||
"metric_labels_loadbalanced_s": "",
|
||||
"value_doubleValue_d": "",
|
||||
"metric_labels_device_type_s": "permanent",
|
||||
"metric_labels_instance_name_s": "centosps",
|
||||
"metric_labels_device_name_s": "disk-1",
|
||||
"metric_labels_storage_type_s": "pd-balanced",
|
||||
"metric_type_s": "compute.googleapis.com/instance/disk/write_bytes_count",
|
||||
"resource_type_s": "gce_instance",
|
||||
"resource_labels_project_id_s": "sometestproject-111111",
|
||||
"resource_labels_instance_id_s": "1111111111111111111",
|
||||
"resource_labels_zone_s": "europe-west3-c",
|
||||
"metricKind_s": "DELTA",
|
||||
"valueType_s": "INT64",
|
||||
"interval_startTime_t": "2021-06-16T14:25:00.00000Z",
|
||||
"interval_endTime_t": "2021-06-16T14:26:00.00000Z",
|
||||
"value_int64Value_d": 0,
|
||||
"Type": "GCP_MONITORING_CL",
|
||||
"_ResourceId": ""
|
||||
},
|
||||
{
|
||||
"TenantId": "00000000-0000-0000-0000-000000000000",
|
||||
"SourceSystem": "RestAPI",
|
||||
"MG": "",
|
||||
"ManagementGroupName": "",
|
||||
"Computer": "",
|
||||
"RawData": "",
|
||||
"metric_labels_loadbalanced_s": "",
|
||||
"value_doubleValue_d": "",
|
||||
"metric_labels_device_type_s": "permanent",
|
||||
"metric_labels_instance_name_s": "centosps",
|
||||
"metric_labels_device_name_s": "disk-1",
|
||||
"metric_labels_storage_type_s": "pd-balanced",
|
||||
"metric_type_s": "compute.googleapis.com/instance/disk/write_bytes_count",
|
||||
"resource_type_s": "gce_instance",
|
||||
"resource_labels_project_id_s": "sometestproject-111111",
|
||||
"resource_labels_instance_id_s": "1111111111111111111",
|
||||
"resource_labels_zone_s": "europe-west3-c",
|
||||
"metricKind_s": "DELTA",
|
||||
"valueType_s": "INT64",
|
||||
"interval_startTime_t": "2021-06-16T14:24:00.00000Z",
|
||||
"interval_endTime_t": "2021-06-16T14:25:00.00000Z",
|
||||
"value_int64Value_d": 0,
|
||||
"Type": "GCP_MONITORING_CL",
|
||||
"_ResourceId": ""
|
||||
},
|
||||
{
|
||||
"TenantId": "00000000-0000-0000-0000-000000000000",
|
||||
"SourceSystem": "RestAPI",
|
||||
"MG": "",
|
||||
"ManagementGroupName": "",
|
||||
"Computer": "",
|
||||
"RawData": "",
|
||||
"metric_labels_loadbalanced_s": "false",
|
||||
"value_doubleValue_d": "",
|
||||
"metric_labels_device_type_s": "",
|
||||
"metric_labels_instance_name_s": "centosps",
|
||||
"metric_labels_device_name_s": "",
|
||||
"metric_labels_storage_type_s": "",
|
||||
"metric_type_s": "compute.googleapis.com/instance/network/received_bytes_count",
|
||||
"resource_type_s": "gce_instance",
|
||||
"resource_labels_project_id_s": "sometestproject-111111",
|
||||
"resource_labels_instance_id_s": "1111111111111111111",
|
||||
"resource_labels_zone_s": "europe-west3-c",
|
||||
"metricKind_s": "DELTA",
|
||||
"valueType_s": "INT64",
|
||||
"interval_startTime_t": "2021-06-16T14:26:00.00000Z",
|
||||
"interval_endTime_t": "2021-06-16T14:27:00.00000Z",
|
||||
"value_int64Value_d": 14498,
|
||||
"Type": "GCP_MONITORING_CL",
|
||||
"_ResourceId": ""
|
||||
}
|
||||
]
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
{
|
||||
"scriptFile": "main.py",
|
||||
"bindings": [
|
||||
{
|
||||
"name": "mytimer",
|
||||
"type": "timerTrigger",
|
||||
"direction": "in",
|
||||
"schedule": "0 */5 * * * *",
|
||||
"runOnStartup": false
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,170 @@
|
|||
import os
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
import re
|
||||
import azure.functions as func
|
||||
from google.cloud.monitoring_v3.services.metric_service import MetricServiceAsyncClient
|
||||
from google.cloud.monitoring_v3.types import TimeInterval
|
||||
from google.cloud.monitoring_v3.types import TimeSeries
|
||||
from google.protobuf.json_format import MessageToDict
|
||||
|
||||
from .sentinel_connector_async import AzureSentinelConnectorAsync
|
||||
from .state_manager import StateManager
|
||||
|
||||
|
||||
logging.getLogger('azure.core.pipeline.policies.http_logging_policy').setLevel(logging.ERROR)
|
||||
|
||||
|
||||
CREDENTIALS_FILE_CONTENT = os.environ['GCP_CREDENTIALS_FILE_CONTENT']
|
||||
GCP_PROJECT_ID_LIST = os.environ['GCP_PROJECT_ID']
|
||||
METRICS = os.environ['GCP_METRICS']
|
||||
WORKSPACE_ID = os.environ['WORKSPACE_ID']
|
||||
SHARED_KEY = os.environ['SHARED_KEY']
|
||||
LOG_TYPE = 'GCP_MONITORING'
|
||||
CREDS_FILE_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), '.creds')
|
||||
|
||||
MAX_SEARCH_PERIOD_MINUTES = 120
|
||||
DEFAULT_SEARCH_PERIOD_MINUTES = 5
|
||||
|
||||
|
||||
LOG_ANALYTICS_URI = os.environ.get('logAnalyticsUri')
|
||||
|
||||
if not LOG_ANALYTICS_URI or str(LOG_ANALYTICS_URI).isspace():
|
||||
LOG_ANALYTICS_URI = 'https://' + WORKSPACE_ID + '.ods.opinsights.azure.com'
|
||||
|
||||
pattern = r'https:\/\/([\w\-]+)\.ods\.opinsights\.azure.([a-zA-Z\.]+)$'
|
||||
match = re.match(pattern, str(LOG_ANALYTICS_URI))
|
||||
if not match:
|
||||
raise Exception("Invalid Log Analytics Uri.")
|
||||
|
||||
|
||||
async def main(mytimer: func.TimerRequest):
|
||||
logging.info('Starting script')
|
||||
|
||||
create_credentials_file()
|
||||
|
||||
projects = GCP_PROJECT_ID_LIST.split()
|
||||
metrics = METRICS.split()
|
||||
|
||||
state_manager = StateManager(os.environ['AzureWebJobsStorage'])
|
||||
|
||||
logging.info('Getting last timestamp')
|
||||
last_ts = state_manager.get()
|
||||
logging.info('Last timestamp: {}'.format(last_ts))
|
||||
|
||||
time_interval, last_ts = gen_time_interval(last_ts)
|
||||
|
||||
cors = []
|
||||
client = MetricServiceAsyncClient()
|
||||
for project in projects:
|
||||
for metric in metrics:
|
||||
cors.append(process_metric(client, project, metric, time_interval))
|
||||
|
||||
await asyncio.gather(*cors)
|
||||
|
||||
logging.info('Saving last timestamp: {}'.format(last_ts))
|
||||
state_manager.post(str(last_ts))
|
||||
remove_credentials_file()
|
||||
|
||||
logging.info('Script finished')
|
||||
|
||||
|
||||
def gen_time_interval(last_ts: int):
|
||||
now = int(time.time())
|
||||
|
||||
if last_ts:
|
||||
try:
|
||||
last_ts = int(last_ts)
|
||||
except Exception:
|
||||
last_ts = None
|
||||
|
||||
if last_ts:
|
||||
last_ts = int(last_ts)
|
||||
if now - last_ts < MAX_SEARCH_PERIOD_MINUTES * 60:
|
||||
start_time = last_ts
|
||||
logging.info('Getting data from {}'.format(start_time))
|
||||
else:
|
||||
start_time = now - MAX_SEARCH_PERIOD_MINUTES * 60
|
||||
logging.warning('Last timestamp is too old. Getting data from {}'.format(start_time))
|
||||
else:
|
||||
start_time = now - DEFAULT_SEARCH_PERIOD_MINUTES * 60
|
||||
logging.info('Last timestamp is not known. Getting data from {}'.format(start_time))
|
||||
|
||||
end_time = now - 30
|
||||
|
||||
interval = {
|
||||
"start_time": {"seconds": start_time, "nanos": 0},
|
||||
"end_time": {"seconds": end_time, "nanos": 0}
|
||||
}
|
||||
logging.info('Getting data for interval: {}'.format(interval))
|
||||
return TimeInterval(interval), end_time
|
||||
|
||||
|
||||
async def process_metric(client: MetricServiceAsyncClient, project_name: str, metric_type: str, time_interval: TimeInterval):
|
||||
logging.info('Start processing metric: {} in {}'.format(metric_type, project_name))
|
||||
|
||||
sentinel = AzureSentinelConnectorAsync(LOG_ANALYTICS_URI, WORKSPACE_ID, SHARED_KEY, LOG_TYPE, queue_size=10000)
|
||||
|
||||
metric_string = 'metric.type = ' + '"' + metric_type + '"'
|
||||
|
||||
time_series_async_iterator = await client.list_time_series(name=project_name, filter=metric_string, interval=time_interval)
|
||||
async for time_series in time_series_async_iterator:
|
||||
async for event in parse_time_series(time_series):
|
||||
await sentinel.send(event)
|
||||
|
||||
await sentinel.flush()
|
||||
|
||||
logging.info('Finish processing metric: {} in {}. Sent events: {}'.format(metric_type, project_name, sentinel.successfull_sent_events_number))
|
||||
|
||||
|
||||
async def parse_time_series(time_series: TimeSeries):
|
||||
d = MessageToDict(time_series._pb)
|
||||
points = d.pop('points', [])
|
||||
|
||||
for p in points:
|
||||
p = parse_point(p)
|
||||
event = {}
|
||||
event.update(d)
|
||||
event.update(p)
|
||||
yield event
|
||||
|
||||
|
||||
def parse_bool(val):
|
||||
if val == 'true':
|
||||
val = True
|
||||
elif val == 'false':
|
||||
val = False
|
||||
return val
|
||||
|
||||
|
||||
def parse_point(point: dict):
|
||||
parse_functions = {
|
||||
'boolValue': parse_bool,
|
||||
'int64Value': int,
|
||||
'doubleValue': float
|
||||
}
|
||||
|
||||
if 'value' in point:
|
||||
for field in point['value']:
|
||||
if field in parse_functions:
|
||||
try:
|
||||
val = point['value'][field]
|
||||
func = parse_functions[field]
|
||||
point['value'][field] = func(val)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return point
|
||||
|
||||
|
||||
def create_credentials_file():
|
||||
with open(CREDS_FILE_PATH, 'w') as f:
|
||||
content = CREDENTIALS_FILE_CONTENT.strip().replace('\n', '\\n')
|
||||
f.write(content)
|
||||
os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = CREDS_FILE_PATH
|
||||
|
||||
|
||||
def remove_credentials_file():
|
||||
if os.path.exists(CREDS_FILE_PATH):
|
||||
os.remove(CREDS_FILE_PATH)
|
||||
|
|
@ -0,0 +1,91 @@
|
|||
import datetime
|
||||
import logging
|
||||
import json
|
||||
import hashlib
|
||||
import hmac
|
||||
import base64
|
||||
import aiohttp
|
||||
import asyncio
|
||||
from collections import deque
|
||||
|
||||
|
||||
class AzureSentinelConnectorAsync:
|
||||
def __init__(self, log_analytics_uri, workspace_id, shared_key, log_type, queue_size=1000, queue_size_bytes=25 * (2**20)):
|
||||
self.log_analytics_uri = log_analytics_uri
|
||||
self.workspace_id = workspace_id
|
||||
self.shared_key = shared_key
|
||||
self.log_type = log_type
|
||||
self.queue_size = queue_size
|
||||
self.queue_size_bytes = queue_size_bytes
|
||||
self._queue = deque()
|
||||
self.successfull_sent_events_number = 0
|
||||
self.failed_sent_events_number = 0
|
||||
self.lock = asyncio.Lock()
|
||||
|
||||
async def send(self, event):
|
||||
events = None
|
||||
async with self.lock:
|
||||
self._queue.append(event)
|
||||
if len(self._queue) >= self.queue_size:
|
||||
events = list(self._queue)
|
||||
self._queue.clear()
|
||||
if events:
|
||||
await self._flush(events)
|
||||
|
||||
async def flush(self):
|
||||
await self._flush(list(self._queue))
|
||||
|
||||
async def _flush(self, data: list):
|
||||
if data:
|
||||
data = self._split_big_request(data)
|
||||
async with aiohttp.ClientSession() as session:
|
||||
await asyncio.gather(*[self._post_data(session, self.workspace_id, self.shared_key, d, self.log_type) for d in data])
|
||||
|
||||
def _build_signature(self, workspace_id, shared_key, date, content_length, method, content_type, resource):
|
||||
x_headers = 'x-ms-date:' + date
|
||||
string_to_hash = method + "\n" + str(content_length) + "\n" + content_type + "\n" + x_headers + "\n" + resource
|
||||
bytes_to_hash = bytes(string_to_hash, encoding="utf-8")
|
||||
decoded_key = base64.b64decode(shared_key)
|
||||
encoded_hash = base64.b64encode(hmac.new(decoded_key, bytes_to_hash, digestmod=hashlib.sha256).digest()).decode()
|
||||
authorization = "SharedKey {}:{}".format(workspace_id, encoded_hash)
|
||||
return authorization
|
||||
|
||||
async def _post_data(self, session: aiohttp.ClientSession, workspace_id, shared_key, body, log_type):
|
||||
logging.debug('Start sending data to sentinel')
|
||||
events_number = len(body)
|
||||
body = json.dumps(body)
|
||||
method = 'POST'
|
||||
content_type = 'application/json'
|
||||
resource = '/api/logs'
|
||||
rfc1123date = datetime.datetime.utcnow().strftime('%a, %d %b %Y %H:%M:%S GMT')
|
||||
content_length = len(body)
|
||||
signature = self._build_signature(workspace_id, shared_key, rfc1123date, content_length, method, content_type, resource)
|
||||
uri = self.log_analytics_uri + resource + '?api-version=2016-04-01'
|
||||
|
||||
headers = {
|
||||
'content-type': content_type,
|
||||
'Authorization': signature,
|
||||
'Log-Type': log_type,
|
||||
'x-ms-date': rfc1123date
|
||||
}
|
||||
|
||||
async with session.post(uri, data=body, headers=headers) as response:
|
||||
if (response.status >= 200 and response.status <= 299):
|
||||
logging.debug('{} events have been successfully sent to Azure Sentinel'.format(events_number))
|
||||
self.successfull_sent_events_number += events_number
|
||||
else:
|
||||
logging.error("Error during sending events to Azure Sentinel. Response code: {}".format(response.status))
|
||||
self.failed_sent_events_number += events_number
|
||||
raise Exception("Error during sending events to Azure Sentinel. Response code: {}".format(response.status))
|
||||
|
||||
def _check_size(self, queue):
|
||||
data_bytes_len = len(json.dumps(queue).encode())
|
||||
return data_bytes_len < self.queue_size_bytes
|
||||
|
||||
def _split_big_request(self, queue):
|
||||
if self._check_size(queue):
|
||||
return [queue]
|
||||
else:
|
||||
middle = int(len(queue) / 2)
|
||||
queues_list = [queue[:middle], queue[middle:]]
|
||||
return self._split_big_request(queues_list[0]) + self._split_big_request(queues_list[1])
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
from azure.storage.fileshare import ShareClient
|
||||
from azure.storage.fileshare import ShareFileClient
|
||||
from azure.core.exceptions import ResourceNotFoundError
|
||||
|
||||
|
||||
class StateManager:
|
||||
def __init__(self, connection_string, share_name='funcstatemarkershare', file_path='funcstatemarkerfile'):
|
||||
self.share_cli = ShareClient.from_connection_string(conn_str=connection_string, share_name=share_name)
|
||||
self.file_cli = ShareFileClient.from_connection_string(conn_str=connection_string, share_name=share_name, file_path=file_path)
|
||||
|
||||
def post(self, marker_text: str):
|
||||
try:
|
||||
self.file_cli.upload_file(marker_text)
|
||||
except ResourceNotFoundError:
|
||||
self.share_cli.create_share()
|
||||
self.file_cli.upload_file(marker_text)
|
||||
|
||||
def get(self):
|
||||
try:
|
||||
return self.file_cli.download_file().readall().decode()
|
||||
except ResourceNotFoundError:
|
||||
return None
|
||||
|
|
@ -0,0 +1,130 @@
|
|||
{
|
||||
"id": "GCPMonitorDataConnector",
|
||||
"title": "Google Cloud Platform Cloud Monitoring",
|
||||
"publisher": "Google",
|
||||
"descriptionMarkdown": "The Google Cloud Platform Cloud Monitoring data connector provides the capability to ingest [GCP Monitoring metrics](https://cloud.google.com/monitoring/api/metrics_gcp) into Azure Sentinel using the GCP Monitoring API. Refer to [GCP Monitoring API documentation](https://cloud.google.com/monitoring/api/v3) for more information.",
|
||||
"additionalRequirementBanner": ">This data connector depends on a parser based on a Kusto Function to work as expected [**GCP_MONITORING**](https://aka.ms/sentinel-GCPMonitorDataConnector-parser) which is deployed with the Azure Sentinel Solution.",
|
||||
"graphQueries": [
|
||||
{
|
||||
"metricName": "GCP Monitoring logs",
|
||||
"legend": "GCP_MONITORING_CL",
|
||||
"baseQuery": "GCP_MONITORING_CL"
|
||||
}
|
||||
],
|
||||
"sampleQueries": [
|
||||
{
|
||||
"description" : "All GCP Monitoring logs",
|
||||
"query": "GCP_MONITORING_CL\n| sort by TimeGenerated desc"
|
||||
}
|
||||
],
|
||||
"dataTypes": [
|
||||
{
|
||||
"name": "GCP_MONITORING_CL",
|
||||
"lastDataReceivedQuery": "GCP_MONITORING_CL\n | summarize Time = max(TimeGenerated)\n | where isnotempty(Time)"
|
||||
}
|
||||
],
|
||||
"connectivityCriterias": [
|
||||
{
|
||||
"type": "IsConnectedQuery",
|
||||
"value": [
|
||||
"GCP_MONITORING_CL\n | summarize LastLogReceived = max(TimeGenerated)\n | project IsConnected = LastLogReceived > ago(1d)"
|
||||
]
|
||||
}
|
||||
],
|
||||
"availability": {
|
||||
"status": 1,
|
||||
"isPreview": true
|
||||
},
|
||||
"permissions": {
|
||||
"resourceProvider": [
|
||||
{
|
||||
"provider": "Microsoft.OperationalInsights/workspaces",
|
||||
"permissionsDisplayText": "read and write permissions on the workspace are required.",
|
||||
"providerDisplayName": "Workspace",
|
||||
"scope": "Workspace",
|
||||
"requiredPermissions": {
|
||||
"write": true,
|
||||
"read": true,
|
||||
"delete": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"provider": "Microsoft.OperationalInsights/workspaces/sharedKeys",
|
||||
"permissionsDisplayText": "read permissions to shared keys for the workspace are required. [See the documentation to learn more about workspace keys](https://docs.microsoft.com/azure/azure-monitor/platform/agent-windows#obtain-workspace-id-and-key).",
|
||||
"providerDisplayName": "Keys",
|
||||
"scope": "Workspace",
|
||||
"requiredPermissions": {
|
||||
"action": true
|
||||
}
|
||||
}
|
||||
],
|
||||
"customs": [
|
||||
{
|
||||
"name": "Microsoft.Web/sites permissions",
|
||||
"description": "Read and write permissions to Azure Functions to create a Function App is required. [See the documentation to learn more about Azure Functions](https://docs.microsoft.com/azure/azure-functions/)."
|
||||
},
|
||||
{
|
||||
"name": "GCP service account",
|
||||
"description": "GCP service account with permissions to read Cloud Monitoring metrics is required for GCP Monitoring API (required *Monitoring Viewer* role). Also json file with service account key is required. See the documentation to learn more about [creating service account](https://cloud.google.com/iam/docs/creating-managing-service-accounts) and [creating service account key](https://cloud.google.com/iam/docs/creating-managing-service-account-keys)."
|
||||
}
|
||||
]
|
||||
},
|
||||
"instructionSteps": [
|
||||
{
|
||||
"title": "",
|
||||
"description": ">**NOTE:** This connector uses Azure Functions to connect to the GCP API to pull logs into Azure Sentinel. This might result in additional data ingestion costs. Check the [Azure Functions pricing page](https://azure.microsoft.com/pricing/details/functions/) for details."
|
||||
},
|
||||
{
|
||||
"title": "",
|
||||
"description": ">**(Optional Step)** Securely store workspace and API authorization key(s) or token(s) in Azure Key Vault. Azure Key Vault provides a secure mechanism to store and retrieve key values. [Follow these instructions](https://docs.microsoft.com/azure/app-service/app-service-key-vault-references) to use Azure Key Vault with an Azure Function App."
|
||||
},
|
||||
{
|
||||
"title": "",
|
||||
"description": ">**NOTE:** This data connector depends on a parser based on a Kusto Function to work as expected [**GCP_MONITORING**](https://aka.ms/sentinel-GCPMonitorDataConnector-parser) which is deployed with the Azure Sentinel Solution."
|
||||
},
|
||||
{
|
||||
"title": "",
|
||||
"description": "**STEP 1 - Configuring GCP and obtaining credentials**\n\n1. [Create service account](https://cloud.google.com/iam/docs/creating-managing-service-accounts) with Monitoring Viewer role and [get service account key json file](https://cloud.google.com/iam/docs/creating-managing-service-account-keys).\n\n2. Prepare the list of GCP projects to get metrics from. [Learn more about GCP projects](https://cloud.google.com/resource-manager/docs/cloud-platform-resource-hierarchy).\n\n3. Prepare the list of [GCP metric types](https://cloud.google.com/monitoring/api/metrics_gcp)"
|
||||
},
|
||||
{
|
||||
"title": "",
|
||||
"description": "**STEP 2 - Choose ONE from the following two deployment options to deploy the connector and the associated Azure Function**\n\n>**IMPORTANT:** Before deploying the data connector, have the Workspace ID and Workspace Primary Key (can be copied from the following), as well as Azure Blob Storage connection string and container name, readily available.",
|
||||
"instructions":[
|
||||
{
|
||||
"parameters": {
|
||||
"fillWith": [
|
||||
"WorkspaceId"
|
||||
],
|
||||
"label": "Workspace ID"
|
||||
},
|
||||
"type": "CopyableLabel"
|
||||
},
|
||||
{
|
||||
"parameters": {
|
||||
"fillWith": [
|
||||
"PrimaryKey"
|
||||
],
|
||||
"label": "Primary Key"
|
||||
},
|
||||
"type": "CopyableLabel"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "Option 1 - Azure Resource Manager (ARM) Template",
|
||||
"description": "Use this method for automated deployment of the data connector using an ARM Tempate.\n\n1. Click the **Deploy to Azure** button below. \n\n\t[](https://aka.ms/sentinel-GCPMonitorDataConnector-azuredeploy)\n2. Select the preferred **Subscription**, **Resource Group** and **Location**. \n3. Enter the **Google Cloud Platform Project Id List**, **Google Cloud Platform Metric Types List**, **Google Cloud Platform Credentials File Content**, **Azure Sentinel Workspace Id**, **Azure Sentinel Shared Key**\n4. Mark the checkbox labeled **I agree to the terms and conditions stated above**.\n5. Click **Purchase** to deploy."
|
||||
},
|
||||
{
|
||||
"title": "Option 2 - Manual Deployment of Azure Functions",
|
||||
"description": "Use the following step-by-step instructions to deploy the data connector manually with Azure Functions (Deployment via Visual Studio Code)."
|
||||
},
|
||||
{
|
||||
"title": "",
|
||||
"description": "**1. Deploy a Function App**\n\n> **NOTE:** You will need to [prepare VS code](https://docs.microsoft.com/azure/azure-functions/create-first-function-vs-code-python) for Azure function development.\n\n1. Download the [Azure Function App](https://aka.ms/sentinel-GCPMonitorDataConnector-functionapp) file. Extract archive to your local development computer.\n2. Start VS Code. Choose File in the main menu and select Open Folder.\n3. Select the top level folder from extracted files.\n4. Choose the Azure icon in the Activity bar, then in the **Azure: Functions** area, choose the **Deploy to function app** button.\nIf you aren't already signed in, choose the Azure icon in the Activity bar, then in the **Azure: Functions** area, choose **Sign in to Azure**\nIf you're already signed in, go to the next step.\n5. Provide the following information at the prompts:\n\n\ta. **Select folder:** Choose a folder from your workspace or browse to one that contains your function app.\n\n\tb. **Select Subscription:** Choose the subscription to use.\n\n\tc. Select **Create new Function App in Azure** (Don't choose the Advanced option)\n\n\td. **Enter a globally unique name for the function app:** Type a name that is valid in a URL path. The name you type is validated to make sure that it's unique in Azure Functions.\n\n\te. **Select a runtime:** Choose Python 3.8.\n\n\tf. Select a location for new resources. For better performance and lower costs choose the same [region](https://azure.microsoft.com/regions/) where Azure Sentinel is located.\n\n6. Deployment will begin. A notification is displayed after your function app is created and the deployment package is applied.\n7. Go to Azure Portal for the Function App configuration."
|
||||
},
|
||||
{
|
||||
"title": "",
|
||||
"description": "**2. Configure the Function App**\n\n1. In the Function App, select the Function App Name and select **Configuration**.\n2. In the **Application settings** tab, select **+ New application setting**.\n3. Add each of the following application settings individually, with their respective string values (case-sensitive): \n\t\tGCP_PROJECT_ID\n\t\tGCP_METRICS\n\t\tGCP_CREDENTIALS_FILE_CONTENT\n\t\tWORKSPACE_ID\n\t\tSHARED_KEY\n\t\tlogAnalyticsUri (Optional)\n - Use logAnalyticsUri to override the log analytics API endpoint for dedicated cloud. For example, for public cloud, leave the value empty; for Azure GovUS cloud environment, specify the value in the following format: `https://WORKSPACE_ID.ods.opinsights.azure.us`. \n4. Once all application settings have been entered, click **Save**."
|
||||
}
|
||||
]
|
||||
}
|
||||
Двоичные данные
Solutions/GoogleCloudPlatformMonitor/Data Connectors/GCP_Monitor_func.zip
Normal file
Двоичные данные
Solutions/GoogleCloudPlatformMonitor/Data Connectors/GCP_Monitor_func.zip
Normal file
Двоичный файл не отображается.
|
|
@ -0,0 +1,214 @@
|
|||
{
|
||||
"$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
|
||||
"contentVersion": "1.0.0.0",
|
||||
"parameters": {
|
||||
"FunctionName": {
|
||||
"defaultValue": "GCPMONITOR",
|
||||
"type": "string",
|
||||
"minLength": 1,
|
||||
"maxLength": 11
|
||||
},
|
||||
"GoogleCloudPlatformProjectIdList": {
|
||||
"type": "string",
|
||||
"defaultValue": "",
|
||||
"metadata": {
|
||||
"description": "List of GCP projects separeted with space. For example: projects/testproject1 projects/testproject2"
|
||||
}
|
||||
},
|
||||
"GoogleCloudPlatformMetricTypesList": {
|
||||
"type": "string",
|
||||
"defaultValue": "",
|
||||
"metadata": {
|
||||
"description": "List of GCP metric types separeted with space. For example: compute.googleapis.com/instance/disk/write_bytes_count compute.googleapis.com/instance/uptime_total"
|
||||
}
|
||||
},
|
||||
"GoogleCloudPlatformCredentialsFileContent": {
|
||||
"type": "securestring",
|
||||
"defaultValue": "",
|
||||
"metadata": {
|
||||
"description": "Content of json file with credentials for service account that has permissions to read logs using Logging API"
|
||||
}
|
||||
},
|
||||
"AzureSentinelWorkspaceId": {
|
||||
"type": "string",
|
||||
"defaultValue": ""
|
||||
},
|
||||
"AzureSentinelSharedKey": {
|
||||
"type": "securestring",
|
||||
"defaultValue": ""
|
||||
}
|
||||
},
|
||||
"variables": {
|
||||
"FunctionName": "[concat(toLower(parameters('FunctionName')), uniqueString(resourceGroup().id))]",
|
||||
"StorageSuffix": "[environment().suffixes.storage]",
|
||||
"LogAnaltyicsUri": "[replace(environment().portal, 'https://portal', concat('https://', toLower(parameters('AzureSentinelWorkspaceId')), '.ods.opinsights'))]"
|
||||
},
|
||||
"resources": [
|
||||
{
|
||||
"type": "Microsoft.Insights/components",
|
||||
"apiVersion": "2015-05-01",
|
||||
"name": "[variables('FunctionName')]",
|
||||
"location": "[resourceGroup().location]",
|
||||
"kind": "web",
|
||||
"properties": {
|
||||
"Application_Type": "web",
|
||||
"ApplicationId": "[variables('FunctionName')]"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "Microsoft.Storage/storageAccounts",
|
||||
"apiVersion": "2019-06-01",
|
||||
"name": "[tolower(variables('FunctionName'))]",
|
||||
"location": "[resourceGroup().location]",
|
||||
"sku": {
|
||||
"name": "Standard_LRS",
|
||||
"tier": "Standard"
|
||||
},
|
||||
"kind": "StorageV2",
|
||||
"properties": {
|
||||
"networkAcls": {
|
||||
"bypass": "AzureServices",
|
||||
"virtualNetworkRules": [],
|
||||
"ipRules": [],
|
||||
"defaultAction": "Allow"
|
||||
},
|
||||
"supportsHttpsTrafficOnly": true,
|
||||
"encryption": {
|
||||
"services": {
|
||||
"file": {
|
||||
"keyType": "Account",
|
||||
"enabled": true
|
||||
},
|
||||
"blob": {
|
||||
"keyType": "Account",
|
||||
"enabled": true
|
||||
}
|
||||
},
|
||||
"keySource": "Microsoft.Storage"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "Microsoft.Storage/storageAccounts/blobServices",
|
||||
"apiVersion": "2019-06-01",
|
||||
"name": "[concat(variables('FunctionName'), '/default')]",
|
||||
"dependsOn": [
|
||||
"[resourceId('Microsoft.Storage/storageAccounts', tolower(variables('FunctionName')))]"
|
||||
],
|
||||
"sku": {
|
||||
"name": "Standard_LRS",
|
||||
"tier": "Standard"
|
||||
},
|
||||
"properties": {
|
||||
"cors": {
|
||||
"corsRules": []
|
||||
},
|
||||
"deleteRetentionPolicy": {
|
||||
"enabled": false
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "Microsoft.Storage/storageAccounts/fileServices",
|
||||
"apiVersion": "2019-06-01",
|
||||
"name": "[concat(variables('FunctionName'), '/default')]",
|
||||
"dependsOn": [
|
||||
"[resourceId('Microsoft.Storage/storageAccounts', tolower(variables('FunctionName')))]"
|
||||
],
|
||||
"sku": {
|
||||
"name": "Standard_LRS",
|
||||
"tier": "Standard"
|
||||
},
|
||||
"properties": {
|
||||
"cors": {
|
||||
"corsRules": []
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "Microsoft.Web/sites",
|
||||
"apiVersion": "2018-11-01",
|
||||
"name": "[variables('FunctionName')]",
|
||||
"location": "[resourceGroup().location]",
|
||||
"dependsOn": [
|
||||
"[resourceId('Microsoft.Storage/storageAccounts', tolower(variables('FunctionName')))]",
|
||||
"[resourceId('Microsoft.Insights/components', variables('FunctionName'))]"
|
||||
],
|
||||
"kind": "functionapp,linux",
|
||||
"identity": {
|
||||
"type": "SystemAssigned"
|
||||
},
|
||||
"properties": {
|
||||
"name": "[variables('FunctionName')]",
|
||||
"httpsOnly": true,
|
||||
"clientAffinityEnabled": true,
|
||||
"alwaysOn": true,
|
||||
"reserved": true,
|
||||
"siteConfig": {
|
||||
"linuxFxVersion": "python|3.8"
|
||||
}
|
||||
},
|
||||
|
||||
"resources": [
|
||||
{
|
||||
"apiVersion": "2018-11-01",
|
||||
"type": "config",
|
||||
"name": "appsettings",
|
||||
"dependsOn": [
|
||||
"[concat('Microsoft.Web/sites/', variables('FunctionName'))]"
|
||||
],
|
||||
"properties": {
|
||||
"FUNCTIONS_EXTENSION_VERSION": "~3",
|
||||
"FUNCTIONS_WORKER_RUNTIME": "python",
|
||||
"APPINSIGHTS_INSTRUMENTATIONKEY": "[reference(resourceId('Microsoft.insights/components', variables('FunctionName')), '2015-05-01').InstrumentationKey]",
|
||||
"APPLICATIONINSIGHTS_CONNECTION_STRING": "[reference(resourceId('microsoft.insights/components', variables('FunctionName')), '2015-05-01').ConnectionString]",
|
||||
"AzureWebJobsStorage": "[concat('DefaultEndpointsProtocol=https;AccountName=', toLower(variables('FunctionName')),';AccountKey=',listKeys(resourceId('Microsoft.Storage/storageAccounts', toLower(variables('FunctionName'))), '2019-06-01').keys[0].value, ';EndpointSuffix=',toLower(variables('StorageSuffix')))]",
|
||||
"GCP_PROJECT_ID": "[parameters('GoogleCloudPlatformProjectIdList')]",
|
||||
"GCP_METRICS": "[parameters('GoogleCloudPlatformMetricTypesList')]",
|
||||
"GCP_CREDENTIALS_FILE_CONTENT": "[parameters('GoogleCloudPlatformCredentialsFileContent')]",
|
||||
"WORKSPACE_ID": "[parameters('AzureSentinelWorkspaceId')]",
|
||||
"SHARED_KEY": "[parameters('AzureSentinelSharedKey')]",
|
||||
"logAnalyticsUri": "[variables('LogAnaltyicsUri')]",
|
||||
"WEBSITE_RUN_FROM_PACKAGE": "https://aka.ms/sentinel-GCPMonitorDataConnector-functionapp"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "Microsoft.Storage/storageAccounts/blobServices/containers",
|
||||
"apiVersion": "2019-06-01",
|
||||
"name": "[concat(variables('FunctionName'), '/default/azure-webjobs-hosts')]",
|
||||
"dependsOn": [
|
||||
"[resourceId('Microsoft.Storage/storageAccounts/blobServices', variables('FunctionName'), 'default')]",
|
||||
"[resourceId('Microsoft.Storage/storageAccounts', variables('FunctionName'))]"
|
||||
],
|
||||
"properties": {
|
||||
"publicAccess": "None"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "Microsoft.Storage/storageAccounts/blobServices/containers",
|
||||
"apiVersion": "2019-06-01",
|
||||
"name": "[concat(variables('FunctionName'), '/default/azure-webjobs-secrets')]",
|
||||
"dependsOn": [
|
||||
"[resourceId('Microsoft.Storage/storageAccounts/blobServices', variables('FunctionName'), 'default')]",
|
||||
"[resourceId('Microsoft.Storage/storageAccounts', variables('FunctionName'))]"
|
||||
],
|
||||
"properties": {
|
||||
"publicAccess": "None"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "Microsoft.Storage/storageAccounts/fileServices/shares",
|
||||
"apiVersion": "2019-06-01",
|
||||
"name": "[concat(variables('FunctionName'), '/default/', tolower(variables('FunctionName')))]",
|
||||
"dependsOn": [
|
||||
"[resourceId('Microsoft.Storage/storageAccounts/fileServices', variables('FunctionName'), 'default')]",
|
||||
"[resourceId('Microsoft.Storage/storageAccounts', variables('FunctionName'))]"
|
||||
],
|
||||
"properties": {
|
||||
"shareQuota": 5120
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
{
|
||||
"version": "2.0",
|
||||
"logging": {
|
||||
"applicationInsights": {
|
||||
"samplingSettings": {
|
||||
"isEnabled": true,
|
||||
"excludedTypes": "Request"
|
||||
}
|
||||
}
|
||||
},
|
||||
"extensionBundle": {
|
||||
"id": "Microsoft.Azure.Functions.ExtensionBundle",
|
||||
"version": "[1.*, 2.0.0)"
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
{
|
||||
"$schema": "http://json.schemastore.org/proxies",
|
||||
"proxies": {}
|
||||
}
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
google-cloud-monitoring==2.2.1
|
||||
azure-functions==1.6.0
|
||||
aiohttp==3.7.4.post0
|
||||
azure-storage-file-share==12.5.0
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
// Usage Instruction :
|
||||
// Paste below query in log analytics, click on Save button and select as Function from drop down by specifying function name and alias as GCP_MONITORING.
|
||||
// Function usually takes 10-15 minutes to activate. You can then use function alias from any other queries (e.g. GCP_MONITORING | take 10).
|
||||
// Reference : Using functions in Azure monitor log queries : https://docs.microsoft.com/azure/azure-monitor/log-query/functions
|
||||
GCP_MONITORING_CL
|
||||
| extend EventVendor = 'GCP'
|
||||
| extend EventProduct = 'Cloud Monitoring'
|
||||
| project-rename MetricLabelsInstanceName=metric_labels_instance_name_s,
|
||||
MetricType=metric_type_s,
|
||||
ResourceType=resource_type_s,
|
||||
ResourceLabelsProjectId=resource_labels_project_id_s,
|
||||
ResourceLabelsInstanceId=resource_labels_instance_id_s,
|
||||
ResourceLabelsZone=resource_labels_zone_s,
|
||||
MetricKind=metricKind_s,
|
||||
ValueType=valueType_s,
|
||||
IntervalStartTime=interval_startTime_t,
|
||||
IntervalEndTime=interval_endTime_t,
|
||||
ValueInt64Value=value_int64Value_d
|
||||
Загрузка…
Ссылка в новой задаче