520 строки
23 KiB
Plaintext
520 строки
23 KiB
Plaintext
input {
|
|
# Collect Syslog Data
|
|
syslog {
|
|
port => 5514
|
|
grok_pattern => "<%{NONNEGINT:deviceFacility}(?:[.])?(?:\[%{NONNEGINT:priority}\])?>(?:%{SYSLOGTIMESTAMP:deviceReceiptTime})? %{DATA:deviceHostName}(:)? %{DATA:deviceProcessName}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:message}"
|
|
type => unifisyslog
|
|
}
|
|
}
|
|
filter {
|
|
# Handle Syslog
|
|
if [type] == "unifisyslog" {
|
|
# Add CEF Fields to all types
|
|
mutate {
|
|
add_field => {
|
|
"deviceVendor" => "Unifi"
|
|
}
|
|
add_field => {
|
|
"deviceProduct" => "USG"
|
|
}
|
|
add_field => {
|
|
"deviceVersion" => "0"
|
|
}
|
|
}
|
|
# Handle DNS logs
|
|
if [deviceProcessName] == "dnsmasq" {
|
|
# Parse DNS Logs
|
|
grok {
|
|
match => {
|
|
"message" =>
|
|
[
|
|
"%{WORD:deviceAction}\[(?:%{GREEDYDATA:querytype})?\] %{URIHOST:destinationDnsDomain} from %{IP:sourceAddress}",
|
|
"%{WORD:deviceAction} %{URIHOST:destinationDnsDomain} is (?:%{IP:destinationAddress})?(?:%{GREEDYDATA:result})?",
|
|
"%{WORD:deviceAction} %{URIHOST:destinationDnsDomain} is %{IP:destinationAddress}",
|
|
"%{WORD:deviceAction} %{URIHOST:destinationDnsDomain} to %{IP:destinationAddress}"
|
|
#"%{blocklist:BLOCKLIST} %{URIHOST:destinationDnsDomain} is %{ip:destinationAddress}"
|
|
]
|
|
}
|
|
}
|
|
# Drop the message field to save data since its parse and add CEF field to seperate
|
|
mutate {
|
|
remove_field => ["message"]
|
|
add_field => {
|
|
"deviceEventClassId" => "DNS"
|
|
}
|
|
}
|
|
# Enrich logs with Geo and ASN data
|
|
if [sourceAddress] {
|
|
geoip {
|
|
source => "sourceAddress"
|
|
target => "sourceGeo"
|
|
add_field => {
|
|
"sourceGeoLongitude" => "%{[sourceGeo][longitude]}"
|
|
}
|
|
add_field => {
|
|
"sourceGeoLatitude" => "%{[sourceGeo][latitude]}"
|
|
}
|
|
add_field => {
|
|
"sourceGeoLongitude" => "%{[sourceGeo][longitude]}"
|
|
}
|
|
add_field => {
|
|
"sourceGeoLatitude" => "%{[sourceGeo][latitude]}"
|
|
}
|
|
add_field => {
|
|
"sourceGeoRegionName" => "%{[sourceGeo][region_name]}"
|
|
}
|
|
add_field => {
|
|
"sourceGeoCountryCode2" => "%{[sourceGeo][country_code2]}"
|
|
}
|
|
add_field => {
|
|
"sourceGeoCountryName" => "%{[sourceGeo][country_name]}"
|
|
}
|
|
add_field => {
|
|
"sourceGeoPostalCode" => "%{[sourceGeo][postal_code]}"
|
|
}
|
|
add_field => {
|
|
"sourceGeoCityName" => "%{[sourceGeo][city_name]}"
|
|
}
|
|
add_field => {
|
|
"sourceGeoTimeZone" => "%{[sourceGeo][timezone]}"
|
|
}
|
|
add_field => {
|
|
"sourceGeoRegionCode" => "%{[sourceGeo][region_code]}"
|
|
}
|
|
add_field => {
|
|
"sourceGeoContinentCode" => "%{[sourceGeo][continent_code]}"
|
|
}
|
|
add_field => {
|
|
"sourceGeoCountryCode3" => "%{[sourceGeo][country_code3]}"
|
|
}
|
|
}
|
|
geoip {
|
|
default_database_type => "ASN"
|
|
source => "sourceAddress"
|
|
target => "sourceGeoASN"
|
|
add_field => {
|
|
"sourceASNAsOrg" => "%{[sourceGeoASN][as_org]}"
|
|
}
|
|
add_field => {
|
|
"sourceASN" => "%{[sourceGeoASN][asn]}"
|
|
}
|
|
}
|
|
}
|
|
if [destinationAddress] {
|
|
geoip {
|
|
source => "destinationAddress"
|
|
target => "destinationGeo"
|
|
add_field => {
|
|
"destinationGeoLongitude" => "%{[destinationGeo][longitude]}"
|
|
}
|
|
add_field => {
|
|
"destinationGeoLatitude" => "%{[destinationGeo][latitude]}"
|
|
}
|
|
add_field => {
|
|
"destinationGeoRegionName" => "%{[destinationGeo][region_name]}"
|
|
}
|
|
add_field => {
|
|
"destinationGeoCountryCode2" => "%{[destinationGeo][country_code2]}"
|
|
}
|
|
add_field => {
|
|
"destinationGeoCountryName" => "%{[destinationGeo][country_name]}"
|
|
}
|
|
add_field => {
|
|
"destinationGeoPostalCode" => "%{[destinationGeo][postal_code]}"
|
|
}
|
|
add_field => {
|
|
"destinationGeoCityName" => "%{[destinationGeo][city_name]}"
|
|
}
|
|
add_field => {
|
|
"destinationGeoTimeZone" => "%{[destinationGeo][timezone]}"
|
|
}
|
|
add_field => {
|
|
"destinationGeoRegionCode" => "%{[destinationGeo][region_code]}"
|
|
}
|
|
add_field => {
|
|
"destinationGeoContinentCode" => "%{[destinationGeo][continent_code]}"
|
|
}
|
|
add_field => {
|
|
"destinationGeoCountryCode3" => "%{[destinationGeo][country_code3]}"
|
|
}
|
|
}
|
|
geoip {
|
|
default_database_type => "ASN"
|
|
source => "destinationAddress"
|
|
target => "destinationGeoASN"
|
|
add_field => {
|
|
"destinationASNAsOrg" => "%{[destinationGeoASN][as_org]}"
|
|
}
|
|
add_field => {
|
|
"destinationASN" => "%{[destinationGeoASN][asn]}"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
# Handle DHCP events
|
|
else if [deviceProcessName] == "dhcpd" {
|
|
if "_grokparsefailure" not in [tags] {
|
|
grok {
|
|
match => {
|
|
"message" => "%{WORD:deviceAction} %{GREEDYDATA:params}"
|
|
}
|
|
}
|
|
if "_grokparsefailure" not in [tags] {
|
|
grok {
|
|
match => {
|
|
"params" =>
|
|
[
|
|
"from %{IP:sourceAddress} via %{IPORHOST:deviceInboundInterface}" ,
|
|
"to %{IP:sourceAddress} \(%{MAC:sourceMacAddress}\) via %{IPORHOST:deviceInboundInterface}" ,
|
|
"from %{MAC:sourceMacAddress} (\(%{DATA:sourceHostName}\) )?via %{IPORHOST:deviceInboundInterface}(: %{GREEDYDATA:errormessage})?" ,
|
|
"on %{IP:sourceAddress} to %{MAC:sourceMacAddress} \(%{DATA:sourceHostName}\) via %{IPORHOST:deviceInboundInterface}"
|
|
]
|
|
}
|
|
}
|
|
# Drop the message and params fields to save data since its parse and add CEF field to seperate
|
|
mutate {
|
|
remove_field => ["message"]
|
|
remove_field => ["params"]
|
|
add_field => {
|
|
"deviceEventClassId" => "DHCP"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
}
|
|
# Handle Firewall Logs
|
|
else {
|
|
# Parse all the types of firewall logs
|
|
grok {
|
|
match => {
|
|
"message" =>
|
|
[
|
|
"\[%{DATA:ruleset}-%{DATA:rulenumber}-%{DATA:deviceAction}\]IN=%{DATA:deviceInboundInterface} OUT=%{DATA:deviceOutboundInterface} MAC=(?:%{MAC:destinationMacAddress}:%{MAC:sourceMacAddress}.*)? SRC=%{IP:sourceAddress} DST=%{IP:destinationAddress} LEN=%{INT:length} TOS=%{WORD:typeofservice} PREC=%{WORD:tosprecedence} TTL=%{INT:ttl} ID=%{INT:id}(?:%{DATA:fragmentFlags})? PROTO=%{WORD:transportProtocol} SPT=%{INT:sourcePort} DPT=%{INT:destinationPort} LEN=%{INT:udplength}",
|
|
"\[%{DATA:ruleset}-%{DATA:rulenumber}-%{DATA:deviceAction}\]IN=%{DATA:deviceInboundInterface} OUT=%{DATA:deviceOutboundInterface} MAC=%{MAC:destinationMacAddress}:%{MAC:sourceMacAddress}.* SRC=%{IP:sourceAddress} DST=%{IP:destinationAddress} LEN=%{INT:length} TOS=%{WORD:typeofservice} PREC=%{WORD:tosprecedence} TTL=%{INT:ttl} ID=%{INT:id}(?:%{DATA:fragmentFlags})? PROTO=%{WORD:transportProtocol} SPT=%{INT:sourcePort} DPT=%{INT:destinationPort} WINDOW=%{INT:window} RES=%{WORD:reservedBits} %{DATA:tcpFlags} URGP=%{INT:urgp}",
|
|
"\[%{DATA:ruleset}-%{DATA:rulenumber}-%{DATA:deviceAction}\]IN=%{DATA:deviceInboundInterface} OUT=%{DATA:deviceOutboundInterface} MAC=%{MAC:destinationMacAddress}:%{MAC:sourceMacAddress}.* SRC=%{IP:sourceAddress} DST=%{IP:destinationAddress} LEN=%{INT:length} TOS=%{WORD:typeofservice} PREC=%{WORD:tosprecedence} TTL=%{INT:ttl} ID=%{INT:id}(?:%{DATA:fragmentFlags})? PROTO=%{WORD:transportProtocol} TYPE=%{INT:etherType} CODE=%{INT:icmpcode} ID=%{INT:icmpId} SEQ=%{INT:icmpSequence}",
|
|
"%{DATA:IPS} %{DATA:deviceAction}: IN=%{DATA:deviceInboundInterface} OUT=%{DATA:deviceOutboundInterface} MAC=%{MAC:destinationMacAddress}:%{MAC:sourceMacAddress}.* SRC=%{IP:sourceAddress} DST=%{IP:destinationAddress} LEN=%{INT:length} TOS=%{WORD:typeofservice} PREC=%{WORD:tosprecedence} TTL=%{INT:ttl} ID=%{INT:id}(?:%{DATA:fragmentFlags})? PROTO=%{WORD:transportProtocol} SPT=%{INT:sourcePort} DPT=%{INT:destinationPort} WINDOW=%{INT:window} RES=%{WORD:reservedBits} %{DATA:tcpFlags} URGP=%{INT:urgp}",
|
|
"%{DATA:IPS} %{DATA:deviceAction}: IN=%{DATA:deviceInboundInterface} OUT=%{DATA:deviceOutboundInterface} MAC=%{MAC:destinationMacAddress}:%{MAC:sourceMacAddress}.* SRC=%{IP:sourceAddress} DST=%{IP:destinationAddress} LEN=%{INT:length} TOS=%{WORD:typeofservice} PREC=%{WORD:tosprecedence} TTL=%{INT:ttl} ID=%{INT:id}(?:%{DATA:fragmentFlags})? PROTO=%{WORD:transportProtocol} SPT=%{INT:sourcePort} DPT=%{INT:destinationPort} LEN=%{INT:udplength}"
|
|
]
|
|
}
|
|
# Drop the message field to save data
|
|
remove_field => ["message"]
|
|
}
|
|
# Correct action to be allow or drop
|
|
if [deviceAction] == "A" {
|
|
mutate {
|
|
update => {
|
|
"deviceAction" => "Allow"
|
|
}
|
|
}
|
|
}
|
|
if [deviceAction] == "D" {
|
|
mutate {
|
|
update => {
|
|
"deviceAction" => "Drop"
|
|
}
|
|
}
|
|
}
|
|
# Tag IPS Events for seperation in CEF
|
|
if [IPS] {
|
|
mutate {
|
|
add_field => {
|
|
"deviceEventClassId" => "IPS"
|
|
}
|
|
}
|
|
}
|
|
else {
|
|
mutate {
|
|
add_field => {
|
|
"deviceEventClassId" => "Firewall"
|
|
}
|
|
}
|
|
}
|
|
# Enrich logs with GeoIP geolocation data
|
|
if [sourceAddress] {
|
|
geoip {
|
|
source => "sourceAddress"
|
|
target => "sourceGeo"
|
|
add_field => {
|
|
"sourceGeoLongitude" => "%{[sourceGeo][longitude]}"
|
|
}
|
|
add_field => {
|
|
"sourceGeoLatitude" => "%{[sourceGeo][latitude]}"
|
|
}
|
|
add_field => {
|
|
"sourceGeoLongitude" => "%{[sourceGeo][longitude]}"
|
|
}
|
|
add_field => {
|
|
"sourceGeoLatitude" => "%{[sourceGeo][latitude]}"
|
|
}
|
|
add_field => {
|
|
"sourceGeoRegionName" => "%{[sourceGeo][region_name]}"
|
|
}
|
|
add_field => {
|
|
"sourceGeoCountryCode2" => "%{[sourceGeo][country_code2]}"
|
|
}
|
|
add_field => {
|
|
"sourceGeoCountryName" => "%{[sourceGeo][country_name]}"
|
|
}
|
|
add_field => {
|
|
"sourceGeoPostalCode" => "%{[sourceGeo][postal_code]}"
|
|
}
|
|
add_field => {
|
|
"sourceGeoCityName" => "%{[sourceGeo][city_name]}"
|
|
}
|
|
add_field => {
|
|
"sourceGeoTimeZone" => "%{[sourceGeo][timezone]}"
|
|
}
|
|
add_field => {
|
|
"sourceGeoRegionCode" => "%{[sourceGeo][region_code]}"
|
|
}
|
|
add_field => {
|
|
"sourceGeoContinentCode" => "%{[sourceGeo][continent_code]}"
|
|
}
|
|
add_field => {
|
|
"sourceGeoCountryCode3" => "%{[sourceGeo][country_code3]}"
|
|
}
|
|
}
|
|
geoip {
|
|
default_database_type => "ASN"
|
|
source => "sourceAddress"
|
|
target => "sourceGeoASN"
|
|
add_field => {
|
|
"sourceASNAsOrg" => "%{[sourceGeoASN][as_org]}"
|
|
}
|
|
add_field => {
|
|
"sourceASN" => "%{[sourceGeoASN][asn]}"
|
|
}
|
|
}
|
|
}
|
|
if [destinationAddress] {
|
|
geoip {
|
|
source => "destinationAddress"
|
|
target => "destinationGeo"
|
|
add_field => {
|
|
"destinationGeoLongitude" => "%{[destinationGeo][longitude]}"
|
|
}
|
|
add_field => {
|
|
"destinationGeoLatitude" => "%{[destinationGeo][latitude]}"
|
|
}
|
|
add_field => {
|
|
"destinationGeoRegionName" => "%{[destinationGeo][region_name]}"
|
|
}
|
|
add_field => {
|
|
"destinationGeoCountryCode2" => "%{[destinationGeo][country_code2]}"
|
|
}
|
|
add_field => {
|
|
"destinationGeoCountryName" => "%{[destinationGeo][country_name]}"
|
|
}
|
|
add_field => {
|
|
"destinationGeoPostalCode" => "%{[destinationGeo][postal_code]}"
|
|
}
|
|
add_field => {
|
|
"destinationGeoCityName" => "%{[destinationGeo][city_name]}"
|
|
}
|
|
add_field => {
|
|
"destinationGeoTimeZone" => "%{[destinationGeo][timezone]}"
|
|
}
|
|
add_field => {
|
|
"destinationGeoRegionCode" => "%{[destinationGeo][region_code]}"
|
|
}
|
|
add_field => {
|
|
"destinationGeoContinentCode" => "%{[destinationGeo][continent_code]}"
|
|
}
|
|
add_field => {
|
|
"destinationGeoCountryCode3" => "%{[destinationGeo][country_code3]}"
|
|
}
|
|
}
|
|
geoip {
|
|
default_database_type => "ASN"
|
|
source => "destinationAddress"
|
|
target => "destinationGeoASN"
|
|
add_field => {
|
|
"destinationASNAsOrg" => "%{[destinationGeoASN][as_org]}"
|
|
}
|
|
add_field => {
|
|
"destinationASN" => "%{[destinationGeoASN][asn]}"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
|
|
output {
|
|
# Send to local agent if parsing was all success
|
|
if [type] == "unifisyslog" {
|
|
if "_grokparsefailure" not in [tags] {
|
|
syslog {
|
|
host => "127.0.0.1"
|
|
port => 25226
|
|
protocol => "tcp"
|
|
codec => cef {
|
|
reverse_mapping => true
|
|
delimiter => "\r\n"
|
|
vendor => "%{deviceVendor}"
|
|
product => "%{deviceProduct}"
|
|
version => "%{deviceVersion}"
|
|
signature => "%{deviceEventClassId}"
|
|
name => "%{deviceAction}"
|
|
severity => "%{severity}"
|
|
fields => [
|
|
"deviceAction",
|
|
"applicationProtocol",
|
|
"deviceCustomIPv6Address1",
|
|
"deviceCustomIPv6Address1Label",
|
|
"deviceCustomIPv6Address2",
|
|
"deviceCustomIPv6Address2Label",
|
|
"deviceCustomIPv6Address3",
|
|
"deviceCustomIPv6Address3Label",
|
|
"deviceCustomIPv6Address4",
|
|
"deviceCustomIPv6Address4Label",
|
|
"deviceEventCategory",
|
|
"deviceCustomFloatingPoint1",
|
|
"deviceCustomFloatingPoint1Label",
|
|
"deviceCustomFloatingPoint2",
|
|
"deviceCustomFloatingPoint2Label",
|
|
"deviceCustomFloatingPoint3",
|
|
"deviceCustomFloatingPoint3Label",
|
|
"deviceCustomFloatingPoint4",
|
|
"deviceCustomFloatingPoint4Label",
|
|
"deviceCustomNumber1",
|
|
"fieldDeviceCustomNumber1",
|
|
"deviceCustomNumber1Label",
|
|
"deviceCustomNumber2",
|
|
"fieldDeviceCustomNumber2",
|
|
"deviceCustomNumber2Label",
|
|
"deviceCustomNumber3",
|
|
"fieldDeviceCustomNumber3"
|
|
"deviceCustomNumber3Label",
|
|
"baseEventCount",
|
|
"deviceCustomString1",
|
|
"deviceCustomString1Label",
|
|
"deviceCustomString2",
|
|
"deviceCustomString2Label",
|
|
"deviceCustomString3",
|
|
"deviceCustomString3Label",
|
|
"deviceCustomString4",
|
|
"deviceCustomString4Label",
|
|
"deviceCustomString5",
|
|
"deviceCustomString5Label",
|
|
"deviceCustomString6",
|
|
"deviceCustomString6Label",
|
|
"destinationHostName",
|
|
"destinationMacAddress",
|
|
"destinationNtDomain",
|
|
"destinationProcessId",
|
|
"destinationUserPrivileges",
|
|
"destinationProcessName",
|
|
"destinationPort",
|
|
"destinationAddress",
|
|
"destinationUserId",
|
|
"destinationUserName",
|
|
"deviceAddress",
|
|
"deviceHostName",
|
|
"deviceProcessId",
|
|
"endTime",
|
|
"fileName",
|
|
"fileSize",
|
|
"bytesIn",
|
|
"message",
|
|
"bytesOut",
|
|
"eventOutcome",
|
|
"transportProtocol",
|
|
"requestUrl",
|
|
"deviceReceiptTime",
|
|
"sourceHostName",
|
|
"sourceMacAddress",
|
|
"sourceNtDomain",
|
|
"sourceProcessId",
|
|
"sourceUserPrivileges",
|
|
"sourceProcessName",
|
|
"sourcePort",
|
|
"sourceAddress",
|
|
"startTime",
|
|
"sourceUserId",
|
|
"sourceUserName",
|
|
"agentHostName",
|
|
"agentReceiptTime",
|
|
"agentType",
|
|
"agentId",
|
|
"cefVersion",
|
|
"agentAddress",
|
|
"agentVersion",
|
|
"agentTimeZone",
|
|
"destinationTimeZone",
|
|
"sourceLongitude",
|
|
"sourceLatitude",
|
|
"destinationLongitude",
|
|
"destinationLatitude",
|
|
"categoryDeviceType",
|
|
"managerReceiptTime",
|
|
"agentMacAddress",
|
|
"querytype",
|
|
"result",
|
|
"BLOCKLIST",
|
|
"errormessage",
|
|
"ruleset",
|
|
"rulenumber",
|
|
"IPS",
|
|
"length",
|
|
"typeofservice",
|
|
"tosprecedence",
|
|
"ttl",
|
|
"id",
|
|
"fragmentFlags",
|
|
"udplength",
|
|
"window",
|
|
"reservedBits",
|
|
"tcpFlags",
|
|
"urgp",
|
|
"etherType",
|
|
"icmpcode",
|
|
"icmpId",
|
|
"icmpSequence",
|
|
"deviceInboundInterface",
|
|
"deviceOutboundInterface",
|
|
"sourceGeoLongitude",
|
|
"sourceGeoLatitude",
|
|
"sourceGeoLongitude",
|
|
"sourceGeoLatitude",
|
|
"sourceGeoRegionName",
|
|
"sourceGeoCountryCode2",
|
|
"sourceGeoCountryName",
|
|
"sourceGeoPostalCode",
|
|
"sourceGeoCityName",
|
|
"sourceGeoTimeZone",
|
|
"sourceGeoRegionCode",
|
|
"sourceGeoContinentCode",
|
|
"sourceGeoCountryCode3",
|
|
"sourceASNAsOrg",
|
|
"sourceASN",
|
|
"destinationGeoLongitude",
|
|
"destinationGeoLatitude",
|
|
"destinationGeoRegionName",
|
|
"destinationGeoCountryCode2",
|
|
"destinationGeoCountryName",
|
|
"destinationGeoPostalCode",
|
|
"destinationGeoCityName",
|
|
"destinationGeoTimeZone",
|
|
"destinationGeoRegionCode",
|
|
"destinationGeoContinentCode",
|
|
"destinationGeoCountryCode3",
|
|
"destinationASNsOrg",
|
|
"destinationASN",
|
|
"destinationDnsDomain",
|
|
"reason"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
# Optional: log failures to see if any additional parsing is needed
|
|
# if "_grokparsefailure" in [tags] {
|
|
# file {
|
|
# path => "/var/log/logstash/grokparsefailure.log"
|
|
# }
|
|
# }
|
|
}
|
|
} |