@Hieveryone ,
I am trying to create a custom parser for Mambu logs but got stuck with an error "generic::unknown: pipeline.ParseLogEntry failed: LOG_PARSING_CBN_ERROR: "generic::invalid_argument: failed to convert raw output to events: failed to convert raw message 0: field \\"idm\\": index 0: recursive rawDataToProto failed: field \\"read_only_udm\\": index 0: recursive rawDataToProto failed: field \\"user\\": no descriptor found"
Sample Logs:
[
{
"occurred_at": "2024-11-07T05:51:11.132Z",
"response_code": 302,
"resource": "login",
"event_source": "UI",
"client_ip": "10.30.40.60",
"request_method": "POST",
"request_payload": "{}",
"resource_fragment": "/saml/login",
"request_uri": "/saml/login",
"user_agent": "Mozilla/5.0 (Windows NT 11.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36",
"response_payload": "",
"username": "john.doe@sampledomain.com"
},
{
"occurred_at": "2024-11-07T05:51:03.373Z",
"response_code": 302,
"resource": "login",
"event_source": "UI",
"client_ip": "10.30.40.60",
"request_method": "POST",
"request_payload": "{}",
"resource_fragment": "/saml/login",
"request_uri": "/saml/login",
"user_agent": "Mozilla/5.0 (Windows NT 11.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36",
"response_payload": "",
"username": "john.doe@sampledomain.com"
},
{
"occurred_at": "2024-11-07T05:50:39.913Z",
"response_code": 302,
"resource": "login",
"event_source": "UI",
"client_ip": "10.30.40.60",
"request_method": "POST",
"request_payload": "{\\"RelayState\\":[\\"https://service.sample.com/saml/login\\"]}",
"resource_fragment": "/saml/login",
"request_uri": "/saml/login",
"user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 11.6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36",
"response_payload": "",
"username": "jane.doe@sampledomain.com"
},
{
"occurred_at": "2024-11-07T05:50:33.135Z",
"response_code": 401,
"resource": "login",
"event_source": "UI",
"client_ip": "10.30.40.60",
"request_method": "POST",
"request_payload": "{\\"username\\":[\\"john.doe\\"],\\"loginType\\":[\\"onlineLogin\\"]}",
"resource_fragment": "/servlet/login",
"request_uri": "/servlet/login",
"user_agent": "Mozilla/5.0 (Windows NT 11.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36",
"response_payload": "FAIL",
"username": ""
},
{
"occurred_at": "2024-11-07T05:50:06.470Z",
"response_code": 429,
"resource": "login",
"event_source": "UI",
"client_ip": "10.30.40.60",
"request_method": "POST",
"request_payload": "{\\"username\\":[\\"john.doe\\"],\\"loginType\\":[\\"onlineLogin\\"]}",
"resource_fragment": "/servlet/login",
"request_uri": "/servlet/login",
"user_agent": "Mozilla/5.0 (Windows NT 11.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36",
"response_payload": "CAPTCHA_REQUIRED",
"username": ""
},
{
"occurred_at": "2024-11-07T05:48:51.489Z",
"response_code": 401,
"resource": "login",
"event_source": "UI",
"client_ip": "10.30.40.60",
"request_method": "POST",
"request_payload": "{\\"username\\":[\\"john.doe\\"],\\"loginType\\":[\\"onlineLogin\\"]}",
"resource_fragment": "/servlet/login",
"request_uri": "/servlet/login",
"user_agent": "Mozilla/5.0 (Windows NT 11.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36",
"response_payload": "FAIL",
"username": ""
},
{
"occurred_at": "2024-11-07T05:48:34.643Z",
"response_code": 429,
"resource": "login",
"event_source": "UI",
"client_ip": "10.30.40.60",
"request_method": "POST",
"request_payload": "{\\"username\\":[\\"john.doe\\"],\\"loginType\\":[\\"onlineLogin\\"]}",
"resource_fragment": "/servlet/login",
"request_uri": "/servlet/login",
"user_agent": "Mozilla/5.0 (Windows NT 11.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36",
"response_payload": "CAPTCHA_REQUIRED",
"username": ""
},
{
"occurred_at": "2024-11-07T05:48:01.810Z",
"response_code": 302,
"resource": "login",
"event_source": "UI",
"client_ip": "10.30.40.60",
"request_method": "POST",
"request_payload": "{}",
"resource_fragment": "/saml/login",
"request_uri": "/saml/login",
"user_agent": "Mozilla/5.0 (Windows NT 11.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36",
"response_payload": "",
"username": "john.doe@sampledomain.com"
}
]
Parser Code:
filter {
# Wrap the message in JSON format
mutate {
replace => {
"message" => "{\\"records\\":%{message}}"
}
}
# Parse the JSON message
json {
source => "message"
array_function => "split_columns"
on_error => "not_json"
}
# Drop malformed messages
if [not_json] {
drop {
tag => "TAG_MALFORMED_MESSAGE"
}
}
# Process valid JSON messages
if ![not_json] {
for index, data in records {
# Set metadata fields
mutate {
replace => {
"event.idm.read_only_udm.metadata.vendor_name" => "Mambu"
"event.idm.read_only_udm.metadata.product_name" => "Mambu"
"event.idm.read_only_udm.metadata.event_type" => "GENERIC_EVENT"
}
}
# Map event source
mutate {
replace => {
"event.idm.read_only_udm.metadata.event_source" => "%{data.event_source}"
}
on_error => "invalid_event_source"
}
# Set principal IP address
mutate {
replace => {
"event.idm.read_only_udm.principal.ip" => "%{data.client_ip}"
}
on_error => "invalid_principal_ip"
}
# Map HTTP method and user agent
mutate {
replace => {
"event.idm.read_only_udm.network.http.method" => "%{data.request_method}"
"event.idm.read_only_udm.network.http.user_agent" => "%{data.user_agent}"
"event.idm.read_only_udm.network.http.request_payload" => "%{data.request_payload}"
}
on_error => "invalid_http_fields"
}
# Convert response code to string and handle errors
mutate {
convert => {
"data.response_code" => "string"
}
on_error => "invalid_response_code_conversion"
}
if ![invalid_response_code_conversion] {
mutate {
replace => {
"event.idm.read_only_udm.network.http.response_code" => "%{data.response_code}"
}
on_error => "invalid_response_code"
}
}
# Map target URL and resource
mutate {
replace => {
"event.idm.read_only_udm.target.url" => "%{data.request_uri}"
"event.idm.read_only_udm.target.resource" => "%{data.resource}"
}
on_error => "invalid_target_fields"
}
# Handle resource fragment label
mutate {
replace => {
"resource_fragment_label.key" => "resource_fragment"
"resource_fragment_label.value" => "%{data.resource_fragment}"
}
on_error => "invalid_resource_fragment_value"
}
if ![invalid_resource_fragment_value] {
mutate {
merge => {
"event.idm.read_only_udm.principal.resource.attribute.labels" => "resource_fragment_label"
}
on_error => "resource_fragment_label_merge_failed"
}
}
# Map user ID
mutate {
replace => {
"event.idm.read_only_udm.user.user_id" => "%{data.username}"
}
on_error => "invalid_user_id"
}
# Output the event
statedump {}
mutate {
merge => {
"@output" => "event"
}
on_error => "output_merge_failed"
}
}
}
}