No UDM events or entities were generated for the current parser configuration. ? Yet idm.read_only_udm filled?

Hi ​@JSpoorSonic,

It’s likely that you are not using the required minimum UDM fields for the NETWORK_CONNECTION event type. See the below reference:

https://docs.cloud.google.com/chronicle/docs/unified-data-model/udm-usage#network_connection

Kind Regards

Ayman

Ok, I added some more code, and got added these UDMs:

"network": {            "ip_protocol": "UNKNOWN_IP_PROTOCOL",            "received_bytes": "2296840",            "sent_bytes": "26275625",            "session_id": "blr0connect04:691598de:00000000"          },

But still erroring out with the same no UDM. error

​@JSpoorSonic Could you share your parser code ?
I am pretty sure the final merge statement is not executed due to another unforeseen error.

# SonicWall SMA 1000 Parser Extension
# Author: J Spoor
# Version: 2.3 WIP

filter {
    # Check for logserver: [datetime] hostname
    grok {
      match => {"message" => "logserver: [[]%{DATA:bracket_timestamp}[]] %{HOSTNAME:hostname}"}
      on_error => "no_logserver_datetime"
    }

    if ![no_logserver_datetime] {
      # Extract and convert logserver: [datetime]  
      mutate {
        gsub => ["bracket_timestamp","/","-"]
      }
      grok {
        match => {"bracket_timestamp" => "(?<bracket_date_part>[^:]+):(?<bracket_time_part>.*)"}
      }  
      date {
        match => [ "bracket_timestamp", "dd-MMM-yyyy:HH:mm:ss Z" ]
        target => "event1.idm.read_only_udm.metadata.event_timestamp"
        time_precision => "microsecond"
        on_error => "failed_datematch"
      }
    }

    # Remove @(SonicWall Connect) from user
    grok {
        match => {"message" => "User='\\(\\s*%{EMAILADDRESS:user}\\s*\\)@"}
        on_error => "no_user_found"
    }
    if ![no_user_found] {
          mutate {
            replace => {
                "event1.idm.read_only_udm.principal.user.userid" => "%{user}"
            }
        }
    }else { mutate { replace => { "event1.idm.read_only_udm.principal.user.userid" => "UKNOWN" } } }

    # Extract Event Type
    grok {
        match => {
            "message" => "EventMessage: %{DATA:event_type} - User="
        }
        on_error => "no_event_type_found"
    }
    if ![no_event_type_found] {
          mutate {
            replace => {
                "event1.idm.read_only_udm.metadata.description" => "%{event_type}"
            }
          }
        if [event_type] == "Resource Access" {
          mutate {
            replace => {
                "event1.idm.read_only_udm.metadata.event_type" => "USER_RESOURCE_ACCESS"
            }
          }
        } else if [event_type] =~ "Session Start Success" {
          mutate {
            replace => {
                "event1.idm.read_only_udm.metadata.event_type" => "USER_LOGIN"
            }
          }
        } else if [event_type] =~ "Session End" {
          mutate {
            replace => {
                "event1.idm.read_only_udm.metadata.event_type" => "USER_LOGOUT"
            }
          }
        } else {
          mutate {
            replace => {
                "event1.idm.read_only_udm.metadata.event_type" => "GENERIC_EVENT"
            }
          }
        }
    }

    grok {
        match => {
            "message" => "Allowed='%{INT:allowed}"
        }
        on_error => "no_action_found"
    }
    grok {
      match => { "message" => "Rule Info='%{DATA:rule_info}'" }
      on_error => "no_rule_info"
    }

    if ![no_action_found] {
      if [allowed] == "1" {
        mutate {
          replace => { "result_action" => "ALLOW" }
        }
      }  else if [allowed] == "0" {
         mutate {
          replace => { "result_action" => "BLOCK" }
         }
      } else { 
        mutate { 
          replace => { "result_action" => "UNKNOWN_ACTION" } 
        } 
      }    
      mutate {
        merge => {"security_result.action" => "result_action"}
      }      
      if ![no_rule_info] {
        mutate {
          replace => {"security_result.rule_name" => "%{rule_info}"}
        }
      }
      mutate {
        merge => {
          "event1.idm.read_only_udm.security_result" => "security_result"
        }
      }
    }

    grok {
      match => {
        "message" => "Full Destination='%{IP:dest_ip}:%{INT:dest_port}'"
      }
      on_error => "no_full_destination"
    }
    if ![no_full_destination] {
      if [dest_ip] != "" {
        mutate {
          merge => {"event1.idm.read_only_udm.target.ip" => "dest_ip"}
        }
      }
      if [dest_port] !="" {
        mutate {
          replace => {"event1.idm.read_only_udm.target.port" => "%{dest_port}"}
        }
        mutate {
          convert => {"event1.idm.read_only_udm.target.port" => "integer"}
        }
      }
    }

# Unparsed Logs
  # Unparsed due to :ffff: before IPv4

  grok {
    # Escape bracket [ with [[] and bracket ] with []]
    match => { "message" => "Src='[[]::ffff:%{IP:src_ip}[]]:%{INT:src_port}'" }
    on_error => "no_unparsed_source"
  }

  if ![no_unparsed_source]{
     mutate {replace => {"event_type" => "NETWORK_CONNECTION"}}
     mutate {
        replace => {"event1.idm.read_only_udm.metadata.event_type" => "NETWORK_CONNECTION"}
     }
     mutate {
        replace => {"event1.idm.read_only_udm.network.ip_protocol" => "UNKNOWN_IP_PROTOCOL"}
     }
     mutate {
        merge => {"event1.idm.read_only_udm.principal.ip" => "src_ip"}
     }
     mutate {
        merge => {"event1.idm.read_only_udm.src.ip" => "src_ip"}
     }

     grok {
        match => {"message" => "Error='%{INT:log_error}"}
        on_error => "no_logerror"
     }
     if ![no_logerror]{
       if [log_error] == "0" {
         mutate {replace => {"logaction" => "ALLOW"}}
       } else if [log_error] == "1" {
         mutate {replace => {"logaction" => "BLOCK"}}
       } else {
        mutate {replace => {"logaction" => "UNKNOWN_ACTION"}}
       }

     } else {mutate {replace => {"logaction" => "UNKNOWN_ACTION"}}}
     mutate {
        merge => {"event1.idm.read_only_udm.security_result.action" => "logaction"}
     }

     grok {
        match => {"message" => "Dest='%{IP:dest_ip}:%{INT:dest_port}"}
        on_error => "no_dest_ip"
     }
     if ![no_dest_ip] {
        mutate {merge => {"event1.idm.read_only_udm.target.ip" => "dest_ip"}}
        mutate {replace => {"event1.idm.read_only_udm.target.port" => "%{dest_port}"}}
        mutate {convert => {"event1.idm.read_only_udm.target.port" => "integer"}}
     }
     if [hostname] != "" {
        mutate {
            replace => {"event1.idm.read_only_udm.principal.hostname" => "%{hostname}" }
        }
     }

     grok {
        match => {"message" => "SessionKey='%{DATA:log_sessionkey}'"}
        on_error => "no_session_key"
     }
     if ![no_session_key] {
        mutate {replace => {"event1.idm.read_only_udm.network.session_id" => "%{log_sessionkey}"}}
     }

     grok {
        match => {"message" => "SrcBytes='%{INT:srcbytes}' DstBytes='%{INT:dstbytes}'"}
        on_error => "no_bytes"
     }
     if ![no_bytes] {
       mutate {
         replace => {
            "event1.idm.read_only_udm.network.received_bytes" => "%{srcbytes}"
            "event1.idm.read_only_udm.network.sent_bytes" => "%{dstbytes}"}
       }
     }
  }

# Debug
#  mutate {
#    replace => {
#      "event1.idm.read_only_udm.metadata.description" => "%{extracted_timestamp}"
#    }
#  }
# End Debug

   mutate { merge => { "@output" => "event1"} }

#  statedump {
#    label => "Post Merge"
#  }

}

 

FYI, I have my statedump after the merge. And the statedump does function.

Another FYI, this is for a parser extension.

I think the underlying parser is not taking the log message proper?

​@JSpoorSonic Sorry I was ooo in the past 2 days. Yes the extension will not kick in if the main parser dropped or failed the initial message.

no worries! Your help was greately appreciated.