Skip to main content

I'm trying to extract ip and hostname from a nested json. There are multiple ips and hostnames depending on the alert category. How can we extract and assign all the IPs and hostnames  it to target.ip  and target.hostname?

{
"target": {
    "total_count": 2,
    "data": [
      {
        "device": [
          {
            "value": "10.10.10.10",
            "type": "ip"
          },
          {
            "value": "abc",
            "type": "hostname"
          }
        ],
        "type": "endpoint",
      },
      {
        "device": [
          {
            "value": "11.11.11.11",
            "type": "ip"
          },
          {
            "value": "z44",
            "type": "hostname"
          }
        ],
    "type": "endpoint",
    }
  ],  
}

}

Parser syntax documentation doesn't have references for this type of nested json data where key value pairs are separated.

I tokenized the IPs/Hostnames separately. It is doable but a bit complex when the keys are double-repeated like this.


However target.hostname has a single value unlike the target.IP . Do you need all the 2 IPs/Hostnames to be in a single event so should I go ahead and map the 2 hostnames to another repeated field like target.labels ? or is it ok to split the log entry into multiple separate events ?


 


 


filter {
json {
source => "message"
array_function => "split_columns"
}

mutate {
replace => {
"ipDedupRegex" => " "
}
}


mutate {
replace => {
"hostDedupRegex" => " "
}
}



for k,v in target.data map {




for k2,v2 in v.device map {


if [v2][type] == "ip" {



if [v2][value] !~ ipDedupRegex {



if k=="0" {

mutate {
replace => {
"ipDedupRegex" => "%{v2.value}"
}
}




}

else {

mutate {
replace => {
"ipDedupRegex" => "%{ipDedupRegex}|%{v2.value}"
}
}

}


mutate {
merge => {
"ipList_" => "v2.value"
}
}

}




statedump {}



}

#if [v2][type] == "hostname" {
#mutate {
# replace => {
# "hostname_" => "%{v2.value}"
# }
#}
#}

if [v2][type] == "hostname" {



if [v2][value] !~ ipDedupRegex {



if k=="0" {

mutate {
replace => {
"hostDedupRegex" => "%{v2.value}"
}
}




}

else {

mutate {
replace => {
"hostDedupRegex" => "%{hostDedupRegex}|%{v2.value}"
}
}

}


mutate {
merge => {
"hostList_" => "v2.value"
}
}

}




statedump {}



}



}
}
}

 

@AbdElHafez could you just use indexing for the IP's or hostnames like


target.ip[0]


target.ip[1]


 

I tokenized the IPs/Hostnames separately. It is doable but a bit complex when the keys are double-repeated like this.


However target.hostname has a single value unlike the target.IP . Do you need all the 2 IPs/Hostnames to be in a single event so should I go ahead and map the 2 hostnames to another repeated field like target.labels ? or is it ok to split the log entry into multiple separate events ?


 


 


filter {
json {
source => "message"
array_function => "split_columns"
}

mutate {
replace => {
"ipDedupRegex" => " "
}
}


mutate {
replace => {
"hostDedupRegex" => " "
}
}



for k,v in target.data map {




for k2,v2 in v.device map {


if [v2][type] == "ip" {



if [v2][value] !~ ipDedupRegex {



if k=="0" {

mutate {
replace => {
"ipDedupRegex" => "%{v2.value}"
}
}




}

else {

mutate {
replace => {
"ipDedupRegex" => "%{ipDedupRegex}|%{v2.value}"
}
}

}


mutate {
merge => {
"ipList_" => "v2.value"
}
}

}




statedump {}



}

#if [v2][type] == "hostname" {
#mutate {
# replace => {
# "hostname_" => "%{v2.value}"
# }
#}
#}

if [v2][type] == "hostname" {



if [v2][value] !~ ipDedupRegex {



if k=="0" {

mutate {
replace => {
"hostDedupRegex" => "%{v2.value}"
}
}




}

else {

mutate {
replace => {
"hostDedupRegex" => "%{hostDedupRegex}|%{v2.value}"
}
}

}


mutate {
merge => {
"hostList_" => "v2.value"
}
}

}




statedump {}



}



}
}
}

 


i.e. This mapping below is more consistent, however looking at the type of logs you are trying to ingest I feel it would fit more in the asset context logs not the event ones ;


 


 


filter {
json { source => "message" array_function => "split_columns"}
mutate {convert => {"target.total_count" => "string"}}
mutate {replace => {"count" => "%{target.total_count}"}}
for k,v in target.data map {
mutate {replace => {"event1" => ""}}
mutate {replace => {"event1.idm.read_only_udm.metadata.event_type" => "GENERIC_EVENT"}}

for k2,v2 in v.device map {
if [v2][type] == "ip" {
mutate {replace => {"ip" => "%{v2.value}"}}
mutate {merge => {"event1.idm.read_only_udm.target.ip" => "ip"}}
}
if [v2][type] == "hostname"{
mutate {replace => {"host" => "%{v2.value}"}}
mutate {rename => {"v2.value" => "event1.idm.read_only_udm.target.hostname"}}
}
}


mutate {
merge => {
"@output" => "event1"
}
}

statedump {label=>"outerLoop"}


}
statedump {}
}

 



 


 

@AbdElHafez could you just use indexing for the IP's or hostnames like


target.ip[0]


target.ip[1]


 


I know but the hostname is unique so it will have to go in the labels section eventually.


I think the 2nd version I added just now is the better version if the data is not ingested within an asset context.

Hi @AbdElHafez Many Thanks for the detailed post. We are getting alerts from a external system and this particular part in json contains the ip/hostname of the machines which are affected. It can range from 1 to 30+. So, we thought to map it to "target" namespace. 

When I tested with the config you have provided, it's extracting the same way as you provided in your screenshot. Since it is having timestamp assigned for each extraction. When I validate the parser, it is giving 1000+ events against ingested events of 100 due to individual time stamp assignment. 

So, I simplifed like below, 
 

for k,v in target.data map {
for k1,v2 in v.device map {
if [v2][type] == "ip" {
mutate {replace => {"ip" => "%{v2.value}"}}
mutate {merge => {"event1.idm.read_only_udm.target.ip" => "ip"}}
}
if [v2][type] == "hostname"{
mutate {replace => {"host" => "%{v2.value}"}}
mutate {rename => {"v2.value" => "event1.idm.read_only_udm.target.hostname"}}
}
}
}

It's extracting the IP's with target.ip[0]/target.ip[1] and so on but the hostname is getting failed to parse.



Hi @AbdElHafez Many Thanks for the detailed post. We are getting alerts from a external system and this particular part in json contains the ip/hostname of the machines which are affected. It can range from 1 to 30+. So, we thought to map it to "target" namespace. 

When I tested with the config you have provided, it's extracting the same way as you provided in your screenshot. Since it is having timestamp assigned for each extraction. When I validate the parser, it is giving 1000+ events against ingested events of 100 due to individual time stamp assignment. 

So, I simplifed like below, 
 

for k,v in target.data map {
for k1,v2 in v.device map {
if [v2][type] == "ip" {
mutate {replace => {"ip" => "%{v2.value}"}}
mutate {merge => {"event1.idm.read_only_udm.target.ip" => "ip"}}
}
if [v2][type] == "hostname"{
mutate {replace => {"host" => "%{v2.value}"}}
mutate {rename => {"v2.value" => "event1.idm.read_only_udm.target.hostname"}}
}
}
}

It's extracting the IP's with target.ip[0]/target.ip[1] and so on but the hostname is getting failed to parse.




That is because target.hostname is a unique field unlike target.ip which is repeated that is why I changed the parser to generate an event per host.


ou would need to pick a different field for the hostnames like in the about.labels for example or something similar to parse them. If you could pick which fields to be used for the hostnames then I could change the mapping back to 1 event per entry.

Using about.labels would be a correct call I guess @AbdElHafez 

Using about.labels would be a correct call I guess @AbdElHafez 


Hi @Aswin_Asokan,
Note that 'labels fields for UDM nouns' (which includes 'about.labels') was deprecated on November 29, 2024[1]. It is recommended to use 'additional.fields'.

[1] - https://cloud.google.com/chronicle/docs/deprecations

Kind Regards,

Ayman

Using about.labels would be a correct call I guess @AbdElHafez 


@Aswin_Asokan Could you give some more context about the type of the datasource and use case for the events ? As @AymanC indicated true it seems the about labels may not be a sustainable choice in the future. But I am trying to figure out if these logs are better off as asset context that could be used for enriching other events, or  UDM events ? and figure out a proper mapping for the hostnames field.

@AbdElHafez We are looking at the XDR incidents

https://conure.us.security.cisco.com/index.html#/v2/get_v2_incident__incident_id__summary 

@AbdElHafez We are looking at the XDR incidents

https://conure.us.security.cisco.com/index.html#/v2/get_v2_incident__incident_id__summary 


@Aswin_Asokan In that case the security_result field is probably more suitable.


Please try this version ;


 


 


filter {
json { source => "message" array_function => "split_columns" }

mutate {replace => {"ipDedupRegex" => " "}}
mutate {replace => {"hostDedupRegex" => " "}}
mutate {replace => {"event1.idm.read_only_udm.metadata.event_type" => "GENERIC_EVENT"}}
mutate {replace => {"event1.idm.read_only_udm.metadata.vendor_name" => "Cisco"}}
mutate {replace => {"event1.idm.read_only_udm.metadata.product_name" => "XDR"}}

for k,v in target.data map {
for k2,v2 in v.device map {
if [v2][type] == "ip" {
if [v2][value] !~ ipDedupRegex {
if k=="0" {
mutate {replace => {"ipDedupRegex" => "%{v2.value}"}}
}
else {
mutate {replace => {"ipDedupRegex" => "%{ipDedupRegex}|%{v2.value}"}}
}
mutate {merge => {"ipList_" => "v2.value"}}
}

mutate {replace => {"ip_" => "%{v2.value}"}}
mutate {merge => {"security_result_.about.ip" => "ip_"}}
#mutate {rename => {"ip_list" => "zz"}}
#statedump {}
}
else if [v2][type] == "hostname" {
if [v2][value] !~ hostDedupRegex {
if k=="0" {
mutate {replace => {"hostDedupRegex" => "%{v2.value}"}}
}
else {
mutate {replace => {"hostDedupRegex" => "%{hostDedupRegex}|%{v2.value}"}}
}
mutate {merge => {"hostList_" => "v2.value"}}
}
mutate {replace => {"security_result_.about.hostname" => "%{v2.value}"}}
#mutate {rename => {"host_" => "about_.hostname"}}
#statedump {}
}

#mutate {rename => {"about" => "x.about"}}
#mutate {merge => {"about_" => "about"}}
#mutate {rename => {"about_" => "x.about_"}}
}
#mutate {rename => {"about" => "x"}}
mutate {merge => {"event1.idm.read_only_udm.security_result" => "security_result_"}}
#mutate {rename => {"security_result" => "event1.idm.read_only_udm.security_result"}}
mutate {replace => {"security_result_" => ""}}
#mutate {replace => {"about_" => "about"}}

}
#mutate {merge => {"about_" => "about"}}
statedump {}
mutate {
merge => {
"@output" => "event1"
}
}

#SecurityResult
}

 


 


 



 


I left some unused tokens "hostList_" and "ipList_" in case you needed them separate.

Reply


Terms & ConditionsCookie settings
Privacy Policy Terms of Service Gen AI Policy Community Guidelines Cookie Settings

© 2025 Google. All rights reserved.