Is it possible to ingest data from sources such as Fivetran, CloudQuery, Steampipe into Chronicle? or alternatively, query a data lake via federated query?
Page 1 / 1
Hi @cahehay553 ,
The options I know are ;
1. You could configure your ETL tools to push the logs to GCP/S3 buckets -among other options- that Chronicle could ingest from, the list of Chronicle-supported feeds are listed here.
https://cloud.google.com/chronicle/docs/administration/feed-management
2. For the datalake query, technically as long as you could periodically export the federated query results to a GCP/S3 bucket -or any other supported feed source from above- then you could ingest the logs from this intermediate feed source to Chronicle, however the latency of the data querying, processing and transmission must be taken into consideration, plus the format in the export must be supported per data source.
Thanks,
Hi @cahehay553 ,
The options I know are ;
1. You could configure your ETL tools to push the logs to GCP/S3 buckets -among other options- that Chronicle could ingest from, the list of Chronicle-supported feeds are listed here.
https://cloud.google.com/chronicle/docs/administration/feed-management
2. For the datalake query, technically as long as you could periodically export the federated query results to a GCP/S3 bucket -or any other supported feed source from above- then you could ingest the logs from this intermediate feed source to Chronicle, however the latency of the data querying, processing and transmission must be taken into consideration, plus the format in the export must be supported per data source.
Thanks,
Please try this parser, it should parse your fields like this ;
# Product: HAPROXY
# Category: Load balancing
# Supported Format: SYSLOG
# Reference: N/A
# Last Updated: 2024-07-22
filter {
mutate {
replace => {
"syslog_timestamp" => ""
"process" => ""
"pid" => ""
"client_ip" => ""
"client_port" => ""
"timestamp" => ""
"frontend_name" => ""
"backend_name" => ""
"server_name" => ""
"time_request" => ""
"time_queue" => ""
"time_backend_connect" => ""
"time_backend_response" => ""
"time_duration" => ""
"http_status_code" => ""
#"bytes_read" => ""
#"captured_request_cookie" => ""
#"captured_response_cookie" => ""
#"termination_state" => ""
#"actconn" => ""
#"feconn" => ""
#"beconn" => ""
#"srvconn" => ""
#"retries" => ""
#"srv_queue" => ""
#"backend_queue" => ""
"http_verb" => ""
#"http_proto" => ""
#"http_host" => ""
"http_request" => ""
"http_version" => ""
}
}
grok {
match => {
"message" => ["<%{INT}>%{SYSLOGTIMESTAMP:syslog_timestamp} %{GREEDYDATA:process}\\\\[%{INT:pid}\\\\]: %{IP:client_ip}:%{INT:client_port} \\\\[%{GREEDYDATA:timestamp}\\\\] %{NOTSPACE:frontend_name} %{NOTSPACE:backend_name}\\/%{NOTSPACE:server_name} %{INT:time_request}\\/%{INT:time_queue}\\/%{INT:time_backend_connect}\\/%{INT:time_backend_response}\\/%{NOTSPACE:time_duration} %{INT:http_status_code}%{GREEDYDATA} \\"%{WORD:http_verb} %{GREEDYDATA:http_request} HTTP/%{NUMBER:http_version}\\""
]
}
overwrite => ["syslog_timestamp","process", "pid", "client_ip", "client_port","timestamp"
,"frontend_name","backend_name","server_name","time_request","time_queue","time_backend_connect",
"time_backend_response","time_duration","http_status_code","http_verb","http_request","http_version"]
on_error => "match_error"
}
if [match_error] {
drop{
tag => "TAG_NO_SECURITY_VALUE"
}
}
if [timestamp] != "" {
date {
match => ["timestamp", "yyyy MMM dd HH:mm:ss.SSS"]
on_error => "time_stamp_wrong_format"
}
}
mutate {
merge => {
"event.idm.read_only_udm.principal.ip" => "client_ip"
}
}
if [http_status_code] != "" {
mutate {
replace => {
"event.idm.read_only_udm.network.http.response_code" => "%{http_status_code}"
"event.idm.read_only_udm.principal.port" => "%{client_port}"
}
}
}
mutate {
convert => {
"event.idm.read_only_udm.network.http.response_code" => "integer"
"event.idm.read_only_udm.principal.port" => "integer"
}
}
mutate {
replace => {
"event.idm.read_only_udm.metadata.event_type" => "NETWORK_HTTP"
"event.idm.read_only_udm.target.application" => "%{process}"
"event.idm.read_only_udm.network.http.method" => "%{http_verb}"
"event.idm.read_only_udm.target.url" => "%{http_request}"
"event.idm.read_only_udm.target.hostname" => "%{frontend_name}"
"event.idm.read_only_udm.target.process.pid" => "%{pid}"
"event.idm.read_only_udm.network.application_protocol" => "HTTP"
"event.idm.read_only_udm.metadata.product_version" => "HTTP/%{http_version}"
"event.idm.read_only_udm.metadata.product_name" => "HAProxy"
"event.idm.read_only_udm.metadata.vendor_name" => "HAProxy Enterprise"
}
}
mutate {
merge => {
"@output" => "event"
}
}
#statedump {label => "3"}
}
Reply
Enter your E-mail address. We'll send you an e-mail with instructions to reset your password.