Skip to main content

Ingesting data from ELT tools

  • July 25, 2024
  • 2 replies
  • 4 views
C
Forum|alt.badge.img

Is it possible to ingest data from sources such as Fivetran, CloudQuery, Steampipe into Chronicle? or alternatively, query a data lake via federated query?

    N

    2 replies

    AbdElHafez
    Staff
    Forum|alt.badge.img+12
    • AbdElHafez
    • Staff
    • July 25, 2024

    Hi @cahehay553 ,
    The options I know are ;
    1. You could configure your ETL tools to push the logs to GCP/S3 buckets -among other options- that Chronicle could ingest from, the list of Chronicle-supported feeds are listed here.
    https://cloud.google.com/chronicle/docs/administration/feed-management
    2. For the datalake query, technically as long as you could periodically export the federated query results to a GCP/S3 bucket -or any other supported feed source from above- then you could ingest the logs from this intermediate feed source to Chronicle, however the latency of the data querying, processing and transmission must be taken into consideration, plus the format in the export must be supported per data source. 

    Thanks,

    AbdElHafez
    Staff
    Forum|alt.badge.img+12
    • AbdElHafez
    • Staff
    • July 26, 2024

    Hi @cahehay553 ,
    The options I know are ;
    1. You could configure your ETL tools to push the logs to GCP/S3 buckets -among other options- that Chronicle could ingest from, the list of Chronicle-supported feeds are listed here.
    https://cloud.google.com/chronicle/docs/administration/feed-management
    2. For the datalake query, technically as long as you could periodically export the federated query results to a GCP/S3 bucket -or any other supported feed source from above- then you could ingest the logs from this intermediate feed source to Chronicle, however the latency of the data querying, processing and transmission must be taken into consideration, plus the format in the export must be supported per data source. 

    Thanks,


    Please try this parser, it should parse your fields like this ;

     



    # Product: HAPROXY
    # Category:  Load balancing
    # Supported Format: SYSLOG
    # Reference: N/A
    # Last Updated: 2024-07-22
     
     
    filter {
      mutate {
        replace => {
      "syslog_timestamp" => ""
      "process" => ""
      "pid" => ""
      "client_ip" => ""
      "client_port" => ""
      "timestamp" => ""
      "frontend_name" => ""
      "backend_name" => ""
      "server_name" => ""
      "time_request" => ""
      "time_queue" => ""
      "time_backend_connect" => ""
      "time_backend_response" => ""
      "time_duration" => ""
      "http_status_code" => ""
      #"bytes_read" => ""
      #"captured_request_cookie" => ""
      #"captured_response_cookie" => ""
      #"termination_state" => ""
      #"actconn" => ""
      #"feconn" => ""
      #"beconn" => ""
      #"srvconn" => ""
      #"retries" => ""
      #"srv_queue" => ""
      #"backend_queue" => ""
      "http_verb" => ""
      #"http_proto" => ""
      #"http_host" => ""
      "http_request" => ""
      "http_version" => ""
        }
      }
      grok {
        match => {
          "message" => ["<%{INT}>%{SYSLOGTIMESTAMP:syslog_timestamp} %{GREEDYDATA:process}\\\\[%{INT:pid}\\\\]: %{IP:client_ip}:%{INT:client_port} \\\\[%{GREEDYDATA:timestamp}\\\\] %{NOTSPACE:frontend_name} %{NOTSPACE:backend_name}\\/%{NOTSPACE:server_name} %{INT:time_request}\\/%{INT:time_queue}\\/%{INT:time_backend_connect}\\/%{INT:time_backend_response}\\/%{NOTSPACE:time_duration} %{INT:http_status_code}%{GREEDYDATA} \\"%{WORD:http_verb} %{GREEDYDATA:http_request} HTTP/%{NUMBER:http_version}\\""
          ]
        }
        overwrite => ["syslog_timestamp","process", "pid", "client_ip", "client_port","timestamp"
        ,"frontend_name","backend_name","server_name","time_request","time_queue","time_backend_connect",
        "time_backend_response","time_duration","http_status_code","http_verb","http_request","http_version"]
        on_error => "match_error"
     

      }
      if [match_error] {
        drop{
          tag => "TAG_NO_SECURITY_VALUE"
        }
      }


      if [timestamp] != "" {
        date {
          match => ["timestamp", "yyyy MMM dd HH:mm:ss.SSS"]
          on_error => "time_stamp_wrong_format"
        }
      }


        mutate {
            merge => {
                "event.idm.read_only_udm.principal.ip" => "client_ip"

            }
        }




      if [http_status_code] != "" {
        mutate {
          replace => {
            "event.idm.read_only_udm.network.http.response_code" => "%{http_status_code}"
            "event.idm.read_only_udm.principal.port" => "%{client_port}"
          }
        }
      }


        mutate {
        convert => {
            "event.idm.read_only_udm.network.http.response_code" => "integer"
            "event.idm.read_only_udm.principal.port" => "integer"
        }
        }


      mutate {
        replace => {
        "event.idm.read_only_udm.metadata.event_type" => "NETWORK_HTTP"
        "event.idm.read_only_udm.target.application" => "%{process}"
        "event.idm.read_only_udm.network.http.method" => "%{http_verb}"
        "event.idm.read_only_udm.target.url" => "%{http_request}"
        "event.idm.read_only_udm.target.hostname" => "%{frontend_name}"
        "event.idm.read_only_udm.target.process.pid" => "%{pid}"
        "event.idm.read_only_udm.network.application_protocol" => "HTTP"
        "event.idm.read_only_udm.metadata.product_version" => "HTTP/%{http_version}"
        "event.idm.read_only_udm.metadata.product_name" => "HAProxy"
        "event.idm.read_only_udm.metadata.vendor_name" => "HAProxy Enterprise"
        }
      }


       mutate {
        merge => {
          "@output" => "event"
        }
      }

       
      #statedump {label => "3"}


    }
    Terms & ConditionsCookie settingsAccessibility statement
    Privacy Policy Terms of Service Gen AI Policy Community Guidelines Cookie Settings

    © 2025 Google. All rights reserved.