infra/roles/elk/templates/pipeline-logstash.yml.j2

# ansible

input {
  #tcp {
  #  port => {{ logstash_tcp }}
  #  codec => json
  #}
  #udp {
  #  port => {{ logstash_udp }}
  #  codec => json
  #}
  #http {
  #  host => "0.0.0.0"
  #  port => {{ logstash_http }}
  #  codec => json
  #}
  beats {
    port => {{ logstash_beats }}
    ecs_compatibility => disabled
    tags => [ "beats" ]
    type => beats
    ssl => true
    ssl_certificate => "/etc/letsencrypt/live/{{ domain }}/fullchain.pem"
    ssl_key => "/etc/letsencrypt/live/{{ domain }}/privkey.pem"
  }
  syslog {
    port => {{ logstash_syslog_port }}
    type => "syslog"
    tags => [ "syslog" ]
    #codec => "plain" #"json"
  }
}

filter {
  if "jellyfin" in [tags] {
    grok {
      match => [ "message", "\[%{LOGLEVEL:level}\] - %{DATA:logger_name} - %{GREEDYDATA:message}" ]
      overwrite => [ "message" ]
    }
  }
  if "arrformat" in [tags] {
    grok {
      match => [ "message", "%{DATA}\|%{LOGLEVEL:level}\|%{DATA:logger_name}\|%{GREEDYDATA:message}" ]
      overwrite => [ "message" ]
    }
    mutate {
       uppercase => [ "level" ]
    }
  }
  if "jackett" in [tags] {
    grok {
      match => [ "message", "%{DATE} %{TIME} %{LOGLEVEL:level} %{DATA:logger_name} %{GREEDYDATA:message}" ]
      overwrite => [ "message" ]
    }
    mutate {
       uppercase => [ "level" ]
    }
  }
  if "bazarr" in [tags] {
    grok {
      match => [ "message", "%{DATA}\|%{LOGLEVEL:level}%{SPACE}\|%{DATA:logger_name}%{SPACE}\|BAZARR %{GREEDYDATA:message}\|" ]
      overwrite => [ "message" ]
    }
  }
  if "mautrix-bridge" in [tags] {
    grok {
      match => ["message", "\[%{LOGLEVEL:level}\] - %{DATA:namespace} - %{GREEDYDATA:message}"]
      overwrite => [ "message" ]
    }
  }
  if "transmission" in [tags] {
    grok {
      match => ["message", "\[%{DATA:log_time_rm}\] %{GREEDYDATA:message} \(%{DATA:logger_name}:%{NUMBER:line_no_rm}\)"]
      overwrite => [ "message" ]
    }
    mutate {
      remove_field => [ "log_time_rm", "line_no_rm" ]
    }
  }
  if "gitea" in [tags] {
    grok {
      match => ["message", "%{DATA:logger_name}\:%{NUMBER:lineno}\:%{DATA:function} \[%{LOGLEVEL:level}\] %{GREEDYDATA:message}"]
      overwrite => [ "message" ]
    }
  }
  if "unbound" in [tags] {
    grok {
      match => ["message", "\[%{NUMBER}\] unbound\[%{DATA}\:%{DATA}\] %{LOGLEVEL:level}\: %{GREEDYDATA:message}"]
      overwrite => [ "message" ]
    }
  }
  if "nextcloud" in [tags] {
    mutate {
      remove_field => [ "time",  "exception.Trace", "exception.Previous" ]
      rename => { "level" => "level_num" }
    }
    translate {
      dictionary => {
        "0" => "DEBUG"
        "1" => "INFO"
        "2" => "WARN"
        "3" => "ERROR"
        "4" => "FATAL"
      }
      source => "[level_num]"
      target => "[level]"
    }
    mutate {
      remove_field => [ "level_num" ]
    }
  }
  if "authelia" in [tags] {
    mutate {
      remove_field => [ "time" ]
      rename => {"msg" => "message"}
    }
    grok {
      match => {
        "message" => [
          "username=%{WORD:auth_request_user} groups=%{GREEDYDATA:auth_request_group} ip=%{IPV4:auth_request_ip} and object %{URI:auth_request_object_url} \(method %{WORD:auth_request_method}\)",
          "user %{WORD:auth_request_user}",
          "user \'%{WORD:auth_request_user}\'"
        ]
      }
    }
  }
  if "nginx" in [tags] {
    mutate {
      remove_field => [ "time_local" ]
    }
  }
  {# if "nginx-error" in [tags] {
   #   grok {
   #     match => {
   #       "message" => [
   #         "%{DATA}\[%{LOGLEVEL:level}\] %{DATA}\: \*%{NUMBER} %{GREEDYDATA:message}",
   #         "client: %{IPV4:remote_addr}",
   #         "server: %{DATA:server_name}",
   #         "request: \"%{DATA:request}\"",
   #         "upstream: \"%{DATA:upstream_addr}\"",
   #         "host: \"%{DATA:http_host}\"",
   #         "referrer: \"%{DATA:http_referrer}\""
   #       ]
   #     }
   #     overwrite => [ "message" ]
   #   }
   #   mutate {
   #     uppercase => [ "level" ]
   #   }
   # } #}

 if [event][module] == "mysql" {
    mutate {
        add_tag => [ "mariadb" ]
    }
  }

  if [type] == "syslog" {
    mutate {
      rename => { "logsource" => "hostname" }
    }
  }
}

output {
  if [type] == "syslog" {
    elasticsearch {
      hosts => "{{ bridgewithdns.elasticsearch }}:9200"
      user => "{{ elk_users.elastic.user }}"
      password => "{{ elk_users.elastic.passwd }}"
      ecs_compatibility => disabled
      index => "syslog-%{+YYYY.MM.dd}"
    }
  }
  if [type] == "beats" {
    elasticsearch {
      hosts => "{{ bridgewithdns.elasticsearch }}:9200"
      user => "{{ elk_users.elastic.user }}"
      password => "{{ elk_users.elastic.passwd }}"
      ecs_compatibility => disabled
      index => "beats-%{+YYYY.MM.dd}"
    }
  }
}


  {# if "jellyfin_login" in [tags] {
   #   file {
   #     path => "/usr/share/logstash/logs/jellyfin_login-%{+YYYY-MM-dd}.json"
   #   }
   # }
   # file {
   #   path => "/usr/share/logstash/logs/logfile-%{+YYYY-MM-dd}.log"
   # } #}

    # match multiple:
    #
    # filter {
    #   grok {
    #     match => {
    #       "message" => [
    #         "Duration: %{NUMBER:duration}",
    #         "Speed: %{NUMBER:speed}"
    #       ]
    #     }
    #   }
    # }