Logstash: Filter Plugins

Logstash Filter Plugins with Most Common Filtering Methods

csv

• parsing csv format

filter {
  csv {
    column => ["col1","col2"]
    column => {"col3" => "integer", "col4"=>"boolean"}
    type => "syslog"
  }
}

date

• parsing date format as ISO8601 format
• ex) “Jul 02 13:19:01” => “MMM dd HH:mm:ss”

filter {
  date {
    match => ["logdate","MMM dd HH:mm:ss"]
    target => "logdate_modified"
  }
}

drop

• filtering

filter {
  drop {
    if[loglevel] == "debug" {
      drop {}
    }
  }
}

grok

• parsing arbitrary and unstructured text
• Grok works based on patterns
• Syntax for grok pattern is %{SYNTAX:SEMANTIC}
• Custom pattern can also be added

filter {
  grok {
    match => {"message" => "%{IP:client} %{WORD:method} %{URIPATHPARAM:request} %{NUMBER:duration}"
  }
}

• custom pattern example

pattern in ~/patterns as (?<message_id>[0-9A-F]{10,11}

filter {
  grok {
    patterns_dir => ["~/patterns"]
    match => {"message" => "%{SYSLOGBASE} %{POSTFIXQUEUEID: queue_id}: %{GREEDYDATA:syslog_message}"}
  }
}

mutate

• transforming

filter {
  mutate {
    add_field => {
      "foo_%{somefield}" => "Hello world, from %{host}"
      "new_field" => "new_static_value"
    }
    convert => {fieldname" => "integer"}
    gsub => ["fieldname", "/", "_"]
    lowercase => ["fieldname"]
    remove_field => ["foo_%{somefield}", "my_etraneous_field"]
    rename => {"HOSTORIP" => "client_ip"}
    replace => {"message" => "%{source_host}: My new message"}
  }
}

range

• parsing range

filter {
  range {
    ranges => ["req_time",0,10,"tag:short",
               "req_time",11,100,"tag:medium",
               "req_time",101,1000,"tag:long",
               "req_time",1001,10000,"drop"]
  }
}

sleep

• taking break

filter {
  sleep {
    time => "1"
    every => 10
  }
}

translate

• translate according to yam file

filter {
  translate {
    dictionary => ["100", "One Hundred"]
    field => ["ping", "pong"]
  }
}