Logstash input plugin exec examples

Logstash is an awesome tool to grab not only lines from log files but also from tons of different sources. A very useful input plugin is exec. It can execute scripts or any other binary directly and capture the output to move it directly into Elasticsearch. Here are two examples where I am parsing arbitrary data, structurize it and put it in a remote Elasticsearch for further exploration with Kibana.

input {
  exec {
    command => '/home/pi/smaread'
    interval => 300
    type => 'solar'
  }
}

filter {
  if [message] != "" {
    csv {
      separator => ';'
      columns => ['Iac', 'Uac', 'Fac', 'Pac', 'Zac', 'Riso', 'dI', 'Upv-Ist', 'PPV', 'E-Total', 'h-Total', 'h-On', 'Netz-Ein', 'E-Total DC', 'unknown', 'Status', 'Fehler']
      remove_field => ['column18']
    }

    mutate {
      rename => { "Iac" => "iac" }
      rename => { "Uac" => "uac" }
      rename => { "Fac" => "fac" }
      rename => { "Pac" => "pac" }
      rename => { "Zac" => "zac" }
      rename => { "Riso" => "riso" }
      rename => { "dI" => "di" }
      rename => { "Upv-Ist" => "upv_ist" }
      rename => { "PPV" => "ppv" }
      rename => { "E-Total" => "e_total" }
      rename => { "h-Total" => "h_total" }
      rename => { "h-On" => "h_on" }
      rename => { "Netz-Ein" => "netz_ein" }
      rename => { "E-Total DC" => "e_total_dc" }
      rename => { "Status" => "status" }
      rename => { "Fehler" => "fehler" }

      convert => {
        'iac' => 'float'
        'uac' => 'float'
        'fac' => 'float'
        'pac' => 'float'
        'zac' => 'float'
        'riso' => 'float'
        'di' => 'float'
        'upv_ist' => 'float'
        'ppv' => 'float'
        'e_total' => 'float'
        'h_total' => 'float'
        'h_on' => 'float'
        'netz_ein' => 'float'
        'e_total_dc' => 'float'
        'unknown' => 'float'
        'status' => 'float'
        'fehler' => 'float'
      }
    }
  }
}

output {
  #test it with
  #stdout { codec => rubydebug }

  if [type] == 'solar' and [message] != "" {
    elasticsearch {
      hosts => ['es.yourdomain.com']
      user => 'user'
      password => 'password'
      index => "solar"
    }
  }
}



input {
  exec {
    command => '/home/pi/vpro -2 -C tipmod -x -d 20 /dev/ttyWetter'
    interval => 300
    type => 'weather'
  }
}

filter {
  kv {
    field_split => "\n"
    value_split => " = "
  }

  mutate {
    rename => { "rtBaroTrend" => "rtbarotrend" }
    rename => { "rtBaroTrendImg" => "rtbarotrendimg" }
    rename => { "rtBaroCurr" => "rtbarocurr" }
    rename => { "rtInsideTemp" => "rtinsidetemp" }
    rename => { "rtInsideHum" => "rtinsidehum" }
    rename => { "rtOutsideTemp" => "rtoutsidetemp" }
    rename => { "rtOutsideHum" => "rtoutsidehum" }
    rename => { "rtWindSpeed" => "rtwindspeed" }
    rename => { "rtWindAvgSpeed" => "rtwindavgspeed" }
    rename => { "rtWindDir" => "rtwinddir" }
    rename => { "rtWindDirRose" => "rtwinddirrose" }
    rename => { "rtWindChill" => "rtwindchill" }
    rename => { "rtHeatIndex" => "rtheatindex" }
    rename => { "rtRainRate" => "rtrainrate" }
    rename => { "rtIsRaining" => "rtisraining" }
    rename => { "rtUVIndex" => "rtuvindex" }
    rename => { "rtSolarRad" => "rtsolarrad" }
    rename => { "rtRainStorm" => "rtrainstorm" }
    rename => { "rtStormStartDate" => "rtstormstartdate" }
    rename => { "rtDayRain" => "rtdayrain" }
    rename => { "rtMonthRain" => "rtmonthrain" }
    rename => { "rtYearRain" => "rtyearrain" }
    rename => { "rtDayET" => "rtdayet" }
    rename => { "rtDayETmL" => "rtdayetml" }
    rename => { "rtMonthET" => "rtmonthet" }
    rename => { "rtYearET" => "rtyearet" }
    rename => { "rtXmitBattt" => "rtxmitbattt" }
    rename => { "rtBattVoltage" => "rtbattvoltage" }
    rename => { "rtForeIcon" => "rtforeicon" }
    rename => { "rtForeRule" => "rtforerule" }
    rename => { "rtForecast" => "rtforecast" }
    rename => { "rtSunrise" => "rtsunrise" }
    rename => { "rtSunset" => "rtsunset" }
    rename => { "rtCaptureTimestamp" => "rtcapturetimestamp" }
    convert => {
      "rtbarocurr" => "float"
      "rtinsidetemp" => "float"
      "rtoutsidetemp" => "float"
      "rtwindchill" => "float"
      "rtheatindex" => "float"
      "rtrainrate" => "float"
      "rtrainstorm" => "float"
      "rtdayrain" => "float"
      "rtmonthrain" => "float"
      "rtyearrain" => "float"
      "rtdayet" => "float"
      "rtdayetml" => "float"
      "rtmonthet" => "float"
      "rtyearet" => "float"
      "rtbattvoltage" => "float"
      "rtinsidehum" => "integer"
      "rtoutsidehum" => "integer"
      "rtwindspeed" => "integer"
      "rtwindavgspeed" => "integer"
      "rtwinddir" => "integer"
      "rtxmitbattt" => "integer"
      "rtforeicon" => "integer"
      "rtforerule" => "integer"
    }
  }
}

output {
  #test it with
  #stdout { codec => rubydebug }

  if [type] == "weather" {
    elasticsearch {
      hosts => ['es.yourdomain.com']
      user => 'user'
      password => 'password'
      index => "weather"
    }
  }
}