Logstash Output Charrington

Logstash Output Plugin to handle batching messages and dynamically create and alter tables.

Forked From logstash-output-jdbc

Configure

Example Logstash Config

input {
  kafka {
    auto_offset_reset => earliest
    bootstrap_servers => "${LS_KAFKA_BOOTSTRAP_SERVERS:kafka:9092}"
    codec => json
    group_id => "winston_ls_pg_sinks_cg"
    topics => [
      "orwell.analytics.v1.json"
    ]
  }
}

filter {
  if "_jsonparsefailure" in [tags] {
    drop { }
  }

  ruby {
    code => "
      hash = event.to_hash
      hash.each do |k,v|
        if v == nil
          event.remove(k)
        end
      end
    "
  }
}

output {
  charrington {
    driver_jar_path => "${DRIVER_PATH:/usr/share/logstash/vendor/jar/jdbc/postgresql-42.2.5.jar}"
    connection_string => "jdbc:postgresql://${WINSTON_DATABASE_HOST}:5432/${WINSTON_DATABASE}?user=${WINSTON_DATABASE_USERNAME}&password=${WINSTON_DATABASE_PASSWORD}"
    schema => "dea"
  }

  if "${LS_SHOW_DEBUG_OUTPUT}" == "true" {
    stdout { codec => rubydebug }
  }
}

Build & Publish

gem build logstash-output-charrington.gemspec       # build
gem push logstash-output-charrington-x.x.x.gem      # publish