1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
#################
# file agent    #
#################
input {
  file {
    type => "mysql-general"
    path => "/cars/salt/logstash-salt/general.log"
    sincedb_path => "/tmp/.sincedb"
    start_position => "beginning"
  }
}
filter {
  if [type] == "mysql-general" {
    # mysql logs sometimes do not have timestamp fields and use spaces instead.
    # events without timstamps merge with previous & possibly unrelated event.
    # Warning: logstash only waits a little while before sending events separatly
    # regardless of multiline pattern, for more info look at this thread:
    #   https://groups.google.com/forum/#!topic/logstash-users/Xx9SxMxM60s
    multiline {
      pattern => "^\s"
      what => "previous"
    }

    # pull out the mysql_timestamp (like, "120707  0:40:34")
    grok {
      match => [ "message", "^%{NUMBER:date} *%{NOTSPACE:time}" ]
      add_field => [ "mysql_timestamp", "%{date} %{time}" ]
    }

    # parse the mysql_timestamp, "hour" which could be one or two digits.
    date {
      match => [ "mysql_timestamp", "YYMMdd H:mm:ss", "YYMMdd HH:mm:ss" ]
    }

    # remove duplicate or temporary fields to prevent bloat
    mutate {
      remove_field => [ "date", "time", "mysql_timestamp"]
    }

    # Now split up the multiline again, which keeps the timestamp for all split
    # out events. The defaults here are fine as they split '@message' by '\n'
    split { }
  }
}

# all logs go to redis "queue"
output {
  #stdout { debug => true debug_format => "json"}
  redis { host => "127.0.0.1" data_type => "list" key => "logstash" }
}

#################
# indexer agent #
#################
input {
  redis {
    host => "127.0.0.1"
    # these settings should match the output of the agent
    data_type => "list"
    key => "logstash"

    # We use the 'json' codec here because we expect to read
    # json events from redis.
    codec => json
  }
}

output {
  ##stdout { debug => true debug_format => "json"}
  elasticsearch {
    host => "127.0.0.1"
    cluster => "logstash"
  }
}