ubuntu-log-archive.conf

input {
	file {
		path => [ "/var/lib/ubuntu-chatlogs/spool/#*/*" ]
		start_position => beginning
		mode => "read"
		file_completed_action => "delete"
		sincedb_path => "/var/lib/ubuntu-chatlogs/_sincedb.chatlogs"
		ecs_compatibility => "v8"
		file_sort_by => "last_modified"
		file_sort_direction => "desc"
		codec => plain {
			charset => "UTF-8"
			ecs_compatibility => "v8"
		}
	}
}

filter {
	grok {
		match => {
			"[log][file][path]" => "%{GREEDYDATA}/#%{DATA:channel}\.%{YEAR:year}-%{MONTHNUM2:month}-%{MONTHDAY:day}$"
		}
		add_field => {
			server => "Freenode/LiberaChat"
		}
		remove_field => "[log][file][path]"
	}
	grok {
		match => { "message" => "^\[%{HOUR:hour}:%{MINUTE:minute}\]" }
	}
	if ![hour] {
		mutate { add_field => { hour => "00" } }
	}
	if ![minute] {
		mutate { add_field => { minute => "00" } }
	}
	mutate {
		add_field => { "grokdate" => "%{year}/%{month}/%{day} %{hour}:%{minute} +0000" }
		remove_tag => [ "_grokparsefailure" ]
	}
	date {
		match => [ "grokdate",
			"YYYY/MM/dd HH:mm Z",
			"YYYY/M/d H:m Z",
			"YY/MM/DD HH:mm Z",
			"YY/M/D H:m Z"
		]
		timezone => "Etc/UTC"
		remove_field => [ "year", "month", "day", "hour", "minute", "grokdate" ]
	}
	grok {
		match => { "message" => [
			"(===|\*)%{SPACE}(?<action>Topic) for %{NOTSPACE}: %{GREEDYDATA:text}",
			"(===|\*)%{SPACE}(?<action>Topic) \(%{NOTSPACE}\): set by {NOTSPACE:nick}",
			"(===|\*)%{SPACE}(?<nick>\S+) sets (?<action>mode) %{GREEDYDATA:mode}",
			"(===|\*)%{SPACE}(?<action>mode)/#%{NOTSPACE}%{SPACE}\[%{DATA:mode}\]%{SPACE}by (?<nick>\S+)",
			"(===|\*)%{SPACE}(?<nick>\S+) was (?<action>kick)ed off #%{NOTSPACE} by (?<who>\S+)%{SPACE}%{GREEDYDATA:reason}",
			"(===|\*)%{SPACE}(?<nick>\S+) changed the (?<action>topic) of #%{NOTSPACE} to %{GREEDYDATA:topic}",
			"(===|\*)%{SPACE}(?<nick>\S+) (\[%{DATA:hostmask}\]%{SPACE})?has (?<action>join)ed #%{NOTSPACE}"
		] }
		remove_tag => [ "_grokparsefailure" ]
	}
	if ![action] and [message] =~ "^(\[\d\d:\d\d\] )?(===|\*) \S+ \S+ has left" {
		grok {
			match => { "message" => "^(\[%{HOUR}:%{MINUTE}\] )?(===|\*)%{SPACE}(?<nick>\S+)%{SPACE}(\[%{DATA}\]%{SPACE})?has left %{NOTSPACE}(%{SPACE}\[%{GREEDYDATA:reason}\])?" }
			add_field => { action => "part" }
			remove_tag => [ "_grokparsefailure" ]
		}
	}
	if ![action] and [message] =~ "^(\[\d\d:\d\d\] )?(===|\*) \S+ is now known as" {
		grok {
			match => { "message" => "^(\[%{HOUR}:%{MINUTE}\] )?(===|\*)%{SPACE}(?<oldnick>\S+) is now known as (?<nick>\S+)" }
			add_field => { action => "nick" }
			remove_tag => [ "_grokparsefailure" ]
		}
	}
	if ![action] and [message] =~ "^(\[\d\d:\d\d\] )?<\S+>" {
		grok {
			match => { "message" => "^(\[%{HOUR}:%{MINUTE}\] )?\<(?<nick>[^\>]+)\>%{SPACE}%{GREEDYDATA:text}" }
			add_field => { action => "message" }
			remove_tag => [ "_grokparsefailure" ]			
		}
	}
	if ![action] and [message] =~ "^(\[\d\d:\d\d\] )?(===|\*) [^#\(\[]\S+" {
		grok {
			match => { "message" => "^(\[%{HOUR}:%{MINUTE}\] )?(===|\*)%{SPACE}(?<nick>\S+)%{SPACE}%{GREEDYDATA:text}" }
			add_field => { action => "action" }
			remove_tag => [ "_grokparsefailure" ]
		}
	}
	mutate {
		lowercase => [ "action", "nick", "oldnick", "who" ]
		gsub => [ "nick", "\s", "", "oldnick", "\s", "", "who", "\s", "" ]
	}
	mutate {
		remove_field => "[host]"
		remove_field => "log"
		remove_field => "tags"
		remove_field => "message"
	}
}

output {
    elasticsearch {
        hosts => [ "http://elasticsearch-master:9200" ]
        index => "logstash-chat-ubuntu"
		template => "/usr/share/logstash/config/index-template.json"
		template_name => "chatlogs_template"
    }
}