Merge pull request #49 from treasure-data/add_output_settings

[WIP] Add in/out plugin settings
This commit is contained in:
uu59 2014-07-17 15:26:43 +09:00
commit 5e24816478
32 changed files with 869 additions and 14 deletions

View File

@ -0,0 +1,7 @@
(function(){
"use strict";
$(function(){
});
})();

View File

@ -116,3 +116,24 @@ label {
cursor: pointer;
}
.ignore-rails-error-div {
.field_with_errors {
display: inline;
}
}
.form-group {
.form-group {
margin-left: 30px; // used at fluentd/settings/out_forward, nested config form
}
}
.fluentd-setting-inout {
.arrow-right {
text-align: center;
font-size: 36px;
padding-top: 64px;
}
}

View File

@ -0,0 +1,34 @@
class Fluentd::Settings::InSyslogController < ApplicationController
before_action :login_required
before_action :find_fluentd
def show
@setting = Fluentd::Setting::InSyslog.new({
bind: "0.0.0.0",
port: 5140,
})
end
def finish
@setting = Fluentd::Setting::InSyslog.new(setting_params)
unless @setting.valid?
return render "show"
end
@fluentd.agent.config_append @setting.to_conf
if @fluentd.agent.running?
unless @fluentd.agent.restart
@setting.errors.add(:base, @fluentd.agent.log_tail(1).first)
return render "show"
end
end
redirect_to fluentd_setting_path(@fluentd)
end
private
def setting_params
params.require(:fluentd_setting_in_syslog).permit(*Fluentd::Setting::InSyslog::KEYS)
end
end

View File

@ -0,0 +1,34 @@
class Fluentd::Settings::OutForwardController < ApplicationController
before_action :login_required
before_action :find_fluentd
def show
@setting = Fluentd::Setting::OutForward.new({
})
end
def finish
@setting = Fluentd::Setting::OutForward.new(setting_params)
unless @setting.valid?
return render "show"
end
@fluentd.agent.config_append @setting.to_config
if @fluentd.agent.running?
unless @fluentd.agent.restart
@setting.errors.add(:base, @fluentd.agent.log_tail(1).first)
return render "show"
end
end
redirect_to fluentd_setting_path(@fluentd)
end
private
def setting_params
params.require(:fluentd_setting_out_forward).permit(*Fluentd::Setting::OutForward::KEYS).merge(
params.require(:fluentd_setting_out_forward).permit(:server => Fluentd::Setting::OutForward::Server::KEYS)
)
end
end

View File

@ -0,0 +1,36 @@
class Fluentd::Settings::OutMongoController < ApplicationController
before_action :login_required
before_action :find_fluentd
def show
@setting = Fluentd::Setting::OutMongo.new({
host: "127.0.0.1",
port: 27017,
capped: true,
capped_size: "100m",
})
end
def finish
@setting = Fluentd::Setting::OutMongo.new(setting_params)
unless @setting.valid?
return render "show"
end
@fluentd.agent.config_append @setting.to_conf
if @fluentd.agent.running?
unless @fluentd.agent.restart
@setting.errors.add(:base, @fluentd.agent.log_tail(1).first)
return render "show"
end
end
redirect_to fluentd_setting_path(@fluentd)
end
private
def setting_params
params.require(:fluentd_setting_out_mongo).permit(*Fluentd::Setting::OutMongo::KEYS)
end
end

View File

@ -0,0 +1,34 @@
class Fluentd::Settings::OutS3Controller < ApplicationController
before_action :login_required
before_action :find_fluentd
def show
@setting = Fluentd::Setting::OutS3.new({
s3_endpoint: "s3-us-west-1.amazonaws.com",
buffer_path: "/var/log/fluent/s3",
})
end
def finish
@setting = Fluentd::Setting::OutS3.new(setting_params)
unless @setting.valid?
return render "show"
end
@fluentd.agent.config_append @setting.to_conf
if @fluentd.agent.running?
unless @fluentd.agent.restart
@setting.errors.add(:base, @fluentd.agent.log_tail(1).first)
return render "show"
end
end
redirect_to fluentd_setting_path(@fluentd)
end
private
def setting_params
params.require(:fluentd_setting_out_s3).permit(*Fluentd::Setting::OutS3::KEYS)
end
end

View File

@ -0,0 +1,36 @@
class Fluentd::Settings::OutTdController < ApplicationController
before_action :login_required
before_action :find_fluentd
def show
@setting = Fluentd::Setting::OutTd.new({
buffer_type: "file",
buffer_path: "/var/log/td-agent/buffer/td",
use_ssl: true,
auto_create_table: true,
})
end
def finish
@setting = Fluentd::Setting::OutTd.new(setting_params)
unless @setting.valid?
return render "show"
end
@fluentd.agent.config_append @setting.to_conf
if @fluentd.agent.running?
unless @fluentd.agent.restart
@setting.errors.add(:base, @fluentd.agent.log_tail(1).first)
return render "show"
end
end
redirect_to fluentd_setting_path(@fluentd)
end
private
def setting_params
params.require(:fluentd_setting_out_td).permit(*Fluentd::Setting::OutTd::KEYS)
end
end

View File

@ -0,0 +1,35 @@
module SettingsHelper
def field(form, key, opts = {})
html = '<div class="form-group">'
case form.object.column_type(key)
when :boolean, :flag
html << form.check_box(key, {}, "true", "false")
html << " " # NOTE: Adding space for padding
html << h(form.label(key))
when :choice
html << h(form.label(key))
html << " " # NOTE: Adding space for padding
html << form.select(key, form.object.values_of(key), opts)
when :nested
html << h(form.label(key))
child_data = form.object.class.children[key]
klass = child_data[:class]
children = form.object.send(key) || {"0" => {}}
children.each_pair do |index, child|
# TODO: allow append/delete for multiple child
form.fields_for("#{key}[#{index}]", klass.new(child), class: "nested-column #{child_data[:multiple] ? "multiple" : ""} well well-sm") do |ff|
klass::KEYS.each do |k|
html << field(ff, k)
end
end
end
else
html << h(form.label(key))
html << form.text_field(key, class: "form-control")
end
html << "</div>"
html.html_safe
end
end

View File

@ -0,0 +1,152 @@
class Fluentd
module Setting
module Common
extend ActiveSupport::Concern
include ActiveModel::Model
module ClassMethods
attr_accessor :values, :types, :children
def choice(key, values)
@values ||= {}
@values[key] = values
set_type(:choice, [key])
end
def nested(key, klass, options = {})
# e.g.:
# <match>
# type forward
# <server>
# ..
# </server>
# </match>
@children ||= {}
@children[key] = {
class: klass,
options: options,
}
set_type(:nested, [key])
end
def booleans(*keys)
# e.g.:
# use_ssl true
# include_time_key false
set_type(:boolean, keys)
end
def flags(*keys)
# e.g.:
# tag_mapped
# utc
set_type(:flag, keys)
end
private
def set_type(type, keys)
@types ||= {}
keys.each do |key|
@types[key] = type
end
end
end
def child_class(key)
self.class.children[key][:class]
end
def values_of(key)
self.class.values.try(:[], key) || []
end
def column_type(key)
self.class.types.try(:[], key) || "string"
end
def conf(key)
case column_type(key)
when :boolean
boolenan(key)
when :flag
flag(key)
when :nested
klass = child_class(key)
send(key).map do |(_, child)|
# send("servers")
#
# "servers" => {
# "0" => {
# "name" => "foo",
# "host" => "bar",
# ..
# },
# "1" => {
# ..
# }
# }
child_instance = klass.new(child)
unless child_instance.empty_value?
"\n" + child_instance.to_config(key).gsub(/^/m, " ")
end
end.join
else
print_if_present(key)
end
end
def plugin_type_name
# Fluentd::Setting::OutS3 -> s3
# Override this method if not above style
self.class.to_s.split("::").last.sub(/(In|Out)/, "").downcase
end
def print_if_present(key)
# e.g.:
# path /var/log/td/aaa
# user nobody
# retry_limit 3
send(key).present? ? "#{key} #{send(key)}" : ""
end
def boolenan(key)
send(key).presence == "true" ? "#{key} true" : "#{key} false"
end
def flag(key)
send(key).presence == "true" ? key.to_s : ""
end
def empty_value?
config = ""
self.class.const_get(:KEYS).each do |key|
config << conf(key)
end
config.empty?
end
def to_config(elm_name = nil)
indent = " "
if elm_name
config = "<#{elm_name}>\n"
else
config = "<match #{match}>\n"
config << "#{indent}type #{plugin_type_name}\n"
end
self.class.const_get(:KEYS).each do |key|
next if key == :match
config << indent
config << conf(key)
config << "\n"
end
if elm_name
config << "</#{elm_name}>\n"
else
config << "</match>\n"
end
config.gsub(/^[ ]*\n/m, "")
end
end
end
end

View File

@ -0,0 +1,28 @@
class Fluentd
module Setting
class InSyslog
include ActiveModel::Model
include Common
KEYS = [
:port, :bind, :tag, :types
].freeze
attr_accessor(*KEYS)
validates :tag, presence: true
def to_conf
<<-XML.strip_heredoc.gsub(/^[ ]*\n/m, "")
<source>
type syslog
#{print_if_present :tag}
#{print_if_present :bind}
#{print_if_present :port}
#{print_if_present :types}
</source>
XML
end
end
end
end

View File

@ -0,0 +1,48 @@
class Fluentd
module Setting
class OutForward
class Server
include Common
KEYS = [
:name, :host, :port, :weight, :standby
].freeze
attr_accessor(*KEYS)
flags :standby
validates :host, presence: true
end
include Common
KEYS = [
:match,
:send_timeout, :recover_wait, :heartbeat_type, :heartbeat_interval,
:phi_threshold, :hard_timeout,
:server
].freeze
attr_accessor(*KEYS)
choice :heartbeat_type, %w(udp tcp)
nested :server, Server
validates :match, presence: true
validate :validate_at_least_one_server
validate :validate_nested_values
def validate_at_least_one_server
# FIXME: real validation
true
end
def validate_nested_values
# FIXME: real validation with child class instance
self.class.children.inject(true) do |result, child|
# result & child.valid?
end
true
end
end
end
end

View File

@ -0,0 +1,39 @@
class Fluentd
module Setting
class OutMongo
include Common
KEYS = [
:match,
:host, :port, :database, :collection, :capped, :capped_size, :capped_max, :user, :password, :tag_mapped,
:buffer_type, :buffer_queue_limit, :buffer_chunk_limit, :flush_interval, :retry_wait, :retry_limit, :max_retry_wait, :num_threads,
].freeze
attr_accessor(*KEYS)
flags :capped, :tag_mapped
validates :match, presence: true
validates :host, presence: true
validates :port, presence: true
validates :database, presence: true
validate :validate_capped
validate :validate_collection
def to_conf
to_config
end
def validate_capped
return true if capped.blank?
errors.add(:capped_size, :blank) if capped_size.blank?
end
def validate_collection
if tag_mapped.blank? && collection.blank?
errors.add(:collection, :blank)
end
end
end
end
end

View File

@ -0,0 +1,33 @@
class Fluentd
module Setting
class OutS3
include ActiveModel::Model
include Common
KEYS = [
:match,
:aws_key_id, :aws_sec_key, :s3_bucket, :s3_endpoint, :path,
# :reduced_redundancy, :check_apikey_on_start, :command_parameter, # not configurable?
:format, :include_time_key, :time_key, :delimiter, :label_delimiter, :add_newline,
:time_slice_format, :time_slice_wait, :time_format, :utc, :store_as, :proxy_uri, :use_ssl,
:buffer_type, :buffer_path, :buffer_queue_limit, :buffer_chunk_limit, :flush_interval,
:retry_wait, :retry_limit, :max_retry_wait, :num_threads,
].freeze
attr_accessor(*KEYS)
choice :format, %w(out_file json ltsv single_value)
choice :store_as, %w(gzip lzo lzma2 json txt)
booleans :include_time_key, :add_newline, :use_ssl
flags :utc
validates :match, presence: true
validates :s3_bucket, presence: true
validates :buffer_path, presence: true
def to_conf
to_config
end
end
end
end

View File

@ -0,0 +1,32 @@
class Fluentd
module Setting
class OutTd
include ActiveModel::Model
include Common
KEYS = [
:match,
:apikey, :auto_create_table, :use_ssl, :database, :table, :endpoint,
:connect_timeout, :read_timeout, :send_timeout, :flush_interval, :buffer_type, :buffer_path,
].freeze
attr_accessor(*KEYS)
booleans :use_ssl
flags :auto_create_table
validates :match, presence: true
validates :apikey, presence: true
validates :auto_create_table, presence: true
validates :use_ssl, presence: true
def plugin_name
"tdlog"
end
def to_conf
to_config
end
end
end
end

View File

@ -0,0 +1,9 @@
= render "shared/setting_errors"
= form_for(@setting, url: finish_fluentd_setting_in_syslog_path(@fluentd), html: {class: "ignore-rails-error-div"}) do |f|
= field(f, :tag)
= field(f, :bind)
= field(f, :port)
= field(f, :types)
= f.submit t('fluentd.common.finish') , class: "btn btn-lg btn-primary pull-right"

View File

@ -0,0 +1,6 @@
- page_title t(".page_title")
.well
= raw t('fluentd.settings.in_syslog.option_guide')
= render "form"

View File

@ -1,4 +1,3 @@
- @setting.errors.full_messages.each do |e|
%div.alert.alert-danger= e

View File

@ -0,0 +1,20 @@
= render "shared/setting_errors"
= form_for(@setting, url: finish_fluentd_setting_out_forward_path(@fluentd), html: {class: "ignore-rails-error-div"}) do |f|
= field(f, :match)
= field(f, :server)
.well.well-sm
%h4{"data-toggle" => "collapse", "href" => "#advanced-setting"}
= icon('fa-caret-down')
= t('terms.advanced_setting')
#advanced-setting.collapse
= field(f, :send_timeout)
= field(f, :recover_wait)
= field(f, :heartbeat_type)
= field(f, :heartbeat_interval)
= field(f, :phi_threshold)
= field(f, :hard_timeout)
= f.submit t('fluentd.common.finish') , class: "btn btn-lg btn-primary pull-right"

View File

@ -0,0 +1,6 @@
- page_title t(".page_title")
.well
= raw t('fluentd.settings.out_forward.option_guide')
= render "form"

View File

@ -0,0 +1,30 @@
= render "shared/setting_errors"
= form_for(@setting, url: finish_fluentd_setting_out_mongo_path(@fluentd), html: {class: "ignore-rails-error-div"}) do |f|
= field(f, :match)
= field(f, :host)
= field(f, :port)
= field(f, :database)
= field(f, :collection)
= field(f, :tag_mapped)
= field(f, :user)
= field(f, :password)
.well.well-sm
%h4{"data-toggle" => "collapse", "href" => "#advanced-setting"}
= icon('fa-caret-down')
= t('terms.advanced_setting')
#advanced-setting.collapse
= field(f, :capped)
= field(f, :capped_size)
= field(f, :capped_max)
= field(f, :buffer_type)
= field(f, :buffer_queue_limit)
= field(f, :buffer_chunk_limit)
= field(f, :flush_interval)
= field(f, :retry_wait)
= field(f, :retry_limit)
= field(f, :max_retry_wait)
= field(f, :num_threads)
= f.submit t('fluentd.common.finish') , class: "btn btn-lg btn-primary pull-right"

View File

@ -0,0 +1,4 @@
- page_title t(".page_title")
= render "form"

View File

@ -0,0 +1,6 @@
- page_title t(".page_title")
.well
= raw t('fluentd.settings.out_mongo.option_guide')
= render "form"

View File

@ -0,0 +1,37 @@
= render "shared/setting_errors"
= form_for(@setting, url: finish_fluentd_setting_out_s3_path(@fluentd), html: {class: "ignore-rails-error-div"}) do |f|
= field(f, :match)
= field(f, :aws_key_id)
= field(f, :aws_sec_key)
= field(f, :s3_bucket)
= field(f, :s3_endpoint)
= field(f, :path)
= field(f, :format, prompt: "--")
= field(f, :time_slice_format)
= field(f, :time_slice_wait)
= field(f, :include_time_key)
= field(f, :utc)
= field(f, :delimiter)
= field(f, :label_delimiter)
= field(f, :store_as)
= field(f, :proxy_uri)
= field(f, :use_ssl)
.well.well-sm
%h4{"data-toggle" => "collapse", "href" => "#advanced-setting"}
= icon('fa-caret-down')
= t('terms.advanced_setting')
#advanced-setting.collapse
= field(f, :buffer_type)
= field(f, :buffer_path)
= field(f, :buffer_queue_limit)
= field(f, :buffer_chunk_limit)
= field(f, :flush_interval)
= field(f, :retry_wait)
= field(f, :retry_limit)
= field(f, :max_retry_wait)
= field(f, :num_threads)
= f.submit t('fluentd.common.finish') , class: "btn btn-lg btn-primary pull-right"

View File

@ -0,0 +1,6 @@
- page_title t(".page_title")
.well
= raw t('fluentd.settings.out_s3.option_guide')
= render "form"

View File

@ -0,0 +1,23 @@
= render "shared/setting_errors"
= form_for(@setting, url: finish_fluentd_setting_out_td_path(@fluentd), html: {class: "ignore-rails-error-div"}) do |f|
= field(f, :match)
= field(f, :apikey)
= field(f, :auto_create_table)
= field(f, :use_ssl)
= field(f, :database)
= field(f, :table)
= field(f, :endpoint)
.well.well-sm
%h4{"data-toggle" => "collapse", "href" => "#advanced-setting"}
= icon('fa-caret-down')
= t('terms.advanced_setting')
#advanced-setting.collapse
= field(f, :connect_timeout)
= field(f, :read_timeout)
= field(f, :send_timeout)
= field(f, :flush_interval)
= field(f, :buffer_type)
= field(f, :buffer_path)
= f.submit t('fluentd.common.finish') , class: "btn btn-lg btn-primary pull-right"

View File

@ -0,0 +1,6 @@
- page_title t(".page_title")
.well
= raw t('fluentd.settings.out_td.option_guide')
= render "form"

View File

@ -1,15 +1,57 @@
- page_title t('.page_title', label: @fluentd.label)
.row
%h2
= t(".edit_config")
= link_to icon('fa-pencil') << t(".edit"), edit_fluentd_setting_path(@fluentd)
%pre
= preserve do
= @config
.col-lg-12
%h2
= t(".edit_config")
= link_to icon('fa-pencil') << t(".edit"), edit_fluentd_setting_path(@fluentd)
%pre
= preserve do
= @config
.row
.col-lg-4.well
= link_to(fluentd_setting_in_tail_path(@fluentd)) do
= icon('fa-file-text-o fa-lg')
= t("fluentd.common.setup_in_tail")
%hr
%h2
= t ".in_out_head"
.row.fluentd-setting-inout
.col-lg-4
.panel.panel-default
.panel-heading
%h4= t('.in')
.panel-body
%p
= link_to(fluentd_setting_in_tail_path(@fluentd)) do
= icon('fa-file-text-o fa-lg')
= t("fluentd.common.setup_in_tail")
%p
= link_to(fluentd_setting_in_syslog_path(@fluentd)) do
= icon('fa-file-text-o fa-lg')
= t("fluentd.common.setup_in_syslog")
.col-lg-1.arrow-right
= icon "fa-arrow-circle-right"
.col-lg-2
= image_tag "/fluentd-logo.png", style: "max-width: 100%"
.col-lg-1.arrow-right
= icon "fa-arrow-circle-right"
.col-lg-4
.panel.panel-default
.panel-heading
%h4= t('.out')
.panel-body
%p
= link_to(fluentd_setting_out_td_path(@fluentd)) do
= icon('fa-file-text-o fa-lg')
= t("fluentd.common.setup_out_td")
%p
= link_to(fluentd_setting_out_s3_path(@fluentd)) do
= icon('fa-file-text-o fa-lg')
= t("fluentd.common.setup_out_s3")
%p
= link_to(fluentd_setting_out_mongo_path(@fluentd)) do
= icon('fa-file-text-o fa-lg')
= t("fluentd.common.setup_out_mongo")
%p
= link_to(fluentd_setting_out_forward_path(@fluentd)) do
= icon('fa-file-text-o fa-lg')
= t("fluentd.common.setup_out_forward")

View File

@ -0,0 +1,5 @@
- if request.post? && !@setting.valid?
.well.well-sm
- @setting.errors.full_messages.each do |msg|
.text.text-danger= msg

View File

@ -92,7 +92,12 @@ en:
edit_config: Edit Config
config_file: Config file
page_title: "%{label}"
setup_in_tail: Setting file reading
setup_in_tail: File
setup_in_syslog: Syslog protocol
setup_out_td: Treasure Data
setup_out_mongo: MongoDB
setup_out_forward: forwarding
setup_out_s3: AWS S3
finish: Update config
fluentd_info: Setting info
recent_errors: "Recently %{days} days errors"
@ -121,8 +126,36 @@ en:
show:
<<: *fluentd_common
page_title: fluentd config
in_out_head: In/Out setting
in: Input
out: Output
edit:
<<: *fluentd_common
out_forward:
option_guide: |
show:
page_title: out_forward(TODO)
out_s3:
option_guide: |
fluent-plugin-s3 install is required.
See also <a target="_blank" href="http://docs.fluentd.org/articles/out_s3">out_s3 plugin</a>.
show:
page_title: Write to AWS S3
out_td:
option_guide: |
show:
page_title: Write to Treasure Data setting
out_mongo:
option_guide: |
fluent-plugin-mongo install is required.<br />
See also <a target="_blank" href="http://docs.fluentd.org/articles/out_mongo">out_mongo plugin</a>.
show:
page_title: Write to MongoDB setting
in_syslog:
option_guide: |
See also <a target="_blank" href="http://docs.fluentd.org/articles/in_syslog">in_syslog Plugin</a> page.
show:
page_title: Read from Syslog protocol
in_tail_option_guide: |
See <a target="_blank" href="http://docs.fluentd.org/articles/in_tail">in_tail Plugin</a> or
<a target="_blank" href="http://fluentular.herokuapp.com/">Fluentular</a> for more details.

View File

@ -92,7 +92,12 @@ ja:
edit_config: 設定ファイルの編集
config_file: 設定ファイル
page_title: "%{label}"
setup_in_tail: ファイル読み込みの設定
setup_in_tail: ファイル
setup_in_syslog: syslogプロトコル
setup_out_td: Treasure Data
setup_out_mongo: MongoDB
setup_out_forward: 転送
setup_out_s3: AWS S3
finish: 設定する
fluentd_info: 設定情報
recent_errors: "直近 %{days} 日内のエラー"
@ -122,8 +127,37 @@ ja:
show:
<<: *fluentd_common
page_title: fluentd 設定
in_out_head: 入力/出力設定
in: 入力
out: 出力
edit:
<<: *fluentd_common
out_forward:
option_guide: |
show:
page_title: out_forward(TODO)
out_s3:
option_guide: |
fluent-plugin-s3プラグインのインストールが必要です。<br />
<a target="_blank" href="http://docs.fluentd.org/articles/out_s3">out_s3プラグインの解説</a>もご参照ください。
show:
page_title: AWS S3書き出し設定
out_td:
option_guide: |
fluent-plugin-tdプラグインのインストールが必要です。<br />
show:
page_title: Treasure Data書き出し設定
out_mongo:
option_guide: |
fluent-plugin-mongoプラグインのインストールが必要です。<br />
<a target="_blank" href="http://docs.fluentd.org/ja/articles/out_mongo">out_mongoプラグインの解説</a>もご参照ください。
show:
page_title: MongoDB書き出し設定
in_syslog:
option_guide: |
<a target="_blank" href="http://docs.fluentd.org/ja/articles/in_syslog">in_syslogプラグイン解説ページ</a>もご参照ください。
show:
page_title: Syslog読み込み設定
in_tail_option_guide: |
<a target="_blank" href="http://docs.fluentd.org/ja/articles/in_tail">in_tailプラグインの解説ページ</a>や
<a target="_blank" href="http://fluentular.herokuapp.com/">Fluentular</a>もご参照ください。

View File

@ -17,6 +17,26 @@ Rails.application.routes.draw do
post "confirm"
post "finish"
end
resource :in_syslog, only: ["show"], module: :settings, controller: :in_syslog do
post "finish"
end
resource :out_mongo, only: ["show"], module: :settings, controller: :out_mongo do
post "finish"
end
resource :out_td, only: ["show"], module: :settings, controller: :out_td do
post "finish"
end
resource :out_s3, only: ["show"], module: :settings, controller: :out_s3 do
post "finish"
end
resource :out_forward, only: ["show"], module: :settings, controller: :out_forward do
post "finish"
end
end
end

BIN
public/fluentd-logo.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 57 KiB