From 4f2a06fd82edec3c0707215e62747e30ec4a996c Mon Sep 17 00:00:00 2001 From: Kenji Okimoto Date: Thu, 24 May 2018 18:08:33 +0900 Subject: [PATCH] Add concerns to define models for fluent-plugin Signed-off-by: Kenji Okimoto --- .../concerns/fluentd/setting/configurable.rb | 103 ++++++++++++++++++ .../concerns/fluentd/setting/pattern.rb | 11 ++ app/models/concerns/fluentd/setting/plugin.rb | 75 +++++++++++++ .../concerns/fluentd/setting/plugin_config.rb | 57 ++++++++++ .../fluentd/setting/plugin_parameter.rb | 43 ++++++++ .../fluentd/setting/section_parser.rb | 21 ++++ .../fluentd/setting/section_validator.rb | 21 ++++ 7 files changed, 331 insertions(+) create mode 100644 app/models/concerns/fluentd/setting/configurable.rb create mode 100644 app/models/concerns/fluentd/setting/pattern.rb create mode 100644 app/models/concerns/fluentd/setting/plugin.rb create mode 100644 app/models/concerns/fluentd/setting/plugin_config.rb create mode 100644 app/models/concerns/fluentd/setting/plugin_parameter.rb create mode 100644 app/models/concerns/fluentd/setting/section_parser.rb create mode 100644 app/models/concerns/fluentd/setting/section_validator.rb diff --git a/app/models/concerns/fluentd/setting/configurable.rb b/app/models/concerns/fluentd/setting/configurable.rb new file mode 100644 index 0000000..a9fdd19 --- /dev/null +++ b/app/models/concerns/fluentd/setting/configurable.rb @@ -0,0 +1,103 @@ +class Fluentd + module Setting + module Configurable + extend ActiveSupport::Concern + + included do + class_attribute :_types, :_defaults, :_secrets, :_aliases, :_required + class_attribute :_deprecated_params, :_obsoleted_params, :_descriptions + class_attribute :_list, :_value_types, :_symbolize_keys + class_attribute :_argument_name, :_built_in_params, :_sections, :_section_params + self._types = {} + self._defaults = {} + self._secrets = {} + self._aliases = {} + self._required = {} + self._deprecated_params = {} + self._obsoleted_params = {} + self._descriptions = {} + self._list = {} + self._value_types = {} + self._symbolize_keys = {} + self._argument_name = nil + self._built_in_params = [] + self._sections = {} + self._section_params = Hash.new {|h, k| h[k] = [] } + end + + def initialize(attributes = {}) + super rescue ActiveModel::UnknownAttributeError # the superclass does not know specific attributes of the model + self.class._sections.each do |name, klass| + klass.init + if klass.multi + next if attributes[name].nil? + attributes[name].each do |attr| + next unless attr + attr.each do |index, _attr| + self._section_params[name] << klass.new(_attr) + end + end + else + attr = attributes.dig(name, "0") + self._section_params[name] << klass.new(attr) if attr + end + end + end + + module ClassMethods + # config_param :name, :string, default: "value", secret: true + def config_param(name, type = ActiveModel::Type::Value.new, **options) + # NOTE: We cannot overwrite types defined by ActiveModel in config/initializers/types.rb + if type == :time + type = Fluentd::Setting::Type::Time.new + end + if name.to_s.start_with?("@") + _name = name.to_s[1..-1] + config_param(_name.to_sym, type, **options.merge(alias: name)) + self._built_in_params << _name + elsif ["id", "type", "log_level"].include?(name.to_s) + self._built_in_params << _name + else + attribute(name, type, **options.slice(:precision, :limit, :scale)) + validates(name, presence: true) if options[:required] + end + self._types[name] = type + self._defaults[name] = options[:default] if options.key?(:default) + self._secrets[name] = options[:secret] if options.key?(:secret) + self._aliases[name] = options[:alias] if options.key?(:alias) + self._required[name] = options[:required] if options.key?(:required) + self._deprecated_params[name] = options[:deprecated] if options.key?(:deprecated) + self._obsoleted_params[name] = options[:obsoleted] if options.key?(:obsoleted) + self._list[name] = options[:list] if options.key?(:list) + self._value_types[name] = options[:value_types] if options.key?(:value_types) + self._symbolize_keys = options[:symbolize_keys] if options.key?(:symbolize_keys) + end + + def config_section(name, **options, &block) + if self._sections.key?(name) + self._sections[name].merge(**options, &block) + else + attribute(name, :section) + section_class = Class.new(::Fluentd::Setting::Section) + section_class.section_name = name + section_class.required = options[:required] + section_class.multi = options[:multi] + section_class.alias = options[:alias] + section_class._block = block + self.const_set(name.to_s.classify, section_class) + self._sections[name] = section_class + end + end + + def config_argument(name, type = ActiveModel::Type::Value.new, **options) + config_param(name, **options) + self._argument_name = name + end + + def set_argument_name(name) + self._argument_name = name + end + end + end + end +end diff --git a/app/models/concerns/fluentd/setting/pattern.rb b/app/models/concerns/fluentd/setting/pattern.rb new file mode 100644 index 0000000..ff0b89b --- /dev/null +++ b/app/models/concerns/fluentd/setting/pattern.rb @@ -0,0 +1,11 @@ +class Fluentd + module Setting + module Pattern + extend ActiveSupport::Concern + + included do + config_argument(:pattern, :string) + end + end + end +end diff --git a/app/models/concerns/fluentd/setting/plugin.rb b/app/models/concerns/fluentd/setting/plugin.rb new file mode 100644 index 0000000..3517655 --- /dev/null +++ b/app/models/concerns/fluentd/setting/plugin.rb @@ -0,0 +1,75 @@ +require "fluent/plugin" + +class Fluentd + module Setting + module Plugin + extend ActiveSupport::Concern + + include ActiveModel::Model + include ActiveModel::Attributes + include Fluentd::Setting::Configurable + include Fluentd::Setting::PluginConfig + include Fluentd::Setting::SectionParser + include Fluentd::Setting::PluginParameter + include Fluentd::Setting::SectionValidator + + included do + cattr_accessor :plugin_type, :plugin_name, :config_definition + end + + module ClassMethods + def register_plugin(type, name) + self.plugin_type = type + self.plugin_name = name + + if ["filter", "output"].include?(type) + include Fluentd::Setting::Pattern + end + + self.load_plugin_config do |_name, params| + params.each do |param_name, definition| + if definition[:section] + parse_section(param_name, definition) + else + config_param(param_name, definition[:type], **definition.except(:type)) + end + end + end + end + + def load_plugin_config + dumped_config = {} + plugin_class.ancestors.reverse_each do |klass| + next unless klass.respond_to?(:dump_config_definition) + dumped_config_definition = klass.dump_config_definition + dumped_config[klass.name] = dumped_config_definition unless dumped_config_definition.empty? + end + self.config_definition = dumped_config + dumped_config.each do |name, config| + yield name, config + end + end + + def plugin_instance + @plugin_instance ||= Fluent::Plugin.__send__("new_#{plugin_type}", plugin_name) + end + + def plugin_class + @plugin_class ||= plugin_instance.class + end + + def plugin_helpers + @plugin_helpers ||= if plugin_instance.respond_to?(:plugin_helpers) + plugin_instance.plugin_helpers + else + [] + end + end + + def section? + false + end + end + end + end +end diff --git a/app/models/concerns/fluentd/setting/plugin_config.rb b/app/models/concerns/fluentd/setting/plugin_config.rb new file mode 100644 index 0000000..c016f8b --- /dev/null +++ b/app/models/concerns/fluentd/setting/plugin_config.rb @@ -0,0 +1,57 @@ +class Fluentd + module Setting + module PluginConfig + extend ActiveSupport::Concern + + def to_config + name = case plugin_type + when "input" + "source" + when "output" + "match" + when "filter" + "filter" + when "parser" + "parse" + when "formatter" + "format" + when "buffer" + "buffer" + end + _attributes = { "@type" => self.plugin_name }.merge(attributes) + _attributes["@log_level"] = _attributes.delete("log_level") + argument = case plugin_type + when "output", "filter", "buffer" + _attributes.delete(self._argument_name.to_s) || "" + else + "" + end + attrs, elements = parse_attributes(_attributes) + config_element(name, argument, attrs, elements) + end + + def parse_attributes(attributes) + base_klasses = config_definition.keys + sections, params = attributes.partition do |key, _section_attributes| + base_klasses.any? do |base_klass| + config_definition.dig(base_klass, key.to_sym, :section) || config_definition.dig(key.to_sym, :section) + end + end + elements = [] + sections.to_h.each do |key, section_params| + next if section_params.blank? + section_params.each do |index, _section_params| + sub_attrs, sub_elements = parse_attributes(_section_params) + elements << config_element(key, "", sub_attrs, sub_elements) + end + end + return params.to_h.reject{|key, value| value.blank? }, elements + end + + # copy from Fluent::Test::Helpers#config_element + def config_element(name = 'test', argument = '', params = {}, elements = []) + Fluent::Config::Element.new(name, argument, params, elements) + end + end + end +end diff --git a/app/models/concerns/fluentd/setting/plugin_parameter.rb b/app/models/concerns/fluentd/setting/plugin_parameter.rb new file mode 100644 index 0000000..2100541 --- /dev/null +++ b/app/models/concerns/fluentd/setting/plugin_parameter.rb @@ -0,0 +1,43 @@ +class Fluentd + module Setting + module PluginParameter + extend ActiveSupport::Concern + + include Fluentd::Setting::Configurable + + def column_type(name) + self.class._types[name] + end + + def list_of(name) + self.class._list[name] + end + + def common_options + [] + end + + def advanced_options + all_options - common_options - hidden_options + end + + def hidden_options + [] + end + + def all_options + self.class._types.keys + self.class._sections.keys + end + + module ClassMethods + def column_type(name) + self._types[name] + end + + def list_of(name) + self._list[name] + end + end + end + end +end diff --git a/app/models/concerns/fluentd/setting/section_parser.rb b/app/models/concerns/fluentd/setting/section_parser.rb new file mode 100644 index 0000000..6ae145c --- /dev/null +++ b/app/models/concerns/fluentd/setting/section_parser.rb @@ -0,0 +1,21 @@ +class Fluentd + module Setting + module SectionParser + extend ActiveSupport::Concern + + module ClassMethods + def parse_section(name, definition) + config_section(name, **definition.slice(:required, :multi, :alias)) do + definition.except(:section, :argument, :required, :multi, :alias).each do |_param_name, _definition| + if _definition[:section] + parse_section(_param_name, _definition) + else + config_param(_param_name, _definition[:type], **_definition.except(:type)) + end + end + end + end + end + end + end +end diff --git a/app/models/concerns/fluentd/setting/section_validator.rb b/app/models/concerns/fluentd/setting/section_validator.rb new file mode 100644 index 0000000..8238441 --- /dev/null +++ b/app/models/concerns/fluentd/setting/section_validator.rb @@ -0,0 +1,21 @@ +class Fluentd + module Setting + module SectionValidator + extend ActiveSupport::Concern + + included do + validate :validate_sections + end + + def validate_sections + self._section_params.each do |name, sections| + sections.each do |section| + if section.invalid? + errors.add(name, :invalid, message: section.errors.full_messages) + end + end + end + end + end + end +end