Add concerns to define models for fluent-plugin

Signed-off-by: Kenji Okimoto <okimoto@clear-code.com>

app/models/concerns/fluentd/setting/configurable.rb

@@ -0,0 +1,103 @@
+class Fluentd
+  module Setting
+    module Configurable
+      extend ActiveSupport::Concern
+
+      included do
+        class_attribute :_types, :_defaults, :_secrets, :_aliases, :_required
+        class_attribute :_deprecated_params, :_obsoleted_params, :_descriptions
+        class_attribute :_list, :_value_types, :_symbolize_keys
+        class_attribute :_argument_name, :_built_in_params, :_sections, :_section_params
+        self._types = {}
+        self._defaults = {}
+        self._secrets = {}
+        self._aliases = {}
+        self._required = {}
+        self._deprecated_params = {}
+        self._obsoleted_params = {}
+        self._descriptions = {}
+        self._list = {}
+        self._value_types = {}
+        self._symbolize_keys = {}
+        self._argument_name = nil
+        self._built_in_params = []
+        self._sections = {}
+        self._section_params = Hash.new {|h, k| h[k] = [] }
+      end
+
+      def initialize(attributes = {})
+        super rescue ActiveModel::UnknownAttributeError # the superclass does not know specific attributes of the model
+        self.class._sections.each do |name, klass|
+          klass.init
+          if klass.multi
+            next if attributes[name].nil?
+            attributes[name].each do |attr|
+              next unless attr
+              attr.each do |index, _attr|
+                self._section_params[name] << klass.new(_attr)
+              end
+            end
+          else
+            attr = attributes.dig(name, "0")
+            self._section_params[name] << klass.new(attr) if attr
+          end
+        end
+      end
+
+      module ClassMethods
+        # config_param :name, :string, default: "value", secret: true
+        def config_param(name, type = ActiveModel::Type::Value.new, **options)
+          # NOTE: We cannot overwrite types defined by ActiveModel in config/initializers/types.rb
+          if type == :time
+            type = Fluentd::Setting::Type::Time.new
+          end
+          if name.to_s.start_with?("@")
+            _name = name.to_s[1..-1]
+            config_param(_name.to_sym, type, **options.merge(alias: name))
+            self._built_in_params << _name
+          elsif ["id", "type", "log_level"].include?(name.to_s)
+            self._built_in_params << _name
+          else
+            attribute(name, type, **options.slice(:precision, :limit, :scale))
+            validates(name, presence: true) if options[:required]
+          end
+          self._types[name] = type
+          self._defaults[name] = options[:default] if options.key?(:default)
+          self._secrets[name] = options[:secret] if options.key?(:secret)
+          self._aliases[name] = options[:alias] if options.key?(:alias)
+          self._required[name] = options[:required] if options.key?(:required)
+          self._deprecated_params[name] = options[:deprecated] if options.key?(:deprecated)
+          self._obsoleted_params[name] = options[:obsoleted] if options.key?(:obsoleted)
+          self._list[name] = options[:list] if options.key?(:list)
+          self._value_types[name] = options[:value_types] if options.key?(:value_types)
+          self._symbolize_keys = options[:symbolize_keys] if options.key?(:symbolize_keys)
+        end
+
+        def config_section(name, **options, &block)
+          if self._sections.key?(name)
+            self._sections[name].merge(**options, &block)
+          else
+            attribute(name, :section)
+            section_class = Class.new(::Fluentd::Setting::Section)
+            section_class.section_name = name
+            section_class.required = options[:required]
+            section_class.multi = options[:multi]
+            section_class.alias = options[:alias]
+            section_class._block = block
+            self.const_set(name.to_s.classify, section_class)
+            self._sections[name] = section_class
+          end
+        end
+
+        def config_argument(name, type = ActiveModel::Type::Value.new, **options)
+          config_param(name, **options)
+          self._argument_name = name
+        end
+
+        def set_argument_name(name)
+          self._argument_name = name
+        end
+      end
+    end
+  end
+end

app/models/concerns/fluentd/setting/pattern.rb

@@ -0,0 +1,11 @@
+class Fluentd
+  module Setting
+    module Pattern
+      extend ActiveSupport::Concern
+
+      included do
+        config_argument(:pattern, :string)
+      end
+    end
+  end
+end

app/models/concerns/fluentd/setting/plugin.rb

@@ -0,0 +1,75 @@
+require "fluent/plugin"
+
+class Fluentd
+  module Setting
+    module Plugin
+      extend ActiveSupport::Concern
+
+      include ActiveModel::Model
+      include ActiveModel::Attributes
+      include Fluentd::Setting::Configurable
+      include Fluentd::Setting::PluginConfig
+      include Fluentd::Setting::SectionParser
+      include Fluentd::Setting::PluginParameter
+      include Fluentd::Setting::SectionValidator
+
+      included do
+        cattr_accessor :plugin_type, :plugin_name, :config_definition
+      end
+
+      module ClassMethods
+        def register_plugin(type, name)
+          self.plugin_type = type
+          self.plugin_name = name
+
+          if ["filter", "output"].include?(type)
+            include Fluentd::Setting::Pattern
+          end
+
+          self.load_plugin_config do |_name, params|
+            params.each do |param_name, definition|
+              if definition[:section]
+                parse_section(param_name, definition)
+              else
+                config_param(param_name, definition[:type], **definition.except(:type))
+              end
+            end
+          end
+        end
+
+        def load_plugin_config
+          dumped_config = {}
+          plugin_class.ancestors.reverse_each do |klass|
+            next unless klass.respond_to?(:dump_config_definition)
+            dumped_config_definition = klass.dump_config_definition
+            dumped_config[klass.name] = dumped_config_definition unless dumped_config_definition.empty?
+          end
+          self.config_definition = dumped_config
+          dumped_config.each do |name, config|
+            yield name, config
+          end
+        end
+
+        def plugin_instance
+          @plugin_instance ||= Fluent::Plugin.__send__("new_#{plugin_type}", plugin_name)
+        end
+
+        def plugin_class
+          @plugin_class ||= plugin_instance.class
+        end
+
+        def plugin_helpers
+          @plugin_helpers ||= if plugin_instance.respond_to?(:plugin_helpers)
+                                plugin_instance.plugin_helpers
+                              else
+                                []
+                              end
+        end
+
+        def section?
+          false
+        end
+      end
+    end
+  end
+end

app/models/concerns/fluentd/setting/plugin_config.rb

@@ -0,0 +1,57 @@
+class Fluentd
+  module Setting
+    module PluginConfig
+      extend ActiveSupport::Concern
+
+      def to_config
+        name = case plugin_type
+               when "input"
+                 "source"
+               when "output"
+                 "match"
+               when "filter"
+                 "filter"
+               when "parser"
+                 "parse"
+               when "formatter"
+                 "format"
+               when "buffer"
+                 "buffer"
+               end
+        _attributes = { "@type" => self.plugin_name }.merge(attributes)
+        _attributes["@log_level"] = _attributes.delete("log_level")
+        argument = case plugin_type
+                   when "output", "filter", "buffer"
+                     _attributes.delete(self._argument_name.to_s) || ""
+                   else
+                     ""
+                   end
+        attrs, elements = parse_attributes(_attributes)
+        config_element(name, argument, attrs, elements)
+      end
+
+      def parse_attributes(attributes)
+        base_klasses = config_definition.keys
+        sections, params = attributes.partition do |key, _section_attributes|
+          base_klasses.any? do |base_klass|
+            config_definition.dig(base_klass, key.to_sym, :section) || config_definition.dig(key.to_sym, :section)
+          end
+        end
+        elements = []
+        sections.to_h.each do |key, section_params|
+          next if section_params.blank?
+          section_params.each do |index, _section_params|
+            sub_attrs, sub_elements = parse_attributes(_section_params)
+            elements << config_element(key, "", sub_attrs, sub_elements)
+          end
+        end
+        return params.to_h.reject{|key, value| value.blank? }, elements
+      end
+
+      # copy from Fluent::Test::Helpers#config_element
+      def config_element(name = 'test', argument = '', params = {}, elements = [])
+        Fluent::Config::Element.new(name, argument, params, elements)
+      end
+    end
+  end
+end

app/models/concerns/fluentd/setting/plugin_parameter.rb

@@ -0,0 +1,43 @@
+class Fluentd
+  module Setting
+    module PluginParameter
+      extend ActiveSupport::Concern
+
+      include Fluentd::Setting::Configurable
+
+      def column_type(name)
+        self.class._types[name]
+      end
+
+      def list_of(name)
+        self.class._list[name]
+      end
+
+      def common_options
+        []
+      end
+
+      def advanced_options
+        all_options - common_options - hidden_options
+      end
+
+      def hidden_options
+        []
+      end
+
+      def all_options
+        self.class._types.keys + self.class._sections.keys
+      end
+
+      module ClassMethods
+        def column_type(name)
+          self._types[name]
+        end
+
+        def list_of(name)
+          self._list[name]
+        end
+      end
+    end
+  end
+end

app/models/concerns/fluentd/setting/section_parser.rb

@@ -0,0 +1,21 @@
+class Fluentd
+  module Setting
+    module SectionParser
+      extend ActiveSupport::Concern
+
+      module ClassMethods
+        def parse_section(name, definition)
+          config_section(name, **definition.slice(:required, :multi, :alias)) do
+            definition.except(:section, :argument, :required, :multi, :alias).each do |_param_name, _definition|
+              if _definition[:section]
+                parse_section(_param_name, _definition)
+              else
+                config_param(_param_name, _definition[:type], **_definition.except(:type))
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

app/models/concerns/fluentd/setting/section_validator.rb

@@ -0,0 +1,21 @@
+class Fluentd
+  module Setting
+    module SectionValidator
+      extend ActiveSupport::Concern
+
+      included do
+        validate :validate_sections
+      end
+
+      def validate_sections
+        self._section_params.each do |name, sections|
+          sections.each do |section|
+            if section.invalid?
+              errors.add(name, :invalid, message: section.errors.full_messages)
+            end
+          end
+        end
+      end
+    end
+  end
+end