#162 in_tail multiline preview

app/assets/javascripts/vue/in_tail_format.js

@@ -58,6 +58,12 @@
       },
 
       methods: {
+        onKeyup: function(ev){
+          var el = ev.target;
+          if(el.name.match(/\[format/)){
+            this.preview();
+          }
+        },
         updateHighlightedLines: function() {
           if(!this.regexpMatches) {
             this.highlightedLines = null;
@@ -132,6 +138,7 @@
                 regexp: self.params.setting.regexp,
                 time_format: self.params.setting.time_format,
                 format: _.isEmpty(self.format) ? "regexp" : self.format,
+                params: self.params.setting,
                 file: self.targetFile
               }
             }).done(resolve).fail(reject);

app/controllers/api_controller.rb

@@ -19,7 +19,7 @@ class ApiController < ApplicationController
   end
 
   def regexp_preview
-    preview = RegexpPreview.new(params[:file], params[:format], regexp: params[:regexp], time_format: params[:time_format])
+    preview = RegexpPreview.new(params[:file], params[:format], params[:params], regexp: params[:regexp], time_format: params[:time_format])
     matches = preview.matches
     render json: {
       params: {

app/models/fluentd/setting/in_tail.rb

@@ -18,6 +18,7 @@ class Fluentd
           :ltsv => [:delimiter, :time_key],
           :json => [:time_key],
           :regexp => [:time_format, :regexp],
+          :multiline => [:format_firstline] + (1..20).map{|n| "format#{n}".to_sym }
           # TODO: Grok could generate Regexp including \d, \s, etc. fluentd config parser raise error with them for escape sequence check.
           #       TBD How to handle Grok/Regexp later, just comment out for hide
           # :grok => [:grok_str],
@@ -59,8 +60,21 @@ class Fluentd
 
         indent = " " * 2
         format_specific_conf = ""
-        extra_format_options.each do |key|
+
-          format_specific_conf << "#{indent}#{key} #{send(key)}\n"
+        if format.to_sym == :multiline
+          known_formats[:multiline].each do |key|
+            value = send(key)
+            if value.present?
+              format_specific_conf << "#{indent}#{key} /#{value}/\n"
+            end
+          end
+        else
+          extra_format_options.each do |key|
+            format_specific_conf << "#{indent}#{key} #{send(key)}\n"
+          end
+        end
+
+        if format.to_sym == :multiline
         end
         format_specific_conf
       end

app/views/fluentd/settings/in_tail/after_file_choose.html.haml

@@ -12,7 +12,7 @@
     = f.text_field :path, class: "form-control", disabled: true
   = render partial: "shared/vue/in_tail_format", locals: { file: f.object.path, formats: @setting.known_formats, initialSelected: f.object.format || @setting.guess_format }
 
-  %pre= file_tail(@setting.path).join("\n")
+  %pre= file_tail(@setting.path, Settings.in_tail_preview_target_line).join("\n")
 
   %p
     = f.submit t('terms.next'), class: "btn btn-lg btn-primary pull-right"

app/views/shared/vue/_in_tail_format.html.erb

@@ -7,7 +7,7 @@
     </div>
     <div class="form-inline form-group" v-repeat="options">
       <label for="in_tail_setting_{{ $value }}">{{ $value }} </label>
-      <input id="in_tail_setting_{{ $value }}" type="text" name="setting[{{ $value }}]" v-model="params.setting[$value]" size="100%" class="form-control" />
+      <input id="in_tail_setting_{{ $value }}" type="text" name="setting[{{ $value }}]" v-model="params.setting[$value]" v-on="keyup: onKeyup" size="100%" class="form-control" />
     </div>
 </script>
 

config/application.yml

@@ -3,6 +3,7 @@ defaults: &defaults
   default_log_tail_count: 30
   histories_count_in_preview: 5
   max_backup_files_count: 100
+  in_tail_preview_target_line: 40
   recommended_plugins:
     - category: filter
       name: "rewrite-tag-filter"

lib/regexp_preview.rb

@@ -4,15 +4,19 @@ require "fluent/configurable"
 require "fluent/parser"
 
 class RegexpPreview
-  attr_reader :file, :format, :time_format, :regexp
+  attr_reader :file, :format, :params, :time_format, :regexp
 
-  def initialize(file, format, options = {})
+  def initialize(file, format, params = {}, options = {})
     @file = file
     @format = format
+    @params = params
     case format
     when "regexp"
       @regexp = Regexp.new(options[:regexp])
       @time_format = options[:time_format]
+      @strategy = :matches_single_line
+    when "multiline"
+      @strategy = :matches_multiline
     when "ltsv", "json", "csv", "tsv"
     else
       definition = Fluent::TextParser::TEMPLATE_REGISTRY.lookup(format).call
@@ -24,9 +28,15 @@ class RegexpPreview
   end
 
   def matches
-    return [] unless @regexp # such as ltsv, json, etc
+    return [] unless @strategy # such as ltsv, json, etc
+    send(@strategy)
+  end
+
+  private
+
+  def matches_single_line
     reader = FileReverseReader.new(File.open(file))
-    matches = reader.tail.map do |line|
+    matches = reader.tail(Settings.in_tail_preview_target_line).map do |line|
       result = {
         :whole => line,
         :matches => [],
@@ -45,4 +55,50 @@ class RegexpPreview
     end
     matches
   end
+
+  def matches_multiline
+    return [] if multiline_regexps.empty?
+    reader = FileReverseReader.new(File.open(file))
+    result = []
+    target_lines = reader.tail(Settings.in_tail_preview_target_line).map{|line| line << "\n" }
+    target_lines.each_with_index do |line, line_no|
+      if line.match(params[:format_firstline])
+        lines = target_lines[line_no, multiline_regexps.length]
+        next if lines.length < multiline_regexps.length
+        ret = multiline_detect(lines, multiline_regexps)
+        next unless ret
+        result << ret
+      end
+    end
+    result
+  end
+
+  def multiline_detect(lines, regexps)
+    whole = ""
+    matches = []
+    lines.each_with_index do |line, j|
+      m = line.match(multiline_regexps[j])
+      unless m
+        return nil
+      end
+      m.names.each_with_index do |name, index|
+        matches << {
+          key: name,
+          matched: m[name],
+          pos: m.offset(index + 1).map{|pos| pos + whole.length},
+        }
+      end
+      whole << line
+    end
+    {
+      whole: whole,
+      matches: matches,
+    }
+  end
+
+  def multiline_regexps
+    @multiline_regexps ||= (1..20).map do |n|
+      params["format#{n}"].presence
+    end.compact
+  end
 end