diff --git a/app/assets/javascripts/vue/in_tail_format.js b/app/assets/javascripts/vue/in_tail_format.js index 2e79ffe..a5944e6 100644 --- a/app/assets/javascripts/vue/in_tail_format.js +++ b/app/assets/javascripts/vue/in_tail_format.js @@ -1,5 +1,6 @@ (function(){ "use strict"; + var maxFormatCount = 20; $(function(){ if($('#in_tail_format').length === 0) return; @@ -30,7 +31,20 @@ highlightedLines: null, }, + computed: { + useTextArea: function() { + return this.format === "multiline"; + } + }, + compiled: function(){ + this.$watch('params.setting.formats', function(formats){ + _.range(1, maxFormatCount + 1).forEach(function(i) {params.setting["format" + String(i)] = "";}); + + _.compact(formats.split("\n")).forEach(function(formatLine, index) { + params.setting["format" + String(index + 1)] = formatLine; + }); + }), this.$watch('params.setting.regexp', function(){ this.preview(); }); @@ -46,6 +60,10 @@ if(!params.setting) { params.setting = {}; } + + var formats = _.chain(_.range(1, maxFormatCount + 1)).map(function(i) {return params.setting["format" + String(i)];}).compact().value(); + params.setting.formats = formats.join("\n"); + _.each(this.formatOptions, function(options){ _.each(options, function(key){ if(!params.setting.hasOwnProperty(key)){ @@ -58,6 +76,12 @@ }, methods: { + onKeyup: function(ev){ + var el = ev.target; + if(el.name.match(/\[format/)){ + this.preview(); + } + }, updateHighlightedLines: function() { if(!this.regexpMatches) { this.highlightedLines = null; @@ -132,6 +156,7 @@ regexp: self.params.setting.regexp, time_format: self.params.setting.time_format, format: _.isEmpty(self.format) ? "regexp" : self.format, + params: self.params.setting, file: self.targetFile } }).done(resolve).fail(reject); diff --git a/app/controllers/api_controller.rb b/app/controllers/api_controller.rb index 998b784..c162390 100644 --- a/app/controllers/api_controller.rb +++ b/app/controllers/api_controller.rb @@ -19,17 +19,9 @@ class ApiController < ApplicationController end def regexp_preview - preview = RegexpPreview.new(params[:file], params[:format], regexp: params[:regexp], time_format: params[:time_format]) - matches = preview.matches - render json: { - params: { - setting: { - regexp: preview.regexp.try(:source), - time_format: preview.time_format, - } - }, - matches: matches.compact, - } + preview = RegexpPreview.processor(params[:format]).new(params[:file], params[:format], params) + + render json: preview.matches_json end def grok_to_regexp diff --git a/app/models/fluentd/setting/in_tail.rb b/app/models/fluentd/setting/in_tail.rb index e5a009f..1d160bb 100644 --- a/app/models/fluentd/setting/in_tail.rb +++ b/app/models/fluentd/setting/in_tail.rb @@ -1,6 +1,8 @@ class Fluentd module Setting class InTail + MULTI_LINE_MAX_FORMAT_COUNT = 20 + include ActiveModel::Model attr_accessor :path, :tag, :format, :regexp, :time_format, :rotate_wait, :pos_file, :read_from_head, :refresh_interval @@ -18,12 +20,13 @@ class Fluentd :ltsv => [:delimiter, :time_key], :json => [:time_key], :regexp => [:time_format, :regexp], + :multiline => [:format_firstline] + (1..MULTI_LINE_MAX_FORMAT_COUNT).map{|n| "format#{n}".to_sym } # TODO: Grok could generate Regexp including \d, \s, etc. fluentd config parser raise error with them for escape sequence check. # TBD How to handle Grok/Regexp later, just comment out for hide # :grok => [:grok_str], } end - attr_accessor *known_formats.values.flatten.compact + attr_accessor *known_formats.values.flatten.compact.uniq def known_formats self.class.known_formats @@ -59,9 +62,20 @@ class Fluentd indent = " " * 2 format_specific_conf = "" - extra_format_options.each do |key| - format_specific_conf << "#{indent}#{key} #{send(key)}\n" + + if format.to_sym == :multiline + known_formats[:multiline].each do |key| + value = send(key) + if value.present? + format_specific_conf << "#{indent}#{key} /#{value}/\n" + end + end + else + extra_format_options.each do |key| + format_specific_conf << "#{indent}#{key} #{send(key)}\n" + end end + format_specific_conf end diff --git a/app/views/fluentd/settings/in_tail/after_file_choose.html.haml b/app/views/fluentd/settings/in_tail/after_file_choose.html.haml index 79da579..8c494c9 100644 --- a/app/views/fluentd/settings/in_tail/after_file_choose.html.haml +++ b/app/views/fluentd/settings/in_tail/after_file_choose.html.haml @@ -12,7 +12,7 @@ = f.text_field :path, class: "form-control", disabled: true = render partial: "shared/vue/in_tail_format", locals: { file: f.object.path, formats: @setting.known_formats, initialSelected: f.object.format || @setting.guess_format } - %pre= file_tail(@setting.path).join("\n") + %pre= file_tail(@setting.path, Settings.in_tail_preview_line_count).join("\n") %p = f.submit t('terms.next'), class: "btn btn-lg btn-primary pull-right" diff --git a/app/views/shared/vue/_in_tail_format.html.erb b/app/views/shared/vue/_in_tail_format.html.erb index 8537739..eb642c6 100644 --- a/app/views/shared/vue/_in_tail_format.html.erb +++ b/app/views/shared/vue/_in_tail_format.html.erb @@ -6,8 +6,21 @@
- - + + +
+ +
+
+ + +
+ +
+

<%= t("fluentd.settings.in_tail.notice_for_multiline_limit") %>

+ + +
diff --git a/config/application.yml b/config/application.yml index ee5fc90..c2df865 100644 --- a/config/application.yml +++ b/config/application.yml @@ -3,6 +3,7 @@ defaults: &defaults default_log_tail_count: 30 histories_count_in_preview: 5 max_backup_files_count: 100 + in_tail_preview_line_count: 40 recommended_plugins: - category: filter name: "rewrite-tag-filter" diff --git a/config/locales/translation_en.yml b/config/locales/translation_en.yml index 73896d8..eaaef1a 100644 --- a/config/locales/translation_en.yml +++ b/config/locales/translation_en.yml @@ -211,6 +211,7 @@ en: For each config parameter, please refer to the Tail input plugin documentation page. in_tail: notice_for_permission: "Please check permission or group setting for %{user} user can read it." + notice_for_multiline_limit: "Please input Regexp(s) separated by newline. blank lines are ignored. Lines more than 20 are dropped." restart_from_first: Restart from first grok_manual: |

diff --git a/config/locales/translation_ja.yml b/config/locales/translation_ja.yml index c8da92b..9a41a0b 100644 --- a/config/locales/translation_ja.yml +++ b/config/locales/translation_ja.yml @@ -216,6 +216,7 @@ ja: in_tailプラグインの解説ページや Fluentularもご参照ください。 in_tail: + notice_for_multiline_limit: "改行区切りで正規表現を入力してください。空行はカウントされません。21行目以降の入力は無視されます。" notice_for_permission: "※%{user}ユーザーが読み込み可能なようにパーミッションやグループの設定をご確認ください。" restart_from_first: 最初からやり直す grok_manual: | diff --git a/lib/regexp_preview.rb b/lib/regexp_preview.rb index 74af30b..bb0c81f 100644 --- a/lib/regexp_preview.rb +++ b/lib/regexp_preview.rb @@ -2,47 +2,16 @@ require "fluent/registry" require "fluent/configurable" require "fluent/parser" +require "regexp_preview/single_line" +require "regexp_preview/multi_line" -class RegexpPreview - attr_reader :file, :format, :time_format, :regexp - - def initialize(file, format, options = {}) - @file = file - @format = format +module RegexpPreview + def self.processor(format) case format - when "regexp" - @regexp = Regexp.new(options[:regexp]) - @time_format = options[:time_format] - when "ltsv", "json", "csv", "tsv" + when "multiline" + RegexpPreview::MultiLine else - definition = Fluent::TextParser::TEMPLATE_REGISTRY.lookup(format).call - raise "Unknown format '#{format}'" unless definition - definition.configure({}) # NOTE: SyslogParser define @regexp in configure method so call it to grab Regexp object - @regexp = definition.patterns["format"] - @time_format = definition.patterns["time_format"] + RegexpPreview::SingleLine end end - - def matches - return [] unless @regexp # such as ltsv, json, etc - reader = FileReverseReader.new(File.open(file)) - matches = reader.tail.map do |line| - result = { - :whole => line, - :matches => [], - } - m = line.match(regexp) - next result unless m - - m.names.each_with_index do |name, index| - result[:matches] << { - key: name, - matched: m[name], - pos: m.offset(index + 1), - } - end - result - end - matches - end end diff --git a/lib/regexp_preview/multi_line.rb b/lib/regexp_preview/multi_line.rb new file mode 100644 index 0000000..7c291bb --- /dev/null +++ b/lib/regexp_preview/multi_line.rb @@ -0,0 +1,70 @@ +module RegexpPreview + class MultiLine + attr_reader :file, :format, :params + + def initialize(file, format, params = {}) + @file = file + @format = format + @params = params[:params] + end + + def matches_json + { + params: { + setting: { # for vue.js + regexp: nil, + time_format: nil, + } + }, + matches: matches.compact, + } + end + + private + + def matches + return [] if patterns.empty? + reader = FileReverseReader.new(File.open(file)) + result = [] + target_lines = reader.tail(Settings.in_tail_preview_line_count).map{|line| line << "\n" } + target_lines.each_with_index do |line, line_no| + if line.match(params[:format_firstline]) + lines = target_lines[line_no, patterns.length] + next if lines.length < patterns.length + ret = detect_chunk(lines) + next unless ret + result << ret + end + end + result + end + + def detect_chunk(lines) + whole = "" + matches = [] + lines.each_with_index do |line, i| + match = line.match(patterns[i]) + return nil unless match + match.names.each_with_index do |name, index| + matches << { + key: name, + matched: match[name], + pos: match.offset(index + 1).map{|pos| pos + whole.length}, + } + end + whole << line + end + { + whole: whole, + matches: matches, + } + end + + def patterns + @patterns ||= (1..20).map do |n| + params["format#{n}"].presence + end.compact.map {|pattern| Regexp.new(pattern)} + end + end +end + diff --git a/lib/regexp_preview/single_line.rb b/lib/regexp_preview/single_line.rb new file mode 100644 index 0000000..1981e73 --- /dev/null +++ b/lib/regexp_preview/single_line.rb @@ -0,0 +1,65 @@ +module RegexpPreview + class SingleLine + attr_reader :file, :format, :params, :regexp, :time_format + + def initialize(file, format, params = {}) + @file = file + @format = format + @time_format = params[:time_format] + @params = params + + case format + when "regexp" + @regexp = Regexp.new(params[:regexp]) + @time_format = nil + when "ltsv", "json", "csv", "tsv" + @regexp = nil + @time_format = nil + else # apache, nginx, etc + definition = Fluent::TextParser::TEMPLATE_REGISTRY.lookup(format).call + raise "Unknown format '#{format}'" unless definition + definition.configure({}) # NOTE: SyslogParser define @regexp in configure method so call it to grab Regexp object + @regexp = definition.patterns["format"] + @time_format = definition.patterns["time_format"] + end + end + + def matches_json + { + params: { + setting: { + # NOTE: regexp and time_format are used when format == 'apache' || 'nginx' || etc. + regexp: regexp.try(:source), + time_format: time_format, + } + }, + matches: matches.compact, + } + end + + private + + def matches + return [] unless @regexp # such as ltsv, json, etc + reader = FileReverseReader.new(File.open(file)) + matches = reader.tail(Settings.in_tail_preview_line_count).map do |line| + result = { + :whole => line, + :matches => [], + } + match = line.match(regexp) + next result unless match + + match.names.each_with_index do |name, index| + result[:matches] << { + key: name, + matched: match[name], + pos: match.offset(index + 1), + } + end + result + end + matches + end + end +end diff --git a/spec/lib/regexp_preview/multi_line_spec.rb b/spec/lib/regexp_preview/multi_line_spec.rb new file mode 100644 index 0000000..100c6ff --- /dev/null +++ b/spec/lib/regexp_preview/multi_line_spec.rb @@ -0,0 +1,40 @@ +require 'spec_helper' + +describe RegexpPreview::MultiLine do + describe "#matches_json" do + subject { RegexpPreview::MultiLine.new(File.expand_path("./spec/support/fixtures/error0.log", Rails.root), "multiline", params).matches_json } + + let :params do + params = { + format_firstline: ".+", + time_format: "time_format", + } + params["format1"] = "(?foo)" + params["format2"] = "(?bar)" + 3.upto(Fluentd::Setting::InTail::MULTI_LINE_MAX_FORMAT_COUNT) do |i| + params["format#{i}"] = "" + end + { params: params } + end + + it 'should not have regexp and time_format in [:params][:setting]' do + expect(subject[:params][:setting]).to eq({ regexp: nil, time_format: nil }) + end + + it "should include matches info" do + matches_info = { + whole: "foo\nbar\n", + matches: [ + { + key: "foo", matched: "foo", pos: [0, 3] + }, + { + key: "bar", matched: "bar", pos: [4, 7] + } + ] + } + + expect(subject[:matches]).to include matches_info + end + end +end diff --git a/spec/lib/regexp_preview/single_line_spec.rb b/spec/lib/regexp_preview/single_line_spec.rb new file mode 100644 index 0000000..57cce8f --- /dev/null +++ b/spec/lib/regexp_preview/single_line_spec.rb @@ -0,0 +1,183 @@ +require 'spec_helper' + +describe RegexpPreview::SingleLine do + describe ".initialize" do + subject { RegexpPreview::SingleLine.new("log_file.log", format, params) } + + describe "format" do + let :params do + { + regexp: "(?\[.+\])", + time_format: "%y/%m/%d", + } + end + + shared_examples "should set regexp and time_format from selected format" do + it do + expect(subject.regexp).to eq regexp + expect(subject.time_format).to eq time_format + expect(subject.params).to eq params + end + end + + shared_examples "should set params only" do + include_examples "should set regexp and time_format from selected format" do + let(:regexp) { nil } + let(:time_format) { nil } + end + end + + context "regexp" do + let(:format) { "regexp" } + + it 'should set regexp from params' do + expect(subject.regexp).to eq /#{params[:regexp]}/ + expect(subject.time_format).to be_nil + expect(subject.params).to eq params + end + end + + context "ltsv" do + let(:format) { "ltsv" } + + include_examples "should set params only" + end + + context "json" do + let(:format) { "json" } + + include_examples "should set params only" + end + + context "csv" do + let(:format) { "csv" } + + include_examples "should set params only" + end + + context "tsv" do + let(:format) { "tsv" } + + include_examples "should set params only" + end + + context "syslog" do # "apache", "nginx", etc + let(:format) { "syslog" } + + include_examples "should set regexp and time_format from selected format" do + let(:regexp) do + /^(?