From 9b2d968bc1746bd629fa2c85e72e038c7936e2c5 Mon Sep 17 00:00:00 2001 From: uu59 Date: Tue, 10 Feb 2015 12:56:16 +0900 Subject: [PATCH] Separate Common module by concerns --- app/models/fluentd/agent/common.rb | 167 +----------------- app/models/fluentd/agent/log.rb | 120 +++++++++++++ app/models/fluentd/agent/process_operation.rb | 61 +++++++ 3 files changed, 187 insertions(+), 161 deletions(-) create mode 100644 app/models/fluentd/agent/log.rb create mode 100644 app/models/fluentd/agent/process_operation.rb diff --git a/app/models/fluentd/agent/common.rb b/app/models/fluentd/agent/common.rb index abec278..d4a3eb0 100644 --- a/app/models/fluentd/agent/common.rb +++ b/app/models/fluentd/agent/common.rb @@ -19,8 +19,13 @@ class Fluentd module Common attr_reader :extra_options + def self.included(base) + base.include Fluentd::Agent::ProcessOperation + base.include Fluentd::Agent::Log + end + # define these methods on each Agent class - %w(start stop restart version dryrun!).each do |method| + %w(start stop restart version).each do |method| define_method(method) do raise NotImplementedError, "'#{method}' method is required to be defined" end @@ -42,71 +47,6 @@ class Fluentd extra_options[:config_file] || self.class.default_options[:config_file] end - def pid - return unless File.exists?(pid_file) - return if File.zero?(pid_file) - File.read(pid_file).to_i rescue nil - end - - def running? - begin - pid && Process.kill(0, pid) - rescue Errno::ESRCH - File.unlink(pid_file) # no needed any more - false - end - end - - def dryrun(file_path = nil) - dryrun!(file_path) - true - rescue ::Fluentd::Agent::ConfigError - false - end - - # -- log - def log - return "" unless File.exists?(log_file) - File.read(log_file) # TODO: large log file - end - - def errors_since(since = 1.day.ago) - errors = [] - logged_errors do |error| - break if Time.parse(error[:subject]) < since - errors << error - end - errors - end - - def recent_errors(limit = 3) - errors = [] - logged_errors do |error| - errors << error - break if errors.length >= limit - end - errors - end - - def last_error_message - recent_errors(1).first.try(:[], :subject) || "" - end - - def log_tail(limit = nil) - return [] unless File.exists?(log_file) - - limit = limit.to_i rescue 0 - limit = limit.zero? ? Settings.default_log_tail_count : limit - io = File.open(log_file) - buf = [] - reader = ::FileReverseReader.new(io) - reader.each_line do |line| - buf << line - break if buf.length >= limit - end - buf - end - # -- config def config File.read(config_file) @@ -165,13 +105,6 @@ class Fluentd # -------------- private -------------- private - def exec_dryrun(command, file_path = nil) - Bundler.with_clean_env do - system("#{command} -q --dry-run #{options_to_argv(config_file: file_path)}", out: File::NULL, err: File::NULL) - raise ::Fluentd::Agent::ConfigError, last_error_message unless $?.exitstatus.zero? - end - end - def backup_running_config #back up config file only when start success return unless yield @@ -202,94 +135,6 @@ class Fluentd FileUtils.rm(file) if File.exist? file end end - - def logged_errors(&block) - return [] unless File.exist?(log_file) - buf = [] - io = File.open(log_file) - reader = ::FileReverseReader.new(io) - reader.each_line do |line| - unless line["error"] - if buf.present? - # NOTE: if a following log is given - # 2014-06-30 11:24:08 +0900 [error]: unexpected error error_class=Errno::EADDRINUSE error=# - # 2014-06-30 11:24:08 +0900 [error]: /Users/uu59/.rbenv/versions/2.1.2/lib/ruby/2.1.0/socket.rb:206:in `bind' - # 2014-06-30 11:24:08 +0900 [error]: /Users/uu59/.rbenv/versions/2.1.2/lib/ruby/2.1.0/socket.rb:206:in `listen' - # 2014-06-30 11:24:08 +0900 [error]: /Users/uu59/.rbenv/versions/2.1.2/lib/ruby/2.1.0/socket.rb:461:in `block in tcp_server_sockets' - # the first line become a "subject", trailing lines are "notes" - # { - # subject: "2014-06-30 11:24:08 +0900 [error]: unexpected error error_class=Errno::EADDRINUSE error=#", - # notes: [ - # 2014-06-30 11:24:08 +0900 [error]: /Users/uu59/.rbenv/versions/2.1.2/lib/ruby/2.1.0/socket.rb:206:in `bind' - # 2014-06-30 11:24:08 +0900 [error]: /Users/uu59/.rbenv/versions/2.1.2/lib/ruby/2.1.0/socket.rb:206:in `listen' - # 2014-06-30 11:24:08 +0900 [error]: /Users/uu59/.rbenv/versions/2.1.2/lib/ruby/2.1.0/socket.rb:461:in `block in tcp_server_sockets' - # ] - # } - split_error_lines_to_error_units(buf.reverse).each do |error_unit| - block.call({ - subject: error_unit[:subject], - notes: error_unit[:notes], - }) - end - end - - buf = [] - next - end - buf << line - end - ensure - io && io.close - end - - def split_error_lines_to_error_units(buf) - # NOTE: if a following log is given - # - #2014-05-27 10:54:37 +0900 [error]: unexpected error error_class=Errno::EADDRINUSE error=# - #2014-05-27 10:55:40 +0900 [error]: unexpected error error_class=Errno::EADDRINUSE error=# - # 2014-05-27 10:55:40 +0900 [error]: /Users/uu59/.rbenv/versions/2.1.0/lib/ruby/gems/2.1.0/gems/cool.io-1.2.4/lib/cool.io/server.rb:57:in `initialize' - # 2014-05-27 10:55:40 +0900 [error]: /Users/uu59/.rbenv/versions/2.1.0/lib/ruby/gems/2.1.0/gems/cool.io-1.2.4/lib/cool.io/server.rb:57:in `new' - # - #the first line and second line must be each "error_unit". and after third lines lines are "notes" of second error unit of . - # [ - # { subject: "2014-05-27 10:54:37 +0900 [error]: unexpected error error_class=Errno::EADDRINUSE error=# ", - # notes: [] }, - # { subject: "2014-05-27 10:55:40 +0900 [error]: unexpected error error_class=Errno::EADDRINUSE error=# ", - # notes: [ - # "2014-05-27 10:55:40 +0900 [error]: /Users/uu59/.rbenv/versions/2.1.0/lib/ruby/gems/2.1.0/gems/cool.io-1.2.4/lib/cool.io/server.rb:57:in `initialize'", - # "2014-05-27 10:55:40 +0900 [error]: /Users/uu59/.rbenv/versions/2.1.0/lib/ruby/gems/2.1.0/gems/cool.io-1.2.4/lib/cool.io/server.rb:57:in `new'" - # ] - # }, - # ] - # - return_array = [] - buf.each_with_index do |b, i| - if b.match(/\A /) - return_array[-1][:notes] << b - else - return_array << { subject: b, notes: [] } - end - end - return return_array.reverse - end - - def detached_command(cmd) - thread = Bundler.with_clean_env do - pid = spawn(cmd) - Process.detach(pid) - end - thread.join - thread.value.exitstatus.zero? - end - - def options_to_argv(opts = {}) - argv = "" - argv << " --use-v1-config" - argv << " -c #{opts[:config_file] || config_file}" - argv << " -d #{opts[:pid_file] || pid_file}" - argv << " -o #{opts[:log_file] || log_file}" - argv - end end end end diff --git a/app/models/fluentd/agent/log.rb b/app/models/fluentd/agent/log.rb new file mode 100644 index 0000000..3b5aa23 --- /dev/null +++ b/app/models/fluentd/agent/log.rb @@ -0,0 +1,120 @@ +class Fluentd + class Agent + module Log + def log + return "" unless File.exists?(log_file) + File.read(log_file) # TODO: large log file + end + + def errors_since(since = 1.day.ago) + errors = [] + logged_errors do |error| + break if Time.parse(error[:subject]) < since + errors << error + end + errors + end + + def recent_errors(limit = 3) + errors = [] + logged_errors do |error| + errors << error + break if errors.length >= limit + end + errors + end + + def last_error_message + recent_errors(1).first.try(:[], :subject) || "" + end + + def log_tail(limit = nil) + return [] unless File.exists?(log_file) + + limit = limit.to_i rescue 0 + limit = limit.zero? ? Settings.default_log_tail_count : limit + io = File.open(log_file) + buf = [] + reader = ::FileReverseReader.new(io) + reader.each_line do |line| + buf << line + break if buf.length >= limit + end + buf + end + + private + + def logged_errors(&block) + return [] unless File.exist?(log_file) + buf = [] + io = File.open(log_file) + reader = ::FileReverseReader.new(io) + reader.each_line do |line| + unless line["error"] + if buf.present? + # NOTE: if a following log is given + # 2014-06-30 11:24:08 +0900 [error]: unexpected error error_class=Errno::EADDRINUSE error=# + # 2014-06-30 11:24:08 +0900 [error]: /Users/uu59/.rbenv/versions/2.1.2/lib/ruby/2.1.0/socket.rb:206:in `bind' + # 2014-06-30 11:24:08 +0900 [error]: /Users/uu59/.rbenv/versions/2.1.2/lib/ruby/2.1.0/socket.rb:206:in `listen' + # 2014-06-30 11:24:08 +0900 [error]: /Users/uu59/.rbenv/versions/2.1.2/lib/ruby/2.1.0/socket.rb:461:in `block in tcp_server_sockets' + # the first line become a "subject", trailing lines are "notes" + # { + # subject: "2014-06-30 11:24:08 +0900 [error]: unexpected error error_class=Errno::EADDRINUSE error=#", + # notes: [ + # 2014-06-30 11:24:08 +0900 [error]: /Users/uu59/.rbenv/versions/2.1.2/lib/ruby/2.1.0/socket.rb:206:in `bind' + # 2014-06-30 11:24:08 +0900 [error]: /Users/uu59/.rbenv/versions/2.1.2/lib/ruby/2.1.0/socket.rb:206:in `listen' + # 2014-06-30 11:24:08 +0900 [error]: /Users/uu59/.rbenv/versions/2.1.2/lib/ruby/2.1.0/socket.rb:461:in `block in tcp_server_sockets' + # ] + # } + split_error_lines_to_error_units(buf.reverse).each do |error_unit| + block.call({ + subject: error_unit[:subject], + notes: error_unit[:notes], + }) + end + end + + buf = [] + next + end + buf << line + end + ensure + io && io.close + end + + def split_error_lines_to_error_units(buf) + # NOTE: if a following log is given + # + #2014-05-27 10:54:37 +0900 [error]: unexpected error error_class=Errno::EADDRINUSE error=# + #2014-05-27 10:55:40 +0900 [error]: unexpected error error_class=Errno::EADDRINUSE error=# + # 2014-05-27 10:55:40 +0900 [error]: /Users/uu59/.rbenv/versions/2.1.0/lib/ruby/gems/2.1.0/gems/cool.io-1.2.4/lib/cool.io/server.rb:57:in `initialize' + # 2014-05-27 10:55:40 +0900 [error]: /Users/uu59/.rbenv/versions/2.1.0/lib/ruby/gems/2.1.0/gems/cool.io-1.2.4/lib/cool.io/server.rb:57:in `new' + # + #the first line and second line must be each "error_unit". and after third lines lines are "notes" of second error unit of . + # [ + # { subject: "2014-05-27 10:54:37 +0900 [error]: unexpected error error_class=Errno::EADDRINUSE error=# ", + # notes: [] }, + # { subject: "2014-05-27 10:55:40 +0900 [error]: unexpected error error_class=Errno::EADDRINUSE error=# ", + # notes: [ + # "2014-05-27 10:55:40 +0900 [error]: /Users/uu59/.rbenv/versions/2.1.0/lib/ruby/gems/2.1.0/gems/cool.io-1.2.4/lib/cool.io/server.rb:57:in `initialize'", + # "2014-05-27 10:55:40 +0900 [error]: /Users/uu59/.rbenv/versions/2.1.0/lib/ruby/gems/2.1.0/gems/cool.io-1.2.4/lib/cool.io/server.rb:57:in `new'" + # ] + # }, + # ] + # + return_array = [] + buf.each_with_index do |b, i| + if b.match(/\A /) + return_array[-1][:notes] << b + else + return_array << { subject: b, notes: [] } + end + end + return return_array.reverse + end + end + end +end + diff --git a/app/models/fluentd/agent/process_operation.rb b/app/models/fluentd/agent/process_operation.rb new file mode 100644 index 0000000..9889ffb --- /dev/null +++ b/app/models/fluentd/agent/process_operation.rb @@ -0,0 +1,61 @@ +class Fluentd + class Agent + module ProcessOperation + def self.included(base) + define_method(:dryrun!) do + raise NotImplementedError, "'dryrun!' method is required to be defined" + end + end + + def running? + begin + pid && Process.kill(0, pid) + rescue Errno::ESRCH + File.unlink(pid_file) # no needed any more + false + end + end + + def dryrun(file_path = nil) + dryrun!(file_path) + true + rescue ::Fluentd::Agent::ConfigError + false + end + + def pid + return unless File.exists?(pid_file) + return if File.zero?(pid_file) + File.read(pid_file).to_i rescue nil + end + + private + + def exec_dryrun(command, file_path = nil) + Bundler.with_clean_env do + unless system("#{command} -q --dry-run #{options_to_argv(config_file: file_path)}", out: File::NULL, err: File::NULL) + raise ::Fluentd::Agent::ConfigError, last_error_message + end + end + end + + def detached_command(cmd) + thread = Bundler.with_clean_env do + pid = spawn(cmd) + Process.detach(pid) + end + thread.join + thread.value.exitstatus.zero? + end + + def options_to_argv(opts = {}) + argv = "" + argv << " --use-v1-config" + argv << " -c #{opts[:config_file] || config_file}" + argv << " -d #{opts[:pid_file] || pid_file}" + argv << " -o #{opts[:log_file] || log_file}" + argv + end + end + end +end