Merge pull request #120 from fluent/config_history

Config history

app/controllers/fluentd/settings/histories_controller.rb

@@ -0,0 +1,26 @@
+class Fluentd::Settings::HistoriesController < ApplicationController
+  before_action :login_required
+  before_action :find_fluentd
+  before_action :find_backup_file, only: [:show, :reuse]
+
+  def index
+    @backup_files = @fluentd.agent.backup_files_in_new_order.map do |file_path|
+      Fluentd::Setting::BackupFile.new(file_path)
+    end
+  end
+
+  def show
+  end
+
+  def reuse
+    @fluentd.agent.config_write @backup_file.content
+    redirect_to daemon_setting_path, flash: { success: t('messages.config_successfully_copied',  brand: fluentd_ui_brand) }
+  end
+
+  private
+
+  def find_backup_file
+    #Do not use BackupFile.new(params[:id]) because params[:id] can be any path.
+    @backup_file = Fluentd::Setting::BackupFile.find_by_file_id(@fluentd.agent.config_backup_dir, params[:id])
+  end
+end

app/controllers/fluentd/settings/running_backup_controller.rb

@@ -0,0 +1,19 @@
+class Fluentd::Settings::RunningBackupController < ApplicationController
+  before_action :login_required
+  before_action :find_fluentd
+  before_action :find_backup_file, only: [:show, :reuse]
+
+  def show
+  end
+
+  def reuse
+    @fluentd.agent.config_write @backup_file.content
+    redirect_to daemon_setting_path, flash: { success: t('messages.config_successfully_copied',  brand: fluentd_ui_brand) }
+  end
+
+  private
+
+  def find_backup_file
+    @backup_file = Fluentd::Setting::BackupFile.new(@fluentd.agent.running_config_backup_file)
+  end
+end

app/controllers/fluentd/settings_controller.rb

@@ -6,6 +6,11 @@ class Fluentd::SettingsController < ApplicationController
   before_action :set_config, only: [:show, :edit, :update]
 
   def show
+    @backup_files = @fluentd.agent.backup_files_in_new_order.first(Settings.histories_count_in_preview).map do |file_path|
+      Fluentd::Setting::BackupFile.new(file_path)
+    end
+
+    @running_backedup_file = Fluentd::Setting::BackupFile.new(@fluentd.agent.running_config_backup_file)
   end
 
   def edit

app/models/fluentd/agent/common.rb

@@ -57,6 +57,21 @@ class Fluentd
         extra_options[:config_file] || self.class.default_options[:config_file]
       end
 
+      def config_backup_dir
+        dir = File.join(FluentdUI.data_dir, "#{Rails.env}_confg_backups")
+        FileUtils.mkdir_p(dir)
+        dir
+      end
+
+      def running_config_backup_dir
+        dir = File.join(FluentdUI.data_dir, "#{Rails.env}_running_confg_backup")
+        FileUtils.mkdir_p(dir)
+        dir
+      end
+
+      def running_config_backup_file
+        File.join(running_config_backup_dir, "running.conf")
+      end
 
       # define these methods on each Agent class
 
@@ -71,6 +86,19 @@ class Fluentd
           raise NotImplementedError, "'#{method}' method is required to be defined"
         end
       end
+
+      private
+
+      def backup_running_config
+        #back up config file only when start success
+        return unless yield
+
+        return unless File.exists? config_file
+
+        FileUtils.cp config_file, running_config_backup_file
+
+        true
+      end
     end
   end
 end

app/models/fluentd/agent/fluentd_gem.rb

@@ -15,7 +15,10 @@ class Fluentd
       # return value is status_after_this_method_called == started
       def start
         return true if running?
-        actual_start
+
+        backup_running_config do
+          actual_start
+        end
       end
 
       # return value is status_after_this_method_called == stopped
@@ -67,6 +70,7 @@ class Fluentd
         Bundler.with_clean_env do
           spawn("fluentd #{options_to_argv}")
         end
+
         wait_starting
       end
 

app/models/fluentd/agent/local_common.rb

@@ -20,12 +20,14 @@ class Fluentd
       end
 
       def config_write(content)
+        backup_config
         File.open(config_file, "w") do |f|
           f.write content
         end
       end
 
       def config_append(content)
+        backup_config
         File.open(config_file, "a") do |f|
           f.write "\n"
           f.write content
@@ -59,8 +61,39 @@ class Fluentd
         File.read(pid_file).to_i rescue nil
       end
 
+      def backup_files
+        Dir.glob(File.join("#{config_backup_dir}", "*.conf"))
+      end
+
+      def backup_files_in_old_order
+        backup_files.sort
+      end
+
+      def backup_files_in_new_order
+        backup_files_in_old_order.reverse
+      end
+
       private
 
+      def backup_config
+        return unless File.exists? config_file
+
+        FileUtils.cp config_file, File.join(config_backup_dir, "#{Time.zone.now.strftime('%Y%m%d_%H%M%S')}.conf")
+
+        remove_over_backup_files
+      end
+
+      def remove_over_backup_files
+        over_file_count = backup_files.size - ::Settings.max_backup_files_count
+
+        return if over_file_count <= 0
+
+        backup_files_in_old_order.first(over_file_count).each do |file|
+          next unless File.exist? file
+          FileUtils.rm(file)
+        end
+      end
+
       def logged_errors(&block)
         return [] unless File.exist?(log_file)
         buf = []

app/models/fluentd/agent/td_agent/macosx.rb

@@ -4,7 +4,9 @@ class Fluentd
       module Macosx
 
         def start
-          detached_command("launchctl load #{plist}") && pid_from_launchctl
+          backup_running_config do
+            detached_command("launchctl load #{plist}") && pid_from_launchctl
+          end
         end
 
         def stop

app/models/fluentd/agent/td_agent/unix.rb

@@ -3,7 +3,9 @@ class Fluentd
     class TdAgent
       module Unix
         def start
-          detached_command('/etc/init.d/td-agent start')
+          backup_running_config do
+            detached_command('/etc/init.d/td-agent start')
+          end
         end
 
         def stop

app/models/fluentd/setting/backup_file.rb

@@ -0,0 +1,41 @@
+class Fluentd
+  module Setting
+    class BackupFile
+      attr_accessor :file_path
+
+      def self.find_by_file_id(backup_dir, file_id)
+        file_path = Pathname.new(backup_dir).join("#{file_id}.conf")
+        raise "No such a file #{file_path}" unless File.exist?(file_path)
+
+        new(file_path)
+      end
+
+      def initialize(file_path)
+        @file_path = file_path
+      end
+
+      def file_id
+        @file_id ||= with_file { name.gsub(/.conf\Z/,'') }
+      end
+
+      def name
+        @name ||= with_file { File.basename(file_path) }
+      end
+
+      def content
+        @content ||= with_file { File.open(file_path, "r") { |f| f.read } }
+      end
+
+      def ctime
+        with_file { File.ctime(file_path) }
+      end
+
+      private
+
+      def with_file
+        return nil unless file_path && File.exist?(file_path)
+        yield
+      end
+    end
+  end
+end

app/models/fluentd/setting/config.rb

@@ -28,7 +28,8 @@ class Fluentd
       end
 
       def write_to_file
-        File.open(file, "w"){|f| f.write formatted }
+        return unless Fluentd.instance
+        Fluentd.instance.agent.config_write formatted
       end
 
       def formatted

app/views/fluentd/settings/histories/_list.html.haml

@@ -0,0 +1,6 @@
+%ul
+  - @backup_files.each do |file|
+    %li
+      = link_to(file.name, daemon_setting_history_path(id: file.file_id))
+      %label= t('terms.backup_time')
+      = file.ctime.strftime(I18n.t 'time.formats.default')

app/views/fluentd/settings/histories/index.html.haml

@@ -0,0 +1,3 @@
+- page_title t('.page_title', label: @fluentd.label)
+
+= render 'list'

app/views/fluentd/settings/histories/show.html.haml

@@ -0,0 +1,11 @@
+- page_title t('.page_title', label: @fluentd.label) do
+  - link_to reuse_daemon_setting_history_path(id: @backup_file.file_id), method: 'post', class: "btn btn-primary pull-right" do
+    = icon('fa-pencil')
+    = t("terms.reuse")
+
+.row
+  .col-xs-12
+    %pre
+      = preserve do
+        = @backup_file.content
+

app/views/fluentd/settings/running_backup/show.html.haml

@@ -0,0 +1,15 @@
+- page_title t('.page_title', label: @fluentd.label) do
+  - if @backup_file.content
+    - link_to reuse_daemon_setting_running_backup_path, method: 'post', class: "btn btn-primary pull-right" do
+      = icon('fa-pencil')
+      = t("terms.reuse")
+
+.row
+  .col-xs-12
+    - if @backup_file.content
+      %pre
+        = preserve do
+          = @backup_file.content
+    - else
+      %p
+        =t('fluentd.common.never_started_yet', brand: fluentd_ui_brand)

app/views/fluentd/settings/show.html.haml

@@ -8,3 +8,16 @@
     %pre
       = preserve do
         = @config
+
+.row
+  .col-xs-12
+    %h3= link_to(t('fluentd.settings.running_backup.title'), daemon_setting_running_backup_path)
+    %p
+      %label= t('terms.backup_time')
+      = @running_backedup_file.ctime.try(:strftime, I18n.t('time.formats.default')) || t('fluentd.common.never_started_yet', brand: fluentd_ui_brand)
+
+.row
+  .col-xs-12
+    %h3= t('fluentd.settings.history')
+    = render '/fluentd/settings/histories/list'
+    .link= link_to t('.link_to_histories'), daemon_setting_histories_path

config/application.yml

@@ -1,6 +1,8 @@
 defaults: &defaults
   default_password: changeme
   default_log_tail_count: 30
+  histories_count_in_preview: 5
+  max_backup_files_count: 100
   recommended_plugins:
     - category: filter
       name: "rewrite-tag-filter"

config/locales/translation_en.yml

@@ -11,6 +11,7 @@ en:
     password_successfully_updated: "Your password has been changed successfully."
     fluentd_status_running: Running
     fluentd_status_stopped: Stopped
+    config_successfully_copied: "Config has been copied successfully. Please restart %{brand} to use the new config"
 
   terms: &terms
     name: Name
@@ -58,6 +59,8 @@ en:
     notice_restart_for_config_edit: "NOTICE: running %{brand} will restart after update config"
     lines: Lines
     languages: Language
+    backup_time: Backed up at
+    reuse: reuse
 
   plugins:
     view_on_rubygems_org: View on rubygems.org
@@ -117,6 +120,7 @@ en:
         Delete %{brand} setting.
 
         <p>Running %{brand} will be stopped, but log and config file are still exists.</p>
+      never_started_yet: "%{brand} had never been started yet."
     show:
       page_title: Dashboard
     new:
@@ -138,6 +142,7 @@ en:
       show:
         page_title: Config File
         in_out_head: In/Out setting
+        link_to_histories: See more histories
       edit:
         page_title: Edit Config File
       out_forward:
@@ -245,6 +250,16 @@ en:
           page_title: "Config Parameters"
         confirm:
           page_title: "Confirmation"
+      history: "Setting History"
+      histories:
+        index:
+          page_title: "Setting Histories"
+        show:
+          page_title: "Reuse Setting History"
+      running_backup:
+        title: "Running Config"
+        show:
+          page_title: "Running Config"
 
   misc:
     information:

config/locales/translation_ja.yml

@@ -11,6 +11,7 @@ ja:
     password_successfully_updated: "パスワードを変更しました。"
     fluentd_status_running: 稼働中
     fluentd_status_stopped: 停止中
+    config_successfully_copied: "設定をコピーしました。反映させるには、 %{brand}を再起動してください。"
 
   terms: &terms
     name: アカウント名
@@ -58,6 +59,8 @@ ja:
     notice_restart_for_config_edit: "※更新すると稼働中の%{brand}が再起動されます"
     lines: 行
     languages: 言語
+    backup_time: バックアップ日時
+    reuse: 再利用
 
   plugins:
     view_on_rubygems_org: rubygems.orgで見る
@@ -116,6 +119,7 @@ ja:
         %{brand}の設定を削除します。
 
         <p>起動中の%{brand}は停止し、ログや設定ファイルはそのまま残存します。</p>
+      never_started_yet: "%{brand} は起動されたことがありません。"
     show:
       page_title: "ダッシュボード"
     new:
@@ -136,6 +140,7 @@ ja:
         setting_empty: 設定されていません
       show:
         page_title: 設定ファイルの編集
+        link_to_histories: さらに過去の履歴をみる
       edit:
         page_title: 設定ファイルの編集
       out_forward:
@@ -250,6 +255,16 @@ ja:
           page_title: "ファイル読み込み | その他の設定"
         confirm:
           page_title: "ファイル読み込み | 確認"
+      history: 設定履歴
+      histories:
+        index:
+          page_title: "設定履歴 | 一覧"
+        show:
+          page_title: "設定履歴 | 詳細"
+      running_backup:
+        title: "使用中の設定"
+        show:
+          page_title: "使用中の設定"
 
   misc:
     information:

config/routes.rb

@@ -63,6 +63,14 @@ Rails.application.routes.draw do
         resource :out_elasticsearch, only: [:show], module: :settings, controller: :out_elasticsearch do
           post "finish"
         end
+
+        resources :histories, only: [:index, :show], module: :settings, controller: :histories do
+          post "reuse", action: 'reuse', on: :member, as: 'reuse'
+        end
+
+        resource :running_backup, only: [:show], module: :settings, controller: :running_backup do
+          post "reuse", action: 'reuse', on: :member, as: 'reuse'
+        end
       end
     end
   end

spec/features/fluentd/setting/hisotries_spec.rb

@@ -0,0 +1,48 @@
+require "spec_helper"
+
+describe "histories", stub: :daemon do
+  let!(:exists_user) { build(:user) }
+  include_context 'daemon has some config histories'
+
+  before do
+    login_with exists_user
+  end
+
+  describe 'index' do
+    before do
+      visit '/daemon/setting/histories'
+    end
+
+    it 'show histories#index' do
+      page.should have_css('h1', text: I18n.t('fluentd.settings.histories.index.page_title'))
+      expect(all('.row li').count).to eq 9 #links to hisotries#show
+    end
+
+    it 'will go to histories#show' do
+      all('.row li a').first.click
+
+      page.should have_css('h1', text: I18n.t('fluentd.settings.histories.show.page_title'))
+    end
+  end
+
+  describe 'show' do
+    let(:last_backup_file) { Fluentd::Setting::BackupFile.new(daemon.agent.backup_files_in_new_order.first) }
+
+    before do
+      visit "/daemon/setting/histories/#{last_backup_file.file_id}"
+    end
+
+    it 'show histories#show' do
+      page.should have_css('h1', text: I18n.t('fluentd.settings.histories.show.page_title'))
+      page.should have_text(last_backup_file.content)
+    end
+
+    it 'update config and redirect to setting#show' do
+      click_link I18n.t("terms.reuse")
+
+      page.should have_css('h1', text: I18n.t('fluentd.settings.show.page_title'))
+      page.should have_text(I18n.t('messages.config_successfully_copied', brand: 'fluentd') )
+      page.should have_text(last_backup_file.content)
+    end
+  end
+end

spec/features/fluentd/setting/ranning_backup_spec.rb

@@ -0,0 +1,48 @@
+require "spec_helper"
+
+describe "running_backup", stub: :daemon do
+  let!(:exists_user) { build(:user) }
+  include_context 'daemon has some config histories'
+
+  before do
+    login_with exists_user
+  end
+
+  context 'has running backup file' do
+    before do
+      visit '/daemon/setting/running_backup'
+    end
+
+    describe 'show' do
+      it 'has no content, no reuse bottun' do
+        expect(page).to have_text(I18n.t('fluentd.common.never_started_yet', brand: 'fluentd'))
+        expect(page).not_to have_css('pre')
+        expect(page).not_to have_text(I18n.t("terms.reuse"))
+      end
+    end
+  end
+
+  context 'has running backup file' do
+    include_context 'daemon had been started once'
+
+    before do
+      visit '/daemon/setting/running_backup'
+    end
+
+    describe 'show' do
+      it 'has content, reuse bottun' do
+        expect(page).not_to have_text(I18n.t('fluentd.common.never_started_yet', brand: 'fluentd'))
+        expect(page).to have_text(backup_content)
+        expect(page).to have_text(I18n.t("terms.reuse"))
+      end
+
+      it 'update config and redirect to setting#show' do
+        click_link I18n.t("terms.reuse")
+
+        page.should have_css('h1', text: I18n.t('fluentd.settings.show.page_title'))
+        page.should have_text(I18n.t('messages.config_successfully_copied', brand: 'fluentd') )
+        page.should have_text(backup_content)
+      end
+    end
+  end
+end

spec/features/setting_spec.rb

@@ -2,6 +2,8 @@ require 'spec_helper'
 
 describe 'setting', stub: :daemon do
   let!(:exists_user) { build(:user) }
+  include_context 'daemon has some config histories'
+  include_context 'daemon had been started once'
 
   before do
     login_with exists_user
@@ -15,6 +17,21 @@ describe 'setting', stub: :daemon do
     page.should have_css('h1', text: I18n.t('fluentd.settings.show.page_title'))
     page.should have_link(I18n.t('terms.edit'))
     page.should have_css('pre', text: 'GREAT CONFIG HERE')
+    expect(all('.row li').count).to eq Settings.histories_count_in_preview #links to hisotries#show
+    page.should have_link(I18n.t('fluentd.settings.show.link_to_histories'))
+    page.should have_text(I18n.t('fluentd.settings.running_backup.title'))
+  end
+
+  it 'will go to histories#index' do
+    click_link I18n.t('fluentd.settings.show.link_to_histories')
+
+    page.should have_css('h1', text: I18n.t('fluentd.settings.histories.index.page_title'))
+  end
+
+  it 'will go to histories#show' do
+    all('.row li a').first.click
+
+    page.should have_css('h1', text: I18n.t('fluentd.settings.histories.show.page_title'))
   end
 
   it 'edits setting' do

spec/models/fluentd/agent/local_common_spec.rb

@@ -2,6 +2,10 @@ require 'spec_helper'
 require 'fileutils'
 
 describe 'Fluentd::Agent::LocalCommon' do
+  let!(:now) { Time.zone.now }
+  before { Timecop.freeze(now) }
+  after { Timecop.return }
+
   subject { target_class.new.tap{|t| t.pid_file = pid_file_path} }
 
   let!(:target_class) { Struct.new(:pid_file){ include Fluentd::Agent::LocalCommon } }
@@ -26,4 +30,45 @@ describe 'Fluentd::Agent::LocalCommon' do
       its(:pid) { should eq(9999) }
     end
   end
+
+  describe '#config_write', stub: :daemon do
+    let(:config_contents) { <<-CONF.strip_heredoc }
+      <source>
+        type forward
+        port 24224
+      </source>
+    CONF
+
+    let(:new_config) { <<-CONF.strip_heredoc }
+      <source>
+        type http
+        port 8899
+      </source>
+    CONF
+
+    before do
+      ::Settings.max_backup_files_count.times do |i|
+        backpued_time = now - (i + 1).hours
+        FileUtils.touch File.join(daemon.agent.config_backup_dir , "#{backpued_time.strftime('%Y%m%d_%H%M%S')}.conf")
+      end
+
+      daemon.agent.config_write config_contents #add before conf
+      daemon.agent.config_write new_config #update conf
+    end
+
+    after do
+      FileUtils.rm_r daemon.agent.config_backup_dir, force: true
+    end
+
+    it 'backed up old conf' do
+      backup_file = File.join(daemon.agent.config_backup_dir, "#{now.strftime('%Y%m%d_%H%M%S')}.conf")
+      expect(File.exists? backup_file).to be_truthy
+      expect(File.read(backup_file)).to eq config_contents
+    end
+
+    it 'keep files num up to max' do
+      backup_files = Dir.glob("#{daemon.agent.config_backup_dir}/*").sort
+      expect(backup_files.size).to eq ::Settings.max_backup_files_count
+    end
+  end
 end

spec/models/fluentd/agent_spec.rb

@@ -2,7 +2,11 @@ require 'spec_helper'
 
 describe Fluentd::Agent do
   let(:instance) { described_class.new(options) }
-  let(:options) { {} }
+  let(:options) {
+    {
+      :config_file => Rails.root.join('tmp', 'fluentd-test', 'fluentd.conf').to_s
+    }
+   }
 
   describe "FluentdGem" do
     let(:described_class) { Fluentd::Agent::FluentdGem } # override nested described_class behavior as https://github.com/rspec/rspec-core/issues/1114
@@ -38,8 +42,19 @@ describe Fluentd::Agent do
         end
 
         context "actual start success" do
+          after do
+            FileUtils.rm_r instance.running_config_backup_dir, force: true
+          end
+
           let(:start_result) { true }
           it { should be_truthy }
+
+          it 'backed up running conf' do
+            subject
+            backup_file = instance.running_config_backup_file
+            expect(File.exists? backup_file).to be_truthy
+            expect(File.read(backup_file)).to eq File.read(instance.config_file)
+          end
         end
 
         context "actual start failed" do
@@ -86,6 +101,33 @@ describe Fluentd::Agent do
     let(:described_class) { Fluentd::Agent::TdAgent } # override nested described_class behavior as https://github.com/rspec/rspec-core/issues/1114
 
     it_should_behave_like "Fluentd::Agent has common behavior"
+
+    describe "#backup_running_config" do
+      before do
+        instance.stub(:detached_command).and_return(true)
+        instance.stub(:pid_from_launchctl).and_return(true)
+      end
+
+      after do
+        FileUtils.rm_r instance.running_config_backup_dir, force: true
+      end
+
+      let(:options) do
+        {
+          :config_file => Rails.root.join('tmp', 'fluentd-test', 'fluentd.conf').to_s
+        }
+      end
+
+      before do
+        instance.start
+      end
+
+      it 'backed up running conf' do
+        backup_file = instance.running_config_backup_file
+        expect(File.exists? backup_file).to be_truthy
+        expect(File.read(backup_file)).to eq File.read(instance.config_file)
+      end
+    end
   end
 end
 

spec/spec_helper.rb

@@ -38,6 +38,7 @@ RSpec.configure do |config|
   config.include LoginMacro
   config.include JavascriptMacro
   config.include StubDaemon
+  config.include ConfigHistories
 
   # If true, the base class of anonymous controllers will be inferred
   # automatically. This will be the default behavior in future versions of

spec/support/config_histories.rb

@@ -0,0 +1,57 @@
+module ConfigHistories
+  shared_context 'daemon has some config histories' do
+    let!(:three_hours_ago) { Time.zone.now - 3.hours }
+    let(:config_contents) { <<-CONF.strip_heredoc }
+      <source>
+        type forward
+        port 24224
+      </source>
+    CONF
+
+    let(:new_config) { <<-CONF.strip_heredoc }
+      <source>
+        type http
+        port 8899
+      </source>
+    CONF
+
+    before do
+      Timecop.freeze(three_hours_ago)
+
+      #remove backups on each to avoid depending on spec execution order
+      FileUtils.rm_r daemon.agent.config_backup_dir, force: true
+
+      7.times do |i|
+        backpued_time = three_hours_ago - (i + 1).hours
+        FileUtils.touch daemon.agent.config_backup_dir + "/#{backpued_time.strftime('%Y%m%d_%H%M%S')}.conf"
+      end
+
+      Timecop.freeze(three_hours_ago + 1.hour)
+      daemon.agent.config_write config_contents #add before conf
+
+      Timecop.freeze(three_hours_ago + 2.hour)
+      daemon.agent.config_write new_config #update conf
+
+      Timecop.freeze(three_hours_ago + 3.hour)
+    end
+
+    after do
+      FileUtils.rm_r daemon.agent.config_backup_dir, force: true
+      Timecop.return
+    end
+  end
+
+  shared_context 'daemon had been started once' do
+    let!(:backup_content){ "Running backup file content" }
+
+    before do
+      File.open(daemon.agent.running_config_backup_file, "w") do |file|
+        file.write(backup_content)
+      end
+    end
+
+    after do
+      FileUtils.rm_r daemon.agent.running_config_backup_dir, force: true
+    end
+  end
+end