diff --git a/Gemfile b/Gemfile index 27e5005..a958a89 100644 --- a/Gemfile +++ b/Gemfile @@ -7,7 +7,7 @@ gemspec group :development, :test do gem "rake" gem "pry" - gem "rspec-rails", "~> 3.0" + gem "rspec-rails", "~> 2.99" end group :test do diff --git a/Gemfile.lock b/Gemfile.lock index ba14444..f18f66a 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -12,6 +12,8 @@ PATH i18n_generators (= 1.2.1) jbuilder (~> 2.0) jquery-rails (~> 3.1.0) + kramdown (> 1.0.0) + kramdown-haml puma rails (= 4.1.1) sass-rails (~> 4.0.3) @@ -75,7 +77,7 @@ GEM factory_girl_rails (4.4.1) factory_girl (~> 4.4.0) railties (>= 3.0.0) - fluentd (0.10.48) + fluentd (0.10.49) cool.io (>= 1.1.1, < 2.0.0, != 1.2.0) http_parser.rb (>= 0.5.1, < 0.7.0) json (>= 1.4.3) @@ -107,6 +109,9 @@ GEM railties (>= 3.0, < 5.0) thor (>= 0.14, < 2.0) json (1.8.1) + kramdown (1.3.3) + kramdown-haml (0.0.3) + haml mail (2.5.4) mime-types (~> 1.16) treetop (~> 1.4.8) @@ -156,22 +161,21 @@ GEM rake (>= 0.8.7) thor (>= 0.18.1, < 2.0) rake (10.3.2) - rspec-core (3.0.0) - rspec-support (~> 3.0.0) - rspec-expectations (3.0.0) - diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.0.0) - rspec-mocks (3.0.0) - rspec-support (~> 3.0.0) - rspec-rails (3.0.1) + rspec-collection_matchers (0.0.4) + rspec-expectations (>= 2.99.0.beta1) + rspec-core (2.99.0) + rspec-expectations (2.99.0) + diff-lcs (>= 1.1.3, < 2.0) + rspec-mocks (2.99.0) + rspec-rails (2.99.0) actionpack (>= 3.0) + activemodel (>= 3.0) activesupport (>= 3.0) railties (>= 3.0) - rspec-core (~> 3.0.0) - rspec-expectations (~> 3.0.0) - rspec-mocks (~> 3.0.0) - rspec-support (~> 3.0.0) - rspec-support (3.0.0) + rspec-collection_matchers + rspec-core (~> 2.99.0) + rspec-expectations (~> 2.99.0) + rspec-mocks (~> 2.99.0) safe_yaml (1.0.3) sass (3.2.19) sass-rails (4.0.3) @@ -216,7 +220,7 @@ GEM webrobots (0.1.1) xpath (2.0.0) nokogiri (~> 1.3) - yajl-ruby (1.2.0) + yajl-ruby (1.2.1) PLATFORMS ruby @@ -228,6 +232,6 @@ DEPENDENCIES fluentd-ui! pry rake - rspec-rails (~> 3.0) + rspec-rails (~> 2.99) simplecov (~> 0.7.1) webmock (~> 1.18.0) diff --git a/app/assets/javascripts/application.js b/app/assets/javascripts/application.js index 43d6e1d..b8c2f45 100644 --- a/app/assets/javascripts/application.js +++ b/app/assets/javascripts/application.js @@ -17,4 +17,5 @@ //= require sb-admin-v2/plugins/dataTables/dataTables.bootstrap //= require bower/vue/dist/vue //= require bower/es6-promise/promise +//= require vue_common //= require_tree . diff --git a/app/assets/javascripts/tutorial.js b/app/assets/javascripts/tutorial.js new file mode 100644 index 0000000..a19c068 --- /dev/null +++ b/app/assets/javascripts/tutorial.js @@ -0,0 +1,69 @@ +(function(){ + "use strict"; + + $(function(){ + if($('#chapter1').length === 0) return; + + new Vue({ + el: "#chapter1", + data: { + "logs": [], + "payloads": [ + { + "path": "/debug.foo", + "data" : { + "message": "test message", // NOTE: "'" will break curl command + } + }, + { + "path": "/debug.bar", + "data" : { + "my_number": 42, + "my_array": [1, 2, 3] + } + }, + { + "path": "/xxxxx", + "data" : { + "xx": "will be unmatched" + } + }, + { + "path": "/slash/convert/to/dot", + "data" : { + "greeting": "hello" + } + } + ] + }, + + created: function(){ + this.fetchLogs(); + }, + + methods: { + fetchLogs: function() { + var self = this; + new Promise(function(resolve, reject) { + $.getJSON("/tutorials/log_tail", resolve).fail(reject); + }).then(function(logs){ + self.logs = logs; + }); + }, + sendRequest: function(payload){ + new Promise(function(resolve, reject) { + $.ajax({ + url: "/tutorials/request_fluentd", + data: JSON.stringify(payload), + contentType: "application/json", + dataType: "json", + type: "POST" + }).done(resolve).fail(reject); + })["catch"](function(e){ + console.error(e); + }); + } + } + }); + }); +})(); diff --git a/app/assets/javascripts/vue_common.js b/app/assets/javascripts/vue_common.js new file mode 100644 index 0000000..ffecc97 --- /dev/null +++ b/app/assets/javascripts/vue_common.js @@ -0,0 +1,4 @@ +Vue.filter('to_json', function (value) { + return JSON.stringify(value); +}) + diff --git a/app/controllers/tutorials_controller.rb b/app/controllers/tutorials_controller.rb new file mode 100644 index 0000000..1f5fab5 --- /dev/null +++ b/app/controllers/tutorials_controller.rb @@ -0,0 +1,41 @@ +class TutorialsController < ApplicationController + before_action :find_fluentd + before_action :check_ready, only: [:chapter1, :chapter2] + helper_method :tutorial_ready? + + def index + @log = @fluentd.agent.log_tail.reverse if @fluentd + end + + def chapter1 + end + + def chapter2 + @default_conf = Fluentd::DEFAULT_CONF + end + + def log_tail + @logs = @fluentd.agent.log_tail.reverse if @fluentd + render json: @logs + end + + def request_fluentd + HTTPClient.post("http://localhost:8888#{params[:path]}", json: params[:data].to_json) + render nothing: true, status: 204 + end + + private + + def find_fluentd + # NOTE: use first fluentd for tutorial + @fluentd = Fluentd.first + end + + def check_ready + return redirect_to tutorials_url unless tutorial_ready? + end + + def tutorial_ready? + @fluentd && @fluentd.agent.running? + end +end diff --git a/app/models/fluentd.rb b/app/models/fluentd.rb index b59d609..6af20c7 100644 --- a/app/models/fluentd.rb +++ b/app/models/fluentd.rb @@ -8,6 +8,29 @@ class Fluentd < ActiveRecord::Base before_validation :expand_paths after_save :ensure_default_config_file + DEFAULT_CONF = <<-CONF.strip_heredoc + + type forward + port 24224 + + + type monitor_agent + port 24220 + + + type http + port 8888 + + + type debug_agent + port 24230 + + + + type stdout + + CONF + def self.variants %w(fluentd td-agent) end @@ -82,24 +105,7 @@ class Fluentd < ActiveRecord::Base return true if File.size?(config_file) File.open(config_file, "w") do |f| - f.write <<-XML.strip_heredoc - - type forward - port 24224 - - - type monitor_agent - port 24220 - - - type http - port 9880 - - - type debug_agent - port 24230 - - XML + f.write DEFAULT_CONF end end end diff --git a/app/models/fluentd/agent/common.rb b/app/models/fluentd/agent/common.rb index 525eaff..d094452 100644 --- a/app/models/fluentd/agent/common.rb +++ b/app/models/fluentd/agent/common.rb @@ -67,6 +67,17 @@ class Fluentd io && io.close end + def log_tail(limit = 30) + buf = [] + io = File.open(log_file) + reader = ::FileReverseReader.new(io) + reader.each_line do |line| + buf << line + break if buf.length >= 30 + end + buf + end + def pid_file extra_options[:pid_file] || self.class.default_options[:pid_file] end diff --git a/app/views/shared/_global_nav.html.erb b/app/views/shared/_global_nav.html.erb index 4c30147..e5a7ec2 100644 --- a/app/views/shared/_global_nav.html.erb +++ b/app/views/shared/_global_nav.html.erb @@ -4,6 +4,11 @@
  • <%= link_to_other icon("fa-puzzle-piece fa-fw") << " fluentd", fluentd_index_path %> +
  • <%= link_to_other icon("fa-cogs fa-fw") << " " << t('terms.plugins') << icon('fa pull-right fa-caret-down'), plugins_path %> diff --git a/app/views/tutorials/chapter1.html.erb b/app/views/tutorials/chapter1.html.erb new file mode 100644 index 0000000..285e7b1 --- /dev/null +++ b/app/views/tutorials/chapter1.html.erb @@ -0,0 +1,35 @@ +<% # NOTE: Using .erb is for styling
     %>
    +
    +<% page_title t(".page_title") %>
    +
    +

    + <%= link_to t('tutorials.chapter2.page_title') << " >>", tutorials_chapter2_path, class: "pull-right" %> +

    + +

    +<%= t ".description" %> +

    + + +
    +
    +

    + " /> + + $ curl -X POST http://localhost:8888{{ path }} -F 'json={{ data | to_json }}' + +

    +
    + +
    +
    +
    +{{ $value }}
    +
    +
    + + +

    + <%= link_to t('tutorials.chapter2.page_title') << " >>", tutorials_chapter2_path, class: "pull-right" %> +

    + diff --git a/app/views/tutorials/chapter2.html.haml b/app/views/tutorials/chapter2.html.haml new file mode 100644 index 0000000..cd1dd8b --- /dev/null +++ b/app/views/tutorials/chapter2.html.haml @@ -0,0 +1,12 @@ +- page_title t(".page_title") + +%p.clearfix + = link_to "<< " << t('tutorials.chapter1.page_title'), tutorials_chapter1_path, class: "pull-left" + = link_to t('tutorials.chapter3.page_title') << " >>", tutorials_chapter3_path, class: "pull-right" + +:markdown + #{t('.lesson_markdown')} + +%p.clearfix + = link_to "<< " << t('tutorials.chapter1.page_title'), tutorials_chapter1_path, class: "pull-left" + = link_to t('tutorials.chapter3.page_title') << " >>", tutorials_chapter3_path, class: "pull-right" diff --git a/app/views/tutorials/chapter3.html.haml b/app/views/tutorials/chapter3.html.haml new file mode 100644 index 0000000..b34f292 --- /dev/null +++ b/app/views/tutorials/chapter3.html.haml @@ -0,0 +1,12 @@ +- page_title t(".page_title") + +%p.clearfix + = link_to "<< " << t('tutorials.chapter2.page_title'), tutorials_chapter2_path, class: "pull-left" + = link_to t('tutorials.chapter4.page_title') << " >>", tutorials_chapter4_path, class: "pull-right" + +:markdown + #{t(".lesson_markdown", edit_config_url: edit_fluentd_setting_path(@fluentd))} + +%p.clearfix + = link_to "<< " << t('tutorials.chapter2.page_title'), tutorials_chapter2_path, class: "pull-left" + = link_to t('tutorials.chapter4.page_title') << " >>", tutorials_chapter4_path, class: "pull-right" diff --git a/app/views/tutorials/chapter4.html.haml b/app/views/tutorials/chapter4.html.haml new file mode 100644 index 0000000..de617b1 --- /dev/null +++ b/app/views/tutorials/chapter4.html.haml @@ -0,0 +1,12 @@ +- page_title t(".page_title") + +%p.clearfix + = link_to "<< " << t('tutorials.chapter3.page_title'), tutorials_chapter3_path, class: "pull-left" + = link_to t('tutorials.chapter5.page_title') << " >>", tutorials_chapter5_path, class: "pull-right" + +:markdown + #{t ".lesson_markdown"} + +%p.clearfix + = link_to "<< " << t('tutorials.chapter3.page_title'), tutorials_chapter3_path, class: "pull-left" + = link_to t('tutorials.chapter5.page_title') << " >>", tutorials_chapter5_path, class: "pull-right" diff --git a/app/views/tutorials/chapter5.html.haml b/app/views/tutorials/chapter5.html.haml new file mode 100644 index 0000000..9213ac1 --- /dev/null +++ b/app/views/tutorials/chapter5.html.haml @@ -0,0 +1,10 @@ +- page_title t(".page_title") + +%p.clearfix + = link_to "<< " << t('tutorials.chapter4.page_title'), tutorials_chapter4_path, class: "pull-left" + +:markdown + #{t ".lesson_markdown"} + +%p.clearfix + = link_to "<< " << t('tutorials.chapter4.page_title'), tutorials_chapter4_path, class: "pull-left" diff --git a/app/views/tutorials/index.html.haml b/app/views/tutorials/index.html.haml new file mode 100644 index 0000000..5e9840e --- /dev/null +++ b/app/views/tutorials/index.html.haml @@ -0,0 +1,26 @@ +- page_title t(".page_title") + +%h2 + Hello, world! + +%ol + %li + - if @fluentd + = icon('fa-check text text-success') + = t('.step1') + - else + = icon('fa-warning text text-danger') + = link_to t('.step1'), fluentd_index_path + %li + - if @fluentd && @fluentd.agent.running? + = icon('fa-check text text-success') + = t('.step2') + - else + = icon('fa-warning text text-danger') + - if @fluentd + = link_to t('.step2'), fluentd_agent_path(@fluentd) + - else + = t('.step2') + +- if tutorial_ready? + = link_to t('.start_tutorial'), tutorials_chapter1_path diff --git a/config/application.rb b/config/application.rb index 0a586f5..d29965a 100644 --- a/config/application.rb +++ b/config/application.rb @@ -18,6 +18,7 @@ require "haml-rails" require "jquery-rails" require "sucker_punch" require "settingslogic" +require "kramdown-haml" module FluentdUi class Application < Rails::Application diff --git a/config/locales/translation_en.yml b/config/locales/translation_en.yml index 2283a89..65c8fae 100644 --- a/config/locales/translation_en.yml +++ b/config/locales/translation_en.yml @@ -105,6 +105,132 @@ en: env_value: Value page_title: System Information + tutorials: + common: &tutorials_common + <<: *terms + index: + <<: *tutorials_common + step1: "Setup fluentd" + step2: "Start fluentd" + page_title: Tutorial + start_tutorial: Start tutorial + chapter1: + <<: *tutorials_common + page_title: "Chapter 1 | Try to send data" + reload_log: Reload fluend log + description: You can send an arbitrary JSON data via HTTP. URL path will be tag name. + send: Send + chapter2: + <<: *tutorials_common + page_title: "Chapter 2 | in_http and out_stdout" + lesson_markdown: | + You can see the log when fluentd started. + + 2014-06-05 14:43:14 +0900 [info]: adding source type="http" + 2014-06-05 14:43:14 +0900 [info]: adding match pattern="debug.*" type="stdout" + + Line 1 enable http plugin that allows to receive HTTP requests. + + Line 2 enable stdout plugin that process the data with matched `debug.*` tag. + + These settings are defined as following fluent.conf: + + + type http + port 8888 + + + + type stdout + + chapter3: + <<: *tutorials_common + page_title: "Chapter 3 | Build your fluentd!" + lesson_markdown: | + fluentd can receive from [syslog protocol](http://docs.fluentd.org/articles/in_syslog), [file](http://docs.fluentd.org/articles/in_tail), etc. + + Also fluentd can output to [MongoDB](http://docs.fluentd.org/articles/out_mongo), [AWS S3](http://docs.fluentd.org/articles/out_s3), etc. + + ![fluentd](/fluentd.png) + + These input/output are provided as plugin. Install them and write a setting, then restart fluentd, you can get the power! + + [Many plugins](/plugins/recommended) are available. And you can [edit config file from here](%{edit_config_url}). + chapter4: + <<: *tutorials_common + page_title: "Chapter 4 | Use case" + lesson_markdown: | + ### Monitoring Apache 5xx response and email it + + **Required plugins** + + - fluent-plugin-grepcounter + - fluent-plugin-mail + + **config file example** + + + type tail + format apache2 + path /var/log/apache2/access.log #This is the location of your Apache log + tag apache.access + + + + type grepcounter + count_interval 3 # Time window to grep and count the # of events + input_key code # We look at the (http status) "code" field + regexp ^5\d\d$ # This regexp matches 5xx status codes + threshold 1 # The # of events to trigger emitting an output + add_tag_prefix error_5xx # The output event's tag will be error_5xx.apache.access + + + + # The event that comes here looks like + # { + # "count":1, + # "input_tag":"error_5xx.apache.access", + # "input_tag_last":"access", + # "message":[500] + # } + + type mail + host smtp.gmail.com # This is for Gmail and Google Apps. Any SMTP server should work + port 587 # port for smtp.gmail.com + user example@gmail.com # your Gmail here for login + password XXXXXX # Gmail password + enable_starttls_auto true # Gmail required this + + from YOUR_SENDER_EMAIL_HERE + to YOUR_RECIPIENT_EMAIL_HERE + subject [URGENT] APACHE 5XX ERROR + message Total 5xx error count: %s\n\nPlease check your Apache webserver ASAP + message_out_keys count # The value of 'count' will be substituted into %s above. + + + **process flow** + + [log file] -> + (in_tail) -> + Capturing file content with tagged as apache.access -> + (match apache.access) -> + "grepcounter" re-send data with appending prefix -> + (match error_5xx.apache.access) -> + "mail" send a mail + chapter5: + <<: *tutorials_common + page_title: "Chapter 5 | Finish!" + lesson_markdown: | + Tutorial is over. congratulation! + + Other resources: + + - [Quick start](http://docs.fluentd.org/articles/quickstart) + - [Forum](https://groups.google.com/forum/?fromgroups#!forum/fluentd) + - [Source code(GitHub)](https://github.com/fluent/fluentd) + - [Twitter @fluentd](https://twitter.com/fluentd) + + messages: need_restart: need to restart fluentd-ui please_sign_in: Sign in diff --git a/config/locales/translation_ja.yml b/config/locales/translation_ja.yml index 0beae2f..dea1dd6 100644 --- a/config/locales/translation_ja.yml +++ b/config/locales/translation_ja.yml @@ -105,6 +105,136 @@ ja: env_value: 値 page_title: システム情報 + tutorials: + common: &tutorials_common + <<: *terms + index: + <<: *tutorials_common + page_title: チュートリアル + step1: "fluentdをセットアップ" + step2: "fluentdを起動" + start_tutorial: "チュートリアルを始める" + chapter1: + <<: *tutorials_common + page_title: "Chapter 1 | データを渡してみる" + reload_log: fluentdのログを更新 + description: fluentdに任意のデータをJSONで送ることができます。URLのパスがタグの名前になります。 + learn_more: | + 他にもin_tail, in_syslogなどのinputプラグインがあります。 + Learn More + send: 送信 + chapter2: + <<: *tutorials_common + page_title: "Chapter 2 | in_httpとout_stdout" + lesson_markdown: | + fluentdの起動時にこのようなログがあるかと思います。 + + 2014-06-05 14:43:14 +0900 [info]: adding source type="http" + 2014-06-05 14:43:14 +0900 [info]: adding match pattern="debug.*" type="stdout" + + この1行目でhttpが有効化されています。これでHTTPリクエストを受け付けるようになります。 + + 2行目でstdoutが有効化されています。受け取ったデータのうち、タグが`debug.*`にマッチするものはstdoutへと渡されます。 + + この2つはfluent.confでそれぞれ次のように設定されています。 + + + type http + port 8888 + + + + type stdout + + chapter3: + <<: *tutorials_common + page_title: "Chapter 3 | fluentdを構築しよう!" + lesson_markdown: | + fluentdはHTTP以外にも[syslogプロトコル](http://docs.fluentd.org/ja/articles/in_syslog)や[ファイル](http://docs.fluentd.org/ja/articles/in_tail)を入力として受け取ることができます。 + + また出力についても、stdout以外に[MongoDB](http://docs.fluentd.org/ja/articles/out_mongo)や[AWS S3](http://docs.fluentd.org/ja/articles/out_s3)などを出力先として指定できます。 + + ![fluentd](/fluentd.png) + + これらはプラグインとして提供されています。プラグインをインストールし、設定ファイルに追記してfluentdを再起動すると使用可能となります。 + + [数多くのプラグイン](/plugins/recommended)がありますので、用途にあったものを探して使いましょう! 設定ファイルは[ここから編集できます。](%{edit_config_url}) + chapter4: + <<: *tutorials_common + page_title: "Chapter 4 | 設定事例" + lesson_markdown: | + ### 例:Apacheの5xxレスポンスを検知してメールを送る + + **必要なプラグイン** + + - fluent-plugin-grepcounter + - fluent-plugin-mail + + **設定ファイル例** + + + type tail + format apache2 + path /var/log/apache2/access.log #This is the location of your Apache log + tag apache.access + + + + type grepcounter + count_interval 3 # Time window to grep and count the # of events + input_key code # We look at the (http status) "code" field + regexp ^5\d\d$ # This regexp matches 5xx status codes + threshold 1 # The # of events to trigger emitting an output + add_tag_prefix error_5xx # The output event's tag will be error_5xx.apache.access + + + + # The event that comes here looks like + # { + # "count":1, + # "input_tag":"error_5xx.apache.access", + # "input_tag_last":"access", + # "message":[500] + # } + + type mail + host smtp.gmail.com # This is for Gmail and Google Apps. Any SMTP server should work + port 587 # port for smtp.gmail.com + user example@gmail.com # your Gmail here for login + password XXXXXX # Gmail password + enable_starttls_auto true # Gmail required this + + from YOUR_SENDER_EMAIL_HERE + to YOUR_RECIPIENT_EMAIL_HERE + subject [URGENT] APACHE 5XX ERROR + message Total 5xx error count: %s\n\nPlease check your Apache webserver ASAP + message_out_keys count # The value of 'count' will be substituted into %s above. + + + **処理の流れ** + + [log file] -> + (in_tail) -> + apache.accessタグでfluentdに取り込む -> + (apache.accessにマッチ) -> + grepcounterがタグにprefixを追加して再送 -> + (error_5xx.apache.accessにマッチ) -> + mailがメール送信 + chapter5: + <<: *tutorials_common + page_title: "Chapter 5 | チュートリアル完了" + lesson_markdown: | + 以上でチュートリアルは終了です。お疲れさまでした! + + 関連リソース: + + - [クイックスタートガイド](http://docs.fluentd.org/ja/articles/quickstart) + - [メーリングリスト](https://groups.google.com/forum/?fromgroups#!forum/fluentd) + - [ソースコード(GitHub)](https://github.com/fluent/fluentd) + - [Twitter @fluentd](https://twitter.com/fluentd) + + + messages: need_restart: fluentd-uiの再起動が必要です please_sign_in: ログイン diff --git a/config/routes.rb b/config/routes.rb index ed64061..c0e3440 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -33,4 +33,15 @@ Rails.application.routes.draw do namespace :polling do get "alerts" end + + namespace :tutorials do + get "/" => :index + get "chapter1" + get "chapter2" + get "chapter3" + get "chapter4" + get "chapter5" + get "log_tail" + post "request_fluentd" + end end diff --git a/fluentd-ui.gemspec b/fluentd-ui.gemspec index 995053e..366f8b0 100644 --- a/fluentd-ui.gemspec +++ b/fluentd-ui.gemspec @@ -35,4 +35,6 @@ Gem::Specification.new do |spec| spec.add_dependency "settingslogic" spec.add_dependency "puma" spec.add_dependency "thor" + spec.add_dependency "kramdown", "> 1.0.0" + spec.add_dependency "kramdown-haml" end diff --git a/public/fluentd.png b/public/fluentd.png new file mode 100644 index 0000000..d2f5f52 Binary files /dev/null and b/public/fluentd.png differ diff --git a/spec/controllers/tutorials_controller_spec.rb b/spec/controllers/tutorials_controller_spec.rb new file mode 100644 index 0000000..d8db500 --- /dev/null +++ b/spec/controllers/tutorials_controller_spec.rb @@ -0,0 +1,5 @@ +require 'spec_helper' + +RSpec.describe TutorialsController, :type => :controller do + +end