Merge pull request #25 from treasure-data/tutorial

[WIP] Add tutorial page

Gemfile

@@ -7,7 +7,7 @@ gemspec
 group :development, :test do
   gem "rake"
   gem "pry"
-  gem "rspec-rails", "~> 3.0"
+  gem "rspec-rails", "~> 2.99"
 end
 
 group :test do

Gemfile.lock

@@ -12,6 +12,8 @@ PATH
       i18n_generators (= 1.2.1)
       jbuilder (~> 2.0)
       jquery-rails (~> 3.1.0)
+      kramdown (> 1.0.0)
+      kramdown-haml
       puma
       rails (= 4.1.1)
       sass-rails (~> 4.0.3)
@@ -75,7 +77,7 @@ GEM
     factory_girl_rails (4.4.1)
       factory_girl (~> 4.4.0)
       railties (>= 3.0.0)
-    fluentd (0.10.48)
+    fluentd (0.10.49)
       cool.io (>= 1.1.1, < 2.0.0, != 1.2.0)
       http_parser.rb (>= 0.5.1, < 0.7.0)
       json (>= 1.4.3)
@@ -107,6 +109,9 @@ GEM
       railties (>= 3.0, < 5.0)
       thor (>= 0.14, < 2.0)
     json (1.8.1)
+    kramdown (1.3.3)
+    kramdown-haml (0.0.3)
+      haml
     mail (2.5.4)
       mime-types (~> 1.16)
       treetop (~> 1.4.8)
@@ -156,22 +161,21 @@ GEM
       rake (>= 0.8.7)
       thor (>= 0.18.1, < 2.0)
     rake (10.3.2)
-    rspec-core (3.0.0)
-      rspec-support (~> 3.0.0)
-    rspec-expectations (3.0.0)
-      diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.0.0)
-    rspec-mocks (3.0.0)
-      rspec-support (~> 3.0.0)
-    rspec-rails (3.0.1)
+    rspec-collection_matchers (0.0.4)
+      rspec-expectations (>= 2.99.0.beta1)
+    rspec-core (2.99.0)
+    rspec-expectations (2.99.0)
+      diff-lcs (>= 1.1.3, < 2.0)
+    rspec-mocks (2.99.0)
+    rspec-rails (2.99.0)
       actionpack (>= 3.0)
+      activemodel (>= 3.0)
       activesupport (>= 3.0)
       railties (>= 3.0)
-      rspec-core (~> 3.0.0)
-      rspec-expectations (~> 3.0.0)
-      rspec-mocks (~> 3.0.0)
-      rspec-support (~> 3.0.0)
-    rspec-support (3.0.0)
+      rspec-collection_matchers
+      rspec-core (~> 2.99.0)
+      rspec-expectations (~> 2.99.0)
+      rspec-mocks (~> 2.99.0)
     safe_yaml (1.0.3)
     sass (3.2.19)
     sass-rails (4.0.3)
@@ -216,7 +220,7 @@ GEM
     webrobots (0.1.1)
     xpath (2.0.0)
       nokogiri (~> 1.3)
-    yajl-ruby (1.2.0)
+    yajl-ruby (1.2.1)
 
 PLATFORMS
   ruby
@@ -228,6 +232,6 @@ DEPENDENCIES
   fluentd-ui!
   pry
   rake
-  rspec-rails (~> 3.0)
+  rspec-rails (~> 2.99)
   simplecov (~> 0.7.1)
   webmock (~> 1.18.0)

app/assets/javascripts/application.js

@@ -17,4 +17,5 @@
 //= require sb-admin-v2/plugins/dataTables/dataTables.bootstrap
 //= require bower/vue/dist/vue
 //= require bower/es6-promise/promise
+//= require vue_common
 //= require_tree .

app/assets/javascripts/tutorial.js

@@ -0,0 +1,69 @@
+(function(){
+  "use strict";
+
+  $(function(){
+    if($('#chapter1').length === 0) return;
+
+    new Vue({
+      el: "#chapter1",
+      data: {
+        "logs": [],
+        "payloads": [
+          {
+            "path": "/debug.foo",
+            "data" : {
+              "message": "test message", // NOTE: "'" will break curl command
+            }
+          },
+          {
+            "path": "/debug.bar",
+            "data" : {
+              "my_number": 42,
+              "my_array": [1, 2, 3]
+            }
+          },
+          {
+            "path": "/xxxxx",
+            "data" : {
+              "xx": "will be unmatched"
+            }
+          },
+          {
+            "path": "/slash/convert/to/dot",
+            "data" : {
+              "greeting": "hello"
+            }
+          }
+        ]
+      },
+
+      created: function(){
+        this.fetchLogs();
+      },
+
+      methods: {
+        fetchLogs: function() {
+          var self = this;
+          new Promise(function(resolve, reject) {
+            $.getJSON("/tutorials/log_tail", resolve).fail(reject);
+          }).then(function(logs){
+            self.logs = logs;
+          });
+        },
+        sendRequest: function(payload){
+          new Promise(function(resolve, reject) {
+            $.ajax({
+              url: "/tutorials/request_fluentd",
+              data: JSON.stringify(payload),
+              contentType: "application/json",
+              dataType: "json",
+              type: "POST"
+            }).done(resolve).fail(reject);
+          })["catch"](function(e){
+            console.error(e);
+          });
+        }
+      }
+    });
+  });
+})();

app/assets/javascripts/vue_common.js

@@ -0,0 +1,4 @@
+Vue.filter('to_json', function (value) {
+    return JSON.stringify(value);
+})
+

app/controllers/tutorials_controller.rb

@@ -0,0 +1,41 @@
+class TutorialsController < ApplicationController
+  before_action :find_fluentd
+  before_action :check_ready, only: [:chapter1, :chapter2]
+  helper_method :tutorial_ready?
+
+  def index
+    @log = @fluentd.agent.log_tail.reverse if @fluentd
+  end
+
+  def chapter1
+  end
+
+  def chapter2
+    @default_conf = Fluentd::DEFAULT_CONF
+  end
+
+  def log_tail
+    @logs = @fluentd.agent.log_tail.reverse if @fluentd
+    render json: @logs
+  end
+
+  def request_fluentd
+    HTTPClient.post("http://localhost:8888#{params[:path]}", json: params[:data].to_json)
+    render nothing: true, status: 204
+  end
+
+  private
+
+  def find_fluentd
+    # NOTE: use first fluentd for tutorial
+    @fluentd = Fluentd.first
+  end
+
+  def check_ready
+    return redirect_to tutorials_url unless tutorial_ready?
+  end
+
+  def tutorial_ready?
+    @fluentd && @fluentd.agent.running?
+  end
+end

app/models/fluentd.rb

@@ -8,6 +8,29 @@ class Fluentd < ActiveRecord::Base
   before_validation :expand_paths
   after_save :ensure_default_config_file
 
+  DEFAULT_CONF = <<-CONF.strip_heredoc
+    <source>
+      type forward
+      port 24224
+    </source>
+    <source>
+      type monitor_agent
+      port 24220
+    </source>
+    <source>
+      type http
+      port 8888
+    </source>
+    <source>
+      type debug_agent
+      port 24230
+    </source>
+
+    <match debug.*>
+      type stdout
+    </match>
+  CONF
+
   def self.variants
     %w(fluentd td-agent)
   end
@@ -82,24 +105,7 @@ class Fluentd < ActiveRecord::Base
     return true if File.size?(config_file)
 
     File.open(config_file, "w") do |f|
-      f.write <<-XML.strip_heredoc
-        <source>
-          type forward
-          port 24224
-        </source>
-        <source>
-          type monitor_agent
-          port 24220
-        </source>
-        <source>
-          type http
-          port 9880
-        </source>
-        <source>
-          type debug_agent
-          port 24230
-        </source>
-      XML
+      f.write DEFAULT_CONF
     end
   end
 end

app/models/fluentd/agent/common.rb

@@ -67,6 +67,17 @@ class Fluentd
         io && io.close
       end
 
+      def log_tail(limit = 30)
+        buf = []
+        io = File.open(log_file)
+        reader = ::FileReverseReader.new(io)
+        reader.each_line do |line|
+          buf << line
+          break if buf.length >= 30
+        end
+        buf
+      end
+
       def pid_file
         extra_options[:pid_file] || self.class.default_options[:pid_file]
       end

app/views/shared/_global_nav.html.erb

@@ -4,6 +4,11 @@
   </li>
   <li>
     <%= link_to_other icon("fa-puzzle-piece fa-fw") << " fluentd", fluentd_index_path %>
+    <ul class="nav nav-second-level">
+      <li>
+        <%= link_to_other t('tutorials.index.page_title'), tutorials_path %>
+      </li>
+    </ul>
   </li>
   <li>
     <%= link_to_other icon("fa-cogs fa-fw") << " " << t('terms.plugins') << icon('fa pull-right fa-caret-down'), plugins_path %>

app/views/tutorials/chapter1.html.erb

@@ -0,0 +1,35 @@
+<% # NOTE: Using .erb is for styling <pre> %>
+
+<% page_title t(".page_title") %>
+
+<p class="clearfix">
+  <%= link_to t('tutorials.chapter2.page_title') << " >>", tutorials_chapter2_path, class: "pull-right" %>
+</p>
+
+<p>
+<%= t ".description" %>
+</p>
+
+<!-- vue.js -->
+<div id="chapter1">
+  <form>
+    <p v-repeat="payloads">
+      <input type="button" class="btn btn-default" v-on="click: sendRequest($data)" value="<%= t ".send" %>" />
+      <code>
+        $ curl -X POST http://localhost:8888{{ path }} -F 'json={{ data | to_json }}'
+      </code>
+    </p>
+  </form>
+
+  <pre>
+<button class="btn btn-primary" v-on="click: fetchLogs"><%= t ".reload_log" %></button>
+
+<span v-repeat="logs">{{ $value }}
+</span></pre>
+</div>
+<!-- /vue.js -->
+
+<p class="clearfix">
+  <%= link_to t('tutorials.chapter2.page_title') << " >>", tutorials_chapter2_path, class: "pull-right" %>
+</p>
+

app/views/tutorials/chapter2.html.haml

@@ -0,0 +1,12 @@
+- page_title t(".page_title")
+
+%p.clearfix
+  = link_to "<< " << t('tutorials.chapter1.page_title'), tutorials_chapter1_path, class: "pull-left"
+  = link_to t('tutorials.chapter3.page_title') << " >>", tutorials_chapter3_path, class: "pull-right"
+
+:markdown
+  #{t('.lesson_markdown')}
+
+%p.clearfix
+  = link_to "<< " << t('tutorials.chapter1.page_title'), tutorials_chapter1_path, class: "pull-left"
+  = link_to t('tutorials.chapter3.page_title') << " >>", tutorials_chapter3_path, class: "pull-right"

app/views/tutorials/chapter3.html.haml

@@ -0,0 +1,12 @@
+- page_title t(".page_title")
+
+%p.clearfix
+  = link_to "<< " << t('tutorials.chapter2.page_title'), tutorials_chapter2_path, class: "pull-left"
+  = link_to t('tutorials.chapter4.page_title') << " >>", tutorials_chapter4_path, class: "pull-right"
+
+:markdown
+  #{t(".lesson_markdown", edit_config_url: edit_fluentd_setting_path(@fluentd))}
+
+%p.clearfix
+  = link_to "<< " << t('tutorials.chapter2.page_title'), tutorials_chapter2_path, class: "pull-left"
+  = link_to t('tutorials.chapter4.page_title') << " >>", tutorials_chapter4_path, class: "pull-right"

app/views/tutorials/chapter4.html.haml

@@ -0,0 +1,12 @@
+- page_title t(".page_title")
+
+%p.clearfix
+  = link_to "<< " << t('tutorials.chapter3.page_title'), tutorials_chapter3_path, class: "pull-left"
+  = link_to t('tutorials.chapter5.page_title') << " >>", tutorials_chapter5_path, class: "pull-right"
+
+:markdown
+  #{t ".lesson_markdown"}
+
+%p.clearfix
+  = link_to "<< " << t('tutorials.chapter3.page_title'), tutorials_chapter3_path, class: "pull-left"
+  = link_to t('tutorials.chapter5.page_title') << " >>", tutorials_chapter5_path, class: "pull-right"

app/views/tutorials/chapter5.html.haml

@@ -0,0 +1,10 @@
+- page_title t(".page_title")
+
+%p.clearfix
+  = link_to "<< " << t('tutorials.chapter4.page_title'), tutorials_chapter4_path, class: "pull-left"
+
+:markdown
+  #{t ".lesson_markdown"}
+
+%p.clearfix
+  = link_to "<< " << t('tutorials.chapter4.page_title'), tutorials_chapter4_path, class: "pull-left"

app/views/tutorials/index.html.haml

@@ -0,0 +1,26 @@
+- page_title t(".page_title")
+
+%h2 
+  Hello, world!
+
+%ol
+  %li
+    - if @fluentd
+      = icon('fa-check text text-success')
+      = t('.step1')
+    - else
+      = icon('fa-warning text text-danger')
+      = link_to t('.step1'), fluentd_index_path
+  %li
+    - if @fluentd && @fluentd.agent.running?
+      = icon('fa-check text text-success')
+      = t('.step2')
+    - else
+      = icon('fa-warning text text-danger')
+      - if @fluentd
+        = link_to t('.step2'), fluentd_agent_path(@fluentd)
+      - else
+        = t('.step2')
+
+- if tutorial_ready?
+  = link_to t('.start_tutorial'), tutorials_chapter1_path

config/application.rb

@@ -18,6 +18,7 @@ require "haml-rails"
 require "jquery-rails"
 require "sucker_punch"
 require "settingslogic"
+require "kramdown-haml"
 
 module FluentdUi
   class Application < Rails::Application

config/locales/translation_en.yml

@@ -105,6 +105,132 @@ en:
       env_value: Value
       page_title: System Information
 
+  tutorials:
+    common: &tutorials_common
+      <<: *terms
+    index:
+      <<: *tutorials_common
+      step1: "Setup fluentd"
+      step2: "Start fluentd"
+      page_title: Tutorial
+      start_tutorial: Start tutorial
+    chapter1:
+      <<: *tutorials_common
+      page_title: "Chapter 1 | Try to send data"
+      reload_log: Reload fluend log
+      description: You can send an arbitrary JSON data via HTTP. URL path will be tag name.
+      send: Send
+    chapter2:
+      <<: *tutorials_common
+      page_title: "Chapter 2 | in_http and out_stdout"
+      lesson_markdown: |
+        You can see the log when fluentd started.
+
+            2014-06-05 14:43:14 +0900 [info]: adding source type="http"
+            2014-06-05 14:43:14 +0900 [info]: adding match pattern="debug.*" type="stdout"
+
+        Line 1 enable http plugin that allows to receive HTTP requests.
+
+        Line 2 enable stdout plugin that process the data with matched `debug.*` tag.
+
+        These settings are defined as following fluent.conf:
+
+            <source>
+              type http
+              port 8888
+            </source>
+
+            <match debug.*>
+              type stdout
+            </match>
+    chapter3:
+      <<: *tutorials_common
+      page_title: "Chapter 3 | Build your fluentd!"
+      lesson_markdown: |
+        fluentd can receive from [syslog protocol](http://docs.fluentd.org/articles/in_syslog), [file](http://docs.fluentd.org/articles/in_tail), etc.
+
+        Also fluentd can output to [MongoDB](http://docs.fluentd.org/articles/out_mongo), [AWS S3](http://docs.fluentd.org/articles/out_s3), etc.
+
+        ![fluentd](/fluentd.png)
+
+        These input/output are provided as plugin. Install them and write a setting, then restart fluentd, you can get the power!
+
+        [Many plugins](/plugins/recommended) are available. And you can [edit config file from here](%{edit_config_url}).
+    chapter4:
+      <<: *tutorials_common
+      page_title: "Chapter 4 | Use case"
+      lesson_markdown: |
+        ### Monitoring Apache 5xx response and email it
+
+        **Required plugins**
+
+        - fluent-plugin-grepcounter
+        - fluent-plugin-mail
+
+        **config file example**
+
+            <source>
+              type tail
+              format apache2
+              path /var/log/apache2/access.log #This is the location of your Apache log
+              tag apache.access
+            </source>
+
+            <match apache.access>
+              type grepcounter
+              count_interval 3         # Time window to grep and count the # of events
+              input_key code           # We look at the (http status) "code" field
+              regexp ^5\d\d$           # This regexp matches 5xx status codes
+              threshold 1              # The # of events to trigger emitting an output
+              add_tag_prefix error_5xx # The output event's tag will be error_5xx.apache.access
+            </match>
+        
+            <match error_5xx.apache.access>
+              # The event that comes here looks like
+              # {
+              #   "count":1,
+              #   "input_tag":"error_5xx.apache.access",
+              #   "input_tag_last":"access",
+              #   "message":[500]
+              # }
+
+              type mail
+              host smtp.gmail.com       # This is for Gmail and Google Apps. Any SMTP server should work
+              port 587                  # port for smtp.gmail.com
+              user example@gmail.com    # your Gmail here for login
+              password XXXXXX           # Gmail password
+              enable_starttls_auto true # Gmail required this
+
+              from YOUR_SENDER_EMAIL_HERE
+              to   YOUR_RECIPIENT_EMAIL_HERE
+              subject [URGENT] APACHE 5XX ERROR
+              message Total 5xx error count: %s\n\nPlease check your Apache webserver ASAP
+              message_out_keys count # The value of 'count' will be substituted into %s above.
+            </match>
+
+        **process flow**
+
+            [log file] -> 
+              (in_tail) ->
+                Capturing file content with tagged as apache.access ->
+              (match apache.access) ->
+                "grepcounter" re-send data with appending prefix ->
+              (match error_5xx.apache.access) ->
+                "mail" send a mail
+    chapter5:
+      <<: *tutorials_common
+      page_title: "Chapter 5 | Finish!"
+      lesson_markdown: |
+        Tutorial is over. congratulation!
+
+        Other resources:
+
+        - [Quick start](http://docs.fluentd.org/articles/quickstart)
+        - [Forum](https://groups.google.com/forum/?fromgroups#!forum/fluentd)
+        - [Source code(GitHub)](https://github.com/fluent/fluentd)
+        - [Twitter @fluentd](https://twitter.com/fluentd)
+
+
   messages:
     need_restart: need to restart fluentd-ui
     please_sign_in: Sign in

config/locales/translation_ja.yml

@@ -105,6 +105,136 @@ ja:
       env_value: 値
       page_title: システム情報
 
+  tutorials:
+    common: &tutorials_common
+      <<: *terms
+    index:
+      <<: *tutorials_common
+      page_title: チュートリアル
+      step1: "fluentdをセットアップ"
+      step2: "fluentdを起動"
+      start_tutorial: "チュートリアルを始める"
+    chapter1:
+      <<: *tutorials_common
+      page_title: "Chapter 1 | データを渡してみる"
+      reload_log: fluentdのログを更新
+      description: fluentdに任意のデータをJSONで送ることができます。URLのパスがタグの名前になります。
+      learn_more: |
+        他にもin_tail, in_syslogなどのinputプラグインがあります。
+        <a href="http://docs.fluentd.org/ja/articles/input-plugin-overview">Learn More</a>
+      send: 送信
+    chapter2:
+      <<: *tutorials_common
+      page_title: "Chapter 2 | in_httpとout_stdout"
+      lesson_markdown: |
+        fluentdの起動時にこのようなログがあるかと思います。
+
+            2014-06-05 14:43:14 +0900 [info]: adding source type="http"
+            2014-06-05 14:43:14 +0900 [info]: adding match pattern="debug.*" type="stdout"
+
+        この1行目でhttpが有効化されています。これでHTTPリクエストを受け付けるようになります。
+
+        2行目でstdoutが有効化されています。受け取ったデータのうち、タグが`debug.*`にマッチするものはstdoutへと渡されます。
+
+        この2つはfluent.confでそれぞれ次のように設定されています。
+
+            <source>
+              type http
+              port 8888
+            </source>
+
+            <match debug.*>
+              type stdout
+            </match>
+    chapter3:
+      <<: *tutorials_common
+      page_title: "Chapter 3 | fluentdを構築しよう！"
+      lesson_markdown: |
+        fluentdはHTTP以外にも[syslogプロトコル](http://docs.fluentd.org/ja/articles/in_syslog)や[ファイル](http://docs.fluentd.org/ja/articles/in_tail)を入力として受け取ることができます。
+
+        また出力についても、stdout以外に[MongoDB](http://docs.fluentd.org/ja/articles/out_mongo)や[AWS S3](http://docs.fluentd.org/ja/articles/out_s3)などを出力先として指定できます。
+
+        ![fluentd](/fluentd.png)
+
+        これらはプラグインとして提供されています。プラグインをインストールし、設定ファイルに追記してfluentdを再起動すると使用可能となります。
+
+        [数多くのプラグイン](/plugins/recommended)がありますので、用途にあったものを探して使いましょう！ 設定ファイルは[ここから編集できます。](%{edit_config_url})
+    chapter4:
+      <<: *tutorials_common
+      page_title: "Chapter 4 | 設定事例"
+      lesson_markdown: |
+        ### 例：Apacheの5xxレスポンスを検知してメールを送る
+
+        **必要なプラグイン**
+
+        - fluent-plugin-grepcounter
+        - fluent-plugin-mail
+
+        **設定ファイル例**
+
+            <source>
+              type tail
+              format apache2
+              path /var/log/apache2/access.log #This is the location of your Apache log
+              tag apache.access
+            </source>
+
+            <match apache.access>
+              type grepcounter
+              count_interval 3         # Time window to grep and count the # of events
+              input_key code           # We look at the (http status) "code" field
+              regexp ^5\d\d$           # This regexp matches 5xx status codes
+              threshold 1              # The # of events to trigger emitting an output
+              add_tag_prefix error_5xx # The output event's tag will be error_5xx.apache.access
+            </match>
+        
+            <match error_5xx.apache.access>
+              # The event that comes here looks like
+              # {
+              #   "count":1,
+              #   "input_tag":"error_5xx.apache.access",
+              #   "input_tag_last":"access",
+              #   "message":[500]
+              # }
+
+              type mail
+              host smtp.gmail.com       # This is for Gmail and Google Apps. Any SMTP server should work
+              port 587                  # port for smtp.gmail.com
+              user example@gmail.com    # your Gmail here for login
+              password XXXXXX           # Gmail password
+              enable_starttls_auto true # Gmail required this
+
+              from YOUR_SENDER_EMAIL_HERE
+              to   YOUR_RECIPIENT_EMAIL_HERE
+              subject [URGENT] APACHE 5XX ERROR
+              message Total 5xx error count: %s\n\nPlease check your Apache webserver ASAP
+              message_out_keys count # The value of 'count' will be substituted into %s above.
+            </match>
+
+        **処理の流れ**
+
+            [log file] -> 
+              (in_tail) ->
+                apache.accessタグでfluentdに取り込む ->
+              (apache.accessにマッチ) ->
+                grepcounterがタグにprefixを追加して再送 ->
+              (error_5xx.apache.accessにマッチ) ->
+                mailがメール送信
+    chapter5:
+      <<: *tutorials_common
+      page_title: "Chapter 5 | チュートリアル完了"
+      lesson_markdown: |
+        以上でチュートリアルは終了です。お疲れさまでした！
+
+        関連リソース：
+
+        - [クイックスタートガイド](http://docs.fluentd.org/ja/articles/quickstart)
+        - [メーリングリスト](https://groups.google.com/forum/?fromgroups#!forum/fluentd)
+        - [ソースコード(GitHub)](https://github.com/fluent/fluentd)
+        - [Twitter @fluentd](https://twitter.com/fluentd)
+
+
+
   messages:
     need_restart: fluentd-uiの再起動が必要です
     please_sign_in: ログイン

config/routes.rb

@@ -33,4 +33,15 @@ Rails.application.routes.draw do
   namespace :polling do
     get "alerts"
   end
+
+  namespace :tutorials do
+    get "/" => :index
+    get "chapter1"
+    get "chapter2"
+    get "chapter3"
+    get "chapter4"
+    get "chapter5"
+    get "log_tail"
+    post "request_fluentd"
+  end
 end

fluentd-ui.gemspec

@@ -35,4 +35,6 @@ Gem::Specification.new do |spec|
   spec.add_dependency "settingslogic"
   spec.add_dependency "puma"
   spec.add_dependency "thor"
+  spec.add_dependency "kramdown", "> 1.0.0"
+  spec.add_dependency "kramdown-haml"
 end

spec/controllers/tutorials_controller_spec.rb

@@ -0,0 +1,5 @@
+require 'spec_helper'
+
+RSpec.describe TutorialsController, :type => :controller do
+
+end