Logstash

Name of Innovation

Logstash

November 22, 2017 Uncategorized 0

# Source the following file to setup the java version correctly
    source ~/sourceme (to setup the java version correctly)

# Logstash to try from command line
    logstash -e 'input { stdin { } } output { stdout {} }'


# Logstash to try from command line
    logstash -e 'input { stdin { } } output { stdout { codec => rubydebug } }'

    logstash -e 'input { stdin { } } output { elasticsearch { hosts => localhost } }'


# Check if Logstash has been able to insert data into the diretory
    curl -XGET "http://localhost:9200/_cat/indices?pretty

# Run
    logstash -e 'input { stdin { } } output { elasticsearch { hosts => localhost } }'

# List the indices 
    curl -XGET "http://localhost:9200/_cat/indices?pretty"

# After it is started enter “Hello Your Name”



# Run -	
    curl -XGET "http://localhost:9200/logstash-*/_search"?pretty


# Download sample log file -
    cd /home/woir/Downloads/ && wget https://download.elastic.co/demos/logstash/gettingstarted/logstash-tutorial.log.gz
    gunzip logstash-tutorial.log.gz


# Save  following file in your home directory as apache.conf


########## apache.conf ##############
input {
    file {
        path => "/home/woir/Downloads/logstash-tutorial.log"
        start_position => beginning
        sincedb_path => "/dev/null"
        ignore_older => 0
    }
}

filter {
    grok {
        match => { "message" => "%{COMBINEDAPACHELOG}"}
    }
    date {
        match => [ "timestamp" , "dd/MMM/yyyy:HH:mm:ss Z" ]
    }
    geoip {
        source => "clientip"
    }
}

output {
    elasticsearch {
        index=>"gabbar"
    }
}

# Start pipeline 
    logstash -f ~/apache.conf


# See the contents –
    curl -XGET http://localhost:9200/logstash-2015.01.04/_search




######## Apple Stocks Hands on session  ####### 


1. Download the data to be inserted into ES
    wget -O /home/woir/Downloads/table-3.csv http://woir.in/wp-content/uploads/2016/12/table-3.csv





## save the following file as /home/woir/apple.conf
input {
  file {
    path => "/home/woir/Downloads/table*.csv"
    type => "core2"
    start_position => "beginning"    
  }
}
filter {
  csv {
      separator => ","
      columns => ["Date","Open","High","Low","Close","Volume","Adj Close"]
  }
  mutate {convert => ["High", "float"]}
  mutate {convert => ["Open", "float"]}
  mutate {convert => ["Low", "float"]}
  mutate {convert => ["Close", "float"]}
  mutate {convert => ["Volume", "float"]}
}
output {  
    elasticsearch {
        action => "index"
        hosts => ["localhost:9200"]
        index => "stock"
        workers => 1
    }
    stdout {}
}

# Point the config file and run the logstash – it will insert data into elasticsearch
    /home/woir/logstash-5.0.2/bin/logstash -f  /home/woir/apple.conf

# Check data insertion is done or not –
    curl -XGET http://localhost:9200/stock/_search?pretty