Backport/fix when file is gone

CHANGELOG.md

@@ -0,0 +1,4 @@
+## 1.1.0
+  - Add create_if_deleted option to create a destination file in case it
+   was deleted by another agent in the machine. In case of being false
+   the system will add the incomming messages to the failure file.

Gemfile

@@ -1,2 +1,3 @@
 source 'https://rubygems.org'
-gemspec
+gemspec
+gem "concurrent-ruby", "0.9.1"

lib/logstash/outputs/file.rb

@@ -11,6 +11,8 @@ class LogStash::Outputs::File < LogStash::Outputs::Base
 
   config_name "file"
 
+  attr_reader :failure_path
+
   # The path to the file to write. Event fields can be used here,
   # like `/var/log/logstash/%{host}/%{application}`
   # One may also utilize the path option for date-based log
@@ -42,6 +44,11 @@ class LogStash::Outputs::File < LogStash::Outputs::Base
   # into this file and inside the defined path.
   config :filename_failure, :validate => :string, :default => '_filepath_failures'
 
+  # If the a file is deleted, but an event is comming with the need to be stored
+  # in such a file, the plugin will created a gain this file. Default => true
+  config :create_if_deleted, :validate => :boolean, :default => true
+
+
   public
   def register
     require "fileutils" # For mkdir_p
@@ -56,8 +63,10 @@ def register
 
     if path_with_field_ref?
       @file_root = extract_file_root
-      @failure_path = File.join(@file_root, @filename_failure)
+    else
+      @file_root = File.dirname(path)
     end
+    @failure_path = File.join(@file_root, @filename_failure)
 
     now = Time.now
     @last_flush_cycle = now
@@ -94,6 +103,8 @@ def receive(event)
     if path_with_field_ref? && !inside_file_root?(file_output_path)
       @logger.warn("File: the event tried to write outside the files root, writing the event to the failure file",  :event => event, :filename => @failure_path)
       file_output_path = @failure_path
+    elsif !@create_if_deleted && deleted?(file_output_path)
+      file_output_path = @failure_path
     end
 
     output = format_message(event)
@@ -124,7 +135,6 @@ def inside_file_root?(log_path)
   def write_event(log_path, event)
     @logger.debug("File, writing event to file.", :filename => log_path)
     fd = open(log_path)
-
     # TODO(sissel): Check if we should rotate the file.
 
     fd.write(event)
@@ -199,23 +209,40 @@ def close_stale_files
   end
 
   private
-  def open(path)
-    return @files[path] if @files.include?(path) and not @files[path].nil?
+  def cached?(path)
+     @files.include?(path) && !@files[path].nil?
+  end
 
+  private
+  def deleted?(path)
+    !File.exist?(path)
+  end
+
+  private
+  def open(path)
+    if !deleted?(path) && cached?(path)
+      return @files[path]
+    elsif deleted?(path)
+      if @create_if_deleted
+        @logger.debug("Required path was deleted, creating the file again", :path => path)
+        @files.delete(path)
+      else
+        return @files[path] if cached?(path)
+      end
+    end
     @logger.info("Opening file", :path => path)
 
     dir = File.dirname(path)
     if !Dir.exists?(dir)
       @logger.info("Creating directory", :directory => dir)
       FileUtils.mkdir_p(dir)
     end
-
     # work around a bug opening fifos (bug JRUBY-6280)
     stat = File.stat(path) rescue nil
     if stat && stat.ftype == "fifo" && LogStash::Environment.jruby?
       fd = java.io.FileWriter.new(java.io.File.new(path))
     else
-      fd = File.new(path, "a")
+      fd = File.new(path, "a+")
     end
     if gzip
       fd = Zlib::GzipWriter.new(fd)

logstash-output-file.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-output-file'
-  s.version         = '1.0.0'
+  s.version         = '1.1.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "This output will write events to files on disk"
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -20,7 +20,7 @@ Gem::Specification.new do |s|
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
 
   # Gem dependencies
-  s.add_runtime_dependency "logstash-core", '>= 1.4.0', '< 2.0.0'
+  s.add_runtime_dependency "logstash-core", '>= 1.4.0', '< 1.999.0'
   s.add_runtime_dependency 'logstash-input-generator'
 
   s.add_development_dependency 'logstash-devutils'

spec/outputs/file_spec.rb

@@ -6,6 +6,7 @@
 require "stud/temporary"
 require "tempfile"
 require "uri"
+require "fileutils"
 
 describe LogStash::Outputs::File do
   describe "ship lots of events to a file" do
@@ -106,6 +107,58 @@
   end
 
   describe "receiving events" do
+
+    context "when the output file is deleted" do
+
+      let(:temp_file) { Tempfile.new('logstash-spec-output-file_deleted') }
+
+      let(:config) do
+        { "path" => temp_file.path, "flush_interval" => 0 }
+      end
+
+      it "should recreate the required file if deleted" do
+        output = LogStash::Outputs::File.new(config)
+        output.register
+
+        10.times do |i|
+          event = LogStash::Event.new("event_id" => i)
+          output.receive(event)
+        end
+        FileUtils.rm(temp_file)
+        10.times do |i|
+          event = LogStash::Event.new("event_id" => i+10)
+          output.receive(event)
+        end
+        expect(FileTest.size(temp_file.path)).to be > 0
+      end
+
+      context "when appending to the error log" do
+
+        let(:config) do
+          { "path" => temp_file.path, "flush_interval" => 0, "create_if_deleted" => false }
+        end
+
+        it "should append the events to the filename_failure location" do
+          output = LogStash::Outputs::File.new(config)
+          output.register
+
+          10.times do |i|
+            event = LogStash::Event.new("event_id" => i)
+            output.receive(event)
+          end
+          FileUtils.rm(temp_file)
+          10.times do |i|
+            event = LogStash::Event.new("event_id" => i+10)
+            output.receive(event)
+          end
+          expect(FileTest.exist?(temp_file.path)).to be_falsey
+          expect(FileTest.size(output.failure_path)).to be > 0
+        end
+
+      end
+
+    end
+
     context "when using an interpolated path" do
       context "when trying to write outside the files root directory" do
         let(:bad_event) do