Download, configure and launch collector (#4)

lib/scout_apm/logging/config.rb

@@ -14,27 +14,58 @@ class Config < ScoutApm::Config
         logging_ingest_key
         logging_monitor
         monitor_pid_file
+        collector_download_dir
+        collector_config_file
+        collector_version
+        monitored_logs
+        logs_reporting_endpoint
       ].freeze
 
       SETTING_COERCIONS = {
-        'monitor_logs' => BooleanCoercion.new
+        'monitor_logs' => BooleanCoercion.new,
+        'monitored_logs' => JsonCoercion.new
       }.freeze
 
       def self.with_file(context, file_path = nil, config = {})
         overlays = [
           ConfigEnvironment.new,
           ConfigFile.new(context, file_path, config),
+          ConfigDynamic.new,
           ConfigDefaults.new,
           ConfigNull.new
         ]
         new(context, overlays)
       end
 
+      # We try and make assumptions about where the Rails log file is located.
+      class ConfigDynamic
+        @@values_to_set = {
+          'monitored_logs': []
+        }
+
+        def self.set_value(key, value)
+          @@values_to_set[key] = value
+        end
+
+        def value(key)
+          @@values_to_set[key]
+        end
+
+        def has_key?(key)
+          @@values_to_set.key?(key)
+        end
+      end
+
       # Defaults in case no config file has been found.
       class ConfigDefaults
         DEFAULTS = {
           'log_level' => 'info',
-          'monitor_pid_file' => '/tmp/scout_apm_log_monitor.pid'
+          'monitor_pid_file' => '/tmp/scout_apm/scout_apm_log_monitor.pid',
+          'collector_download_dir' => '/tmp/scout_apm',
+          'collector_config_file' => '/tmp/scout_apm/config.yml',
+          'collector_version' => '0.99.0',
+          'monitored_logs' => [],
+          'logs_reporting_endpoint' => 'https://otlp.telemetryhub.com:4317'
         }.freeze
 
         def value(key)

lib/scout_apm/logging/context.rb

@@ -4,6 +4,10 @@ module ScoutApm
   module Logging
     # Contains context around Scout APM logging, such as environment, configuration, and the logger.
     class Context
+      # Useful for the monitor daemon process, where we get the Rails root through the pipe.
+      attr_accessor :application_root
+      attr_accessor :application_env
+
       # Initially start up without attempting to load a configuration file. We
       # need to be able to lookup configuration options like "application_root"
       # which would then in turn influence where the yaml configuration file is
@@ -15,15 +19,15 @@ def initialize
       end
 
       def config
-        @config ||= ScoutApm::Logging::Config.without_file(self)
+        @config ||= Config.without_file(self)
       end
 
       def environment
         @environment ||= ScoutApm::Environment.instance
       end
 
       def logger
-        @logger ||= LoggerFactory.build(config, environment)
+        @logger ||= LoggerFactory.build(config, environment, application_root)
       end
 
       def config=(config)
@@ -38,19 +42,20 @@ def config=(config)
 
     # Create a logger based on the configuration settings.
     class LoggerFactory
-      def self.build(config, environment)
-        ScoutApm::Logging::Logger.new(environment.root,
-                                      {
-                                        log_level: config.value('log_level'),
-                                        log_file_path: config.value('log_file_path'),
-                                        stdout: config.value('log_stdout') || environment.platform_integration.log_to_stdout?,
-                                        stderr: config.value('log_stderr'),
-                                        logger_class: config.value('log_class')
-                                      })
+      def self.build(config, environment, application_root = nil)
+        root = application_root || environment.root
+        Logger.new(root,
+                   {
+                     log_level: config.value('log_level'),
+                     log_file_path: config.value('log_file_path'),
+                     stdout: config.value('log_stdout') || environment.platform_integration.log_to_stdout?,
+                     stderr: config.value('log_stderr'),
+                     logger_class: config.value('log_class')
+                   })
       end
 
       def self.build_minimal_logger
-        ScoutApm::Logging::Logger.new(nil, stdout: true)
+        Logger.new(nil, stdout: true)
       end
     end
   end

lib/scout_apm/logging/monitor/collector/configuration.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module ScoutApm
+  module Logging
+    module Collector
+      # Creates the configuration to be used when launching the collector.
+      class Configuration
+        attr_reader :context
+
+        def initialize(context)
+          @context = context
+        end
+
+        def setup!
+          create_config_file
+        end
+
+        def create_config_file
+          File.write(config_file, config_contents)
+        end
+
+        private
+
+        def config_file
+          context.config.value('collector_config_file')
+        end
+
+        def config_contents
+          <<~CONFIG
+            receivers:
+              filelog:
+                include: [#{context.config.value('monitored_logs').join(',')}]
+            processors:
+              batch:
+            exporters:
+              otlp:
+                endpoint: #{context.config.value('logs_reporting_endpoint')}
+                headers:
+                  x-telemetryhub-key: #{context.config.value('logging_ingest_key')}
+            service:
+              pipelines:
+                logs:
+                  receivers:
+                    - filelog
+                  processors:
+                    - batch
+                  exporters:
+                    - otlp
+          CONFIG
+        end
+      end
+    end
+  end
+end

lib/scout_apm/logging/monitor/collector/downloader.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module ScoutApm
+  module Logging
+    module Collector
+      # Downloads the collector-contrib binary from the OpenTelemetry project.
+      class Downloader
+        attr_reader :context
+
+        def initialize(context)
+          @context = context
+        end
+
+        def run!
+          download_collector
+          extract_collector
+        end
+
+        def download_collector(url = nil) # rubocop:disable Metrics/AbcSize
+          # TODO: Check if we have already downloaded the collector.
+          url_to_download = url || collector_url
+          uri = URI(url_to_download)
+
+          Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == 'https') do |http|
+            request = Net::HTTP::Get.new(uri)
+            http.request(request) do |response|
+              return download_collector(response['location']) if response.code == '302'
+
+              File.open(destination, 'wb') do |file|
+                response.read_body do |chunk|
+                  file.write(chunk)
+                end
+              end
+            end
+          end
+        end
+
+        def extract_collector
+          Utils.ensure_directory_exists(destination)
+          `tar -xzf #{destination} -C #{context.config.value('collector_download_dir')}`
+        end
+
+        private
+
+        def collector_url
+          collector_version = context.config.value('collector_version')
+
+          # https://opentelemetry.io/docs/collector/installation/#manual-linux-installation
+          "https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/v#{collector_version}/otelcol-contrib_#{collector_version}_#{host_os}_#{architecture}.tar.gz"
+        end
+
+        # TODO: Add support for other platforms
+        def architecture
+          if /arm/ =~ RbConfig::CONFIG['arch']
+            'arm64'
+          else
+            'amd64'
+          end
+        end
+
+        def host_os
+          if /darwin|mac os/ =~ RbConfig::CONFIG['host_os']
+            'darwin'
+          else
+            'linux'
+          end
+        end
+
+        def destination
+          "#{context.config.value('collector_download_dir')}/otelcol.tar.gz"
+        end
+      end
+    end
+  end
+end

lib/scout_apm/logging/monitor/collector/manager.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require_relative './configuration'
+require_relative './downloader'
+
+module ScoutApm
+  module Logging
+    module Collector
+      # Manager class for the downloading, configuring, and starting of the collector.
+      class Manager
+        attr_reader :context
+
+        def initialize(context)
+          @context = context
+        end
+
+        def setup!
+          Configuration.new(context).setup!
+          Downloader.new(context).run!
+
+          start_collector
+        end
+
+        def start_collector
+          Process.spawn("#{extracted_collector_path}/otelcol-contrib --config #{config_file}")
+        end
+
+        private
+
+        def extracted_collector_path
+          context.config.value('collector_download_dir')
+        end
+
+        def config_file
+          context.config.value('collector_config_file')
+        end
+      end
+    end
+  end
+end

lib/scout_apm/logging/monitor/monitor.rb

@@ -9,6 +9,9 @@
 require_relative '../logger'
 require_relative '../context'
 require_relative '../config'
+require_relative '../utils'
+
+require_relative './collector/manager'
 
 module ScoutApm
   module Logging
@@ -23,13 +26,20 @@ def self.instance
       end
 
       def initialize
-        @context = ScoutApm::Logging::Context.new
-        context.config = ScoutApm::Logging::Config.with_file(context, context.config.value('config_file'))
+        @context = Context.new
+
+        @context.application_root = $stdin.gets.chomp
+        @context.application_env = $stdin.gets.chomp
+
+        Config::ConfigDynamic.set_value('monitored_logs', [assumed_rails_log_path])
+        context.config = Config.with_file(context, determine_scout_config_filepath)
       end
 
       def setup!
         add_exit_handler
 
+        Collector::Manager.new(context).setup!
+
         run!
       end
 
@@ -47,6 +57,14 @@ def add_exit_handler
           File.delete(context.config.value('monitor_pid_file'))
         end
       end
+
+      def determine_scout_config_filepath
+        "#{context.application_root}/config/scout_apm.yml"
+      end
+
+      def assumed_rails_log_path
+        context.application_root + "/log/#{context.application_env}.log"
+      end
     end
   end
 end

lib/scout_apm/logging/monitor_manager/manager.rb

@@ -13,22 +13,32 @@ def self.instance
       end
 
       def initialize
-        @context = ScoutApm::Logging::Context.new
-        context.config = ScoutApm::Logging::Config.with_file(context, context.config.value('config_file'))
+        @context = Context.new
+        context.config = Config.with_file(context, context.config.value('config_file'))
       end
 
       def setup!
         create_process
       end
 
-      def create_process
+      def create_process # rubocop:disable Metrics/AbcSize
+        # TODO: Do an actual check that the process actually exists.
         return if File.exist? context.config.value('monitor_pid_file')
 
+        Utils.ensure_directory_exists(context.config.value('monitor_pid_file'))
+
+        reader, writer = IO.pipe
+
         gem_directory = File.expand_path('../../../..', __dir__)
-        daemon_process = Process.spawn("ruby #{gem_directory}/bin/scout_apm_logging_monitor", pgroup: true)
+        daemon_process = Process.spawn("ruby #{gem_directory}/bin/scout_apm_logging_monitor", pgroup: true, in: reader)
 
         File.write(context.config.value('monitor_pid_file'), daemon_process)
 
+        reader.close
+        writer.puts Rails.root if defined?(Rails)
+        writer.puts Rails.env if defined?(Rails)
+        writer.close
+
         # TODO: Add exit handlers?
       end
     end

lib/scout_apm/logging/utils.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+
+module ScoutApm
+  module Logging
+    # Miscellaneous utilities for the logging module.
+    module Utils
+      def self.ensure_directory_exists(file_path)
+        directory = File.dirname(file_path)
+        FileUtils.mkdir_p(directory) unless File.directory?(directory)
+      end
+    end
+  end
+end