Merge pull request #6145 from avalonmediasystem/caption_extraction

Create caption files during transcoding

Gemfile

@@ -75,7 +75,7 @@ gem 'omniauth-lti', git: "https://github.com/avalonmediasystem/omniauth-lti.git"
 gem "omniauth-saml", "~> 2.0", ">= 2.2.1"
 
 # Media Access & Transcoding
-gem 'active_encode', '~> 1.2'
+gem 'active_encode', git: "https://github.com/samvera-labs/active_encode.git", branch: 'main'
 gem 'audio_waveform-ruby', '~> 1.0.7', require: 'audio_waveform'
 gem 'browse-everything', git: "https://github.com/avalonmediasystem/browse-everything.git", tag: '1.4-avalon'
 gem 'fastimage'

Gemfile.lock

@@ -57,6 +57,15 @@ GIT
       ims-lti
       omniauth
 
+GIT
+  remote: https://github.com/samvera-labs/active_encode.git
+  revision: adecfb1503c2a706661f716ff041ca7ad3c9c3d7
+  branch: main
+  specs:
+    active_encode (1.2.3)
+      addressable (~> 2.8)
+      rails
+
 GIT
   remote: https://github.com/samvera/active_fedora.git
   revision: 0f5ccb1536224efec750941ce9a1f58f2e09cd3c
@@ -134,9 +143,6 @@ GEM
     active_elastic_job (3.3.0)
       aws-sdk-sqs (~> 1)
       rails (>= 5.2.6, < 8)
-    active_encode (1.2.3)
-      addressable (~> 2.8)
-      rails
     active_fedora-datastreams (0.5.0)
       active-fedora (>= 11.0.0.pre)
       activemodel (>= 5.2)
@@ -354,7 +360,7 @@ GEM
       activerecord (>= 5.a)
       database_cleaner-core (~> 2.0.0)
     database_cleaner-core (2.0.1)
-    date (3.3.4)
+    date (3.4.1)
     declarative (0.0.20)
     deep_merge (1.2.2)
     deprecation (1.1.0)
@@ -494,8 +500,8 @@ GEM
     ims-lti (1.1.13)
       builder
       oauth (>= 0.4.5, < 0.6)
-    io-console (0.7.2)
-    irb (1.14.0)
+    io-console (0.8.0)
+    irb (1.14.1)
       rdoc (>= 4.0.0)
       reline (>= 0.4.2)
     jbuilder (2.12.0)
@@ -553,13 +559,13 @@ GEM
     llhttp-ffi (0.5.0)
       ffi-compiler (~> 1.0)
       rake (~> 13.0)
-    logger (1.6.1)
+    logger (1.6.2)
     lograge (0.14.0)
       actionpack (>= 4)
       activesupport (>= 4)
       railties (>= 4)
       request_store (~> 1.0)
-    loofah (2.22.0)
+    loofah (2.23.1)
       crass (~> 1.0.2)
       nokogiri (>= 1.12.0)
     mail (2.8.1)
@@ -578,9 +584,9 @@ GEM
       mime-types-data (~> 3.2015)
     mime-types-data (3.2024.0820)
     mini_mime (1.1.5)
-    mini_portile2 (2.8.7)
+    mini_portile2 (2.8.8)
     minitar (1.0.2)
-    minitest (5.25.1)
+    minitest (5.25.4)
     msgpack (1.7.2)
     multi_json (1.15.0)
     multi_xml (0.7.1)
@@ -590,7 +596,7 @@ GEM
     mysql2 (0.5.6)
     net-http (0.4.1)
       uri
-    net-imap (0.4.16)
+    net-imap (0.5.1)
       date
       net-protocol
     net-ldap (0.19.0)
@@ -606,12 +612,12 @@ GEM
       net-protocol
     net-ssh (7.2.3)
     netrc (0.11.0)
-    nio4r (2.7.3)
+    nio4r (2.7.4)
     noid (0.9.0)
     noid-rails (3.2.0)
       actionpack (>= 5.0.0, < 8)
       noid (~> 0.9)
-    nokogiri (1.16.7)
+    nokogiri (1.16.8)
       mini_portile2 (~> 2.8.2)
       racc (~> 1.4)
     nom-xml (1.2.0)
@@ -675,7 +681,7 @@ GEM
       rack (< 3)
     rack-test (2.1.0)
       rack (>= 1.3)
-    rackup (1.0.0)
+    rackup (1.0.1)
       rack (< 3)
       webrick
     rails (7.2.2)
@@ -700,9 +706,9 @@ GEM
       activesupport (>= 5.0.0)
       minitest
       nokogiri (>= 1.6)
-    rails-html-sanitizer (1.6.0)
+    rails-html-sanitizer (1.6.1)
       loofah (~> 2.21)
-      nokogiri (~> 1.14)
+      nokogiri (>= 1.15.7, != 1.16.7, != 1.16.6, != 1.16.5, != 1.16.4, != 1.16.3, != 1.16.2, != 1.16.1, != 1.16.0.rc1, != 1.16.0)
     rails_same_site_cookie (0.1.9)
       rack (>= 1.5)
       user_agent_parser (~> 2.6)
@@ -774,7 +780,7 @@ GEM
     redlock (2.0.6)
       redis-client (>= 0.14.1, < 1.0.0)
     regexp_parser (2.9.2)
-    reline (0.5.10)
+    reline (0.5.12)
       io-console (~> 0.5)
     representable (3.2.0)
       declarative (< 0.1.0)
@@ -869,7 +875,7 @@ GEM
       tilt
     scanf (1.0.0)
     scrub_rb (1.0.1)
-    securerandom (0.3.1)
+    securerandom (0.4.0)
     selenium-webdriver (3.142.7)
       childprocess (>= 0.5, < 4.0)
       rubyzip (>= 1.2.2)
@@ -949,7 +955,7 @@ GEM
       execjs (>= 0.3.0, < 3)
     thor (1.3.2)
     tilt (2.4.0)
-    timeout (0.4.1)
+    timeout (0.4.2)
     trailblazer-option (0.1.2)
     twitter-typeahead-rails (0.11.1.pre.corejavascript)
       actionpack (>= 3.1)
@@ -965,7 +971,7 @@ GEM
     unicode-types (1.10.0)
     uri (0.13.1)
     user_agent_parser (2.18.0)
-    useragent (0.16.10)
+    useragent (0.16.11)
     view_component (3.14.0)
       activesupport (>= 5.2.0, < 8.0)
       concurrent-ruby (~> 1.0)
@@ -986,7 +992,7 @@ GEM
       addressable (>= 2.8.0)
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)
-    webrick (1.8.2)
+    webrick (1.9.1)
     websocket-driver (0.7.6)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.5)
@@ -995,7 +1001,7 @@ GEM
       rexml
     xpath (3.2.0)
       nokogiri (~> 1.8)
-    zeitwerk (2.6.18)
+    zeitwerk (2.7.1)
     zk (1.10.0)
       zookeeper (~> 1.5.0)
     zookeeper (1.5.5)
@@ -1009,7 +1015,7 @@ DEPENDENCIES
   active-fedora!
   active_annotations (~> 0.5.0)
   active_elastic_job
-  active_encode (~> 1.2)
+  active_encode!
   active_fedora-datastreams (~> 0.5)
   activejob-traffic_control
   activejob-uniqueness

app/controllers/media_objects_controller.rb

@@ -240,7 +240,8 @@ def update_media_object
           break
         end
         if file_spec[:files].present?
-          if master_file.update_derivatives(file_spec[:files], false)
+          master_file.update_derivatives(file_spec[:files], false)
+          if master_file.save
             master_file.update_stills_from_offset!
             WaveformJob.perform_later(master_file.id)
           else

app/jobs/active_encode_jobs.rb

@@ -20,7 +20,7 @@ def perform(input, master_file_id, options = {})
       return unless MasterFile.exists? master_file_id
       master_file = MasterFile.find(master_file_id)
       return if master_file.workflow_id.present?
-      master_file.encoder_class.create(input, options.merge!(master_file_id: master_file_id, preset: master_file.workflow_name))
+      master_file.encoder_class.create(input, options.merge!(master_file_id: master_file_id, preset: master_file.workflow_name, extract_subtitles: true))
     end
   end
 

app/models/master_file.rb

@@ -318,11 +318,11 @@ def update_progress_on_success!(encode)
     # We can get the proper aspect ratio from the transcoded files, so we set the master file off the 
     # encode output.
     if is_video?
-      high_output = Array(encode.output).select { |out| out.label.include?("high") }.first
+      high_output = Array(encode.output).select { |out| out.label&.include?("high") }.first
       self.display_aspect_ratio = (high_output.width.to_f / high_output.height.to_f).to_s
     end
 
-    outputs = Array(encode.output).collect do |output|
+    outputs = Array(encode.output).reject { |output| output.format == "vtt" }.collect do |output|
       {
         id: output.id,
         label: output.label,
@@ -339,6 +339,13 @@ def update_progress_on_success!(encode)
       }
     end
     update_derivatives(outputs)
+
+    supplemental_file_outputs = Array(encode.output).select { |out| out.format == 'vtt' }
+    supplemental_file_ids = supplemental_file_outputs.collect { |sf| sf.id }.compact
+    add_supplemental_files(supplemental_file_ids) if supplemental_file_ids.present?
+
+    save
+
     run_hook :after_transcoding
   end
 
@@ -352,8 +359,10 @@ def update_derivatives(outputs, managed = true)
         existing.delete
       end
     end
+  end
 
-    save
+  def add_supplemental_files(ids)
+    self.supplemental_files += ids.collect { |id| GlobalID::Locator.locate(id) }
   end
 
   alias_method :'_poster_offset', :'poster_offset'

app/models/supplemental_file.rb

@@ -33,9 +33,14 @@ class SupplementalFile < ApplicationRecord
   after_update_commit :update_index, prepend: true
   after_destroy_commit :remove_from_index
 
-  def attach_file(new_file)
-    file.attach(new_file)
-    extension = File.extname(new_file.original_filename)
+  def attach_file(new_file, io: false)
+    if io
+      file.attach(io: File.open(new_file), filename: File.basename(new_file))
+      extension = File.extname(new_file)
+    else
+      file.attach(new_file)
+      extension = File.extname(new_file.original_filename)
+    end
     self.file.content_type = Mime::Type.lookup_by_extension(extension.slice(1..-1)).to_s if extension == '.srt'
     self.label = file.filename.to_s if label.blank?
     self.language ||= Settings.caption_default.language

app/models/watched_encode.rb

@@ -32,23 +32,37 @@ class WatchedEncode < ActiveEncode::Base
     if Settings.encoding.derivative_bucket &&
        (Settings.encoding.engine_adapter.to_sym == :ffmpeg || is_a?(PassThroughEncode))
       bucket = Aws::S3::Bucket.new(name: Settings.encoding.derivative_bucket)
-      encode.output.collect! do |output|
-        file = FileLocator.new output.url
-        key = file.location.sub(/\/(.*?)\//, "")
-        obj = bucket.object key
 
-        if File.exist? file.location
-          obj.upload_file file.location
-          File.delete file.location
+      encode.output.collect! do |output|
+        if output.format == "vtt"
+          new_file = SupplementalFile.new(tags: ['caption'], parent_id: record.master_file_id)
+          new_file.attach_file(FileLocator.new(output.url).location, io: true)
+          new_file.save
+          output.url = if Settings.active_storage.bucket.present?
+                      "s3://#{Settings.active_storage.bucket}/#{new_file.file.blob.key}"
+                    else
+                      new_file.file.blob.url
+                    end
+          output.id = new_file.to_global_id.to_s
+          output
         else
-          # Calls to Addressable::URI.escape here are to counter the unescaping that happens in FileLocator
-          # This is needed because files uploaded to minio (and probably AWS) escape spaces (%20) instead of keeping spaces
-          obj.upload_file Addressable::URI.escape(file.location)
-          File.delete Addressable::URI.escape(file.location)
-        end
+          file = FileLocator.new output.url
+          key = file.location.sub(/\/(.*?)\//, "")
+          obj = bucket.object key
 
-        output.url = "s3://#{obj.bucket.name}/#{obj.key}"
-        output
+          if File.exist? file.location
+            obj.upload_file file.location
+            File.delete file.location
+          else
+            # Calls to Addressable::URI.escape here are to counter the unescaping that happens in FileLocator
+            # This is needed because files uploaded to minio (and probably AWS) escape spaces (%20) instead of keeping spaces
+            obj.upload_file Addressable::URI.escape(file.location)
+            File.delete Addressable::URI.escape(file.location)
+          end
+
+          output.url = "s3://#{obj.bucket.name}/#{obj.key}"
+          output
+        end
       end
 
       # Save translated output urls

lib/avalon/batch/entry.rb

@@ -297,7 +297,7 @@ def self.process_datastream(datastream, type, parent_id)
         machine_generated = Avalon::Batch.true_field?(datastream[:machine_generated]) ? 'machine_generated' : nil
         # Create SupplementalFile
         supplemental_file = SupplementalFile.new(label: label, tags: [type, treat_as_transcript, machine_generated].uniq.compact, language: language, parent_id: parent_id)
-        supplemental_file.file.attach(io: FileLocator.new(datastream[file_key]).reader, filename: filename)
+        supplemental_file.attach_file(FileLocator.new(datastream[file_key]).reader, io: true)
         supplemental_file.save ? supplemental_file : nil
       end
       private_class_method :process_datastream

spec/factories/encode.rb

@@ -26,29 +26,36 @@
       state { :running }
       percent_complete { 50.5 }
       current_operations { ['encoding'] }
-      input { FactoryBot.build(:encode_output) }
+      input { FactoryBot.build(:encode_input) }
     end
 
     trait :succeeded do
       state { :completed }
       percent_complete { 100 }
       current_operations { ['DONE'] }
-      input { FactoryBot.build(:encode_output) }
+      input { FactoryBot.build(:encode_input) }
       output { [ FactoryBot.build(:encode_output) ] }
     end
 
+    trait :embedded_captions do
+      state { :completed }
+      percent_complete { 100 }
+      current_operations { ['DONE'] }
+      input { FactoryBot.build(:encode_input) }
+      output { [ FactoryBot.build(:encode_output), FactoryBot.build(:encode_supplemental_output) ] }
+    end
+
     trait :failed do
       state { :failed }
       percent_complete { 50.5 }
       current_operations { ['FAILED'] }
-      input { FactoryBot.build(:encode_output) }
+      input { FactoryBot.build(:encode_input) }
       errors { ['Out of disk space.'] }
     end
   end
 
   factory :encode_input, class: ActiveEncode::Input do
     id { SecureRandom.uuid }
-    label { 'quality-high' }
     url { 'file://path/to/output.mp4' }
     duration { '21575.0' }
     audio_bitrate { '163842.0' }
@@ -57,6 +64,7 @@
     video_codec { 'AVC' }
     width { '1024' }
     height { '768' }
+    subtitles { [] }
   end
 
   factory :encode_output, class: ActiveEncode::Output do
@@ -70,5 +78,12 @@
     video_codec { 'AVC' }
     width { '1024' }
     height { '768' }
+    subtitles { [] }
+  end
+
+  factory :encode_supplemental_output, class: ActiveEncode::Output do
+    id { "gid://avalon/SupplementalFile/1" }
+    url { "file://#{Rails.root.join('spec', 'fixtures', 'caption.vtt')}"}
+    format { "vtt" }
   end
 end

spec/jobs/active_encode_job_spec.rb

@@ -22,7 +22,7 @@
 
     describe "perform" do
       it 'creates the active_encode job' do
-        expect(encoder_class).to receive(:create).with(input, { master_file_id: master_file.id, preset: master_file.workflow_name }).once
+        expect(encoder_class).to receive(:create).with(input, { master_file_id: master_file.id, preset: master_file.workflow_name, extract_subtitles: true }).once
         ActiveEncodeJobs::CreateEncodeJob.perform_now(input, master_file.id)
       end