Add invidious companion support

config/config.example.yml

@@ -54,6 +54,53 @@ db:
 ##
 #signature_server:
 
+##
+## Invidious companion is an external program
+## for loading the video streams from YouTube servers.
+##
+## When this setting is commented out, Invidious companion is not used.
+## Otherwise, Invidious will proxy the requests to Invidious companion.
+## 
+## Note: multiple URL can be configured. In this case, invidious will
+## randomly pick one every time video data needs to be retrieved. This
+## URL is then kept in the video metadata cache to allow video playback
+## to work. Once said cache has expired, requesting that video's data
+## again will cause a new companion URL to be picked.
+##
+## The parameter private_url needs to be configured for the internal
+## communication between the companion and Invidious.
+## And public_url is the public URL from which companion is listening
+## to the requests from the user(s).
+##
+## If you are using a reverse proxy then you will probably need to
+## configure the public_url to be the same as the domain used for Invidious.
+## Also apply when used from an external IP address (without a domain).
+## Examples: https://MYINVIDIOUSDOMAIN or http://192.168.1.100:8282
+##
+## Both parameter can have identical URL when Invidious is hosted in
+## an internal network or at home or locally (localhost).
+##
+## Accepted values: "http(s)://<IP-HOSTNAME>:<Port>"
+## Default: <none>
+##
+#invidious_companion:
+#  - private_url: "http://localhost:8282"
+#    public_url: "http://localhost:8282"
+
+##
+## API key for Invidious companion, used for securing the communication
+## between Invidious and Invidious companion.
+## The size of the key needs to be more or equal to 16.
+##
+## Note: This parameter is mandatory when Invidious companion is enabled
+## and should be a random string.
+## Such random string can be generated on linux with the following
+## command: `pwgen 16 1`
+##
+## Accepted values: a string
+## Default: <none>
+##
+#invidious_companion_key: "CHANGE_ME!!"
 
 #########################################
 #

src/invidious.cr

@@ -97,6 +97,10 @@ YT_POOL = YoutubeConnectionPool.new(YT_URL, capacity: CONFIG.pool_size)
 
 GGPHT_POOL = YoutubeConnectionPool.new(URI.parse("https://yt3.ggpht.com"), capacity: CONFIG.pool_size)
 
+COMPANION_POOL = CompanionConnectionPool.new(
+  capacity: CONFIG.pool_size
+)
+
 # CLI
 Kemal.config.extra_options do |parser|
   parser.banner = "Usage: invidious [arguments]"

src/invidious/config.cr

@@ -74,6 +74,16 @@ end
 class Config
   include YAML::Serializable
 
+  class CompanionConfig
+    include YAML::Serializable
+
+    @[YAML::Field(converter: Preferences::URIConverter)]
+    property private_url : URI = URI.parse("")
+
+    @[YAML::Field(converter: Preferences::URIConverter)]
+    property public_url : URI = URI.parse("")
+  end
+
   # Number of threads to use for crawling videos from channels (for updating subscriptions)
   property channel_threads : Int32 = 1
   # Time interval between two executions of the job that crawls channel videos (subscriptions update).
@@ -160,6 +170,12 @@ class Config
   # poToken for passing bot attestation
   property po_token : String? = nil
 
+  # Invidious companion
+  property invidious_companion : Array(CompanionConfig) = [] of CompanionConfig
+
+  # Invidious companion API key
+  property invidious_companion_key : String = ""
+
   # Saved cookies in "name1=value1; name2=value2..." format
   @[YAML::Field(converter: Preferences::StringToCookies)]
   property cookies : HTTP::Cookies = HTTP::Cookies.new
@@ -240,6 +256,23 @@ class Config
         end
     {% end %}
 
+    if config.invidious_companion.present?
+      # invidious_companion and signature_server can't work together
+      if config.signature_server
+        puts "Config: You can not run inv_sig_helper and invidious_companion at the same time."
+        exit(1)
+      elsif config.invidious_companion_key.empty?
+        puts "Config: Please configure a key if you are using invidious companion."
+        exit(1)
+      elsif config.invidious_companion_key == "CHANGE_ME!!"
+        puts "Config: The value of 'invidious_companion_key' needs to be changed!!"
+        exit(1)
+      elsif config.invidious_companion_key.size < 16
+        puts "Config: The value of 'invidious_companion_key' needs to be a size of 16 or more."
+        exit(1)
+      end
+    end
+
     # HMAC_key is mandatory
     # See: https://github.com/iv-org/invidious/issues/3854
     if config.hmac_key.empty?

src/invidious/helpers/utils.cr

@@ -383,3 +383,22 @@ def parse_link_endpoint(endpoint : JSON::Any, text : String, video_id : String)
   end
   return text
 end
+
+def encrypt_ecb_without_salt(data, key)
+  cipher = OpenSSL::Cipher.new("aes-128-ecb")
+  cipher.encrypt
+  cipher.key = key
+
+  io = IO::Memory.new
+  io.write(cipher.update(data))
+  io.write(cipher.final)
+  io.rewind
+
+  return io
+end
+
+def invidious_companion_encrypt(data)
+  timestamp = Time.utc.to_unix
+  encrypted_data = encrypt_ecb_without_salt("#{timestamp}|#{data}", CONFIG.invidious_companion_key)
+  return Base64.urlsafe_encode(encrypted_data)
+end

src/invidious/routes/api/manifest.cr

@@ -8,6 +8,11 @@ module Invidious::Routes::API::Manifest
     id = env.params.url["id"]
     region = env.params.query["region"]?
 
+    if CONFIG.invidious_companion.present?
+      invidious_companion = CONFIG.invidious_companion.sample
+      return env.redirect "#{invidious_companion.public_url}/api/manifest/dash/id/#{id}?#{env.params.query}"
+    end
+
     # Since some implementations create playlists based on resolution regardless of different codecs,
     # we can opt to only add a source to a representation if it has a unique height within that representation
     unique_res = env.params.query["unique_res"]?.try { |q| (q == "true" || q == "1").to_unsafe }

src/invidious/routes/embed.cr

@@ -203,6 +203,13 @@ module Invidious::Routes::Embed
       return env.redirect url
     end
 
+    if companion_base_url = video.invidious_companion.try &.["baseUrl"].as_s
+      env.response.headers["Content-Security-Policy"] =
+        env.response.headers["Content-Security-Policy"]
+          .gsub("media-src", "media-src #{companion_base_url}")
+          .gsub("connect-src", "connect-src #{companion_base_url}")
+    end
+
     rendered "embed"
   end
 end

src/invidious/routes/video_playback.cr

@@ -256,6 +256,11 @@ module Invidious::Routes::VideoPlayback
   # YouTube /videoplayback links expire after 6 hours,
   # so we have a mechanism here to redirect to the latest version
   def self.latest_version(env)
+    if CONFIG.invidious_companion.present?
+      invidious_companion = CONFIG.invidious_companion.sample
+      return env.redirect "#{invidious_companion.public_url}/latest_version?#{env.params.query}"
+    end
+
     id = env.params.query["id"]?
     itag = env.params.query["itag"]?.try &.to_i?
 

src/invidious/routes/watch.cr

@@ -192,6 +192,13 @@ module Invidious::Routes::Watch
       captions: video.captions
     )
 
+    if companion_base_url = video.invidious_companion.try &.["baseUrl"].as_s
+      env.response.headers["Content-Security-Policy"] =
+        env.response.headers["Content-Security-Policy"]
+          .gsub("media-src", "media-src #{companion_base_url}")
+          .gsub("connect-src", "connect-src #{companion_base_url}")
+    end
+
     templated "watch"
   end
 
@@ -314,14 +321,18 @@ module Invidious::Routes::Watch
       env.params.query["label"] = URI.decode_www_form(label.as_s)
 
       return Invidious::Routes::API::V1::Videos.captions(env)
-    elsif itag = download_widget["itag"]?.try &.as_i
+    elsif itag = download_widget["itag"]?.try &.as_i.to_s
       # URL params specific to /latest_version
       env.params.query["id"] = video_id
-      env.params.query["itag"] = itag.to_s
       env.params.query["title"] = filename
       env.params.query["local"] = "true"
 
-      return Invidious::Routes::VideoPlayback.latest_version(env)
+      if (CONFIG.invidious_companion.present?)
+        video = get_video(video_id)
+        return env.redirect "#{video.invidious_companion["baseUrl"].as_s}/latest_version?#{env.params.query}"
+      else
+        return Invidious::Routes::VideoPlayback.latest_version(env)
+      end
     else
       return error_template(400, "Invalid label or itag")
     end

src/invidious/videos.cr

@@ -15,7 +15,7 @@ struct Video
   # NOTE: don't forget to bump this number if any change is made to
   # the `params` structure in videos/parser.cr!!!
   #
-  SCHEMA_VERSION = 2
+  SCHEMA_VERSION = 3
 
   property id : String
 
@@ -192,6 +192,10 @@ struct Video
     }
   end
 
+  def invidious_companion : Hash(String, JSON::Any)?
+    info["invidiousCompanion"]?.try &.as_h || {} of String => JSON::Any
+  end
+
   # Macros defining getters/setters for various types of data
 
   private macro getset_string(name)

src/invidious/videos/parser.cr

@@ -108,30 +108,32 @@ def extract_video_info(video_id : String)
   params = parse_video_info(video_id, player_response)
   params["reason"] = JSON::Any.new(reason) if reason
 
-  new_player_response = nil
-
-  # Don't use Android test suite client if po_token is passed because po_token doesn't
-  # work for Android test suite client.
-  if reason.nil? && CONFIG.po_token.nil?
-    # Fetch the video streams using an Android client in order to get the
-    # decrypted URLs and maybe fix throttling issues (#2194). See the
-    # following issue for an explanation about decrypted URLs:
-    # https://github.com/TeamNewPipe/NewPipeExtractor/issues/562
-    client_config.client_type = YoutubeAPI::ClientType::AndroidTestSuite
-    new_player_response = try_fetch_streaming_data(video_id, client_config)
-  end
+  if CONFIG.invidious_companion.present?
+    new_player_response = nil
+
+    # Don't use Android test suite client if po_token is passed because po_token doesn't
+    # work for Android test suite client.
+    if reason.nil? && CONFIG.po_token.nil?
+      # Fetch the video streams using an Android client in order to get the
+      # decrypted URLs and maybe fix throttling issues (#2194). See the
+      # following issue for an explanation about decrypted URLs:
+      # https://github.com/TeamNewPipe/NewPipeExtractor/issues/562
+      client_config.client_type = YoutubeAPI::ClientType::AndroidTestSuite
+      new_player_response = try_fetch_streaming_data(video_id, client_config)
+    end
 
-  # Replace player response and reset reason
-  if !new_player_response.nil?
-    # Preserve captions & storyboard data before replacement
-    new_player_response["storyboards"] = player_response["storyboards"] if player_response["storyboards"]?
-    new_player_response["captions"] = player_response["captions"] if player_response["captions"]?
+    # Replace player response and reset reason
+    if !new_player_response.nil?
+      # Preserve captions & storyboard data before replacement
+      new_player_response["storyboards"] = player_response["storyboards"] if player_response["storyboards"]?
+      new_player_response["captions"] = player_response["captions"] if player_response["captions"]?
 
-    player_response = new_player_response
-    params.delete("reason")
+      player_response = new_player_response
+      params.delete("reason")
+    end
   end
 
-  {"captions", "playabilityStatus", "playerConfig", "storyboards"}.each do |f|
+  {"captions", "playabilityStatus", "playerConfig", "storyboards", "invidiousCompanion"}.each do |f|
     params[f] = player_response[f] if player_response[f]?
   end
 

src/invidious/views/components/player.ecr

@@ -22,6 +22,8 @@
                audio_streams.each_with_index do |fmt, i|
                 src_url  = "/latest_version?id=#{video.id}&itag=#{fmt["itag"]}"
                 src_url += "&local=true" if params.local
+                src_url = video.invidious_companion["baseUrl"].as_s + src_url + 
+                            "&check=#{invidious_companion_encrypt(video.id)}" if (CONFIG.invidious_companion.present?)
 
                 bitrate = fmt["bitrate"]
                 mimetype = HTML.escape(fmt["mimeType"].as_s)
@@ -34,8 +36,12 @@
                 <% end %>
             <% end %>
         <% else %>
-            <% if params.quality == "dash" %>
-                <source src="/api/manifest/dash/id/<%= video.id %>?local=true&unique_res=1" type='application/dash+xml' label="dash">
+            <% if params.quality == "dash"
+               src_url = "/api/manifest/dash/id/" + video.id + "?local=true&unique_res=1"
+               src_url = video.invidious_companion["baseUrl"].as_s + src_url +
+                            "&check=#{invidious_companion_encrypt(video.id)}" if (CONFIG.invidious_companion.present?)
+            %>
+                <source src="<%= src_url %>" type='application/dash+xml' label="dash">
             <% end %>
 
             <%
@@ -44,6 +50,8 @@
             fmt_stream.each_with_index do |fmt, i|
                 src_url  = "/latest_version?id=#{video.id}&itag=#{fmt["itag"]}"
                 src_url += "&local=true" if params.local
+                src_url = video.invidious_companion["baseUrl"].as_s + src_url +
+                            "&check=#{invidious_companion_encrypt(video.id)}" if (CONFIG.invidious_companion.present?)
 
                 quality = fmt["quality"]
                 mimetype = HTML.escape(fmt["mimeType"].as_s)

src/invidious/yt_backend/connection_pool.cr

@@ -46,6 +46,45 @@ struct YoutubeConnectionPool
   end
 end
 
+struct CompanionConnectionPool
+  property pool : DB::Pool(HTTP::Client)
+
+  def initialize(capacity = 5, timeout = 5.0)
+    options = DB::Pool::Options.new(
+      initial_pool_size: 0,
+      max_pool_size: capacity,
+      max_idle_pool_size: capacity,
+      checkout_timeout: timeout
+    )
+
+    @pool = DB::Pool(HTTP::Client).new(options) do
+      companion = CONFIG.invidious_companion.sample
+      next make_client(companion.private_url, force_resolve: true)
+    end
+  end
+
+  def client(&)
+    conn = pool.checkout
+    # Proxy needs to be reinstated every time we get a client from the pool
+    conn.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy
+
+    begin
+      response = yield conn
+    rescue ex
+      conn.close
+
+      companion = CONFIG.invidious_companion.sample
+      conn = make_client(companion.private_url, force_resolve: true)
+
+      response = yield conn
+    ensure
+      pool.release(conn)
+    end
+
+    response
+  end
+end
+
 def add_yt_headers(request)
   request.headers.delete("User-Agent") if request.headers["User-Agent"] == "Crystal"
   request.headers["User-Agent"] ||= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36"
@@ -61,9 +100,9 @@ def add_yt_headers(request)
   end
 end
 
-def make_client(url : URI, region = nil, force_resolve : Bool = false, force_youtube_headers : Bool = false)
+def make_client(url : URI, region = nil, force_resolve : Bool = false, force_youtube_headers : Bool = false, use_http_proxy : Bool = true)
   client = HTTP::Client.new(url)
-  client.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy
+  client.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy && use_http_proxy
 
   # Force the usage of a specific configured IP Family
   if force_resolve
@@ -78,8 +117,8 @@ def make_client(url : URI, region = nil, force_resolve : Bool = false, force_you
   return client
 end
 
-def make_client(url : URI, region = nil, force_resolve : Bool = false, &)
-  client = make_client(url, region, force_resolve: force_resolve)
+def make_client(url : URI, region = nil, force_resolve : Bool = false, use_http_proxy : Bool = true, &)
+  client = make_client(url, region, force_resolve: force_resolve, use_http_proxy: use_http_proxy)
   begin
     yield client
   ensure