FEATURE: Multi-file javascript support for themes (discourse#7526)

You can now add javascript files under `/javascripts/*` in a theme, and they will be loaded as if they were included in core, or a plugin. If you give something the same name as a core/plugin file, it will be overridden. Support file extensions are `.js.es6`, `.hbs` and `.raw.hbs`.

app/helpers/application_helper.rb

@@ -431,6 +431,11 @@ def theme_translations_lookup
       &.html_safe
   end
 
+  def theme_js_lookup
+    Theme.lookup_field(theme_ids, :extra_js, nil)
+      &.html_safe
+  end
+
   def discourse_stylesheet_link_tag(name, opts = {})
     if opts.key?(:theme_ids)
       ids = opts[:theme_ids] unless customization_disabled?

app/models/javascript_cache.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 class JavascriptCache < ActiveRecord::Base
   belongs_to :theme_field
+  belongs_to :theme
 
   validate :content_cannot_be_nil
 
@@ -26,14 +27,21 @@ def content_cannot_be_nil
 # Table name: javascript_caches
 #
 #  id             :bigint           not null, primary key
-#  theme_field_id :bigint           not null
+#  theme_field_id :bigint
 #  digest         :string
 #  content        :text             not null
 #  created_at     :datetime         not null
 #  updated_at     :datetime         not null
+#  theme_id       :bigint
 #
 # Indexes
 #
 #  index_javascript_caches_on_digest          (digest)
 #  index_javascript_caches_on_theme_field_id  (theme_field_id)
+#  index_javascript_caches_on_theme_id        (theme_id)
+#
+# Foreign Keys
+#
+#  fk_rails_...  (theme_field_id => theme_fields.id) ON DELETE => cascade
+#  fk_rails_...  (theme_id => themes.id) ON DELETE => cascade
 #

app/models/theme.rb

@@ -25,6 +25,7 @@ class Theme < ActiveRecord::Base
   belongs_to :remote_theme, autosave: true
 
   has_one :settings_field, -> { where(target_id: Theme.targets[:settings], name: "yaml") }, class_name: 'ThemeField'
+  has_one :javascript_cache, dependent: :destroy
   has_many :locale_fields, -> { filter_locale_fields(I18n.fallbacks[I18n.locale]) }, class_name: 'ThemeField'
 
   validate :component_validations
@@ -51,19 +52,26 @@ def notify_color_change(color)
     changed_fields.each(&:save!)
     changed_fields.clear
 
+    if saved_change_to_name?
+      theme_fields.select(&:basic_html_field?).each(&:invalidate_baked!)
+    end
+
     Theme.expire_site_cache! if saved_change_to_user_selectable? || saved_change_to_name?
     notify_with_scheme = saved_change_to_color_scheme_id?
-    name_changed = saved_change_to_name?
 
     reload
     settings_field&.ensure_baked! # Other fields require setting to be **baked**
     theme_fields.each(&:ensure_baked!)
 
-    if name_changed
-      theme_fields.select { |f| f.basic_html_field? }.each do |f|
-        f.value_baked = nil
-        f.ensure_baked!
-      end
+    all_extra_js = theme_fields.where(target_id: Theme.targets[:extra_js]).pluck(:value_baked).join("\n")
+    if all_extra_js.present?
+      js_compiler = ThemeJavascriptCompiler.new(id, name)
+      js_compiler.append_raw_script(all_extra_js)
+      js_compiler.prepend_settings(cached_settings) if cached_settings.present?
+      javascript_cache || build_javascript_cache
+      javascript_cache.update!(content: js_compiler.content)
+    else
+      javascript_cache&.destroy!
     end
 
     remove_from_cache!
@@ -238,7 +246,7 @@ def self.clear_cache!
   end
 
   def self.targets
-    @targets ||= Enum.new(common: 0, desktop: 1, mobile: 2, settings: 3, translations: 4, extra_scss: 5)
+    @targets ||= Enum.new(common: 0, desktop: 1, mobile: 2, settings: 3, translations: 4, extra_scss: 5, extra_js: 6)
   end
 
   def self.lookup_target(target_id)
@@ -276,6 +284,11 @@ def self.refresh_message_for_targets(targets, theme_ids)
   end
 
   def self.resolve_baked_field(theme_ids, target, name)
+    if target == :extra_js
+      caches = JavascriptCache.where(theme_id: theme_ids)
+      caches = caches.sort_by { |cache| theme_ids.index(cache.theme_id) }
+      return caches.map { |c| "<script src='#{c.url}'></script>" }.join("\n")
+    end
     list_baked_fields(theme_ids, target, name).map { |f| f.value_baked || f.value }.join("\n")
   end
 

app/models/theme_field.rb

@@ -46,7 +46,8 @@ def self.types
                         theme_upload_var: 2,
                         theme_color_var: 3, # No longer used
                         theme_var: 4, # No longer used
-                        yaml: 5)
+                        yaml: 5,
+                        js: 6)
   end
 
   def self.theme_var_type_ids
@@ -122,6 +123,29 @@ def process_html(html)
     [doc.to_s, errors&.join("\n")]
   end
 
+  def process_extra_js(content)
+    errors = []
+
+    js_compiler = ThemeJavascriptCompiler.new(theme_id, theme.name)
+    filename, extension = name.split(".", 2)
+    begin
+      case extension
+      when "js.es6"
+        js_compiler.append_module(content, filename)
+      when "hbs"
+        js_compiler.append_ember_template(filename.sub("discourse/templates/", ""), content)
+      when "raw.hbs"
+        js_compiler.append_raw_template(filename, content)
+      else
+        raise ThemeJavascriptCompiler::CompileError.new(I18n.t("themes.compile_error.unrecognized_extension", extension: extension))
+      end
+    rescue ThemeJavascriptCompiler::CompileError => ex
+      errors << ex.message
+    end
+
+    [js_compiler.content, errors&.join("\n")]
+  end
+
   def raw_translation_data(internal: false)
     # Might raise ThemeTranslationParser::InvalidYaml
     ThemeTranslationParser.new(self, internal: internal).load
@@ -227,6 +251,8 @@ def self.guess_type(name:, target:)
       types[:scss]
     elsif target.to_s == "extra_scss"
       types[:scss]
+    elsif target.to_s == "extra_js"
+      types[:js]
     elsif target.to_s == "settings" || target.to_s == "translations"
       types[:yaml]
     end
@@ -249,6 +275,10 @@ def basic_html_field?
       ThemeField.html_fields.include?(self.name)
   end
 
+  def extra_js_field?
+    Theme.targets[self.target_id] == :extra_js
+  end
+
   def basic_scss_field?
     ThemeField.basic_targets.include?(Theme.targets[self.target_id].to_s) &&
       ThemeField.scss_fields.include?(self.name)
@@ -278,6 +308,10 @@ def ensure_baked!
       self.value_baked, self.error = translation_field? ? process_translation : process_html(self.value)
       self.error = nil unless self.error.present?
       self.compiler_version = COMPILER_VERSION
+    elsif extra_js_field?
+      self.value_baked, self.error = process_extra_js(self.value)
+      self.error = nil unless self.error.present?
+      self.compiler_version = COMPILER_VERSION
     elsif basic_scss_field?
       ensure_scss_compiles!
       Stylesheet::Manager.clear_theme_cache!
@@ -382,6 +416,9 @@ def filename_from_opts(opts)
     ThemeFileMatcher.new(regex: /^(?:scss|stylesheets)\/(?<name>.+)\.scss$/,
                          targets: :extra_scss, names: nil, types: :scss,
                          canonical: -> (h) { "stylesheets/#{h[:name]}.scss" }),
+    ThemeFileMatcher.new(regex: /^javascripts\/(?<name>.+)$/,
+                         targets: :extra_js, names: nil, types: :js,
+                         canonical: -> (h) { "javascripts/#{h[:name]}" }),
     ThemeFileMatcher.new(regex: /^settings\.ya?ml$/,
                          names: "yaml", types: :yaml, targets: :settings,
                          canonical: -> (h) { "settings.yml" }),

app/views/layouts/application.html.erb

@@ -39,6 +39,7 @@
 
     <%- unless customization_disabled? %>
       <%= raw theme_translations_lookup %>
+      <%= raw theme_js_lookup %>
       <%= raw theme_lookup("head_tag") %>
     <%- end %>
 

config/locales/server.en.yml

@@ -76,6 +76,8 @@ en:
   themes:
     bad_color_scheme: "Can not update theme, invalid color palette"
     other_error: "Something went wrong updating theme"
+    compile_error:
+      unrecognized_extension: "Unrecognized file extension: %{extension}"
     import_error:
       generic: An error occured while importing that theme
       about_json: "Import Error: about.json does not exist, or is invalid"

db/migrate/20190513143015_add_theme_id_to_javascript_cache.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+class AddThemeIdToJavascriptCache < ActiveRecord::Migration[5.2]
+  def up
+    make_changes
+    execute "ALTER TABLE javascript_caches ADD CONSTRAINT enforce_theme_or_theme_field CHECK ((theme_id IS NOT NULL AND theme_field_id IS NULL) OR (theme_id IS NULL AND theme_field_id IS NOT NULL))"
+  end
+  def down
+    execute "ALTER TABLE javascript_caches DROP CONSTRAINT enforce_theme_or_theme_field"
+    revert { make_changes }
+  end
+
+  private
+
+  def make_changes
+    add_reference :javascript_caches, :theme, foreign_key: { on_delete: :cascade }
+    add_foreign_key :javascript_caches, :theme_fields, on_delete: :cascade
+
+    begin
+      Migration::SafeMigrate.disable!
+      change_column_null :javascript_caches, :theme_field_id, true
+    ensure
+      Migration::SafeMigrate.enable!
+    end
+  end
+end

lib/theme_javascript_compiler.rb

@@ -202,22 +202,36 @@ def append_raw_script(script)
     @content << script + "\n"
   end
 
+  def append_module(script, name)
+    script.prepend theme_variables
+    template = Tilt::ES6ModuleTranspilerTemplate.new {}
+    @content << template.module_transpile(script, "", name)
+  rescue MiniRacer::RuntimeError => ex
+    raise CompileError.new ex.message
+  end
+
   def append_js_error(message)
     @content << "console.error('Theme Transpilation Error:', #{message.inspect});"
   end
 
   private
 
+  def theme_variables
+    <<~JS
+      const __theme_name__ = "#{@theme_name.gsub('"', "\\\"")}";
+      const settings = Discourse.__container__
+        .lookup("service:theme-settings")
+        .getObjectForTheme(#{@theme_id});
+      const themePrefix = (key) => `theme_translations.#{@theme_id}.${key}`;
+    JS
+  end
+
   def transpile(es6_source, version)
     template = Tilt::ES6ModuleTranspilerTemplate.new {}
     wrapped = <<~PLUGIN_API_JS
       (function() {
         if ('Discourse' in window && typeof Discourse._registerPluginCode === 'function') {
-          const __theme_name__ = "#{@theme_name.gsub('"', "\\\"")}";
-          const settings = Discourse.__container__
-            .lookup("service:theme-settings")
-            .getObjectForTheme(#{@theme_id});
-          const themePrefix = (key) => `theme_translations.#{@theme_id}.${key}`;
+          #{theme_variables}
           Discourse._registerPluginCode('#{version}', api => {
             try {
             #{es6_source}

spec/models/theme_field_spec.rb

@@ -147,6 +147,40 @@
     expect(result).to include(".class5")
   end
 
+  it "correctly handles extra JS fields" do
+    theme = Fabricate(:theme)
+    js_field = theme.set_field(target: :extra_js, name: "discourse/controllers/discovery.js.es6", value: "import 'discourse/lib/ajax'; console.log('hello');")
+    hbs_field = theme.set_field(target: :extra_js, name: "discourse/templates/discovery.hbs", value: "{{hello-world}}")
+    raw_hbs_field = theme.set_field(target: :extra_js, name: "discourse/templates/discovery.raw.hbs", value: "{{hello-world}}")
+    unknown_field = theme.set_field(target: :extra_js, name: "discourse/controllers/discovery.blah", value: "this wont work")
+    theme.save!
+
+    expected_js = <<~JS
+      define("discourse/controllers/discovery", ["discourse/lib/ajax"], function () {
+        "use strict";
+
+        var __theme_name__ = "#{theme.name}";
+        var settings = Discourse.__container__.lookup("service:theme-settings").getObjectForTheme(#{theme.id});
+        var themePrefix = function themePrefix(key) {
+          return "theme_translations.#{theme.id}." + key;
+        };
+        console.log('hello');
+      });
+    JS
+    expect(js_field.reload.value_baked).to eq(expected_js.strip)
+
+    expect(hbs_field.reload.value_baked).to include('Ember.TEMPLATES["discovery"]')
+    expect(raw_hbs_field.reload.value_baked).to include('Discourse.RAW_TEMPLATES["discourse/templates/discovery"]')
+    expect(unknown_field.reload.value_baked).to eq("")
+    expect(unknown_field.reload.error).to eq(I18n.t("themes.compile_error.unrecognized_extension", extension: "blah"))
+
+    # All together
+    expect(theme.javascript_cache.content).to include('Ember.TEMPLATES["discovery"]')
+    expect(theme.javascript_cache.content).to include('Discourse.RAW_TEMPLATES["discourse/templates/discovery"]')
+    expect(theme.javascript_cache.content).to include('define("discourse/controllers/discovery"')
+    expect(theme.javascript_cache.content).to include("var settings =")
+  end
+
   def create_upload_theme_field!(name)
     ThemeField.create!(
       theme_id: 1,

spec/models/theme_spec.rb

@@ -367,6 +367,7 @@ def transpile(html)
           var themePrefix = function themePrefix(key) {
             return 'theme_translations.#{theme.id}.' + key;
           };
+
           Discourse._registerPluginCode('1.0', function (api) {
             try {
               alert(settings.name);var a = function a() {};
@@ -402,6 +403,7 @@ def transpile(html)
           var themePrefix = function themePrefix(key) {
             return 'theme_translations.#{theme.id}.' + key;
           };
+
           Discourse._registerPluginCode('1.0', function (api) {
             try {
               alert(settings.name);var a = function a() {};