✨ Add basic ESearch support

lib/net/imap.rb

@@ -1930,13 +1930,13 @@ def uid_expunge(uid_set)
     end
 
     # :call-seq:
-    #   search(criteria, charset = nil) -> result
+    #   search(criteria, charset = nil, esearch: false) -> result
     #
     # Sends a {SEARCH command [IMAP4rev1 §6.4.4]}[https://www.rfc-editor.org/rfc/rfc3501#section-6.4.4]
     # to search the mailbox for messages that match the given search +criteria+,
-    # and returns a SearchResult.  SearchResult inherits from Array (for
-    # backward compatibility) but adds SearchResult#modseq when the +CONDSTORE+
-    # capability has been enabled.
+    # and returns either a SearchResult or an ESearchResult.  SearchResult
+    # inherits from Array (for backward compatibility) but adds
+    # SearchResult#modseq when the +CONDSTORE+ capability has been enabled.
     #
     # +criteria+ is one or more search keys and their arguments, which may be
     # provided as an array or a string.
@@ -1967,8 +1967,16 @@ def uid_expunge(uid_set)
     # set}[https://www.iana.org/assignments/character-sets/character-sets.xhtml]
     # used by strings in the search +criteria+.  When +charset+ isn't specified,
     # either <tt>"US-ASCII"</tt> or <tt>"UTF-8"</tt> is assumed, depending on
-    # the server's capabilities.  +charset+ may be sent inside +criteria+
-    # instead of as a separate argument.
+    # the server's capabilities.
+    #
+    # _NOTE:_ Return options and +charset+ may be sent as part of +criteria+.
+    # Do not use the +charset+ argument when either return options or charset
+    # are embedded in +criteria+.
+    #
+    # +esearch+ controls the return type when the server does not return any
+    # search results.  If +esearch+ is +true+ or +criteria+ begins with
+    # +RETURN+, an empty ESearchResult will be returned.  When +esearch+ is
+    # +false+, an empty SearchResult will be returned.
     #
     # Related: #uid_search
     #
@@ -1988,6 +1996,12 @@ def uid_expunge(uid_set)
     #    # criteria string contains charset arg
     #    imap.search("CHARSET UTF-8 OR UNSEEN (FLAGGED SUBJECT foo)")
     #
+    # Sending return optionsand charset embedded in the +crriteria+ arg:
+    #    imap.search("RETURN (MIN MAX) CHARSET UTF-8 (OR UNSEEN FLAGGED)")
+    #    imap.search(["RETURN", %w(MIN MAX),
+    #                 "CHARSET", "UTF-8",
+    #                 %w(OR UNSEEN FLAGGED)])
+    #
     # ===== Search keys
     #
     # For full definitions of the standard search +criteria+,
@@ -2178,6 +2192,12 @@ def uid_expunge(uid_set)
     #
     # ===== Capabilities
     #
+    # Return options should only be specified when the server supports
+    # +IMAP4rev2+ or an extension that allows them, such as +ESEARCH+.
+    #
+    # When +IMAP4rev2+ is enabled, or when the server supports +IMAP4rev2+ but
+    # not +IMAP4rev1+, ESearchResult is always returned instead of SearchResult.
+    #
     # If CONDSTORE[https://www.rfc-editor.org/rfc/rfc7162.html] is supported
     # and enabled for the selected mailbox, a non-empty SearchResult will
     # include a +MODSEQ+ value.
@@ -3131,12 +3151,35 @@ def enforce_logindisabled?
       end
     end
 
-    def search_internal(cmd, keys, charset = nil)
+    HasSearchReturnOpts = ->keys {
+      keys in RawData[/\ARETURN /] | Array[/\ARETURN\z/i, *]
+    }
+    private_constant :HasSearchReturnOpts
+
+    def search_internal(cmd, keys, charset = nil, esearch: nil)
       keys = normalize_searching_criteria(keys)
       args = charset ? ["CHARSET", charset, *keys] : keys
+      # TODO: check if certain extensions are enabled
+      esearch = (keys in HasSearchReturnOpts) if esearch.nil?
       synchronize do
-        send_command(cmd, *args)
-        clear_responses("SEARCH").last || []
+        tagged = send_command(cmd, *args)
+        tag    = tagged.tag
+        # Only the last ESEARCH or SEARCH is used.  Excess results are ignored.
+        esearch_result = extract_responses("ESEARCH") {|response|
+          response in ESearchResult(tag: ^tag)
+        }.last
+        search_result = clear_responses("SEARCH").last
+        if esearch_result
+          esearch_result # silently ignore SEARCH results
+        elsif search_result
+          # TODO: warn if ESEARCH result was expected, i.e: buggy server?
+          # warn EXPECTED_ESEARCH_RESULT if esearch
+          search_result
+        elsif esearch
+          ESearchResult[tag:, uid: cmd == "UID SEARCH"]
+        else
+          SearchResult[]
+        end
       end
     end
 
@@ -3276,6 +3319,7 @@ def self.saslprep(string, **opts)
 require_relative "imap/config"
 require_relative "imap/command_data"
 require_relative "imap/data_encoding"
+require_relative "imap/data_lite"
 require_relative "imap/flags"
 require_relative "imap/response_data"
 require_relative "imap/response_parser"

lib/net/imap/command_data.rb

@@ -119,19 +119,33 @@ def send_symbol_data(symbol)
       put_string("\\" + symbol.to_s)
     end
 
-    class RawData # :nodoc:
+    # RawData can be used with some Net::IMAP commands to bypass _all_ argument
+    # validation, normalization, and encoding.  RawData will be sent _directly_
+    # to the socket.
+    #
+    # *WARNING:* Using RawData directly should be avoided.  Use only with
+    # extreme caution.  Do _NOT_ use with unsanitized user input.  Using
+    # RawData can lead to {injection
+    # flaws}[https://owasp.org/www-community/Injection_Flaws].
+    class RawData
+      attr_reader :data
+
+      def initialize(data)
+        @data = data
+      end
+
       def send_data(imap, tag)
-        imap.__send__(:put_string, @data)
+        imap.__send__(:put_string, data)
       end
 
       def validate
       end
 
-      private
-
-      def initialize(data)
-        @data = data
+      def ==(other)
+        other.class == self.class && other.data == data
       end
+
+      def deconstruct = [data]
     end
 
     class Atom # :nodoc:

lib/net/imap/data_lite.rb

@@ -0,0 +1,165 @@
+# frozen_string_literal: true
+
+# Some of the code in this file was copied from the polyfill-data gem.
+#
+# MIT License
+#
+# Copyright (c) 2023 Jim Gay, Joel Drapper, Nicholas Evans
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+# TODO: fix tests to allow same yaml for both Data and DataLite
+# NOTE: psych 5.1.2 doesn't encode ::Data correctly!
+# if RUBY_VERSION >= "3.2.0"
+#   return
+# end
+
+module Net
+  class IMAP
+
+    # See {ruby's documentation for Data}[https://docs.ruby-lang.org/en/3.3/Data.html].
+    #
+    # DataLite is a temporary polyfill for ruby 3.2's
+    # Data[https://docs.ruby-lang.org/en/3.3/Data.html].  <em>This class is
+    # not defined for ruby versions >= 3.2.</em>  It will only be defined when
+    # using ruby 3.1 (+net-imap+ no longer supports ruby versions < 3.1).  It
+    # <em>will be removed</em> in +net-imap+ 0.6, when support for ruby 3.1 is
+    # dropped.
+    #
+    # It is aliased as Net::IMAP::Data so that, in ruby 3.1, any reference to
+    # "Data" that is namespaced inside Net::IMAP will use it.  This way,
+    # Net::IMAP's code shouldn't need to change to work with both
+    # Net::IMAP::DataLite and
+    # {::Data}[https://docs.ruby-lang.org/en/3.3/Data.html].
+    #
+    # Some of the code in this class was copied or adapted from the
+    # {polyfill-data gem}[https://rubygems.org/gems/polyfill-data], by Jim Gay
+    # and Joel Drapper, under the MIT license terms.
+    class DataLite
+      singleton_class.undef_method :new
+
+      TYPE_ERROR    = "%p is not a symbol nor a string"
+      ATTRSET_ERROR = "invalid data member: %p"
+      DUP_ERROR     = "duplicate member: %p"
+      ARITY_ERROR   = "wrong number of arguments (given %d, expected %s)"
+      private_constant :TYPE_ERROR, :ATTRSET_ERROR, :DUP_ERROR, :ARITY_ERROR
+
+      # *NOTE:+ DataLite.define does not support member names which are not
+      # valid local variable names.
+      def self.define(*args, &block)
+        members = args.each_with_object({}) do |arg, members|
+          arg = arg.to_str unless arg in Symbol | String if arg.respond_to?(:to_str)
+          arg = arg.to_sym if     arg in String
+          arg in Symbol     or  raise TypeError,     TYPE_ERROR    % [arg]
+          arg in %r{=}      and raise ArgumentError, ATTRSET_ERROR % [arg]
+          members.key?(arg) and raise ArgumentError, DUP_ERROR     % [arg]
+          members[arg] = true
+        end
+        members = members.keys.freeze
+
+        klass = ::Class.new(self)
+
+        klass.singleton_class.undef_method :define
+        klass.define_singleton_method(:members) { members }
+
+        def klass.new(*args, **kwargs, &block)
+          if kwargs.size.positive?
+            if args.size.positive?
+              raise ArgumentError, ARITY_ERROR % [args.size, 0]
+            end
+          elsif members.size < args.size
+            expected = members.size.zero? ? 0 : 0..members.size
+            raise ArgumentError, ARITY_ERROR % [args.size, expected]
+          else
+            kwargs = Hash[members.take(args.size).zip(args)]
+          end
+          allocate.tap do |instance|
+            instance.__send__(:initialize, **kwargs, &block)
+          end.freeze
+        end
+
+        klass.singleton_class.alias_method :[], :new
+        klass.attr_reader(*members)
+
+        # Dynamically defined initializer methods are in an included module,
+        # rather than directly on DataLite (like in ruby 3.2+):
+        # * simpler to handle required kwarg ArgumentErrors
+        # * easier to ensure consistent ivar assignment order (object shape)
+        # * faster than instance_variable_set
+        klass.include(Module.new do
+          if members.any?
+            kwargs = members.map{"#{_1.name}:"}.join(", ")
+            params = members.map(&:name).join(", ")
+            ivars  = members.map{"@#{_1.name}"}.join(", ")
+            attrs  = members.map{"attrs[:#{_1.name}]"}.join(", ")
+            module_eval <<~RUBY, __FILE__, __LINE__ + 1
+              protected
+              def initialize(#{kwargs}) #{ivars} = #{params}; freeze end
+              def marshal_load(attrs)   #{ivars} = #{attrs};  freeze end
+            RUBY
+          end
+        end)
+
+        klass.module_eval do _1.module_eval(&block) end if block_given?
+
+        klass
+      end
+
+      def members;     self.class.members                              end
+      def attributes;  Hash[members.map {|m| [m, send(m)] }]           end
+      def to_h(&block) attributes.to_h(&block)                         end
+      def hash;        to_h.hash                                       end
+      def ==(other)    self.class == other.class && to_h == other.to_h end
+      def eql?(other)  self.class == other.class && hash == other.hash end
+      def deconstruct; attributes.values                               end
+
+      def deconstruct_keys(keys)
+        raise TypeError unless keys.is_a?(Array) || keys.nil?
+        return attributes if keys&.first.nil?
+        attributes.slice(*keys)
+      end
+
+      def with(**kwargs)
+        return self if kwargs.empty?
+        self.class.new(**attributes.merge(kwargs))
+      end
+
+      # +NOTE:+ Unlike ruby 3.2's <tt>Data#inspect</tt>, this has no guard
+      # against infinite recursion.
+      def inspect
+        attrs   = attributes.map {|kv| "%s=%p" % kv }.join(", ")
+        display = ["data", self.class.name, attrs].compact.join(" ")
+        "#<#{display}>"
+      end
+      alias_method :to_s, :inspect
+
+      def encode_with(coder) coder.map = attributes.transform_keys(&:to_s) end
+      def init_with(coder) marshal_load(coder.map.transform_keys(&:to_sym)) end
+
+      private
+
+      def initialize_copy(source) super.freeze end
+      def marshal_dump; attributes end
+
+    end
+
+    Data = DataLite
+
+  end
+end