diff --git a/keras_tuner/applications/augment.py b/keras_tuner/applications/augment.py index b5671e681..8a27e894f 100644 --- a/keras_tuner/applications/augment.py +++ b/keras_tuner/applications/augment.py @@ -36,14 +36,17 @@ class HyperImageAugment(hypermodel.HyperModel): - """Builds HyperModel for image augmentation. + """A image augmentation hypermodel. - Only supporting augmentations available in Keras preprocessing layers currently. + The `HyperImageAugment` class searches for the best combination of image + augmentation operations in Keras preprocessing layers. The input shape of + the model should be (height, width, channels). The output of the model is + of the same shape as the input. - Arguments: + Args: input_shape: Optional shape tuple, e.g. `(256, 256, 3)`. - input_tensor: Optional Keras tensor (i.e. output of - `layers.Input()`) to use as image input for the model. + input_tensor: Optional Keras tensor (i.e. output of `layers.Input()`) + to use as image input for the model. rotate: A number between [0, 1], a list of two numbers between [0, 1] or None. Configures the search space of the factor of random rotation transform in the augmentation. A factor is chosen for each @@ -85,8 +88,8 @@ class HyperImageAugment(hypermodel.HyperModel): determines number of layers of augment transforms are applied, each randomly picked from all available transform types with equal probability on each sample. - **kwargs: Additional keyword arguments that apply to all - HyperModels. See `keras_tuner.HyperModel`. + **kwargs: Additional keyword arguments that apply to all hypermodels. + See `keras_tuner.HyperModel`. Example: @@ -231,8 +234,9 @@ def _build_fixedaug_layers(self, inputs, hp): def _register_transform(self, transform_name, transform_params): """Register a transform and format parameters for tuning the transform. - Arguments: - transform_name: str, the name of the transform. + + Args: + transform_name: A string, the name of the transform. trnasform_params: A number between [0, 1], a list of two numbers between [0, 1] or None. If set to a single number x, the corresponding transform factor will be between [0, x]. diff --git a/keras_tuner/applications/efficientnet.py b/keras_tuner/applications/efficientnet.py index d271e2b5e..9930b3839 100644 --- a/keras_tuner/applications/efficientnet.py +++ b/keras_tuner/applications/efficientnet.py @@ -46,26 +46,25 @@ class HyperEfficientNet(hypermodel.HyperModel): - """An EfficientNet HyperModel. - - Models built by this HyperModel takes input image data in - ints [0, 255]. The output data should be one-hot encoded - with number of classes matching `classes`. - - Arguments: - input_shape: shape tuple, e.g. `(256, 256, 3)`. - Input images will be resized if different from - the default input size of the version of - efficientnet base model used. - One of `input_shape` or `input_tensor` must be - specified. - input_tensor: Keras tensor to use as image input for the model. - One of `input_shape` or `input_tensor` must be - specified. - classes: number of classes to classify images into. - augmentation_model: optional Model or HyperModel for image augmentation. - **kwargs: Additional keyword arguments that apply to all - HyperModels. See `keras_tuner.HyperModel`. + """An EfficientNet hypermodel. + + Models built by `HyperEfficientNet` take images with shape (height, width, + channels) as input. The output are one-hot encoded with the length matching + the number of classes specified by the `classes` argument. + + Args: + input_shape: Optional shape tuple, e.g. `(256, 256, 3)`. One of + `input_shape` or `input_tensor` must be specified. + input_tensor: Optional Keras tensor (i.e. output of `layers.Input()`) + to use as image input for the model. One of `input_shape` or + `input_tensor` must be specified. + classes: Optional number of classes to classify images into, only to be + specified if `include_top` is True, and if no `weights` argument is + specified. + augmentation_model: Optional `Model` or `HyperModel` instance for image + augmentation. + **kwargs: Additional keyword arguments that apply to all hypermodels. + See `keras_tuner.HyperModel`. """ def __init__( @@ -81,7 +80,7 @@ def __init__( ): raise ValueError( "Keyword augmentation_model should be " - "a HyperModel, a Keras Model or empty. " + "a `HyperModel`, a Keras `Model` or empty. " "Received {}.".format(augmentation_model) ) @@ -153,8 +152,8 @@ def build(self, hp): def _compile(self, model, hp): """Compile model using hyperparameters in hp. - When subclassing the hypermodel, this may - be overriden to change behavior of compiling. + When subclassing the hypermodel, this may be overriden to change + behavior of compiling. """ learning_rate = hp.Choice("learning_rate", [0.1, 0.01, 0.001], default=0.01) optimizer = tf.keras.optimizers.SGD( diff --git a/keras_tuner/applications/resnet.py b/keras_tuner/applications/resnet.py index 970448646..e33c8c29b 100644 --- a/keras_tuner/applications/resnet.py +++ b/keras_tuner/applications/resnet.py @@ -23,23 +23,25 @@ class HyperResNet(hypermodel.HyperModel): - """A ResNet HyperModel. - - Arguments: - include_top: whether to include the fully-connected - layer at the top of the network. - input_shape: Optional shape tuple, e.g. `(256, 256, 3)`. - One of `input_shape` or `input_tensor` must be - specified. - input_tensor: Optional Keras tensor (i.e. output of - `layers.Input()`) to use as image input for the model. - One of `input_shape` or `input_tensor` must be - specified. - classes: optional number of classes to classify images - into, only to be specified if `include_top` is True, - and if no `weights` argument is specified. - **kwargs: Additional keyword arguments that apply to all - HyperModels. See `keras_tuner.HyperModel`. + """A ResNet hypermodel. + + Models built by `HyperResNet` take images with shape (height, width, + channels) as input. The output are one-hot encoded with the length matching + the number of classes specified by the `classes` argument. + + Args: + include_top: Boolean, whether to include the fully-connected layer at + the top of the network. + input_shape: Optional shape tuple, e.g. `(256, 256, 3)`. One of + `input_shape` or `input_tensor` must be specified. + input_tensor: Optional Keras tensor (i.e. output of `layers.Input()`) + to use as image input for the model. One of `input_shape` or + `input_tensor` must be specified. + classes: Optional number of classes to classify images into, only to be + specified if `include_top` is True, and if no `weights` argument is + specified. + **kwargs: Additional keyword arguments that apply to all hypermodels. + See `keras_tuner.HyperModel`. """ def __init__( @@ -147,7 +149,8 @@ def build(self, hp): def block1(x, filters, kernel_size=3, stride=1, conv_shortcut=True, name=None): """A residual block. - # Arguments + + Args: x: input tensor. filters: integer, filters of the bottleneck layer. kernel_size: default 3, kernel size of the bottleneck layer. @@ -155,7 +158,8 @@ def block1(x, filters, kernel_size=3, stride=1, conv_shortcut=True, name=None): conv_shortcut: default True, use convolution shortcut if True, otherwise identity shortcut. name: string, block label. - # Returns + + Returns: Output tensor for the residual block. """ bn_axis = 3 if backend.image_data_format() == "channels_last" else 1 @@ -195,7 +199,7 @@ def block1(x, filters, kernel_size=3, stride=1, conv_shortcut=True, name=None): def stack1(x, filters, blocks, stride1=2, name=None): """A set of stacked residual blocks. - Arguments: + Args: x: input tensor. filters: integer, filters of the bottleneck layer in a block. blocks: integer, blocks in the stacked blocks. @@ -214,7 +218,7 @@ def stack1(x, filters, blocks, stride1=2, name=None): def block2(x, filters, kernel_size=3, stride=1, conv_shortcut=False, name=None): """A residual block. - Arguments: + Args: x: input tensor. filters: integer, filters of the bottleneck layer. kernel_size: default 3, kernel size of the bottleneck layer. @@ -265,7 +269,7 @@ def block2(x, filters, kernel_size=3, stride=1, conv_shortcut=False, name=None): def stack2(x, filters, blocks, stride1=2, name=None): """A set of stacked residual blocks. - Arguments: + Args: x: input tensor. filters: integer, filters of the bottleneck layer in a block. blocks: integer, blocks in the stacked blocks. @@ -287,7 +291,7 @@ def block3( ): """A residual block. - Arguments: + Args: x: input tensor. filters: integer, filters of the bottleneck layer. kernel_size: default 3, kernel size of the bottleneck layer. @@ -365,7 +369,7 @@ def block3( def stack3(x, filters, blocks, stride1=2, groups=32, name=None): """A set of stacked residual blocks. - Arguments: + Args: x: input tensor. filters: integer, filters of the bottleneck layer in a block. blocks: integer, blocks in the stacked blocks. diff --git a/keras_tuner/applications/xception.py b/keras_tuner/applications/xception.py index 14c3af61c..e10d9bd45 100644 --- a/keras_tuner/applications/xception.py +++ b/keras_tuner/applications/xception.py @@ -20,23 +20,25 @@ class HyperXception(hypermodel.HyperModel): - """An Xception HyperModel. - - Arguments: - include_top: whether to include the fully-connected - layer at the top of the network. - input_shape: Optional shape tuple, e.g. `(256, 256, 3)`. - One of `input_shape` or `input_tensor` must be - specified. - input_tensor: Optional Keras tensor (i.e. output of - `layers.Input()`) to use as image input for the model. - One of `input_shape` or `input_tensor` must be - specified. - classes: optional number of classes to classify images - into, only to be specified if `include_top` is True, - and if no `weights` argument is specified. - **kwargs: Additional keyword arguments that apply to all - HyperModels. See `keras_tuner.HyperModel`. + """An Xception hypermodel. + + Models built by `HyperXception` take images with shape (height, width, + channels) as input. The output are one-hot encoded with the length matching + the number of classes specified by the `classes` argument. + + Args: + include_top: Boolean, whether to include the fully-connected layer at + the top of the network. + input_shape: Optional shape tuple, e.g. `(256, 256, 3)`. One of + `input_shape` or `input_tensor` must be specified. + input_tensor: Optional Keras tensor (i.e. output of `layers.Input()`) + to use as image input for the model. One of `input_shape` or + `input_tensor` must be specified. + classes: Optional number of classes to classify images into, only to be + specified if `include_top` is True, and if no `weights` argument is + specified. + **kwargs: Additional keyword arguments that apply to all hypermodels. + See `keras_tuner.HyperModel`. """ def __init__( diff --git a/keras_tuner/distribute/utils.py b/keras_tuner/distribute/utils.py index 43f906105..efbc3cd72 100644 --- a/keras_tuner/distribute/utils.py +++ b/keras_tuner/distribute/utils.py @@ -23,11 +23,11 @@ def has_chief_oracle(): """Checks for distributed tuning with a chief Oracle. - `CloudOracle` manages its own distribution and so should not set - "KERASTUNER_ORACLE_IP" + `CloudOracle` manages its own distribution so should not set + "KERASTUNER_ORACLE_IP". Returns: - bool. Whether distributed tuning with a chief Oracle should be run. + Boolean, whether distributed tuning with a chief Oracle should be run. """ if "KERASTUNER_ORACLE_IP" in os.environ: if "KERASTUNER_ORACLE_PORT" not in os.environ: diff --git a/keras_tuner/engine/base_tuner.py b/keras_tuner/engine/base_tuner.py index 93a3017d9..6b309d2d1 100644 --- a/keras_tuner/engine/base_tuner.py +++ b/keras_tuner/engine/base_tuner.py @@ -35,20 +35,27 @@ class BaseTuner(stateful.Stateful): """Tuner base class. - May be subclassed to create new tuners, including for non-Keras models. + `BaseTuner` is the base class for all Tuners, which manages the search + loop, Oracle, logging, saving, etc. Tuners for non-Keras models can be + created by subclassing `BaseTuner`. - Arguments: + Args: oracle: Instance of Oracle class. hypermodel: Instance of HyperModel class (or callable that takes hyperparameters and returns a Model instance). - directory: String. Path to the working directory (relative). - project_name: Name to use as prefix for files saved - by this Tuner. - logger: Optional. Instance of Logger class, used for streaming data - to Cloud Service for monitoring. - overwrite: Bool, default `False`. If `False`, reloads an existing project - of the same name if one is found. Otherwise, overwrites the project. + directory: A string, the relative path to the working directory. + project_name: A string, the name to use as prefix for files saved by + this Tuner. + logger: Optional instance of `Logger` class, used for streaming data to + Cloud Service for monitoring. + overwrite: Boolean, defaults to `False`. If `False`, reloads an + existing project of the same name if one is found. Otherwise, + overwrites the project. + + Attributes: + remaining_trials: Number of trials remaining, `None` if `max_trials` is + not set. This is useful when resuming a previously stopped search. """ def __init__( @@ -114,10 +121,10 @@ def _populate_initial_space(self): def search(self, *fit_args, **fit_kwargs): """Performs a search for best hyperparameter configuations. - Arguments: + Args: *fit_args: Positional arguments that should be passed to `run_trial`, for example the training and validation data. - *fit_kwargs: Keyword arguments that should be passed to + **fit_kwargs: Keyword arguments that should be passed to `run_trial`, for example the training and validation data. """ if "verbose" in fit_kwargs: @@ -141,12 +148,11 @@ def search(self, *fit_args, **fit_kwargs): def run_trial(self, trial, *fit_args, **fit_kwargs): """Evaluates a set of hyperparameter values. - This method is called during `search` to evaluate a set of - hyperparameters. + This method is called multiple times during `search` to build and + evaluate the models with different hyperparameters. - For subclass implementers: This method is responsible for - reporting metrics related to the `Trial` to the `Oracle` - via `self.oracle.update_trial`. + The method is responsible for reporting metrics related to the `Trial` + to the `Oracle` via `self.oracle.update_trial`. Example: @@ -160,46 +166,44 @@ def run_trial(self, trial, x, y, val_x, val_y): self.save_model(trial.trial_id, model) ``` - Arguments: - trial: A `Trial` instance that contains the information - needed to run this trial. Hyperparameters can be accessed - via `trial.hyperparameters`. + Args: + trial: A `Trial` instance that contains the information needed to + run this trial. Hyperparameters can be accessed via + `trial.hyperparameters`. *fit_args: Positional arguments passed by `search`. - *fit_kwargs: Keyword arguments passed by `search`. + **fit_kwargs: Keyword arguments passed by `search`. """ raise NotImplementedError def save_model(self, trial_id, model, step=0): """Saves a Model for a given trial. - Arguments: - trial_id: The ID of the `Trial` that corresponds to this Model. + Args: + trial_id: The ID of the `Trial` corresponding to this Model. model: The trained model. - step: For models that report intermediate results to the `Oracle`, - the step that this saved file should correspond to. For example, - for Keras models this is the number of epochs trained. + step: Integer, for models that report intermediate results to the + `Oracle`, the step the saved file correspond to. For example, for + Keras models this is the number of epochs trained. """ raise NotImplementedError def load_model(self, trial): """Loads a Model from a given trial. - Arguments: - trial: A `Trial` instance. For models that report intermediate - results to the `Oracle`, generally `load_model` should load the - best reported `step` by relying of `trial.best_step` + For models that report intermediate results to the `Oracle`, generally + `load_model` should load the best reported `step` by relying of + `trial.best_step`. + + Args: + trial: A `Trial` instance, the `Trial` corresponding to the model + to load. """ raise NotImplementedError - def on_search_begin(self): - """A hook called at the beginning of `search`.""" - if self.logger: - self.logger.register_tuner(self.get_state()) - def on_trial_begin(self, trial): - """A hook called before starting each trial. + """Called at the beginning of a trial. - # Arguments: + Args: trial: A `Trial` instance. """ if self.logger: @@ -207,9 +211,9 @@ def on_trial_begin(self, trial): self._display.on_trial_begin(self.oracle.get_trial(trial.trial_id)) def on_trial_end(self, trial): - """A hook called after each trial is run. + """Called at the end of a trial. - Arguments: + Args: trial: A `Trial` instance. """ # Send status to Logger @@ -222,24 +226,30 @@ def on_trial_end(self, trial): self._display.on_trial_end(self.oracle.get_trial(trial.trial_id)) self.save() + def on_search_begin(self): + """Called at the beginning of the `search` method.""" + if self.logger: + self.logger.register_tuner(self.get_state()) + def on_search_end(self): - """A hook called at the end of `search`.""" + """Called at the end of the `search` method.""" if self.logger: self.logger.exit() def get_best_models(self, num_models=1): """Returns the best model(s), as determined by the objective. - This method is only a convenience shortcut. For best performance, It is - recommended to retrain your Model on the full dataset using the best - hyperparameters found during `search`. + This method is for querying the the models trained during the search. + For best performance, it is recommended to retrain your Model on the + full dataset using the best hyperparameters found during `search`, + which can be obtained using `tuner.get_best_hyperparameters()`. - Arguments: - num_models (int, optional). Number of best models to return. - Models will be returned in sorted order. Defaults to 1. + Args: + num_models: Optional number of best models to return. + Defaults to 1. Returns: - List of trained model instances. + List of trained models sorted from the best to the worst. """ best_trials = self.oracle.get_best_trials(num_models) models = [self.load_model(trial) for trial in best_trials] @@ -258,21 +268,22 @@ def get_best_hyperparameters(self, num_trials=1): model = tuner.hypermodel.build(best_hp) ``` - Arguments: - num_trials: (int, optional). Number of `HyperParameters` objects to - return. `HyperParameters` will be returned in sorted order based on - trial performance. + Args: + num_trials: Optional number of `HyperParameters` objects to return. Returns: - List of `HyperParameter` objects. + List of `HyperParameter` objects sorted from the best to the worst. """ return [t.hyperparameters for t in self.oracle.get_best_trials(num_trials)] def search_space_summary(self, extended=False): """Print search space summary. - Arguments: - extended: Bool, optional. Display extended summary. + The methods prints a summary of the hyperparameters in the search + space, which can be called before calling the `search` method. + + Args: + extended: Optional boolean, whether to display an extended summary. Defaults to False. """ print("Search space summary") @@ -287,9 +298,11 @@ def search_space_summary(self, extended=False): def results_summary(self, num_trials=10): """Display tuning results summary. - Arguments: - num_trials (int, optional): Number of trials to display. - Defaults to 10. + The method prints a summary of the search results including the + hyperparameter values and evaluation results for each trial. + + Args: + num_trials: Optional number of trials to display. Defaults to 10. """ print("Results summary") print("Results in %s" % self.project_dir) @@ -304,7 +317,8 @@ def results_summary(self, num_trials=10): def remaining_trials(self): """Returns the number of trials remaining. - Will return `None` if `max_trials` is not set. + Will return `None` if `max_trials` is not set. This is useful when + resuming a previously stopped search. """ return self.oracle.remaining_trials() diff --git a/keras_tuner/engine/conditions.py b/keras_tuner/engine/conditions.py index 1cc6310db..9732e818c 100644 --- a/keras_tuner/engine/conditions.py +++ b/keras_tuner/engine/conditions.py @@ -28,18 +28,9 @@ class Condition(object): """Abstract condition for a conditional hyperparameter. - Subclasses of this object can be passed to a `HyperParameter` to - specify that this condition must be met in order for that hyperparameter - to be considered active for the `Trial`. - - Example: - - ``` - - a = Choice('model', ['linear', 'dnn']) - condition = kt.conditions.Parent(name='a', value=['dnn']) - b = Int('num_layers', 5, 10, conditions=[condition]) - ``` + Subclasses of this object can be passed to a `HyperParameter` to specify + that this condition must be met for the hyperparameter to be active for the + `Trial`. """ @abc.abstractmethod @@ -48,12 +39,12 @@ def is_active(self, values): Determines whether this condition is true for the current `Trial`. - # Arguments: + Args: values: Dict. The active values for this `Trial`. Keys are the - names of the hyperparameters. + names of the hyperparameters. - # Returns: - bool. + Returns: + A boolean value of whether the condition is true. """ raise NotImplementedError("Must be implemented in subclasses.") @@ -82,23 +73,25 @@ def from_proto(self, proto): class Parent(Condition): - """Condition that checks a value is equal to one of a list of values. + """Condition checking a `HyperParameter`'s value is in a list of values. - This object can be passed to a `HyperParameter` to specify that this - condition must be met in order for that hyperparameter to be considered - active for the `Trial`. + It specifies a condition that a `HyperParameter`'s value is in a list of + values. It can be used as the condition to activate another + `HyperParameter` for the `Trial`. Example: - ``` + ```python a = Choice('model', ['linear', 'dnn']) - b = Int('num_layers', 5, 10, conditions=[kt.conditions.Parent('a', ['dnn'])]) + condition = Parent(name='model', value=['dnn']) + b = Int('num_layers', 5, 10, conditions=[condition]) ``` - # Arguments: - name: The name of a `HyperParameter`. - values: Values for which the `HyperParameter` this object is - passed to should be considered active. + Args: + name: A string, the name of the `HyperParameter` to use in the + condition. + values: A list of values of the `HyperParameter` to activate the + condition. """ def __init__(self, name, values): diff --git a/keras_tuner/engine/hypermodel.py b/keras_tuner/engine/hypermodel.py index d24c6975e..62e5cbab2 100644 --- a/keras_tuner/engine/hypermodel.py +++ b/keras_tuner/engine/hypermodel.py @@ -28,14 +28,34 @@ class HyperModel(object): - """Defines a searchable space of Models and builds Models from this space. - - Arguments: - name: The name of this HyperModel. - tunable: Whether the hyperparameters defined in this hypermodel - should be added to search space. If `False`, either the search - space for these parameters must be defined in advance, or the - default values will be used. + """Defines a search space of models. + + A search space is a collection of models. The `build` function will build + one of the models from the space using the given `HyperParameters` object. + + Users should subclass the `HyperModel` class to define their search spaces + by overriding the `.build(...)` function. + + Examples: + + ```python + class MyHyperModel(kt.HyperModel): + def build(self, hp): + model = keras.Sequential() + model.add(keras.layers.Dense( + hp.Choice('units', [8, 16, 32]), + activation='relu')) + model.add(keras.layers.Dense(1, activation='relu')) + model.compile(loss='mse') + return model + ``` + + Args: + name: Optional string, the name of this HyperModel. + tunable: Boolean, whether the hyperparameters defined in this + hypermodel should be added to search space. If `False`, either the + search space for these parameters must be defined in advance, or + the default values will be used. Defaults to True. """ def __init__(self, name=None, tunable=True): @@ -48,7 +68,7 @@ def __init__(self, name=None, tunable=True): def build(self, hp): """Builds a model. - Arguments: + Args: hp: A `HyperParameters` instance. Returns: @@ -65,6 +85,8 @@ def _build_wrapper(self, hp, *args, **kwargs): class DefaultHyperModel(HyperModel): + """Produces HyperModel from a model building function.""" + def __init__(self, build, name=None, tunable=True): super(DefaultHyperModel, self).__init__(name=name) self.build = build diff --git a/keras_tuner/engine/hyperparameters.py b/keras_tuner/engine/hyperparameters.py index 8645308f9..431028fb8 100644 --- a/keras_tuner/engine/hyperparameters.py +++ b/keras_tuner/engine/hyperparameters.py @@ -68,14 +68,14 @@ def _check_int(val, arg): class HyperParameter(object): - """HyperParameter base class. - - Arguments: - name: Str. Name of parameter. Must be unique. - default: Default value to return for the - parameter. - conditions: A list of `Condition`s for this object to be - considered active. + """Hyperparameter base class. + + Args: + name: A string. the name of parameter. Must be unique for each + `HyperParameter` instance in the search space. + default: The default value to return for the parameter. + conditions: A list of `Condition`s for this object to be considered + active. """ def __init__(self, name, default=None, conditions=None): @@ -104,14 +104,15 @@ def from_config(cls, config): class Choice(HyperParameter): """Choice of one value among a predefined set of possible values. - Arguments: - name: Str. Name of parameter. Must be unique. - values: List of possible values. Values must be int, float, + Args: + name: A string. the name of parameter. Must be unique for each + `HyperParameter` instance in the search space. + values: A list of possible values. Values must be int, float, str, or bool. All values must be of the same type. - ordered: Whether the values passed should be considered to - have an ordering. This defaults to `True` for float/int - values. Must be `False` for any other values. - default: Default value to return for the parameter. + ordered: Optional boolean, whether the values passed should be + considered to have an ordering. Defaults to `True` for float/int + values. Must be `False` for any other values. + default: Optional default value to return for the parameter. If unspecified, the default value will be: - None if None is one of the choices in `values` - The first entry in `values` otherwise. @@ -225,19 +226,19 @@ class Int(HyperParameter): Note that unlike Python's `range` function, `max_value` is *included* in the possible values this parameter can take on. - Arguments: - name: Str. Name of parameter. Must be unique. - min_value: Int. Lower limit of range (included). - max_value: Int. Upper limit of range (included). - step: Int. Step of range. - sampling: Optional. One of "linear", "log", - "reverse_log". Acts as a hint for an initial prior - probability distribution for how this value should - be sampled, e.g. "log" will assign equal + Args: + name: A string. the name of parameter. Must be unique for each + `HyperParameter` instance in the search space. + min_value: Integer, the lower limit of range, inclusive. + max_value: Integer, the upper limit of range, inclusive. + step: Integer, the distance between two consecutive samples in the + range. Defaults to 1. + sampling: Optional string. One of "linear", "log", "reverse_log". Acts + as a hint for an initial prior probability distribution for how + this value should be sampled, e.g. "log" will assign equal probabilities to each order of magnitude range. - default: Default value to return for the parameter. - If unspecified, the default value will be - `min_value`. + default: Integer, default value to return for the parameter. If + unspecified, the default value will be `min_value`. """ def __init__( @@ -321,22 +322,21 @@ def to_proto(self): class Float(HyperParameter): """Floating point range, can be evenly divided. - Arguments: - name: Str. Name of parameter. Must be unique. - min_value: Float. Lower bound of the range. - max_value: Float. Upper bound of the range. - step: Optional. Float, e.g. 0.1. - smallest meaningful distance between two values. - Whether step should be specified is Oracle dependent, - since some Oracles can infer an optimal step automatically. - sampling: Optional. One of "linear", "log", - "reverse_log". Acts as a hint for an initial prior - probability distribution for how this value should - be sampled, e.g. "log" will assign equal + Args: + name: A string. the name of parameter. Must be unique for each + `HyperParameter` instance in the search space. + min_value: Float, the lower bound of the range. + max_value: Float, the upper bound of the range. + step: Optional float, e.g. 0.1, the smallest meaningful distance + between two values. Whether step should be specified is Oracle + dependent, since some Oracles can infer an optimal step + automatically. + sampling: Optional string. One of "linear", "log", "reverse_log". Acts + as a hint for an initial prior probability distribution for how + this value should be sampled, e.g. "log" will assign equal probabilities to each order of magnitude range. - default: Default value to return for the parameter. - If unspecified, the default value will be - `min_value`. + default: Float, the default value to return for the parameter. If + unspecified, the default value will be `min_value`. """ def __init__( @@ -422,9 +422,10 @@ def to_proto(self): class Boolean(HyperParameter): """Choice between True and False. - Arguments: - name: Str. Name of parameter. Must be unique. - default: Default value to return for the parameter. + Args: + name: A string. the name of parameter. Must be unique for each + `HyperParameter` instance in the search space. + default: Boolean, the default value to return for the parameter. If unspecified, the default value will be False. """ @@ -461,10 +462,10 @@ def to_proto(self): class Fixed(HyperParameter): """Fixed, untunable value. - Arguments: - name: Str. Name of parameter. Must be unique. - value: Value to use (can be any JSON-serializable - Python type). + Args: + name: A string. the name of parameter. Must be unique for each + `HyperParameter` instance in the search space. + value: The value to use (can be any JSON-serializable Python type). """ def __init__(self, name, value, **kwargs): @@ -529,8 +530,10 @@ def to_proto(self): class HyperParameters(object): """Container for both a hyperparameter space, and current values. - Arguments: - space: A list of HyperParameter instances. + A `HyperParameters` instance can be pass to `HyperModel.build(hp)` as an + argument to build a model. + + Attributes: values: A dict mapping hyperparameter names to current values. """ @@ -566,21 +569,50 @@ def name_scope(self, name): def conditional_scope(self, parent_name, parent_values): """Opens a scope to create conditional HyperParameters. - All HyperParameters created under this scope will only be active - when the parent HyperParameter specified by `parent_name` is - equal to one of the values passed in `parent_values`. - - When the condition is not met, creating a HyperParameter under - this scope will register the HyperParameter, but will return - `None` rather than a concrete value. - - Note that any Python code under this scope will execute - regardless of whether the condition is met. - - Arguments: - parent_name: The name of the HyperParameter to condition on. - parent_values: Values of the parent HyperParameter for which - HyperParameters under this scope should be considered active. + All `HyperParameter`s created under this scope will only be active when + the parent `HyperParameter` specified by `parent_name` is equal to one + of the values passed in `parent_values`. + + When the condition is not met, creating a `HyperParameter` under this + scope will register the `HyperParameter`, but will return `None` rather + than a concrete value. + + Note that any Python code under this scope will execute regardless of + whether the condition is met. + + This feature is for the `Tuner` to collect more information of the + search space and the current trial. It is especially useful for model + selection. If the parent `HyperParameter` is for model selection, the + `HyperParameter`s in a model should only be active when the model + selected, which can be implemented using `conditional_scope`. + + Examples: + + ```python + def MyHyperModel(HyperModel): + def build(self, hp): + model = Sequential() + model.add(Input(shape=(32, 32, 3))) + model_type = hp.Choice("model_type", ["mlp", "cnn"]) + if model_type == "mlp": + with hp.conditional_scope("model_type", ["mlp"]): + model.add(Flatten()) + model.add(Dense(32, activation='relu')) + if model_type == "cnn": + with hp.conditional_scope("model_type", ["cnn"]): + model.add(Conv2D(64, 3, activation='relu')) + model.add(GlobalAveragePooling2D()) + model.add(Dense(10, activation='softmax')) + return model + ``` + + Args: + parent_name: A string, specifying the name of the parent + `HyperParameter` to use as the condition to activate the + current `HyperParameter`. + parent_values: A list of the values of the parent `HyperParameter` + to use as the condition to activate the current + `HyperParameter`. """ parent_name = self._get_name(parent_name) # Add name_scopes. if not self._exists(parent_name): @@ -597,10 +629,12 @@ def conditional_scope(self, parent_name, parent_values): def is_active(self, hyperparameter): """Checks if a hyperparameter is currently active for a `Trial`. - # Arguments: - hp: Str or `HyperParameter`. If str, checks if any - `HyperParameter` with that name is active. If `HyperParameter`, - checks that this object is active. + Args: + hp: A string or `HyperParameter` instance. If string, checks if any + `HyperParameter` with that name is active. If `HyperParameter`, + checks that this object is active. + Returns: + A boolean, whether the hyperparameter is active. """ hp = hyperparameter if isinstance(hp, six.string_types): @@ -684,21 +718,24 @@ def Choice( ): """Choice of one value among a predefined set of possible values. - Arguments: - name: Str. Name of parameter. Must be unique. - values: List of possible values. Values must be int, float, + Args: + name: A string. the name of parameter. Must be unique for each + `HyperParameter` instance in the search space. + values: A list of possible values. Values must be int, float, str, or bool. All values must be of the same type. - ordered: Whether the values passed should be considered to - have an ordering. This defaults to `True` for float/int - values. Must be `False` for any other values. - default: Default value to return for the parameter. + ordered: Optional boolean, whether the values passed should be + considered to have an ordering. Defaults to `True` for float/int + values. Must be `False` for any other values. + default: Optional default value to return for the parameter. If unspecified, the default value will be: - None if None is one of the choices in `values` - The first entry in `values` otherwise. - parent_name: (Optional) String. Specifies that this hyperparameter is - conditional. The name of the this hyperparameter's parent. - parent_values: (Optional) List. The values of the parent hyperparameter - for which this hyperparameter should be considered active. + parent_name: Optional string, specifying the name of the parent + `HyperParameter` to use as the condition to activate the + current `HyperParameter`. + parent_values: Optional list of the values of the parent + `HyperParameter` to use as the condition to activate the + current `HyperParameter`. Returns: The current value of this hyperparameter. @@ -729,23 +766,25 @@ def Int( Note that unlike Python's `range` function, `max_value` is *included* in the possible values this parameter can take on. - Arguments: - name: Str. Name of parameter. Must be unique. - min_value: Int. Lower limit of range (included). - max_value: Int. Upper limit of range (included). - step: Int. Step of range. - sampling: Optional. One of "linear", "log", - "reverse_log". Acts as a hint for an initial prior - probability distribution for how this value should - be sampled, e.g. "log" will assign equal + Args: + name: A string. the name of parameter. Must be unique for each + `HyperParameter` instance in the search space. + min_value: Integer, the lower limit of range, inclusive. + max_value: Integer, the upper limit of range, inclusive. + step: Integer, the distance between two consecutive samples in the + range. Defaults to 1. + sampling: Optional string. One of "linear", "log", "reverse_log". Acts + as a hint for an initial prior probability distribution for how + this value should be sampled, e.g. "log" will assign equal probabilities to each order of magnitude range. - default: Default value to return for the parameter. - If unspecified, the default value will be - `min_value`. - parent_name: (Optional) String. Specifies that this hyperparameter is - conditional. The name of the this hyperparameter's parent. - parent_values: (Optional) List. The values of the parent hyperparameter - for which this hyperparameter should be considered active. + default: Integer, default value to return for the parameter. If + unspecified, the default value will be `min_value`. + parent_name: Optional string, specifying the name of the parent + `HyperParameter` to use as the condition to activate the + current `HyperParameter`. + parent_values: Optional list of the values of the parent + `HyperParameter` to use as the condition to activate the + current `HyperParameter`. Returns: The current value of this hyperparameter. @@ -775,26 +814,27 @@ def Float( ): """Floating point range, can be evenly divided. - Arguments: - name: Str. Name of parameter. Must be unique. - min_value: Float. Lower bound of the range. - max_value: Float. Upper bound of the range. - step: Optional. Float, e.g. 0.1. - smallest meaningful distance between two values. - Whether step should be specified is Oracle dependent, - since some Oracles can infer an optimal step automatically. - sampling: Optional. One of "linear", "log", - "reverse_log". Acts as a hint for an initial prior - probability distribution for how this value should - be sampled, e.g. "log" will assign equal + Args: + name: A string. the name of parameter. Must be unique for each + `HyperParameter` instance in the search space. + min_value: Float, the lower bound of the range. + max_value: Float, the upper bound of the range. + step: Optional float, e.g. 0.1, the smallest meaningful distance + between two values. Whether step should be specified is Oracle + dependent, since some Oracles can infer an optimal step + automatically. + sampling: Optional string. One of "linear", "log", "reverse_log". Acts + as a hint for an initial prior probability distribution for how + this value should be sampled, e.g. "log" will assign equal probabilities to each order of magnitude range. - default: Default value to return for the parameter. - If unspecified, the default value will be - `min_value`. - parent_name: (Optional) String. Specifies that this hyperparameter is - conditional. The name of the this hyperparameter's parent. - parent_values: (Optional) List. The values of the parent hyperparameter - for which this hyperparameter should be considered active. + default: Float, the default value to return for the parameter. If + unspecified, the default value will be `min_value`. + parent_name: Optional string, specifying the name of the parent + `HyperParameter` to use as the condition to activate the + current `HyperParameter`. + parent_values: Optional list of the values of the parent + `HyperParameter` to use as the condition to activate the + current `HyperParameter`. Returns: The current value of this hyperparameter. @@ -814,14 +854,17 @@ def Float( def Boolean(self, name, default=False, parent_name=None, parent_values=None): """Choice between True and False. - Arguments: - name: Str. Name of parameter. Must be unique. - default: Default value to return for the parameter. + Args: + name: A string. the name of parameter. Must be unique for each + `HyperParameter` instance in the search space. + default: Boolean, the default value to return for the parameter. If unspecified, the default value will be False. - parent_name: (Optional) String. Specifies that this hyperparameter is - conditional. The name of the this hyperparameter's parent. - parent_values: (Optional) List. The values of the parent hyperparameter - for which this hyperparameter should be considered active. + parent_name: Optional string, specifying the name of the parent + `HyperParameter` to use as the condition to activate the + current `HyperParameter`. + parent_values: Optional list of the values of the parent + `HyperParameter` to use as the condition to activate the + current `HyperParameter`. Returns: The current value of this hyperparameter. @@ -837,14 +880,16 @@ def Boolean(self, name, default=False, parent_name=None, parent_values=None): def Fixed(self, name, value, parent_name=None, parent_values=None): """Fixed, untunable value. - Arguments: - name: Str. Name of parameter. Must be unique. - value: Value to use (can be any JSON-serializable - Python type). - parent_name: (Optional) String. Specifies that this hyperparameter is - conditional. The name of the this hyperparameter's parent. - parent_values: (Optional) List. The values of the parent hyperparameter - for which this hyperparameter should be considered active. + Args: + name: A string. the name of parameter. Must be unique for each + `HyperParameter` instance in the search space. + value: The value to use (can be any JSON-serializable Python type). + parent_name: Optional string, specifying the name of the parent + `HyperParameter` to use as the condition to activate the + current `HyperParameter`. + parent_values: Optional list of the values of the parent + `HyperParameter` to use as the condition to activate the + current `HyperParameter`. Returns: The current value of this hyperparameter. @@ -886,12 +931,11 @@ def copy(self): def merge(self, hps, overwrite=True): """Merges hyperparameters into this object. - Arguments: - hps: A `HyperParameters` object or list of `HyperParameter` - objects. - overwrite: bool. Whether existing `HyperParameter`s should - be overridden by those in `hps` with the same name and - conditions. + Args: + hps: A `HyperParameters` object or list of `HyperParameter` + objects. + overwrite: bool. Whether existing `HyperParameter`s should be + overridden by those in `hps` with the same name and conditions. """ if isinstance(hps, HyperParameters): hps = hps.space @@ -1070,7 +1114,7 @@ def cumulative_prob_to_value(prob, hp): return int(value) return value else: - raise ValueError("Unrecognized HyperParameter type: {}".format(hp)) + raise ValueError("Unrecognized `HyperParameter` type: {}".format(hp)) def value_to_cumulative_prob(value, hp): @@ -1102,7 +1146,7 @@ def value_to_cumulative_prob(value, hp): else: raise ValueError("Unrecognized sampling value: {}".format(sampling)) else: - raise ValueError("Unrecognized HyperParameter type: {}".format(hp)) + raise ValueError("Unrecognized `HyperParameter` type: {}".format(hp)) def _sampling_from_proto(sampling): diff --git a/keras_tuner/engine/multi_execution_tuner.py b/keras_tuner/engine/multi_execution_tuner.py index 8c93518ee..eec836e18 100644 --- a/keras_tuner/engine/multi_execution_tuner.py +++ b/keras_tuner/engine/multi_execution_tuner.py @@ -31,22 +31,18 @@ class MultiExecutionTuner(tuner_module.Tuner): - """A Tuner class that averages multiple runs of the process. + """A `Tuner` class that averages multiple runs of the process. Args: - oracle: Instance of Oracle class. - hypermodel: Instance of HyperModel class - (or callable that takes hyperparameters - and returns a Model instance). - executions_per_trial: Int. Number of executions - (training a model from scratch, - starting from a new initialization) - to run per trial (model configuration). - Model metrics may vary greatly depending - on random initialization, hence it is - often a good idea to run several executions - per trial in order to evaluate the performance - of a given set of hyperparameter values. + oracle: An Oracle instance. + hypermodel: A `HyperModel` instance (or callable that takes + hyperparameters and returns a `Model` instance). + executions_per_trial: Integer, the number of executions (training a + model from scratch, starting from a new initialization) to run per + trial (model configuration). Model metrics may vary greatly + depending on random initialization, hence it is often a good idea + to run several executions per trial in order to evaluate the + performance of a given set of hyperparameter values. **kwargs: Keyword arguments relevant to all `Tuner` subclasses. Please see the docstring for `Tuner`. """ diff --git a/keras_tuner/engine/oracle.py b/keras_tuner/engine/oracle.py index de6a2ed5b..6b1c18368 100644 --- a/keras_tuner/engine/oracle.py +++ b/keras_tuner/engine/oracle.py @@ -38,23 +38,24 @@ class Oracle(stateful.Stateful): """Implements a hyperparameter optimization algorithm. - Arguments: - objective: String. Name of model metric to minimize - or maximize, e.g. "val_accuracy". - max_trials: The maximum number of hyperparameter - combinations to try. - hyperparameters: HyperParameters class instance. - Can be used to override (or register in advance) - hyperparamters in the search space. - tune_new_entries: Whether hyperparameter entries - that are requested by the hypermodel - but that were not specified in `hyperparameters` - should be added to the search space, or not. - If not, then the default value for these parameters - will be used. - allow_new_entries: Whether the hypermodel is allowed - to request hyperparameter entries not listed in - `hyperparameters`. + Args: + objective: A string or `keras_tuner.Objective` instance. If a string, + the direction of the optimization (min or max) will be inferred. + max_trials: Integer, the total number of trials (model configurations) + to test at most. Note that the oracle may interrupt the search + before `max_trial` models have been tested if the search space has + been exhausted. + hyperparameters: Optional `HyperParameters` instance. Can be used to + override (or register in advance) hyperparameters in the search + space. + tune_new_entries: Boolean, whether hyperparameter entries that are + requested by the hypermodel but that were not specified in + `hyperparameters` should be added to the search space, or not. If + not, then the default value for these parameters will be used. + Defaults to True. + allow_new_entries: Boolean, whether the hypermodel is allowed to + request hyperparameter entries not listed in `hyperparameters`. + Defaults to True. seed: Int. Random seed. """ @@ -130,7 +131,7 @@ def populate_space(self, trial_id): values. Args: - `trial_id`: The id for this Trial. + trial_id: A string, the ID for this Trial. Returns: A dictionary with keys "values" and "status", where "values" is @@ -167,14 +168,14 @@ def create_trial(self, tuner_id): A `Trial` corresponds to a unique set of hyperparameters to be run by `Tuner.run_trial`. - Arguments: - tuner_id: A ID that identifies the `Tuner` requesting a - `Trial`. `Tuners` that should run the same trial (for instance, - when running a multi-worker model) should have the same ID. + Args: + tuner_id: A string, the ID that identifies the `Tuner` requesting a + `Trial`. `Tuners` that should run the same trial (for instance, + when running a multi-worker model) should have the same ID. Returns: - A `Trial` object containing a set of hyperparameter values to run - in a `Tuner`. + A `Trial` object containing a set of hyperparameter values to run + in a `Tuner`. """ # Allow for multi-worker DistributionStrategy within a Trial. if tuner_id in self.ongoing_trials: @@ -208,13 +209,12 @@ def create_trial(self, tuner_id): def update_trial(self, trial_id, metrics, step=0): """Used by a worker to report the status of a trial. - Arguments: - trial_id: A previously seen trial id. - metrics: Dict of float. The current value of this - trial's metrics. - step: (Optional) Float. Used to report intermediate results. The - current value in a timeseries representing the state of the - trial. This is the value that `metrics` will be associated with. + Args: + trial_id: A string, a previously seen trial id. + metrics: Dict of float. The current value of this trial's metrics. + step: Optional float, reporting intermediate results. The current + value in a timeseries representing the state of the trial. This + is the value that `metrics` will be associated with. Returns: Trial object. Trial.status will be set to "STOPPED" if the Trial @@ -236,10 +236,10 @@ def update_trial(self, trial_id, metrics, step=0): def end_trial(self, trial_id, status="COMPLETED"): """Record the measured objective for a set of parameter values. - Arguments: - trial_id: String. Unique id for this trial. - status: String, one of "COMPLETED", "INVALID". A status of - "INVALID" means a trial has crashed or been deemed + Args: + trial_id: A string, the unique ID for this trial. + status: A string, one of `"COMPLETED"`, `"INVALID"`. A status of + `"INVALID"` means a trial has crashed or been deemed infeasible. """ trial = None @@ -267,8 +267,8 @@ def update_space(self, hyperparameters): Already recorded parameters get ignored. - Arguments: - hyperparameters: An updated HyperParameters object. + Args: + hyperparameters: An updated `HyperParameters` object. """ hps = hyperparameters.space new_hps = [] diff --git a/keras_tuner/engine/stateful.py b/keras_tuner/engine/stateful.py index 555aa4dc3..710bbeb64 100644 --- a/keras_tuner/engine/stateful.py +++ b/keras_tuner/engine/stateful.py @@ -23,10 +23,15 @@ class Stateful(object): + """The base class for saving and restoring the state.""" + def get_state(self): """Returns the current state of this object. This method is called during `save`. + + Returns: + A dictionary of serializable objects as the state. """ raise NotImplementedError @@ -35,16 +40,16 @@ def set_state(self, state): This method is called during `reload`. - Arguments: - state: Dict. The state to restore for this object. + Args: + state: A dictionary of serialized objects as the state to restore. """ raise NotImplementedError def save(self, fname): """Saves this object using `get_state`. - Arguments: - fname: The file name to save to. + Args: + fname: A string, the file name to save to. """ state = self.get_state() state_json = json.dumps(state) @@ -55,8 +60,8 @@ def save(self, fname): def reload(self, fname): """Reloads this object using `set_state`. - Arguments: - fname: The file name to restore from. + Args: + fname: A string, the file name to restore from. """ with tf.io.gfile.GFile(fname, "r") as f: state_data = f.read() diff --git a/keras_tuner/engine/tuner.py b/keras_tuner/engine/tuner.py index 15ed0afe9..e52e92db5 100644 --- a/keras_tuner/engine/tuner.py +++ b/keras_tuner/engine/tuner.py @@ -31,46 +31,45 @@ class Tuner(base_tuner.BaseTuner): """Tuner class for Keras models. - May be subclassed to create new tuners. + This is the base `Tuner` class for all tuners for Keras models. It manages + the building, training, evaluation and saving of the Keras models. New + tuners can be created by subclassing the class. - Arguments: + Args: oracle: Instance of Oracle class. - hypermodel: Instance of HyperModel class - (or callable that takes hyperparameters - and returns a Model instance). - max_model_size: Int. Maximum number of scalars - in the parameters of a model. Models larger - than this are rejected. - optimizer: Optional. Optimizer instance. - May be used to override the `optimizer` - argument in the `compile` step for the - models. If the hypermodel - does not compile the models it generates, - then this argument must be specified. - loss: Optional. May be used to override the `loss` - argument in the `compile` step for the - models. If the hypermodel - does not compile the models it generates, - then this argument must be specified. - metrics: Optional. May be used to override the - `metrics` argument in the `compile` step - for the models. If the hypermodel - does not compile the models it generates, - then this argument must be specified. - distribution_strategy: Optional. A TensorFlow - `tf.distribute` DistributionStrategy instance. If - specified, each trial will run under this scope. For - example, `tf.distribute.MirroredStrategy(['/gpu:0', '/gpu:1'])` - will run each trial on two GPUs. Currently only - single-worker strategies are supported. - directory: String. Path to the working directory (relative). - project_name: Name to use as prefix for files saved - by this Tuner. - logger: Optional. Instance of Logger class, used for streaming data - to Cloud Service for monitoring. - tuner_id: Optional. If set, use this value as the id of this Tuner. - overwrite: Bool, default `False`. If `False`, reloads an existing project - of the same name if one is found. Otherwise, overwrites the project. + hypermodel: Instance of HyperModel class (or callable that takes + hyperparameters and returns a Model instance). + max_model_size: Integer, maximum number of scalars in the parameters of + a model. Models larger than this are rejected. + optimizer: Optional `Optimizer` instance. May be used to override the + `optimizer` argument in the `compile` step for the models. If the + hypermodel does not compile the models it generates, then this + argument must be specified. + loss: Optional loss. May be used to override the `loss` argument in the + `compile` step for the models. If the hypermodel does not compile + the models it generates, then this argument must be specified. + metrics: Optional metrics. May be used to override the `metrics` + argument in the `compile` step for the models. If the hypermodel + does not compile the models it generates, then this argument must + be specified. + distribution_strategy: Optional instance of `tf.distribute.Strategy`. + If specified, each trial will run under this scope. For example, + `tf.distribute.MirroredStrategy(['/gpu:0', '/gpu:1'])` will run + each trial on two GPUs. Currently only single-worker strategies are + supported. + directory: A string, the relative path to the working directory. + project_name: A string, the name to use as prefix for files saved by + this `Tuner`. + logger: Optional instance of `Logger` class, used for streaming data to + Cloud Service for monitoring. + tuner_id: Optional string, used as the ID of this `Tuner`. + overwrite: Boolean, defaults to `False`. If `False`, reloads an + existing project of the same name if one is found. Otherwise, + overwrites the project. + + Attributes: + remaining_trials: Number of trials remaining, `None` if `max_trials` is + not set. This is useful when resuming a previously stopped search. """ def __init__( @@ -134,10 +133,10 @@ def _build_and_fit_model(self, trial, fit_args, fit_kwargs): the input shape before building the model, adapt preprocessing layers, and tune other fit_args and fit_kwargs. - Arguments: - trial: A `Trial` instance that contains the information - needed to run this trial. `Hyperparameters` can be accessed - via `trial.hyperparameters`. + Args: + trial: A `Trial` instance that contains the information needed to + run this trial. `Hyperparameters` can be accessed via + `trial.hyperparameters`. fit_args: Positional arguments passed by `search`. fit_kwargs: Keyword arguments passed by `search`. @@ -150,15 +149,18 @@ def _build_and_fit_model(self, trial, fit_args, fit_kwargs): def run_trial(self, trial, *fit_args, **fit_kwargs): """Evaluates a set of hyperparameter values. - This method is called during `search` to evaluate a set of - hyperparameters. + This method is called multiple times during `search` to build and + evaluate the models with different hyperparameters. - Arguments: - trial: A `Trial` instance that contains the information - needed to run this trial. `Hyperparameters` can be accessed - via `trial.hyperparameters`. + The method is responsible for reporting metrics related to the `Trial` + to the `Oracle` via `self.oracle.update_trial`. + + Args: + trial: A `Trial` instance that contains the information needed to + run this trial. `Hyperparameters` can be accessed via + `trial.hyperparameters`. *fit_args: Positional arguments passed by `search`. - *fit_kwargs: Keyword arguments passed by `search`. + **fit_kwargs: Keyword arguments passed by `search`. """ # Handle any callbacks passed to `fit`. copied_fit_kwargs = copy.copy(fit_kwargs) @@ -207,45 +209,43 @@ def load_model(self, trial): ) return model - def on_epoch_begin(self, trial, model, epoch, logs=None): - """A hook called at the start of every epoch. + def on_batch_begin(self, trial, model, batch, logs): + """Called at the beginning of a batch. - Arguments: + Args: trial: A `Trial` instance. model: A Keras `Model`. - epoch: The current epoch number. + batch: The current batch number within the curent epoch. logs: Additional metrics. """ pass - def on_batch_begin(self, trial, model, batch, logs): - """A hook called at the start of every batch. + def on_batch_end(self, trial, model, batch, logs=None): + """Called at the end of a batch. - Arguments: + Args: trial: A `Trial` instance. model: A Keras `Model`. - batch: The current batch number within the - curent epoch. + batch: The current batch number within the curent epoch. logs: Additional metrics. """ pass - def on_batch_end(self, trial, model, batch, logs=None): - """A hook called at the end of every batch. + def on_epoch_begin(self, trial, model, epoch, logs=None): + """Called at the beginning of an epoch. - Arguments: + Args: trial: A `Trial` instance. model: A Keras `Model`. - batch: The current batch number within the - curent epoch. + epoch: The current epoch number. logs: Additional metrics. """ pass def on_epoch_end(self, trial, model, epoch, logs=None): - """A hook called at the end of every epoch. + """Called at the end of an epoch. - Arguments: + Args: trial: A `Trial` instance. model: A Keras `Model`. epoch: The current epoch number. @@ -265,16 +265,17 @@ def get_best_models(self, num_models=1): The models are loaded with the weights corresponding to their best checkpoint (at the end of the best epoch of best trial). - This method is only a convenience shortcut. For best performance, It is - recommended to retrain your Model on the full dataset using the best - hyperparameters found during `search`. + This method is for querying the the models trained during the search. + For best performance, it is recommended to retrain your Model on the + full dataset using the best hyperparameters found during `search`, + which can be obtained using `tuner.get_best_hyperparameters()`. - Arguments: - num_models (int, optional): Number of best models to return. - Models will be returned in sorted order. Defaults to 1. + Args: + num_models: Optional number of best models to return. + Defaults to 1. Returns: - List of trained model instances. + List of trained model instances sorted from the best to the worst. """ # Method only exists in this class for the docstring override. return super(Tuner, self).get_best_models(num_models) diff --git a/keras_tuner/tuners/bayesian.py b/keras_tuner/tuners/bayesian.py index 6ee87a299..88d956a0b 100644 --- a/keras_tuner/tuners/bayesian.py +++ b/keras_tuner/tuners/bayesian.py @@ -43,11 +43,11 @@ def matern_kernel(x, y=None): class GaussianProcessRegressor(object): """A Gaussian process regressor. - Arguments: - alpha: Float. Value added to the diagonal of the kernel matrix - during fitting. It represents the expected amount of noise - in the observed performances in Bayesian optimization. - seed: Int. Random seed. + Args: + alpha: Float, the value added to the diagonal of the kernel matrix + during fitting. It represents the expected amount of noise in the + observed performances in Bayesian optimization. + seed: Optional int, the random seed. """ def __init__(self, alpha, seed=None): @@ -62,7 +62,7 @@ def __init__(self, alpha, seed=None): def fit(self, x, y): """Fit the Gaussian process regressor. - Arguments: + Args: x: np.ndarray with shape (samples, features). y: np.ndarray with shape (samples,). """ @@ -85,7 +85,7 @@ def fit(self, x, y): def predict(self, x): """Predict the mean and standard deviation of the target. - Arguments: + Args: x: np.ndarray with shape (samples, features). Returns: @@ -122,37 +122,34 @@ class BayesianOptimizationOracle(oracle_module.Oracle): be found [here]( https://www.cse.wustl.edu/~garnett/cse515t/spring_2015/files/lecture_notes/12.pdf). - Arguments: - objective: String or `keras_tuner.Objective`. If a string, - the direction of the optimization (min or max) will be - inferred. - max_trials: Int. Total number of trials - (model configurations) to test at most. - Note that the oracle may interrupt the search - before `max_trial` models have been tested if the search space has been - exhausted. - num_initial_points: (Optional) Int. The number of randomly generated samples - as initial training data for Bayesian optimization. If not specified, - a value of 3 times the dimensionality of the hyperparameter space is - used. - alpha: Float. Value added to the diagonal of the kernel matrix - during fitting. It represents the expected amount of noise - in the observed performances in Bayesian optimization. - beta: Float. The balancing factor of exploration and exploitation. - The larger it is, the more explorative it is. - seed: Int. Random seed. - hyperparameters: HyperParameters class instance. - Can be used to override (or register in advance) - hyperparameters in the search space. - tune_new_entries: Whether hyperparameter entries - that are requested by the hypermodel - but that were not specified in `hyperparameters` - should be added to the search space, or not. - If not, then the default value for these parameters - will be used. - allow_new_entries: Whether the hypermodel is allowed - to request hyperparameter entries not listed in - `hyperparameters`. + Args: + objective: A string or `keras_tuner.Objective` instance. If a string, + the direction of the optimization (min or max) will be inferred. + max_trials: Integer, the total number of trials (model configurations) + to test at most. Note that the oracle may interrupt the search + before `max_trial` models have been tested if the search space has + been exhausted. + num_initial_points: Optional number of randomly generated samples as + initial training data for Bayesian optimization. If left + unspecified, a value of 3 times the dimensionality of the + hyperparameter space is used. + alpha: Float, the value added to the diagonal of the kernel matrix + during fitting. It represents the expected amount of noise in the + observed performances in Bayesian optimization. Defaults to 1e-4. + beta: Float, the balancing factor of exploration and exploitation. The + larger it is, the more explorative it is. Defaults to 2.6. + seed: Optional integer, the random seed. + hyperparameters: Optional `HyperParameters` instance. Can be used to + override (or register in advance) hyperparameters in the search + space. + tune_new_entries: Boolean, whether hyperparameter entries that are + requested by the hypermodel but that were not specified in + `hyperparameters` should be added to the search space, or not. If + not, then the default value for these parameters will be used. + Defaults to True. + allow_new_entries: Boolean, whether the hypermodel is allowed to + request hyperparameter entries not listed in `hyperparameters`. + Defaults to True. """ def __init__( @@ -345,38 +342,38 @@ def _get_hp_bounds(self): class BayesianOptimization(multi_execution_tuner.MultiExecutionTuner): """BayesianOptimization tuning with Gaussian process. - Arguments: - hypermodel: Instance of HyperModel class - (or callable that takes hyperparameters - and returns a Model instance). - objective: String. Name of model metric to minimize - or maximize, e.g. "val_accuracy". - max_trials: Int. Total number of trials - (model configurations) to test at most. - Note that the oracle may interrupt the search + Args: + hypermodel: A `HyperModel` instance (or callable that takes + hyperparameters and returns a Model instance). + objective: A string or `keras_tuner.Objective` instance. If a string, + the direction of the optimization (min or max) will be inferred. + max_trials: Integer, the total number of trials (model configurations) + to test at most. Note that the oracle may interrupt the search before `max_trial` models have been tested if the search space has been exhausted. - num_initial_points: Int. The number of randomly generated samples as initial - training data for Bayesian optimization. - alpha: Float or array-like. Value added to the diagonal of - the kernel matrix during fitting. - beta: Float. The balancing factor of exploration and exploitation. - The larger it is, the more explorative it is. - seed: Int. Random seed. - hyperparameters: HyperParameters class instance. - Can be used to override (or register in advance) - hyperparamters in the search space. - tune_new_entries: Whether hyperparameter entries - that are requested by the hypermodel - but that were not specified in `hyperparameters` - should be added to the search space, or not. - If not, then the default value for these parameters - will be used. - allow_new_entries: Whether the hypermodel is allowed - to request hyperparameter entries not listed in - `hyperparameters`. - **kwargs: Keyword arguments relevant to all `Tuner` subclasses. - Please see the docstring for `Tuner`. + num_initial_points: Optional number of randomly generated samples as + initial training data for Bayesian optimization. If left + unspecified, a value of 3 times the dimensionality of the + hyperparameter space is used. + alpha: Float, the value added to the diagonal of the kernel matrix + during fitting. It represents the expected amount of noise in the + observed performances in Bayesian optimization. Defaults to 1e-4. + beta: Float, the balancing factor of exploration and exploitation. The + larger it is, the more explorative it is. Defaults to 2.6. + seed: Optional integer, the random seed. + hyperparameters: Optional `HyperParameters` instance. Can be used to + override (or register in advance) hyperparameters in the search + space. + tune_new_entries: Boolean, whether hyperparameter entries that are + requested by the hypermodel but that were not specified in + `hyperparameters` should be added to the search space, or not. If + not, then the default value for these parameters will be used. + Defaults to True. + allow_new_entries: Boolean, whether the hypermodel is allowed to + request hyperparameter entries not listed in `hyperparameters`. + Defaults to True. + **kwargs: Keyword arguments relevant to all `Tuner` subclasses. Please + see the docstring for `Tuner`. """ def __init__( diff --git a/keras_tuner/tuners/hyperband.py b/keras_tuner/tuners/hyperband.py index 518691d93..77f57ee08 100644 --- a/keras_tuner/tuners/hyperband.py +++ b/keras_tuner/tuners/hyperband.py @@ -35,7 +35,7 @@ class must be able to handle in `Tuner.run_trial` three special hyperparameters These hyperparameters will be set during the "successive halving" portion of the Hyperband algorithm. - Example: + Examples: ```python def run_trial(self, trial, *args, **kwargs): @@ -56,34 +56,34 @@ def run_trial(self, trial, *args, **kwargs): self.on_epoch_end(...) ``` - Arguments: - objective: String or `keras_tuner.Objective`. If a string, - the direction of the optimization (min or max) will be - inferred. - max_epochs: Int. The maximum number of epochs to train one model. It is - recommended to set this to a value slightly higher than the expected epochs - to convergence for your largest Model, and to use early stopping during - training (for example, via `tf.keras.callbacks.EarlyStopping`). - factor: Int. Reduction factor for the number of epochs - and number of models for each bracket. - hyperband_iterations: Int >= 1. The number of times to iterate over the full - Hyperband algorithm. One iteration will run approximately - `max_epochs * (math.log(max_epochs, factor) ** 2)` cumulative epochs - across all trials. It is recommended to set this to as high a value - as is within your resource budget. - seed: Int. Random seed. - hyperparameters: HyperParameters class instance. - Can be used to override (or register in advance) - hyperparameters in the search space. - tune_new_entries: Whether hyperparameter entries - that are requested by the hypermodel - but that were not specified in `hyperparameters` - should be added to the search space, or not. - If not, then the default value for these parameters - will be used. - allow_new_entries: Whether the hypermodel is allowed - to request hyperparameter entries not listed in - `hyperparameters`. + Args: + objective: A string or `keras_tuner.Objective` instance. If a string, + the direction of the optimization (min or max) will be inferred. + max_epochs: Integer, the maximum number of epochs to train one model. + It is recommended to set this to a value slightly higher than the + expected epochs to convergence for your largest Model, and to use + early stopping during training (for example, via + `tf.keras.callbacks.EarlyStopping`). + factor: Integer, the reduction factor for the number of epochs + and number of models for each bracket. Defaults to 3. + hyperband_iterations: Integer, at least 1, the number of times to + iterate over the full Hyperband algorithm. One iteration will run + approximately `max_epochs * (math.log(max_epochs, factor) ** 2)` + cumulative epochs across all trials. It is recommended to set this + to as high a value as is within your resource budget. Defaults to + 1. + seed: Optional integer, the random seed. + hyperparameters: Optional HyperParameters instance. Can be used to + override (or register in advance) hyperparameters in the search + space. + tune_new_entries: Boolean, whether hyperparameter entries that are + requested by the hypermodel but that were not specified in + `hyperparameters` should be added to the search space, or not. If + not, then the default value for these parameters will be used. + Defaults to True. + allow_new_entries: Boolean, whether the hypermodel is allowed to + request hyperparameter entries not listed in `hyperparameters`. + Defaults to True. """ def __init__( @@ -301,36 +301,36 @@ class Hyperband(multi_execution_tuner.MultiExecutionTuner): http://jmlr.org/papers/v18/16-558.html). - # Arguments - hypermodel: Instance of HyperModel class - (or callable that takes hyperparameters - and returns a Model instance). - objective: String. Name of model metric to minimize - or maximize, e.g. "val_accuracy". - max_epochs: Int. The maximum number of epochs to train one model. It is - recommended to set this to a value slightly higher than the expected time - to convergence for your largest Model, and to use early stopping during - training (for example, via `tf.keras.callbacks.EarlyStopping`). - factor: Int. Reduction factor for the number of epochs - and number of models for each bracket. - hyperband_iterations: Int >= 1. The number of times to iterate over the full - Hyperband algorithm. One iteration will run approximately - `max_epochs * (math.log(max_epochs, factor) ** 2)` cumulative epochs - across all trials. It is recommended to set this to as high a value - as is within your resource budget. - seed: Int. Random seed. - hyperparameters: HyperParameters class instance. - Can be used to override (or register in advance) - hyperparamters in the search space. - tune_new_entries: Whether hyperparameter entries - that are requested by the hypermodel - but that were not specified in `hyperparameters` - should be added to the search space, or not. - If not, then the default value for these parameters - will be used. - allow_new_entries: Whether the hypermodel is allowed - to request hyperparameter entries not listed in - `hyperparameters`. + Args: + hypermodel: A `HyperModel` instance (or callable that takes + hyperparameters and returns a Model instance). + objective: A string or `keras_tuner.Objective` instance. If a string, + the direction of the optimization (min or max) will be inferred. + max_epochs: Integer, the maximum number of epochs to train one model. + It is recommended to set this to a value slightly higher than the + expected epochs to convergence for your largest Model, and to use + early stopping during training (for example, via + `tf.keras.callbacks.EarlyStopping`). + factor: Integer, the reduction factor for the number of epochs + and number of models for each bracket. Defaults to 3. + hyperband_iterations: Integer, at least 1, the number of times to + iterate over the full Hyperband algorithm. One iteration will run + approximately `max_epochs * (math.log(max_epochs, factor) ** 2)` + cumulative epochs across all trials. It is recommended to set this + to as high a value as is within your resource budget. Defaults to + 1. + seed: Optional integer, the random seed. + hyperparameters: Optional HyperParameters instance. Can be used to + override (or register in advance) hyperparameters in the search + space. + tune_new_entries: Boolean, whether hyperparameter entries that are + requested by the hypermodel but that were not specified in + `hyperparameters` should be added to the search space, or not. If + not, then the default value for these parameters will be used. + Defaults to True. + allow_new_entries: Boolean, whether the hypermodel is allowed to + request hyperparameter entries not listed in `hyperparameters`. + Defaults to True. **kwargs: Keyword arguments relevant to all `Tuner` subclasses. Please see the docstring for `Tuner`. """ diff --git a/keras_tuner/tuners/randomsearch.py b/keras_tuner/tuners/randomsearch.py index 010db7d69..65f9a825c 100644 --- a/keras_tuner/tuners/randomsearch.py +++ b/keras_tuner/tuners/randomsearch.py @@ -26,27 +26,25 @@ class RandomSearchOracle(oracle_module.Oracle): """Random search oracle. - Arguments: - objective: String or `keras_tuner.Objective`. If a string, - the direction of the optimization (min or max) will be - inferred. - max_trials: Int. Total number of trials - (model configurations) to test at most. - Note that the oracle may interrupt the search - before `max_trial` models have been tested. - seed: Int. Random seed. - hyperparameters: HyperParameters class instance. - Can be used to override (or register in advance) - hyperparameters in the search space. - tune_new_entries: Whether hyperparameter entries - that are requested by the hypermodel - but that were not specified in `hyperparameters` - should be added to the search space, or not. - If not, then the default value for these parameters - will be used. - allow_new_entries: Whether the hypermodel is allowed - to request hyperparameter entries not listed in - `hyperparameters`. + Args: + objective: A string or `keras_tuner.Objective` instance. If a string, + the direction of the optimization (min or max) will be inferred. + max_trials: Integer, the total number of trials (model configurations) + to test at most. Note that the oracle may interrupt the search + before `max_trial` models have been tested if the search space has + been exhausted. + seed: Optional integer, the random seed. + hyperparameters: Optional `HyperParameters` instance. Can be used to + override (or register in advance) hyperparameters in the search + space. + tune_new_entries: Boolean, whether hyperparameter entries that are + requested by the hypermodel but that were not specified in + `hyperparameters` should be added to the search space, or not. If + not, then the default value for these parameters will be used. + Defaults to True. + allow_new_entries: Boolean, whether the hypermodel is allowed to + request hyperparameter entries not listed in `hyperparameters`. + Defaults to True. """ def __init__( @@ -67,11 +65,11 @@ def __init__( seed=seed, ) - def populate_space(self, _): + def populate_space(self, trial_id): """Fill the hyperparameter space with values. - Arguments: - `trial_id`: The id for this Trial. + Args: + trial_id: A string, the ID for this Trial. Returns: A dictionary with keys "values" and "status", where "values" is @@ -88,29 +86,27 @@ def populate_space(self, _): class RandomSearch(multi_execution_tuner.MultiExecutionTuner): """Random search tuner. - Arguments: - hypermodel: Instance of HyperModel class - (or callable that takes hyperparameters - and returns a Model instance). - objective: String. Name of model metric to minimize - or maximize, e.g. "val_accuracy". - max_trials: Int. Total number of trials - (model configurations) to test at most. - Note that the oracle may interrupt the search - before `max_trial` models have been tested. - seed: Int. Random seed. - hyperparameters: HyperParameters class instance. - Can be used to override (or register in advance) - hyperparamters in the search space. - tune_new_entries: Whether hyperparameter entries - that are requested by the hypermodel - but that were not specified in `hyperparameters` - should be added to the search space, or not. - If not, then the default value for these parameters - will be used. - allow_new_entries: Whether the hypermodel is allowed - to request hyperparameter entries not listed in - `hyperparameters`. + Args: + hypermodel: A `HyperModel` instance (or callable that takes + hyperparameters and returns a Model instance). + objective: A string or `keras_tuner.Objective` instance. If a string, + the direction of the optimization (min or max) will be inferred. + max_trials: Integer, the total number of trials (model configurations) + to test at most. Note that the oracle may interrupt the search + before `max_trial` models have been tested if the search space has + been exhausted. + seed: Optional integer, the random seed. + hyperparameters: Optional `HyperParameters` instance. Can be used to + override (or register in advance) hyperparameters in the search + space. + tune_new_entries: Boolean, whether hyperparameter entries that are + requested by the hypermodel but that were not specified in + `hyperparameters` should be added to the search space, or not. If + not, then the default value for these parameters will be used. + Defaults to True. + allow_new_entries: Boolean, whether the hypermodel is allowed to + request hyperparameter entries not listed in `hyperparameters`. + Defaults to True. **kwargs: Keyword arguments relevant to all `Tuner` subclasses. Please see the docstring for `Tuner`. """ diff --git a/keras_tuner/tuners/sklearn_tuner.py b/keras_tuner/tuners/sklearn_tuner.py index 9cd81914b..4ece2f8cd 100644 --- a/keras_tuner/tuners/sklearn_tuner.py +++ b/keras_tuner/tuners/sklearn_tuner.py @@ -31,31 +31,9 @@ class SklearnTuner(base_tuner.BaseTuner): """Tuner for Scikit-learn Models. - Performs cross-validated hyperparameter search for Scikit-learn - models. - - Arguments: - oracle: An instance of the `keras_tuner.Oracle` class. Note that for - this `Tuner`, the `objective` for the `Oracle` should always be set - to `Objective('score', direction='max')`. Also, `Oracle`s that exploit - Neural-Network-specific training (e.g. `Hyperband`) should not be - used with this `Tuner`. - hypermodel: Instance of `HyperModel` class (or callable that takes a - `Hyperparameters` object and returns a Model instance). - scoring: An sklearn `scoring` function. For more information, see - `sklearn.metrics.make_scorer`. If not provided, the Model's default - scoring will be used via `model.score`. Note that if you are searching - across different Model families, the default scoring for these Models - will often be different. In this case you should supply `scoring` here - in order to make sure your Models are being scored on the same metric. - metrics: Additional `sklearn.metrics` functions to monitor during search. - Note that these metrics do not affect the search process. - cv: An `sklearn.model_selection` Splitter class. Used to - determine how samples are split up into groups for cross-validation. - **kwargs: Keyword arguments relevant to all `Tuner` subclasses. Please - see the docstring for `Tuner`. + Performs cross-validated hyperparameter search for Scikit-learn models. - Example: + Examples: ```python import keras_tuner as kt @@ -77,7 +55,7 @@ def build_model(hp): return model tuner = kt.tuners.SklearnTuner( - oracle=kt.oracles.BayesianOptimization( + oracle=kt.oracles.BayesianOptimizationOracle( objective=kt.Objective('score', 'max'), max_trials=10), hypermodel=build_model, @@ -94,6 +72,29 @@ def build_model(hp): best_model = tuner.get_best_models(num_models=1)[0] ``` + + Args: + oracle: A `keras_tuner.Oracle` instance. Note that for this `Tuner`, + the `objective` for the `Oracle` should always be set to + `Objective('score', direction='max')`. Also, `Oracle`s that exploit + Neural-Network-specific training (e.g. `Hyperband`) should not be + used with this `Tuner`. + hypermodel: A `HyperModel` instance (or callable that takes + hyperparameters and returns a Model instance). + scoring: An sklearn `scoring` function. For more information, see + `sklearn.metrics.make_scorer`. If not provided, the Model's default + scoring will be used via `model.score`. Note that if you are + searching across different Model families, the default scoring for + these Models will often be different. In this case you should + supply `scoring` here in order to make sure your Models are being + scored on the same metric. + metrics: Additional `sklearn.metrics` functions to monitor during search. + Note that these metrics do not affect the search process. + cv: An `sklearn.model_selection` Splitter class. Used to + determine how samples are split up into groups for + cross-validation. + **kwargs: Keyword arguments relevant to all `Tuner` subclasses. Please + see the docstring for `Tuner`. """ def __init__( @@ -121,14 +122,14 @@ def __init__( def search(self, X, y, sample_weight=None, groups=None): """Performs hyperparameter search. - Arguments: + Args: X: See docstring for `model.fit` for the `sklearn` Models being tuned. y: See docstring for `model.fit` for the `sklearn` Models being tuned. - sample_weight: (Optional). See docstring for `model.fit` for the - `sklearn` Models being tuned. - groups: (Optional). Required for `sklearn.model_selection` Splitter - classes that split based on group labels (For example, see - `sklearn.model_selection.GroupKFold`). + sample_weight: Optional. See docstring for `model.fit` for the + `sklearn` Models being tuned. + groups: Optional. Required for `sklearn.model_selection` Splitter + classes that split based on group labels (For example, see + `sklearn.model_selection.GroupKFold`). """ # Only overridden for the docstring. return super().search(X, y, sample_weight=sample_weight, groups=groups)