Skip to content

Commit

Permalink
* Fixed an issue to support autostart with LLM generated tasks.
Browse files Browse the repository at this point in the history 
* Fixed an issue whereby complex steps with many substeps would overrun the task panel display borders.
  • Loading branch information
@danbohus
danbohus committed Mar 25, 2024
1 parent 6703aa7 commit 3b04faf
Show file tree
Hide file tree
Showing 11 changed files with 240 additions and 152 deletions.
104 changes: 4 additions & 100 deletions Applications/Sigma/Sigma/InteractionModels/Diamond/DiamondDialogStates.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,8 +71,8 @@ public override IEnumerable<DialogAction> GetNextDialogActions(IInputEvent input
{
if (speechRecognitionInputEvent.SpeechRecognitionResult.Contains("ready"))
{
yield return DialogAction.Execute(interactionModel.StartTask);
yield return DialogAction.Execute(interactionModel.ContinueWithSelectedStep);
var isKnownTask = interactionModel.TryGetKnownTask();
yield return DialogAction.Execute(interactionModel.BeginTask(isKnownTask));
}
else
{
Expand Down Expand Up @@ -129,104 +129,8 @@ public override IEnumerable<DialogAction> GetNextDialogActions(IInputEvent input
// Move to a glanceable position
yield return DialogAction.Execute(interactionModel.MoveToGlanceablePosition);

if (interactionModel.Configuration.TaskGenerationPolicy == TaskGenerationPolicy.FromLibraryOnly)
{
if (isKnownTask)
{
yield return DialogAction.Execute(interactionModel.StartTask);
yield return DialogAction.Execute(interactionModel.ContinueWithSelectedStep);
}
else
{
yield return DialogAction.Speak("I'm sorry but I don't know how to help with this task.");
yield return DialogAction.Speak("Is there anything else you'd like to do today?");
yield return DialogAction.ContinueWith<WhatAreWeDoing>(noSpeechSynthesis: true);
}
}
else if (interactionModel.Configuration.TaskGenerationPolicy == TaskGenerationPolicy.FromLibraryOrLLMGenerate)
{
if (isKnownTask)
{
yield return DialogAction.Execute(interactionModel.StartTask);
yield return DialogAction.Execute(interactionModel.ContinueWithSelectedStep);
}
else
{
// O/w if we are setup to ask context questions
if (interactionModel.Configuration.AskContextQuestionsBeforeGeneratingTask)
{
// Get the context questions
yield return DialogAction.Execute(interactionModel.GetContextQuestions);
if (interactionModel.InteractionState.ContextQuestions == null)
{
yield return DialogAction.Speak("I'm sorry but I don't think I can actually help with this task.");
yield return DialogAction.Speak("Is there anything else you'd like to do today?");
yield return DialogAction.ContinueWith<WhatAreWeDoing>(noSpeechSynthesis: true);
}
else
{
yield return DialogAction.Speak("First, a couple of quick questions.");
yield return DialogAction.ContinueWith<AskContextQuestions>();
}
}
else
{
// Generate the task
yield return DialogAction.Execute(interactionModel.GenerateTask);
if (interactionModel.InteractionState.Task == null)
{
yield return DialogAction.Speak("I'm sorry but I don't think I can actually help with this task.");
yield return DialogAction.Speak("Is there anything else you'd like to do today?");
yield return DialogAction.ContinueWith<WhatAreWeDoing>(noSpeechSynthesis: true);
}
else
{
yield return DialogAction.Execute(interactionModel.StartTask);
yield return DialogAction.Execute(interactionModel.ContinueWithSelectedStep);
}
}
}
}
else if (interactionModel.Configuration.TaskGenerationPolicy == TaskGenerationPolicy.AlwaysLLMGenerate)
{
// O/w if we are setup to ask context questions
if (interactionModel.Configuration.AskContextQuestionsBeforeGeneratingTask)
{
// Get the context questions
yield return DialogAction.Execute(interactionModel.GetContextQuestions);
if (interactionModel.InteractionState.ContextQuestions == null)
{
yield return DialogAction.Speak("I'm sorry but I don't think I can actually help with this task.");
yield return DialogAction.Speak("Is there anything else you'd like to do today?");
yield return DialogAction.ContinueWith<WhatAreWeDoing>(noSpeechSynthesis: true);
}
else
{
yield return DialogAction.Speak("First, a couple of quick questions.");
yield return DialogAction.ContinueWith<AskContextQuestions>();
}
}
else
{
// Generate the task
yield return DialogAction.Execute(interactionModel.GenerateTask);
if (interactionModel.InteractionState.Task == null)
{
yield return DialogAction.Speak("I'm sorry but I don't think I can actually help with this task.");
yield return DialogAction.Speak("Is there anything else you'd like to do today?");
yield return DialogAction.ContinueWith<WhatAreWeDoing>(noSpeechSynthesis: true);
}
else
{
yield return DialogAction.Execute(interactionModel.StartTask);
yield return DialogAction.Execute(interactionModel.ContinueWithSelectedStep);
}
}
}
else
{
throw new System.Exception("Unexpected GuidanceTaskGenerationPolicy.");
}
// And begin the task
yield return DialogAction.Execute(interactionModel.BeginTask(isKnownTask));
}
else if (interactionModel.InteractionState.TopLevelIntent == "list")
{
Expand Down
109 changes: 109 additions & 0 deletions Applications/Sigma/Sigma/InteractionModels/Diamond/DiamondInteractionModel.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ namespace Sigma.Diamond
{
using System;
using System.Collections.Generic;
using static Sigma.Diamond.DiamondDialogStates;

/// <summary>
/// Represents the interaction model for the Diamond version of the Sigma app.
Expand All @@ -17,6 +18,114 @@ public class DiamondInteractionModel : SigmaInteractionModel<
DiamondUserInterfaceState,
DiamondUserInterfaceCommands>
{
/// <summary>
/// Begins the task.
/// </summary>
/// <param name="isKnownTask">Indicates whether this is a known (library) task.</param>
/// <returns>The set of corresponding dialog actions.</returns>
/// <exception cref="Exception">An exception is thrown if the task generation policy is unknown.</exception>
public virtual IEnumerable<DialogAction> BeginTask(bool isKnownTask)
{
if (this.Configuration.TaskGenerationPolicy == TaskGenerationPolicy.FromLibraryOnly)
{
if (isKnownTask)
{
yield return DialogAction.Execute(this.StartTask);
yield return DialogAction.Execute(this.ContinueWithSelectedStep);
}
else
{
yield return DialogAction.Speak("I'm sorry but I don't know how to help with this task.");
yield return DialogAction.Speak("Is there anything else you'd like to do today?");
yield return DialogAction.ContinueWith<WhatAreWeDoing>(noSpeechSynthesis: true);
}
}
else if (this.Configuration.TaskGenerationPolicy == TaskGenerationPolicy.FromLibraryOrLLMGenerate)
{
if (isKnownTask)
{
yield return DialogAction.Execute(this.StartTask);
yield return DialogAction.Execute(this.ContinueWithSelectedStep);
}
else
{
// O/w if we are setup to ask context questions
if (this.Configuration.AskContextQuestionsBeforeGeneratingTask)
{
// Get the context questions
yield return DialogAction.Execute(this.GetContextQuestions);
if (this.InteractionState.ContextQuestions == null)
{
yield return DialogAction.Speak("I'm sorry but I don't think I can actually help with this task.");
yield return DialogAction.Speak("Is there anything else you'd like to do today?");
yield return DialogAction.ContinueWith<WhatAreWeDoing>(noSpeechSynthesis: true);
}
else
{
yield return DialogAction.Speak("First, a couple of quick questions.");
yield return DialogAction.ContinueWith<AskContextQuestions>();
}
}
else
{
// Generate the task
yield return DialogAction.Execute(this.GenerateTask);
if (this.InteractionState.Task == null)
{
yield return DialogAction.Speak("I'm sorry but I don't think I can actually help with this task.");
yield return DialogAction.Speak("Is there anything else you'd like to do today?");
yield return DialogAction.ContinueWith<WhatAreWeDoing>(noSpeechSynthesis: true);
}
else
{
yield return DialogAction.Execute(this.StartTask);
yield return DialogAction.Execute(this.ContinueWithSelectedStep);
}
}
}
}
else if (this.Configuration.TaskGenerationPolicy == TaskGenerationPolicy.AlwaysLLMGenerate)
{
// O/w if we are setup to ask context questions
if (this.Configuration.AskContextQuestionsBeforeGeneratingTask)
{
// Get the context questions
yield return DialogAction.Execute(this.GetContextQuestions);
if (this.InteractionState.ContextQuestions == null)
{
yield return DialogAction.Speak("I'm sorry but I don't think I can actually help with this task.");
yield return DialogAction.Speak("Is there anything else you'd like to do today?");
yield return DialogAction.ContinueWith<WhatAreWeDoing>(noSpeechSynthesis: true);
}
else
{
yield return DialogAction.Speak("First, a couple of quick questions.");
yield return DialogAction.ContinueWith<AskContextQuestions>();
}
}
else
{
// Generate the task
yield return DialogAction.Execute(this.GenerateTask);
if (this.InteractionState.Task == null)
{
yield return DialogAction.Speak("I'm sorry but I don't think I can actually help with this task.");
yield return DialogAction.Speak("Is there anything else you'd like to do today?");
yield return DialogAction.ContinueWith<WhatAreWeDoing>(noSpeechSynthesis: true);
}
else
{
yield return DialogAction.Execute(this.StartTask);
yield return DialogAction.Execute(this.ContinueWithSelectedStep);
}
}
}
else
{
throw new Exception("Unexpected TaskGenerationPolicy.");
}
}

/// <summary>
/// Abandons the task.
/// </summary>
Expand Down
1 change: 1 addition & 0 deletions Applications/Sigma/Sigma/SigmaInteractionModel.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -348,6 +348,7 @@ public virtual IEnumerable<DialogAction> GenerateTask()
this.InteractionState.Task = new TTask()
{
Name = this.InteractionState.TaskName,
Steps = new (),
};

this.InteractionState.Task.Steps.Add(new GatherStep("1", "Gather", "Objects", objects));
Expand Down
11 changes: 9 additions & 2 deletions Applications/Sigma/Sigma/Task/ComplexStep.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,12 @@ public ComplexStep(string label, string description)
public override string GetSpokenInstructions() => this.Description.TrimEnd('.');

/// <inheritdoc/>
public override StepPanel UpdateStepPanel(StepPanel stepPanel, TaskPanelUserInterfaceCommand taskPanelUserInterfaceCommand, TaskPanelUserInterfaceConfiguration taskPanelUserInterfaceConfiguration, string name)
public override StepPanel UpdateStepPanel(
StepPanel stepPanel,
TaskPanelUserInterfaceCommand taskPanelUserInterfaceCommand,
TaskPanelUserInterfaceConfiguration taskPanelUserInterfaceConfiguration,
float maxHeight,
string name)
{
// If it's a complex step panel, update
var showObjectsChecklist = new List<(string Name, bool Checked, bool Highlight)>();
Expand All @@ -78,12 +83,14 @@ public override StepPanel UpdateStepPanel(StepPanel stepPanel, TaskPanelUserInte
taskPanelUserInterfaceConfiguration.ComplexStepTaughtObjectTextStyle,
taskPanelUserInterfaceConfiguration.SelectionColor,
name);

complexStepPanel.Update(
this.Label,
this.GetDisplayInstructions(),
taskPanelUserInterfaceCommand.SelectedSubStepIndex,
showObjectsChecklist,
subSteps);
subSteps,
maxHeight);
return complexStepPanel;
}

Expand Down
7 changes: 6 additions & 1 deletion Applications/Sigma/Sigma/Task/DoStep.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,12 @@ public DoStep(string label, string description, TimeSpan timerDuration)
public override string GetDisplayInstructions() => this.Description.TrimEnd('.');

/// <inheritdoc/>
public override StepPanel UpdateStepPanel(StepPanel stepPanel, TaskPanelUserInterfaceCommand taskPanelUserInterfaceCommand, TaskPanelUserInterfaceConfiguration taskPanelUserInterfaceConfiguration, string name)
public override StepPanel UpdateStepPanel(
StepPanel stepPanel,
TaskPanelUserInterfaceCommand taskPanelUserInterfaceComman,
TaskPanelUserInterfaceConfiguration taskPanelUserInterfaceConfiguration,
float maxHeight,
string name)
{
var doStepPanel = stepPanel as DoStepPanel ?? new DoStepPanel(
taskPanelUserInterfaceConfiguration.Width,
Expand Down
7 changes: 6 additions & 1 deletion Applications/Sigma/Sigma/Task/GatherStep.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,12 @@ public GatherStep(string label, string verb, string noun, List<string> objects)
public override string GetDisplayInstructions() => $"{this.Noun}:";

/// <inheritdoc/>
public override StepPanel UpdateStepPanel(StepPanel stepPanel, TaskPanelUserInterfaceCommand taskPanelUserInterfaceCommand, TaskPanelUserInterfaceConfiguration taskPanelUserInterfaceConfiguration, string name)
public override StepPanel UpdateStepPanel(
StepPanel stepPanel,
TaskPanelUserInterfaceCommand taskPanelUserInterfaceCommand,
TaskPanelUserInterfaceConfiguration taskPanelUserInterfaceConfiguration,
float maxHeigth,
string name)
{
var gatherStepPanel = (stepPanel as GatherStepPanel) ?? new GatherStepPanel(
taskPanelUserInterfaceConfiguration.Width,
Expand Down
8 changes: 7 additions & 1 deletion Applications/Sigma/Sigma/Task/Step.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,9 +36,15 @@ public Step()
/// <param name="stepPanel">The step panel to update.</param>
/// <param name="taskPanelUserInterfaceCommand">The task panel user interface command.</param>
/// <param name="taskPanelUserInterfaceConfiguration">The task panel configuration options.</param>
/// <param name="maxHeight">The maximum height for the step panel user interface.</param>
/// <param name="name">The name for the user interface.</param>
/// <returns>The user interface for the step.</returns>
public virtual StepPanel UpdateStepPanel(StepPanel stepPanel, TaskPanelUserInterfaceCommand taskPanelUserInterfaceCommand, TaskPanelUserInterfaceConfiguration taskPanelUserInterfaceConfiguration, string name)
public virtual StepPanel UpdateStepPanel(
StepPanel stepPanel,
TaskPanelUserInterfaceCommand taskPanelUserInterfaceCommand,
TaskPanelUserInterfaceConfiguration taskPanelUserInterfaceConfiguration,
float maxHeight,
string name)
=> stepPanel;

/// <inheritdoc/>
Expand Down
Loading

0 comments on commit 3b04faf

Please sign in to comment.