Skip to content

Commit

Permalink
Dev flags (#17)
Browse files Browse the repository at this point in the history 
* Save Flags - Gives us options to not save on deserialize

Adding flags that can be sent to the serializer, mainly 'DoNotSave'

this gives us the option not to save as part of the deserialization - this is mainly to allow for some optimisations where we can batch save a load of changes, - which will
trigger less cache rebuilds, and more importantly less ModelsBuilder rebuilds.

* Add BatchSave to the config (so we can turn it on at some point)

* Add batch save to config.

* Flag Messages - so we know what the wait is for.
  • Loading branch information
@KevinJump
KevinJump authored Apr 29, 2019
1 parent 4c8b9f1 commit 0cf410f
Show file tree
Hide file tree
Showing 33 changed files with 215 additions and 54 deletions.
2 changes: 1 addition & 1 deletion uSync8.BackOffice/App_Plugins/uSync8/dashboard.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ <h3 class="bold"><localize key="usync_name">uSync 8</localize>
<div class="text-center">
<h4 class="usync-action-message">{{vm.status.Message}}</h4>
<small>{{vm.update.Message}}</small>
<div class="progress" style="height: 3px;">
<div class="progress usync-not-animated" style="height: 3px;">
<div class="bar" role="progressbar" style="width: {{vm.calcPercentage(vm.update)}}%;" aria-valuenow="{{vm.calcPercentage(vm.update)}}" aria-valuemin="0" aria-valuemax="100"></div>
</div>
</div>
Expand Down
6 changes: 5 additions & 1 deletion uSync8.BackOffice/App_Plugins/uSync8/usync.css
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,4 +118,8 @@

.usync-root-folder input {
width: 100%;
}
}

.usync-not-animated .bar {
transition: none;
}
1 change: 1 addition & 0 deletions uSync8.BackOffice/Config/uSync8.config
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
<ExportAtStartup>False</ExportAtStartup>
<ExportOnSave>True</ExportOnSave>
<UseGuidFilenames>False</UseGuidFilenames>
<BatchSave>False</BatchSave>
<Handlers EnableMissing="true">
<Handler Alias="dataTypeHandler" Enabled="true" />
<Handler Alias="languageHandler" Enabled="true" />
Expand Down
12 changes: 11 additions & 1 deletion uSync8.BackOffice/Configuration/BackOfficeConfig.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@ public uSyncSettings LoadSettings()
settings.ExportAtStartup = node.Element("ExportAtStartup").ValueOrDefault(false);
settings.ExportOnSave = node.Element("ExportOnSave").ValueOrDefault(true);
settings.UseGuidNames = node.Element("UseGuidFilenames").ValueOrDefault(false);
settings.BatchSave = node.Element("BatchSave").ValueOrDefault(false);
settings.ReportDebug = node.Element("ReportDebug").ValueOrDefault(false);

var handlerConfig = node.Element("Handlers");

Expand Down Expand Up @@ -64,6 +66,8 @@ public uSyncSettings SaveSettings(uSyncSettings settings, bool fireReload = fals
node.CreateOrSetElement("ExportAtStartup", settings.ExportAtStartup);
node.CreateOrSetElement("ExportOnSave", settings.ExportOnSave);
node.CreateOrSetElement("UseGuidFilenames", settings.UseGuidNames);
node.CreateOrSetElement("BatchSave", settings.BatchSave);
node.CreateOrSetElement("ReportDebug", settings.ReportDebug);

if (settings.Handlers != null && settings.Handlers.Any())
{
Expand All @@ -84,6 +88,9 @@ public uSyncSettings SaveSettings(uSyncSettings settings, bool fireReload = fals
if (!handler.UseFlatStructure.IsOverridden)
handler.UseFlatStructure.SetDefaultValue(settings.UseFlatStructure);

if (!handler.BatchSave.IsOverridden)
handler.BatchSave.SetDefaultValue(settings.BatchSave);

var handlerNode = handlerConfig.Elements("Handler").FirstOrDefault(x => x.Attribute("Alias").Value == handler.Alias);
if (handlerNode == null)
{
Expand Down Expand Up @@ -128,7 +135,7 @@ public HandlerSettings LoadHandlerConfig(XElement node, uSyncSettings defaultSet
// we get them from the global setting.
settings.GuidNames = GetLocalValue(node.Attribute("GuidNames"), defaultSettings.UseGuidNames);
settings.UseFlatStructure = GetLocalValue(node.Attribute("UseFlatStructure"), defaultSettings.UseFlatStructure);

settings.BatchSave = GetLocalValue(node.Attribute("BatchSave"), defaultSettings.BatchSave);
settings.Actions = node.Attribute("Actions").ValueOrDefault("All").ToDelimitedList().ToArray();

var settingNode = node.Element("Settings");
Expand Down Expand Up @@ -174,6 +181,9 @@ public void SaveHandlerConfig(XElement node, HandlerSettings config, uSyncSettin
if (config.UseFlatStructure.IsOverridden)
node.SetAttributeValue("UseFlatStructure", config.UseFlatStructure);

if (config.BatchSave.IsOverridden)
node.SetAttributeValue("BatchSave", config.BatchSave);

if (config.Actions.Length > 0 && !(config.Actions.Length == 1 && config.Actions[0].InvariantEquals("all")))
node.SetAttributeValue("Actions", string.Join(",", config.Actions));

Expand Down
2 changes: 2 additions & 0 deletions uSync8.BackOffice/Configuration/uSyncHandlerSettings.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ public class HandlerSettings
public OverriddenValue<bool> UseFlatStructure { get; set; } = new OverriddenValue<bool>();
public OverriddenValue<bool> GuidNames { get; set; } = new OverriddenValue<bool>();

public OverriddenValue<bool> BatchSave { get; set; } = new OverriddenValue<bool>();

public Dictionary<string, string> Settings { get; set; } = new Dictionary<string, string>();

public HandlerSettings(string alias, bool enabled)
Expand Down
3 changes: 3 additions & 0 deletions uSync8.BackOffice/Configuration/uSyncSettings.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,13 +14,16 @@ public class uSyncSettings

public bool UseFlatStructure { get; set; } = true;
public bool UseGuidNames { get; set; } = false;
public bool BatchSave { get; set; } = false;

public bool ImportAtStartup { get; set; } = false;
public bool ExportAtStartup { get; set; } = false;
public bool ExportOnSave { get; set; } = true;

public bool EnableMissingHandlers { get; set; } = true;
public List<HandlerSettings> Handlers { get; set; } = new List<HandlerSettings>();

public bool ReportDebug { get; set; } = false;
}

}
16 changes: 15 additions & 1 deletion uSync8.BackOffice/Services/uSyncService.cs
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
Expand Down Expand Up @@ -46,6 +47,16 @@ public IEnumerable<uSyncAction> Report(string folder, SyncEventCallback callback

var summary = new SyncProgressSummary(configuredHandlers.Select(x => x.Handler), "Reporting", configuredHandlers.Count);

if (settings.ReportDebug)
{
// debug - full export into a dated folder.
summary.Message = "Debug: Creating Extract in Tracker folder";
callback?.Invoke(summary);
}

this.Export($"~/uSync/Tracker/{DateTime.Now.ToString("yyyyMMdd_HHmmss")}/", null, null);


foreach (var configuredHandler in configuredHandlers)
{
var handler = configuredHandler.Handler;
Expand Down Expand Up @@ -74,6 +85,8 @@ public IEnumerable<uSyncAction> Import(string folder, bool force, SyncEventCallb
{
lock (_importLock)
{
var sw = Stopwatch.StartNew();

try
{
uSync8BackOffice.eventsPaused = true;
Expand Down Expand Up @@ -137,7 +150,8 @@ public IEnumerable<uSyncAction> Import(string folder, bool force, SyncEventCallb
}
}

summary.UpdateHandler("Post Import", HandlerStatus.Complete, "Import Completed");
sw.Stop();
summary.UpdateHandler("Post Import", HandlerStatus.Complete, $"Import Completed ({sw.ElapsedMilliseconds}ms)");
callback?.Invoke(summary);

return actions;
Expand Down
2 changes: 1 addition & 1 deletion uSync8.BackOffice/SyncHandlers/Handlers/DataTypeHandler.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ public override IEnumerable<uSyncAction> ProcessPostImport(string folder, IEnume

foreach (var action in actions)
{
var attempt = Import(action.FileName, config);
var attempt = Import(action.FileName, config, SerializerFlags.None);
if (attempt.Success)
{
ImportSecondPass(action.FileName, attempt.Item, config, null);
Expand Down
18 changes: 14 additions & 4 deletions uSync8.BackOffice/SyncHandlers/SyncHandlerBase.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -144,9 +144,12 @@ public IEnumerable<uSyncAction> ImportAll(string folder, HandlerSettings config,
protected virtual IEnumerable<uSyncAction> ImportFolder(string folder, HandlerSettings config, Dictionary<string, TObject> updates, bool force, SyncUpdateCallback callback)
{
List<uSyncAction> actions = new List<uSyncAction>();

var files = syncFileService.GetFiles(folder, "*.config");

var flags = SerializerFlags.None;
if (force) flags |= SerializerFlags.Force;
if (config.BatchSave) flags |= SerializerFlags.DoNotSave;

int count = 0;
int total = files.Count();
foreach (string file in files)
Expand All @@ -155,7 +158,7 @@ protected virtual IEnumerable<uSyncAction> ImportFolder(string folder, HandlerSe

callback?.Invoke($"Importing {Path.GetFileName(file)}", count, total);

var attempt = Import(file, config, force);
var attempt = Import(file, config, flags);
if (attempt.Success && attempt.Item != null)
{
updates.Add(file, attempt.Item);
Expand All @@ -168,6 +171,13 @@ protected virtual IEnumerable<uSyncAction> ImportFolder(string folder, HandlerSe
actions.Add(action);
}

// bulk save ..
if (flags.HasFlag(SerializerFlags.DoNotSave) && updates.Any())
{
callback?.Invoke($"Saving {updates.Count()} changes", 1, 1);
serializer.Save(updates.Select(x => x.Value));
}

var folders = syncFileService.GetDirectories(folder);
foreach (var children in folders)
{
Expand All @@ -179,7 +189,7 @@ protected virtual IEnumerable<uSyncAction> ImportFolder(string folder, HandlerSe
return actions;
}

virtual public SyncAttempt<TObject> Import(string filePath, HandlerSettings config, bool force = false)
virtual public SyncAttempt<TObject> Import(string filePath, HandlerSettings config, SerializerFlags flags)
{
try
{
Expand All @@ -188,7 +198,7 @@ virtual public SyncAttempt<TObject> Import(string filePath, HandlerSettings conf
using (var stream = syncFileService.OpenRead(filePath))
{
var node = XElement.Load(stream);
var attempt = serializer.Deserialize(node, force, false);
var attempt = serializer.Deserialize(node, flags);
return attempt;
}
}
Expand Down
13 changes: 12 additions & 1 deletion uSync8.BackOffice/SyncHandlers/SyncHandlerLevelBase.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ protected SyncHandlerLevelBase(
/// <returns></returns>
protected override IEnumerable<uSyncAction> ImportFolder(string folder, HandlerSettings config, Dictionary<string, TObject> updates, bool force, SyncUpdateCallback callback)
{

// if not using flat, then directory structure is doing
// this for us.
if (config.UseFlatStructure == false)
Expand Down Expand Up @@ -90,14 +91,17 @@ protected override IEnumerable<uSyncAction> ImportFolder(string folder, HandlerS
}

// loaded - now process.
var flags = SerializerFlags.None;
if (force) flags |= SerializerFlags.Force;
if (config.BatchSave) flags |= SerializerFlags.DoNotSave;

var count = 0;
foreach (var node in nodes.OrderBy(x => x.Level))
{
count++;
callback?.Invoke($"{Path.GetFileName(node.File)}", count, nodes.Count);

var attempt = Import(node.File, config, force);
var attempt = Import(node.File, config, flags);
if (attempt.Success && attempt.Item != null)
{
updates.Add(node.File, attempt.Item);
Expand All @@ -106,6 +110,13 @@ protected override IEnumerable<uSyncAction> ImportFolder(string folder, HandlerS
actions.Add(uSyncActionHelper<TObject>.SetAction(attempt, node.File, IsTwoPass));
}

if (flags.HasFlag(SerializerFlags.DoNotSave) && updates.Any())
{
// bulk save - should be the fastest way to do this
callback?.Invoke($"Saving {updates.Count()} changes", 1, 1);
serializer.Save(updates.Select(x => x.Value));
}

var folders = syncFileService.GetDirectories(folder);
foreach (var children in folders)
{
Expand Down
7 changes: 6 additions & 1 deletion uSync8.ContentEdition/Serializers/ContentSerializer.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ protected override SyncAttempt<IContent> DeserializeCore(XElement node)

DeserializeBase(item, node);

contentService.Save(item);
// contentService.Save(item);

return SyncAttempt<IContent>.Succeed(
item.Name,
Expand Down Expand Up @@ -198,6 +198,11 @@ protected override IContent FindAtRoot(string alias)

#endregion

public override void Save(IEnumerable<IContent> items)
=> contentService.Save(items);

protected override void SaveItem(IContent item)
=> contentService.Save(item);

}
}
31 changes: 17 additions & 14 deletions uSync8.ContentEdition/Serializers/ContentSerializerBase.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ public abstract class ContentSerializerBase<TObject> : SyncTreeSerializerBase<TO

protected UmbracoObjectTypes umbracoObjectType;

public ContentSerializerBase(IEntityService entityService, ILogger logger, UmbracoObjectTypes umbracoObjectType)
public ContentSerializerBase(IEntityService entityService, ILogger logger, UmbracoObjectTypes umbracoObjectType)
: base(entityService, logger)
{
this.umbracoObjectType = umbracoObjectType;
Expand Down Expand Up @@ -58,7 +58,7 @@ protected virtual XElement SerializeInfo(TObject item)
info.Add(new XElement("Path", GetItemPath(item)));

var title = new XElement("NodeName", new XAttribute("Default", item.Name));
foreach(var culture in item.AvailableCultures)
foreach (var culture in item.AvailableCultures)
{
title.Add(new XElement("Name", item.GetCultureName(culture),
new XAttribute("Culture", culture)));
Expand All @@ -74,27 +74,29 @@ protected virtual XElement SerializeProperties(TObject item)
{
var node = new XElement("Properties");

foreach(var property in item.Properties.OrderBy(x => x.Alias))
foreach (var property in item.Properties.OrderBy(x => x.Alias))
{
var propertyNode = new XElement(property.Alias);

foreach(var value in property.Values)
// this can cause us false change readings
// but we need to preserve the values if they are blank
// because we have to be able to set them to blank on deserialization
foreach (var value in property.Values)
{
var valueNode = new XElement("Value");

if (!string.IsNullOrWhiteSpace(value.Culture)) {
if (!string.IsNullOrWhiteSpace(value.Culture))
{
valueNode.Add(new XAttribute("Culture", value.Culture ?? string.Empty));
}

if (!string.IsNullOrWhiteSpace(value.Segment)) {
valueNode.Add(new XAttribute("Segment", value.Segment ?? string.Empty));
if (!string.IsNullOrWhiteSpace(value.Segment))
{
valueNode.Add(new XAttribute("Segment", value.Segment ?? string.Empty));
}

valueNode.Value = value.EditedValue?.ToString() ?? string.Empty;

propertyNode.Add(valueNode);
}

node.Add(propertyNode);
}

Expand Down Expand Up @@ -147,7 +149,7 @@ protected Attempt<TObject> DeserializeName(TObject item, XElement node)
if (name != string.Empty)
item.Name = name;

foreach(var cultureNode in nameNode.Elements("Name"))
foreach (var cultureNode in nameNode.Elements("Name"))
{
var culture = cultureNode.Attribute("Culture").ValueOrDefault(string.Empty);
if (culture == string.Empty) continue;
Expand All @@ -168,7 +170,7 @@ protected Attempt<TObject> DeserializeProperties(TObject item, XElement node)
if (properties == null || !properties.HasElements)
return Attempt.Fail(item, new Exception("No Properties in the content node"));

foreach(var property in properties.Elements())
foreach (var property in properties.Elements())
{
var alias = property.Name.LocalName;
if (item.HasProperty(alias))
Expand Down Expand Up @@ -204,7 +206,7 @@ protected string GetImportValue(PropertyType propertyType, string value, string
}

public override bool IsValid(XElement node)
=> node != null
=> node != null
&& node.GetKey() != null
&& node.GetAlias() != null
&& node.Element("Info") != null;
Expand All @@ -229,7 +231,8 @@ protected override TObject FindOrCreate(XElement node)
// create
var parent = default(TObject);

if (parentKey != Guid.Empty) {
if (parentKey != Guid.Empty)
{
parent = FindItem(parentKey);
}

Expand Down
6 changes: 5 additions & 1 deletion uSync8.ContentEdition/Serializers/ContentTemplateSerializer.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ protected override SyncAttempt<IContent> DeserializeCore(XElement node)

DeserializeBase(item, node);

contentService.SaveBlueprint(item);
// contentService.SaveBlueprint(item);

return SyncAttempt<IContent>.Succeed(
item.Name,
Expand Down Expand Up @@ -108,5 +108,9 @@ protected override Attempt<string> DoSaveOrPublish(IContent item, XElement node)
contentService.SaveBlueprint(item);
return Attempt.Succeed<string>("blueprint saved");
}

protected override void SaveItem(IContent item)
=> contentService.SaveBlueprint(item);
}

}
Loading

0 comments on commit 0cf410f

Please sign in to comment.