Fixed a product bug by implementing response serialization that correctly adheres to the design specification using Response Payload switches.

* Introduced `PartialResponseMessageSerialisation to enable full serialization responses during development and testing.
* Added PartialResponseMessageSerialisation environment variable to relevant documentation.
* Relocated EntityAnalysisModelAdaptations to its proper location within the context model.
* Implemented a conditional serializer that dynamically builds JObject properties based on active Response Payload switches. When no switches are enabled, the array is excluded entirely from the response.
* Response payloads now default to partial serialization (PartialResponseMessageSerialisation defaults to true).
* Renamed methods in the BuildJsonResponses given new functionality.
This commit is contained in:
richard-churchman
2026-02-14 12:44:36 +02:00
parent 76ab3ac1c6
commit 02af6dc5f4
14 changed files with 286 additions and 19 deletions
+1 -7
View File
@@ -69,10 +69,4 @@ Content-Type: application/json
"TransactionTypeId": "1000",
"TransactionResultId": "1006",
"Test":"Test1"
}
<> 2025-06-13T101843.200.json
<> 2025-06-13T101839.200.json
<> 2025-06-13T101827.200.json
###
}
@@ -301,6 +301,9 @@ namespace Jube.DynamicEnvironment
},
{
"CaseCreationThreads", "4"
},
{
"PartialResponseMessageSerialisation", "True"
}
};
@@ -48,7 +48,7 @@ namespace Jube.Engine.EntityAnalysisModelInvoke.Context.Extensions
private static async Task IterateAndProcessAsync(Context context)
{
foreach (var (adaptationKey, modelAdaptation) in context.EntityAnalysisModel.Dependencies.EntityAnalysisModelAdaptations)
foreach (var (adaptationKey, modelAdaptation) in context.EntityAnalysisModel.Collections.EntityAnalysisModelAdaptations)
{
try
{
@@ -23,8 +23,27 @@ namespace Jube.Engine.EntityAnalysisModelInvoke.Context.Extensions
{
public static async Task WriteResponseJsonAndQueueAsynchronousResponseMessageAsync(this Context context, IModel rabbitMqChannel)
{
context.JsonResult = BuildJsonResponses.BuildJson(context.EntityAnalysisModelInstanceEntryPayload, context.EntityAnalysisModel.JsonSerializationHelper.ArchiveJsonSerializer);
if (context.Environment.AppSettings("PartialResponseMessageSerialisation").Equals("True",StringComparison.CurrentCultureIgnoreCase))
{
if (context.Log.IsInfoEnabled)
{
context.Log.Info(
$"HTTP Handler Entity: GUID payload {context.EntityAnalysisModelInstanceEntryPayload.EntityAnalysisModelInstanceEntryGuid} model id is {context.EntityAnalysisModel.Instance.Id} has partial serialised response environment variable.");
}
context.JsonResult = BuildJsonResponses.BuildPartialResponsePayloadJson(context);
}
else
{
if (context.Log.IsInfoEnabled)
{
context.Log.Info(
$"HTTP Handler Entity: GUID payload {context.EntityAnalysisModelInstanceEntryPayload.EntityAnalysisModelInstanceEntryGuid} model id is {context.EntityAnalysisModel.Instance.Id} has partial serialised response environment variable false and will serialise full response.");
}
context.JsonResult = BuildJsonResponses.BuildFullJson(context.EntityAnalysisModelInstanceEntryPayload, context.EntityAnalysisModel.JsonSerializationHelper.ArchiveJsonSerializer);
}
if (context.Environment.AppSettings("AMQP").Equals("True", StringComparison.OrdinalIgnoreCase))
{
PublishToAmqp(context, rabbitMqChannel);
@@ -112,7 +112,7 @@ namespace Jube.Engine.EntityAnalysisModelInvoke
EntityAnalysisModelInstanceEntryGuid = context.EntityAnalysisModelInstanceEntryPayload.EntityAnalysisModelInstanceEntryGuid
};
context.JsonResult = BuildJsonResponses.BuildJson(asyncInvocationCallbackToken, context.EntityAnalysisModel.JsonSerializationHelper.ArchiveJsonSerializer);
context.JsonResult = BuildJsonResponses.BuildFullJson(asyncInvocationCallbackToken, context.EntityAnalysisModel.JsonSerializationHelper.ArchiveJsonSerializer);
}
public static async Task InvokeAsync(Context.Context context)
@@ -39,7 +39,7 @@ namespace Jube.Engine.EntityAnalysisModelInvoke.Extraction.Helpers
Sanction = new PooledDictionary<string, double>(model.Collections.EntityAnalysisModelSanctions.Count),
AbstractionCalculation =
new PooledDictionary<string, double>(model.Collections.EntityAnalysisModelAbstractionCalculations.Count),
HttpAdaptation = new PooledDictionary<string, double>(model.Dependencies.EntityAnalysisModelAdaptations.Count),
HttpAdaptation = new PooledDictionary<string, double>(model.Collections.EntityAnalysisModelAdaptations.Count),
ExhaustiveAdaptation = new PooledDictionary<string, double>(model.Collections.ExhaustiveModels.Count),
InvokeTaskPerformance = new InvokeTaskPerformance
{
@@ -14,12 +14,15 @@
namespace Jube.Engine.EntityAnalysisModelInvoke.Models.Payload.EntityAnalysisModelInstanceEntry
{
using System.IO;
using System.Linq;
using AsyncInvocationCallbackToken;
using Context;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
public static class BuildJsonResponses
{
public static MemoryStream BuildJson(AsyncInvocationCallbackToken payload,
public static MemoryStream BuildFullJson(AsyncInvocationCallbackToken payload,
JsonSerializer serializer)
{
var stream = new MemoryStream();
@@ -33,7 +36,7 @@ namespace Jube.Engine.EntityAnalysisModelInvoke.Models.Payload.EntityAnalysisMod
return stream;
}
public static MemoryStream BuildJson(EntityAnalysisModelInstanceEntryPayload payload,
public static MemoryStream BuildFullJson(EntityAnalysisModelInstanceEntryPayload payload,
JsonSerializer serializer)
{
var stream = new MemoryStream();
@@ -46,5 +49,252 @@ namespace Jube.Engine.EntityAnalysisModelInvoke.Models.Payload.EntityAnalysisMod
stream.Seek(0, SeekOrigin.Begin);
return stream;
}
public static MemoryStream BuildPartialResponsePayloadJson(Context context)
{
var jObject = CreateJObject(context);
AddCreateCaseToJObject(context, jObject);
AddDictionaryToJObject(context, jObject);
AddTtlCounterToJObject(context, jObject);
AddSanctionToJObject(context, jObject);
AddAbstractionToJObject(context, jObject);
AddAbstractionCalculationToJObject(context, jObject);
AddHttpAdaptationToJObject(context, jObject);
AddExhaustiveAdaptationToJObject(context, jObject);
AddActivationToJObject(context, jObject);
AddTagToJObject(context, jObject);
var stream = new MemoryStream();
var streamWriter = new StreamWriter(stream);
var jsonWriter = new JsonTextWriter(streamWriter);
jObject.WriteTo(jsonWriter);
jsonWriter.Flush();
streamWriter.Flush();
stream.Seek(0, SeekOrigin.Begin);
return stream;
}
private static void AddTagToJObject(Context context, JObject jObject)
{
var kvpEntityAnalysisModelTags= context.EntityAnalysisModel.Collections.EntityAnalysisModelTags.Where(w => w.ResponsePayload).ToArray();
if (!kvpEntityAnalysisModelTags.Any())
{
return;
}
var kvpEntityAnalysisModelTagsArray = new JArray();
jObject.Add("Tag",kvpEntityAnalysisModelTagsArray);
foreach (var kvpEntityAnalysisModelTag in kvpEntityAnalysisModelTags)
{
var value = context.EntityAnalysisModelInstanceEntryPayload.Activation.FirstOrDefault(f => f.Key == kvpEntityAnalysisModelTag.Name);
if (value.Key != null)
{
kvpEntityAnalysisModelTagsArray.Add(JToken.FromObject(value));
}
}
}
private static void AddActivationToJObject(Context context, JObject jObject)
{
var kvpModelActivationRules = context.EntityAnalysisModel.Collections.ModelActivationRules.Where(w => w.ResponsePayload).ToArray();
if (!kvpModelActivationRules.Any())
{
return;
}
var kvpModelActivationRulesArray = new JArray();
jObject.Add("Activation",kvpModelActivationRulesArray);
foreach (var kvpModelActivationRule in kvpModelActivationRules)
{
var value = context.EntityAnalysisModelInstanceEntryPayload.Activation.FirstOrDefault(f => f.Key == kvpModelActivationRule.Name);
if (value.Key != null)
{
kvpModelActivationRulesArray.Add(JToken.FromObject(value));
}
}
}
private static void AddExhaustiveAdaptationToJObject(Context context, JObject jObject)
{
var kvpExhaustiveModels = context.EntityAnalysisModel.Collections.ExhaustiveModels.Where(w => w.ResponsePayload).ToArray();
if (!kvpExhaustiveModels.Any())
{
return;
}
var kvpExhaustiveModelsArray = new JArray();
jObject.Add("ExhaustiveAdaptation",kvpExhaustiveModelsArray);
foreach (var kvpExhaustiveModel in kvpExhaustiveModels)
{
var value = context.EntityAnalysisModelInstanceEntryPayload.ExhaustiveAdaptation.FirstOrDefault(f => f.Key == kvpExhaustiveModel.Name);
if (value.Key != null)
{
kvpExhaustiveModelsArray.Add(JToken.FromObject(value));
}
}
}
private static void AddHttpAdaptationToJObject(Context context, JObject jObject)
{
var kvpEntityAnalysisModelAdaptations = context.EntityAnalysisModel.Collections.EntityAnalysisModelAdaptations.Where(w => w.Value.ResponsePayload).ToArray();
if (!kvpEntityAnalysisModelAdaptations.Any())
{
return;
}
var kvpEntityAnalysisModelAdaptationsArray = new JArray();
jObject.Add("HttpAdaptation",kvpEntityAnalysisModelAdaptationsArray);
foreach (var kvpEntityAnalysisModelAdaptation in kvpEntityAnalysisModelAdaptations)
{
var value = context.EntityAnalysisModelInstanceEntryPayload.ExhaustiveAdaptation.FirstOrDefault(f => f.Key == kvpEntityAnalysisModelAdaptation.Value.Name);
if (value.Key != null)
{
kvpEntityAnalysisModelAdaptationsArray.Add(JToken.FromObject(value));
}
}
}
private static void AddAbstractionCalculationToJObject(Context context, JObject jObject)
{
var kvpEntityAnalysisModelAbstractionCalculations = context.EntityAnalysisModel.Collections.EntityAnalysisModelAbstractionCalculations.Where(w => w.ResponsePayload).ToArray();
if (!kvpEntityAnalysisModelAbstractionCalculations.Any())
{
return;
}
var kvpEntityAnalysisModelAbstractionCalculationsArray = new JArray();
jObject.Add("AbstractionCalculation",kvpEntityAnalysisModelAbstractionCalculationsArray);
foreach (var kvpEntityAnalysisModelAbstractionCalculation in kvpEntityAnalysisModelAbstractionCalculations)
{
var value = context.EntityAnalysisModelInstanceEntryPayload.AbstractionCalculation.FirstOrDefault(f => f.Key == kvpEntityAnalysisModelAbstractionCalculation.Name);
if (value.Key != null)
{
kvpEntityAnalysisModelAbstractionCalculationsArray.Add(JToken.FromObject(value));
}
}
}
private static void AddAbstractionToJObject(Context context, JObject jObject)
{
var kvpAbstractions = context.EntityAnalysisModel.Collections.ModelAbstractionRules.Where(w => w.ResponsePayload).ToArray();
if (!kvpAbstractions.Any())
{
return;
}
var kvpAbstractionsJArray = new JArray();
jObject.Add("Abstraction",kvpAbstractionsJArray);
foreach (var kvpAbstraction in kvpAbstractions)
{
var value = context.EntityAnalysisModelInstanceEntryPayload.Abstraction.FirstOrDefault(f => f.Key == kvpAbstraction.Name);
if (value.Key != null)
{
kvpAbstractionsJArray.Add(JToken.FromObject(value));
}
}
}
private static void AddSanctionToJObject(Context context, JObject jObject)
{
var kvpEntityAnalysisModelSanctions = context.EntityAnalysisModel.Collections.EntityAnalysisModelSanctions.Where(w => w.ResponsePayload).ToArray();
if (!kvpEntityAnalysisModelSanctions.Any())
{
return;
}
var kvpEntityAnalysisModelSanctionsArray = new JArray();
jObject.Add("Sanction",kvpEntityAnalysisModelSanctionsArray);
foreach (var kvpEntityAnalysisModelSanction in kvpEntityAnalysisModelSanctions)
{
var value = context.EntityAnalysisModelInstanceEntryPayload.Sanction.FirstOrDefault(f => f.Key == kvpEntityAnalysisModelSanction.Name);
if (value.Key != null)
{
kvpEntityAnalysisModelSanctionsArray.Add(JToken.FromObject(value));
}
}
}
private static void AddTtlCounterToJObject(Context context, JObject jObject)
{
var kvpModelTtlCounters = context.EntityAnalysisModel.Collections.ModelTtlCounters.Where(w => w.ResponsePayload).ToArray();
if (!kvpModelTtlCounters.Any())
{
return;
}
var kvpModelTtlCountersArray = new JArray();
jObject.Add("TtlCounter",kvpModelTtlCountersArray);
foreach (var kvpModelTtlCounter in kvpModelTtlCounters)
{
var value = context.EntityAnalysisModelInstanceEntryPayload.TtlCounter.FirstOrDefault(f => f.Key == kvpModelTtlCounter.Name);
if (value.Key != null)
{
kvpModelTtlCountersArray.Add(JToken.FromObject(value));
}
}
}
private static void AddDictionaryToJObject(Context context, JObject jObject)
{
var kvpDictionaries = context.EntityAnalysisModel.Dependencies.KvpDictionaries.Where(w => w.Value.ResponsePayload).ToArray();
if (kvpDictionaries.Length <= 0)
{
return;
}
var kvpDictionariesJArray = new JArray();
jObject.Add("Dictionary",kvpDictionariesJArray);
foreach (var kvpDictionary in kvpDictionaries)
{
var value = context.EntityAnalysisModelInstanceEntryPayload.Dictionary.FirstOrDefault(f => f.Key == kvpDictionary.Value.Name);
if (value.Key != null)
{
kvpDictionariesJArray.Add(JToken.FromObject(value));
}
}
}
private static void AddCreateCaseToJObject(Context context, JObject jObject)
{
if (context.EntityAnalysisModelInstanceEntryPayload.CreateCase != null)
{
jObject.Add("CreateCase", JToken.FromObject(context.EntityAnalysisModelInstanceEntryPayload.CreateCase));
}
}
private static JObject CreateJObject(Context context)
{
var jObject = new JObject
{
{
"CreatedDate", context.EntityAnalysisModelInstanceEntryPayload.CreatedDate
},
{
"EntityAnalysisModelInstanceEntryGuid", context.EntityAnalysisModelInstanceEntryPayload.EntityAnalysisModelInstanceEntryGuid
},
{
"EntityInstanceEntryId", context.EntityAnalysisModelInstanceEntryPayload.EntityInstanceEntryId
},
{
"ReferenceDate", context.EntityAnalysisModelInstanceEntryPayload.ReferenceDate
},
{
"ResponseElevation", JToken.FromObject(context.EntityAnalysisModelInstanceEntryPayload.ResponseElevation)
}
};
return jObject;
}
}
}
@@ -122,7 +122,7 @@ namespace Jube.Engine.EntityAnalysisModelManager.BackgroundTasks.TaskStarters.Ab
TtlCounter = new PooledDictionary<string, long>(entityAnalysisModel.Collections.ModelTtlCounters.Count),
Sanction = new PooledDictionary<string, double>(entityAnalysisModel.Collections.EntityAnalysisModelSanctions.Count),
AbstractionCalculation = new PooledDictionary<string, double>(entityAnalysisModel.Collections.EntityAnalysisModelAbstractionCalculations.Count),
HttpAdaptation = new PooledDictionary<string, double>(entityAnalysisModel.Dependencies.EntityAnalysisModelAdaptations.Count),
HttpAdaptation = new PooledDictionary<string, double>(entityAnalysisModel.Collections.EntityAnalysisModelAdaptations.Count),
ExhaustiveAdaptation = new PooledDictionary<string, double>(entityAnalysisModel.Collections.ExhaustiveModels.Count),
InvokeTaskPerformance = new InvokeTaskPerformance
{
@@ -38,7 +38,7 @@ namespace Jube.Engine.EntityAnalysisModelManager.BackgroundTasks.TaskStarters.Ar
{
try
{
var json = BuildJsonResponses.BuildJson(payload, jsonSerializer);
var json = BuildJsonResponses.BuildFullJson(payload, jsonSerializer);
var jsonString = Encoding.UTF8.GetString(json.ToArray());
if (payload.CreateCase != null)
@@ -202,7 +202,7 @@ namespace Jube.Engine.EntityAnalysisModelManager.EntityAnalysisModel.Context.Ext
$"Model {key} and Adaptations Model {key} has completed creating the adaptations into a shadow list of adaptations and it will now be allocated the fields in the order that they appeared in model training.");
}
value.Dependencies.EntityAnalysisModelAdaptations = shadowEntityAnalysisModelAdaptations;
value.Collections.EntityAnalysisModelAdaptations = shadowEntityAnalysisModelAdaptations;
if (context.Services.Log.IsDebugEnabled)
{
@@ -25,6 +25,7 @@ namespace Jube.Engine.EntityAnalysisModelManager.EntityAnalysisModel.Models
public List<EntityAnalysisModelSanction> EntityAnalysisModelSanctions { get; set; } = [];
public List<EntityAnalysisModelActivationRule> ModelActivationRules { get; set; } = [];
public List<EntityModelGatewayRule> ModelGatewayRules { get; set; } = [];
public Dictionary<int, EntityAnalysisModelHttpAdaptation> EntityAnalysisModelAdaptations { get; set; } = new Dictionary<int, EntityAnalysisModelHttpAdaptation>();
public List<ExhaustiveSearchInstance> ExhaustiveModels { get; set; } = [];
public List<EntityAnalysisModelRequestXPath> EntityAnalysisModelRequestXPaths { get; set; } = [];
public List<EntityAnalysisModelAbstractionCalculation> EntityAnalysisModelAbstractionCalculations { get; set; } = [];
@@ -21,7 +21,6 @@ namespace Jube.Engine.EntityAnalysisModelManager.EntityAnalysisModel.Models
public class Dependencies
{
public Dictionary<int, EntityAnalysisModelHttpAdaptation> EntityAnalysisModelAdaptations { get; set; } = new Dictionary<int, EntityAnalysisModelHttpAdaptation>();
public Dictionary<string, List<string>> EntityAnalysisModelLists { get; set; } = new Dictionary<string, List<string>>();
public Dictionary<int, EntityAnalysisModelDictionary> KvpDictionaries { get; set; } = new Dictionary<int, EntityAnalysisModelDictionary>();
public Dictionary<string, List<string>> EntityAnalysisModelSuppressionModels { get; set; }
+1 -1
View File
@@ -1427,7 +1427,7 @@ namespace Jube.Engine.Exhaustive
EntityAnalysisModelInstanceEntryGuid = Guid.NewGuid()
};
var sr = new StreamReader(BuildJsonResponses.BuildJson(row, jsonSerializationHelper.ArchiveJsonSerializer));
var sr = new StreamReader(BuildJsonResponses.BuildFullJson(row, jsonSerializationHelper.ArchiveJsonSerializer));
model.Json = await sr.ReadToEndAsync(token).ConfigureAwait(false);
await repository.InsertAsync(model, token).ConfigureAwait(false);
@@ -101,6 +101,7 @@ following hardcoded Environment Variables, this would be to say defaults, are av
| AMQPHeartbeatSeconds | 30 | The time in seconds to for the AMQP client to send heartbeats to the server to avoid the connection being closed. |
| CaseCreationThreads | 4 | The number of background thread responsible for the expensive case creation and upward classification processing. |
| MaxInvokeControllerRequestBytes | 4000 | The maximum number of bytes that can be JSON parsed for the invocation pipeline, beyond which an exception will be thrown. |
| PartialResponseMessageSerialisation | True | When true, only return elements in the response payload on the basis of ResponsePayload switch being true in varius model entities, otherwise return the same comprehensive payload as stored in the Archive table. Disabling PartialResponseMessageSerialisation is useful when developing configurations where testing needs total visibility of invocation context payload. |
Environment Variables passed to Jube at startup will override the hardcoded defaults above. Environment Variables are
not written to disk in any form, and their contents are available only in the Jube applications memory and operating