Skip to content

Handle scenario JSON with no point inputs and missing node/baseline data in IWFM output generation #16

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Gorfaal merged 1 commit into from
Sep 21, 2026
Merged

Handle scenario JSON with no point inputs and missing node/baseline data in IWFM output generation #16

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
116 changes: 73 additions & 43 deletions Olsson.GET.Engines/ModelInputOutputEngines/IWFMModelInputOutputEngine.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
using System.Globalization;
using CsvHelper.Configuration;
using CsvHelper;
using Microsoft.Extensions.Logging;

namespace Olsson.GET.Engines.ModelInputOutputEngines
{
Expand All @@ -23,6 +24,8 @@ public class IWFMModelInputOutputEngine : BaseInputOutputEngine, IModelInputOutp
public const string BudgetGroundwaterFileName = "Budget/Groundwater.bud";
public const string BaselineBudgetGroundwaterFileName = "Budget/Groundwater.Baseline.bud";

private static readonly ILogger Logger = Logging.GetLogger<IWFMModelInputOutputEngine>();

private Model Model { get; }

public IWFMModelInputOutputEngine(Model model)
Expand Down Expand Up @@ -58,56 +61,83 @@ private void CreateGroundwaterLevelPoints(Run run, IBlobFileAccessor fileAccesso
var userDataObject = JsonConvert.DeserializeObject<UserDataJson>(Encoding.UTF8.GetString(file));


var nodeLocations = modelFileAccessor.GetIWFMNodeLocations();

var nodePoints = nodeLocations.ToDictionary(x => x.Key, x => new Point(x.Value.Item2, x.Value.Item1));
// find the closest node to each of the input locations
userDataObject.UserDataPointInputs.ForEach(inputPoint =>
{
var inputPointGeometry = new Point(inputPoint.Lng, inputPoint.Lat);
var closestNode = nodePoints.Keys.Select(x => new
{ Node = x, Distance = nodePoints[x].Distance(inputPointGeometry) })
.OrderBy(x => x.Distance)
.First().Node;

inputPoint.ClosestNode = closestNode;
});

var parsedHeadAllOutputFile = ParseHeadAllOutputFile(modelFileAccessor);
var baselineHeadAllOutputFile = ParseHeadAllOutputFile(modelFileAccessor, true);

// get the NodeWaterLevelLayer.csv file
// this file maps which layer each node will use, instead of it always defaulting to the last layer
var nodeToLayerMapping = modelFileAccessor.GetNodeWaterLevelLayerMapping();
userDataObject.UserDataPointInputs.ForEach(inputPoint =>
// Some scenario JSON formats (e.g. sustainability-project / polygon uploads)
// do not contain any point inputs. Treat that as "nothing to do" for this
// sub-engine rather than crashing output generation for the whole run.
if (userDataObject?.UserDataPointInputs != null && userDataObject.UserDataPointInputs.Count > 0)
{
inputPoint.TimeSteps = new List<UserDataPointTimeStep>();
var nodeLocations = modelFileAccessor.GetIWFMNodeLocations();

foreach (var dateTime in parsedHeadAllOutputFile.Keys)
var nodePoints = nodeLocations.ToDictionary(x => x.Key, x => new Point(x.Value.Item2, x.Value.Item1));
// find the closest node to each of the input locations
userDataObject.UserDataPointInputs.ForEach(inputPoint =>
{
var runValue = parsedHeadAllOutputFile[dateTime][inputPoint.ClosestNode][nodeToLayerMapping[inputPoint.ClosestNode]];
double? baselineValue = null;
double? baselineValueDifference = null;
if (run.IsDifferential)
{
// get the difference between baseline and the run value for differential results
baselineValue = baselineHeadAllOutputFile[dateTime][inputPoint.ClosestNode][nodeToLayerMapping[inputPoint.ClosestNode]];
baselineValueDifference = baselineValue - runValue;
}
else
var inputPointGeometry = new Point(inputPoint.Lng, inputPoint.Lat);
var closestNode = nodePoints.Keys.Select(x => new
{ Node = x, Distance = nodePoints[x].Distance(inputPointGeometry) })
.OrderBy(x => x.Distance)
.First().Node;

inputPoint.ClosestNode = closestNode;
});

var parsedHeadAllOutputFile = ParseHeadAllOutputFile(modelFileAccessor);
var baselineHeadAllOutputFile = ParseHeadAllOutputFile(modelFileAccessor, true);

@Gorfaal Gorfaal Sep 21, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not addressing in this PR, but the mechanism is correct and worth recording.

GetIWFMHeadAllOutputFile resolves the file with Directory.EnumerateFiles(resultsPath).Single(...), which throws when nothing matches, so a non-differential run with no *HeadAll.Baseline.out in Results/ would throw here before any of the new guards run.

Two qualifications:

  • Pre-existing. The unconditional baseline load is line 77 on develop today; this PR only moved it inside the new if block, which if anything narrows the exposure — a run with no point inputs now skips the baseline parse entirely.
  • Not reachable from the Qanat integration. Qanat hardcodes IsDifferential = true on every run it submits, and GETEngine defaults data.IsDifferential ?? true (Functions.cs:278), so the baseline file is always expected on that path. This is only reachable for a non-differential run submitted from elsewhere.

Leaving it out because it can't be verified from here: this repo has no CI, and restore needs the private esadatatechnology/sitkatech feed, which I can't reach — so a change to a path with no test coverage and no way to exercise it locally isn't one to land on top of a hand-off PR.

The fix, for whoever picks it up:

var baselineHeadAllOutputFile = run.IsDifferential
    ? ParseHeadAllOutputFile(modelFileAccessor, true)
    : new Dictionary<DateTime, Dictionary<int, List<double>>>();

That also avoids parsing a large head-all file on non-differential runs.


// get the NodeWaterLevelLayer.csv file
// this file maps which layer each node will use, instead of it always defaulting to the last layer
var nodeToLayerMapping = modelFileAccessor.GetNodeWaterLevelLayerMapping();
userDataObject.UserDataPointInputs.ForEach(inputPoint =>
{
inputPoint.TimeSteps = new List<UserDataPointTimeStep>();

if (!nodeToLayerMapping.ContainsKey(inputPoint.ClosestNode))
{
runValue = parsedHeadAllOutputFile[dateTime][inputPoint.ClosestNode][nodeToLayerMapping[inputPoint.ClosestNode]];
Logger.LogWarning($"No water level layer mapping found for node {inputPoint.ClosestNode} (point '{inputPoint.Name}'); skipping.");
return;
}

inputPoint.TimeSteps.Add(new UserDataPointTimeStep()
var layer = nodeToLayerMapping[inputPoint.ClosestNode];

foreach (var dateTime in parsedHeadAllOutputFile.Keys)
{
DateTime = dateTime,
Value = runValue,
BaselineValue = baselineValue,
BaselineValueDifference = baselineValueDifference
});
}
});
if (!parsedHeadAllOutputFile[dateTime].ContainsKey(inputPoint.ClosestNode))
{
continue;
}

var runValue = parsedHeadAllOutputFile[dateTime][inputPoint.ClosestNode][layer];
double? baselineValue = null;
double? baselineValueDifference = null;
if (run.IsDifferential)
{
// get the difference between baseline and the run value for differential results
if (baselineHeadAllOutputFile.ContainsKey(dateTime) &&
baselineHeadAllOutputFile[dateTime].ContainsKey(inputPoint.ClosestNode))
{
baselineValue = baselineHeadAllOutputFile[dateTime][inputPoint.ClosestNode][layer];
baselineValueDifference = baselineValue - runValue;
}
else
{
Logger.LogWarning($"No baseline head data found for node {inputPoint.ClosestNode} at {dateTime}; skipping baseline comparison for point '{inputPoint.Name}'.");
}
}

inputPoint.TimeSteps.Add(new UserDataPointTimeStep()
{
DateTime = dateTime,
Value = runValue,
BaselineValue = baselineValue,
BaselineValueDifference = baselineValueDifference
});
}
});
}
else
{
Logger.LogInformation($"No UserDataPointInputs found in userdata.json for run {run.RunID}; writing empty groundwater level point output.");
}

fileAccessor
.SaveFile(
Expand Down
Loading