This commit is contained in:
2026-04-17 14:55:32 -04:00
commit bc3ac1d4c9
18017 changed files with 4371742 additions and 0 deletions

View File

@@ -0,0 +1,175 @@
using DataPROWin7.DataModel;
using DTS.Common.Classes;
using DTS.Common.Classes.Sensors;
using DTS.Common.Import.Interfaces;
using DTS.Common.Interface.Groups.GroupList;
using DTS.SensorDB;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace DTS.Common.Import
{
public static class GroupHelper
{
//FB 36879 Single responsibility principle , moved group realted utility methods to this class
public static IGroup CreateEmptyGroup()
{
return new GroupList.Model.Group(true);
}
public static void GetTestSensorParameters(out string isoCode, out string isoChannelName, out string userCode, out string userChannelName,
ParseParameters pp, SensorData sd, out string dasSerialNumber, out int dasChannelIdx)
{
isoCode = sd.ISOCode;
if (pp.SensorISOCode.ContainsKey(sd.SerialNumber)) { isoCode = pp.SensorISOCode[sd.SerialNumber]; }
isoChannelName = sd.ISOChannelName;
if (pp.SensorISOChannelName.ContainsKey(sd.SerialNumber)) { isoChannelName = pp.SensorISOChannelName[sd.SerialNumber]; }
userCode = sd.UserCode;
if (pp.SensorUserCode.ContainsKey(sd.SerialNumber)) { userCode = pp.SensorUserCode[sd.SerialNumber]; }
userChannelName = sd.UserChannelName;
if (pp.SensorUserChannelName.ContainsKey(sd.SerialNumber)) { userChannelName = pp.SensorUserChannelName[sd.SerialNumber]; }
dasSerialNumber = string.Empty;
if (pp.SensorDASSerialNumber.ContainsKey(sd.SerialNumber)) { dasSerialNumber = pp.SensorDASSerialNumber[sd.SerialNumber]; }
dasChannelIdx = -1;
if (pp.SensorDASChannelIndex.ContainsKey(sd.SerialNumber)) { dasChannelIdx = pp.SensorDASChannelIndex[sd.SerialNumber]; }
}
public static IReadOnlyDictionary<string, int> GetDAS(Dictionary<string, List<TsetSetupImportSensorInfo>> groupSensorLookupgroupSensorLookup,
DASHardware[] allDAS)
{
var lookup = new Dictionary<string, int>();
using (var e = groupSensorLookupgroupSensorLookup.GetEnumerator())
{
while (e.MoveNext())
{
var list = e.Current.Value;
foreach (var sensorDASSerialNumber in list.Select(s => s.DASSerialNumber))
{
if (!string.IsNullOrEmpty(sensorDASSerialNumber))
{
if (lookup.ContainsKey(sensorDASSerialNumber)) { continue; }
var das = Array.Find(allDAS, d => d.SerialNumber.Equals(sensorDASSerialNumber));
if (null == das) { continue; }
lookup[sensorDASSerialNumber] = das.DASId;
}
}
}
}
return lookup;
}
public static int? GetGroupId(TestTestObject testObject, Dictionary<string, int> groupIdMapping, List<TestObject> groupList)
{
foreach (var group in groupList)
{
if (group.SerialNumber == testObject.SerialNumber)
{
//Found the embedded Group, now find the static Group
foreach (var serialNumber in groupList.Select(p => p.SerialNumber))
{
if (serialNumber == group.GetISOTestObject().OriginalSerialNumber)
{
if (groupIdMapping.ContainsKey(serialNumber))
{
return groupIdMapping[serialNumber];
}
else
{
//Must not be from a static Group
return null;
}
}
}
}
}
return null;
}
public static List<string> ReverseChannelOrder(TestTemplate testTemplate, Dictionary<string, string> sensorGroupNameLookup, List<SensorData> sensors)
{
var orders = new List<string>();
//store the original channel order
//note that for some reason the channels are reversed
//I don't change that, I just make sure that when I set the channel order
//it's not reversed ...
if (sensors == null) { return orders; }
if (testTemplate == null) { return orders; }
if (sensorGroupNameLookup != null && sensorGroupNameLookup.Any())
{
foreach (var ch in sensors)
{
orders.Add($"{testTemplate.Name}_{sensorGroupNameLookup[ch.SerialNumber]}_{ch.Comment}");
}
orders.Reverse();
}
return orders;
}
public static List<SensorData> CleanUneededSensorDataPlaceHolder(Dictionary<string, List<SensorCalibration>> calibrationLookup, List<SensorData> sensorList)
{
// Cleanup unneeded sensor data place holders
for (int i = sensorList.Count - 1; i >= 0; i--)
{
if (sensorList[i].SerialNumber.StartsWith(Constants.ISO_CH_ONLY_PREFIX))
{
// Only an ISO channel
sensorList.RemoveAt(i);
}
else if (sensorList[i].IsSquib() || sensorList[i].IsStreamOutput() || sensorList[i].IsUart() || sensorList[i].IsStreamInput()
|| sensorList[i].IsDigitalInput())
{
//DON'T REMOVE, doesn't have a calibration.
}
else if (calibrationLookup[sensorList[i].SerialNumber].Exists(sc => sc.CalibrationDate == DateTime.MinValue))
{
// invalid default cal
// likely incomplete sensor reference
// Don't try to import this
sensorList.RemoveAt(i);
}
}
return sensorList;
}
public static List<SensorData> NormalizeSensorIds(List<SensorData> sensorList)
{
if(sensorList == null) { return new List<SensorData>(); }
//normalize sensor ids
for (int i = 0; i < sensorList.Count; i++)
{
var existing = SensorsCollection.SensorsList.GetSensorBySerialNumber(sensorList[i].SerialNumber);
if (null != existing && existing.IsAnalog())
{
//FB 41866 If the calibration is null use the one from parsed file
existing.Calibration = SensorCalibrationList.GetLatestCalibrationBySerialNumber(existing) ?? existing.Calibration;
}
if (existing != null)
{
sensorList[i].DatabaseId = existing.DatabaseId;
sensorList[i].Calibration = existing.Calibration;
}
else
{
//normalize to -2 ... -n (-1 preserved for no sensor)
sensorList[i].DatabaseId = -1 * (2 + i);
}
}
return sensorList;
}
}
}

View File

@@ -0,0 +1,392 @@
using DTS.Common.Classes.Sensors;
using DTS.Common.Enums.Sensors;
using DTS.Common.Interface.Sensors;
using DTS.Common.SharedResource.Strings;
using DTS.SensorDB;
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using static DTS.Common.Enums.Sensors.SensorConstants;
using System.IO;
using DTS.Common.Enums;
using DTS.Slice.Users;
using DTS.Common.Import.ImportOptions;
using DTS.Common.Import.Interfaces;
using DTS.Common.Import.Parsers;
using DTS.Common.Utilities.Logging;
using DTS.Common.Import.Factories;
namespace DTS.Common.Import
{
public class DTSCSVSensorsParser : ParseVariantBase
{
private readonly List<string> Errors = new List<string>();
private readonly User _currentUser;
private readonly CsvImportOptions _csvImportOptions;
private readonly ICalibrationImport _calibrationImport;
private readonly ZeroMethodOptions _zeroMethodOptions;
private readonly IImportNotification _importNotification;
public DTSCSVSensorsParser(IImportNotification importNotification, User user, CsvImportOptions csvImportOptions,
ICalibrationImport calibrationImport, ZeroMethodOptions zeroMethodOptions)
{
_currentUser = user;
_csvImportOptions = csvImportOptions;
_calibrationImport = calibrationImport;
_zeroMethodOptions = zeroMethodOptions;
_importNotification = importNotification;
}
public bool ImportCreateDynamicGroups { get; set; }
public bool UseISOCodeFilterMapping { get; set; }
public bool UseZeroForUnfiltered { get; set; }
private ISquibSettingDefaults GetSquibDefaults()
{
return SquibSettingDefaults.GetSquibSettingsDefault(_currentUser.UserName);
}
private IDigitalOutDefaults GetDigitalOutDefaults()
{
return DigitalOutputDefaults.GetDigitalOutDefault(_currentUser.UserName);
}
public override void Parse(ref ImportObject importObject)
{
if (string.IsNullOrEmpty(FileName))
{
return;
}
if (importObject == null)
{
throw new ArgumentNullException("importObject", "importObject can't be null or empty");
}
importObject = ParseSensor(importObject, FileName);
AssignErrorsToImportObject(ref importObject);
}
private void AssignErrorsToImportObject(ref ImportObject importObject)
{
//FB 42971 Assign errors in this parser to the import object errors property to be reported to user
foreach (var error in Errors)
{
importObject.AddError(new ImportError { ContinueImportOnError = true, Message = error, Severity = ImportSeverityError.Error });
}
}
private ImportObject ParseSensor(ImportObject importObject, string filename)
{
var columns = new List<CSVImportTags.Tags>();
var line = 2;
var version = 0;
var sensorGroupNameLookup = new Dictionary<string, string>();
var sensorGroupTypeLookup = new Dictionary<string, string>();
var groupNameTestObjectLookup = new Dictionary<string, string>();
var sensorISOCode = new Dictionary<string, string>();
var sensorISOChannelName = new Dictionary<string, string>();
var sensorUserCode = new Dictionary<string, string>();
var sensorUserChannelName = new Dictionary<string, string>();
var sensorDASSerialNumber = new Dictionary<string, string>();
var sensorDASChannelIdx = new Dictionary<string, int>();
var pp = new ParseParameters()
{
ImportCulture = _csvImportOptions.ImportCulture,
StripBackslash = _csvImportOptions.StripBackSlash,
SensorGroupNameLookup = sensorGroupNameLookup,
SensorGroupTypeLookup = sensorGroupTypeLookup,
GroupNameToTestObjectLookup = groupNameTestObjectLookup,
UseISOCodeFilterMapping = UseISOCodeFilterMapping,
UseZeroForUnfiltered = UseZeroForUnfiltered,
SensorISOCode = sensorISOCode,
SensorISOChannelName = sensorISOChannelName,
SensorUserCode = sensorUserCode,
SensorUserChannelName = sensorUserChannelName,
SensorDASSerialNumber = sensorDASSerialNumber,
SensorDASChannelIndex = sensorDASChannelIdx
};
using (var parser = CsvUtil.CreateCsvReader(filename))
{
var tokens = CsvUtil.ReadFields(parser);
if (tokens != null && tokens[0] == "Version")
{
var tokens2 = CsvUtil.ReadFields(parser);
if (tokens2 != null)
{
int.TryParse(tokens2[0], out version);
}
if (version == 6)
{
//read until you have no empty start and the tag is < 5
while (parser.Read())
{
tokens = CsvUtil.ReadFields(parser, false);
if (null == tokens || string.IsNullOrWhiteSpace(tokens[0])) { continue; }
var tag = CSVImportTags.GetTagForString(tokens[0]);
if (tag == CSVImportTags.Tags.Unknown) { continue; }
var tagVersion = CSVImportTags.GetVersionForTag(tag);
if (tagVersion > 4) { continue; }
else
{
break;
}
}
}
else
{
parser.Read();//read an empty line
tokens = CsvUtil.ReadFields(parser);
}
}
if (tokens == null)
{
Errors.Clear();
Errors.Add("No tokens to parse");
return importObject;
}
foreach (var token in tokens)
{
columns.Add(CSVImportTags.GetTagForString(token));
}
if (0 == columns.Count)
{
//NONE of the columns row was recognized, PopulateSensor below will fail as it uses this to populate the sensor
//this will not import any further, so shortcut to an error to the user
//14745 Vague error message when Test Setup import is attempted
Errors.Clear();
Errors.Add(StringResources.ImportSensorsPreviewControl_NoColumnsInTDCCSV);
return importObject;
}
var squibDefaults = GetSquibDefaults();
var digitalOutDefaults = GetDigitalOutDefaults();
//18259 DataPRO XML and CSV imports let you import inconsistent SensitvityUnits
var sensorsWithUpdatedSensitivityUnits = new List<string>();
while (parser.Read())
{
tokens = CsvUtil.ReadFields(parser, false);
if (null == tokens) { break; }
var sd = new SensorData();
var sc = new SensorCalibration();
pp.SensorData = sd;
pp.SensorCal = sc;
pp.SquibDefaults = squibDefaults;
pp.DigitalOutDefaults = digitalOutDefaults;
if (!PopulateSensor(columns, tokens.ToArray(), File.GetLastWriteTime(filename), pp))
{
//pp.Errors contains all the errors related to this sensor which is parsed
var errorMessage = $"Line {line}: {string.Join(CultureInfo.CurrentCulture.TextInfo.ListSeparator, pp.Errors)}";
Errors.Add(errorMessage);
line++;
continue;
}
if (!sc.IsProportional && sc.Records.Records[0].SensitivityUnits == SensUnits.mVperVperEU)
{
sc.Records.Records[0].SensitivityUnits = SensUnits.mVperEU;
if (!sensorsWithUpdatedSensitivityUnits.Contains(sd.SerialNumber))
{
sensorsWithUpdatedSensitivityUnits.Add(sd.SerialNumber);
}
}
if (!sd.SerialNumber.StartsWith(Constants.ISO_CH_ONLY_PREFIX))
{
if (!string.IsNullOrEmpty(sd.SerialNumber))
{
importObject.AddSensorLookup(sd.SerialNumber, sd);
}
if (!string.IsNullOrEmpty(sd.SerialNumber))
{
importObject.AddCalibrationLookup(sd.SerialNumber, new List<SensorCalibration>(new[] { sc }));
}
}
// this is for compatability for import test setups
importObject.AddSensor(sd);
importObject.AddSensorChannelCodeLookup(sd.SettingName, sd.SerialNumber);
// this is for compatability for import test setups
importObject.AddCalibration(sc);
line++;
}
if (sensorsWithUpdatedSensitivityUnits.Any())
{
var msg = $"{StringResources.SensorsUpdatedSensitivityUnits} {string.Join(", ", sensorsWithUpdatedSensitivityUnits.ToArray())}";
_importNotification.ReportErrors(new List<string>(new[] { msg }));
}
}
importObject.AssignSensorGroupNameLookup(sensorGroupNameLookup);
importObject.AssignSensorGroupTypeLookup(sensorGroupTypeLookup);
importObject.AssignGroupNameTestObjectLookup(groupNameTestObjectLookup);
importObject.AssignParseParameters(pp);
return importObject;
}
private void Preparse(ParseParameters pp)
{
var sd = (SensorData)pp.SensorData;
sd.Initialize(pp.SquibDefaults);
sd.Initialize(pp.DigitalOutDefaults);
}
private void PostParse(ParseParameters pp)
{
var sd = (SensorData)pp.SensorData;
var sc = (SensorCalibration)pp.SensorCal;
sd.InitializeTag(sd.Bridge);
pp.SensorCal.SerialNumber = pp.SensorData.SerialNumber;
if (!double.IsNaN(pp.IrtraccExponent) && pp.SensorData.SensorCategory == (int)SensorInformationFile.TDCSensorCategory.IRTracc)
{
pp.SensorCal.IsProportional = false;
pp.SensorCal.NonLinear = true;
if (pp.SensorCal.Records.Records[0].Poly.NonLinearStyle == NonLinearStyles.IRTraccAverageOverTime)
{
pp.SensorCal.Records.Records[0].Poly.MMPerV = 1000D / pp.Sensitivity;
pp.SensorCal.Records.Records[0].Poly.LinearizationExponent = pp.IrtraccExponent;
pp.SensorCal.Records.Records[0].Poly.NonLinearStyle = NonLinearStyles.IRTraccAverageOverTime;
}
pp.SensorCal.Records.Records[0].Poly.MarkValid(true);
}
else
{
pp.SensorCal.Records.Records[0].Sensitivity = pp.Sensitivity;
if (pp.SensorCal.IsProportional) { pp.SensorCal.Records.Records[0].Excitation = pp.SensorData.SupportedExcitation.First(); }
}
if (!double.IsNaN(pp.OriginalOffset))
{
pp.SensorCal.InitialOffsets.Offsets[0].EU = pp.OriginalOffset;
pp.SensorCal.InitialOffsets.Offsets[0].Form = InitialOffsetTypes.EU;
}
pp.SensorCal.ZeroMethods = new ZeroMethods(new[] { new ZeroMethod(pp.ZeroType, pp.ZeroStart, pp.ZeroEnd) });
if (sd.SerialNumber.StartsWith(Constants.ISO_CH_ONLY_PREFIX) && !string.IsNullOrEmpty(sd.ISOCode)) { pp.Errors = new List<string>(); }
//warn if we are analog and have 0 sensitivity ...
if (pp.SensorCal.Records.Records[0].Sensitivity == 0D && !sd.IsDigitalOutput() && !sd.IsDigitalInput() && !sd.IsSquib()
&& !sd.IsUart() && !sd.IsStreamInput() && !sd.IsStreamOutput() && !sd.IsCan())
{
pp.Errors.Add(string.Format(StringResources.InvalidEmptySensitivity, sd.SerialNumber));
}
//warn if we are analog and have invalid capacity
if (sd.Capacity < 1 && !sd.IsDigitalInput() && !sd.IsDigitalOutput() && !sd.IsSquib()
&& !sd.IsUart() && !sd.IsStreamOutput() && !sd.IsStreamInput() && !sd.IsCan())
{
pp.Errors.Add(string.Format(StringResources.ImportSensor_InvalidCapacity, sd.SerialNumber,
sd.Capacity));
}
//warn if we are analog and have invalid excitation
if (IsAnalogWithInvalidExcitation(sd, sc))
{
pp.Errors.Add(string.Format(StringResources.InvalidExcitation,
pp.SensorCal.Records.Records[0].Excitation));
}
if (IsAnalogWithInvalidSensitivity(sd, sc))
{
pp.Errors.Add(string.Format(StringResources.InvalidEmptySensitivity,
sd.SerialNumber));
}
}
private void Parse(CSVImportTags.Tags tag, string sValue, ParseParameters pp,
IReadOnlyDictionary<int, IParseCSVSensor> parsers)
{
var version = CSVImportTags.GetVersionForTag(tag);
if (parsers.ContainsKey(version))
{
parsers[version].ParseVersion(tag, sValue, pp);
}
}
/// <summary>
/// scans throw a line of tokens in an expected column order and populates a sensor and a sensor calibration with data from that line
/// errors is populated with any errors encountered during the import
/// returns true if any errors were detected.
/// </summary>
/// <param name="columns"></param>
/// <param name="tokens"></param>
/// <param name="sd"></param>
/// <param name="errors"></param>
/// <param name="sc"></param>
/// <param name="fileDateTime"></param>
/// <param name="importCulture"></param>
/// <param name="sensorGroupNameLookup"></param>
/// <param name="sensorGroupTypeLookup"></param>
/// <param name="squibDefaults"></param>
/// <param name="digitalOutDefaults"></param>
/// <param name="groupNameToTestObjectLookup"></param>
/// <param name="stripBackslash"></param>
/// <param name="useISOCodeFilterMapping"></param>
/// <param name="useZeroForUnfiltered"></param>
/// <returns></returns>
private bool PopulateSensor(IReadOnlyList<CSVImportTags.Tags> columns,
string[] tokens,
DateTime fileDateTime,
ParseParameters pp
)
{
pp.SensorData.LastModified = fileDateTime;
pp.IrtraccExponent = double.NaN;
pp.Sensitivity = double.NaN;
pp.SensorCal.ModifyDate = DateTime.Now;
pp.ZeroType = pp.SensorCal.ZeroMethods.Methods.First().Method;
pp.ZeroEnd = pp.SensorCal.ZeroMethods.Methods.First().End;
pp.ZeroStart = pp.SensorCal.ZeroMethods.Methods.First().Start;
pp.SavedIsProportional = false;
pp.SavedRemoveOffset = false;
//FB 42971 for each senosr starts with no errors
pp.Errors = new List<string>();
Preparse(pp);
var parsers = CSVSensorParserFactory.CreateCSVParsers(_calibrationImport, _zeroMethodOptions,
_importNotification, ImportCreateDynamicGroups, UseISOCodeFilterMapping,
UseZeroForUnfiltered);
for (var i = 0; i < columns.Count; i++)
{
var field = columns[i];
var val = tokens[i];
if (val.StartsWith("'") && !val.StartsWith("''")) { val = val.Substring(1); }
try
{
Parse(field, val, pp, parsers);
}
catch (Exception ex)
{
APILogger.Log($"Failed to parse tags version {i}", ex);
return false;
}
}
PostParse(pp);
return !pp.Errors.Any();
}
private static bool IsAnalogWithInvalidExcitation(SensorData sd, SensorCalibration sc)
{
return sc.Records.Records[0].Excitation ==
ExcitationVoltageOptions.ExcitationVoltageOption.Undefined &&
(sd.Bridge == BridgeType.FullBridge ||
sd.Bridge == BridgeType.HalfBridge ||
sd.Bridge == BridgeType.QuarterBridge);
}
private static bool IsAnalogWithInvalidSensitivity(SensorData sd, SensorCalibration sc)
{
if (sd == null || sc == null || null == sc.Records || null == sc.Records.Records || 0 == sc.Records.Records.Length) { return true; }
return sd.IsAnalog() && Math.Abs(sc.Records.Records[0].Sensitivity) < .0000001;
}
}
}