using DTS.Common.Classes.Sensors; using DTS.Common.Enums.Sensors; using DTS.Common.Interface.Sensors; using DTS.Common.SharedResource.Strings; using DTS.SensorDB; using System; using System.Collections.Generic; using System.Globalization; using System.Linq; using static DTS.Common.Enums.Sensors.SensorConstants; using System.IO; using DTS.Common.Enums; using DTS.Slice.Users; using DTS.Common.Import.ImportOptions; using DTS.Common.Import.Interfaces; using DTS.Common.Import.Parsers; using DTS.Common.Utilities.Logging; using DTS.Common.Import.Factories; namespace DTS.Common.Import { public class DTSCSVSensorsParser : ParseVariantBase { private readonly List Errors = new List(); private readonly User _currentUser; private readonly CsvImportOptions _csvImportOptions; private readonly ICalibrationImport _calibrationImport; private readonly ZeroMethodOptions _zeroMethodOptions; private readonly IImportNotification _importNotification; public DTSCSVSensorsParser(IImportNotification importNotification, User user, CsvImportOptions csvImportOptions, ICalibrationImport calibrationImport, ZeroMethodOptions zeroMethodOptions) { _currentUser = user; _csvImportOptions = csvImportOptions; _calibrationImport = calibrationImport; _zeroMethodOptions = zeroMethodOptions; _importNotification = importNotification; } public bool ImportCreateDynamicGroups { get; set; } public bool UseISOCodeFilterMapping { get; set; } public bool UseZeroForUnfiltered { get; set; } private ISquibSettingDefaults GetSquibDefaults() { return SquibSettingDefaults.GetSquibSettingsDefault(_currentUser.UserName); } private IDigitalOutDefaults GetDigitalOutDefaults() { return DigitalOutputDefaults.GetDigitalOutDefault(_currentUser.UserName); } public override void Parse(ref ImportObject importObject) { if (string.IsNullOrEmpty(FileName)) { return; } if (importObject == null) { throw new ArgumentNullException("importObject", "importObject can't be null or empty"); } importObject = ParseSensor(importObject, FileName); AssignErrorsToImportObject(ref importObject); } private void AssignErrorsToImportObject(ref ImportObject importObject) { //FB 42971 Assign errors in this parser to the import object errors property to be reported to user foreach (var error in Errors) { importObject.AddError(new ImportError { ContinueImportOnError = true, Message = error, Severity = ImportSeverityError.Error }); } } private ImportObject ParseSensor(ImportObject importObject, string filename) { var columns = new List(); var line = 2; var version = 0; var sensorGroupNameLookup = new Dictionary(); var sensorGroupTypeLookup = new Dictionary(); var groupNameTestObjectLookup = new Dictionary(); var sensorISOCode = new Dictionary(); var sensorISOChannelName = new Dictionary(); var sensorUserCode = new Dictionary(); var sensorUserChannelName = new Dictionary(); var sensorDASSerialNumber = new Dictionary(); var sensorDASChannelIdx = new Dictionary(); var pp = new ParseParameters() { ImportCulture = _csvImportOptions.ImportCulture, StripBackslash = _csvImportOptions.StripBackSlash, SensorGroupNameLookup = sensorGroupNameLookup, SensorGroupTypeLookup = sensorGroupTypeLookup, GroupNameToTestObjectLookup = groupNameTestObjectLookup, UseISOCodeFilterMapping = UseISOCodeFilterMapping, UseZeroForUnfiltered = UseZeroForUnfiltered, SensorISOCode = sensorISOCode, SensorISOChannelName = sensorISOChannelName, SensorUserCode = sensorUserCode, SensorUserChannelName = sensorUserChannelName, SensorDASSerialNumber = sensorDASSerialNumber, SensorDASChannelIndex = sensorDASChannelIdx }; using (var parser = CsvUtil.CreateCsvReader(filename)) { var tokens = CsvUtil.ReadFields(parser); if (tokens != null && tokens[0] == "Version") { var tokens2 = CsvUtil.ReadFields(parser); if (tokens2 != null) { int.TryParse(tokens2[0], out version); } if (version == 6) { //read until you have no empty start and the tag is < 5 while (parser.Read()) { tokens = CsvUtil.ReadFields(parser, false); if (null == tokens || string.IsNullOrWhiteSpace(tokens[0])) { continue; } var tag = CSVImportTags.GetTagForString(tokens[0]); if (tag == CSVImportTags.Tags.Unknown) { continue; } var tagVersion = CSVImportTags.GetVersionForTag(tag); if (tagVersion > 4) { continue; } else { break; } } } else { parser.Read();//read an empty line tokens = CsvUtil.ReadFields(parser); } } if (tokens == null) { Errors.Clear(); Errors.Add("No tokens to parse"); return importObject; } foreach (var token in tokens) { columns.Add(CSVImportTags.GetTagForString(token)); } if (0 == columns.Count) { //NONE of the columns row was recognized, PopulateSensor below will fail as it uses this to populate the sensor //this will not import any further, so shortcut to an error to the user //14745 Vague error message when Test Setup import is attempted Errors.Clear(); Errors.Add(StringResources.ImportSensorsPreviewControl_NoColumnsInTDCCSV); return importObject; } var squibDefaults = GetSquibDefaults(); var digitalOutDefaults = GetDigitalOutDefaults(); //18259 DataPRO XML and CSV imports let you import inconsistent SensitvityUnits var sensorsWithUpdatedSensitivityUnits = new List(); while (parser.Read()) { tokens = CsvUtil.ReadFields(parser, false); if (null == tokens) { break; } var sd = new SensorData(); var sc = new SensorCalibration(); pp.SensorData = sd; pp.SensorCal = sc; pp.SquibDefaults = squibDefaults; pp.DigitalOutDefaults = digitalOutDefaults; if (!PopulateSensor(columns, tokens.ToArray(), File.GetLastWriteTime(filename), pp)) { //pp.Errors contains all the errors related to this sensor which is parsed var errorMessage = $"Line {line}: {string.Join(CultureInfo.CurrentCulture.TextInfo.ListSeparator, pp.Errors)}"; Errors.Add(errorMessage); line++; continue; } if (!sc.IsProportional && sc.Records.Records[0].SensitivityUnits == SensUnits.mVperVperEU) { sc.Records.Records[0].SensitivityUnits = SensUnits.mVperEU; if (!sensorsWithUpdatedSensitivityUnits.Contains(sd.SerialNumber)) { sensorsWithUpdatedSensitivityUnits.Add(sd.SerialNumber); } } if (!sd.SerialNumber.StartsWith(Constants.ISO_CH_ONLY_PREFIX)) { if (!string.IsNullOrEmpty(sd.SerialNumber)) { importObject.AddSensorLookup(sd.SerialNumber, sd); } if (!string.IsNullOrEmpty(sd.SerialNumber)) { importObject.AddCalibrationLookup(sd.SerialNumber, new List(new[] { sc })); } } // this is for compatability for import test setups importObject.AddSensor(sd); importObject.AddSensorChannelCodeLookup(sd.SettingName, sd.SerialNumber); // this is for compatability for import test setups importObject.AddCalibration(sc); line++; } if (sensorsWithUpdatedSensitivityUnits.Any()) { var msg = $"{StringResources.SensorsUpdatedSensitivityUnits} {string.Join(", ", sensorsWithUpdatedSensitivityUnits.ToArray())}"; _importNotification.ReportErrors(new List(new[] { msg })); } } importObject.AssignSensorGroupNameLookup(sensorGroupNameLookup); importObject.AssignSensorGroupTypeLookup(sensorGroupTypeLookup); importObject.AssignGroupNameTestObjectLookup(groupNameTestObjectLookup); importObject.AssignParseParameters(pp); return importObject; } private void Preparse(ParseParameters pp) { var sd = (SensorData)pp.SensorData; sd.Initialize(pp.SquibDefaults); sd.Initialize(pp.DigitalOutDefaults); } private void PostParse(ParseParameters pp) { var sd = (SensorData)pp.SensorData; var sc = (SensorCalibration)pp.SensorCal; sd.InitializeTag(sd.Bridge); pp.SensorCal.SerialNumber = pp.SensorData.SerialNumber; if (!double.IsNaN(pp.IrtraccExponent) && pp.SensorData.SensorCategory == (int)SensorInformationFile.TDCSensorCategory.IRTracc) { pp.SensorCal.IsProportional = false; pp.SensorCal.NonLinear = true; if (pp.SensorCal.Records.Records[0].Poly.NonLinearStyle == NonLinearStyles.IRTraccAverageOverTime) { pp.SensorCal.Records.Records[0].Poly.MMPerV = 1000D / pp.Sensitivity; pp.SensorCal.Records.Records[0].Poly.LinearizationExponent = pp.IrtraccExponent; pp.SensorCal.Records.Records[0].Poly.NonLinearStyle = NonLinearStyles.IRTraccAverageOverTime; } pp.SensorCal.Records.Records[0].Poly.MarkValid(true); } else { pp.SensorCal.Records.Records[0].Sensitivity = pp.Sensitivity; if (pp.SensorCal.IsProportional) { pp.SensorCal.Records.Records[0].Excitation = pp.SensorData.SupportedExcitation.First(); } } if (!double.IsNaN(pp.OriginalOffset)) { pp.SensorCal.InitialOffsets.Offsets[0].EU = pp.OriginalOffset; pp.SensorCal.InitialOffsets.Offsets[0].Form = InitialOffsetTypes.EU; } pp.SensorCal.ZeroMethods = new ZeroMethods(new[] { new ZeroMethod(pp.ZeroType, pp.ZeroStart, pp.ZeroEnd) }); if (sd.SerialNumber.StartsWith(Constants.ISO_CH_ONLY_PREFIX) && !string.IsNullOrEmpty(sd.ISOCode)) { pp.Errors = new List(); } //warn if we are analog and have 0 sensitivity ... if (pp.SensorCal.Records.Records[0].Sensitivity == 0D && !sd.IsDigitalOutput() && !sd.IsDigitalInput() && !sd.IsSquib() && !sd.IsUart() && !sd.IsStreamInput() && !sd.IsStreamOutput() && !sd.IsCan()) { pp.Errors.Add(string.Format(StringResources.InvalidEmptySensitivity, sd.SerialNumber)); } //warn if we are analog and have invalid capacity if (sd.Capacity < 1 && !sd.IsDigitalInput() && !sd.IsDigitalOutput() && !sd.IsSquib() && !sd.IsUart() && !sd.IsStreamOutput() && !sd.IsStreamInput() && !sd.IsCan()) { pp.Errors.Add(string.Format(StringResources.ImportSensor_InvalidCapacity, sd.SerialNumber, sd.Capacity)); } //warn if we are analog and have invalid excitation if (IsAnalogWithInvalidExcitation(sd, sc)) { pp.Errors.Add(string.Format(StringResources.InvalidExcitation, pp.SensorCal.Records.Records[0].Excitation)); } if (IsAnalogWithInvalidSensitivity(sd, sc)) { pp.Errors.Add(string.Format(StringResources.InvalidEmptySensitivity, sd.SerialNumber)); } } private void Parse(CSVImportTags.Tags tag, string sValue, ParseParameters pp, IReadOnlyDictionary parsers) { var version = CSVImportTags.GetVersionForTag(tag); if (parsers.ContainsKey(version)) { parsers[version].ParseVersion(tag, sValue, pp); } } /// /// scans throw a line of tokens in an expected column order and populates a sensor and a sensor calibration with data from that line /// errors is populated with any errors encountered during the import /// returns true if any errors were detected. /// /// /// /// /// /// /// /// /// /// /// /// /// /// /// /// /// private bool PopulateSensor(IReadOnlyList columns, string[] tokens, DateTime fileDateTime, ParseParameters pp ) { pp.SensorData.LastModified = fileDateTime; pp.IrtraccExponent = double.NaN; pp.Sensitivity = double.NaN; pp.SensorCal.ModifyDate = DateTime.Now; pp.ZeroType = pp.SensorCal.ZeroMethods.Methods.First().Method; pp.ZeroEnd = pp.SensorCal.ZeroMethods.Methods.First().End; pp.ZeroStart = pp.SensorCal.ZeroMethods.Methods.First().Start; pp.SavedIsProportional = false; pp.SavedRemoveOffset = false; //FB 42971 for each senosr starts with no errors pp.Errors = new List(); Preparse(pp); var parsers = CSVSensorParserFactory.CreateCSVParsers(_calibrationImport, _zeroMethodOptions, _importNotification, ImportCreateDynamicGroups, UseISOCodeFilterMapping, UseZeroForUnfiltered); for (var i = 0; i < columns.Count; i++) { var field = columns[i]; var val = tokens[i]; if (val.StartsWith("'") && !val.StartsWith("''")) { val = val.Substring(1); } try { Parse(field, val, pp, parsers); } catch (Exception ex) { APILogger.Log($"Failed to parse tags version {i}", ex); return false; } } PostParse(pp); return !pp.Errors.Any(); } private static bool IsAnalogWithInvalidExcitation(SensorData sd, SensorCalibration sc) { return sc.Records.Records[0].Excitation == ExcitationVoltageOptions.ExcitationVoltageOption.Undefined && (sd.Bridge == BridgeType.FullBridge || sd.Bridge == BridgeType.HalfBridge || sd.Bridge == BridgeType.QuarterBridge); } private static bool IsAnalogWithInvalidSensitivity(SensorData sd, SensorCalibration sc) { if (sd == null || sc == null || null == sc.Records || null == sc.Records.Records || 0 == sc.Records.Records.Length) { return true; } return sd.IsAnalog() && Math.Abs(sc.Records.Records[0].Sensitivity) < .0000001; } } }