DP44/GLM5Analysis/PromptTemplates/AddNewImportFormat.md

Add New Import Format - DataPRO Prompt Template

Context

DataPRO supports importing test data from various file formats through the Common/DTS.Common.Import/ library. The import system uses a parser-based architecture where each format has a dedicated parser class that converts external data into the internal ImportObject structure.

System Architecture

Common/DTS.Common.Import/
├── Parsers/
│   ├── CSV/                  # CSV format parsers
│   ├── EQX/                  # Equipment Exchange format
│   │   ├── EQXSensorsParser.cs
│   │   ├── EQXTestSetupParser.cs
│   │   └── EQXGroupImport.cs
│   ├── DefaultParseImport.cs
│   ├── DTSXMLParseImport.cs
│   └── ParseVariantBase.cs   # Base class for parsers
├── ImportObject.cs           # Container for imported data
├── ImportError.cs            # Error handling
├── ImportOptions/            # Format-specific options
├── Interfaces/               # Parser interfaces
├── Persist/                  # Database persistence
└── XML/                      # XML processing utilities

Step-by-Step Instructions

1. Create the Parser Class

File: Common/DTS.Common.Import/Parsers/{FormatName}/{FormatName}Parser.cs

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using DTS.Common.Enums;
using DTS.Common.Import.Enums;
using DTS.Common.Import.ImportOptions;
using DTS.Common.Import.Parsers;
using DTS.Common.Interface.Sensors;
using DTS.Common.Storage;
using DTS.SensorDB;

namespace DTS.Common.Import
{
    public class {FORMAT_NAME}Parser : ParseVariantBase
    {
        private readonly User _currentUser;
        private readonly IImportNotification _importNotification;
        private readonly {FORMAT_NAME}ImportOptions _importOptions;
        
        public {FORMAT_NAME}Parser(
            IImportNotification importNotification,
            User user,
            {FORMAT_NAME}ImportOptions importOptions)
        {
            _currentUser = user;
            _importNotification = importNotification;
            _importOptions = importOptions;
        }
        
        public override void Parse(ref ImportObject importObject)
        {
            if (string.IsNullOrEmpty(FileName))
            {
                return;
            }
            
            if (importObject == null)
            {
                throw new ArgumentNullException(nameof(importObject), 
                    "importObject can't be null");
            }
            
            importObject = ParseFile(importObject, FileName);
        }
        
        private ImportObject ParseFile(ImportObject importObject, string filePath)
        {
            try
            {
                // Validate file exists
                if (!File.Exists(filePath))
                {
                    importObject.AddError(new ImportError(
                        $"File not found: {filePath}"));
                    return importObject;
                }
                
                // Read and parse file content
                var content = File.ReadAllText(filePath);
                var parsedData = ParseContent(content);
                
                // Populate import object
                PopulateImportObject(importObject, parsedData);
                
                // Set source format
                importObject.SourceFormat = ImportFormats.{FORMAT_NAME};
                
                _importNotification?.NotifyProgress(100, "Import complete");
            }
            catch (Exception ex)
            {
                importObject.AddError(new ImportError(
                    $"Parse error: {ex.Message}"));
            }
            
            return importObject;
        }
        
        private ParsedData ParseContent(string content)
        {
            var data = new ParsedData();
            
            // Format-specific parsing logic here
            // Example: Parse lines, extract sensors, test setups, etc.
            
            return data;
        }
        
        private void PopulateImportObject(ImportObject importObject, ParsedData data)
        {
            // Add sensors
            foreach (var sensor in data.Sensors)
            {
                importObject.AddSensor(ConvertToSensorData(sensor));
            }
            
            // Add test setups
            foreach (var setup in data.TestSetups)
            {
                importObject.AddTestSetup(ConvertToTestTemplate(setup));
            }
            
            // Add hardware
            foreach (var hardware in data.Hardware)
            {
                importObject.AddHardware(ConvertToDASHardware(hardware));
            }
        }
        
        private SensorData ConvertToSensorData(ParsedSensor parsed)
        {
            var sensorData = new SensorData
            {
                Name = parsed.Name,
                SerialNumber = parsed.SerialNumber,
                ChannelCode = parsed.ChannelCode,
                CalibrationFactor = parsed.CalibrationFactor,
                EngineeringUnits = parsed.EngineeringUnits,
                Sensitivity = parsed.Sensitivity
            };
            
            return sensorData;
        }
        
        private TestTemplate ConvertToTestTemplate(ParsedTestSetup parsed)
        {
            // Convert parsed test setup to TestTemplate
            return new TestTemplate();
        }
        
        private DASHardware ConvertToDASHardware(ParsedHardware parsed)
        {
            // Convert parsed hardware to DASHardware
            return new DASHardware();
        }
    }
    
    internal class ParsedData
    {
        public List<ParsedSensor> Sensors { get; set; } = new List<ParsedSensor>();
        public List<ParsedTestSetup> TestSetups { get; set; } = new List<ParsedTestSetup>();
        public List<ParsedHardware> Hardware { get; set; } = new List<ParsedHardware>();
    }
    
    internal class ParsedSensor
    {
        public string Name { get; set; }
        public string SerialNumber { get; set; }
        public string ChannelCode { get; set; }
        public double CalibrationFactor { get; set; }
        public string EngineeringUnits { get; set; }
        public double Sensitivity { get; set; }
    }
    
    internal class ParsedTestSetup { }
    internal class ParsedHardware { }
}

2. Create Import Options Class

File: Common/DTS.Common.Import/ImportOptions/{FormatName}ImportOptions.cs

using System;

namespace DTS.Common.Import.ImportOptions
{
    public class {FORMAT_NAME}ImportOptions
    {
        public bool ImportSensors { get; set; } = true;
        public bool ImportTestSetups { get; set; } = true;
        public bool ImportHardware { get; set; } = false;
        public bool CreateGroups { get; set; } = true;
        public bool ValidateData { get; set; } = true;
        
        // Format-specific options
        public string DateTimeFormat { get; set; } = "yyyy-MM-dd HH:mm:ss";
        public string Delimiter { get; set; } = ",";
        public bool HasHeaderRow { get; set; } = true;
        public int SkipRows { get; set; } = 0;
        
        public void Validate()
        {
            if (string.IsNullOrEmpty(Delimiter))
                throw new ArgumentException("Delimiter is required");
        }
    }
}

3. Update Import Formats Enum

File: Common/DTS.Common.Import/Enums/ImportFormats.cs

namespace DTS.Common.Import.Enums
{
    public enum ImportFormats
    {
        NOT_SPECIFIED,
        CSV,
        EQX,
        DTS_XML,
        {FORMAT_NAME}  // Add new format
    }
    
    public enum ImportFileFormat
    {
        NoTestSetup,
        SingleTestSetup,
        MultipleTestSetup
    }
}

4. Create Parser Factory Registration

File: Common/DTS.Common.Import/Factories/{FormatName}ParserFactory.cs

using System;
using DTS.Common.Import.ImportOptions;
using DTS.Common.Import.Parsers;
using DTS.Common.Interface.Sensors;
using DTS.SensorDB;

namespace DTS.Common.Import.Factories
{
    public class {FORMAT_NAME}ParserFactory
    {
        public static {FORMAT_NAME}Parser Create(
            IImportNotification importNotification,
            User user,
            {FORMAT_NAME}ImportOptions options)
        {
            if (options == null)
            {
                options = new {FORMAT_NAME}ImportOptions();
            }
            
            return new {FORMAT_NAME}Parser(importNotification, user, options);
        }
    }
}

5. Add File Detection Logic

File: Common/DTS.Common.Import/ImportObject.cs (modify)

Add method to detect format from file:

public static ImportFormats DetectFormat(string filePath)
{
    var extension = Path.GetExtension(filePath).ToLowerInvariant();
    
    switch (extension)
    {
        case ".csv":
            return ImportFormats.CSV;
        case ".eqx":
            return ImportFormats.EQX;
        case ".{FORMAT_EXTENSION}":
            return ImportFormats.{FORMAT_NAME};
        default:
            // Check file content for format signature
            return DetectFormatFromContent(filePath);
    }
}

private static ImportFormats DetectFormatFromContent(string filePath)
{
    // Read first few lines to detect format
    var firstLine = File.ReadLines(filePath).FirstOrDefault();
    
    if (firstLine != null)
    {
        // Check for format-specific signatures
        if (firstLine.StartsWith("{FORMAT_SIGNATURE}"))
        {
            return ImportFormats.{FORMAT_NAME};
        }
    }
    
    return ImportFormats.NOT_SPECIFIED;
}

6. Create Unit Tests

File: Common/DTS.Common.Tests/{FormatName}ParserShould.cs

using DTS.Common.Import;
using DTS.Common.Import.ImportOptions;
using NUnit.Framework;
using System;
using System.IO;

namespace DTS.Common.Tests
{
    [TestFixture]
    public class {FORMAT_NAME}ParserShould
    {
        private {FORMAT_NAME}Parser _parser;
        private {FORMAT_NAME}ImportOptions _options;
        
        [SetUp]
        public void Setup()
        {
            _options = new {FORMAT_NAME}ImportOptions();
            _parser = new {FORMAT_NAME}Parser(null, null, _options);
        }
        
        [Test]
        public void Parse_ShouldReturnEmptyImportObject_WhenFileNameIsNull()
        {
            // Arrange
            var importObject = new ImportObject();
            _parser.FileName = null;
            
            // Act
            _parser.Parse(ref importObject);
            
            // Assert
            Assert.IsNotNull(importObject);
        }
        
        [Test]
        public void Parse_ShouldThrowArgumentNullException_WhenImportObjectIsNull()
        {
            // Arrange
            ImportObject importObject = null;
            
            // Act & Assert
            Assert.Throws<ArgumentNullException>(() => _parser.Parse(ref importObject));
        }
        
        [Test]
        public void Parse_ShouldImportSensors_WhenValidFile()
        {
            // Arrange
            var importObject = new ImportObject();
            var testFile = CreateTestFile();
            _parser.FileName = testFile;
            
            // Act
            _parser.Parse(ref importObject);
            
            // Assert
            // Add assertions for expected data
            
            // Cleanup
            File.Delete(testFile);
        }
        
        private string CreateTestFile()
        {
            var path = Path.GetTempFileName();
            // Write test content
            File.WriteAllText(path, "test content");
            return path;
        }
    }
}

Files to Create/Modify Summary

File	Action
Parsers/{FormatName}/{FormatName}Parser.cs	Create
ImportOptions/{FormatName}ImportOptions.cs	Create
Factories/{FormatName}ParserFactory.cs	Create
Enums/ImportFormats.cs	Modify (add enum value)
ImportObject.cs	Modify (add detection logic)
DTS.Common.Tests/{FormatName}ParserShould.cs	Create

Parser Base Class Reference

public abstract class ParseVariantBase
{
    public string FileName { get; set; }
    public abstract void Parse(ref ImportObject importObject);
}

ImportObject Key Methods

// Adding data to import object
void AddSensor(SensorData sensor);
void AddTestSetup(TestTemplate template);
void AddHardware(DASHardware hardware);
void AddCalibration(SensorCalibration calibration);

// Error handling
void AddError(ImportError error);
IEnumerable<ImportError> Errors();

// Format detection
ImportFormats GetImportFileFormat();

Validation Checklist

• Parser inherits from ParseVariantBase
• Parse() method validates null inputs
• File existence checked before parsing
• Errors added to ImportObject for failures
• Progress notification via IImportNotification
• Source format set on ImportObject
• Import options class with validation
• Format added to ImportFormats enum
• File detection logic implemented
• Unit tests for happy path and error cases
• Test file cleanup in tests

Common Patterns

1. Dependency Injection: Parser receives notification and user objects
2. Error Accumulation: Add errors to ImportObject rather than throwing
3. Progress Notification: Call NotifyProgress() during long operations
4. File Validation: Check existence before reading
5. Format Detection: Check extension and content signature

Supported Data Types

The ImportObject can hold:

• SensorData - Sensor configurations
• TestTemplate - Test setup definitions
• DASHardware - Data acquisition hardware
• SensorCalibration - Calibration data
• Group - Test object groups
• ISO.CustomerDetails - Customer information