465 lines
14 KiB
Markdown
465 lines
14 KiB
Markdown
|
# Add New Import Format - DataPRO Prompt Template
|
||
|
|
|
||
|
|
## Context
|
||
|
|
DataPRO supports importing test data from various file formats through the `Common/DTS.Common.Import/` library. The import system uses a parser-based architecture where each format has a dedicated parser class that converts external data into the internal `ImportObject` structure.
|
||
|
|
|
||
|
|
## System Architecture
|
||
|
|
```
|
||
|
|
Common/DTS.Common.Import/
|
||
|
|
├── Parsers/
|
||
|
|
│ ├── CSV/ # CSV format parsers
|
||
|
|
│ ├── EQX/ # Equipment Exchange format
|
||
|
|
│ │ ├── EQXSensorsParser.cs
|
||
|
|
│ │ ├── EQXTestSetupParser.cs
|
||
|
|
│ │ └── EQXGroupImport.cs
|
||
|
|
│ ├── DefaultParseImport.cs
|
||
|
|
│ ├── DTSXMLParseImport.cs
|
||
|
|
│ └── ParseVariantBase.cs # Base class for parsers
|
||
|
|
├── ImportObject.cs # Container for imported data
|
||
|
|
├── ImportError.cs # Error handling
|
||
|
|
├── ImportOptions/ # Format-specific options
|
||
|
|
├── Interfaces/ # Parser interfaces
|
||
|
|
├── Persist/ # Database persistence
|
||
|
|
└── XML/ # XML processing utilities
|
||
|
|
```
|
||
|
|
|
||
|
|
## Step-by-Step Instructions
|
||
|
|
|
||
|
|
### 1. Create the Parser Class
|
||
|
|
**File:** `Common/DTS.Common.Import/Parsers/{FormatName}/{FormatName}Parser.cs`
|
||
|
|
|
||
|
|
```csharp
|
||
|
|
using System;
|
||
|
|
using System.Collections.Generic;
|
||
|
|
using System.IO;
|
||
|
|
using System.Linq;
|
||
|
|
using DTS.Common.Enums;
|
||
|
|
using DTS.Common.Import.Enums;
|
||
|
|
using DTS.Common.Import.ImportOptions;
|
||
|
|
using DTS.Common.Import.Parsers;
|
||
|
|
using DTS.Common.Interface.Sensors;
|
||
|
|
using DTS.Common.Storage;
|
||
|
|
using DTS.SensorDB;
|
||
|
|
|
||
|
|
namespace DTS.Common.Import
|
||
|
|
{
|
||
|
|
public class {FORMAT_NAME}Parser : ParseVariantBase
|
||
|
|
{
|
||
|
|
private readonly User _currentUser;
|
||
|
|
private readonly IImportNotification _importNotification;
|
||
|
|
private readonly {FORMAT_NAME}ImportOptions _importOptions;
|
||
|
|
|
||
|
|
public {FORMAT_NAME}Parser(
|
||
|
|
IImportNotification importNotification,
|
||
|
|
User user,
|
||
|
|
{FORMAT_NAME}ImportOptions importOptions)
|
||
|
|
{
|
||
|
|
_currentUser = user;
|
||
|
|
_importNotification = importNotification;
|
||
|
|
_importOptions = importOptions;
|
||
|
|
}
|
||
|
|
|
||
|
|
public override void Parse(ref ImportObject importObject)
|
||
|
|
{
|
||
|
|
if (string.IsNullOrEmpty(FileName))
|
||
|
|
{
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (importObject == null)
|
||
|
|
{
|
||
|
|
throw new ArgumentNullException(nameof(importObject),
|
||
|
|
"importObject can't be null");
|
||
|
|
}
|
||
|
|
|
||
|
|
importObject = ParseFile(importObject, FileName);
|
||
|
|
}
|
||
|
|
|
||
|
|
private ImportObject ParseFile(ImportObject importObject, string filePath)
|
||
|
|
{
|
||
|
|
try
|
||
|
|
{
|
||
|
|
// Validate file exists
|
||
|
|
if (!File.Exists(filePath))
|
||
|
|
{
|
||
|
|
importObject.AddError(new ImportError(
|
||
|
|
$"File not found: {filePath}"));
|
||
|
|
return importObject;
|
||
|
|
}
|
||
|
|
|
||
|
|
// Read and parse file content
|
||
|
|
var content = File.ReadAllText(filePath);
|
||
|
|
var parsedData = ParseContent(content);
|
||
|
|
|
||
|
|
// Populate import object
|
||
|
|
PopulateImportObject(importObject, parsedData);
|
||
|
|
|
||
|
|
// Set source format
|
||
|
|
importObject.SourceFormat = ImportFormats.{FORMAT_NAME};
|
||
|
|
|
||
|
|
_importNotification?.NotifyProgress(100, "Import complete");
|
||
|
|
}
|
||
|
|
catch (Exception ex)
|
||
|
|
{
|
||
|
|
importObject.AddError(new ImportError(
|
||
|
|
$"Parse error: {ex.Message}"));
|
||
|
|
}
|
||
|
|
|
||
|
|
return importObject;
|
||
|
|
}
|
||
|
|
|
||
|
|
private ParsedData ParseContent(string content)
|
||
|
|
{
|
||
|
|
var data = new ParsedData();
|
||
|
|
|
||
|
|
// Format-specific parsing logic here
|
||
|
|
// Example: Parse lines, extract sensors, test setups, etc.
|
||
|
|
|
||
|
|
return data;
|
||
|
|
}
|
||
|
|
|
||
|
|
private void PopulateImportObject(ImportObject importObject, ParsedData data)
|
||
|
|
{
|
||
|
|
// Add sensors
|
||
|
|
foreach (var sensor in data.Sensors)
|
||
|
|
{
|
||
|
|
importObject.AddSensor(ConvertToSensorData(sensor));
|
||
|
|
}
|
||
|
|
|
||
|
|
// Add test setups
|
||
|
|
foreach (var setup in data.TestSetups)
|
||
|
|
{
|
||
|
|
importObject.AddTestSetup(ConvertToTestTemplate(setup));
|
||
|
|
}
|
||
|
|
|
||
|
|
// Add hardware
|
||
|
|
foreach (var hardware in data.Hardware)
|
||
|
|
{
|
||
|
|
importObject.AddHardware(ConvertToDASHardware(hardware));
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
private SensorData ConvertToSensorData(ParsedSensor parsed)
|
||
|
|
{
|
||
|
|
var sensorData = new SensorData
|
||
|
|
{
|
||
|
|
Name = parsed.Name,
|
||
|
|
SerialNumber = parsed.SerialNumber,
|
||
|
|
ChannelCode = parsed.ChannelCode,
|
||
|
|
CalibrationFactor = parsed.CalibrationFactor,
|
||
|
|
EngineeringUnits = parsed.EngineeringUnits,
|
||
|
|
Sensitivity = parsed.Sensitivity
|
||
|
|
};
|
||
|
|
|
||
|
|
return sensorData;
|
||
|
|
}
|
||
|
|
|
||
|
|
private TestTemplate ConvertToTestTemplate(ParsedTestSetup parsed)
|
||
|
|
{
|
||
|
|
// Convert parsed test setup to TestTemplate
|
||
|
|
return new TestTemplate();
|
||
|
|
}
|
||
|
|
|
||
|
|
private DASHardware ConvertToDASHardware(ParsedHardware parsed)
|
||
|
|
{
|
||
|
|
// Convert parsed hardware to DASHardware
|
||
|
|
return new DASHardware();
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
internal class ParsedData
|
||
|
|
{
|
||
|
|
public List<ParsedSensor> Sensors { get; set; } = new List<ParsedSensor>();
|
||
|
|
public List<ParsedTestSetup> TestSetups { get; set; } = new List<ParsedTestSetup>();
|
||
|
|
public List<ParsedHardware> Hardware { get; set; } = new List<ParsedHardware>();
|
||
|
|
}
|
||
|
|
|
||
|
|
internal class ParsedSensor
|
||
|
|
{
|
||
|
|
public string Name { get; set; }
|
||
|
|
public string SerialNumber { get; set; }
|
||
|
|
public string ChannelCode { get; set; }
|
||
|
|
public double CalibrationFactor { get; set; }
|
||
|
|
public string EngineeringUnits { get; set; }
|
||
|
|
public double Sensitivity { get; set; }
|
||
|
|
}
|
||
|
|
|
||
|
|
internal class ParsedTestSetup { }
|
||
|
|
internal class ParsedHardware { }
|
||
|
|
}
|
||
|
|
```
|
||
|
|
|
||
|
|
### 2. Create Import Options Class
|
||
|
|
**File:** `Common/DTS.Common.Import/ImportOptions/{FormatName}ImportOptions.cs`
|
||
|
|
|
||
|
|
```csharp
|
||
|
|
using System;
|
||
|
|
|
||
|
|
namespace DTS.Common.Import.ImportOptions
|
||
|
|
{
|
||
|
|
public class {FORMAT_NAME}ImportOptions
|
||
|
|
{
|
||
|
|
public bool ImportSensors { get; set; } = true;
|
||
|
|
public bool ImportTestSetups { get; set; } = true;
|
||
|
|
public bool ImportHardware { get; set; } = false;
|
||
|
|
public bool CreateGroups { get; set; } = true;
|
||
|
|
public bool ValidateData { get; set; } = true;
|
||
|
|
|
||
|
|
// Format-specific options
|
||
|
|
public string DateTimeFormat { get; set; } = "yyyy-MM-dd HH:mm:ss";
|
||
|
|
public string Delimiter { get; set; } = ",";
|
||
|
|
public bool HasHeaderRow { get; set; } = true;
|
||
|
|
public int SkipRows { get; set; } = 0;
|
||
|
|
|
||
|
|
public void Validate()
|
||
|
|
{
|
||
|
|
if (string.IsNullOrEmpty(Delimiter))
|
||
|
|
throw new ArgumentException("Delimiter is required");
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
```
|
||
|
|
|
||
|
|
### 3. Update Import Formats Enum
|
||
|
|
**File:** `Common/DTS.Common.Import/Enums/ImportFormats.cs`
|
||
|
|
|
||
|
|
```csharp
|
||
|
|
namespace DTS.Common.Import.Enums
|
||
|
|
{
|
||
|
|
public enum ImportFormats
|
||
|
|
{
|
||
|
|
NOT_SPECIFIED,
|
||
|
|
CSV,
|
||
|
|
EQX,
|
||
|
|
DTS_XML,
|
||
|
|
{FORMAT_NAME} // Add new format
|
||
|
|
}
|
||
|
|
|
||
|
|
public enum ImportFileFormat
|
||
|
|
{
|
||
|
|
NoTestSetup,
|
||
|
|
SingleTestSetup,
|
||
|
|
MultipleTestSetup
|
||
|
|
}
|
||
|
|
}
|
||
|
|
```
|
||
|
|
|
||
|
|
### 4. Create Parser Factory Registration
|
||
|
|
**File:** `Common/DTS.Common.Import/Factories/{FormatName}ParserFactory.cs`
|
||
|
|
|
||
|
|
```csharp
|
||
|
|
using System;
|
||
|
|
using DTS.Common.Import.ImportOptions;
|
||
|
|
using DTS.Common.Import.Parsers;
|
||
|
|
using DTS.Common.Interface.Sensors;
|
||
|
|
using DTS.SensorDB;
|
||
|
|
|
||
|
|
namespace DTS.Common.Import.Factories
|
||
|
|
{
|
||
|
|
public class {FORMAT_NAME}ParserFactory
|
||
|
|
{
|
||
|
|
public static {FORMAT_NAME}Parser Create(
|
||
|
|
IImportNotification importNotification,
|
||
|
|
User user,
|
||
|
|
{FORMAT_NAME}ImportOptions options)
|
||
|
|
{
|
||
|
|
if (options == null)
|
||
|
|
{
|
||
|
|
options = new {FORMAT_NAME}ImportOptions();
|
||
|
|
}
|
||
|
|
|
||
|
|
return new {FORMAT_NAME}Parser(importNotification, user, options);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
```
|
||
|
|
|
||
|
|
### 5. Add File Detection Logic
|
||
|
|
**File:** `Common/DTS.Common.Import/ImportObject.cs` (modify)
|
||
|
|
|
||
|
|
Add method to detect format from file:
|
||
|
|
```csharp
|
||
|
|
public static ImportFormats DetectFormat(string filePath)
|
||
|
|
{
|
||
|
|
var extension = Path.GetExtension(filePath).ToLowerInvariant();
|
||
|
|
|
||
|
|
switch (extension)
|
||
|
|
{
|
||
|
|
case ".csv":
|
||
|
|
return ImportFormats.CSV;
|
||
|
|
case ".eqx":
|
||
|
|
return ImportFormats.EQX;
|
||
|
|
case ".{FORMAT_EXTENSION}":
|
||
|
|
return ImportFormats.{FORMAT_NAME};
|
||
|
|
default:
|
||
|
|
// Check file content for format signature
|
||
|
|
return DetectFormatFromContent(filePath);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
private static ImportFormats DetectFormatFromContent(string filePath)
|
||
|
|
{
|
||
|
|
// Read first few lines to detect format
|
||
|
|
var firstLine = File.ReadLines(filePath).FirstOrDefault();
|
||
|
|
|
||
|
|
if (firstLine != null)
|
||
|
|
{
|
||
|
|
// Check for format-specific signatures
|
||
|
|
if (firstLine.StartsWith("{FORMAT_SIGNATURE}"))
|
||
|
|
{
|
||
|
|
return ImportFormats.{FORMAT_NAME};
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
return ImportFormats.NOT_SPECIFIED;
|
||
|
|
}
|
||
|
|
```
|
||
|
|
|
||
|
|
### 6. Create Unit Tests
|
||
|
|
**File:** `Common/DTS.Common.Tests/{FormatName}ParserShould.cs`
|
||
|
|
|
||
|
|
```csharp
|
||
|
|
using DTS.Common.Import;
|
||
|
|
using DTS.Common.Import.ImportOptions;
|
||
|
|
using NUnit.Framework;
|
||
|
|
using System;
|
||
|
|
using System.IO;
|
||
|
|
|
||
|
|
namespace DTS.Common.Tests
|
||
|
|
{
|
||
|
|
[TestFixture]
|
||
|
|
public class {FORMAT_NAME}ParserShould
|
||
|
|
{
|
||
|
|
private {FORMAT_NAME}Parser _parser;
|
||
|
|
private {FORMAT_NAME}ImportOptions _options;
|
||
|
|
|
||
|
|
[SetUp]
|
||
|
|
public void Setup()
|
||
|
|
{
|
||
|
|
_options = new {FORMAT_NAME}ImportOptions();
|
||
|
|
_parser = new {FORMAT_NAME}Parser(null, null, _options);
|
||
|
|
}
|
||
|
|
|
||
|
|
[Test]
|
||
|
|
public void Parse_ShouldReturnEmptyImportObject_WhenFileNameIsNull()
|
||
|
|
{
|
||
|
|
// Arrange
|
||
|
|
var importObject = new ImportObject();
|
||
|
|
_parser.FileName = null;
|
||
|
|
|
||
|
|
// Act
|
||
|
|
_parser.Parse(ref importObject);
|
||
|
|
|
||
|
|
// Assert
|
||
|
|
Assert.IsNotNull(importObject);
|
||
|
|
}
|
||
|
|
|
||
|
|
[Test]
|
||
|
|
public void Parse_ShouldThrowArgumentNullException_WhenImportObjectIsNull()
|
||
|
|
{
|
||
|
|
// Arrange
|
||
|
|
ImportObject importObject = null;
|
||
|
|
|
||
|
|
// Act & Assert
|
||
|
|
Assert.Throws<ArgumentNullException>(() => _parser.Parse(ref importObject));
|
||
|
|
}
|
||
|
|
|
||
|
|
[Test]
|
||
|
|
public void Parse_ShouldImportSensors_WhenValidFile()
|
||
|
|
{
|
||
|
|
// Arrange
|
||
|
|
var importObject = new ImportObject();
|
||
|
|
var testFile = CreateTestFile();
|
||
|
|
_parser.FileName = testFile;
|
||
|
|
|
||
|
|
// Act
|
||
|
|
_parser.Parse(ref importObject);
|
||
|
|
|
||
|
|
// Assert
|
||
|
|
// Add assertions for expected data
|
||
|
|
|
||
|
|
// Cleanup
|
||
|
|
File.Delete(testFile);
|
||
|
|
}
|
||
|
|
|
||
|
|
private string CreateTestFile()
|
||
|
|
{
|
||
|
|
var path = Path.GetTempFileName();
|
||
|
|
// Write test content
|
||
|
|
File.WriteAllText(path, "test content");
|
||
|
|
return path;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
```
|
||
|
|
|
||
|
|
## Files to Create/Modify Summary
|
||
|
|
|
||
|
|
| File | Action |
|
||
|
|
|------|--------|
|
||
|
|
| `Parsers/{FormatName}/{FormatName}Parser.cs` | Create |
|
||
|
|
| `ImportOptions/{FormatName}ImportOptions.cs` | Create |
|
||
|
|
| `Factories/{FormatName}ParserFactory.cs` | Create |
|
||
|
|
| `Enums/ImportFormats.cs` | Modify (add enum value) |
|
||
|
|
| `ImportObject.cs` | Modify (add detection logic) |
|
||
|
|
| `DTS.Common.Tests/{FormatName}ParserShould.cs` | Create |
|
||
|
|
|
||
|
|
## Parser Base Class Reference
|
||
|
|
|
||
|
|
```csharp
|
||
|
|
public abstract class ParseVariantBase
|
||
|
|
{
|
||
|
|
public string FileName { get; set; }
|
||
|
|
public abstract void Parse(ref ImportObject importObject);
|
||
|
|
}
|
||
|
|
```
|
||
|
|
|
||
|
|
## ImportObject Key Methods
|
||
|
|
|
||
|
|
```csharp
|
||
|
|
// Adding data to import object
|
||
|
|
void AddSensor(SensorData sensor);
|
||
|
|
void AddTestSetup(TestTemplate template);
|
||
|
|
void AddHardware(DASHardware hardware);
|
||
|
|
void AddCalibration(SensorCalibration calibration);
|
||
|
|
|
||
|
|
// Error handling
|
||
|
|
void AddError(ImportError error);
|
||
|
|
IEnumerable<ImportError> Errors();
|
||
|
|
|
||
|
|
// Format detection
|
||
|
|
ImportFormats GetImportFileFormat();
|
||
|
|
```
|
||
|
|
|
||
|
|
## Validation Checklist
|
||
|
|
|
||
|
|
- [ ] Parser inherits from `ParseVariantBase`
|
||
|
|
- [ ] `Parse()` method validates null inputs
|
||
|
|
- [ ] File existence checked before parsing
|
||
|
|
- [ ] Errors added to `ImportObject` for failures
|
||
|
|
- [ ] Progress notification via `IImportNotification`
|
||
|
|
- [ ] Source format set on `ImportObject`
|
||
|
|
- [ ] Import options class with validation
|
||
|
|
- [ ] Format added to `ImportFormats` enum
|
||
|
|
- [ ] File detection logic implemented
|
||
|
|
- [ ] Unit tests for happy path and error cases
|
||
|
|
- [ ] Test file cleanup in tests
|
||
|
|
|
||
|
|
## Common Patterns
|
||
|
|
|
||
|
|
1. **Dependency Injection:** Parser receives notification and user objects
|
||
|
|
2. **Error Accumulation:** Add errors to ImportObject rather than throwing
|
||
|
|
3. **Progress Notification:** Call `NotifyProgress()` during long operations
|
||
|
|
4. **File Validation:** Check existence before reading
|
||
|
|
5. **Format Detection:** Check extension and content signature
|
||
|
|
|
||
|
|
## Supported Data Types
|
||
|
|
|
||
|
|
The `ImportObject` can hold:
|
||
|
|
- `SensorData` - Sensor configurations
|
||
|
|
- `TestTemplate` - Test setup definitions
|
||
|
|
- `DASHardware` - Data acquisition hardware
|
||
|
|
- `SensorCalibration` - Calibration data
|
||
|
|
- `Group` - Test object groups
|
||
|
|
- `ISO.CustomerDetails` - Customer information
|