Improved error messages when reading and validating an invalid grid

Author: staudtMarius

CHANGELOG.md

@@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Changed
 - Updated dependabot workflow and added CODEOWNERS [#1328](https://github.com/ie3-institute/PowerSystemDataModel/issues/1328)
 - Extend azimuth angle range to [-180°, 180°] for PV inputs [#1330](https://github.com/ie3-institute/PowerSystemDataModel/issues/1330)
+- Improved error messages when reading and validating an invalid grid [#1354](https://github.com/ie3-institute/PowerSystemDataModel/issues/1354)
 
 ## [7.0.0] - 2025-05-08
 

src/main/java/edu/ie3/datamodel/io/source/csv/CsvDataSource.java

@@ -79,8 +79,19 @@ public Optional<Set<String>> getSourceFields(Class<? extends Entity> entityClass
    */
   public Optional<Set<String>> getSourceFields(Path filePath) throws SourceException {
     try (BufferedReader reader = connector.initReader(filePath)) {
-      return Optional.of(
-          Arrays.stream(parseCsvRow(reader.readLine(), csvSep)).collect(Collectors.toSet()));
+      String line = reader.readLine();
+      String[] headline = parseCsvRow(line, csvSep);
+
+      if (headline.length <= 1) {
+        throw new SourceException(
+            "The given file has less than two columns! (Used separator '"
+                + csvSep
+                + "' on headline '"
+                + line
+                + "')");
+      }
+
+      return Optional.of(Arrays.stream(headline).collect(Collectors.toSet()));
     } catch (FileNotFoundException e) {
       // A file not existing can be acceptable in many cases, and is handled elsewhere.
       log.debug("The source for the given entity couldn't be found! Cause: {}", e.getMessage());
@@ -208,36 +219,40 @@ protected Set<Path> getTimeSeriesFilePaths(Pattern pattern) {
    */
   protected Map<String, String> buildFieldsToAttributes(
       final String csvRow, final String[] headline) throws SourceException {
-
-    TreeMap<String, String> insensitiveFieldsToAttributes =
-        new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
-
+    // parse row
     String[] fieldVals = parseCsvRow(csvRow, csvSep);
-    insensitiveFieldsToAttributes.putAll(
-        IntStream.range(0, Math.min(fieldVals.length, headline.length))
-            .boxed()
-            .collect(
-                Collectors.toMap(
-                    k -> StringUtils.snakeCaseToCamelCase(headline[k]), v -> fieldVals[v])));
 
+    // check if the number row elements matched the number of headline elements
     if (fieldVals.length != headline.length) {
+      String headlineElements = "['" + String.join("', '", headline) + "']";
+      String parsedRow = "['" + String.join("', '", fieldVals) + "']";
+
       throw new SourceException(
           "The size of the headline ("
               + headline.length
               + ") does not fit to the size of the attribute fields ("
               + fieldVals.length
               + ").\nHeadline: "
-              + String.join(", ", headline)
-              + "\nRow: "
-              + csvRow.trim()
+              + headlineElements
+              + "\nParsed row: "
+              + parsedRow
               + ".\nPlease check:"
-              + "\n - is the csv separator in the file matching the separator provided in the constructor ('"
+              + "\n - is the csv separator in the row matching the provided separator '"
               + csvSep
-              + "')"
+              + "'"
               + "\n - does the number of columns match the number of headline fields "
               + "\n - are you using a valid RFC 4180 formatted csv row?");
     }
 
+    TreeMap<String, String> insensitiveFieldsToAttributes =
+        new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
+    insensitiveFieldsToAttributes.putAll(
+        IntStream.range(0, Math.min(fieldVals.length, headline.length))
+            .boxed()
+            .collect(
+                Collectors.toMap(
+                    k -> StringUtils.snakeCaseToCamelCase(headline[k]), v -> fieldVals[v])));
+
     if (insensitiveFieldsToAttributes.size() != fieldVals.length) {
       throw new SourceException(
           "There might be duplicate headline elements.\nHeadline: "
@@ -298,7 +313,9 @@ protected Try<Stream<Map<String, String>>, SourceException> buildStreamWithField
       // is wanted to avoid a lock on the file), but this causes a closing of the stream as well.
       // As we still want to consume the data at other places, we start a new stream instead of
       // returning the original one
-      return csvRowFieldValueMapping(reader, headline);
+      return csvRowFieldValueMapping(reader, headline, filePath.getFileName())
+          .transformF(
+              e -> new SourceException("The file '" + filePath + "' could not be parsed.", e));
     } catch (FileNotFoundException e) {
       if (allowFileNotExisting) {
         log.warn("Unable to find file '{}': {}", filePath, e.getMessage());
@@ -324,10 +341,11 @@ private Try<Path, SourceException> getFilePath(Class<? extends Entity> entityCla
    *
    * @param reader for the file
    * @param headline of the file
+   * @param fileName the name of the file, that is read
    * @return a list of mapping
    */
   protected Try<Stream<Map<String, String>>, SourceException> csvRowFieldValueMapping(
-      BufferedReader reader, String[] headline) {
+      BufferedReader reader, String[] headline, Path fileName) {
     return Try.scanStream(
             reader
                 .lines()
@@ -337,7 +355,7 @@ protected Try<Stream<Map<String, String>>, SourceException> csvRowFieldValueMapp
                         Try.of(
                             () -> buildFieldsToAttributes(csvRow, headline),
                             SourceException.class)),
-            "Map<String, String>")
+            fileName.toString())
         .transform(
             stream -> stream.filter(map -> !map.isEmpty()),
             e -> new SourceException("Parsing csv row failed.", e));

src/main/java/edu/ie3/datamodel/io/source/csv/CsvIdCoordinateSource.java

@@ -209,7 +209,7 @@ private Collection<Point> getCoordinatesInBoundingBox(
       // is wanted to avoid a lock on the file), but this causes a closing of the stream as well.
       // As we still want to consume the data at other places, we start a new stream instead of
       // returning the original one
-      return dataSource.csvRowFieldValueMapping(reader, headline);
+      return dataSource.csvRowFieldValueMapping(reader, headline, filePath.getFileName());
     } catch (IOException e) {
       return Failure.of(
           new SourceException("Cannot read the file for coordinate id to coordinate mapping.", e));

src/main/java/edu/ie3/datamodel/io/source/csv/CsvWeatherSource.java

@@ -177,9 +177,11 @@ private Map<Point, IndividualTimeSeries<WeatherValue>> readWeatherTimeSeries(
         this::buildWeatherValue;
     /* Reading in weather time series */
     for (CsvIndividualTimeSeriesMetaInformation data : weatherMetaInformation) {
+      Path path = data.getFullFilePath();
+
       // we need a reader for each file
-      try (BufferedReader reader = connector.initReader(data.getFullFilePath())) {
-        buildStreamWithFieldsToAttributesMap(reader)
+      try (BufferedReader reader = connector.initReader(path)) {
+        buildStreamWithFieldsToAttributesMap(reader, path.getFileName())
             .getOrThrow()
             .map(fieldToValueFunction)
             .flatMap(Optional::stream)
@@ -227,7 +229,7 @@ private Map<Point, IndividualTimeSeries<WeatherValue>> readWeatherTimeSeries(
   }
 
   private Try<Stream<Map<String, String>>, SourceException> buildStreamWithFieldsToAttributesMap(
-      BufferedReader bufferedReader) throws ValidationException {
+      BufferedReader bufferedReader, Path fileName) throws ValidationException {
     Class<? extends Entity> entityClass = TimeBasedValue.class;
 
     try (BufferedReader reader = bufferedReader) {
@@ -240,7 +242,7 @@ private Try<Stream<Map<String, String>>, SourceException> buildStreamWithFieldsT
       // is wanted to avoid a lock on the file), but this causes a closing of the stream as well.
       // As we still want to consume the data at other places, we start a new stream instead of
       // returning the original one
-      return dataSource.csvRowFieldValueMapping(reader, headline);
+      return dataSource.csvRowFieldValueMapping(reader, headline, fileName);
     } catch (IOException e) {
       return Failure.of(
           new SourceException(

src/main/java/edu/ie3/datamodel/utils/Try.java

@@ -67,6 +67,20 @@ public static <E extends Exception> Try<Void, E> ofVoid(
     }
   }
 
+  /**
+   * Method to create multiple {@link Try} object easily.
+   *
+   * @param suppliers that either return no data or throw an exception
+   * @param clazz class of the exception
+   * @return a collection of try objects
+   * @param <E> type of exception that could be thrown
+   */
+  @SafeVarargs
+  public static <E extends Exception> Collection<Try<Void, E>> ofVoids(
+      Class<E> clazz, VoidSupplier<E>... suppliers) {
+    return Arrays.stream(suppliers).map(supplier -> ofVoid(supplier, clazz)).toList();
+  }
+
   /**
    * Method to create a {@link Try} object easily.
    *

src/main/java/edu/ie3/datamodel/utils/validation/GridContainerValidationUtils.java

@@ -99,10 +99,15 @@ private GridContainerValidationUtils() {
 
     /* sanity check to ensure uniqueness */
     List<Try<Void, ? extends ValidationException>> exceptions = new ArrayList<>();
-    exceptions.add(
-        Try.ofVoid(
-            () -> checkAssetUniqueness(rawGridElements.allEntitiesAsList()),
-            DuplicateEntitiesException.class));
+    exceptions.addAll(
+        Try.ofVoids(
+            DuplicateEntitiesException.class,
+            () -> checkAssetUniqueness(rawGridElements.getNodes()),
+            () -> checkAssetUniqueness(rawGridElements.getLines()),
+            () -> checkAssetUniqueness(rawGridElements.getSwitches()),
+            () -> checkAssetUniqueness(rawGridElements.getTransformer2Ws()),
+            () -> checkAssetUniqueness(rawGridElements.getTransformer3Ws()),
+            () -> checkAssetUniqueness(rawGridElements.getMeasurementUnits())));
 
     /* Checking nodes */
     Set<NodeInput> nodes = rawGridElements.getNodes();
@@ -287,10 +292,6 @@ protected static Try<Void, InvalidGridException> checkConnectivity(
 
     /* sanity check to ensure uniqueness */
     List<Try<Void, ? extends ValidationException>> exceptions = new ArrayList<>();
-    exceptions.add(
-        Try.ofVoid(
-            () -> checkAssetUniqueness(systemParticipants.allEntitiesAsList()),
-            DuplicateEntitiesException.class));
 
     exceptions.addAll(checkSystemParticipants(systemParticipants.getBmPlants(), nodes));
     exceptions.addAll(checkSystemParticipants(systemParticipants.getChpPlants(), nodes));
@@ -306,8 +307,8 @@ protected static Try<Void, InvalidGridException> checkConnectivity(
   }
 
   /**
-   * Checks the validity of specific system participant. Moreover, it checks, if the systems are
-   * connected to a node that is not in the provided set
+   * Checks the validity and uniqueness of specific system participant. Moreover, it checks, if the
+   * systems are connected to a node that is not in the provided set
    *
    * @param participants a set of specific system participants
    * @param nodes Set of already known nodes
@@ -316,18 +317,17 @@ protected static Try<Void, InvalidGridException> checkConnectivity(
    */
   protected static List<Try<Void, ? extends ValidationException>> checkSystemParticipants(
       Set<? extends SystemParticipantInput> participants, Set<NodeInput> nodes) {
-    return participants.stream()
-        .map(
-            entity -> {
-              List<Try<Void, ? extends ValidationException>> exceptions = new ArrayList<>();
-
-              exceptions.add(checkNodeAvailability(entity, nodes));
-              exceptions.addAll(SystemParticipantValidationUtils.check(entity));
-
-              return exceptions;
-            })
-        .flatMap(List::stream)
-        .toList();
+    List<Try<Void, ? extends ValidationException>> exceptions = new ArrayList<>();
+    exceptions.add(
+        Try.ofVoid(() -> checkAssetUniqueness(participants), DuplicateEntitiesException.class));
+
+    participants.forEach(
+        participant -> {
+          exceptions.add(checkNodeAvailability(participant, nodes));
+          exceptions.addAll(SystemParticipantValidationUtils.check(participant));
+        });
+
+    return exceptions;
   }
 
   /**

src/main/java/edu/ie3/datamodel/utils/validation/UniquenessValidationUtils.java

@@ -24,10 +24,11 @@
 public class UniquenessValidationUtils extends ValidationUtils {
 
   // default field set supplier
-  protected static final FieldSetSupplier<UniqueEntity> uuidFieldSupplier =
+  protected static final FieldSetSupplier<? extends UniqueEntity> uuidFieldSupplier =
       entity -> Set.of(entity.getUuid());
-  protected static final FieldSetSupplier<AssetInput> idFieldSupplier = e -> Set.of(e.getId());
-  protected static final FieldSetSupplier<ResultEntity> resultFieldSupplier =
+  protected static final FieldSetSupplier<? extends AssetInput> idFieldSupplier =
+      e -> Set.of(e.getId());
+  protected static final FieldSetSupplier<? extends ResultEntity> resultFieldSupplier =
       entity -> Set.of(entity.getTime(), entity.getInputModel());
   protected static final FieldSetSupplier<MappingEntry> mappingFieldSupplier =
       entity -> Set.of(entity.getAsset());
@@ -44,9 +45,10 @@ public class UniquenessValidationUtils extends ValidationUtils {
    * @param entities to be checked
    * @throws DuplicateEntitiesException if uniqueness is violated
    */
-  public static void checkUniqueEntities(Collection<? extends UniqueEntity> entities)
+  @SuppressWarnings("unchecked")
+  public static <E extends UniqueEntity> void checkUniqueEntities(Collection<E> entities)
       throws DuplicateEntitiesException {
-    checkUniqueness(entities, uuidFieldSupplier).getOrThrow();
+    checkUniqueness(entities, (FieldSetSupplier<? super E>) uuidFieldSupplier).getOrThrow();
   }
 
   /**
@@ -55,13 +57,14 @@ public static void checkUniqueEntities(Collection<? extends UniqueEntity> entiti
    * @param entities to be checked
    * @throws DuplicateEntitiesException if uniqueness is violated
    */
-  public static void checkAssetUniqueness(Collection<? extends AssetInput> entities)
+  @SuppressWarnings("unchecked")
+  public static <E extends AssetInput> void checkAssetUniqueness(Collection<E> entities)
       throws DuplicateEntitiesException {
 
     List<DuplicateEntitiesException> exceptions =
         Try.getExceptions(
             Try.ofVoid(() -> checkUniqueEntities(entities), DuplicateEntitiesException.class),
-            checkUniqueness(entities, idFieldSupplier));
+            checkUniqueness(entities, (FieldSetSupplier<? super E>) idFieldSupplier));
 
     if (!exceptions.isEmpty()) {
       throw new DuplicateEntitiesException("AssetInput", exceptions);
@@ -74,9 +77,10 @@ public static void checkAssetUniqueness(Collection<? extends AssetInput> entitie
    * @param entities to be checked
    * @throws DuplicateEntitiesException if uniqueness is violated
    */
-  public static void checkResultUniqueness(Collection<? extends ResultEntity> entities)
+  @SuppressWarnings("unchecked")
+  public static <E extends ResultEntity> void checkResultUniqueness(Collection<E> entities)
       throws DuplicateEntitiesException {
-    checkUniqueness(entities, resultFieldSupplier).getOrThrow();
+    checkUniqueness(entities, (FieldSetSupplier<? super E>) resultFieldSupplier).getOrThrow();
   }
 
   /**
@@ -122,33 +126,46 @@ public static void checkWeatherUniqueness(Collection<TimeBasedValue<WeatherValue
    */
   private static <E extends Entity> Try<Void, DuplicateEntitiesException> checkUniqueness(
       Collection<? extends E> entities, FieldSetSupplier<E> supplier) {
+    Optional<String> option = entities.stream().findAny().map(e -> e.getClass().getSimpleName());
+    if (option.isPresent()) {
+      return checkUniqueness(entities, supplier, option.get());
+    } else {
+      return Try.Success.empty();
+    }
+  }
+
+  /**
+   * Checking the uniqueness for a given {@link Entity}.
+   *
+   * @param entities to be checked
+   * @param supplier for the field set
+   * @param entityName name of the class of the entity
+   * @return a try object
+   * @param <E> type of entity
+   */
+  private static <E extends Entity> Try<Void, DuplicateEntitiesException> checkUniqueness(
+      Collection<? extends E> entities, FieldSetSupplier<E> supplier, String entityName) {
     if (entities.size() < 2) {
       return Success.empty();
     }
 
-    return entities.stream()
-        .findAny()
-        .map(
-            entity -> {
-              List<Set<Object>> elements = entities.stream().map(supplier::getFieldSets).toList();
-              Set<Set<Object>> uniqueElements = new HashSet<>(elements);
-
-              return Try.ofVoid(
-                  elements.size() != uniqueElements.size(),
-                  () -> buildDuplicationException(entity.getClass(), elements));
-            })
-        .orElse(Success.empty());
+    List<Set<Object>> elements = entities.stream().map(supplier::getFieldSets).toList();
+    Set<Set<Object>> uniqueElements = new HashSet<>(elements);
+
+    return Try.ofVoid(
+        elements.size() != uniqueElements.size(),
+        () -> buildDuplicationException(entityName, elements));
   }
 
   /**
    * Method for building a {@link DuplicateEntitiesException}.
    *
-   * @param entityClass class of the entity
+   * @param entityClass name of the class of the entity
    * @param notUniqueElements list of not unique elements
    * @return a {@link DuplicateEntitiesException}
    */
   protected static DuplicateEntitiesException buildDuplicationException(
-      Class<? extends Entity> entityClass, List<Set<Object>> notUniqueElements) {
+      String entityClass, List<Set<Object>> notUniqueElements) {
     String fieldName =
         notUniqueElements.get(0).stream()
             .map(f -> f.getClass().getSimpleName())
@@ -167,7 +184,7 @@ protected static DuplicateEntitiesException buildDuplicationException(
 
     return new DuplicateEntitiesException(
         "'"
-            + entityClass.getSimpleName()
+            + entityClass
             + "' entities with duplicated "
             + fieldName
             + " key, but different field "