Add weather interpolation

CHANGELOG.md

@@ -8,7 +8,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Fixed
 - Fixed handling of `CongestionResult.InputModelType` in `EntityProcessor` [#1325](https://github.com/ie3-institute/PowerSystemDataModel/issues/1325)
+- Consider None-equivalent value for missing data points in weather [#1304](https://github.com/ie3-institute/PowerSystemDataModel/issues/1304)
 - -Fixed em fields in input models [#1331](https://github.com/ie3-institute/PowerSystemDataModel/issues/1331)
+- Consider None-equivalent value for missing data points in weather [#1304](https://github.com/ie3-institute/PowerSystemDataModel/issues/1304)
 
 ### Changed
 - Updated dependabot workflow and added CODEOWNERS [#1328](https://github.com/ie3-institute/PowerSystemDataModel/issues/1328)

src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java

@@ -2,7 +2,7 @@
  * © 2021. TU Dortmund University,
  * Institute of Energy Systems, Energy Efficiency and Energy Economics,
  * Research group Distribution grid planning and operation
-*/
+ */
 package edu.ie3.datamodel.io.source;
 
 import edu.ie3.datamodel.exceptions.SourceException;
@@ -14,14 +14,17 @@
 import edu.ie3.datamodel.models.value.WeatherValue;
 import edu.ie3.datamodel.utils.Try;
 import edu.ie3.util.interval.ClosedInterval;
+import java.time.Duration;
 import java.time.ZonedDateTime;
 import java.util.*;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
+import javax.measure.Quantity;
 import org.apache.commons.lang3.tuple.Pair;
 import org.locationtech.jts.geom.Point;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import tech.units.indriya.ComparableQuantity;
 
 /** Abstract class for WeatherSource by Csv and Sql Data */
 public abstract class WeatherSource extends EntitySource {
@@ -35,7 +38,7 @@ public abstract class WeatherSource extends EntitySource {
   protected static final String COORDINATE_ID = "coordinateid";
 
   protected WeatherSource(
-      IdCoordinateSource idCoordinateSource, TimeBasedWeatherValueFactory weatherFactory) {
+          IdCoordinateSource idCoordinateSource, TimeBasedWeatherValueFactory weatherFactory) {
     this.idCoordinateSource = idCoordinateSource;
     this.weatherFactory = weatherFactory;
   }
@@ -52,26 +55,162 @@ public void validate() throws ValidationException {
     validate(WeatherValue.class, this::getSourceFields, weatherFactory);
   }
 
+  /**
+   * Method for interpolating weather values.
+   *
+   * @return a new quantity
+   */
+  protected List<TimeBasedValue<WeatherValue>> interpolateMissingValues(
+          List<TimeBasedValue<WeatherValue>> timeSeries) {
+
+    List<TimeBasedValue<WeatherValue>> result = new ArrayList<>();
+    int i = 0;
+    while (i < timeSeries.size()) {
+      TimeBasedValue<WeatherValue> current = timeSeries.get(i);
+
+      if (current.getValue() != null) {
+        result.add(current);
+        i++;
+        continue;
+      }
+      int prevIdx = i - 1;
+      int nextIdx = i + 1;
+      while (nextIdx < timeSeries.size() && timeSeries.get(nextIdx).getValue() == null) {
+        nextIdx++;
+      }
+
+      if (prevIdx >= 0 && nextIdx < timeSeries.size()) {
+        TimeBasedValue<WeatherValue> prev = timeSeries.get(prevIdx);
+        TimeBasedValue<WeatherValue> next = timeSeries.get(nextIdx);
+        Duration total = Duration.between(prev.getTime(), next.getTime());
+        for (int j = i; j < nextIdx; j++) {
+          TimeBasedValue<WeatherValue> missing = timeSeries.get(j);
+          Duration fromPrev = Duration.between(prev.getTime(), missing.getTime());
+          double ratio = (double) fromPrev.toSeconds() / total.toSeconds();
+          WeatherValue interpolated =
+                  interpolateWeatherValue(prev.getValue(), next.getValue(), ratio);
+          result.add(new TimeBasedValue<>(missing.getTime(), interpolated));
+        }
+        i = nextIdx;
+      } else {
+        result.add(current);
+        i++;
+      }
+    }
+
+    return result;
+  }
+
+  private WeatherValue interpolateWeatherValue(WeatherValue start, WeatherValue end, double ratio) {
+    var direct = interpolateOptional(start.getDirectIrradiance(), end.getDirectIrradiance(), ratio);
+    var diffuse = interpolateOptional(start.getDiffuseIrradiance(), end.getDiffuseIrradiance(), ratio);
+
+    var temp = interpolateDirect(start.getTemperature(), end.getTemperature(), ratio);
+    var dir = interpolateDirect(start.getWindDirection(), end.getWindDirection(), ratio);
+    var vel = interpolateDirect(start.getWindVelocity(), end.getWindVelocity(), ratio);
+
+    return new WeatherValue(start.getCoordinate(), direct, diffuse, temp, dir, vel);
+  }
+
+
+  private <Q extends Quantity<Q>> ComparableQuantity<Q> interpolateOptional(
+          Optional<ComparableQuantity<Q>> startOpt,
+          Optional<ComparableQuantity<Q>> endOpt,
+          double ratio) {
+    return startOpt
+            .flatMap(startVal -> endOpt.map(endVal -> interpolateQuantity(startVal, endVal, ratio)))
+            .orElse(null);
+  }
+
+  private <Q extends Quantity<Q>> ComparableQuantity<Q> interpolateDirect(
+          ComparableQuantity<Q> start, ComparableQuantity<Q> end, double ratio) {
+    return interpolateQuantity(start, end, ratio);
+  }
+
+  private <Q extends Quantity<Q>> ComparableQuantity<Q> interpolateQuantity(
+          ComparableQuantity<Q> a, ComparableQuantity<Q> b, double ratio) {
+    return a.add(b.subtract(a).multiply(ratio));
+  }
+
   // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
 
   public abstract Map<Point, IndividualTimeSeries<WeatherValue>> getWeather(
-      ClosedInterval<ZonedDateTime> timeInterval) throws SourceException;
+          ClosedInterval<ZonedDateTime> timeInterval) throws SourceException;
 
   public abstract Map<Point, IndividualTimeSeries<WeatherValue>> getWeather(
-      ClosedInterval<ZonedDateTime> timeInterval, Collection<Point> coordinates)
-      throws SourceException;
+          ClosedInterval<ZonedDateTime> timeInterval, Collection<Point> coordinates)
+          throws SourceException;
 
   public abstract Optional<TimeBasedValue<WeatherValue>> getWeather(
-      ZonedDateTime date, Point coordinate) throws SourceException;
+          ZonedDateTime date, Point coordinate) throws SourceException;
+
+  public Optional<WeatherValue> getWeatherInterpolated(
+          ZonedDateTime date, Point coordinate, int plus, int minus) throws SourceException {
+
+    ClosedInterval<ZonedDateTime> interpolationInterval =
+            new ClosedInterval<>(date.minusHours(minus), date.plusHours(plus));
+    IndividualTimeSeries<WeatherValue> ts =
+            getWeather(interpolationInterval, List.of(coordinate)).get(coordinate);
+
+    if (ts == null) {
+      log.warn("No time series available for coordinate {}", coordinate);
+      return Optional.empty();
+    }
+
+    Optional<WeatherValue> value = ts.getValue(date);
+
+    if (value.isPresent() && value.get().isComplete()) {
+      return value;
+    }
+
+    Optional<TimeBasedValue<WeatherValue>> prevValue = ts.getPreviousTimeBasedValue(date);
+    Optional<TimeBasedValue<WeatherValue>> nextValue = ts.getNextTimeBasedValue(date);
+
+    if (prevValue.isEmpty() || nextValue.isEmpty()) {
+      log.warn(
+              "Not enough data to interpolate weather value at {} for coordinate {}", date, coordinate);
+      return Optional.empty();
+    }
+
+    TimeBasedValue<WeatherValue> prev = prevValue.get();
+    TimeBasedValue<WeatherValue> next = nextValue.get();
+
+    Duration totalDuration = Duration.between(prev.getTime(), next.getTime());
+    Duration partialDuration = Duration.between(prev.getTime(), date);
+
+    if (totalDuration.isZero()) {
+      return Optional.of(prev.getValue());
+    }
+
+    double ratio = (double) partialDuration.toSeconds() / totalDuration.toSeconds();
+
+    WeatherValue interpolated = interpolateWeatherValue(prev.getValue(), next.getValue(), ratio);
+
+    return Optional.of(interpolated);
+  }
+
 
   public abstract Map<Point, List<ZonedDateTime>> getTimeKeysAfter(ZonedDateTime time)
-      throws SourceException;
+          throws SourceException;
 
   public List<ZonedDateTime> getTimeKeysAfter(ZonedDateTime time, Point coordinate)
-      throws SourceException {
+          throws SourceException {
     return getTimeKeysAfter(time).getOrDefault(coordinate, Collections.emptyList());
   }
 
+  protected Map<Point, IndividualTimeSeries<WeatherValue>> interpolateWeatherData(
+          Map<Point, IndividualTimeSeries<WeatherValue>> rawData) {
+    return rawData.entrySet().stream()
+            .collect(
+                    Collectors.toMap(
+                            Map.Entry::getKey,
+                            entry ->
+                                    new IndividualTimeSeries<>(
+                                            new LinkedHashSet<>(
+                                                    interpolateMissingValues(
+                                                            new ArrayList<>(entry.getValue().getEntries()))))));
+  }
+
   // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
 
   /**
@@ -82,7 +221,7 @@ public List<ZonedDateTime> getTimeKeysAfter(ZonedDateTime time, Point coordinate
    * @return the TimeBasedWeatherValueData
    */
   protected Optional<TimeBasedWeatherValueData> toTimeBasedWeatherValueData(
-      Map<String, String> fieldMap) {
+          Map<String, String> fieldMap) {
     String coordinateValue = fieldMap.remove(COORDINATE_ID);
     fieldMap.putIfAbsent("uuid", UUID.randomUUID().toString());
     int coordinateId = Integer.parseInt(coordinateValue);
@@ -101,16 +240,16 @@ protected Optional<TimeBasedWeatherValueData> toTimeBasedWeatherValueData(
    * @return a map of coordinate point to time series
    */
   protected Map<Point, IndividualTimeSeries<WeatherValue>> mapWeatherValuesToPoints(
-      Collection<TimeBasedValue<WeatherValue>> timeBasedValues) {
+          Collection<TimeBasedValue<WeatherValue>> timeBasedValues) {
     Map<Point, Set<TimeBasedValue<WeatherValue>>> coordinateToValues =
-        timeBasedValues.stream()
-            .collect(
-                Collectors.groupingBy(
-                    timeBasedWeatherValue -> timeBasedWeatherValue.getValue().getCoordinate(),
-                    Collectors.toSet()));
+            timeBasedValues.stream()
+                    .collect(
+                            Collectors.groupingBy(
+                                    timeBasedWeatherValue -> timeBasedWeatherValue.getValue().getCoordinate(),
+                                    Collectors.toSet()));
     Map<Point, IndividualTimeSeries<WeatherValue>> coordinateToTimeSeriesMap = new HashMap<>();
     for (Map.Entry<Point, Set<TimeBasedValue<WeatherValue>>> entry :
-        coordinateToValues.entrySet()) {
+            coordinateToValues.entrySet()) {
       Set<TimeBasedValue<WeatherValue>> values = entry.getValue();
       IndividualTimeSeries<WeatherValue> timeSeries = new IndividualTimeSeries<>(values);
       coordinateToTimeSeriesMap.put(entry.getKey(), timeSeries);
@@ -119,34 +258,34 @@ protected Map<Point, IndividualTimeSeries<WeatherValue>> mapWeatherValuesToPoint
   }
 
   protected Map<Point, List<ZonedDateTime>> toTimeKeys(
-      Stream<Map<String, String>> fieldMaps, TimeBasedWeatherValueFactory factory) {
+          Stream<Map<String, String>> fieldMaps, TimeBasedWeatherValueFactory factory) {
     return groupTime(
-        fieldMaps.map(
-            fieldMap -> {
-              String coordinateValue = fieldMap.get(COORDINATE_ID);
-              int coordinateId = Integer.parseInt(coordinateValue);
-              Optional<Point> coordinate = idCoordinateSource.getCoordinate(coordinateId);
-              ZonedDateTime time = factory.extractTime(fieldMap);
-
-              if (coordinate.isEmpty()) {
-                log.warn("Unable to match coordinate ID {} to a point", coordinateId);
-              }
-              return Pair.of(coordinate, time);
-            }));
+            fieldMaps.map(
+                    fieldMap -> {
+                      String coordinateValue = fieldMap.get(COORDINATE_ID);
+                      int coordinateId = Integer.parseInt(coordinateValue);
+                      Optional<Point> coordinate = idCoordinateSource.getCoordinate(coordinateId);
+                      ZonedDateTime time = factory.extractTime(fieldMap);
+
+                      if (coordinate.isEmpty()) {
+                        log.warn("Unable to match coordinate ID {} to a point", coordinateId);
+                      }
+                      return Pair.of(coordinate, time);
+                    }));
   }
 
   protected Map<Point, List<ZonedDateTime>> groupTime(
-      Stream<Pair<Optional<Point>, ZonedDateTime>> values) {
+          Stream<Pair<Optional<Point>, ZonedDateTime>> values) {
     return values
-        .filter(pair -> pair.getKey().isPresent())
-        .map(pair -> Pair.of(pair.getKey().get(), pair.getValue()))
-        .collect(Collectors.groupingBy(Pair::getKey, Collectors.toSet()))
-        .entrySet()
-        .stream()
-        .collect(
-            Collectors.toMap(
-                Map.Entry::getKey,
-                e -> e.getValue().stream().map(Pair::getValue).sorted().toList()));
+            .filter(pair -> pair.getKey().isPresent())
+            .map(pair -> Pair.of(pair.getKey().get(), pair.getValue()))
+            .collect(Collectors.groupingBy(Pair::getKey, Collectors.toSet()))
+            .entrySet()
+            .stream()
+            .collect(
+                    Collectors.toMap(
+                            Map.Entry::getKey,
+                            e -> e.getValue().stream().map(Pair::getValue).sorted().toList()));
   }
 
   // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
@@ -159,19 +298,19 @@ protected Map<Point, List<ZonedDateTime>> groupTime(
    * @return a list of that TimeBasedValues
    */
   protected List<TimeBasedValue<WeatherValue>> buildTimeBasedValues(
-      TimeBasedWeatherValueFactory factory, Stream<Map<String, String>> inputStream)
-      throws SourceException {
+          TimeBasedWeatherValueFactory factory, Stream<Map<String, String>> inputStream)
+          throws SourceException {
     return Try.scanStream(
-            inputStream.map(
-                fieldsToAttributes -> {
-                  fieldsToAttributes.remove("tid");
-                  Optional<TimeBasedWeatherValueData> data =
-                      toTimeBasedWeatherValueData(fieldsToAttributes);
-                  return factory.get(
-                      Try.from(data, () -> new SourceException("Missing data in: " + data)));
-                }),
-            "TimeBasedValue<WeatherValue>")
-        .transform(Stream::toList, SourceException::new)
-        .getOrThrow();
+                    inputStream.map(
+                            fieldsToAttributes -> {
+                              fieldsToAttributes.remove("tid");
+                              Optional<TimeBasedWeatherValueData> data =
+                                      toTimeBasedWeatherValueData(fieldsToAttributes);
+                              return factory.get(
+                                      Try.from(data, () -> new SourceException("Missing data in: " + data)));
+                            }),
+                    "TimeBasedValue<WeatherValue>")
+            .transform(Stream::toList, SourceException::new)
+            .getOrThrow();
   }
-}
+}