From 0ca4bda06cf917a767b7f5d7022150f8ccc61616 Mon Sep 17 00:00:00 2001 From: pierrepetersmeier <155652256+pierrepetersmeier@users.noreply.github.com> Date: Wed, 14 May 2025 16:14:21 +0200 Subject: [PATCH 1/4] Interpolate functionality and InterpolationTest. --- CHANGELOG.md | 1 + .../datamodel/io/source/WeatherSource.java | 100 ++++++++++++++++++ .../WeatherSourceInterpolationTest.groovy | 62 +++++++++++ 3 files changed, 163 insertions(+) create mode 100644 src/test/groovy/edu/ie3/datamodel/io/source/WeatherSourceInterpolationTest.groovy diff --git a/CHANGELOG.md b/CHANGELOG.md index 064b97b1b..0eadbd330 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - Fixed SonarQube junit path issue in GitHub Actions [#1284](https://github.com/ie3-institute/PowerSystemDataModel/issues/1284) - Fixed no errors thrown in `getMapping()` in `TimeSeriesMappingSource` [#1287](https://github.com/ie3-institute/PowerSystemDataModel/issues/1287) +- Consider None-equivalent value for missing data points in weather [#1304](https://github.com/ie3-institute/PowerSystemDataModel/issues/1304) ### Changed - Replaced `return this` with `return thisInstance` in CopyBuilders [#1250](https://github.com/ie3-institute/PowerSystemDataModel/issues/1250) diff --git a/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java index a7a056d85..60233e4d1 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java @@ -14,14 +14,17 @@ import edu.ie3.datamodel.models.value.WeatherValue; import edu.ie3.datamodel.utils.Try; import edu.ie3.util.interval.ClosedInterval; +import java.time.Duration; import java.time.ZonedDateTime; import java.util.*; import java.util.stream.Collectors; import java.util.stream.Stream; +import javax.measure.Quantity; import org.apache.commons.lang3.tuple.Pair; import org.locationtech.jts.geom.Point; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import tech.units.indriya.ComparableQuantity; /** Abstract class for WeatherSource by Csv and Sql Data */ public abstract class WeatherSource extends EntitySource { @@ -52,6 +55,90 @@ public void validate() throws ValidationException { validate(WeatherValue.class, this::getSourceFields, weatherFactory); } + /** + * Method for interpolating weather values. + * + * @return a new quantity + */ + protected List> interpolateMissingValues( + List> timeSeries) { + + List> result = new ArrayList<>(); + int i = 0; + while (i < timeSeries.size()) { + TimeBasedValue current = timeSeries.get(i); + + if (current.getValue() != null) { + result.add(current); + i++; + continue; + } + int prevIdx = i - 1; + int nextIdx = i + 1; + while (nextIdx < timeSeries.size() && timeSeries.get(nextIdx).getValue() == null) { + nextIdx++; + } + + if (prevIdx >= 0 && nextIdx < timeSeries.size()) { + TimeBasedValue prev = timeSeries.get(prevIdx); + TimeBasedValue next = timeSeries.get(nextIdx); + Duration total = Duration.between(prev.getTime(), next.getTime()); + for (int j = i; j < nextIdx; j++) { + TimeBasedValue missing = timeSeries.get(j); + Duration fromPrev = Duration.between(prev.getTime(), missing.getTime()); + double ratio = (double) fromPrev.toSeconds() / total.toSeconds(); + WeatherValue interpolated = + interpolateWeatherValue(prev.getValue(), next.getValue(), ratio); + result.add(new TimeBasedValue<>(missing.getTime(), interpolated)); + } + i = nextIdx; + } else { + result.add(current); + i++; + } + } + + return result; + } + + private WeatherValue interpolateWeatherValue(WeatherValue start, WeatherValue end, double ratio) { + var startSolar = start.getSolarIrradiance(); + var endSolar = end.getSolarIrradiance(); + + var direct = + interpolateOptional( + startSolar.getDirectIrradiance(), endSolar.getDirectIrradiance(), ratio); + var diffuse = + interpolateOptional( + startSolar.getDiffuseIrradiance(), endSolar.getDiffuseIrradiance(), ratio); + + var temp = interpolateDirect(start.getTemperature(), end.getTemperature(), ratio); + var dir = + interpolateDirect(start.getWind().getDirection(), end.getWind().getDirection(), ratio); + var vel = interpolateDirect(start.getWind().getVelocity(), end.getWind().getVelocity(), ratio); + + return new WeatherValue(start.getCoordinate(), direct, diffuse, temp, dir, vel); + } + + private > ComparableQuantity interpolateOptional( + Optional> startOpt, + Optional> endOpt, + double ratio) { + return startOpt + .flatMap(startVal -> endOpt.map(endVal -> interpolateQuantity(startVal, endVal, ratio))) + .orElse(null); + } + + private > ComparableQuantity interpolateDirect( + ComparableQuantity start, ComparableQuantity end, double ratio) { + return interpolateQuantity(start, end, ratio); + } + + private > ComparableQuantity interpolateQuantity( + ComparableQuantity a, ComparableQuantity b, double ratio) { + return a.add(b.subtract(a).multiply(ratio)); + } + // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- public abstract Map> getWeather( @@ -72,6 +159,19 @@ public List getTimeKeysAfter(ZonedDateTime time, Point coordinate return getTimeKeysAfter(time).getOrDefault(coordinate, Collections.emptyList()); } + protected Map> interpolateWeatherData( + Map> rawData) { + return rawData.entrySet().stream() + .collect( + Collectors.toMap( + Map.Entry::getKey, + entry -> + new IndividualTimeSeries<>( + new LinkedHashSet<>( + interpolateMissingValues( + new ArrayList<>(entry.getValue().getEntries())))))); + } + // -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- /** diff --git a/src/test/groovy/edu/ie3/datamodel/io/source/WeatherSourceInterpolationTest.groovy b/src/test/groovy/edu/ie3/datamodel/io/source/WeatherSourceInterpolationTest.groovy new file mode 100644 index 000000000..bcec92ba0 --- /dev/null +++ b/src/test/groovy/edu/ie3/datamodel/io/source/WeatherSourceInterpolationTest.groovy @@ -0,0 +1,62 @@ +/* + * © 2025. TU Dortmund University, + * Institute of Energy Systems, Energy Efficiency and Energy Economics, + * Research group Distribution grid planning and operation + */ +package edu.ie3.datamodel.io.source + +import edu.ie3.datamodel.models.timeseries.individual.TimeBasedValue +import edu.ie3.datamodel.models.value.WeatherValue +import edu.ie3.util.TimeUtil +import org.locationtech.jts.geom.Coordinate +import org.locationtech.jts.geom.GeometryFactory +import org.locationtech.jts.geom.Point +import spock.lang.Shared +import spock.lang.Specification + +import java.time.ZonedDateTime + +class WeatherSourceInterpolationTest extends Specification { + + @Shared + def geometryFactory = new GeometryFactory() + + def "interpolateMissingValues fills gaps between known weather values"() { + given: + def coordinate = geometryFactory.createPoint(new Coordinate(7.0, 51.0)) + def t1 = TimeUtil.withDefaults.toZonedDateTime("2020-01-01T00:00:00Z") + def t2 = TimeUtil.withDefaults.toZonedDateTime("2020-01-01T01:00:00Z") + def t3 = TimeUtil.withDefaults.toZonedDateTime("2020-01-01T02:00:00Z") + def t4 = TimeUtil.withDefaults.toZonedDateTime("2020-01-01T03:00:00Z") + def value1 = new WeatherValue(coordinate, 100, 200, 10, 5, 180) + def value3 = new WeatherValue(coordinate, 300 , 400, 20, 10, 270) + + def series = [ + new TimeBasedValue(t1, value1), + new TimeBasedValue(t3, value3), + new TimeBasedValue(t4, value3) + ] as Set + + def mockSource = new InterpolatingWeatherSource() + + when: + def interpolatedSeries = mockSource.interpolateMissingValues(series) + + then: + interpolatedSeries.size() == 4 + def interpolated = interpolatedSeries.find { it.time == t2 } + interpolated != null + with(interpolated.value as WeatherValue) { + getTemperature().value.doubleValue() == 15.0 + getWindVelocity().value.doubleValue() == 7.5 + getWindDirection().value.doubleValue() == 225.0 + } + } + + private static class InterpolatingWeatherSource extends WeatherSource { + @Override + Optional> getWeather(ZonedDateTime dateTime, Point coordinate) { + return Optional.empty() + } + } +} \ No newline at end of file From be17bc5a939d26785541292bdbf6ad6eabf6f28b Mon Sep 17 00:00:00 2001 From: pierrepetersmeier <155652256+pierrepetersmeier@users.noreply.github.com> Date: Thu, 22 May 2025 22:04:38 +0200 Subject: [PATCH 2/4] Interpolate functionality and InterpolationTest. --- .../datamodel/io/source/WeatherSource.java | 32 +++++++++++++++++++ .../datamodel/models/value/WeatherValue.java | 1 + 2 files changed, 33 insertions(+) diff --git a/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java index 60233e4d1..c7fd6b764 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java @@ -20,6 +20,10 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import javax.measure.Quantity; +import javax.measure.quantity.Speed; +import javax.measure.quantity.Temperature; + +import edu.ie3.util.quantities.interfaces.Irradiance; import org.apache.commons.lang3.tuple.Pair; import org.locationtech.jts.geom.Point; import org.slf4j.Logger; @@ -151,6 +155,34 @@ public abstract Map> getWeather( public abstract Optional> getWeather( ZonedDateTime date, Point coordinate) throws SourceException; + public Optional getWeatherInterpolated(ZonedDateTime date, Point coordinate, int plus, int minus) throws SourceException { + + ClosedInterval interpolationInterval = new ClosedInterval<>( + date.minusHours(minus), date.plusHours(plus) + ); + IndividualTimeSeriests=getWeather(interpolationInterval,List.of(coordinate)).get(coordinate); + + Optional value = ts.getValue(date); + + if(value.isPresent() && value.get().isComplete()){ + return value; + } + + + Optional,Integer>> dirIrrPre = Optional.empty(); + Optional,Integer>> diffIrrPre = Optional.empty(); + Optional,Integer>> tempPre = Optional.empty(); + var velocityPre = Optional.empty(); + + for (int i = 1;i<=minus*4;i++){ + ZonedDateTime current = date.minusMinutes(i*15L); + ts.getValue(current).map(WeatherValue::getDiffIrradiance) = ; + /// + } + } + + + public abstract Map> getTimeKeysAfter(ZonedDateTime time) throws SourceException; diff --git a/src/main/java/edu/ie3/datamodel/models/value/WeatherValue.java b/src/main/java/edu/ie3/datamodel/models/value/WeatherValue.java index 09ec1787f..9a250ea8f 100644 --- a/src/main/java/edu/ie3/datamodel/models/value/WeatherValue.java +++ b/src/main/java/edu/ie3/datamodel/models/value/WeatherValue.java @@ -7,6 +7,7 @@ import edu.ie3.util.quantities.interfaces.Irradiance; import java.util.Objects; +import java.util.Optional; import javax.measure.quantity.Angle; import javax.measure.quantity.Speed; import javax.measure.quantity.Temperature; From 675e0cb6e9b41068026fcf796b7fbca9fc2e0e19 Mon Sep 17 00:00:00 2001 From: pierrepetersmeier <155652256+pierrepetersmeier@users.noreply.github.com> Date: Wed, 28 May 2025 15:07:42 +0200 Subject: [PATCH 3/4] Add weather interpolation and fix getters --- .../datamodel/io/source/WeatherSource.java | 77 ++++++++++--------- .../individual/IndividualTimeSeries.java | 4 +- .../datamodel/models/value/WeatherValue.java | 28 +++++++ 3 files changed, 72 insertions(+), 37 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java b/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java index c7fd6b764..755095a52 100644 --- a/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java +++ b/src/main/java/edu/ie3/datamodel/io/source/WeatherSource.java @@ -20,10 +20,6 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import javax.measure.Quantity; -import javax.measure.quantity.Speed; -import javax.measure.quantity.Temperature; - -import edu.ie3.util.quantities.interfaces.Irradiance; import org.apache.commons.lang3.tuple.Pair; import org.locationtech.jts.geom.Point; import org.slf4j.Logger; @@ -106,24 +102,17 @@ protected List> interpolateMissingValues( } private WeatherValue interpolateWeatherValue(WeatherValue start, WeatherValue end, double ratio) { - var startSolar = start.getSolarIrradiance(); - var endSolar = end.getSolarIrradiance(); - - var direct = - interpolateOptional( - startSolar.getDirectIrradiance(), endSolar.getDirectIrradiance(), ratio); - var diffuse = - interpolateOptional( - startSolar.getDiffuseIrradiance(), endSolar.getDiffuseIrradiance(), ratio); + var direct = interpolateOptional(start.getDirectIrradiance(), end.getDirectIrradiance(), ratio); + var diffuse = interpolateOptional(start.getDiffuseIrradiance(), end.getDiffuseIrradiance(), ratio); var temp = interpolateDirect(start.getTemperature(), end.getTemperature(), ratio); - var dir = - interpolateDirect(start.getWind().getDirection(), end.getWind().getDirection(), ratio); - var vel = interpolateDirect(start.getWind().getVelocity(), end.getWind().getVelocity(), ratio); + var dir = interpolateDirect(start.getWindDirection(), end.getWindDirection(), ratio); + var vel = interpolateDirect(start.getWindVelocity(), end.getWindVelocity(), ratio); return new WeatherValue(start.getCoordinate(), direct, diffuse, temp, dir, vel); } + private > ComparableQuantity interpolateOptional( Optional> startOpt, Optional> endOpt, @@ -155,32 +144,50 @@ public abstract Map> getWeather( public abstract Optional> getWeather( ZonedDateTime date, Point coordinate) throws SourceException; - public Optional getWeatherInterpolated(ZonedDateTime date, Point coordinate, int plus, int minus) throws SourceException { + public Optional getWeatherInterpolated( + ZonedDateTime date, Point coordinate, int plus, int minus) throws SourceException { - ClosedInterval interpolationInterval = new ClosedInterval<>( - date.minusHours(minus), date.plusHours(plus) - ); - IndividualTimeSeriests=getWeather(interpolationInterval,List.of(coordinate)).get(coordinate); + ClosedInterval interpolationInterval = + new ClosedInterval<>(date.minusHours(minus), date.plusHours(plus)); + IndividualTimeSeries ts = + getWeather(interpolationInterval, List.of(coordinate)).get(coordinate); - Optional value = ts.getValue(date); + if (ts == null) { + log.warn("No time series available for coordinate {}", coordinate); + return Optional.empty(); + } - if(value.isPresent() && value.get().isComplete()){ - return value; - } + Optional value = ts.getValue(date); + if (value.isPresent() && value.get().isComplete()) { + return value; + } - Optional,Integer>> dirIrrPre = Optional.empty(); - Optional,Integer>> diffIrrPre = Optional.empty(); - Optional,Integer>> tempPre = Optional.empty(); - var velocityPre = Optional.empty(); + Optional> prevValue = ts.getPreviousTimeBasedValue(date); + Optional> nextValue = ts.getNextTimeBasedValue(date); - for (int i = 1;i<=minus*4;i++){ - ZonedDateTime current = date.minusMinutes(i*15L); - ts.getValue(current).map(WeatherValue::getDiffIrradiance) = ; - /// - } - } + if (prevValue.isEmpty() || nextValue.isEmpty()) { + log.warn( + "Not enough data to interpolate weather value at {} for coordinate {}", date, coordinate); + return Optional.empty(); + } + + TimeBasedValue prev = prevValue.get(); + TimeBasedValue next = nextValue.get(); + + Duration totalDuration = Duration.between(prev.getTime(), next.getTime()); + Duration partialDuration = Duration.between(prev.getTime(), date); + + if (totalDuration.isZero()) { + return Optional.of(prev.getValue()); + } + + double ratio = (double) partialDuration.toSeconds() / totalDuration.toSeconds(); + WeatherValue interpolated = interpolateWeatherValue(prev.getValue(), next.getValue(), ratio); + + return Optional.of(interpolated); + } public abstract Map> getTimeKeysAfter(ZonedDateTime time) diff --git a/src/main/java/edu/ie3/datamodel/models/timeseries/individual/IndividualTimeSeries.java b/src/main/java/edu/ie3/datamodel/models/timeseries/individual/IndividualTimeSeries.java index 1ae846da3..ab47673fc 100644 --- a/src/main/java/edu/ie3/datamodel/models/timeseries/individual/IndividualTimeSeries.java +++ b/src/main/java/edu/ie3/datamodel/models/timeseries/individual/IndividualTimeSeries.java @@ -53,14 +53,14 @@ public Optional getValue(ZonedDateTime time) { } @Override - protected Optional getPreviousDateTime(ZonedDateTime time) { + public Optional getPreviousDateTime(ZonedDateTime time) { return timeToValue.keySet().stream() .filter(valueTime -> valueTime.compareTo(time) < 0) .max(Comparator.naturalOrder()); } @Override - protected Optional getNextDateTime(ZonedDateTime time) { + public Optional getNextDateTime(ZonedDateTime time) { return timeToValue.keySet().stream() .filter(valueTime -> valueTime.compareTo(time) > 0) .min(Comparator.naturalOrder()); diff --git a/src/main/java/edu/ie3/datamodel/models/value/WeatherValue.java b/src/main/java/edu/ie3/datamodel/models/value/WeatherValue.java index 9a250ea8f..f7fd75073 100644 --- a/src/main/java/edu/ie3/datamodel/models/value/WeatherValue.java +++ b/src/main/java/edu/ie3/datamodel/models/value/WeatherValue.java @@ -82,6 +82,34 @@ public WindValue getWind() { return wind; } + public Optional> getDirectIrradiance() { + return solarIrradiance.getDirectIrradiance(); + } + + public Optional> getDiffuseIrradiance() { + return solarIrradiance.getDiffuseIrradiance(); + } + + public ComparableQuantity getTemperature() { + return temperature.getTemperature(); + } + + public ComparableQuantity getWindDirection() { + return wind.getDirection(); + } + public ComparableQuantity getWindVelocity() { + return wind.getVelocity(); + } + + /** + * Checks if all mandatory values are present. + * + * @return true if all values are present, false otherwise + */ + public boolean isComplete() { + return solarIrradiance != null && temperature != null && wind != null; + } + @Override public boolean equals(Object o) { if (this == o) return true; From 5fae400aa3b5b11e1c1aa3024b7a3c542b4361dc Mon Sep 17 00:00:00 2001 From: pierrepetersmeier <155652256+pierrepetersmeier@users.noreply.github.com> Date: Wed, 28 May 2025 15:12:57 +0200 Subject: [PATCH 4/4] IndividualTimeSeries.java protected -> public Reverted --- .../models/timeseries/individual/IndividualTimeSeries.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/ie3/datamodel/models/timeseries/individual/IndividualTimeSeries.java b/src/main/java/edu/ie3/datamodel/models/timeseries/individual/IndividualTimeSeries.java index ab47673fc..1ae846da3 100644 --- a/src/main/java/edu/ie3/datamodel/models/timeseries/individual/IndividualTimeSeries.java +++ b/src/main/java/edu/ie3/datamodel/models/timeseries/individual/IndividualTimeSeries.java @@ -53,14 +53,14 @@ public Optional getValue(ZonedDateTime time) { } @Override - public Optional getPreviousDateTime(ZonedDateTime time) { + protected Optional getPreviousDateTime(ZonedDateTime time) { return timeToValue.keySet().stream() .filter(valueTime -> valueTime.compareTo(time) < 0) .max(Comparator.naturalOrder()); } @Override - public Optional getNextDateTime(ZonedDateTime time) { + protected Optional getNextDateTime(ZonedDateTime time) { return timeToValue.keySet().stream() .filter(valueTime -> valueTime.compareTo(time) > 0) .min(Comparator.naturalOrder());