diff --git a/quartz_solar_forecast/data.py b/quartz_solar_forecast/data.py index e93ff03b..5bcb07f9 100644 --- a/quartz_solar_forecast/data.py +++ b/quartz_solar_forecast/data.py @@ -4,15 +4,11 @@ from typing import Optional import numpy as np -import openmeteo_requests import pandas as pd -import requests_cache import xarray as xr -from retry_requests import retry from quartz_solar_forecast.pydantic_models import PVSite - -ssl._create_default_https_context = ssl._create_unverified_context +from quartz_solar_forecast.weather import WeatherService def get_nwp(site: PVSite, ts: datetime, nwp_source: str = "icon") -> xr.Dataset: @@ -26,95 +22,74 @@ def get_nwp(site: PVSite, ts: datetime, nwp_source: str = "icon") -> xr.Dataset: """ now = datetime.now() - # Setup the Open-Meteo API client with cache and retry on error - cache_session = requests_cache.CachedSession('.cache', expire_after = -1) - retry_session = retry(cache_session, retries = 5, backoff_factor = 0.2) - openmeteo = openmeteo_requests.Client(session = retry_session) - - # Define the variables we want. Visibility is handled separately after the main request - variables = [ - "temperature_2m", - "precipitation", - "cloud_cover_low", - "cloud_cover_mid", - "cloud_cover_high", - "wind_speed_10m", - "shortwave_radiation", - "direct_radiation" - ] + # Define the variables we'll be fetching from Open-Meteo API (and their aliases), + # visibility is handled separately after the main request + variable_map = { + "temperature_2m": "t", + "precipitation": "prate", + "cloud_cover_low": "lcc", + "cloud_cover_mid": "mcc", + "cloud_cover_high": "hcc", + "wind_speed_10m": "si10", + "shortwave_radiation": "dswrf", + "direct_radiation": "dlwrf" + } start = ts.date() end = start + pd.Timedelta(days=7) - url = "" + weather_service = WeatherService() - # check whether the time stamp is more than 3 months in the past + # Check whether the time stamp is more than 3 months in the past: + # if yes, use open-meteo Historical Weather API if (now - ts).days > 90: print("Warning: The requested timestamp is more than 3 months in the past. The weather data are provided by a reanalyse model and not ICON or GFS.") - - # load data from open-meteo Historical Weather API - url = "https://archive-api.open-meteo.com/v1/archive" - + api_type = "archive" + + # Visibility cannot be fetched with Historical Weather API, just set it to maximum possible value + visibility = 24000.0 + # Else, get NWP from open-meteo Weather Forecast API by ICON, GFS, or UKMO within the last 3 months else: - # Getting NWP from open meteo weather forecast API by ICON, GFS, or UKMO within the last 3 months - url_nwp_source = { + api_type = { "icon": "dwd-icon", "gfs": "gfs", - "ukmo": "ukmo_seamless" + "ukmo": "forecast" }.get(nwp_source) - if not url_nwp_source: + if not api_type: raise Exception(f'Source ({nwp_source}) must be either "icon", "gfs", or "ukmo"') - url = f"https://api.open-meteo.com/v1/{url_nwp_source if nwp_source != 'ukmo' else 'forecast'}" - - params = { - "latitude": site.latitude, - "longitude": site.longitude, - "start_date": f"{start}", - "end_date": f"{end}", - "hourly": variables - } - - # Add the "models" parameter if using "ukmo" - if nwp_source == "ukmo": - params["models"] = "ukmo_seamless" - - # Make API call to URL - response = openmeteo.weather_api(url, params=params) - hourly = response[0].Hourly() - - hourly_data = {"time": pd.date_range( - start = pd.to_datetime(hourly.Time(), unit = "s", utc = False), - end = pd.to_datetime(hourly.TimeEnd(), unit = "s", utc = False), - freq = pd.Timedelta(seconds = hourly.Interval()), - inclusive = "left" - )} - - - # variables index as in the variables array of the request - for idx, var in enumerate(["t", "prate", "lcc", "mcc", "hcc", "si10", "dswrf", "dlwrf"]): - hourly_data[var] = hourly.Variables(idx).ValuesAsNumpy() - - # handle visibility - if (now - ts).days <= 90: - # load data from open-meteo gfs model - params = { - "latitude": site.latitude, - "longitude": site.longitude, - "start_date": f"{start}", - "end_date": f"{end}", - "hourly": "visibility" - } - data_vis_gfs = openmeteo.weather_api("https://api.open-meteo.com/v1/gfs", params=params)[0].Hourly().Variables(0).ValuesAsNumpy() - hourly_data["vis"] = data_vis_gfs - else: - # set to maximum visibility possible - hourly_data["vis"] = 24000.0 + + # Use visibility provided by GFS model + visibility_data = weather_service.get_hourly_weather( + latitude=site.latitude, + longitude=site.longitude, + start_date=f"{start}", + end_date=f"{end}", + variables=["visibility"], + api_type="gfs" + ) + visibility = visibility_data["visibility"].values + + # Make main call to Open-Meteo API + weather_data = weather_service.get_hourly_weather( + latitude=site.latitude, + longitude=site.longitude, + start_date=f"{start}", + end_date=f"{end}", + variables=list(variable_map.keys()), + api_type=api_type, + model="ukmo_seamless" if nwp_source == "ukmo" else None + ) - df = pd.DataFrame(data=hourly_data).set_index("time").astype('float64') + # Add visibility values to data frame + weather_data["vis"] = visibility + # Rename variable columns to be correctly processed further + weather_data.rename(columns=variable_map, inplace=True) + weather_data.rename(columns={"date": "time"}, inplace=True) + weather_data = weather_data.set_index("time").astype('float64') - # convert data into xarray - data_xr = format_nwp_data(df, nwp_source, site) + # Convert data into xarray + data_xr = format_nwp_data(weather_data, nwp_source, site) return data_xr diff --git a/quartz_solar_forecast/forecasts/v2.py b/quartz_solar_forecast/forecasts/v2.py index 0cbe18cc..2b2a7450 100644 --- a/quartz_solar_forecast/forecasts/v2.py +++ b/quartz_solar_forecast/forecasts/v2.py @@ -164,11 +164,30 @@ def get_data( start_date_datetime = datetime.datetime.strptime(start_date, "%Y-%m-%d") end_date_datetime = start_date_datetime + datetime.timedelta(days=2) end_date = end_date_datetime.strftime("%Y-%m-%d") + variables = [ + "temperature_2m", + "relative_humidity_2m", + "dew_point_2m", + "precipitation", + "surface_pressure", + "cloud_cover", + "cloud_cover_low", + "cloud_cover_mid", + "cloud_cover_high", + "wind_speed_10m", + "wind_direction_10m", + "is_day", + "shortwave_radiation", + "direct_radiation", + "diffuse_radiation", + "direct_normal_irradiance", + "terrestrial_radiation", + ] weather_service = WeatherService() weather_data = weather_service.get_hourly_weather( - latitude, longitude, start_date, end_date + latitude, longitude, start_date, end_date, variables ) PANEL_COLUMNS = [ diff --git a/quartz_solar_forecast/weather/open_meteo.py b/quartz_solar_forecast/weather/open_meteo.py index 7da71108..b5563c5d 100644 --- a/quartz_solar_forecast/weather/open_meteo.py +++ b/quartz_solar_forecast/weather/open_meteo.py @@ -17,44 +17,6 @@ def __init__(self): """ pass - def _build_url( - self, - latitude: float, - longitude: float, - start_date: str, - end_date: str, - variables: List[str], - ) -> str: - """ - Build the URL for the OpenMeteo API. - - Parameters - ---------- - latitude : float - The latitude of the location for which to get weather data. - longitude : float - The longitude of the location for which to get weather data. - start_date : str - The start date for the weather data, in the format YYYY-MM-DD. - end_date : str - The end date for the weather data, in the format YYYY-MM-DD. - variables : list - A list of weather variables to include in the API response. - - Returns - ------- - str - The URL for the OpenMeteo API. - """ - url = "https://api.open-meteo.com/v1/forecast?latitude={latitude}&longitude={longitude}&hourly={variables}&start_date={start_date}&end_date={end_date}&timezone=GMT".format( - latitude=latitude, - longitude=longitude, - variables=",".join(variables), - start_date=start_date, - end_date=end_date, - ) - return url - def _validate_coordinates(self, latitude: float, longitude: float) -> None: """ Validate latitude and longitude coordinates. @@ -101,7 +63,7 @@ def _validate_date_format(self, start_date: str, end_date: str) -> None: ) def get_hourly_weather( - self, latitude: float, longitude: float, start_date: str, end_date: str + self, latitude: float, longitude: float, start_date: str, end_date: str, variables: List[str] = [], api_type: str = "forecast", model: str = None ) -> pd.DataFrame: """ Get hourly weather data ranging from 3 months ago up to 15 days ahead (forecast). @@ -116,6 +78,12 @@ def get_hourly_weather( The start date for the weather data, in the format YYYY-MM-DD. end_date : str The end date for the weather data, in the format YYYY-MM-DD. + variables : list + A list of weather variables to include in the API response. + api_type : str + Type of Open-Meteo API to be used, forecast by default. + model : str + Weather model, may be undefined Returns ------- @@ -130,37 +98,28 @@ def get_hourly_weather( self._validate_coordinates(latitude, longitude) self._validate_date_format(start_date, end_date) - variables = [ - "temperature_2m", - "relative_humidity_2m", - "dew_point_2m", - "precipitation", - "surface_pressure", - "cloud_cover", - "cloud_cover_low", - "cloud_cover_mid", - "cloud_cover_high", - "wind_speed_10m", - "wind_direction_10m", - "is_day", - "shortwave_radiation", - "direct_radiation", - "diffuse_radiation", - "direct_normal_irradiance", - "terrestrial_radiation", - ] - url = self._build_url(latitude, longitude, start_date, end_date, variables) - + main_api = "archive-api" if api_type == "archive" else "api" + url = f"https://{main_api}.open-meteo.com/v1/{api_type}" + params = { + "latitude": latitude, + "longitude": longitude, + "start_date": start_date, + "end_date": end_date, + "hourly": variables, + } + if model is not None: + params["models"] = model + cache_session = requests_cache.CachedSession(".cache", expire_after=-1) retry_session = retry(cache_session, retries=5, backoff_factor=0.2) try: openmeteo = openmeteo_requests.Client(session=retry_session) - response = openmeteo.weather_api(url, params={}) - except requests.exceptions.Timeout: - raise TimeoutError(f"Request to OpenMeteo API timed out. URl - {url}") + response = openmeteo.weather_api(url, params=params) + except requests.exceptions.Timeout as e: + raise TimeoutError(f"Request to OpenMeteo API timed out. URl - {e.request.url}") hourly = response[0].Hourly() - hourly_data = {"time": pd.date_range( + hourly_data = {"date": pd.date_range( start=pd.to_datetime(hourly.Time(), unit="s", utc=False), end=pd.to_datetime(hourly.TimeEnd(), unit="s", utc=False), freq=pd.Timedelta(seconds=hourly.Interval()), @@ -171,13 +130,4 @@ def get_hourly_weather( hourly_data[variable] = hourly.Variables(i).ValuesAsNumpy() df = pd.DataFrame(hourly_data) - df["time"] = pd.to_datetime(df["time"]) - - # rename time column to date - df = df.rename( - columns={ - "time": "date", - } - ) - return df diff --git a/tests/unit/mocks.py b/tests/unit/mocks.py new file mode 100644 index 00000000..25c8ed60 --- /dev/null +++ b/tests/unit/mocks.py @@ -0,0 +1,25 @@ +import pandas as pd +import pytest + +from quartz_solar_forecast.weather.open_meteo import WeatherService + + +# Fixture for getting hourly data from Open-Meteo API +@pytest.fixture +def mock_weather_api(monkeypatch): + # Monkeypatch get_hourly_weather method: + # behavior same to original method, returns dummy weather data (zeroes) + def mock_get_hourly_weather(self, latitude, longitude, start_date, end_date, variables=[], api_type="forecast", model=None): + mock_hourly_date = pd.date_range( + start = pd.to_datetime(start_date, format="%Y-%m-%d", utc = False), + end = pd.to_datetime(end_date, format="%Y-%m-%d", utc = False) + pd.Timedelta(days=1), + freq = pd.Timedelta(hours=1), + inclusive = "left" + ) + mock_weather_df = pd.DataFrame({ "date": mock_hourly_date }) + # Fill with zeroes (fake weather data) + for v in variables: + mock_weather_df[v] = 0.0 + return mock_weather_df + + monkeypatch.setattr(WeatherService, "get_hourly_weather", mock_get_hourly_weather) diff --git a/tests/unit/test_forecast.py b/tests/unit/test_forecast.py index c2d0c70b..4c8ed0ad 100644 --- a/tests/unit/test_forecast.py +++ b/tests/unit/test_forecast.py @@ -1,13 +1,29 @@ from quartz_solar_forecast.forecast import run_forecast from quartz_solar_forecast.pydantic_models import PVSite +from tests.unit.mocks import mock_weather_api from datetime import datetime, timedelta import numpy as np +import pytest -def test_run_forecast(): - # make input data - site = PVSite(latitude=51.75, longitude=-1.25, capacity_kwp=1.25) +# Shared fixture for site input +@pytest.fixture +def site(): + return PVSite(latitude=51.75, longitude=-1.25, capacity_kwp=1.25) + + +def test_run_forecast_invalid_model(site): + with pytest.raises(ValueError, match="Unsupported model:"): + run_forecast(site=site, model="invalid_model") + + +def test_run_forecast_invalid_nwp_source(site): + with pytest.raises(Exception, match="Source"): + run_forecast(site=site, model="gb", ts=None, nwp_source="invalid_nwp_source") + + +def test_run_forecast(site, mock_weather_api): ts = datetime.today() - timedelta(weeks=2) # run model with icon, gfs and ukmo nwp @@ -33,10 +49,7 @@ def test_run_forecast(): print(f" Max: {predications_df_xgb['power_kw'].max()}") -def test_run_forecast_historical(): - - # model input data creation - site = PVSite(latitude=51.75, longitude=-1.25, capacity_kwp=1.25) +def test_run_forecast_historical(site, mock_weather_api): ts = datetime.today() - timedelta(days=200) # run model with icon, gfs and ukmo nwp @@ -63,9 +76,7 @@ def test_run_forecast_historical(): print(predications_df_xgb) -def test_large_capacity(): - - # make input data +def test_large_capacity(mock_weather_api): site = PVSite(latitude=51.75, longitude=-1.25, capacity_kwp=4) site_large = PVSite(latitude=51.75, longitude=-1.25, capacity_kwp=4000) ts = datetime.today() - timedelta(weeks=2) diff --git a/tests/unit/test_forecast_no_ts.py b/tests/unit/test_forecast_no_ts.py index 0df021b6..b6eaef31 100644 --- a/tests/unit/test_forecast_no_ts.py +++ b/tests/unit/test_forecast_no_ts.py @@ -1,9 +1,10 @@ import pandas as pd + from quartz_solar_forecast.forecast import run_forecast from quartz_solar_forecast.pydantic_models import PVSite +from tests.unit.mocks import mock_weather_api - -def test_run_forecast_no_ts(): +def test_run_forecast_no_ts(mock_weather_api): # make input data site = PVSite(latitude=51.75, longitude=-1.25, capacity_kwp=1.25) diff --git a/tests/unit/test_generate_forecast.py b/tests/unit/test_generate_forecast.py index 5c5c7aa4..7d38e518 100644 --- a/tests/unit/test_generate_forecast.py +++ b/tests/unit/test_generate_forecast.py @@ -4,8 +4,9 @@ import quartz_solar_forecast.forecast as forecast from quartz_solar_forecast.utils.forecast_csv import write_out_forecasts from quartz_solar_forecast.pydantic_models import PVSite +from tests.unit.mocks import mock_weather_api -def test_generate_forecast(monkeypatch): +def test_generate_forecast(monkeypatch, mock_weather_api): site_name = "TestCase" latitude = 51.75 longitude = -1.25