Skip to content

Mock calls to Open-Meteo API #281

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Mock calls to Open-Meteo API #281

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
133 changes: 54 additions & 79 deletions quartz_solar_forecast/data.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,11 @@
from typing import Optional

import numpy as np
import openmeteo_requests
import pandas as pd
import requests_cache
import xarray as xr
from retry_requests import retry

from quartz_solar_forecast.pydantic_models import PVSite

ssl._create_default_https_context = ssl._create_unverified_context
from quartz_solar_forecast.weather import WeatherService


def get_nwp(site: PVSite, ts: datetime, nwp_source: str = "icon") -> xr.Dataset:
Expand All @@ -26,95 +22,74 @@ def get_nwp(site: PVSite, ts: datetime, nwp_source: str = "icon") -> xr.Dataset:
"""
now = datetime.now()

# Setup the Open-Meteo API client with cache and retry on error
cache_session = requests_cache.CachedSession('.cache', expire_after = -1)
retry_session = retry(cache_session, retries = 5, backoff_factor = 0.2)
openmeteo = openmeteo_requests.Client(session = retry_session)

# Define the variables we want. Visibility is handled separately after the main request
variables = [
"temperature_2m",
"precipitation",
"cloud_cover_low",
"cloud_cover_mid",
"cloud_cover_high",
"wind_speed_10m",
"shortwave_radiation",
"direct_radiation"
]
# Define the variables we'll be fetching from Open-Meteo API (and their aliases),
# visibility is handled separately after the main request
variable_map = {
"temperature_2m": "t",
"precipitation": "prate",
"cloud_cover_low": "lcc",
"cloud_cover_mid": "mcc",
"cloud_cover_high": "hcc",
"wind_speed_10m": "si10",
"shortwave_radiation": "dswrf",
"direct_radiation": "dlwrf"
}

start = ts.date()
end = start + pd.Timedelta(days=7)

url = ""
weather_service = WeatherService()

# check whether the time stamp is more than 3 months in the past
# Check whether the time stamp is more than 3 months in the past:
# if yes, use open-meteo Historical Weather API
if (now - ts).days > 90:
print("Warning: The requested timestamp is more than 3 months in the past. The weather data are provided by a reanalyse model and not ICON or GFS.")

# load data from open-meteo Historical Weather API
url = "https://archive-api.open-meteo.com/v1/archive"

api_type = "archive"

# Visibility cannot be fetched with Historical Weather API, just set it to maximum possible value
visibility = 24000.0
# Else, get NWP from open-meteo Weather Forecast API by ICON, GFS, or UKMO within the last 3 months
else:
# Getting NWP from open meteo weather forecast API by ICON, GFS, or UKMO within the last 3 months
url_nwp_source = {
api_type = {
"icon": "dwd-icon",
"gfs": "gfs",
"ukmo": "ukmo_seamless"
"ukmo": "forecast"
}.get(nwp_source)
if not url_nwp_source:
if not api_type:
raise Exception(f'Source ({nwp_source}) must be either "icon", "gfs", or "ukmo"')
url = f"https://api.open-meteo.com/v1/{url_nwp_source if nwp_source != 'ukmo' else 'forecast'}"

params = {
"latitude": site.latitude,
"longitude": site.longitude,
"start_date": f"{start}",
"end_date": f"{end}",
"hourly": variables
}

# Add the "models" parameter if using "ukmo"
if nwp_source == "ukmo":
params["models"] = "ukmo_seamless"

# Make API call to URL
response = openmeteo.weather_api(url, params=params)
hourly = response[0].Hourly()

hourly_data = {"time": pd.date_range(
start = pd.to_datetime(hourly.Time(), unit = "s", utc = False),
end = pd.to_datetime(hourly.TimeEnd(), unit = "s", utc = False),
freq = pd.Timedelta(seconds = hourly.Interval()),
inclusive = "left"
)}


# variables index as in the variables array of the request
for idx, var in enumerate(["t", "prate", "lcc", "mcc", "hcc", "si10", "dswrf", "dlwrf"]):
hourly_data[var] = hourly.Variables(idx).ValuesAsNumpy()

# handle visibility
if (now - ts).days <= 90:
# load data from open-meteo gfs model
params = {
"latitude": site.latitude,
"longitude": site.longitude,
"start_date": f"{start}",
"end_date": f"{end}",
"hourly": "visibility"
}
data_vis_gfs = openmeteo.weather_api("https://api.open-meteo.com/v1/gfs", params=params)[0].Hourly().Variables(0).ValuesAsNumpy()
hourly_data["vis"] = data_vis_gfs
else:
# set to maximum visibility possible
hourly_data["vis"] = 24000.0

# Use visibility provided by GFS model
visibility_data = weather_service.get_hourly_weather(
latitude=site.latitude,
longitude=site.longitude,
start_date=f"{start}",
end_date=f"{end}",
variables=["visibility"],
api_type="gfs"
)
visibility = visibility_data["visibility"].values

# Make main call to Open-Meteo API
weather_data = weather_service.get_hourly_weather(
latitude=site.latitude,
longitude=site.longitude,
start_date=f"{start}",
end_date=f"{end}",
variables=list(variable_map.keys()),
api_type=api_type,
model="ukmo_seamless" if nwp_source == "ukmo" else None
)

df = pd.DataFrame(data=hourly_data).set_index("time").astype('float64')
# Add visibility values to data frame
weather_data["vis"] = visibility

# Rename variable columns to be correctly processed further
weather_data.rename(columns=variable_map, inplace=True)
weather_data.rename(columns={"date": "time"}, inplace=True)
weather_data = weather_data.set_index("time").astype('float64')

# convert data into xarray
data_xr = format_nwp_data(df, nwp_source, site)
# Convert data into xarray
data_xr = format_nwp_data(weather_data, nwp_source, site)

return data_xr

Expand Down
21 changes: 20 additions & 1 deletion quartz_solar_forecast/forecasts/v2.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -164,11 +164,30 @@ def get_data(
start_date_datetime = datetime.datetime.strptime(start_date, "%Y-%m-%d")
end_date_datetime = start_date_datetime + datetime.timedelta(days=2)
end_date = end_date_datetime.strftime("%Y-%m-%d")
variables = [
"temperature_2m",
"relative_humidity_2m",
"dew_point_2m",
"precipitation",
"surface_pressure",
"cloud_cover",
"cloud_cover_low",
"cloud_cover_mid",
"cloud_cover_high",
"wind_speed_10m",
"wind_direction_10m",
"is_day",
"shortwave_radiation",
"direct_radiation",
"diffuse_radiation",
"direct_normal_irradiance",
"terrestrial_radiation",
]

weather_service = WeatherService()

weather_data = weather_service.get_hourly_weather(
latitude, longitude, start_date, end_date
latitude, longitude, start_date, end_date, variables
)

PANEL_COLUMNS = [
Expand Down
96 changes: 23 additions & 73 deletions quartz_solar_forecast/weather/open_meteo.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,44 +17,6 @@ def __init__(self):
"""
pass

def _build_url(
self,
latitude: float,
longitude: float,
start_date: str,
end_date: str,
variables: List[str],
) -> str:
"""
Build the URL for the OpenMeteo API.
Parameters
----------
latitude : float
The latitude of the location for which to get weather data.
longitude : float
The longitude of the location for which to get weather data.
start_date : str
The start date for the weather data, in the format YYYY-MM-DD.
end_date : str
The end date for the weather data, in the format YYYY-MM-DD.
variables : list
A list of weather variables to include in the API response.
Returns
-------
str
The URL for the OpenMeteo API.
"""
url = "https://api.open-meteo.com/v1/forecast?latitude={latitude}&longitude={longitude}&hourly={variables}&start_date={start_date}&end_date={end_date}&timezone=GMT".format(
latitude=latitude,
longitude=longitude,
variables=",".join(variables),
start_date=start_date,
end_date=end_date,
)
return url

def _validate_coordinates(self, latitude: float, longitude: float) -> None:
"""
Validate latitude and longitude coordinates.
Expand Down Expand Up @@ -101,7 +63,7 @@ def _validate_date_format(self, start_date: str, end_date: str) -> None:
)

def get_hourly_weather(
self, latitude: float, longitude: float, start_date: str, end_date: str
self, latitude: float, longitude: float, start_date: str, end_date: str, variables: List[str] = [], api_type: str = "forecast", model: str = None
) -> pd.DataFrame:
"""
Get hourly weather data ranging from 3 months ago up to 15 days ahead (forecast).
Expand All @@ -116,6 +78,12 @@ def get_hourly_weather(
The start date for the weather data, in the format YYYY-MM-DD.
end_date : str
The end date for the weather data, in the format YYYY-MM-DD.
variables : list
A list of weather variables to include in the API response.
api_type : str
Type of Open-Meteo API to be used, forecast by default.
model : str
Weather model, may be undefined
Returns
-------
Expand All @@ -130,37 +98,28 @@ def get_hourly_weather(
self._validate_coordinates(latitude, longitude)
self._validate_date_format(start_date, end_date)

variables = [
"temperature_2m",
"relative_humidity_2m",
"dew_point_2m",
"precipitation",
"surface_pressure",
"cloud_cover",
"cloud_cover_low",
"cloud_cover_mid",
"cloud_cover_high",
"wind_speed_10m",
"wind_direction_10m",
"is_day",
"shortwave_radiation",
"direct_radiation",
"diffuse_radiation",
"direct_normal_irradiance",
"terrestrial_radiation",
]
url = self._build_url(latitude, longitude, start_date, end_date, variables)

main_api = "archive-api" if api_type == "archive" else "api"
url = f"https://{main_api}.open-meteo.com/v1/{api_type}"
params = {
"latitude": latitude,
"longitude": longitude,
"start_date": start_date,
"end_date": end_date,
"hourly": variables,
}
if model is not None:
params["models"] = model

cache_session = requests_cache.CachedSession(".cache", expire_after=-1)
retry_session = retry(cache_session, retries=5, backoff_factor=0.2)
try:
openmeteo = openmeteo_requests.Client(session=retry_session)
response = openmeteo.weather_api(url, params={})
except requests.exceptions.Timeout:
raise TimeoutError(f"Request to OpenMeteo API timed out. URl - {url}")
response = openmeteo.weather_api(url, params=params)
except requests.exceptions.Timeout as e:
raise TimeoutError(f"Request to OpenMeteo API timed out. URl - {e.request.url}")

hourly = response[0].Hourly()
hourly_data = {"time": pd.date_range(
hourly_data = {"date": pd.date_range(
start=pd.to_datetime(hourly.Time(), unit="s", utc=False),
end=pd.to_datetime(hourly.TimeEnd(), unit="s", utc=False),
freq=pd.Timedelta(seconds=hourly.Interval()),
Expand All @@ -171,13 +130,4 @@ def get_hourly_weather(
hourly_data[variable] = hourly.Variables(i).ValuesAsNumpy()

df = pd.DataFrame(hourly_data)
df["time"] = pd.to_datetime(df["time"])

# rename time column to date
df = df.rename(
columns={
"time": "date",
}
)

return df
25 changes: 25 additions & 0 deletions tests/unit/mocks.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
import pandas as pd
import pytest

from quartz_solar_forecast.weather.open_meteo import WeatherService


# Fixture for getting hourly data from Open-Meteo API
@pytest.fixture
def mock_weather_api(monkeypatch):
# Monkeypatch get_hourly_weather method:
# behavior same to original method, returns dummy weather data (zeroes)
def mock_get_hourly_weather(self, latitude, longitude, start_date, end_date, variables=[], api_type="forecast", model=None):
mock_hourly_date = pd.date_range(
start = pd.to_datetime(start_date, format="%Y-%m-%d", utc = False),
end = pd.to_datetime(end_date, format="%Y-%m-%d", utc = False) + pd.Timedelta(days=1),
freq = pd.Timedelta(hours=1),
inclusive = "left"
)
mock_weather_df = pd.DataFrame({ "date": mock_hourly_date })
# Fill with zeroes (fake weather data)
for v in variables:
mock_weather_df[v] = 0.0
return mock_weather_df

monkeypatch.setattr(WeatherService, "get_hourly_weather", mock_get_hourly_weather)
Loading