diff --git a/README.md b/README.md index 6c7f5de7..474d911f 100644 --- a/README.md +++ b/README.md @@ -305,8 +305,34 @@ Modified from the great works of * cjtapper/solcast-py * home-assistant-libs/forecast_solar +## Known issues + +- The variable 'tally' should never be unavailable during a forecast fetch retry sequence, but it can be for some reason. This causes site 'forecast today' sensor to show as 'Unknown' until the retries are exhausted, or a successful fetch occurs. + +- The call for get current API usage on startup is not correctly retried. This will be fixed in a future release. + +- Startup times can be delayed when the Solcast API is returning 429/Too busy status, which is caused by retries. This will be fixed in a future release by getting the data immediately from cache if it exists. + ## Changes +v4.0.33 +- Performance improvements for sensor updates by @isorin, including: +- Reduced the update interval of sensors to 5 minutes +- Split the sensors into two groups: sensors that need to be updated every 5 minutes and sensors that need to be updated only when the data is refreshed or the date changes (daily values) +- Fixed issues with removing the past forecasts (older than 2 years), broken code +- Improve the functionality of the forecasts, for exmaple "forecast_remaining_today" is updated every 5 minutes by calculating the remaining energy from the current 30 minute interval. Same for "now/next hour" sensors. +- Redaction of Solcast API key in logs by @isorin +- Revert Oziee '4.0.23' async_update_options #54 by @autoSteve, which was causing dampening update issues + +A comment from @isorin: "_I use the forecast_remaining_today to determine the time of the day when to start charging the batteries so that they will reach a predetermined charge in the evening. With my changes, this is possible._" + +To that, I say nicely done. + +New Contributors +- @isorin made their first contribution in https://github.com/BJReplay/ha-solcast-solar/pull/45 + +Full Changelog: https://github.com/BJReplay/ha-solcast-solar/compare/v4.0.32...v4.0.33 + v4.0.32 - Bug fix: Independent API use counter for each Solcast account by @autoSteve - Bug fix: Force all caches to /config/ for all platforms (fixes Docker deployments) #43 by @autoSteve @@ -317,9 +343,6 @@ v4.0.32 Full Changelog: https://github.com/BJReplay/ha-solcast-solar/compare/v4.0.31...v4.0.32 -Known issues -- The variable 'tally' should never be unavailable during a forecast fetch retry sequence, but it can be for some reason. This causes site 'forecast today' sensor to show as 'Unknown' until the retries are exhausted, or a successful fetch occurs. - v4.0.31 - docs: Changes to README.md - docs: Add troubleshooting notes. diff --git a/custom_components/solcast_solar/__init__.py b/custom_components/solcast_solar/__init__.py index 9e68719c..c670108b 100644 --- a/custom_components/solcast_solar/__init__.py +++ b/custom_components/solcast_solar/__init__.py @@ -293,16 +293,8 @@ async def async_remove_config_entry_device(hass: HomeAssistant, entry: ConfigEnt return True async def async_update_options(hass: HomeAssistant, entry: ConfigEntry): - """Handle options update. Only reload if any item was changed""" - if any( - entry.data.get(attrib) != entry.options.get(attrib) - for attrib in (DAMP_FACTOR, HARD_LIMIT,KEY_ESTIMATE, CUSTOM_HOUR_SENSOR, CONF_API_KEY) - ): - # update entry replacing data with new options - hass.config_entries.async_update_entry( - entry, data={**entry.data, **entry.options} - ) - await hass.config_entries.async_reload(entry.entry_id) + """Reload...""" + await hass.config_entries.async_reload(entry.entry_id) async def async_migrate_entry(hass: HomeAssistant, config_entry: ConfigEntry) -> bool: """Migrate old entry.""" diff --git a/custom_components/solcast_solar/coordinator.py b/custom_components/solcast_solar/coordinator.py index 92c5ee83..d2f7c091 100644 --- a/custom_components/solcast_solar/coordinator.py +++ b/custom_components/solcast_solar/coordinator.py @@ -1,5 +1,6 @@ """The Solcast Solar integration.""" from __future__ import annotations +from datetime import datetime as dt import logging import traceback @@ -23,6 +24,9 @@ def __init__(self, hass: HomeAssistant, solcast: SolcastApi, version: str) -> No self._hass = hass self._previousenergy = None self._version = version + self._lastDay = None + self._dayChanged = False + self._dataUpdated = False super().__init__( hass, @@ -38,17 +42,22 @@ async def _async_update_data(self): async def setup(self): d={} self._previousenergy = d - + self._lastDay = dt.now(self.solcast._tz).day try: #4.0.18 - added reset usage call to reset usage sensors at UTC midnight async_track_utc_time_change(self._hass, self.update_utcmidnight_usage_sensor_data, hour=0,minute=0,second=0) - async_track_utc_time_change(self._hass, self.update_integration_listeners, second=0) + async_track_utc_time_change(self._hass, self.update_integration_listeners, minute=range(0, 60, 5), second=0) except Exception as error: _LOGGER.error("SOLCAST - Error coordinator setup: %s", traceback.format_exc()) async def update_integration_listeners(self, *args): try: + crtDay = dt.now(self.solcast._tz).day + self._dateChanged = (crtDay != self._lastDay) + if self._dateChanged: + self._lastDay = crtDay + self.async_update_listeners() except Exception: #_LOGGER.error("SOLCAST - update_integration_listeners: %s", traceback.format_exc()) @@ -56,9 +65,7 @@ async def update_integration_listeners(self, *args): async def update_utcmidnight_usage_sensor_data(self, *args): try: - for k in self.solcast._api_used.keys(): - self.solcast._api_used[k] = 0 - self.async_update_listeners() + await self.solcast.reset_api_usage() except Exception: #_LOGGER.error("SOLCAST - update_utcmidnight_usage_sensor_data: %s", traceback.format_exc()) pass @@ -66,7 +73,9 @@ async def update_utcmidnight_usage_sensor_data(self, *args): async def service_event_update(self, *args): #await self.solcast.sites_weather() await self.solcast.http_data(dopast=False) + self._dataUpdated = True await self.update_integration_listeners() + self._dataUpdated = False async def service_event_delete_old_solcast_json_file(self, *args): await self.solcast.delete_solcast_file() @@ -78,84 +87,75 @@ def get_energy_tab_data(self): return self.solcast.get_energy_data() def get_sensor_value(self, key=""): - if key == "total_kwh_forecast_today": - return self.solcast.get_total_kwh_forecast_day(0) - elif key == "peak_w_today": - return self.solcast.get_peak_w_day(0) - elif key == "peak_w_time_today": - return self.solcast.get_peak_w_time_day(0) - elif key == "forecast_this_hour": - return self.solcast.get_forecast_n_hour(0) - elif key == "forecast_next_hour": - return self.solcast.get_forecast_n_hour(1) - elif key == "forecast_custom_hour": - return self.solcast.get_forecast_custom_hour(self.solcast._customhoursensor) - elif key == "forecast_next_12hour": - return self.solcast.get_forecast_n_hour(12) - elif key == "forecast_next_24hour": - return self.solcast.get_forecast_n_hour(24) - elif key == "total_kwh_forecast_tomorrow": - return self.solcast.get_total_kwh_forecast_day(1) - elif key == "total_kwh_forecast_d3": - return self.solcast.get_total_kwh_forecast_day(2) - elif key == "total_kwh_forecast_d4": - return self.solcast.get_total_kwh_forecast_day(3) - elif key == "total_kwh_forecast_d5": - return self.solcast.get_total_kwh_forecast_day(4) - elif key == "total_kwh_forecast_d6": - return self.solcast.get_total_kwh_forecast_day(5) - elif key == "total_kwh_forecast_d7": - return self.solcast.get_total_kwh_forecast_day(6) - elif key == "power_now": - return self.solcast.get_power_production_n_mins(0) - elif key == "power_now_30m": - return self.solcast.get_power_production_n_mins(30) - elif key == "power_now_1hr": - return self.solcast.get_power_production_n_mins(60) - elif key == "power_now_12hr": - return self.solcast.get_power_production_n_mins(60*12) - elif key == "power_now_24hr": - return self.solcast.get_power_production_n_mins(60*24) - elif key == "peak_w_tomorrow": - return self.solcast.get_peak_w_day(1) - elif key == "peak_w_time_tomorrow": - return self.solcast.get_peak_w_time_day(1) - elif key == "get_remaining_today": - return self.solcast.get_remaining_today() - elif key == "api_counter": - return self.solcast.get_api_used_count() - elif key == "api_limit": - return self.solcast.get_api_limit() - elif key == "lastupdated": - return self.solcast.get_last_updated_datetime() - elif key == "hard_limit": - #return self.solcast._hardlimit < 100 - return False if self.solcast._hardlimit == 100 else f"{round(self.solcast._hardlimit * 1000)}w" - # elif key == "weather_description": - # return self.solcast.get_weather() - - - #just in case - return None + match key: + case "total_kwh_forecast_today": + return self.solcast.get_total_kwh_forecast_day(0) + case "peak_w_today": + return self.solcast.get_peak_w_day(0) + case "peak_w_time_today": + return self.solcast.get_peak_w_time_day(0) + case "forecast_this_hour": + return self.solcast.get_forecast_n_hour(0) + case "forecast_next_hour": + return self.solcast.get_forecast_n_hour(1) + case "forecast_custom_hour": + return self.solcast.get_forecast_custom_hours(self.solcast._customhoursensor) + case "total_kwh_forecast_tomorrow": + return self.solcast.get_total_kwh_forecast_day(1) + case "total_kwh_forecast_d3": + return self.solcast.get_total_kwh_forecast_day(2) + case "total_kwh_forecast_d4": + return self.solcast.get_total_kwh_forecast_day(3) + case "total_kwh_forecast_d5": + return self.solcast.get_total_kwh_forecast_day(4) + case "total_kwh_forecast_d6": + return self.solcast.get_total_kwh_forecast_day(5) + case "total_kwh_forecast_d7": + return self.solcast.get_total_kwh_forecast_day(6) + case "power_now": + return self.solcast.get_power_n_mins(0) + case "power_now_30m": + return self.solcast.get_power_n_mins(30) + case "power_now_1hr": + return self.solcast.get_power_n_mins(60) + case "peak_w_tomorrow": + return self.solcast.get_peak_w_day(1) + case "peak_w_time_tomorrow": + return self.solcast.get_peak_w_time_day(1) + case "get_remaining_today": + return self.solcast.get_forecast_remaining_today() + case "api_counter": + return self.solcast.get_api_used_count() + case "api_limit": + return self.solcast.get_api_limit() + case "lastupdated": + return self.solcast.get_last_updated_datetime() + case "hard_limit": + #return self.solcast._hardlimit < 100 + return False if self.solcast._hardlimit == 100 else f"{round(self.solcast._hardlimit * 1000)}w" + # case "weather_description": + # return self.solcast.get_weather() + case _: + return None def get_sensor_extra_attributes(self, key=""): - if key == "total_kwh_forecast_today": - return self.solcast.get_forecast_day(0) - elif key == "total_kwh_forecast_tomorrow": - return self.solcast.get_forecast_day(1) - elif key == "total_kwh_forecast_d3": - return self.solcast.get_forecast_day(2) - elif key == "total_kwh_forecast_d4": - return self.solcast.get_forecast_day(3) - elif key == "total_kwh_forecast_d5": - return self.solcast.get_forecast_day(4) - elif key == "total_kwh_forecast_d6": - return self.solcast.get_forecast_day(5) - elif key == "total_kwh_forecast_d7": - return self.solcast.get_forecast_day(6) - - #just in case - return None + match key: + case "total_kwh_forecast_today": + return self.solcast.get_forecast_day(0) + case "total_kwh_forecast_tomorrow": + return self.solcast.get_forecast_day(1) + case "total_kwh_forecast_d3": + return self.solcast.get_forecast_day(2) + case "total_kwh_forecast_d4": + return self.solcast.get_forecast_day(3) + case "total_kwh_forecast_d5": + return self.solcast.get_forecast_day(4) + case "total_kwh_forecast_d6": + return self.solcast.get_forecast_day(5) + case "total_kwh_forecast_d7": + return self.solcast.get_forecast_day(6) + case _: + return None def get_site_sensor_value(self, roof_id, key): match key: diff --git a/custom_components/solcast_solar/manifest.json b/custom_components/solcast_solar/manifest.json index fe49c006..ce36cb00 100644 --- a/custom_components/solcast_solar/manifest.json +++ b/custom_components/solcast_solar/manifest.json @@ -10,5 +10,5 @@ "iot_class": "cloud_polling", "issue_tracker": "https://github.com/BJReplay/ha-solcast-solar/issues", "requirements": ["aiohttp>=3.8.5", "datetime>=4.3", "isodate>=0.6.1"], - "version": "4.0.32" + "version": "4.0.33" } diff --git a/custom_components/solcast_solar/sensor.py b/custom_components/solcast_solar/sensor.py index f37a2e21..910997ac 100755 --- a/custom_components/solcast_solar/sensor.py +++ b/custom_components/solcast_solar/sensor.py @@ -5,6 +5,7 @@ import logging import traceback from dataclasses import dataclass +from enum import Enum from homeassistant.components.sensor import ( SensorDeviceClass, @@ -34,7 +35,6 @@ _LOGGER = logging.getLogger(__name__) - SENSORS: dict[str, SensorEntityDescription] = { "total_kwh_forecast_today": SensorEntityDescription( key="total_kwh_forecast_today", @@ -232,6 +232,25 @@ #), } +class SensorUpdatePolicy(Enum): + DEFAULT = 0 + EVERY_TIME_INTERVAL = 1 + +def getSensorUpdatePolicy(key) -> SensorUpdatePolicy: + match key: + case ( + "forecast_this_hour" | + "forecast_next_hour" | + "forecast_custom_hour" | + "forecast_remaining_today" | + "get_remaining_today" | + "power_now" | + "power_now_30m" | + "power_now_1hr" + ): + return SensorUpdatePolicy.EVERY_TIME_INTERVAL + case _: + return SensorUpdatePolicy.DEFAULT async def async_setup_entry( hass: HomeAssistant, @@ -289,6 +308,7 @@ def __init__( self.entity_description = entity_description self.coordinator = coordinator + self.update_policy = getSensorUpdatePolicy(entity_description.key) self._attr_unique_id = f"{entity_description.key}" self._attributes = {} @@ -344,6 +364,15 @@ def should_poll(self) -> bool: @callback def _handle_coordinator_update(self) -> None: """Handle updated data from the coordinator.""" + + # these sensors will pick-up the change on the next interval update (5mins) + if self.update_policy == SensorUpdatePolicy.EVERY_TIME_INTERVAL and self.coordinator._dataUpdated: + return + + # these sensors update when the date changed or when there is new data + if self.update_policy == SensorUpdatePolicy.DEFAULT and not (self.coordinator._dateChanged or self.coordinator._dataUpdated) : + return + try: self._sensor_data = self.coordinator.get_sensor_value( self.entity_description.key diff --git a/custom_components/solcast_solar/solcastapi.py b/custom_components/solcast_solar/solcastapi.py index e92bd648..e657e023 100644 --- a/custom_components/solcast_solar/solcastapi.py +++ b/custom_components/solcast_solar/solcastapi.py @@ -6,7 +6,10 @@ import copy import json import logging +import math import os +import sys +import time import traceback import random from dataclasses import dataclass @@ -21,6 +24,11 @@ from aiohttp.client_reqrep import ClientResponse from isodate import parse_datetime +# for current func name, specify 0 or no argument. +# for name of caller of current func, specify 1. +# for name of caller of caller of current func, specify 2. etc. +currentFuncName = lambda n=0: sys._getframe(n + 1).f_code.co_name + _JSON_VERSION = 4 _LOGGER = logging.getLogger(__name__) @@ -103,6 +111,28 @@ async def serialize_data(self): async with aiofiles.open(self._filename, "w") as f: await f.write(json.dumps(self._data, ensure_ascii=False, cls=DateTimeEncoder)) + def redact_api_key(self, api_key): + return '*'*6 + api_key[-6:] + + def redact_msg_api_key(self, msg, api_key): + return msg.replace(api_key, self.redact_api_key(api_key)) + + async def write_api_usage_cache_file(self, json_file, json_content, api_key): + _LOGGER.debug(f"SOLCAST - writing API usage cache file: {self.redact_msg_api_key(json_file, api_key)}") + async with aiofiles.open(json_file, 'w') as f: + await f.write(json.dumps(json_content, ensure_ascii=False)) + + def get_api_usage_cache_filename(self, num_entries, entry_name): + return "/config/solcast-usage%s.json" % ("" if num_entries <= 1 else "-" + entry_name) + + async def reset_api_usage(self): + for api_key in self._api_used.keys(): + self._api_used[api_key] = 0 + await self.write_api_usage_cache_file( + self.get_api_usage_cache_filename(len(self._api_used), api_key), + {"daily_limit": self._api_limit[api_key], "daily_limit_consumed": self._api_used[api_key]}, + api_key) + async def sites_data(self): """Request data via the Solcast API.""" @@ -124,7 +154,7 @@ async def sites_data(self): resp_json = json.loads(await f.read()) status = 200 else: - _LOGGER.debug(f"SOLCAST - connecting to - {self.options.host}/rooftop_sites?format=json&api_key=REDACTED") + _LOGGER.debug(f"SOLCAST - connecting to - {self.options.host}/rooftop_sites?format=json&api_key={self.redact_api_key(spl)}") retries = 3 retry = retries success = False @@ -164,7 +194,7 @@ async def sites_data(self): resp_json = json.loads(await f.read()) status = 200 else: - LOGGER.error(f"SOLCAST - cached sites data is not yet available to cope with Solcast API being too busy - at least one successful API call is needed") + _LOGGER.error(f"SOLCAST - cached sites data is not yet available to cope with Solcast API being too busy - at least one successful API call is needed") if status == 200: d = cast(dict, resp_json) @@ -201,12 +231,9 @@ async def sites_usage(self): sitekey = spl.strip() #params = {"format": "json", "api_key": self.options.api_key} params = {"api_key": sitekey} - _LOGGER.debug(f"SOLCAST - getting API limit and usage from solcast for {sitekey}") + _LOGGER.debug(f"SOLCAST - getting API limit and usage from solcast for {self.redact_api_key(sitekey)}") async with async_timeout.timeout(60): - if len(sp) == 1: - apiCacheFileName = "/config/solcast-usage.json" - else: - apiCacheFileName = "/config/solcast-usage-%s.json" % (spl,) + apiCacheFileName = self.get_api_usage_cache_filename(len(sp), sitekey) resp: ClientResponse = await self.aiohttp_session.get( url=f"https://api.solcast.com.au/json/reply/GetUserUsageAllowance", params=params, ssl=False ) @@ -217,9 +244,7 @@ async def sites_usage(self): resp_json = await resp.json(content_type=None) status = resp.status if status == 200: - _LOGGER.debug(f"SOLCAST - writing usage cache") - async with aiofiles.open(apiCacheFileName, 'w') as f: - await f.write(json.dumps(resp_json, ensure_ascii=False)) + await self.write_api_usage_cache_file(apiCacheFileName, resp_json, sitekey) retry = 0 success = True else: @@ -240,7 +265,7 @@ async def sites_usage(self): d = cast(dict, resp_json) self._api_limit[sitekey] = d.get("daily_limit", None) self._api_used[sitekey] = d.get("daily_limit_consumed", None) - _LOGGER.debug(f"SOLCAST - API counter for {sitekey} is {self._api_used[sitekey]}/{self._api_limit[sitekey]}") + _LOGGER.debug(f"SOLCAST - API counter for {self.redact_api_key(sitekey)} is {self._api_used[sitekey]}/{self._api_limit[sitekey]}") else: self._api_limit[sitekey] = 10 self._api_used[sitekey] = 0 @@ -423,20 +448,32 @@ def get_rooftop_site_extra_data(self, rooftopid = ""): return ret + def get_now_utc(self): + return dt.now(self._tz).astimezone(timezone.utc) + + def get_hour_start_utc(self): + return dt.now(self._tz).replace(minute=0, second=0, microsecond=0).astimezone(timezone.utc) + + def get_day_start_utc(self): + return dt.now(self._tz).replace(hour=0, minute=0, second=0, microsecond=0).astimezone(timezone.utc) + def get_forecast_day(self, futureday) -> Dict[str, Any]: - """Return Solcast Forecasts data for N days ahead""" + """Return Solcast Forecasts data for the Nth day ahead""" noDataError = True - tz = self._tz - da = dt.now(tz).date() + timedelta(days=futureday) - h = tuple( - d - for d in self._data_forecasts - if d["period_start"].astimezone(tz).date() == da - ) + start_utc = self.get_day_start_utc() + timedelta(days=futureday) + end_utc = start_utc + timedelta(days=1) + st_i, end_i = self.get_forecast_list_slice(start_utc, end_utc) + h = self._data_forecasts[st_i:end_i] + + _LOGGER.debug("SOLCAST get_forecast_day %d st %s end %s st_i %d end_i %d h.len %d", + futureday, + start_utc.strftime('%Y-%m-%d %H:%M:%S'), + end_utc.strftime('%Y-%m-%d %H:%M:%S'), + st_i, end_i, len(h)) tup = tuple( - {**d, "period_start": d["period_start"].astimezone(tz)} for d in h + {**d, "period_start": d["period_start"].astimezone(self._tz)} for d in h ) if len(tup) < 48: @@ -459,137 +496,129 @@ def get_forecast_day(self, futureday) -> Dict[str, Any]: return { "detailedForecast": tup, "detailedHourly": hourlyturp, - "dayname": da.strftime("%A"), + "dayname": start_utc.astimezone(self._tz).strftime("%A"), "dataCorrect": noDataError, } - def get_forecast_n_hour(self, hourincrement) -> int: - # This technically is for the given hour in UTC time, not local time; - # this is because the Solcast API doesn't provide the local time zone - # and returns 30min intervals that doesn't necessarily align with the - # local time zone. This is a limitation of the Solcast API and not - # this code, so we'll just have to live with it. - try: - da = dt.now(timezone.utc).replace( - minute=0, second=0, microsecond=0 - ) + timedelta(hours=hourincrement) - g = tuple( - d - for d in self._data_forecasts - if d["period_start"] >= da and d["period_start"] < da + timedelta(hours=1) - ) - m = sum(z[self._use_data_field] for z in g) / len(g) - - return int(m * 1000) - except Exception as ex: - return 0 - - def get_forecast_custom_hour(self, hourincrement) -> int: - """Return Custom Sensor Hours forecast for N hours ahead""" - try: - danow = dt.now(timezone.utc).replace( - minute=0, second=0, microsecond=0 - ) - da = dt.now(timezone.utc).replace( - minute=0, second=0, microsecond=0 - ) + timedelta(hours=hourincrement) - g=[] - for d in self._data_forecasts: - if d["period_start"] >= danow and d["period_start"] < da: - g.append(d) - - m = sum(z[self._use_data_field] for z in g) - - return int(m * 500) - except Exception as ex: - return 0 - - def get_power_production_n_mins(self, minuteincrement) -> float: - """Return Solcast Power Now data for N minutes ahead""" - try: - da = dt.now(timezone.utc) + timedelta(minutes=minuteincrement) - m = min( - (z for z in self._data_forecasts), key=lambda x: abs(x["period_start"] - da) - ) - return int(m[self._use_data_field] * 1000) - except Exception as ex: - return 0.0 - - def get_peak_w_day(self, dayincrement) -> int: + def get_forecast_n_hour(self, n_hour) -> int: + """Return Solcast Forecast for the Nth hour""" + start_utc = self.get_hour_start_utc() + timedelta(hours=n_hour) + end_utc = start_utc + timedelta(hours=1) + res = round(500 * self.get_forecast_pv_estimates(start_utc, end_utc)) + return res + + def get_forecast_custom_hours(self, n_hours) -> int: + """Return Solcast Forecast for the next N hours""" + start_utc = self.get_now_utc() + end_utc = start_utc + timedelta(hours=n_hours) + res = round(500 * self.get_forecast_pv_estimates(start_utc, end_utc)) + return res + + def get_power_n_mins(self, n_mins) -> int: + """Return Solcast Power for the next N minutes""" + # uses a rolling 20mins interval (arbitrary decision) to smooth out the transitions between the 30mins intervals + start_utc = self.get_now_utc() + timedelta(minutes=n_mins-10) + end_utc = start_utc + timedelta(minutes=20) + # multiply with 1.5 as the power reported is only for a 20mins interval (out of 30mins) + res = round(1000 * 1.5 * self.get_forecast_pv_estimates(start_utc, end_utc)) + return res + + def get_peak_w_day(self, n_day) -> int: + """Return max kw for rooftop site N days ahead""" + start_utc = self.get_day_start_utc() + timedelta(days=n_day) + end_utc = start_utc + timedelta(days=1) + res = self.get_max_forecast_pv_estimate(start_utc, end_utc) + return 0 if res is None else round(1000 * res[self._use_data_field]) + + def get_peak_w_time_day(self, n_day) -> dt: """Return hour of max kw for rooftop site N days ahead""" + start_utc = self.get_day_start_utc() + timedelta(days=n_day) + end_utc = start_utc + timedelta(days=1) + res = self.get_max_forecast_pv_estimate(start_utc, end_utc) + return res if res is None else res["period_start"] + + def get_forecast_remaining_today(self) -> float: + """Return remaining Forecasts data for today""" + # time remaining today + start_utc = self.get_now_utc() + end_utc = self.get_day_start_utc() + timedelta(days=1) + res = 0.5 * self.get_forecast_pv_estimates(start_utc, end_utc) + return res + + def get_total_kwh_forecast_day(self, n_day) -> float: + """Return total kwh total for rooftop site N days ahead""" + start_utc = self.get_day_start_utc() + timedelta(days=n_day) + end_utc = start_utc + timedelta(days=1) + res = 0.5 * self.get_forecast_pv_estimates(start_utc, end_utc) + return res + + def get_forecast_list_slice(self, start_utc, end_utc): + """Return Solcast pv_estimates list slice [st_i, end_i) for interval [start_utc, end_utc)""" + crt_i = -1 + st_i = -1 + end_i = len(self._data_forecasts) + for d in self._data_forecasts: + crt_i += 1 + d1 = d['period_start'] + d2 = d1 + timedelta(seconds=1800) + # after the last segment + if end_utc <= d1: + end_i = crt_i + break + # first segment + if start_utc < d2 and st_i == -1: + st_i = crt_i + # never found + if st_i == -1: + st_i = 0 + end_i = 0 + return st_i, end_i + + def get_forecast_pv_estimates(self, start_utc, end_utc) -> float: + """Return Solcast pv_estimates for interval [start_utc, end_utc)""" try: - tz = self._tz - da = dt.now(tz).date() + timedelta(days=dayincrement) - g = tuple( - d - for d in self._data_forecasts - if d["period_start"].astimezone(tz).date() == da - ) - m = max(z[self._use_data_field] for z in g) - return int(m * 1000) + res = 0 + st_i, end_i = self.get_forecast_list_slice(start_utc, end_utc) + for d in self._data_forecasts[st_i:end_i]: + d1 = d['period_start'] + d2 = d1 + timedelta(seconds=1800) + s = 1800 + f = d[self._use_data_field] + if start_utc > d1: + s -= (start_utc - d1).total_seconds() + if end_utc < d2: + s -= (d2 - end_utc).total_seconds() + if s < 1800: + f *= s / 1800 + res += f + _LOGGER.debug("SOLCAST %s st %s end %s st_i %d end_i %d res %s", + currentFuncName(1), + start_utc.strftime('%Y-%m-%d %H:%M:%S'), + end_utc.strftime('%Y-%m-%d %H:%M:%S'), + st_i, end_i, round(res,3)) + return res except Exception as ex: + _LOGGER.error(f"SOLCAST - get_forecast_pv_estimates: {ex}") return 0 - def get_peak_w_time_day(self, dayincrement) -> dt: - """Return hour of max kw for rooftop site N days ahead""" + def get_max_forecast_pv_estimate(self, start_utc, end_utc): + """Return max Solcast pv_estimate for the interval [start_utc, end_utc)""" try: - tz = self._tz - da = dt.now(tz).date() + timedelta(days=dayincrement) - g = tuple( - d - for d in self._data_forecasts - if d["period_start"].astimezone(tz).date() == da - ) - #HA strips any TZ info set and forces UTC tz, so dont need to return with local tz info - return max((z for z in g), key=lambda x: x[self._use_data_field])["period_start"] + res = None + st_i, end_i = self.get_forecast_list_slice(start_utc, end_utc) + for d in self._data_forecasts[st_i:end_i]: + if res is None or res[self._use_data_field] < d[self._use_data_field]: + res = d + _LOGGER.debug("SOLCAST %s st %s end %s st_i %d end_i %d res %s", + currentFuncName(1), + start_utc.strftime('%Y-%m-%d %H:%M:%S'), + end_utc.strftime('%Y-%m-%d %H:%M:%S'), + st_i, end_i, res) + return res except Exception as ex: + _LOGGER.error(f"SOLCAST - get_max_forecast_pv_estimate: {ex}") return None - def get_remaining_today(self) -> float: - """Return Remaining Forecasts data for today""" - try: - tz = self._tz - da = dt.now(tz).replace(second=0, microsecond=0) - - if da.minute < 30: - da = da.replace(minute=0) - else: - da = da.replace(minute=30) - - g = tuple( - d - for d in self._data_forecasts - if d["period_start"].astimezone(tz).date() == da.date() and d["period_start"].astimezone(tz) >= da - ) - - return sum(z[self._use_data_field] for z in g) / 2 - except Exception as ex: - return 0.0 - - def get_total_kwh_forecast_day(self, dayincrement) -> float: - """Return total kwh total for rooftop site N days ahead""" - tz = self._tz - d = dt.now(tz) + timedelta(days=dayincrement) - d = d.replace(hour=0, minute=0, second=0, microsecond=0) - needed_delta = d.replace(hour=23, minute=59, second=59, microsecond=0) - d - - ret = 0.0 - for idx in range(1, len(self._data_forecasts)): - prev = self._data_forecasts[idx - 1] - curr = self._data_forecasts[idx] - - prev_date = prev["period_start"].astimezone(tz).date() - cur_date = curr["period_start"].astimezone(tz).date() - if prev_date != cur_date or cur_date != d.date(): - continue - - delta: timedelta = curr["period_start"] - prev["period_start"] - diff_hours = delta.total_seconds() / 3600 - ret += (prev[self._use_data_field] + curr[self._use_data_field]) / 2 * diff_hours - needed_delta -= delta - - return ret - def get_energy_data(self) -> dict[str, Any]: try: return self._dataenergy @@ -603,17 +632,11 @@ async def http_data(self, dopast = False): _LOGGER.warning(f"SOLCAST - not requesting forecast because time is within fifteen minutes of last update ({self.get_last_updated_datetime().astimezone(self._tz)})") return - lastday = dt.now(self._tz) + timedelta(days=7) - lastday = lastday.replace(hour=23,minute=59).astimezone(timezone.utc) - failure = False for site in self._sites: _LOGGER.debug(f"SOLCAST - API polling for rooftop {site['resource_id']}") #site=site['resource_id'], apikey=site['apikey'], - if len(self._sites) == 1: - usageCacheFileName = "/config/solcast-usage.json" - else: - usageCacheFileName = "/config/solcast-usage-%s.json" % (site['apikey'],) + usageCacheFileName = self.get_api_usage_cache_filename(len(self._sites), site['apikey']) result = await self.http_data_call(usageCacheFileName, site['resource_id'], site['apikey'], dopast) if not result: failure = True @@ -630,10 +653,9 @@ async def http_data(self, dopast = False): async def http_data_call(self, usageCacheFileName, r_id = None, api = None, dopast = False): """Request forecast data via the Solcast API.""" - lastday = dt.now(self._tz) + timedelta(days=7) - lastday = lastday.replace(hour=23,minute=59).astimezone(timezone.utc) - pastdays = dt.now(self._tz).date() + timedelta(days=-730) - _LOGGER.debug(f"SOLCAST - Polling API for rooftop_id {r_id}") + lastday = self.get_day_start_utc() + timedelta(days=8) + numhours = math.ceil((lastday - self.get_now_utc()).total_seconds() / 3600) + _LOGGER.debug(f"SOLCAST - Polling API for rooftop_id {r_id} lastday {lastday} numhours {numhours}") _data = [] _data2 = [] @@ -676,7 +698,7 @@ async def http_data_call(self, usageCacheFileName, r_id = None, api = None, dopa } ) - resp_dict = await self.fetch_data(usageCacheFileName, "forecasts", 168, site=r_id, apikey=api, cachedname="forecasts") + resp_dict = await self.fetch_data(usageCacheFileName, "forecasts", numhours, site=r_id, apikey=api, cachedname="forecasts") if resp_dict is None: return False @@ -689,6 +711,7 @@ async def http_data_call(self, usageCacheFileName, r_id = None, api = None, dopa _LOGGER.debug(f"SOLCAST - Solcast returned {len(af)} records (should be 168)") + st_time = time.time() for x in af: z = parse_datetime(x["period_end"]).astimezone(timezone.utc) z = z.replace(second=0, microsecond=0) - timedelta(minutes=30) @@ -707,38 +730,41 @@ async def http_data_call(self, usageCacheFileName, r_id = None, api = None, dopa ) _data = sorted(_data2, key=itemgetter("period_start")) - _forecasts = [] + _fcasts_dict = {} try: - _forecasts = self._data['siteinfo'][r_id]['forecasts'] + for x in self._data['siteinfo'][r_id]['forecasts']: + _fcasts_dict[x["period_start"]] = x except: pass + _LOGGER.debug("SOLCAST http_data_call _fcasts_dict len %s", len(_fcasts_dict)) + for x in _data: #loop each rooftop site and its forecasts - - itm = next((item for item in _forecasts if item["period_start"] == x["period_start"]), None) + + itm = _fcasts_dict.get(x["period_start"]) if itm: itm["pv_estimate"] = x["pv_estimate"] itm["pv_estimate10"] = x["pv_estimate10"] itm["pv_estimate90"] = x["pv_estimate90"] else: # _LOGGER.debug("adding itm") - _forecasts.append({"period_start": x["period_start"],"pv_estimate": x["pv_estimate"], + _fcasts_dict[x["period_start"]] = {"period_start": x["period_start"], + "pv_estimate": x["pv_estimate"], "pv_estimate10": x["pv_estimate10"], - "pv_estimate90": x["pv_estimate90"]}) - - #_forecasts now contains all data for the rooftop site up to 730 days worth - #this deletes data that is older than 730 days (2 years) - for x in _forecasts: - zz = x['period_start'].astimezone(self._tz) - timedelta(minutes=30) - if zz.date() < pastdays: - _forecasts.remove(x) - + "pv_estimate90": x["pv_estimate90"]} + + #_fcasts_dict now contains all data for the rooftop site up to 730 days worth + #this deletes data that is older than 730 days (2 years) + pastdays = dt.now(timezone.utc).date() + timedelta(days=-730) + _forecasts = list(filter(lambda x: x["period_start"].date() >= pastdays, _fcasts_dict.values())) + _forecasts = sorted(_forecasts, key=itemgetter("period_start")) self._data['siteinfo'].update({r_id:{'forecasts': copy.deepcopy(_forecasts)}}) + _LOGGER.info(f"SOLCAST - http_data_call processing took {round(time.time()-st_time,4)}s") return True @@ -779,12 +805,13 @@ async def fetch_data(self, usageCacheFileName, path= "error", hours=168, site="" counter += 1 if status == 200: + _LOGGER.info(f"SOLCAST - Fetch successful") + _LOGGER.debug(f"SOLCAST - API returned data. API Counter incremented from {self._api_used[apikey]} to {self._api_used[apikey] + 1}") self._api_used[apikey] = self._api_used[apikey] + 1 - _LOGGER.debug(f"SOLCAST - writing usage cache") - async with aiofiles.open(usageCacheFileName, 'w') as f: - await f.write(json.dumps({"daily_limit": self._api_limit[apikey], "daily_limit_consumed": self._api_used[apikey]}, ensure_ascii=False)) - _LOGGER.info(f"SOLCAST - Fetch successful") + await self.write_api_usage_cache_file(usageCacheFileName, + {"daily_limit": self._api_limit[apikey], "daily_limit_consumed": self._api_used[apikey]}, + apikey) resp_json = await resp.json(content_type=None) @@ -858,10 +885,11 @@ async def buildforecastdata(self): try: today = dt.now(self._tz).date() yesterday = dt.now(self._tz).date() + timedelta(days=-730) - lastday = dt.now(self._tz).date() + timedelta(days=7) + lastday = dt.now(self._tz).date() + timedelta(days=8) - _forecasts = {} + _fcasts_dict = {} + st_time = time.time() for s, siteinfo in self._data['siteinfo'].items(): tally = 0 for x in siteinfo['forecasts']: @@ -876,31 +904,43 @@ async def buildforecastdata(self): if zz.date() == today: tally += min(x[self._use_data_field] * 0.5 * self._damp[h], self._hardlimit) - itm = _forecasts.get(z) + itm = _fcasts_dict.get(z) if itm: itm["pv_estimate"] = min(round(itm["pv_estimate"] + (x["pv_estimate"] * self._damp[h]),4), self._hardlimit) itm["pv_estimate10"] = min(round(itm["pv_estimate10"] + (x["pv_estimate10"] * self._damp[h]),4), self._hardlimit) itm["pv_estimate90"] = min(round(itm["pv_estimate90"] + (x["pv_estimate90"] * self._damp[h]),4), self._hardlimit) else: - _forecasts[z] = {"period_start": z,"pv_estimate": min(round((x["pv_estimate"]* self._damp[h]),4), self._hardlimit), - "pv_estimate10": min(round((x["pv_estimate10"]* self._damp[h]),4), self._hardlimit), - "pv_estimate90": min(round((x["pv_estimate90"]* self._damp[h]),4), self._hardlimit)} + _fcasts_dict[z] = {"period_start": z, + "pv_estimate": min(round((x["pv_estimate"]* self._damp[h]),4), self._hardlimit), + "pv_estimate10": min(round((x["pv_estimate10"]* self._damp[h]),4), self._hardlimit), + "pv_estimate90": min(round((x["pv_estimate90"]* self._damp[h]),4), self._hardlimit)} siteinfo['tally'] = round(tally, 4) - self._data_forecasts = list(_forecasts.values()) - self._data_forecasts.sort(key=itemgetter("period_start")) + self._data_forecasts = sorted(_fcasts_dict.values(), key=itemgetter("period_start")) + + self._dataenergy = {"wh_hours": self.makeenergydict()} + + await self.removePastForecastData() await self.checkDataRecords() - self._dataenergy = {"wh_hours": self.makeenergydict()} + _LOGGER.info(f"SOLCAST - buildforecastdata processing took {round(time.time()-st_time,4)}s") except Exception as e: _LOGGER.error("SOLCAST - http_data error: %s", traceback.format_exc()) + + + async def removePastForecastData(self): + _LOGGER.debug("SOLCAST - removePastForecastData forecasts len in %s", len(self._data_forecasts)) + midnight_utc = dt.now(self._tz).replace(hour=0, minute=0, second=0, microsecond=0).astimezone(timezone.utc) + self._data_forecasts = list(filter(lambda x: x["period_start"] >= midnight_utc, self._data_forecasts)) + _LOGGER.debug("SOLCAST - removePastForecastData midnight_utc %s, forecasts len out %s", midnight_utc, len(self._data_forecasts)) + async def checkDataRecords(self): tz = self._tz - for i in range(0,6): + for i in range(0,8): da = dt.now(tz).date() + timedelta(days=i) h = tuple( d