added full cli package and CI execution
Python Code Quality / python-code-quality (push) Successful in 55s
Hourly Energy Data Sync / sync-energy-data (push) Successful in 17s

This commit is contained in:
Brian Bjarke Jensen
2025-10-27 19:07:25 +01:00
parent bcf02b5714
commit aa6ec36878
22 changed files with 4197 additions and 49 deletions
@@ -0,0 +1,21 @@
"""Energy consumption data ingester for Eloverblik API."""
__version__ = "0.1.0"
from .eloverblik import EloverblikClient
from .database import DatabaseStorage
from .scheduler import ScheduledSync
from .utils import (
transform_metering_point_data,
transform_consumption_data,
calculate_consumption_stats,
)
__all__ = [
"EloverblikClient",
"DatabaseStorage",
"ScheduledSync",
"transform_metering_point_data",
"transform_consumption_data",
"calculate_consumption_stats",
]
@@ -0,0 +1,6 @@
"""CLI module entry point for python -m execution."""
from .cli import main
if __name__ == "__main__":
main()
+374
View File
@@ -0,0 +1,374 @@
"""Command line interface for energy consumption ingester."""
import argparse
import logging
import os
import sys
from typing import Any
# Optional import for development
try:
from dotenv import load_dotenv
_load_dotenv: Any | None = load_dotenv
except ImportError:
_load_dotenv = None
from .eloverblik import EloverblikClient
from .database import DatabaseStorage
from .utils import transform_metering_point_data, transform_consumption_data
from .scheduler import ScheduledSync
def load_environment() -> None:
"""Load environment variables from .env file if available."""
if _load_dotenv:
_load_dotenv()
def setup_logging(verbose: bool = False) -> None:
"""Set up logging configuration."""
level = logging.DEBUG if verbose else logging.INFO
logging.basicConfig(
level=level, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
)
def sync_incremental(hours_back: int = 25, verbose: bool = False) -> int:
"""Sync consumption data incrementally for the last specified hours."""
setup_logging(verbose)
logger = logging.getLogger(__name__)
load_environment()
# Get required environment variables
api_token = os.getenv("ELOVERBLIK_API_TOKEN")
if not api_token:
logger.error("ELOVERBLIK_API_TOKEN environment variable is required")
return 1
try:
# Initialize scheduled sync
sync = ScheduledSync.from_env()
# Connect to database
if not sync.db.connect():
logger.error("Failed to connect to database")
return 1
if not sync.db.create_tables():
logger.error("Failed to create database tables")
return 1
# Perform incremental sync
logger.info("Starting incremental synchronization")
if sync.sync_incremental(hours_back=hours_back):
logger.info("Incremental sync completed successfully")
return 0
else:
logger.error("Incremental sync failed")
return 1
except Exception as e:
logger.error(f"Error during incremental sync: {e}")
return 1
finally:
if "sync" in locals():
sync.db.close()
def setup_database(reset: bool = False, verbose: bool = False) -> int:
"""Setup the database schema."""
setup_logging(verbose)
logger = logging.getLogger(__name__)
load_environment()
db_params: dict[str, Any] = {
"host": os.getenv("DB_HOST", "localhost"),
"port": int(os.getenv("DB_PORT", "5432")),
"database": os.getenv("DB_NAME", "energy_consumption"),
"user": os.getenv("DB_USER"),
"password": os.getenv("DB_PASSWORD"),
}
if not db_params["user"] or not db_params["password"]:
logger.error("DB_USER and DB_PASSWORD environment variables are required")
return 1
try:
db = DatabaseStorage(db_params)
# Connect to database
if not db.connect():
logger.error("Failed to connect to database")
return 1
# Drop tables if reset is requested
if reset or os.getenv("RESET_DB", "").lower() in ("true", "1", "yes"):
logger.info("Resetting database (dropping existing tables)")
if db.connection is not None:
cursor = db.connection.cursor()
cursor.execute("DROP TABLE IF EXISTS consumption_readings CASCADE;")
cursor.execute("DROP TABLE IF EXISTS metering_points CASCADE;")
cursor.execute(
"DROP FUNCTION IF EXISTS update_updated_at_column() CASCADE;"
)
db.connection.commit()
logger.info("Existing tables dropped")
# Create database schema
if db.create_tables():
logger.info("Database schema created successfully")
return 0
else:
logger.error("Failed to create database schema")
return 1
except Exception as e:
logger.error(f"Error setting up database: {e}")
return 1
finally:
if "db" in locals():
db.close()
def health_check(verbose: bool = False) -> int:
"""Perform a health check of the system."""
setup_logging(verbose)
logger = logging.getLogger(__name__)
load_environment()
# Get required environment variables
api_token = os.getenv("ELOVERBLIK_API_TOKEN")
if not api_token:
logger.error("ELOVERBLIK_API_TOKEN environment variable is required")
return 1
db_params: dict[str, Any] = {
"host": os.getenv("DB_HOST", "localhost"),
"port": int(os.getenv("DB_PORT", "5432")),
"database": os.getenv("DB_NAME", "energy_consumption"),
"user": os.getenv("DB_USER"),
"password": os.getenv("DB_PASSWORD"),
}
if not db_params["user"] or not db_params["password"]:
logger.error("DB_USER and DB_PASSWORD environment variables are required")
return 1
try:
from .scheduler import ScheduledSync
logger.info("Starting health check")
# Initialize clients
eloverblik = EloverblikClient(api_token)
db = DatabaseStorage(db_params)
# Create ScheduledSync with proper instances
sync = ScheduledSync(eloverblik, db)
if sync.health_check():
logger.info("Health check passed")
return 0
else:
logger.error("Health check failed")
return 1
except Exception as e:
logger.error(f"Error during health check: {e}")
return 1
def sync_full_backfill(verbose: bool = False) -> int:
"""Perform a full backfill synchronization."""
setup_logging(verbose)
logger = logging.getLogger(__name__)
load_environment()
# Get required environment variables
api_token = os.getenv("ELOVERBLIK_API_TOKEN")
if not api_token:
logger.error("ELOVERBLIK_API_TOKEN environment variable is required")
return 1
try:
# Initialize scheduled sync
sync = ScheduledSync.from_env()
# Connect to database
if not sync.db.connect():
logger.error("Failed to connect to database")
return 1
if not sync.db.create_tables():
logger.error("Failed to create database tables")
return 1
# Perform full backfill sync
logger.info("Starting full backfill synchronization")
if sync.sync_full_backfill():
logger.info("Full backfill sync completed successfully")
return 0
else:
logger.error("Full backfill sync failed")
return 1
except Exception as e:
logger.error(f"Error during full backfill sync: {e}")
return 1
finally:
if "sync" in locals():
sync.db.close()
def sync_data(days: int = 7, verbose: bool = False) -> None:
"""
Sync energy consumption data from Eloverblik to PostgreSQL.
Args:
days: Number of days to sync (from today backwards)
verbose: Enable verbose logging
"""
setup_logging(verbose)
logger = logging.getLogger(__name__)
# Load environment variables
load_environment()
try:
# Initialize clients
eloverblik = EloverblikClient.from_env()
db = DatabaseStorage.from_env()
# Connect to database and create tables
if not db.connect():
logger.error("Failed to connect to database")
sys.exit(1)
if not db.create_tables():
logger.error("Failed to create database tables")
sys.exit(1)
# Get metering points
logger.info("Fetching metering points...")
metering_points = eloverblik.get_metering_points()
if not metering_points:
logger.error("No metering points found")
sys.exit(1)
logger.info(f"Found {len(metering_points)} metering points")
# Process each metering point
for mp_data in metering_points:
metering_point_id = mp_data["meteringPointId"]
logger.info(f"Processing metering point: {metering_point_id}")
# Store metering point data
transformed_mp = transform_metering_point_data(mp_data)
if not db.store_metering_point(transformed_mp):
logger.error(f"Failed to store metering point {metering_point_id}")
continue
# Get consumption data
logger.info(f"Fetching {days} days of consumption data...")
consumption_data = eloverblik.get_consumption_data_last_days(
metering_point_id, days
)
if not consumption_data:
logger.warning(f"No consumption data for {metering_point_id}")
continue
# Transform and store consumption data
_, readings = transform_consumption_data(consumption_data)
if readings:
if db.store_consumption_readings(metering_point_id, readings):
logger.info(
f"Stored {len(readings)} readings for {metering_point_id}"
)
else:
logger.error(f"Failed to store readings for {metering_point_id}")
else:
logger.warning(f"No readings to store for {metering_point_id}")
logger.info("Data synchronization completed successfully")
except Exception as e:
logger.error(f"Error during data sync: {e}")
sys.exit(1)
finally:
if "db" in locals():
db.close()
def main() -> None:
"""Main CLI entry point."""
parser = argparse.ArgumentParser(
description="Energy consumption data ingester for Eloverblik API"
)
subparsers = parser.add_subparsers(dest="command", help="Available commands")
# Sync command (legacy - full sync)
sync_parser = subparsers.add_parser(
"sync", help="Full sync from Eloverblik to PostgreSQL"
)
sync_parser.add_argument(
"--days", type=int, default=7, help="Number of days to sync (default: 7)"
)
sync_parser.add_argument(
"--verbose", action="store_true", help="Enable verbose logging"
)
# Incremental sync command (optimized for CI)
incr_parser = subparsers.add_parser(
"sync-incremental",
help="Incremental sync for CI/CD pipelines (recommended for hourly jobs)",
)
incr_parser.add_argument(
"--hours-back",
type=int,
default=25,
help="Hours to look back for gaps (default: 25)",
)
incr_parser.add_argument(
"--verbose", action="store_true", help="Enable verbose logging"
)
# Health check command
health_parser = subparsers.add_parser(
"health-check", help="Check system health (API connectivity, database, etc.)"
)
health_parser.add_argument(
"--verbose", action="store_true", help="Enable verbose logging"
)
# Database setup command
db_parser = subparsers.add_parser(
"setup-database", help="Set up database tables and schema"
)
db_parser.add_argument(
"--reset",
action="store_true",
help="Reset database (DESTRUCTIVE - drops all tables)",
)
db_parser.add_argument(
"--verbose", action="store_true", help="Enable verbose logging"
)
args = parser.parse_args()
if args.command == "sync":
sync_data(days=args.days, verbose=args.verbose)
elif args.command == "sync-incremental":
sync_incremental(hours_back=args.hours_back, verbose=args.verbose)
elif args.command == "health-check":
health_check(verbose=args.verbose)
elif args.command == "setup-database":
setup_database(reset=args.reset, verbose=args.verbose)
else:
parser.print_help()
if __name__ == "__main__":
main()
+48
View File
@@ -0,0 +1,48 @@
"""Test configuration and example usage."""
from dotenv import load_dotenv
# Load environment variables
load_dotenv()
# Example configuration
EXAMPLE_ENV_FILE = """
# Eloverblik API Configuration
ELOVERBLIK_API_TOKEN=your_refresh_token_here
# Database Configuration
DB_HOST=localhost
DB_NAME=energy_consumption
DB_USER=energy_user
DB_PASSWORD=your_password_here
DB_PORT=5432
"""
# Example usage
EXAMPLE_USAGE = """
# Basic usage example
from energy_consumption_ingester import EloverblikClient, DatabaseStorage
# Initialize clients
client = EloverblikClient.from_env()
db = DatabaseStorage.from_env()
# Connect and sync data
db.connect()
db.create_tables()
# Get metering points
metering_points = client.get_metering_points()
# Get consumption data for last 7 days
consumption_data = client.get_consumption_data_last_days(
metering_points[0]['meteringPointId'],
days=7
)
"""
if __name__ == "__main__":
print("Example .env file content:")
print(EXAMPLE_ENV_FILE)
print("\nExample usage:")
print(EXAMPLE_USAGE)
+310
View File
@@ -0,0 +1,310 @@
"""Database storage module for energy consumption data."""
import logging
from datetime import datetime
from typing import Any, TYPE_CHECKING
if TYPE_CHECKING:
import psycopg2
from psycopg2.extras import RealDictCursor
else:
try:
import psycopg2
from psycopg2.extras import RealDictCursor
except ImportError:
psycopg2 = None # type: ignore
RealDictCursor = None # type: ignore
class DatabaseStorage:
"""PostgreSQL database storage for energy consumption data."""
def __init__(self, db_config: dict[str, Any]):
"""
Initialize database storage.
Args:
db_config: Database connection configuration dict
"""
if psycopg2 is None:
raise ImportError(
"psycopg2 not installed. Install with: uv add psycopg2-binary"
)
self.db_config = db_config
self.connection = None
self.logger = logging.getLogger(__name__)
def connect(self) -> bool:
"""
Establish database connection.
Returns:
True if successful, False otherwise
"""
try:
self.connection = psycopg2.connect(**self.db_config)
self.logger.info("Database connection established")
return True
except Exception as e:
self.logger.error(f"Error connecting to database: {e}")
return False
def create_tables(self) -> bool:
"""
Create database tables if they don't exist.
Returns:
True if successful, False otherwise
"""
if not self.connection:
self.logger.error("No database connection")
return False
create_tables_sql = """
-- Create metering_points table
CREATE TABLE IF NOT EXISTS metering_points (
id SERIAL PRIMARY KEY,
metering_point_id VARCHAR(50) UNIQUE NOT NULL,
street_code VARCHAR(10),
street_name VARCHAR(255),
building_number VARCHAR(20),
floor_id VARCHAR(10),
room_id VARCHAR(10),
city_subdivision_name VARCHAR(255),
municipality_code VARCHAR(10),
location_description TEXT,
settlement_method VARCHAR(10),
meter_reading_occurrence VARCHAR(20),
first_consumer_party_name VARCHAR(255),
second_consumer_party_name VARCHAR(255),
meter_number VARCHAR(50),
consumer_start_date TIMESTAMP WITH TIME ZONE,
type_of_mp VARCHAR(10),
balance_supplier_name VARCHAR(255),
postcode VARCHAR(10),
created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP
);
-- Create consumption_readings table
CREATE TABLE IF NOT EXISTS consumption_readings (
id SERIAL PRIMARY KEY,
metering_point_id VARCHAR(50) REFERENCES metering_points(metering_point_id),
timestamp TIMESTAMP WITH TIME ZONE NOT NULL,
consumption_kwh DECIMAL(10, 3) NOT NULL,
quality VARCHAR(10),
period_resolution VARCHAR(10) DEFAULT 'PT1H',
created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
UNIQUE(metering_point_id, timestamp)
);
-- Create indexes for performance
CREATE INDEX IF NOT EXISTS idx_consumption_metering_point
ON consumption_readings(metering_point_id);
CREATE INDEX IF NOT EXISTS idx_consumption_timestamp
ON consumption_readings(timestamp);
CREATE INDEX IF NOT EXISTS idx_consumption_metering_point_timestamp
ON consumption_readings(metering_point_id, timestamp);
-- Create a function to update the updated_at timestamp
CREATE OR REPLACE FUNCTION update_updated_at_column()
RETURNS TRIGGER AS $$
BEGIN
NEW.updated_at = CURRENT_TIMESTAMP;
RETURN NEW;
END;
$$ language 'plpgsql';
-- Create trigger for metering_points
CREATE TRIGGER update_metering_points_updated_at
BEFORE UPDATE ON metering_points
FOR EACH ROW EXECUTE FUNCTION update_updated_at_column();
"""
try:
with self.connection.cursor() as cursor:
cursor.execute(create_tables_sql)
self.connection.commit()
self.logger.info("Tables created successfully")
return True
except Exception as e:
self.logger.error(f"Error creating tables: {e}")
self.connection.rollback()
return False
def store_metering_point(self, metering_point_data: dict[str, Any]) -> bool:
"""
Store or update metering point information.
Args:
metering_point_data: Dictionary containing metering point data
Returns:
True if successful, False otherwise
"""
if not self.connection:
self.logger.error("No database connection")
return False
try:
with self.connection.cursor() as cursor:
upsert_sql = """
INSERT INTO metering_points (
metering_point_id, street_code, street_name, building_number,
floor_id, room_id, city_subdivision_name, municipality_code,
location_description, settlement_method, meter_reading_occurrence,
first_consumer_party_name, second_consumer_party_name,
meter_number, consumer_start_date, type_of_mp,
balance_supplier_name, postcode
) VALUES (
%(meteringPointId)s, %(streetCode)s, %(streetName)s, %(buildingNumber)s,
%(floorId)s, %(roomId)s, %(citySubDivisionName)s, %(municipalityCode)s,
%(locationDescription)s, %(settlementMethod)s, %(meterReadingOccurrence)s,
%(firstConsumerPartyName)s, %(secondConsumerPartyName)s,
%(meterNumber)s, %(consumerStartDate)s, %(typeOfMP)s,
%(balanceSupplierName)s, %(postcode)s
)
ON CONFLICT (metering_point_id)
DO UPDATE SET
street_code = EXCLUDED.street_code,
street_name = EXCLUDED.street_name,
building_number = EXCLUDED.building_number,
updated_at = CURRENT_TIMESTAMP;
"""
cursor.execute(upsert_sql, metering_point_data)
self.connection.commit()
self.logger.info(
f"Metering point {metering_point_data['meteringPointId']} stored"
)
return True
except Exception as e:
self.logger.error(f"Error storing metering point: {e}")
self.connection.rollback()
return False
def store_consumption_readings(
self, metering_point_id: str, consumption_data: list[dict[str, Any]]
) -> bool:
"""
Store consumption readings with conflict handling.
Args:
metering_point_id: The metering point identifier
consumption_data: List of consumption reading dictionaries
Returns:
True if successful, False otherwise
"""
if not self.connection:
self.logger.error("No database connection")
return False
try:
with self.connection.cursor() as cursor:
insert_sql = """
INSERT INTO consumption_readings (
metering_point_id, timestamp, consumption_kwh, quality, period_resolution
) VALUES (
%s, %s, %s, %s, %s
)
ON CONFLICT (metering_point_id, timestamp)
DO UPDATE SET
consumption_kwh = EXCLUDED.consumption_kwh,
quality = EXCLUDED.quality;
"""
readings_data = []
for reading in consumption_data:
readings_data.append(
(
metering_point_id,
reading["timestamp"],
reading["consumption_kwh"],
reading["quality"],
reading.get("period_resolution", "PT1H"),
)
)
cursor.executemany(insert_sql, readings_data)
self.connection.commit()
self.logger.info(f"Stored {len(readings_data)} consumption readings")
return True
except Exception as e:
self.logger.error(f"Error storing consumption readings: {e}")
self.connection.rollback()
return False
def get_latest_reading_timestamp(self, metering_point_id: str) -> datetime | None:
"""
Get the timestamp of the latest reading for a metering point.
Args:
metering_point_id: The metering point identifier
Returns:
Latest timestamp or None if no readings exist
"""
if not self.connection:
self.logger.error("No database connection")
return None
try:
with self.connection.cursor() as cursor:
cursor.execute(
"""
SELECT MAX(timestamp)
FROM consumption_readings
WHERE metering_point_id = %s
""",
(metering_point_id,),
)
result = cursor.fetchone()
return result[0] if result[0] else None
except Exception as e:
self.logger.error(f"Error getting latest reading timestamp: {e}")
return None
def close(self) -> None:
"""Close database connection."""
if self.connection:
self.connection.close()
self.logger.info("Database connection closed")
@classmethod
def from_env(cls) -> "DatabaseStorage":
"""
Create DatabaseStorage from environment variables.
Expected environment variables:
- DB_HOST
- DB_NAME
- DB_USER
- DB_PASSWORD
- DB_PORT (optional, defaults to 5432)
Returns:
DatabaseStorage instance
"""
import os
db_config = {
"host": os.getenv("DB_HOST", "localhost"),
"database": os.getenv("DB_NAME"),
"user": os.getenv("DB_USER"),
"password": os.getenv("DB_PASSWORD"),
"port": int(os.getenv("DB_PORT", 5432)),
}
# Check for required environment variables
required_vars = ["DB_NAME", "DB_USER", "DB_PASSWORD"]
missing_vars = [var for var in required_vars if not os.getenv(var)]
if missing_vars:
raise ValueError(f"Missing required environment variables: {missing_vars}")
return cls(db_config)
@@ -0,0 +1,220 @@
"""Eloverblik API client for energy consumption data."""
import http.client
import json
import os
from datetime import datetime, timedelta
from typing import Any
from urllib.parse import urlencode
from pydantic import AnyUrl
class EloverblikClient:
"""Client for interacting with the Eloverblik API."""
def __init__(
self, refresh_token: str, server_url: str = "https://api.eloverblik.dk"
):
"""
Initialize the Eloverblik client.
Args:
refresh_token: The refresh token for authentication
server_url: The base URL of the Eloverblik API
"""
self.refresh_token = refresh_token
self.server_url = AnyUrl(server_url)
self.access_token: str | None = None
# Ensure host is available
if not self.server_url.host:
raise ValueError(f"Invalid server URL: {server_url}")
self.host = self.server_url.host
def get_access_token(self) -> bool:
"""
Exchange refresh token for access token.
Returns:
True if successful, False otherwise
"""
conn = http.client.HTTPSConnection(self.host)
headers = {
"Authorization": f"Bearer {self.refresh_token}",
"Content-Type": "application/json",
}
try:
conn.request("GET", "/customerapi/api/token", headers=headers)
res = conn.getresponse()
if res.status == 200:
token_response = res.read().decode("utf-8")
token_data = json.loads(token_response)
self.access_token = token_data["result"]
return True
else:
error_data = res.read().decode("utf-8")
print(f"Error getting access token: {error_data}")
return False
except Exception as e:
print(f"Exception getting access token: {e}")
return False
finally:
conn.close()
def get_metering_points(
self, include_all: bool = True
) -> list[dict[str, Any]] | None:
"""
Get all metering points for the authenticated user.
Args:
include_all: Whether to include all metering points
Returns:
List of metering point data or None if error
"""
if not self.access_token:
if not self.get_access_token():
return None
conn = http.client.HTTPSConnection(self.host)
headers = {
"Authorization": f"Bearer {self.access_token}",
"Content-Type": "application/json",
"api-version": "1.0",
}
params = {"includeAll": include_all}
try:
conn.request(
"GET",
"/customerapi/api/meteringpoints/meteringpoints?" + urlencode(params),
headers=headers,
)
res = conn.getresponse()
if res.status == 200:
data = res.read().decode("utf-8")
metering_data = json.loads(data)
result = metering_data.get("result", [])
return result if isinstance(result, list) else []
else:
error_data = res.read().decode("utf-8")
print(f"Error getting metering points: {error_data}")
return None
except Exception as e:
print(f"Exception getting metering points: {e}")
return None
finally:
conn.close()
def get_consumption_data(
self,
metering_point_id: str,
date_from: str,
date_to: str,
aggregation: str = "Hour",
) -> dict[str, Any] | None:
"""
Get consumption data for a specific metering point.
Args:
metering_point_id: The metering point identifier
date_from: Start date in YYYY-MM-DD format
date_to: End date in YYYY-MM-DD format
aggregation: Data aggregation level (Hour, Day, Month)
Returns:
Consumption data or None if error
"""
if not self.access_token:
if not self.get_access_token():
return None
conn = http.client.HTTPSConnection(self.host)
headers = {
"Authorization": f"Bearer {self.access_token}",
"Content-Type": "application/json",
"api-version": "1.0",
}
request_body = {"meteringPoints": {"meteringPoint": [metering_point_id]}}
try:
conn.request(
"POST",
f"/customerapi/api/meterdata/gettimeseries/{date_from}/{date_to}/{aggregation}",
body=json.dumps(request_body),
headers=headers,
)
res = conn.getresponse()
if res.status == 200:
data = res.read().decode("utf-8")
result = json.loads(data)
return result if isinstance(result, dict) else {}
else:
error_data = res.read().decode("utf-8")
print(f"Error getting consumption data: {error_data}")
return None
except Exception as e:
print(f"Exception getting consumption data: {e}")
return None
finally:
conn.close()
def get_consumption_data_last_days(
self, metering_point_id: str, days: int = 7, aggregation: str = "Hour"
) -> dict[str, Any] | None:
"""
Get consumption data for the last N days.
Args:
metering_point_id: The metering point identifier
days: Number of days to look back
aggregation: Data aggregation level (Hour, Day, Month)
Returns:
Consumption data or None if error
"""
end_date = datetime.now()
start_date = end_date - timedelta(days=days)
date_from = start_date.strftime("%Y-%m-%d")
date_to = end_date.strftime("%Y-%m-%d")
return self.get_consumption_data(
metering_point_id, date_from, date_to, aggregation
)
@classmethod
def from_env(
cls, token_env_var: str = "ELOVERBLIK_API_TOKEN"
) -> "EloverblikClient":
"""
Create client from environment variable.
Args:
token_env_var: Name of environment variable containing the refresh token
Returns:
EloverblikClient instance
Raises:
ValueError: If token environment variable is not set
"""
token = os.getenv(token_env_var)
if not token:
raise ValueError(f"Environment variable {token_env_var} is not set")
return cls(refresh_token=token)
@@ -0,0 +1,235 @@
"""Scheduled synchronization functionality for CI/CD pipelines."""
import logging
from datetime import datetime, timedelta
from .eloverblik import EloverblikClient
from .database import DatabaseStorage
from .utils import transform_metering_point_data, transform_consumption_data
class ScheduledSync:
"""Handles scheduled synchronization of energy consumption data."""
def __init__(self, client: EloverblikClient, db: DatabaseStorage):
"""
Initialize scheduled sync.
Args:
client: Eloverblik API client
db: Database storage instance
"""
self.client = client
self.db = db
self.logger = logging.getLogger(__name__)
def sync_incremental(self, hours_back: int = 25) -> bool:
"""
Perform incremental sync - only sync recent data.
Args:
hours_back: Number of hours to look back (default 25 to handle
timezone issues and ensure no gaps)
Returns:
True if successful, False otherwise
"""
try:
self.logger.info(f"Starting incremental sync for last {hours_back} hours")
# Get metering points
metering_points = self.client.get_metering_points()
if not metering_points:
self.logger.error("No metering points found")
return False
success_count = 0
total_readings = 0
for mp_data in metering_points:
metering_point_id = mp_data["meteringPointId"]
try:
# Store/update metering point data
transformed_mp = transform_metering_point_data(mp_data)
self.db.store_metering_point(transformed_mp)
# Get latest reading timestamp from database
latest_timestamp = self.db.get_latest_reading_timestamp(
metering_point_id
)
# Calculate date range for sync
end_date = datetime.now()
if latest_timestamp:
# Sync from last reading with some overlap
start_date = latest_timestamp - timedelta(hours=1)
self.logger.info(
f"Syncing {metering_point_id} from {start_date} (last reading: {latest_timestamp})"
)
else:
# No previous data, sync last N hours
start_date = end_date - timedelta(hours=hours_back)
self.logger.info(
f"No previous data for {metering_point_id}, syncing last {hours_back} hours"
)
# Format dates for API
date_from = start_date.strftime("%Y-%m-%d")
date_to = end_date.strftime("%Y-%m-%d")
# Get consumption data
consumption_data = self.client.get_consumption_data(
metering_point_id, date_from, date_to
)
if consumption_data:
# Transform and store
_, readings = transform_consumption_data(consumption_data)
if readings:
if self.db.store_consumption_readings(
metering_point_id, readings
):
total_readings += len(readings)
self.logger.info(
f"Stored {len(readings)} readings for {metering_point_id}"
)
else:
self.logger.error(
f"Failed to store readings for {metering_point_id}"
)
continue
success_count += 1
except Exception as e:
self.logger.error(f"Error processing {metering_point_id}: {e}")
continue
self.logger.info(
f"Incremental sync completed: {success_count}/{len(metering_points)} "
f"metering points, {total_readings} total readings"
)
return success_count > 0
except Exception as e:
self.logger.error(f"Incremental sync failed: {e}")
return False
def sync_full_backfill(self, days: int = 30) -> bool:
"""
Perform full backfill sync - sync large amount of historical data.
Args:
days: Number of days to backfill
Returns:
True if successful, False otherwise
"""
try:
self.logger.info(f"Starting full backfill sync for last {days} days")
metering_points = self.client.get_metering_points()
if not metering_points:
self.logger.error("No metering points found")
return False
success_count = 0
total_readings = 0
for mp_data in metering_points:
metering_point_id = mp_data["meteringPointId"]
try:
# Store metering point data
transformed_mp = transform_metering_point_data(mp_data)
self.db.store_metering_point(transformed_mp)
# Get consumption data for the full period
consumption_data = self.client.get_consumption_data_last_days(
metering_point_id, days
)
if consumption_data:
_, readings = transform_consumption_data(consumption_data)
if readings:
if self.db.store_consumption_readings(
metering_point_id, readings
):
total_readings += len(readings)
self.logger.info(
f"Stored {len(readings)} readings for {metering_point_id}"
)
else:
self.logger.error(
f"Failed to store readings for {metering_point_id}"
)
continue
success_count += 1
except Exception as e:
self.logger.error(f"Error processing {metering_point_id}: {e}")
continue
self.logger.info(
f"Full backfill completed: {success_count}/{len(metering_points)} "
f"metering points, {total_readings} total readings"
)
return success_count > 0
except Exception as e:
self.logger.error(f"Full backfill sync failed: {e}")
return False
def health_check(self) -> bool:
"""
Perform a health check of the system.
Returns:
True if all systems are healthy, False otherwise
"""
try:
self.logger.info("Starting health check")
# Check API connection
if not self.client.get_access_token():
self.logger.error("Failed to get API access token")
return False
# Check database connection
if not self.db.connection:
self.logger.error("No database connection")
return False
# Try to get metering points
metering_points = self.client.get_metering_points()
if not metering_points:
self.logger.error("No metering points found")
return False
self.logger.info(
f"Health check passed: {len(metering_points)} metering points available"
)
return True
except Exception as e:
self.logger.error(f"Health check failed: {e}")
return False
@classmethod
def from_env(cls) -> "ScheduledSync":
"""
Create ScheduledSync from environment variables.
Returns:
ScheduledSync instance
"""
client = EloverblikClient.from_env()
db = DatabaseStorage.from_env()
return cls(client, db)
+124
View File
@@ -0,0 +1,124 @@
"""Utility functions for data transformation and processing."""
from datetime import datetime, timedelta
from typing import Any
def transform_metering_point_data(api_data: dict[str, Any]) -> dict[str, Any]:
"""
Transform API metering point data to database format.
Args:
api_data: Raw metering point data from API
Returns:
Dictionary formatted for database storage
"""
# Parse consumer start date if present
consumer_start_date = None
if api_data.get("consumerStartDate"):
try:
consumer_start_date = datetime.fromisoformat(
api_data["consumerStartDate"].replace("Z", "+00:00")
)
except (ValueError, AttributeError):
pass
return {
"meteringPointId": api_data.get("meteringPointId"),
"streetCode": api_data.get("streetCode"),
"streetName": api_data.get("streetName"),
"buildingNumber": api_data.get("buildingNumber"),
"floorId": api_data.get("floorId"),
"roomId": api_data.get("roomId"),
"citySubDivisionName": api_data.get("citySubDivisionName"),
"municipalityCode": api_data.get("municipalityCode"),
"locationDescription": api_data.get("locationDescription"),
"settlementMethod": api_data.get("settlementMethod"),
"meterReadingOccurrence": api_data.get("meterReadingOccurrence"),
"firstConsumerPartyName": api_data.get("firstConsumerPartyName"),
"secondConsumerPartyName": api_data.get("secondConsumerPartyName"),
"meterNumber": api_data.get("meterNumber"),
"consumerStartDate": consumer_start_date,
"typeOfMP": api_data.get("typeOfMP"),
"balanceSupplierName": api_data.get("balanceSupplierName"),
"postcode": api_data.get("postcode"),
}
def transform_consumption_data(
consumption_json: dict[str, Any],
) -> tuple[str | None, list[dict[str, Any]]]:
"""
Transform API consumption data to database format.
Args:
consumption_json: Raw consumption data from API
Returns:
Tuple of (metering_point_id, list_of_consumption_readings)
"""
if not consumption_json or "result" not in consumption_json:
return None, []
try:
# Extract the time series data
market_document = consumption_json["result"][0]["MyEnergyData_MarketDocument"]
time_series = market_document["TimeSeries"][0]
metering_point_id = time_series["mRID"]
periods = time_series["Period"]
consumption_readings = []
for period in periods:
period_start = datetime.fromisoformat(
period["timeInterval"]["start"].replace("Z", "+00:00")
)
resolution = period["resolution"]
if "Point" in period:
for point in period["Point"]:
# Calculate the actual timestamp for this point
position = int(point["position"])
# Position is 1-based, so subtract 1 to get hours offset
hours_offset = position - 1
point_timestamp = period_start + timedelta(hours=hours_offset)
reading = {
"timestamp": point_timestamp,
"consumption_kwh": float(point["out_Quantity.quantity"]),
"quality": point["out_Quantity.quality"],
"period_resolution": resolution,
}
consumption_readings.append(reading)
return metering_point_id, consumption_readings
except Exception as e:
print(f"Error transforming consumption data: {e}")
return None, []
def calculate_consumption_stats(readings: list[dict[str, Any]]) -> dict[str, float]:
"""
Calculate consumption statistics from readings.
Args:
readings: List of consumption reading dictionaries
Returns:
Dictionary with consumption statistics
"""
if not readings:
return {}
consumptions = [r["consumption_kwh"] for r in readings]
return {
"total_kwh": sum(consumptions),
"average_kwh": sum(consumptions) / len(consumptions),
"min_kwh": min(consumptions),
"max_kwh": max(consumptions),
"count": len(consumptions),
}