From b7797a7301a6abcbe79a231ca37d53ac2ccf4a80 Mon Sep 17 00:00:00 2001 From: Alexandre CUER Date: Sun, 20 Sep 2026 17:30:20 +0200 Subject: [PATCH 1/7] add epw merge tool + basic sql_viewer --- .gitignore | 3 +- sql_viewer/eplus_sql_reader.py | 169 +++++++++++++++++++++++++++++++++ sql_viewer/sql.yaml | 33 +++++++ sql_viewer/sql_compare.yaml | 22 +++++ weather/merge_epw.py | 47 +++++++++ 5 files changed, 273 insertions(+), 1 deletion(-) create mode 100644 sql_viewer/eplus_sql_reader.py create mode 100644 sql_viewer/sql.yaml create mode 100644 sql_viewer/sql_compare.yaml create mode 100644 weather/merge_epw.py diff --git a/.gitignore b/.gitignore index 5b8f9b0..96eeacc 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ osm/* osm_* idf_zip/* -weather_data +weather/data +sql_viewer/sql_db *egg-info ep_simulations/* diff --git a/sql_viewer/eplus_sql_reader.py b/sql_viewer/eplus_sql_reader.py new file mode 100644 index 0000000..ac9e627 --- /dev/null +++ b/sql_viewer/eplus_sql_reader.py @@ -0,0 +1,169 @@ +"""read energyplus results databases""" +import argparse +import contextlib +import sqlite3 +from datetime import datetime, timedelta + +import matplotlib.pyplot as plt +import yaml + + +def list_variables(db): + """list variables stored in the sql""" + with sqlite3.connect(db) as con: + rows = con.execute(""" + SELECT DISTINCT + KeyValue, + Name, + Units + FROM ReportDataDictionary + ORDER BY Name, KeyValue + """) + + for key, name, units in rows: + print(f"{name:50} | {key:40} | {units:10}") + + +def explore(db): + """sql exploration""" + with sqlite3.connect(db) as con: + rows = con.execute(""" + SELECT + EnvironmentPeriodIndex, + EnvironmentName + FROM EnvironmentPeriods + """).fetchall() + + for row in rows: + print(row) + + +def fetch(con, name, key, config): + """fetch datas related to name and key""" + rows = con.execute(""" + SELECT + t.Year, t.Month, t.Day, t.Hour, t.Minute, + r.Value + FROM ReportData r + JOIN ReportDataDictionary d + ON r.ReportDataDictionaryIndex = + d.ReportDataDictionaryIndex + JOIN Time t + ON r.TimeIndex = t.TimeIndex + JOIN EnvironmentPeriods e + ON t.EnvironmentPeriodIndex = + e.EnvironmentPeriodIndex + WHERE d.Name = ? + AND d.KeyValue = ? + AND e.EnvironmentName = ? + ORDER BY t.TimeIndex + """, (name, key, config["environment"])).fetchall() + if not rows: + print(f"WARNING: no data for {name} / {key}") + return None, None + dates = [] + for y, mo, d, h, mi, _ in rows: + dt = datetime(y, mo, d, h % 24, mi) + if h == 24: + dt += timedelta(days=1) + dates.append(dt) + values = [value for *_, value in rows] + return dates, values + + +def multidb_plot_variables(dbs: dict[str, str], config: dict): + """Plot variables declared in the yaml for multiple sql databases. + permits to compare 2 scenarios""" + nb = len(config["variables"]) + if nb < 1: + return + with contextlib.ExitStack() as stack: + cons = { + db_name: stack.enter_context(sqlite3.connect(db_path)) + for db_name, db_path in dbs.items() + } + fig, ax = plt.subplots(nb, 1, sharex=True, squeeze=False) + ax = ax[:, 0] + for i, (thema_name, thema) in enumerate(config["variables"].items()): + for variable in thema: + name = variable["name"] + key = variable["key"] + label = variable.get("label", f"{name} — {key}") + for db_name, con in cons.items(): + db_label = label if db_name == "main" else f"{db_name} {label}" + dates, values = fetch(con, name, key, config) + ax[i].plot(dates, values, label=db_label) + ax[i].set_xlabel("date") + ax[i].set_ylabel(thema_name) + ax[i].grid(True) + ax[i].legend() + fig.tight_layout() + plt.show() + + +def plot_variables(db, config): + """plot variables declared in the yaml + for a single database""" + plt.figure() + nb = len(config["variables"]) + if nb < 1: + return + ax = [] + ax.append(plt.subplot(nb, 1, 1)) + with sqlite3.connect(db) as con: + for i, thema_name in enumerate(config["variables"]): + thema = config["variables"][thema_name] + if i > 0: + ax.append(plt.subplot(nb, 1, i+1, sharex=ax[0])) + for variable in thema: + name = variable["name"] + key = variable["key"] + label = variable.get("label", f"{name} — {key}") + dates, values = fetch(con, name, key, config) + plt.plot(dates, values, label=label) + plt.xlabel("date") + plt.ylabel(thema_name) + plt.grid(True) + plt.legend() + plt.tight_layout() + plt.show() + + +def main(): + """main""" + parser = argparse.ArgumentParser() + parser.add_argument("--list", action="store_true") + parser.add_argument("--plot", action="store_true") + parser.add_argument("--explore", action="store_true") + parser.add_argument("--yml", default="sql.yaml") + args = parser.parse_args() + with open(args.yml, encoding="utf-8") as f: + sql_config = yaml.safe_load(f) + databases = {} + if isinstance(sql_config.get('name'), str): + database = f"{sql_config.get('path')}/{sql_config.get('name')}" + databases["main"] = database + if isinstance(sql_config.get('name'), dict): + for key, value in sql_config.get('name').items(): + database = f"{sql_config.get('path')}/{value}" + databases[key] = database + + if args.list: + for database in databases.values(): + list_variables(database) + return + + if args.plot: + if len(databases) == 1: + plot_variables(databases["main"], sql_config) + return + multidb_plot_variables(databases, sql_config) + return + + if args.explore: + for database in databases.values(): + explore(database) + + +if __name__ == "__main__": + main() diff --git a/sql_viewer/sql.yaml b/sql_viewer/sql.yaml new file mode 100644 index 0000000..f3aa616 --- /dev/null +++ b/sql_viewer/sql.yaml @@ -0,0 +1,33 @@ +path: sql_db +#/Agence_ACF_geo_only/SVG_3loops_CtrldeltaSP_500ml_30KWPAC_Boiler45KW_SP7040_FCU +name: Agence_ACF_geo_only_chauffage_seul_10Y_RSM_FD.sql +#Agence_ACF_geo_only.sql + +environment: "RUN PERIOD" + +variables: + temperature: + - name: "Site Outdoor Air Drybulb Temperature" + key: Environment + label: "Outdoor Temp °C" + - name: "Zone Air Temperature" + key: RPLUS1 + label: "RPLUS1 temp °C" + + #borehole: + # - name: "Ground Heat Exchanger Inlet Temperature" + # key: "BOREHOLE" + # label: "Soil inlet °C" + # - name: "Ground Heat Exchanger Outlet Temperature" + # key: "BOREHOLE" + # label: "Soil outlet °C" + + borehole_average: + - name: "Ground Heat Exchanger Average Borehole Temperature" + key: "BOREHOLE" + label: "Average Borehole °C" + + #heat_transfer: + # - name: "Ground Heat Exchanger Heat Transfer Rate" + # key: "BOREHOLE" + # label: "Borehole Heat Transfer W" diff --git a/sql_viewer/sql_compare.yaml b/sql_viewer/sql_compare.yaml new file mode 100644 index 0000000..06bd393 --- /dev/null +++ b/sql_viewer/sql_compare.yaml @@ -0,0 +1,22 @@ +path: sql_db +#/Agence_ACF_geo_only/SVG_3loops_CtrldeltaSP_500ml_30KWPAC_Boiler45KW_SP7040_FCU +name: + heating_only: Agence_ACF_geo_only_chauffage_seul_10Y.sql + heating_n_cooling: Agence_ACF_geo_only_10Y.sql +#Agence_ACF_geo_only.sql + +environment: "RUN PERIOD" + +variables: + T_inlet: + - name: "Ground Heat Exchanger Inlet Temperature" + key: "BOREHOLE" + label: "Soil inlet °C" + T_outlet: + - name: "Ground Heat Exchanger Outlet Temperature" + key: "BOREHOLE" + label: "Soil outlet °C" + average_borehole_T: + - name: "Ground Heat Exchanger Average Borehole Temperature" + key: "BOREHOLE" + label: "Average Borehole Temperature" \ No newline at end of file diff --git a/weather/merge_epw.py b/weather/merge_epw.py new file mode 100644 index 0000000..d14c6e7 --- /dev/null +++ b/weather/merge_epw.py @@ -0,0 +1,47 @@ +"""merge epw files""" +from pathlib import Path +import sys + + +def merge_epw(directory, output, start_year=2025): + """merge all epw files in a directory + adjust years""" + files = sorted(Path(directory).glob("*.epw")) + + if not files: + raise ValueError(f"No EPW files found in {directory}") + + total_hours = 0 + + with open(output, "w", encoding="utf-8") as out: + for i, filename in enumerate(files): + with open(filename, encoding="utf-8") as f: + lines = f.readlines() + + data = lines[8:] + + if i == 0: + out.writelines(lines[:8]) + + year = start_year + i + + for line in data: + fields = line.rstrip("\n").split(",") + if len(fields) >= 35: + fields[0] = str(year) + out.write(",".join(fields) + "\n") + + total_hours += len(data) + + print(f"{len(files)} EPW files") + print(f"{total_hours} hourly records") + print(f"Expected: {len(files) * 8760}") + print(f"Output: {output}") + + +if __name__ == "__main__": + if len(sys.argv) != 3: + print("Usage: python merge_epw.py ") + sys.exit(1) + + merge_epw(sys.argv[1], sys.argv[2]) From 01d9a0745fe879719c9a9b3ad9dd2144890fcd2e Mon Sep 17 00:00:00 2001 From: Alexandre CUER Date: Sun, 20 Sep 2026 17:37:30 +0200 Subject: [PATCH 2/7] cosmetic --- sql_viewer/sql.yaml | 24 +++++++++++------------- sql_viewer/sql_compare.yaml | 4 +--- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/sql_viewer/sql.yaml b/sql_viewer/sql.yaml index f3aa616..3a561d2 100644 --- a/sql_viewer/sql.yaml +++ b/sql_viewer/sql.yaml @@ -1,7 +1,5 @@ path: sql_db -#/Agence_ACF_geo_only/SVG_3loops_CtrldeltaSP_500ml_30KWPAC_Boiler45KW_SP7040_FCU name: Agence_ACF_geo_only_chauffage_seul_10Y_RSM_FD.sql -#Agence_ACF_geo_only.sql environment: "RUN PERIOD" @@ -14,20 +12,20 @@ variables: key: RPLUS1 label: "RPLUS1 temp °C" - #borehole: - # - name: "Ground Heat Exchanger Inlet Temperature" - # key: "BOREHOLE" - # label: "Soil inlet °C" - # - name: "Ground Heat Exchanger Outlet Temperature" - # key: "BOREHOLE" - # label: "Soil outlet °C" + borehole: + - name: "Ground Heat Exchanger Inlet Temperature" + key: "BOREHOLE" + label: "Soil inlet °C" + - name: "Ground Heat Exchanger Outlet Temperature" + key: "BOREHOLE" + label: "Soil outlet °C" borehole_average: - name: "Ground Heat Exchanger Average Borehole Temperature" key: "BOREHOLE" label: "Average Borehole °C" - #heat_transfer: - # - name: "Ground Heat Exchanger Heat Transfer Rate" - # key: "BOREHOLE" - # label: "Borehole Heat Transfer W" + heat_transfer: + - name: "Ground Heat Exchanger Heat Transfer Rate" + key: "BOREHOLE" + label: "Borehole Heat Transfer W" diff --git a/sql_viewer/sql_compare.yaml b/sql_viewer/sql_compare.yaml index 06bd393..bf1b029 100644 --- a/sql_viewer/sql_compare.yaml +++ b/sql_viewer/sql_compare.yaml @@ -1,9 +1,7 @@ path: sql_db -#/Agence_ACF_geo_only/SVG_3loops_CtrldeltaSP_500ml_30KWPAC_Boiler45KW_SP7040_FCU name: heating_only: Agence_ACF_geo_only_chauffage_seul_10Y.sql heating_n_cooling: Agence_ACF_geo_only_10Y.sql -#Agence_ACF_geo_only.sql environment: "RUN PERIOD" @@ -19,4 +17,4 @@ variables: average_borehole_T: - name: "Ground Heat Exchanger Average Borehole Temperature" key: "BOREHOLE" - label: "Average Borehole Temperature" \ No newline at end of file + label: "Average Borehole Temperature" From 3ec17ea12394cf2e5b22e02041ef805db0411b01 Mon Sep 17 00:00:00 2001 From: Alexandre CUER Date: Sun, 20 Sep 2026 17:41:59 +0200 Subject: [PATCH 3/7] add matplotlib to requirements --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index ea6ecce..b776e68 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,7 +41,8 @@ dependencies = [ "honeybee-core", "honeybee-energy", "ladybug-geometry", - "pyyaml" + "pyyaml", + "matplotlib" ] [tool.setuptools] From 375229f29bb7b480a97ecc2c35c94783513f5b67 Mon Sep 17 00:00:00 2001 From: Alexandre CUER Date: Sun, 20 Sep 2026 22:05:45 +0200 Subject: [PATCH 4/7] overlay_mode --- sql_viewer/eplus_sql_reader.py | 90 +++++++++++++++++++++++++++------- sql_viewer/sql_overlay.yaml | 17 +++++++ 2 files changed, 88 insertions(+), 19 deletions(-) create mode 100644 sql_viewer/sql_overlay.yaml diff --git a/sql_viewer/eplus_sql_reader.py b/sql_viewer/eplus_sql_reader.py index ac9e627..6b7caf0 100644 --- a/sql_viewer/eplus_sql_reader.py +++ b/sql_viewer/eplus_sql_reader.py @@ -1,9 +1,11 @@ """read energyplus results databases""" import argparse import contextlib +from collections import defaultdict import sqlite3 from datetime import datetime, timedelta +import matplotlib.dates as mdates import matplotlib.pyplot as plt import yaml @@ -101,31 +103,77 @@ def multidb_plot_variables(dbs: dict[str, str], config: dict): plt.show() -def plot_variables(db, config): - """plot variables declared in the yaml - for a single database""" - plt.figure() +def plot_variables(db, config, overlay_years=True): + """Plot variables declared in the yaml. + For a single database ! + overlay mode to overlap successive years + TODO : prise en compte des années bisextiles + """ nb = len(config["variables"]) if nb < 1: return - ax = [] - ax.append(plt.subplot(nb, 1, 1)) + fig, ax = plt.subplots( + nb, + 1, + sharex=True, + squeeze=False, + ) + ax = ax[:, 0] with sqlite3.connect(db) as con: - for i, thema_name in enumerate(config["variables"]): - thema = config["variables"][thema_name] - if i > 0: - ax.append(plt.subplot(nb, 1, i+1, sharex=ax[0])) + for i, (thema_name, thema) in enumerate( + config["variables"].items() + ): for variable in thema: name = variable["name"] key = variable["key"] - label = variable.get("label", f"{name} — {key}") - dates, values = fetch(con, name, key, config) - plt.plot(dates, values, label=label) - plt.xlabel("date") - plt.ylabel(thema_name) - plt.grid(True) - plt.legend() - plt.tight_layout() + label = variable.get( + "label", + f"{name} — {key}", + ) + dates, values = fetch( + con, + name, + key, + config, + ) + if not overlay_years: + ax[i].plot( + dates, + values, + label=label, + ) + continue + # Regroupement par année + yearly_data = defaultdict( + lambda: ([], []) + ) + for date, value in zip(dates, values): + year = date.year + yearly_data[year][0].append( + date.replace(year=2000) + ) + yearly_data[year][1].append(value) + for year, (year_dates, year_values) in ( + yearly_data.items() + ): + ax[i].plot( + year_dates, + year_values, + label=f"{label} — {year}", + ) + ax[i].set_xlabel("date") + ax[i].set_ylabel(thema_name) + ax[i].grid(True) + ax[i].legend() + if overlay_years: + for axis in ax: + axis.xaxis.set_major_locator( + mdates.MonthLocator() + ) + axis.xaxis.set_major_formatter( + mdates.DateFormatter("%b") + ) + fig.tight_layout() plt.show() @@ -155,7 +203,11 @@ def main(): if args.plot: if len(databases) == 1: - plot_variables(databases["main"], sql_config) + plot_variables( + databases["main"], + sql_config, + overlay_years=sql_config.get('overlay', False) + ) return multidb_plot_variables(databases, sql_config) return diff --git a/sql_viewer/sql_overlay.yaml b/sql_viewer/sql_overlay.yaml new file mode 100644 index 0000000..af355ed --- /dev/null +++ b/sql_viewer/sql_overlay.yaml @@ -0,0 +1,17 @@ +path: sql_db +#name: Agence_ACF_geo_only_chauffage_seul_10Y_RSM_FD.sql +name: Agence_ACF_geo_only_10Y.sql +overlay: 1 + +environment: "RUN PERIOD" + +variables: + #temperature: + # - name: "Site Outdoor Air Drybulb Temperature" + # key: Environment + # label: "Outdoor Temp °C" + + borehole_average: + - name: "Ground Heat Exchanger Average Borehole Temperature" + key: "BOREHOLE" + label: "Average Borehole °C" From 3f2903c6b0782db8c57338b46e63cad569898584 Mon Sep 17 00:00:00 2001 From: Alexandre CUER Date: Thu, 24 Sep 2026 21:27:39 +0200 Subject: [PATCH 5/7] smart SQL request + export to csv in chunks --- .gitignore | 1 + sql_viewer/eplus_sql_reader.py | 260 ++++++++++++++++++++++++++++++--- sql_viewer/sql.yaml | 2 +- sql_viewer/sql_compare.yaml | 4 +- sql_viewer/sql_overlay.yaml | 4 +- 5 files changed, 249 insertions(+), 22 deletions(-) diff --git a/.gitignore b/.gitignore index 96eeacc..8d3cea9 100644 --- a/.gitignore +++ b/.gitignore @@ -4,5 +4,6 @@ osm_* idf_zip/* weather/data sql_viewer/sql_db +sql_viewer/exports *egg-info ep_simulations/* diff --git a/sql_viewer/eplus_sql_reader.py b/sql_viewer/eplus_sql_reader.py index 6b7caf0..c666d96 100644 --- a/sql_viewer/eplus_sql_reader.py +++ b/sql_viewer/eplus_sql_reader.py @@ -1,9 +1,15 @@ """read energyplus results databases""" import argparse import contextlib -from collections import defaultdict +import csv +import io +import re import sqlite3 -from datetime import datetime, timedelta + +from collections import defaultdict +from datetime import datetime, timedelta, timezone +from pathlib import Path +from zoneinfo import ZoneInfo import matplotlib.dates as mdates import matplotlib.pyplot as plt @@ -21,13 +27,13 @@ def list_variables(db): FROM ReportDataDictionary ORDER BY Name, KeyValue """) - for key, name, units in rows: print(f"{name:50} | {key:40} | {units:10}") def explore(db): - """sql exploration""" + """sql exploration + to identify the periods : main run, design day(s)""" with sqlite3.connect(db) as con: rows = con.execute(""" SELECT @@ -35,16 +41,37 @@ def explore(db): EnvironmentName FROM EnvironmentPeriods """).fetchall() - for row in rows: print(row) +def table_info(db, table_name): + """fetch table info""" + with sqlite3.connect(db) as con: + return con.execute( + f"PRAGMA table_info({table_name})" + ).fetchall() + + +def energyplus_date(y, mo, d, h, mi, + tz_name="Europe/Paris", + mode="human" +): + """Manage EnergyPlus date/time.""" + tz = ZoneInfo(tz_name) + # EnergyPlus peut utiliser 24:00 pour la fin de journée. + dt = datetime(y, mo, d, h % 24, mi, tzinfo=tz) + if h == 24: + dt += timedelta(days=1) + return dt if mode=="human" else int(dt.timestamp()) + + def fetch(con, name, key, config): """fetch datas related to name and key""" - rows = con.execute(""" + date_field = "t.Year, t.Month, t.Day, t.Hour, t.Minute" + rows = con.execute(f""" SELECT - t.Year, t.Month, t.Day, t.Hour, t.Minute, + {date_field}, r.Value FROM ReportData r JOIN ReportDataDictionary d @@ -65,14 +92,101 @@ def fetch(con, name, key, config): return None, None dates = [] for y, mo, d, h, mi, _ in rows: - dt = datetime(y, mo, d, h % 24, mi) - if h == 24: - dt += timedelta(days=1) + dt = energyplus_date(y, mo, d, h, mi) dates.append(dt) values = [value for *_, value in rows] return dates, values +def get_dictionary_indexes(con, variables): + """Return dictionary indexes for requested variables.""" + if not variables: + return {} + conditions = [] + params = [] + for variable in variables: + conditions.append( + "(Name = ? AND KeyValue = ?)" + ) + params.extend([ + variable["name"], + variable["key"], + ]) + query = f""" + SELECT + ReportDataDictionaryIndex, + Name, + KeyValue + FROM ReportDataDictionary + WHERE {" OR ".join(conditions)} + """ + rows = con.execute(query, params).fetchall() + return { + (name, key): index + for index, name, key in rows + } + + +def fetch_many(con, config, + tz_name="Europe/Paris", + mode="human" +): + """Fetch several EnergyPlus variables sharing the same dates.""" + variables = [ + variable + for thema in config["variables"].values() + for variable in thema + ] + indexes = get_dictionary_indexes(con, variables) + if not indexes: + return { "dates": [] } + index_to_variable = { + index: variable + for variable, index in indexes.items() + } + placeholders = ", ".join( + "?" for _ in index_to_variable + ) + query = f""" + SELECT + r.TimeIndex, + t.Year, + t.Month, + t.Day, + t.Hour, + t.Minute, + r.ReportDataDictionaryIndex, + r.Value + FROM ReportData r + JOIN Time t + ON r.TimeIndex = t.TimeIndex + JOIN EnvironmentPeriods e + ON t.EnvironmentPeriodIndex = + e.EnvironmentPeriodIndex + WHERE r.ReportDataDictionaryIndex IN ({placeholders}) + AND e.EnvironmentName = ? + ORDER BY r.TimeIndex + """ + params = list(index_to_variable) + params.append(config["environment"]) + rows = con.execute(query, params).fetchall() + data = { + variable: [] + for variable in indexes + } + data["dates"] = [] + last_time_index = None + for time_index, y, mo, d, h, mi, dict_index, value in rows: + # A single date per TimeIndex + if time_index != last_time_index: + dt = energyplus_date(y, mo, d, h, mi, tz_name=tz_name, mode=mode) + data["dates"].append(dt) + last_time_index = time_index + variable = index_to_variable[dict_index] + data[variable].append(value) + return data + + def multidb_plot_variables(dbs: dict[str, str], config: dict): """Plot variables declared in the yaml for multiple sql databases. permits to compare 2 scenarios""" @@ -84,16 +198,21 @@ def multidb_plot_variables(dbs: dict[str, str], config: dict): db_name: stack.enter_context(sqlite3.connect(db_path)) for db_name, db_path in dbs.items() } + datas = {db_name : {} for db_name in cons} + for db_name, con in cons.items(): + datas[db_name] = fetch_many(con, config) fig, ax = plt.subplots(nb, 1, sharex=True, squeeze=False) ax = ax[:, 0] for i, (thema_name, thema) in enumerate(config["variables"].items()): for variable in thema: name = variable["name"] key = variable["key"] + identifier = (name, key) label = variable.get("label", f"{name} — {key}") for db_name, con in cons.items(): db_label = label if db_name == "main" else f"{db_name} {label}" - dates, values = fetch(con, name, key, config) + dates = datas[db_name]["dates"] + values = datas[db_name][identifier] ax[i].plot(dates, values, label=db_label) ax[i].set_xlabel("date") ax[i].set_ylabel(thema_name) @@ -120,6 +239,8 @@ def plot_variables(db, config, overlay_years=True): ) ax = ax[:, 0] with sqlite3.connect(db) as con: + data = fetch_many(con, config) + dates = data["dates"] for i, (thema_name, thema) in enumerate( config["variables"].items() ): @@ -130,12 +251,8 @@ def plot_variables(db, config, overlay_years=True): "label", f"{name} — {key}", ) - dates, values = fetch( - con, - name, - key, - config, - ) + identifier = (name, key) + values = data[identifier] if not overlay_years: ax[i].plot( dates, @@ -177,13 +294,105 @@ def plot_variables(db, config, overlay_years=True): plt.show() +def safe_filename(name): + """Return a filesystem-safe filename.""" + return re.sub(r'[<>:"/\\|?*]', "_", name) + + +def export_variables_csv_chunk(db, config, + output_dir="exports", + tz_name="Europe/Paris", + max_size_mb=1.5, +): + """Export variables declared in the yaml to CSV files. + One CSV file per variable, split into chunks if needed. + First column: Unix timestamp (seconds). + Second column: value. + """ + output_dir = Path(output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + max_size = int(max_size_mb * 1_000_000) + with sqlite3.connect(db) as con: + data = fetch_many( + con, + config, + tz_name=tz_name, + mode="timestamp", + ) + dates = data["dates"] + for _, thema in config["variables"].items(): + for variable in thema: + name = variable["name"] + key = variable["key"] + identifier = (name, key) + values = data[identifier] + filename = safe_filename(name) + # Export in chunks. + chunk_index = 1 + current_size = 0 + csvfile = None + writer = None + def open_chunk(index): + """open chunk""" + nonlocal csvfile, writer, current_size + if csvfile is not None: + csvfile.close() + if index == 1: + filepath = output_dir / f"{filename}.csv" + else: + filepath = output_dir / ( + f"{filename}_{index:03d}.csv" + ) + csvfile = filepath.open( + "w", + newline="", + encoding="utf-8", + ) + writer = csv.writer(csvfile) + writer.writerow(["timestamp", "value"]) + csvfile.flush() + current_size = csvfile.tell() + print(f"Exporting: {filepath}") + try: + open_chunk(chunk_index) + for date, value in zip(dates, values): + # Generate the CSV row in memory + # to determine its encoded size. + buffer = io.StringIO(newline="") + row_writer = csv.writer(buffer) + row_writer.writerow([date, value]) + row = buffer.getvalue() + row_size = len( + row.encode("utf-8") + ) + # Start a new chunk before writing + # if the size limit would be exceeded. + if ( + current_size + row_size > max_size + and current_size > 0 + ): + chunk_index += 1 + open_chunk(chunk_index) + writer.writerow([date, value]) + csvfile.flush() + current_size += row_size + finally: + if csvfile is not None: + csvfile.close() + + def main(): """main""" parser = argparse.ArgumentParser() parser.add_argument("--list", action="store_true") parser.add_argument("--plot", action="store_true") parser.add_argument("--explore", action="store_true") + parser.add_argument("--csv", action="store_true") parser.add_argument("--yml", default="sql.yaml") + parser.add_argument("--tz", default="Europe/Paris") + parser.add_argument("--mb", default=2) + parser.add_argument("--table_info") + args = parser.parse_args() with open(args.yml, encoding="utf-8") as f: sql_config = yaml.safe_load(f) @@ -201,6 +410,23 @@ def main(): list_variables(database) return + if args.table_info: + print(args.table_info) + for database in databases.values(): + for line in table_info(database, args.table_info): + print(line) + return + + if args.csv: + for database in databases.values(): + export_variables_csv_chunk( + database, + sql_config, + tz_name=args.tz, + max_size_mb=args.mb + ) + return + if args.plot: if len(databases) == 1: plot_variables( diff --git a/sql_viewer/sql.yaml b/sql_viewer/sql.yaml index 3a561d2..670885f 100644 --- a/sql_viewer/sql.yaml +++ b/sql_viewer/sql.yaml @@ -1,5 +1,5 @@ path: sql_db -name: Agence_ACF_geo_only_chauffage_seul_10Y_RSM_FD.sql +#name: Agence_ACF_chauffage_seul_10Y_RSM_FD.sql environment: "RUN PERIOD" diff --git a/sql_viewer/sql_compare.yaml b/sql_viewer/sql_compare.yaml index bf1b029..9bcbdf2 100644 --- a/sql_viewer/sql_compare.yaml +++ b/sql_viewer/sql_compare.yaml @@ -1,7 +1,7 @@ path: sql_db name: - heating_only: Agence_ACF_geo_only_chauffage_seul_10Y.sql - heating_n_cooling: Agence_ACF_geo_only_10Y.sql + heating_only: Agence_ACF_chauffage_seul_10Y.sql + heating_n_cooling: Agence_ACF_10Y.sql environment: "RUN PERIOD" diff --git a/sql_viewer/sql_overlay.yaml b/sql_viewer/sql_overlay.yaml index af355ed..adeed75 100644 --- a/sql_viewer/sql_overlay.yaml +++ b/sql_viewer/sql_overlay.yaml @@ -1,6 +1,6 @@ path: sql_db -#name: Agence_ACF_geo_only_chauffage_seul_10Y_RSM_FD.sql -name: Agence_ACF_geo_only_10Y.sql +#name: Agence_ACF_chauffage_seul_10Y_RSM_FD.sql +#name: Agence_ACF_10Y.sql overlay: 1 environment: "RUN PERIOD" From 82ce54c69b2ec7ecc8db5eb7d83371982b318e2e Mon Sep 17 00:00:00 2001 From: Alexandre CUER Date: Thu, 24 Sep 2026 21:34:20 +0200 Subject: [PATCH 6/7] remove comments in sql yml --- sql_viewer/sql.yaml | 2 +- sql_viewer/sql_overlay.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sql_viewer/sql.yaml b/sql_viewer/sql.yaml index 670885f..e9eb77b 100644 --- a/sql_viewer/sql.yaml +++ b/sql_viewer/sql.yaml @@ -1,5 +1,5 @@ path: sql_db -#name: Agence_ACF_chauffage_seul_10Y_RSM_FD.sql +name: Agence_ACF_chauffage_seul_10Y_RSM_FD.sql environment: "RUN PERIOD" diff --git a/sql_viewer/sql_overlay.yaml b/sql_viewer/sql_overlay.yaml index adeed75..9bb4110 100644 --- a/sql_viewer/sql_overlay.yaml +++ b/sql_viewer/sql_overlay.yaml @@ -1,6 +1,6 @@ path: sql_db #name: Agence_ACF_chauffage_seul_10Y_RSM_FD.sql -#name: Agence_ACF_10Y.sql +name: Agence_ACF_10Y.sql overlay: 1 environment: "RUN PERIOD" From 1f578312c53ff813eeb84d8afdaa9f54af443bdc Mon Sep 17 00:00:00 2001 From: Alexandre CUER Date: Fri, 25 Sep 2026 22:18:01 +0200 Subject: [PATCH 7/7] add tzdata to dependancies --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index b776e68..b52d17f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,7 +42,8 @@ dependencies = [ "honeybee-energy", "ladybug-geometry", "pyyaml", - "matplotlib" + "matplotlib", + "tzdata" ] [tool.setuptools]