Running the MISO2 model

import logging
import pickle
import os
import multiprocess as mp
from itertools import repeat
from datetime import datetime
from model.config.MISO2_config import split_config
from model.core.MISO2_multiprocessing import miso_run_pool
from model.config.MISO2_logger import get_MISO2_logger
from model.output.MISO2_database import MISO2Database

main_path = os.path.join(os.getcwd(), os.pardir, os.pardir, os.pardir)
config_path = os.path.join(main_path, "config")
output_path = os.path.join(main_path, "output", "usuk")

timestamp = datetime.now().strftime("%Y_%m_%d_%H_%M_%S")
get_MISO2_logger(
    log_filename='MISO2_log_' + timestamp + ".txt",
    log_pathname=os.path.join(main_path, 'logs'),
    file_level=None,
    console_level=logging.WARNING)

with open(os.path.join(config_path,"selected_countries_config.pickle"), 'rb') as f:
    miso_config = pickle.load(f)

We split the master config into individual regions due to memory constraints.

config_split = []
for i in range(0, miso_config.nr_stop):
    new_config = split_config(miso_config, i)
    config_split.append(new_config)

The model run over the split configs is multi-processed. Increase the number of processes if your system memory can handle processing multiple configs at once.

run_model_arguments = {
    "estimate_aggregates": True,  # default option
    "output_path": output_path,
    "log_path": output_path,
    "save_stock_cohorts": False,    # very large memory / disk-space requirements
    "save_last_year": False,    # this is the buffer year and does not contain full results
    "save_debug_output": False, #  mass balances and other debug output are saved in the model object and can be saved to file
    "sf_parametrization": "MISO2LogNormal"  # this is the default
}

n_processes = 1

with mp.Pool(n_processes) as pool:
    miso_outputs_list = pool.starmap(
        miso_run_pool, zip(config_split, repeat(run_model_arguments), repeat(None), repeat(None), repeat(logging.WARNING)), chunksize=1)

output_path = os.path.join(main_path, "output", "usuk")
os.makedirs(output_path, exist_ok=True)
db_folder = os.path.join(output_path, "selected_countries_cohorts")  # only required if cohorts are saved
os.makedirs(db_folder, exist_ok=True)

Database is just a wrapper object for the dataframes that the MISO2 model outputs. It provides some convenience functionality for handling the data.

miso_database = MISO2Database()
miso_database.add_miso_outputs(miso_outputs_list, db_folder=db_folder)

We can save the database object to parquet (if we want to restore the database into a Python object) or CSV (for data export). Note that xlsx export is not recommended, since saving large XLSX is extremely slow.

miso_database.save_to_file("parquet", output_path, False, "miso_db")
miso_database.save_to_file("csv", output_path, False, "miso_db")