Commit 0ba7fc44 authored by Antoine Berchet's avatar Antoine Berchet
Browse files

Dumping example yaml with join syntax

parent d130c73b
......@@ -45,13 +45,14 @@ input_arguments = {
}
def ini_data(plugin, **kwargs):
"""Initializes the platform
Args:
"""
# Loading NetCDF module according to the model that will be run
if (
plugin.model.plugin.name == "CHIMERE"
......
......@@ -70,7 +70,8 @@ def calc_indexes(mapper, trid_out):
"end": out_index_end.values,
"duration": out_durations})
weights = out_df.iloc[out_df.index.repeat(out_df["duration"])]
weights = copy.deepcopy(
out_df.iloc[out_df.index.repeat(out_df["duration"])])
indexes = np.zeros(len(weights))
start_index = np.array(
[0] + list(np.cumsum(
......
import os
import re
import copy
from collections import OrderedDict
import yaml
......@@ -69,11 +70,40 @@ def ordered_load(
return yaml.load(stream, OrderedLoader)
# Special representation for Ordered Dictionaries to keep order
def represent_dictionary_order(self, dict_data):
return self.represent_mapping('tag:yaml.org,2002:map',
dict_data.items())
def ordered_dump(stream, yml_dict):
def ordered_dump(stream, yml_dict, ref_directories={}, replace_values={}):
dump_dict = copy.deepcopy(yml_dict)
# Replacing reference directories by "join" syntax
for key in ref_directories:
dump_dict = dump_join_syntax(dump_dict, key, ref_directories[key])
dump_dict.update(
{key: "&{} {}".format(
key, replace_values.get(key, ref_directories[key]))})
dump_dict.move_to_end(key, last=False)
yaml.add_representer(OrderedDict, represent_dictionary_order)
yaml.dump(yml_dict, stream)
# Dump yaml to string and remove quotes for later parsing by yaml
todump = yaml.dump(dump_dict).replace("'", "")
stream.write(todump)
def dump_join_syntax(yml_dict, ref_alias, ref_dir):
for key in yml_dict:
if type(yml_dict[key]) == OrderedDict:
yml_dict[key] = dump_join_syntax(yml_dict[key], ref_alias, ref_dir)
elif type(yml_dict[key]) == str:
if ref_dir in yml_dict[key]:
yml_dict[key] = \
"!join [*{}, {}]".format(
ref_alias, yml_dict[key].split(ref_dir)[1])
return yml_dict
from .fixtures import ref_dummy
from .fixtures.fwd import dummy_config_fwd
from .fixtures.inversion import dummy_config_inversion
from .fixtures.adjtltest import dummy_config_adjtltest
from .fixtures.postprocessing import plot_inversion
import pytest
......
import os
from pathlib import Path
import pytest
from pycif.utils.yml import ordered_dump
@pytest.fixture(
params=
[
{"incrmode": "cst", "testspace": "control"},
{"incrmode": "cst", "testspace": "chi"},
{"incrmode": "cst", "testspace": "chi", "tresol": "3H"},
{"incrmode": "cst", "testspace": "chi", "tresol": "5H"},
{"incrmode": "rand", "testspace": "control"},
{"incrmode": "rand", "testspace": "chi"},
{"incrmode": "rand", "testspace": "chi", "tresol": "3H"},
{"incrmode": "rand", "testspace": "chi", "tresol": "5H"}
])
def dummy_config_adjtltest(dummy_config_inversion, request):
"""
Fixture that returns a temporary folder and a configuration file for the
dummy_forward model.
"""
tmpdir_str, config, tag = dummy_config_inversion
# Changing mode
config.update(
{"mode": {
"plugin": {"name": "adj-tl_test", "version": "std"},
"increments": 0.2,
"incrmode": "cst",
"testspace": "control",
}}
)
# Applying various configurations modes
config["mode"]["incrmode"] = request.param.get("incrmode", "cst")
config["mode"]["testspace"] = request.param.get("testspace", "control")
# Altering temporal resolution
params = config["datavect"]["components"]["fluxes"]["parameters"]
CH4 = params["CH4"] if "CH4" in params else params["CH4_ref"]
CH4["period"] = request.param.get("tresol", "5D")
# Save to yml file
with open(os.path.join(tmpdir_str, "dummy_config.yml"), "w") as outfile:
ordered_dump(outfile, config)
yield tmpdir_str, config
# Dump configuration into CIF examples
tag += "_" + request.param.get("incrmode", "cst") + "-incr_" \
+ request.param.get("testspace", "control") + "-testspace_" \
+ ("fullperiod" if request.param.get("tresol", "5D") else
"subperiod")
current_dir = os.path.abspath(os.path.dirname(os.path.realpath(__file__)))
example_dir = \
os.path.abspath(os.path.join(current_dir,
"../../examples_artifact/dummy/"))
Path(example_dir).mkdir(parents=True, exist_ok=True)
dummy_config_file = \
os.path.join(example_dir, "config_adjtltest_{}.yml".format(tag))
with open(dummy_config_file, "w") as outfile:
ordered_dump(outfile, config)
......@@ -83,28 +83,28 @@ def dummy_config_fwd(ref_dummy, request):
yield tmpdir_str, config, tag
# Change config to be compatible in the docker
config["workdir"] = "/workdir/"
config["model"]["file_pg"] = \
"/tmp/CIF/model_sources/dummy_gauss/Pasquill-Gifford.txt"
# Dump configuration into CIF examples
current_dir = os.path.abspath(os.path.dirname(os.path.realpath(__file__)))
root_dir = os.path.join(current_dir, "../../../")
pytest_dir = tmpdir_str + "/../"
example_dir = \
os.path.abspath(os.path.join(current_dir,
"../../examples_artifact/dummy/"))
os.path.abspath(os.path.join(root_dir, "examples_artifact/dummy/"))
Path(example_dir).mkdir(parents=True, exist_ok=True)
dummy_config_file = \
os.path.join(example_dir, "config_fwd_{}.yml".format(tag))
config["workdir"] = "{}/fwd_{}/".format(pytest_dir, tag)
# Write only if mode is forward
if config["mode"]["plugin"]["name"] == "forward":
with open(dummy_config_file, "w") as outfile:
ordered_dump(outfile, config)
ordered_dump(outfile, config,
ref_directories={"rootdir": root_dir,
"outdir": pytest_dir},
replace_values={"rootdir": "/tmp/CIF/"})
# Copy simulated observation vector for reading by inversions
pytest_dir = tmpdir_str + "/../"
target_dir = pytest_dir + "ref_obsvect"
if os.path.isdir(target_dir):
shutil.rmtree(target_dir)
......@@ -122,6 +122,3 @@ def dummy_config_fwd(ref_dummy, request):
os.remove(Hfile_target)
shutil.copy(Hfile_ref, Hfile_target)
......@@ -59,12 +59,18 @@ def test_integration_adjtltest(dummy_config_inversion, settings):
"subperiod")
current_dir = os.path.abspath(os.path.dirname(os.path.realpath(__file__)))
root_dir = os.path.join(current_dir, "../../")
pytest_dir = tmpdir + "/../"
example_dir = \
os.path.abspath(os.path.join(current_dir,
"../../examples_artifact/dummy/"))
os.path.abspath(os.path.join(root_dir, "examples_artifact/dummy/"))
Path(example_dir).mkdir(parents=True, exist_ok=True)
config["workdir"] = "{}/adjtltest_{}/".format(pytest_dir, tag)
dummy_config_file = \
os.path.join(example_dir, "config_adjtltest_{}.yml".format(tag))
with open(dummy_config_file, "w") as outfile:
ordered_dump(outfile, config)
ordered_dump(outfile, config,
ref_directories={"rootdir": root_dir,
"outdir": pytest_dir},
replace_values={"rootdir": "/tmp/CIF/"})
......@@ -151,15 +151,21 @@ def test_integration_inversion(dummy_config_inversion, settings, pytestconfig):
else settings["minimizer"])
current_dir = os.path.abspath(os.path.dirname(os.path.realpath(__file__)))
root_dir = os.path.join(current_dir, "../../")
pytest_dir = tmpdir + "/../"
example_dir = \
os.path.abspath(os.path.join(current_dir,
"../../examples_artifact/dummy/"))
os.path.abspath(os.path.join(root_dir, "examples_artifact/dummy/"))
Path(example_dir).mkdir(parents=True, exist_ok=True)
config["workdir"] = "{}/inversion_{}/".format(pytest_dir, tag)
dummy_config_file = \
os.path.join(example_dir, "config_{}.yml".format(tag))
with open(dummy_config_file, "w") as outfile:
ordered_dump(outfile, config)
ordered_dump(outfile, config,
ref_directories={"rootdir": root_dir,
"outdir": pytest_dir},
replace_values={"rootdir": "/tmp/CIF/"})
# Loop with different number of simulations for non analytical inversions
title = ""
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment