Commit 34bf41ea authored by Antoine Berchet's avatar Antoine Berchet

Merge branch 'LSCE' into 'master'

Lsce

See merge request !16
parents f28f794a e9a2a23c
Pipeline #60 failed with stages
#!/usr/bin/env bash
pytest -s --basetemp=/home/chimereges/aberchet/pytest/ tests/
......@@ -97,7 +97,7 @@ model :
version : std
# H matrix
file_pg : ~/cif/model_sources/dummy_gauss/Pasquill-Gifford.txt
file_pg : ~/CIF/model_sources/dummy_gauss/Pasquill-Gifford.txt
# Chemical scheme
chemistry:
......
......@@ -2,7 +2,7 @@ import datetime
import glob
import os
import cfgrib
#import cfgrib
import numpy as np
from pycif.utils.check import info
......
import os
import subprocess
from shutil import copytree, ignore_patterns, rmtree, copy
from pycif.utils.check import info
def compile(self):
comp_dir = "{}/model".format(self.workdir)
......@@ -80,6 +80,6 @@ def compile(self):
stderr=subprocess.PIPE,
)
stdout = process.communicate()
source = "{}/sources/src{}/{}chimere.e".format(comp_dir, suff, pref)
copy(source, comp_dir)
......@@ -35,7 +35,6 @@ def run(self, runsubdir, mode, workdir, nbproc=1, do_simu=True, **kwargs):
# Running the model
info("Running sub-simulation in {}".format(runsubdir))
with open("{}/chimere.log".format(runsubdir), "w") as log:
process = subprocess.Popen(
[
......
......@@ -126,3 +126,7 @@ def execute(self, **kwargs):
np.floor(np.abs(scaleprod2 / scaleprod1 - 1) / accuracy)
)
)
# Return test
return np.floor(np.abs(scaleprod2 / scaleprod1 - 1) / accuracy)
\ No newline at end of file
......@@ -27,5 +27,5 @@ def ini_data(plugin, **kwargs):
# Initializing Singular Value Decomposition if required
plugin.do_svd = getattr(plugin, "do_svd", False)
if plugin.svd:
if plugin.do_svd:
plugin.svd_vectors = svd_init(plugin.obsvect.datastore)
......@@ -147,10 +147,16 @@ def init_bprod(cntrlv, options={}, **kwargs):
else:
tracer.chi_tresoldim = tracer.ndates
# TODO: dealing with vertical resolution
tracer.chi_vresoldim = tracer.vresoldim
# Update the dimension of the full control vector
tracer.chi_pointer = cntrlv.chi_dim
tracer.chi_dim = tracer.chi_hresoldim * tracer.chi_tresoldim
tracer.chi_dim = \
tracer.chi_hresoldim \
* tracer.chi_tresoldim \
* tracer.chi_vresoldim
cntrlv.chi_dim += tracer.chi_dim
# Defining chi from the total dimension
......
......@@ -14,6 +14,11 @@ def sqrtbprod(statevect, chi, **kwargs):
components = statevect.components
for comp in components.attributes:
component = getattr(components, comp)
# Skip if component does not have parameters
if not hasattr(component, "parameters"):
continue
for trcr in component.parameters.attributes:
tracer = getattr(component.parameters, trcr)
......@@ -63,7 +68,9 @@ def sqrtbprod(statevect, chi, **kwargs):
chi_tmp = np.dot(
evectors, chi_horizstacks * sqrt_evalues[:, np.newaxis]
).T.flatten(order="F")
# TODO: deal with vertical resolution
# Filling corresponding part in the control vector
xout[x_pointer: x_pointer + x_dim] = chi_tmp
......@@ -78,6 +85,11 @@ def sqrtbprod_ad(statevect, dx, **kwargs):
components = statevect.components
for comp in components.attributes:
component = getattr(components, comp)
# Skip if component does not have parameters
if not hasattr(component, "parameters"):
continue
for trcr in component.parameters.attributes:
tracer = getattr(component.parameters, trcr)
......
......@@ -22,7 +22,7 @@ def reindex(array, levels={}, method="ffill"):
index_out = index_out.data
index_tmp = np.unique(np.append(index_in, index_out))
df_index = pd.DataFrame(range(len(index_in)), index=index_in)
df_index = pd.DataFrame(np.arange(len(index_in)), index=index_in)
df_index = df_index.reindex(index_tmp, method="ffill")
idim = var_in.dims.index(lev)
......
......@@ -46,7 +46,6 @@ requirements = [
'psutil',
'Pillow',
'cfgrib',
'pyeccodes',
'future'
]
......
import datetime
import os
import pytest
import yaml
@pytest.fixture
def dummy_config_adjtltest(tmpdir):
"""
Fixture that returns a temporary folder and a configuration file for the
dummy_forward model.
"""
current_dir = os.path.abspath(os.path.dirname(os.path.realpath(__file__)))
root_dir = os.path.abspath(os.path.join(current_dir, "../../.."))
data_dir = os.path.abspath(os.path.join(root_dir, "data/"))
tmpdir_str = tmpdir.strpath
config = {
"verbose": 1,
"logfile": "pycif.logtest",
"workdir": tmpdir_str,
"datei": datetime.date(2010, 1, 1),
"datef": datetime.date(2010, 1, 5),
"mode": {
"plugin": {"name": "adj-tl_test", "version": "std"},
"increments": 0.2,
"incrmode": "cst",
"testspace": "control",
},
"obsoperator": {
"plugin": {"name": "standard", "version": "std"},
"autorestart": True,
},
"model": {
"plugin": {"name": "dummy", "version": "std"},
"file_pg": os.path.join(
root_dir, "model_sources/dummy_gauss/Pasquill-Gifford.txt"
),
"chemistry": {"acspecies": {"CH4": None}},
},
"measurements": {
"plugin": {"name": "random", "version": "std"},
"file_monitor": os.path.join(tmpdir_str, "monitor_reference.nc"),
"dump_type": "nc",
"species": {
"CH4": {
"frequency": "1H",
"nstations": 50,
"duration": "1H",
"random_subperiod_shift": True,
"zmax": 100,
},
"MCF": {
"frequency": "1H",
"nstations": 50,
"duration": "1H",
"random_subperiod_shift": True,
"zmax": 100,
}
},
},
"obsvect": {
"plugin": {"name": "standard", "version": "std"},
"file_obsvect": os.path.join(tmpdir_str, "monitor_reference.nc"),
"dump_type": "nc",
"transform": {
"timeavg": {
"plugin": {"name": "timeavg", "version": "std", "type": "transform"}
}
},
},
"statevect": {
"plugin": {"name": "standard", "version": "std"},
"components": {
"fluxes": {
"parameters": {
"CH4": {
"plugin": {
"name": "dummy",
"version": "txt",
"type": "fluxes",
},
"hresol": "hpixels",
"type": "physical",
"errtype": "max",
"err": 1,
"period": "1D",
"dir": os.path.join(tmpdir_str, "statevect/"),
"file": "flx_real.txt",
"hcorrelations": {
"landsea": False,
"dump_hcorr": True,
"dircorrel": os.path.join(tmpdir_str, "statevect/"),
"sigma": 3000,
},
"tcorrelations": {"sigma_t": 5},
"flx_text": "CIF",
}
}
},
"meteo": {
"plugin": {"name": "dummy", "version": "csv", "type": "meteo"},
"dir": os.path.join(data_dir, "dummy_gauss/"),
"file": "meteo2.csv",
"resolution": "1H",
},
},
},
"domain": {
"plugin": {"name": "dummy", "version": "std"},
"xmin": 0,
"xmax": 25000,
"nlon": 30,
"ymin": 0,
"ymax": 20000,
"nlat": 15,
},
}
with open(os.path.join(tmpdir_str, "dummy_config.yml"), "w") as outfile:
yaml.dump(config, outfile)
# Doing the test
from pycif.utils.classes.setup import Setup
result = Setup.run_simu({"def_file": outfile.name})
# yield tmpdir_str, outfile.name
yield result
......@@ -5,13 +5,13 @@ import yaml
@pytest.fixture
def dummy_config(tmpdir):
def dummy_config_fwd(tmpdir):
"""
Fixture that returns a temporary folder and a configuration file for the
dummy_forward model.
"""
current_dir = os.path.abspath(os.path.dirname(os.path.realpath(__file__)))
root_dir = os.path.abspath(os.path.join(current_dir, ".."))
root_dir = os.path.abspath(os.path.join(current_dir, "../../.."))
data_dir = os.path.abspath(os.path.join(root_dir, "data/"))
tmpdir_str = tmpdir.strpath
......
import datetime
import os
import pytest
import yaml
@pytest.fixture
def dummy_config_variat(tmpdir):
"""
Fixture that returns a temporary folder and a configuration file for the
dummy_forward model.
"""
current_dir = os.path.abspath(os.path.dirname(os.path.realpath(__file__)))
root_dir = os.path.abspath(os.path.join(current_dir, "../../.."))
data_dir = os.path.abspath(os.path.join(root_dir, "data/"))
tmpdir_str = tmpdir.strpath
config = {
"verbose": 1,
"logfile": "pycif.logtest",
"workdir": tmpdir_str,
"datei": datetime.date(2010, 1, 1),
"datef": datetime.date(2010, 1, 5),
"mode": {
"plugin": {"name": "adj-tl_test", "version": "std"},
"increments": 0.2,
"incrmode": "cst",
"testspace": "control",
},
"obsoperator": {
"plugin": {"name": "standard", "version": "std"},
"autorestart": True,
},
"model": {
"plugin": {"name": "dummy", "version": "std"},
"file_pg": os.path.join(
root_dir, "model_sources/dummy_gauss/Pasquill-Gifford.txt"
),
"chemistry": {"acspecies": {"CH4": None}},
},
"measurements": {
"plugin": {"name": "random", "version": "std"},
"file_monitor": os.path.join(tmpdir_str, "monitor_reference.nc"),
"dump_type": "nc",
"species": {
"CH4": {
"frequency": "1H",
"nstations": 50,
"duration": "1H",
"random_subperiod_shift": True,
"zmax": 100,
},
"MCF": {
"frequency": "1H",
"nstations": 50,
"duration": "1H",
"random_subperiod_shift": True,
"zmax": 100,
}
},
},
"obsvect": {
"plugin": {"name": "standard", "version": "std"},
"file_obsvect": os.path.join(tmpdir_str, "monitor_reference.nc"),
"dump_type": "nc",
"transform": {
"timeavg": {
"plugin": {"name": "timeavg", "version": "std", "type": "transform"}
}
},
},
"statevect": {
"plugin": {"name": "standard", "version": "std"},
"components": {
"fluxes": {
"parameters": {
"CH4": {
"plugin": {
"name": "dummy",
"version": "txt",
"type": "fluxes",
},
"hresol": "hpixels",
"type": "physical",
"errtype": "max",
"err": 1,
"period": "1D",
"dir": os.path.join(tmpdir_str, "statevect/"),
"file": "flx_real.txt",
"hcorrelations": {
"landsea": False,
"dump_hcorr": True,
"dircorrel": os.path.join(tmpdir_str, "statevect/"),
"sigma": 3000,
},
"tcorrelations": {"sigma_t": 5},
"flx_text": "CIF",
}
}
},
"meteo": {
"plugin": {"name": "dummy", "version": "csv", "type": "meteo"},
"dir": os.path.join(data_dir, "dummy_gauss/"),
"file": "meteo2.csv",
"resolution": "1H",
},
},
},
"domain": {
"plugin": {"name": "dummy", "version": "std"},
"xmin": 0,
"xmax": 25000,
"nlon": 30,
"ymin": 0,
"ymax": 20000,
"nlat": 15,
},
}
with open(os.path.join(tmpdir_str, "dummy_config.yml"), "w") as outfile:
yaml.dump(config, outfile)
yield tmpdir_str, outfile.name
from .conftests.adjtltest import dummy_config_adjtltest
def test_integration_adjtltest(dummy_config_adjtltest):
"""
Integration test that runs the dummy_forward model.
"""
# tmpdir, dummy_config_file = dummy_config_adjtltest
# Run the dummy model
# result = Setup.run_simu({"def_file": dummy_config_file})
# Test of the adjoint passes only if eps < 100
assert dummy_config_adjtltest < 100
import numpy as np
import os
from pycif.utils.classes.setup import Setup
# from pycif.utils.classes.setup import Setup
from pycif.utils.netcdf import readnc
from .conftests.fwd import dummy_config_fwd
def test_integration(dummy_config):
def test_integration_fwd(dummy_config_fwd):
"""
Integration test that runs the dummy_forward model.
"""
tmpdir, dummy_config_file = dummy_config
tmpdir, dummy_config_file = dummy_config_fwd
# Run the dummy model
Setup.run_simu({"def_file": dummy_config_file})
......
from pycif.utils.classes.setup import Setup
from .conftests.variat import dummy_config_variat
def test_integration_varia(dummy_config_variat):
"""
Integration test that runs the dummy_forward model.
"""
tmpdir, dummy_config_file = dummy_config_variat
# Run the dummy model
result = Setup.run_simu({"def_file": dummy_config_file})
# Test of the adjoint passes only if eps < 100
assert result < 100
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment