Commit 3144f29d authored by Antoine Berchet's avatar Antoine Berchet
Browse files

Merge branch 'new_classes' into 'devel'

New classes

See merge request satinv/cif!166
parents b11b72d1 36794610
......@@ -172,27 +172,25 @@ article:
image:
name: pycif/pycif-ubuntu:0.1
entrypoint: [""]
# before_script:
## - apt-get update
# - pip freeze
before_script:
# - apt-get update
- pip freeze
script:
- echo AAAAAAAAAAAAAAAAAAAAAAAAaaa
- echo ${CI_COMMIT_BRANCH}
- echo AAAAAAAAAAAAAAAAAAAAAAAAaaa
- echo ${CI_MERGE_REQUEST_TARGET_BRANCH_NAME}
- echo AAAAAAAAAAAAAAAAAAAAAAAAaaa
- if [ ${CI_COMMIT_BRANCH} == "new_classes" ]; then echo "TTTTTTTTTTTTTTTTTTTTTTTTT"; fi;
#- tox -e py38 -e coverage -- -m "(dummy and article and inversion and not adjtltest and not uncertainties) or (fwd and ref_config)"
# after_script:
# - mkdir -p coverage
# - xmlstarlet sel -t -v "//coverage/@line-rate" reports/coverage.xml > coverage/.current_coverage
# - calc() { awk "BEGIN{print $*}"; }
# - percent_coverage=`cat coverage/.current_coverage`
# - tot_coverage=`calc ${percent_coverage}*100`
# - echo 'TOTAL COVERAGE:'" ${tot_coverage}%"
# - mv coverage_raw/coverage/.coverage coverage_raw/.coverage.article
# coverage: '/^TOTAL COVERAGE: ([0-9\.]+\%)$/'
- |
if [ ${CI_COMMIT_BRANCH} == "LSCE" ]; then
tox -e py38 -e coverage -- -m "(dummy and article and inversion and not adjtltest and not uncertainties) or (fwd and ref_config) or (allsimulations)";
else
tox -e py38 -e coverage -- -m "(dummy and article and inversion and not adjtltest and not uncertainties and bands) or (fwd and ref_config)";
fi;
after_script:
- mkdir -p coverage
- xmlstarlet sel -t -v "//coverage/@line-rate" reports/coverage.xml > coverage/.current_coverage
- calc() { awk "BEGIN{print $*}"; }
- percent_coverage=`cat coverage/.current_coverage`
- tot_coverage=`calc ${percent_coverage}*100`
- echo 'TOTAL COVERAGE:'" ${tot_coverage}%"
- mv coverage_raw/coverage/.coverage coverage_raw/.coverage.article
coverage: '/^TOTAL COVERAGE: ([0-9\.]+\%)$/'
artifacts:
when: always
paths:
......@@ -202,8 +200,6 @@ article:
- coverage_raw
- examples_artifact
- figures_artifact
# only:
# - LSCE
article_uncertainties:
stage: test
......@@ -214,7 +210,12 @@ article_uncertainties:
# - apt-get update
- pip freeze
script:
- tox -e py38 -e coverage -- -m "(dummy and article and inversion and not adjtltest and uncertainties) or (fwd and ref_config)"
- |
if [ ${CI_COMMIT_BRANCH} == "LSCE" ]; then
tox -e py38 -e coverage -- -m "(dummy and article and inversion and not adjtltest and uncertainties) or (fwd and ref_config) or (allsimulations)";
else
tox -e py38 -e coverage -- -m "(dummy and article and inversion and not adjtltest and uncertainties and bands) or (fwd and ref_config)";
fi;
after_script:
- mkdir -p coverage
- xmlstarlet sel -t -v "//coverage/@line-rate" reports/coverage.xml > coverage/.current_coverage
......@@ -233,8 +234,6 @@ article_uncertainties:
- coverage_raw
- examples_artifact
- figures_artifact
only:
- LSCE
# Run the tests for the dummy model
tests_dummy:
......
......@@ -16,10 +16,10 @@ export PYCIF_DATATEST=/home/chimereges/PYCIF_TEST_DATA/
export PYCIF_PLATFORM=LSCE
###
# select a subset of tests to run by using the tags ("mark")
#mark="(dummy and article and inversion and not adjtltest and not uncertainties) or (fwd and ref_config)"
mark="(dummy and article and inversion and not adjtltest and not uncertainties) or (fwd and ref_config)"
#mark="(dummy and article and inversion and not adjtltest and uncertainties) or (fwd and ref_config)"
#mark="(fwd and ref_config)"
mark="test_in_ci and dummy"
#mark="test_in_ci and dummy"
#mark="test_in_ci and chimere"
#mark="chimere and argfrsd and fwd"
#mark="tm5 and test_in_ci and fwd"
......
......@@ -79,7 +79,29 @@ The :bash:`datavect` paragraph of your working yaml should look like that:
type: flux
version: your_version
3. Test running again your test case. It should generate fluxes with random values
3. Test running again your test case. It should generate fluxes with random values as in the template
Document your plugin
====================
Before going further, be sure to document your plugin properly.
To do so, please replace the docstring header in the file :bash:`__init__.py`.
Include the following information:
- licensing information
- permanent link to download the data (or a contact person if no link is publicly available)
- data format (temporal and horizontal resolution, names and shape of the data files)
- any specific treatment that prevents the plugin from working with another type of files.
Updating functions and data to implement your flux data
=======================================================
......@@ -142,9 +164,9 @@ The :bash:`datavect` paragraph of your working yaml should look like that:
.. code-block:: rest
####################
#####################
Fluxes :bash:`fluxes`
####################
#####################
Available Fluxes
=========================
......
......@@ -22,6 +22,36 @@ It is used for the following purposes:
Required parameters, dependencies and functions
===============================================
Functions
+++++++++
fetch
---------
.. currentmodule:: pycif.plugins.datastreams.fluxes.flux_plugin_template
.. autofunction:: fetch
:noindex:
get_domain (optional)
----------------------
.. autofunction:: get_domain
:noindex:
read
------
.. autofunction:: read
:noindex:
write (optional)
----------------
......
......@@ -44,7 +44,7 @@ def build_H(controlvect, obsvect, base_dir):
"{}/obsvect_{:04d}/{}/{}/monitor.nc"\
.format(base_dir, idim, comp, trcr)
data = read_datastore(monitor_file)
hloc.loc[:, idim] = data.loc[:, "sim"].values
hloc.loc[:, idim] = data.loc[:, ("maindata", "sim")].values
jacobian[(comp, trcr)] = hloc
harray[tracer.ypointer: tracer.ypointer + tracer.dim, :] = \
......
......@@ -40,7 +40,7 @@ def build_Hx(obsvect, ensemble_dir, x_sample):
"{}/H_matrix/obsvect_{:04d}/{}/{}/monitor.nc"\
.format(ensemble_dir, isample, comp, trcr)
data = read_datastore(monitor_file)
hloc.loc[:, isample] = data.loc[:, "sim"].values
hloc.loc[:, isample] = data.loc[:, ("maindata", "sim")].values
jacobian[(comp, trcr)] = hloc
harray[tracer.ypointer: tracer.ypointer + tracer.dim, :] = \
......
......@@ -110,10 +110,11 @@ def dummy_config_fwd(ref_dummy, request):
if os.path.isdir(target_dir):
shutil.rmtree(target_dir)
shutil.copytree(
"{}/obsvect".format(tmpdir_str),
target_dir
)
if os.path.isdir("{}/obsvect".format(tmpdir_str)):
shutil.copytree(
"{}/obsvect".format(tmpdir_str),
target_dir
)
# Copy the H matrix to avoid re-compiling it every time
Hfile_ref = "{}/model/H_matrix.pickle".format(tmpdir_str)
......@@ -121,5 +122,6 @@ def dummy_config_fwd(ref_dummy, request):
if os.path.isfile(Hfile_target):
os.remove(Hfile_target)
shutil.copy(Hfile_ref, Hfile_target)
if os.path.isfile(Hfile_ref):
shutil.copy(Hfile_ref, Hfile_target)
......@@ -15,6 +15,12 @@ from pycif.utils.classes.setup import Setup
scope='session', autouse=True)
def plot_inversion(pytestconfig):
yield
# Do only for LSCE branch
marker = pytestconfig.getoption('-m')
if "allsimulations" not in marker:
return
# Plot varying cost functions at the very end of pytest
marker = pytestconfig.getoption('-m')
if "article" in marker and "not article" not in marker \
......
......@@ -65,11 +65,11 @@ def test_integration_fwd(dummy_config_fwd, pytestconfig):
coords = monitor_ref.loc[:, ["lon", "lat", "alt"]].drop_duplicates()
# Read fluxes
file_flx = "{}/obsoperator/fwd_0000/controlvect/fluxes/" \
"controlvect_fluxes_CH4_ref.nc".format(tmpdir)
file_flx = "{}/obsoperator/fwd_0000/controlvect/flux/" \
"controlvect_flux_CH4_ref.nc".format(tmpdir)
fluxes = xr.open_dataset(file_flx)["xb_phys"].mean(axis=(0, 1))
file_flx = "{}/obsoperator/fwd_0000/controlvect/fluxes/" \
"controlvect_fluxes_CH4_perturb.nc".format(tmpdir)
file_flx = "{}/obsoperator/fwd_0000/controlvect/flux/" \
"controlvect_flux_CH4_perturb.nc".format(tmpdir)
fluxes_perturb = xr.open_dataset(file_flx)["xb_phys"].mean(axis=(0, 1))
# Show the fluxes
......
......@@ -39,11 +39,24 @@ def test_integration_inversion(dummy_config_inversion, settings, pytestconfig):
# from forward to make computation faster
config["model"]["reload_H"] = "{}/../H_matrix.pickle".format(tmpdir)
# Changing mode
# Update number of simulations depending on pytests options
marker = pytestconfig.getoption('-m')
nsimmax = 10
montecarlo = settings.get("montecarlo", 10)
nsample = settings.get("nsample", 5)
if "allsimulations" not in marker:
nsimmax = 5
montecarlo = 3
nsample = 2
# Skip analytical inversions
if settings["mode"] == "analytical":
pytest.skip("Skipping analytical inversions.")
# Changing mode
if config["datavect"]["components"]["flux"]\
["parameters"]["CH4"]["hresol"] == "hpixels":
nsimmax = 25
nsimmax = int(2.5 * nsimmax)
elif config["datavect"]["components"]["flux"]\
["parameters"]["CH4"]["hresol"] == "global" \
......@@ -186,6 +199,11 @@ def test_integration_inversion(dummy_config_inversion, settings, pytestconfig):
"rootdir": root_dir},
replace_values={"rootdir": "/tmp/CIF/"})
# Stop here if not full article computation
marker = pytestconfig.getoption('-m')
if "allsimulations" not in marker:
return
# Loop with different number of simulations for non analytical inversions
title = ""
obs_root = "obsvect_posterior"
......@@ -272,12 +290,12 @@ def test_integration_inversion(dummy_config_inversion, settings, pytestconfig):
# Read observations
file_obs = "{}/obsvect/concs/CH4/monitor.nc".format(tmpdir)
monitor_ref = read_datastore(file_obs)
monitor_ref = read_datastore(file_obs)["metadata"]
coords = monitor_ref.loc[:, ["lon", "lat", "alt"]].drop_duplicates()
# Compute fluxes from control vector
file_flx = "{}/{}/fluxes/" \
"controlvect_fluxes_CH4.nc".format(tmpdir, control_root)
file_flx = "{}/{}/flux/" \
"controlvect_flux_CH4.nc".format(tmpdir, control_root)
ds = xr.open_dataset(file_flx)
dflx = ds["x_phys"].mean(axis=(0, 1)) - ds["xb_phys"].mean(axis=(0, 1))
dx = ds["x"].mean(axis=(0, 1)) - ds["xb"].mean(axis=(0, 1))
......
......@@ -11,6 +11,7 @@ setenv =
PYCIF_DATATEST=/tmp/PYCIF_DATA_TEST/
extras=test
skipsdist=True
usedevelop=True
commands=
pytest --basetemp={envtmpdir} --cov=pycif --no-cov-on-fail --html reports/pytest-{envname}.html --self-contained-html {posargs}
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment