test_integration_3inversions.py 11.2 KB
Newer Older
1
2
3
import os
import time
import shutil
4
5
6
import numpy as np
import matplotlib.pyplot as plt
import xarray as xr
7
8
9
10
11
12
13
14
15
16
17
18
19
20

import pytest

from pycif.utils.classes.setup import Setup
from pycif.utils.datastores.dump import read_datastore
from pycif.utils.yml import ordered_dump
from pycif.utils.path import init_dir


@pytest.mark.dummy
@pytest.mark.article
@pytest.mark.parametrize(
    "settings", [
        {"mode": "4dvar", "minimizer": "M1QN3"},
Antoine Berchet's avatar
Antoine Berchet committed
21
22
23
        {"mode": "4dvar", "minimizer": "congrad"},
        {"mode": "analytical"},
        {"mode": "ensrf"}
24
    ]
25
)
Antoine Berchet's avatar
Antoine Berchet committed
26
def test_integration_inversion(dummy_config_inversion, settings, pytestconfig):
27
28
29
30
31
32
33
    """
    Integration test that runs the dummy_forward model.
    """

    tmpdir, config, tag = dummy_config_inversion
    
    # Changing mode
Antoine Berchet's avatar
Antoine Berchet committed
34
    nsimmax = 10
35
36
    if config["datavect"]["components"]["fluxes"]\
            ["parameters"]["CH4"]["hresol"] == "hpixels":
Antoine Berchet's avatar
Antoine Berchet committed
37
        nsimmax = 25
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
    
    elif config["datavect"]["components"]["fluxes"]\
            ["parameters"]["CH4"]["hresol"] == "global" \
            and settings.get("minimizer", "congrad"):
        nsimmax = 1
        
    if settings["mode"] == "4dvar":
        mode = {
            "plugin": {"name": "4dvar", "version": "std"},
            "minimizer": {
                "plugin": {"name": settings["minimizer"], "version": "std"},
                "simulator": {
                    "plugin": {"name": "gausscost", "version": "std"},
                    "reload_from_previous": True
                },
                "maxiter": nsimmax,
                "nsim": nsimmax,
                "epsg": 0.02,
Antoine Berchet's avatar
Antoine Berchet committed
56
                "df1": 0.00001
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
            },
            "save_out_netcdf": True
        }
        prior_dir = os.path.join(tmpdir, "obsoperator/fwd_-001/obsvect")
        posterior_dir = os.path.join(tmpdir, "obsoperator/fwd_-002/obsvect")
    
    elif settings["mode"] == "analytical":
        mode = {"plugin": {"name": "analytic", "version": "std",
                           "dump_nc_base_control": True}}
        prior_dir = os.path.join(tmpdir, "obsvect_prior")
        posterior_dir = os.path.join(tmpdir, "obsvect_posterior")
    
    elif settings["mode"] == "ensrf":
        nsimmax *= 2
        mode = {"plugin": {"name": "EnSRF", "version": "std"},
                "nsample": nsimmax}
        prior_dir = os.path.join(tmpdir, "obsvect_prior")
        posterior_dir = os.path.join(tmpdir, "obsvect_posterior")
      
    config["mode"] = mode
Antoine Berchet's avatar
Antoine Berchet committed
77
    config["platform"] = {"plugin": {"name": "docker", "version": "cif"}}
Antoine Berchet's avatar
Antoine Berchet committed
78
79
80
81
82
83
84
85
86
87
88
89

    # Save ID for later plot of cost function
    resolution = \
        config["datavect"]["components"]["fluxes"]["parameters"]["CH4"][
            "hresol"]
    sigma = config["datavect"]["components"]\
                ["fluxes"]["parameters"]["CH4"]["hcorrelations"]["sigma"]
    correlations = "lowcorr" if sigma == 500 else "highcorr"
    test_id = (settings["mode"], settings.get("minimizer", ""),
               resolution, correlations)

    pytest.test_ids[test_id] = (tmpdir, nsimmax)
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
    
    # Removing MCF to make computation quicker
    del config["model"]["chemistry"]["acspecies"]["MCF"]
    del config["datavect"]["components"]["fluxes"]["parameters"]["MCF"]
    del config["datavect"]["components"]["concs"]["parameters"]["MCF"]
    
    # Dump yml config file
    dummy_config_file = os.path.join(tmpdir, "dummy_config.yml")
    with open(dummy_config_file, "w") as outfile:
        ordered_dump(outfile, config)
    
    # Run as job
    setup = Setup.yaml_to_setup(dummy_config_file)
    setup.todo_init = ["platform"]
    setup = Setup.load_config(setup)
    platform = setup.platform

    job_file = os.path.join(tmpdir,
                            "job_pycif_{}".format(tmpdir[-5:]))

    exe = "{} -m pycif {}".format(platform.python, dummy_config_file)

    job_id = platform.submit_job(
        exe,
        job_file
    )
Antoine Berchet's avatar
Antoine Berchet committed
116
    
117
118
119
    # Wait for the end of the execution
    while not platform.check_jobs([job_id]):
        time.sleep(platform.sleep_time)
Antoine Berchet's avatar
Antoine Berchet committed
120
    
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
    # Get results and check that posterior closer to prior
    components = os.listdir(prior_dir)
    for comp in components:
        comp_dir_prior = os.path.join(prior_dir, comp)
        comp_dir_post = os.path.join(posterior_dir, comp)
        parameters = os.listdir(comp_dir_prior)
        for param in parameters:
            param_dir = os.path.join(comp_dir_prior, param)
            monitor_prior = \
                read_datastore(os.path.join(param_dir, "monitor.nc"))

            param_dir = os.path.join(comp_dir_post, param)
            monitor_post = \
                read_datastore(os.path.join(param_dir, "monitor.nc"))
            assert (
                    (monitor_prior.loc[:, "obs"]
                     - monitor_prior.loc[:, "sim"]).pow(2).sum()
                    - (monitor_post.loc[:, "obs"]
                       - monitor_post.loc[:, "sim"]).pow(2).sum()
                    > 0
            )

    # Dump configuration into CIF examples
    tag += "_{}_{}".format(settings["mode"],
                           "" if settings["mode"] != "4dvar"
                           else settings["minimizer"])

    current_dir = os.path.abspath(os.path.dirname(os.path.realpath(__file__)))
    example_dir = \
        os.path.abspath(os.path.join(current_dir, "../../examples/dummy/"))

    dummy_config_file = \
Antoine Berchet's avatar
Antoine Berchet committed
153
        os.path.join(example_dir, "config_{}.yml".format(tag))
154
155
156
    with open(dummy_config_file, "w") as outfile:
        ordered_dump(outfile, config)
    
Antoine Berchet's avatar
Antoine Berchet committed
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
    # Loop with different number of simulations for non analytical inversions
    title = ""
    obs_root = "obsvect_posterior"
    control_root = "controlvect"
    if settings["mode"] != "analytical":
        for nsim in range(1, nsimmax + 1, 2):
            file_config = "{}/dummy_config.yml".format(tmpdir)
            inv_setup = Setup.from_yaml(file_config)
            
            if settings.get("minimizer", None) == "congrad":
                title = "CONGRAD"
                obs_root = "obsoperator/fwd_-002/obsvect"
                control_root = "obsoperator/fwd_-002/controlvect"
                inv_setup["mode"]["minimizer"]["maxiter"] = \
                    max(1, int(nsim))
                if os.path.isdir("{}/obsoperator/fwd_-002".format(tmpdir)):
                    shutil.rmtree("{}/obsoperator/fwd_-002".format(tmpdir))
            elif settings.get("minimizer", None) == "M1QN3":
                title = "M1QN3"
                obs_root = "obsoperator/fwd_-002/obsvect"
                control_root = "obsoperator/fwd_-002/controlvect"
                inv_setup["mode"]["minimizer"]["maxiter"] = \
                    max(1, int(nsim))
                inv_setup["mode"]["minimizer"]["nsim"] = \
                    max(1, int(nsim))
                if os.path.isdir("{}/obsoperator/fwd_-002".format(tmpdir)):
                    shutil.rmtree("{}/obsoperator/fwd_-002".format(tmpdir))
            elif settings["mode"] == "ensrf":
                title = "EnSRF"
                inv_setup["mode"]["nsample"] = nsim
            
            # Execute a partial inversion with less simulations
            inv_setup = Setup.from_dict(inv_setup, convert_none=True)
            inv_setup = inv_setup.load_config(inv_setup)
            controlvect, obsvect = inv_setup.mode.execute()
            
            # Compute the cost function
            departures = obsvect.ysim - obsvect.yobs
            j_o = 0.5 * (departures * obsvect.rinvprod(departures)).sum()
            
            dx = controlvect.x - controlvect.xb
            bfull = np.linalg.inv(controlvect.build_b(inv_setup.controlvect))
            j_b = dx[np.newaxis, :].dot(bfull.dot(dx[:, np.newaxis])).sum() / 2
            
            # Dump to text file for later plot
            with open("{}/varying_cost_function.txt".format(tmpdir), "a") as f:
                f.write("{} {} {}\n".format(nsim, j_o, j_b))
    
    else:
206
207
        title = "Analytical"
        
Antoine Berchet's avatar
Antoine Berchet committed
208
209
210
        # Load results
        file_config = "{}/dummy_config.yml".format(tmpdir)
        inv_setup = Setup.from_yaml(file_config)
211
212
        inv_setup = Setup.from_dict(inv_setup, convert_none=True)
        inv_setup = inv_setup.load_config(inv_setup)
213
214
215
216
217
218
219
220
221
222
223
        controlvect, obsvect = inv_setup.mode.execute()
        
        # Compute the cost function
        departures = obsvect.ysim - obsvect.yobs
        j_o = 0.5 * (departures * obsvect.rinvprod(departures)).sum()
        
        dx = controlvect.x - controlvect.xb
        bfull = np.linalg.inv(
            inv_setup.controlvect
                .build_b(inv_setup.controlvect)
        )
Antoine Berchet's avatar
Antoine Berchet committed
224
        j_b = dx[np.newaxis, :].dot(bfull.dot(dx[:, np.newaxis])).sum() / 2
225
226
227
        
        # Dump to text file for later plot
        with open("{}/varying_cost_function.txt".format(tmpdir), "a") as f:
Antoine Berchet's avatar
Antoine Berchet committed
228
229
            f.write("{} {} {}\n".format(0, j_o, j_b))
            f.write("{} {} {}\n".format(nsimmax, j_o, j_b))
230
    
Antoine Berchet's avatar
Antoine Berchet committed
231
232
233
234
235
236
237
    marker = pytestconfig.getoption('-m')
    if "article" in marker and "not article" not in marker:
        # Domain limits
        xmin = inv_setup.domain.xmin
        xmax = inv_setup.domain.xmax
        ymin = inv_setup.domain.ymin
        ymax = inv_setup.domain.ymax
238
    
Antoine Berchet's avatar
Antoine Berchet committed
239
240
241
242
243
244
245
246
247
248
249
250
251
        # Read observations
        file_obs = "{}/obsvect/concs/CH4/monitor.nc".format(tmpdir)
        monitor_ref = read_datastore(file_obs)
        coords = monitor_ref.loc[:, ["lon", "lat", "alt"]].drop_duplicates()
        
        # Compute fluxes from control vector
        file_flx = "{}/{}/fluxes/" \
                   "controlvect_fluxes_CH4.nc".format(tmpdir, control_root)
        ds = xr.open_dataset(file_flx)
        dflx = ds["x_phys"].mean(axis=(0, 1)) - ds["xb_phys"].mean(axis=(0, 1))
        
        # Fetch resolution for figure name
        resol = ""
252
        if config["datavect"]["components"][
Antoine Berchet's avatar
Antoine Berchet committed
253
254
255
256
257
258
                "fluxes"]["parameters"]["CH4"]["hresol"] == "hpixels":
            if config["datavect"]["components"][
                    "fluxes"]["parameters"]["CH4"]["hcorrelations"]["sigma"] == 500:
                resol = "lowcorr"
            else:
                resol = "highcorr"
259
    
Antoine Berchet's avatar
Antoine Berchet committed
260
261
262
        elif config["datavect"]["components"]["fluxes"] \
                ["parameters"]["CH4"]["hresol"] == "ibands":
            resol = "bands"
263
    
Antoine Berchet's avatar
Antoine Berchet committed
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
        elif config["datavect"]["components"]["fluxes"] \
                ["parameters"]["CH4"]["hresol"] == "global":
            resol = "global"
        
        # Plot the figure
        plt.figure(figsize=(21, 11))
        
        ax0 = plt.axes([0.05, 0.05, 0.73, 0.87])
        im = plt.imshow(dflx, extent=(xmin, xmax, ymin, ymax), cmap="YlOrRd",
                        vmin=-0.2, vmax=0.5)
        sc = plt.scatter(coords["lon"], coords["lat"], c=coords["alt"],
                         cmap="Blues", linewidths=1, edgecolors="k", s=600)
        plt.yticks(fontsize=25)
        plt.xticks(fontsize=25)
        
        ax1 = plt.axes([0.74, 0.05, 0.05, 0.87])
        cb1 = plt.colorbar(im, cax=ax1)
        plt.yticks(fontsize=25)
        plt.ylabel("Fluxes (a.u.)", fontsize=30)
        
        ax2 = plt.axes([0.86, 0.05, 0.05, 0.87])
        cb2 = plt.colorbar(sc, cax=ax2)
        plt.yticks(fontsize=25)
        plt.ylabel("Station altitude (m a.g.l)", fontsize=30)
        
        ax0.set_title(title, fontsize=45)
        plt.savefig("{}/../map_dx_{}_{}.pdf".format(tmpdir, title, resol))
        plt.close()