Commit bad00a30 authored by Espen Sollum's avatar Espen Sollum
Browse files

Merge branch 'espen' of git.nilu.no:VERIFY/CIF into espen

parents b68b53d5 122d0b9d
...@@ -15,7 +15,6 @@ ...@@ -15,7 +15,6 @@
# y|Y|yes|Yes|YES|n|N|no|No|NO|true|True|TRUE|false|False|FALSE|on|On|ON|off|Off|OFF # y|Y|yes|Yes|YES|n|N|no|No|NO|true|True|TRUE|false|False|FALSE|on|On|ON|off|Off|OFF
# This offers flexibility, but one should stick to a style for consistency # This offers flexibility, but one should stick to a style for consistency
#####################################################################
##################################################################### #####################################################################
# pyCIF parameters # pyCIF parameters
...@@ -33,10 +32,10 @@ workdir : /home/eso/repos/CIF/flextest/ ...@@ -33,10 +32,10 @@ workdir : /home/eso/repos/CIF/flextest/
# Initial date # Initial date
# Use the following compatible format for the date: # Use the following compatible format for the date:
# YYYY-mm-dd or YYYY-mm-dd HH:mm # YYYY-mm-dd or YYYY-mm-dd HH:mm
datei : 2012-01-01 datei : 2012-02-01
# End date (same as for the initial date) # End date (same as for the initial date)
datef : 2012-01-31 datef : 2012-02-11 00:00
##################################################################### #####################################################################
##################################################################### #####################################################################
# Transport model # Transport model
...@@ -54,32 +53,19 @@ model : ...@@ -54,32 +53,19 @@ model :
name : FLEXPART name : FLEXPART
version : std version : std
periods : 1MS periods : 1MS
# periods : 3MS
nested : yes nested : yes
run_dir_nest : /home/eso/FLEX_INV/TEST_OUTPUT/FLEXOUT/GHG/NEST/ run_dir_nest : /home/eso/FLEX_INV/CH4/TEST_OUTPUT/FLEXOUT/GHG/NEST/
run_dir_glob : /home/eso/FLEX_INV/TEST_OUTPUT/FLEXOUT/GHG/NEST/ run_dir_glob : /home/eso/FLEX_INV/CH4/TEST_OUTPUT/FLEXOUT/GHG/NEST/
coeff : 1.e-3 coeff : 1.e-3
mmair : 28.97 mmair : 28.97
molarmass : 16. molarmass : 16.
numscale : 1.E12 numscale : 1.E12
####################################################################
####################################################################
#####################################################################
#####################################################################
# How to build your observation vector and observation uncertainties if needed
# Also projects information from the observation to the model space
# - fic_obsvect: observation vector from previous simulations
obsvect:
plugin:
name: standard
version: std
# file_obsvect : /home/eso/repos/CIF/flextest/monitor_obsvect.nc
dump: True
dump_type: nc
#####################################################################
#####################################################################
#####################################################################
#####################################################################
# Measurements to account for # Measurements to account for
# Main keys are: # Main keys are:
# - infos: infos on tracers to include in the simulation # - infos: infos on tracers to include in the simulation
...@@ -98,7 +84,7 @@ measurements : ...@@ -98,7 +84,7 @@ measurements :
dailyPM : dailyPM :
provider : WDCGG provider : WDCGG
format : std format : std
dir_obs : /home/eso/FLEX_INV/TEST_INPUT/OBS/GHG/*.cn.* dir_obs : /home/eso/FLEX_INV/CH4/TEST_INPUT/OBS/GHG/*.cn.*
# rescale : false # rescale : false
rescale : true rescale : true
na_values : -999999.99 na_values : -999999.99
...@@ -108,8 +94,50 @@ measurements : ...@@ -108,8 +94,50 @@ measurements :
# Minimum measeurement error # Minimum measeurement error
measerr : 5.0 measerr : 5.0
#################################################################### #####################################################################
#################################################################### #####################################################################
# How to build your observation vector and observation uncertainties if needed
# Also projects information from the observation to the model space
# - fic_obsvect: observation vector from previous simulations
# - dump_debug: write out extra information (for debugging)
# - sort_order: (optional) datastore sort order, ['index', 'station'] is default
# For FLEXPART the background plugin takes the following parameters
# - dir_initconc: directory containing concentration files
# - file_initconc: name of concentration files
# - varname_initconc: name of fata variable in the netCDF file
# - optimize_cini: optimize initial concentration or not
# - cini_lat: northern edges of latitude bands for cini optimization
# - cini_err: Initial concentration error (unit: same as observations, or fraction???)
obsvect:
plugin:
name: standard
version: std
# file_obsvect : /home/eso/repos/CIF/flextest/monitor_obsvect.nc
dump: True
dump_debug: True
dump_type: nc
sort_order: ['station', 'index']
background:
plugin:
name: FLEXPART
version: nc
# parameters:
# CH4 :
dir_netcdf : /home/eso/repos/CIF/flextest/
# Initial concentrations
dir_initconc : /home/eso/FLEX_INV/CH4/TEST_INPUT/INIT_CONC/GHG/
file_initconc: ch4_noaa_%Y%m.nc
varname_conc: CH4
optimize_cini: True
# Time resolution for initial mixing ratio scalars (days)
cini_res : 28.
cini_lat: [-30., 0., 30., 90.]
cini_err: .01
#####################################################################
#####################################################################
# spaces to control/observation spaces # spaces to control/observation spaces
# - read_only: do not run the transport model, read output from existing run # - read_only: do not run the transport model, read output from existing run
...@@ -119,24 +147,9 @@ obsoperator: ...@@ -119,24 +147,9 @@ obsoperator:
version: std version: std
autorestart: false autorestart: false
read_only: true read_only: true
#####################################################################
#####################################################################
#################################################################### ####################################################################
##################################################################### #####################################################################
# Domain definition
# domain :
# plugin :
# name : FLEXPART
# version : std
# xmin : -180
# xmax : 180
# ymin : -90
# ymax : 90
# nlon : 360
# nlat : 180
# nlev : 1
# type : deg
# Domain definition # Domain definition
# coordinates refers to edge of inversion domain # coordinates refers to edge of inversion domain
...@@ -153,12 +166,14 @@ domain : ...@@ -153,12 +166,14 @@ domain :
nlat : 90 nlat : 90
nlev : 1 nlev : 1
type : deg type : deg
xmin_glob : -179. xmin_glob : -180.
ymin_glob : -90. ymin_glob : -90.
nlon_glob : 360 nlon_glob : 360
nlat_glob : 180 nlat_glob : 180
dx_glob : 1. dx_glob : 1.
dy_glob : 1. dy_glob : 1.
heights : [100., 500., 1000., 2000., 3000., 5000., 7000., 9000.,
12000., 15000., 20000., 50000.]
...@@ -177,9 +192,10 @@ mode: ...@@ -177,9 +192,10 @@ mode:
version: flexpart version: flexpart
reload_from_previous: False reload_from_previous: False
maxiter: 10 maxiter: 10
# zreduc: 0.001 df1: 1.0
# epsg: 0.01 # df1: 0.01
df1: 0.01 nsim: 100
m: 1
save_out_netcdf: True save_out_netcdf: True
#################################################################### ####################################################################
...@@ -189,6 +205,8 @@ mode: ...@@ -189,6 +205,8 @@ mode:
# Please refer to the documentation to know what to include here # Please refer to the documentation to know what to include here
# For the standard FLEXPART, include the following: # For the standard FLEXPART, include the following:
# - ficlsm: land-sea mask (must be consistent with LMDZ grid) # - ficlsm: land-sea mask (must be consistent with LMDZ grid)
# - regions_lsm : if true, use land/sea mask encoded in fileregions (sea: < 0, land: > 0)
# This overrides usage of lsm file, if this is also provided
# - correl: Use correlation or not in B # - correl: Use correlation or not in B
# - dircorrel: path to pre-computed correlations # - dircorrel: path to pre-computed correlations
# - sigma_land: spatial correlation length for prior errors over land (km) # - sigma_land: spatial correlation length for prior errors over land (km)
...@@ -201,8 +219,8 @@ mode: ...@@ -201,8 +219,8 @@ mode:
# if bands, define a list of latitudes as band limits (n+1 for n bands) # if bands, define a list of latitudes as band limits (n+1 for n bands)
# - inc_ocean: for 'regions', optimize ocean boxes (default true) # - inc_ocean: for 'regions', optimize ocean boxes (default true)
# - periodflux: period of variation for increments within a month (days) # - periodflux: period of variation for increments within a month (days)
# - glob_err (optional) = uncertainty on global budget # - flxerr_ll: lower limit flux error: unit (kg/m2/h)
# - glob_err (optional) = uncertainty on global budget # - glob_err (optional): uncertainty on global budget. Units (Tg/y).
controlvect: controlvect:
plugin: plugin:
name: standard name: standard
...@@ -214,25 +232,21 @@ controlvect: ...@@ -214,25 +232,21 @@ controlvect:
plugin: plugin:
name: 'FLEXPART' name: 'FLEXPART'
version: 'nc' version: 'nc'
# resol : hpixels
type : flexpart type : flexpart
to_netcdf : True to_netcdf : True
dir_netcdf : /home/eso/repos/CIF/flextest/ dir_netcdf : /home/eso/repos/CIF/flextest/
hresol : regions hresol : regions
inc_ocean : true inc_ocean : true
fileregions : /home/eso/FLEX_INV/TEST_OUTPUT/regions_ghg.nc fileregions : /home/eso/FLEX_INV/CH4/TEST_OUTPUT/regions_ghg.nc
# errtype : max regions_lsm : True
errtype : avg
err : 0.5 err : 0.5
# Lower limit flux error: unit (kg/m2/h)
flxerr_ll: 1.e-8 flxerr_ll: 1.e-8
# Total error inversion domain: unit (Tg/y) -- todo: not implemented # glob_err: 50.
glob_err: 10.
numscale : 1.E12 numscale : 1.E12
xb_scale : 1. xb_scale : 1.
# periodflux : 5D period : 1D
period : 10D # period : 10D
dir : /home/eso/FLEX_INV/TEST_OUTPUT/FLUXES/GHG/ dir : /home/eso/FLEX_INV/CH4/TEST_OUTPUT/FLUXES/GHG/
file : CH4_TOTAL_%Y_05x05.nc file : CH4_TOTAL_%Y_05x05.nc
file_glob : CH4_TOTAL_%Y_10x10.nc file_glob : CH4_TOTAL_%Y_10x10.nc
...@@ -243,24 +257,25 @@ controlvect: ...@@ -243,24 +257,25 @@ controlvect:
offsets : True offsets : True
# offsets : False # offsets : False
# Initial concentrations
dir_initconc : /home/eso/FLEX_INV/TEST_INPUT/INIT_CONC/GHG/
file_initconc: ch4_noaa_%Y%M.nc
varname_init: CH4
hcorrelations : hcorrelations :
# landsea: True landsea: True
landsea: False # landsea: False
filelsm : /home/eso/FLEX_INV/TEST_INPUT/lsm_0.5x0.5_VERIFY.nc filelsm : /home/eso/FLEX_INV/CH4/TEST_INPUT/lsm_0.5x0.5_VERIFY.nc
dircorrel : /home/eso/repos/CIF/flextest/ dircorrel : /home/eso/repos/CIF/flextest/
dump_hcorr : True dump_hcorr : True
sigma_land: 250 sigma_land: 250.
sigma_sea: 1000 sigma_sea: 1000.
# sigma_land: 1.
# sigma_sea: 1.
evalmin : 1.e-6
# evalmin : 1.e-15
tcorrelations : tcorrelations :
sigma_t : 30 # sigma_t : 30
sigma_t : 10
dump_tcorr : True dump_tcorr : True
dircorrel : /home/eso/repos/CIF/flextest/ dircorrel : /home/eso/repos/CIF/flextest/
evalmin : 1.e-5 # evalmin : 1.e-6
evalmin : 1.e-99
...@@ -25,7 +25,7 @@ def read_conc(self, model, dir_initconc, file_initconc, **kwargs): ...@@ -25,7 +25,7 @@ def read_conc(self, model, dir_initconc, file_initconc, **kwargs):
TS = 288.15 # surface temperature (K) TS = 288.15 # surface temperature (K)
GC = 9.81 # gravitational acceleration (m/s2) GC = 9.81 # gravitational acceleration (m/s2)
MMOL = 0.02894 # (kg/mol) MMOL = 0.02894 # (kg/mol)
PSURF = 101325 # surface pressure (Pa) PSURF = 101325. # surface pressure (Pa)
# Available files in the directory # Available files in the directory
list_files = os.listdir(dir_initconc) list_files = os.listdir(dir_initconc)
...@@ -37,7 +37,7 @@ def read_conc(self, model, dir_initconc, file_initconc, **kwargs): ...@@ -37,7 +37,7 @@ def read_conc(self, model, dir_initconc, file_initconc, **kwargs):
continue continue
dates_available = np.array(dates_available) dates_available = np.array(dates_available)
# If available, read files from one month before and after simulation window. # If available, read files from one month before and after simulation window.
# Note: subsimu_dates not properly implemented yet # Note: subsimu_dates not properly implemented yet
files_conc = pd.date_range(model.subsimu_dates[0] - pd.DateOffset(months=1), files_conc = pd.date_range(model.subsimu_dates[0] - pd.DateOffset(months=1),
...@@ -86,7 +86,6 @@ def read_conc(self, model, dir_initconc, file_initconc, **kwargs): ...@@ -86,7 +86,6 @@ def read_conc(self, model, dir_initconc, file_initconc, **kwargs):
xmod_ver['height'] = np.array(model.domain.heights) xmod_ver['height'] = np.array(model.domain.heights)
# Use middle of month to represent the time of concentration data TODO: check # Use middle of month to represent the time of concentration data TODO: check
xmod_ver['time'] = datetime.strptime(file_conc, file_initconc) + timedelta(days=14) xmod_ver['time'] = datetime.strptime(file_conc, file_initconc) + timedelta(days=14)
if conc_all is None: if conc_all is None:
......
from builtins import range from builtins import range
from .init_xb import init_xb from .init_xb import init_xb
from .background_xb import background_xb
from .init_bprod import init_bprod from .init_bprod import init_bprod
from .control2native import control2native from .control2native import control2native
from .native2control import native2control from .native2control import native2control
...@@ -29,6 +30,7 @@ def ini_data(plugin, **kwargs): ...@@ -29,6 +30,7 @@ def ini_data(plugin, **kwargs):
""" """
# Initializes reference directories if needed # Initializes reference directories if needed
init_dir('{}/controlvect/'.format(plugin.workdir)) init_dir('{}/controlvect/'.format(plugin.workdir))
......
...@@ -5,6 +5,7 @@ import os ...@@ -5,6 +5,7 @@ import os
from itertools import islice from itertools import islice
from pycif.utils.geometry import dist_matrix from pycif.utils.geometry import dist_matrix
from pycif.utils.netcdf import readnc from pycif.utils.netcdf import readnc
from .utils.scalemaps import map2scale
import pycif.utils.check as check import pycif.utils.check as check
...@@ -13,6 +14,7 @@ def build_hcorrelations(zlat, zlon, lsm, ...@@ -13,6 +14,7 @@ def build_hcorrelations(zlat, zlon, lsm,
evalmin=0.5, regions=False, evalmin=0.5, regions=False,
hresoldim=None, hresoldim=None,
dump=False, dir_dump='', projection='gps', dump=False, dir_dump='', projection='gps',
tracer=None, errscalar=None,
**kwargs): **kwargs):
"""Build horizontal correlation matrix based on distance between grid """Build horizontal correlation matrix based on distance between grid
cells. cells.
...@@ -22,8 +24,8 @@ def build_hcorrelations(zlat, zlon, lsm, ...@@ -22,8 +24,8 @@ def build_hcorrelations(zlat, zlon, lsm,
un-correlated un-correlated
Args: Args:
zlat (np.array): 2D array of latitudes zlat (np.array): 2D array of latitudes or 1D array (hresoldim)
zlon (np.array): 2D array of longitudes zlon (np.array): 2D array of longitudes or 1D array (hresoldim)
file_lsm (str): path to NetCDF file with land-sea mask (grid must be file_lsm (str): path to NetCDF file with land-sea mask (grid must be
consistent with LMDZ grid); the land-sea mask is assumed to be stored consistent with LMDZ grid); the land-sea mask is assumed to be stored
in the varible 'lsm' in the varible 'lsm'
...@@ -36,6 +38,7 @@ def build_hcorrelations(zlat, zlon, lsm, ...@@ -36,6 +38,7 @@ def build_hcorrelations(zlat, zlon, lsm,
dump (bool): dumps computed correlations if True dump (bool): dumps computed correlations if True
dir_dump (str): directory where correlation matrices are stored dir_dump (str): directory where correlation matrices are stored
projection (str): the projection used for the longitudes and latitudes projection (str): the projection used for the longitudes and latitudes
tracer: tracer class
Return: Return:
tuple with: tuple with:
...@@ -49,7 +52,10 @@ def build_hcorrelations(zlat, zlon, lsm, ...@@ -49,7 +52,10 @@ def build_hcorrelations(zlat, zlon, lsm,
""" """
# Define domain dimensions # Define domain dimensions
nlon, nlat = zlat.shape if regions:
nlon, nlat = len(zlon), len(zlat)
else:
nlon, nlat = zlat.shape
# Try reading existing file # Try reading existing file
try: try:
...@@ -61,34 +67,47 @@ def build_hcorrelations(zlat, zlon, lsm, ...@@ -61,34 +67,47 @@ def build_hcorrelations(zlat, zlon, lsm,
except IOError: except IOError:
check.verbose("Computing hcorr") check.verbose("Computing hcorr")
# TODO 'regions' # No correlation between land and sea if lsm = True
if regions: if lsm:
if hresoldim is None: if regions:
raise Exception("hresoldim missing!") regions_lsm = getattr(tracer, 'regions_lsm', False)
if regions_lsm:
landseamask2d = tracer.regions_lsmask
landseamask = map2scale(
landseamask2d[np.newaxis, np.newaxis, :, :],
tracer, tracer.domain, region_scale_area=False,
region_max_val=True).flatten()
else:
landseamask2d = readnc(file_lsm, ['lsm'])
landseamask = map2scale(
landseamask2d[np.newaxis, np.newaxis, :, :],
tracer, tracer.domain, region_scale_area=False,
region_max_val=True).flatten()
else:
landseamask = readnc(file_lsm, ['lsm']).flatten()
corr = np.identity(hresoldim) sigma = sigma_land * (landseamask[:, np.newaxis] >= 0.5) \
* (landseamask[np.newaxis, :] >= 0.5) \
+ sigma_sea * (landseamask[:, np.newaxis] < 0.5) \
* (landseamask[np.newaxis, :] < 0.5)
# Otherwise, isotropic correlation
# Takes sigma_land
else: else:
# No correlation between land and sea if lsm = True sigma = sigma_land
if lsm:
landseamask = readnc(file_lsm, ['lsm']).flatten()
sigma = sigma_land * (landseamask[:, np.newaxis] >= 0.5) \
* (landseamask[np.newaxis, :] >= 0.5) \
+ sigma_sea * (landseamask[:, np.newaxis] < 0.5) \
* (landseamask[np.newaxis, :] < 0.5)
# Otherwise, isotropic correlation # Compute matrix of distance
# Takes sigma_land dx = dist_matrix(zlat, zlon, projection)
else:
sigma = sigma_land
# Compute matrix of distance # Compute the correlation matrix itself
dx = dist_matrix(zlat, zlon, projection) corr = np.exp(old_div(-dx, sigma))
corr[sigma <= 0] = 0
# Compute the correlation matrix itself # If total error specified, scale correlation matrix
corr = np.exp(old_div(-dx, sigma)) if errscalar is not None:
corr[sigma <= 0] = 0 corr *= errscalar**2
# Component analysis # Component analysis
evalues, evectors = np.linalg.eigh(corr) evalues, evectors = np.linalg.eigh(corr)
...@@ -98,7 +117,7 @@ def build_hcorrelations(zlat, zlon, lsm, ...@@ -98,7 +117,7 @@ def build_hcorrelations(zlat, zlon, lsm,
evalues = evalues[index] evalues = evalues[index]
evectors = evectors[:, index] evectors = evectors[:, index]
# Dumping to a txt file # Dumping to a txt file
if dump: if dump:
dump_hcorr(nlon, nlat, sigma_sea, sigma_land, dump_hcorr(nlon, nlat, sigma_sea, sigma_land,
...@@ -108,7 +127,11 @@ def build_hcorrelations(zlat, zlon, lsm, ...@@ -108,7 +127,11 @@ def build_hcorrelations(zlat, zlon, lsm,
raise e raise e
# Truncating values < evalmin # Truncating values < evalmin
mask = evalues >= evalmin # mask = evalues >= evalmin
# ESO: This is how it is done in flexinvert
mask = evalues >= evalmin*evalues.max()
check.verbose("Truncating eigenvalues at " + str(evalmin*evalues.max()))
return evalues[mask] ** 0.5, evectors[:, mask] return evalues[mask] ** 0.5, evectors[:, mask]
...@@ -133,10 +156,16 @@ def dump_hcorr(nlon, nlat, sigma_sea, sigma_land, ...@@ -133,10 +156,16 @@ def dump_hcorr(nlon, nlat, sigma_sea, sigma_land,
"I don't want to overwrite it".format(file_dump)) "I don't want to overwrite it".format(file_dump))
datasave = np.concatenate((evalues[np.newaxis, :], evectors), axis=0) datasave = np.concatenate((evalues[np.newaxis, :], evectors), axis=0)
# ESO: Writing to text file (for testing)
# if isinstance(datasave, np.ma.MaskedArray):
# datasave.filled().tofile(file_dump, sep="\n", format="%11.3E")
# else:
# datasave.tofile(file_dump, sep="\n", format="%11.3E")
if isinstance(datasave, np.ma.MaskedArray): if isinstance(datasave, np.ma.MaskedArray):
datasave.filled().tofile(file_dump) datasave.filled().tofile(file_dump)
else: else:
datasave.tofile(file_dump) datasave.tofile(file_dump)
def read_hcorr(nlon, nlat, sigma_sea, sigma_land, dir_dump, hresoldim=None): def read_hcorr(nlon, nlat, sigma_sea, sigma_land, dir_dump, hresoldim=None):
......
...@@ -50,7 +50,9 @@ def build_tcorrelations(period, dates, sigma_t, ...@@ -50,7 +50,9 @@ def build_tcorrelations(period, dates, sigma_t,
- pd.DatetimeIndex(dates)[np.newaxis, :]), np.timedelta64(sigma_t, 'D')) - pd.DatetimeIndex(dates)[np.newaxis, :]), np.timedelta64(sigma_t, 'D'))
# Compute the correlation matrix itself # Compute the correlation matrix itself
corr = np.exp(-dt ** 2) # corr = np.exp(-dt ** 2)