Commit e545544d authored by Espen Sollum's avatar Espen Sollum
Browse files

Added branch for flexpart debuging

parent ba66ebb2
...@@ -15,7 +15,6 @@ ...@@ -15,7 +15,6 @@
# y|Y|yes|Yes|YES|n|N|no|No|NO|true|True|TRUE|false|False|FALSE|on|On|ON|off|Off|OFF # y|Y|yes|Yes|YES|n|N|no|No|NO|true|True|TRUE|false|False|FALSE|on|On|ON|off|Off|OFF
# This offers flexibility, but one should stick to a style for consistency # This offers flexibility, but one should stick to a style for consistency
#####################################################################
##################################################################### #####################################################################
# pyCIF parameters # pyCIF parameters
...@@ -33,10 +32,10 @@ workdir : /home/eso/repos/CIF/flextest/ ...@@ -33,10 +32,10 @@ workdir : /home/eso/repos/CIF/flextest/
# Initial date # Initial date
# Use the following compatible format for the date: # Use the following compatible format for the date:
# YYYY-mm-dd or YYYY-mm-dd HH:mm # YYYY-mm-dd or YYYY-mm-dd HH:mm
datei : 2012-01-01 datei : 2012-02-01
# End date (same as for the initial date) # End date (same as for the initial date)
datef : 2012-01-31 datef : 2012-02-11 00:00
##################################################################### #####################################################################
##################################################################### #####################################################################
# Transport model # Transport model
...@@ -54,9 +53,10 @@ model : ...@@ -54,9 +53,10 @@ model :
name : FLEXPART name : FLEXPART
version : std version : std
periods : 1MS periods : 1MS
# periods : 3MS
nested : yes nested : yes
run_dir_nest : /home/eso/FLEX_INV/TEST_OUTPUT/FLEXOUT/GHG/NEST/ run_dir_nest : /home/eso/FLEX_INV/CH4/TEST_OUTPUT/FLEXOUT/GHG/NEST/
run_dir_glob : /home/eso/FLEX_INV/TEST_OUTPUT/FLEXOUT/GHG/NEST/ run_dir_glob : /home/eso/FLEX_INV/CH4/TEST_OUTPUT/FLEXOUT/GHG/NEST/
coeff : 1.e-3 coeff : 1.e-3
mmair : 28.97 mmair : 28.97
molarmass : 16. molarmass : 16.
...@@ -68,15 +68,40 @@ model : ...@@ -68,15 +68,40 @@ model :
# How to build your observation vector and observation uncertainties if needed # How to build your observation vector and observation uncertainties if needed
# Also projects information from the observation to the model space # Also projects information from the observation to the model space
# - fic_obsvect: observation vector from previous simulations # - fic_obsvect: observation vector from previous simulations
# - dump_debug: write out extra information (for debugging)
# For FLEXPART the background plugin takes the following parameters
# - dir_initconc: directory containing concentration files
# - file_initconc: name of concentration files
# - varname_initconc: name of fata variable in the netCDF file
# - optimize_cini: optimize initial concentration or not
# - cini_lat: northern edges of latitude bands for cini optimization
# - cini_err: Initial concentration error (unit same as observations)
obsvect: obsvect:
plugin: plugin:
name: standard name: standard
version: std version: std
# file_obsvect : /home/eso/repos/CIF/flextest/monitor_obsvect.nc # file_obsvect : /home/eso/repos/CIF/flextest/monitor_obsvect.nc
dump: True dump: True
dump_debug: True
dump_type: nc dump_type: nc
#####################################################################
##################################################################### background:
plugin:
name: FLEXPART
version: nc
# parameters:
# CH4 :
dir_netcdf : /home/eso/repos/CIF/flextest/
# Initial concentrations
dir_initconc : /home/eso/FLEX_INV/CH4/TEST_INPUT/INIT_CONC/GHG/
file_initconc: ch4_noaa_%Y%m.nc
varname_conc: CH4
optimize_cini: False
cini_lat: [-30., 0., 30., 90.]
cini_err: 5.
##################################################################### #####################################################################
##################################################################### #####################################################################
...@@ -98,7 +123,7 @@ measurements : ...@@ -98,7 +123,7 @@ measurements :
dailyPM : dailyPM :
provider : WDCGG provider : WDCGG
format : std format : std
dir_obs : /home/eso/FLEX_INV/TEST_INPUT/OBS/GHG/*.cn.* dir_obs : /home/eso/FLEX_INV/CH4/TEST_INPUT/OBS/GHG/*.cn.*
# rescale : false # rescale : false
rescale : true rescale : true
na_values : -999999.99 na_values : -999999.99
...@@ -119,24 +144,9 @@ obsoperator: ...@@ -119,24 +144,9 @@ obsoperator:
version: std version: std
autorestart: false autorestart: false
read_only: true read_only: true
#####################################################################
#####################################################################
#################################################################### ####################################################################
##################################################################### #####################################################################
# Domain definition
# domain :
# plugin :
# name : FLEXPART
# version : std
# xmin : -180
# xmax : 180
# ymin : -90
# ymax : 90
# nlon : 360
# nlat : 180
# nlev : 1
# type : deg
# Domain definition # Domain definition
# coordinates refers to edge of inversion domain # coordinates refers to edge of inversion domain
...@@ -159,6 +169,8 @@ domain : ...@@ -159,6 +169,8 @@ domain :
nlat_glob : 180 nlat_glob : 180
dx_glob : 1. dx_glob : 1.
dy_glob : 1. dy_glob : 1.
heights : [100., 500., 1000., 2000., 3000., 5000., 7000., 9000.,
12000., 15000., 20000., 50000.]
...@@ -202,7 +214,6 @@ mode: ...@@ -202,7 +214,6 @@ mode:
# - inc_ocean: for 'regions', optimize ocean boxes (default true) # - inc_ocean: for 'regions', optimize ocean boxes (default true)
# - periodflux: period of variation for increments within a month (days) # - periodflux: period of variation for increments within a month (days)
# - glob_err (optional) = uncertainty on global budget # - glob_err (optional) = uncertainty on global budget
# - glob_err (optional) = uncertainty on global budget
controlvect: controlvect:
plugin: plugin:
name: standard name: standard
...@@ -214,25 +225,24 @@ controlvect: ...@@ -214,25 +225,24 @@ controlvect:
plugin: plugin:
name: 'FLEXPART' name: 'FLEXPART'
version: 'nc' version: 'nc'
# resol : hpixels
type : flexpart type : flexpart
to_netcdf : True to_netcdf : True
dir_netcdf : /home/eso/repos/CIF/flextest/ dir_netcdf : /home/eso/repos/CIF/flextest/
hresol : regions hresol : regions
inc_ocean : true inc_ocean : true
fileregions : /home/eso/FLEX_INV/TEST_OUTPUT/regions_ghg.nc fileregions : /home/eso/FLEX_INV/CH4/TEST_OUTPUT/regions_ghg.nc
# errtype : max errtype : max
errtype : avg # errtype : avg
err : 0.5 err : 0.5
# Lower limit flux error: unit (kg/m2/h) # Lower limit flux error: unit (kg/m2/h)
flxerr_ll: 1.e-8 flxerr_ll: 1.e-8
# Total error inversion domain: unit (Tg/y) -- todo: not implemented # Total error inversion domain: unit (Tg/y)
glob_err: 10. # glob_err: 10.
numscale : 1.E12 numscale : 1.E12
xb_scale : 1. xb_scale : 1.
# periodflux : 5D # periodflux : 5D
period : 10D period : 1D
dir : /home/eso/FLEX_INV/TEST_OUTPUT/FLUXES/GHG/ dir : /home/eso/FLEX_INV/CH4/TEST_OUTPUT/FLUXES/GHG/
file : CH4_TOTAL_%Y_05x05.nc file : CH4_TOTAL_%Y_05x05.nc
file_glob : CH4_TOTAL_%Y_10x10.nc file_glob : CH4_TOTAL_%Y_10x10.nc
...@@ -243,24 +253,25 @@ controlvect: ...@@ -243,24 +253,25 @@ controlvect:
offsets : True offsets : True
# offsets : False # offsets : False
# Initial concentrations
dir_initconc : /home/eso/FLEX_INV/TEST_INPUT/INIT_CONC/GHG/
file_initconc: ch4_noaa_%Y%M.nc
varname_init: CH4
hcorrelations : hcorrelations :
# landsea: True landsea: True
landsea: False # landsea: False
filelsm : /home/eso/FLEX_INV/TEST_INPUT/lsm_0.5x0.5_VERIFY.nc filelsm : /home/eso/FLEX_INV/CH4/TEST_INPUT/lsm_0.5x0.5_VERIFY.nc
dircorrel : /home/eso/repos/CIF/flextest/ dircorrel : /home/eso/repos/CIF/flextest/
dump_hcorr : True dump_hcorr : True
sigma_land: 250 # sigma_land: 250.
sigma_sea: 1000 # sigma_sea: 1000.
sigma_land: 1.
sigma_sea: 1.
evalmin : 1.e-6
# evalmin : 1.e-15
tcorrelations : tcorrelations :
sigma_t : 30 # sigma_t : 30
sigma_t : 1
dump_tcorr : True dump_tcorr : True
dircorrel : /home/eso/repos/CIF/flextest/ dircorrel : /home/eso/repos/CIF/flextest/
evalmin : 1.e-5 # evalmin : 1.e-6
evalmin : 1.e-99
...@@ -25,7 +25,7 @@ def read_conc(self, model, dir_initconc, file_initconc, **kwargs): ...@@ -25,7 +25,7 @@ def read_conc(self, model, dir_initconc, file_initconc, **kwargs):
TS = 288.15 # surface temperature (K) TS = 288.15 # surface temperature (K)
GC = 9.81 # gravitational acceleration (m/s2) GC = 9.81 # gravitational acceleration (m/s2)
MMOL = 0.02894 # (kg/mol) MMOL = 0.02894 # (kg/mol)
PSURF = 101325 # surface pressure (Pa) PSURF = 101325. # surface pressure (Pa)
# Available files in the directory # Available files in the directory
list_files = os.listdir(dir_initconc) list_files = os.listdir(dir_initconc)
...@@ -37,7 +37,7 @@ def read_conc(self, model, dir_initconc, file_initconc, **kwargs): ...@@ -37,7 +37,7 @@ def read_conc(self, model, dir_initconc, file_initconc, **kwargs):
continue continue
dates_available = np.array(dates_available) dates_available = np.array(dates_available)
# If available, read files from one month before and after simulation window. # If available, read files from one month before and after simulation window.
# Note: subsimu_dates not properly implemented yet # Note: subsimu_dates not properly implemented yet
files_conc = pd.date_range(model.subsimu_dates[0] - pd.DateOffset(months=1), files_conc = pd.date_range(model.subsimu_dates[0] - pd.DateOffset(months=1),
...@@ -86,7 +86,6 @@ def read_conc(self, model, dir_initconc, file_initconc, **kwargs): ...@@ -86,7 +86,6 @@ def read_conc(self, model, dir_initconc, file_initconc, **kwargs):
xmod_ver['height'] = np.array(model.domain.heights) xmod_ver['height'] = np.array(model.domain.heights)
# Use middle of month to represent the time of concentration data TODO: check # Use middle of month to represent the time of concentration data TODO: check
xmod_ver['time'] = datetime.strptime(file_conc, file_initconc) + timedelta(days=14) xmod_ver['time'] = datetime.strptime(file_conc, file_initconc) + timedelta(days=14)
if conc_all is None: if conc_all is None:
......
...@@ -5,6 +5,7 @@ import os ...@@ -5,6 +5,7 @@ import os
from itertools import islice from itertools import islice
from pycif.utils.geometry import dist_matrix from pycif.utils.geometry import dist_matrix
from pycif.utils.netcdf import readnc from pycif.utils.netcdf import readnc
from .utils.scalemaps import map2scale
import pycif.utils.check as check import pycif.utils.check as check
...@@ -13,6 +14,7 @@ def build_hcorrelations(zlat, zlon, lsm, ...@@ -13,6 +14,7 @@ def build_hcorrelations(zlat, zlon, lsm,
evalmin=0.5, regions=False, evalmin=0.5, regions=False,
hresoldim=None, hresoldim=None,
dump=False, dir_dump='', projection='gps', dump=False, dir_dump='', projection='gps',
tracer=None,
**kwargs): **kwargs):
"""Build horizontal correlation matrix based on distance between grid """Build horizontal correlation matrix based on distance between grid
cells. cells.
...@@ -22,8 +24,8 @@ def build_hcorrelations(zlat, zlon, lsm, ...@@ -22,8 +24,8 @@ def build_hcorrelations(zlat, zlon, lsm,
un-correlated un-correlated
Args: Args:
zlat (np.array): 2D array of latitudes zlat (np.array): 2D array of latitudes or 1D array (hresoldim)
zlon (np.array): 2D array of longitudes zlon (np.array): 2D array of longitudes or 1D array (hresoldim)
file_lsm (str): path to NetCDF file with land-sea mask (grid must be file_lsm (str): path to NetCDF file with land-sea mask (grid must be
consistent with LMDZ grid); the land-sea mask is assumed to be stored consistent with LMDZ grid); the land-sea mask is assumed to be stored
in the varible 'lsm' in the varible 'lsm'
...@@ -36,6 +38,7 @@ def build_hcorrelations(zlat, zlon, lsm, ...@@ -36,6 +38,7 @@ def build_hcorrelations(zlat, zlon, lsm,
dump (bool): dumps computed correlations if True dump (bool): dumps computed correlations if True
dir_dump (str): directory where correlation matrices are stored dir_dump (str): directory where correlation matrices are stored
projection (str): the projection used for the longitudes and latitudes projection (str): the projection used for the longitudes and latitudes
tracer: tracer class
Return: Return:
tuple with: tuple with:
...@@ -49,7 +52,10 @@ def build_hcorrelations(zlat, zlon, lsm, ...@@ -49,7 +52,10 @@ def build_hcorrelations(zlat, zlon, lsm,
""" """
# Define domain dimensions # Define domain dimensions
nlon, nlat = zlat.shape if regions:
nlon, nlat = len(zlon), len(zlat)
else:
nlon, nlat = zlat.shape
# Try reading existing file # Try reading existing file
try: try:
...@@ -62,33 +68,49 @@ def build_hcorrelations(zlat, zlon, lsm, ...@@ -62,33 +68,49 @@ def build_hcorrelations(zlat, zlon, lsm,
check.verbose("Computing hcorr") check.verbose("Computing hcorr")
# TODO 'regions' # TODO 'regions'
if regions: # if regions:
if hresoldim is None: # if hresoldim is None:
raise Exception("hresoldim missing!") # raise Exception("hresoldim missing!")
corr = np.identity(hresoldim) # corr = np.identity(hresoldim)
else: # else:
# No correlation between land and sea if lsm = True # No correlation between land and sea if lsm = True
if lsm: if lsm:
landseamask = readnc(file_lsm, ['lsm']).flatten() if regions:
sigma = sigma_land * (landseamask[:, np.newaxis] >= 0.5) \ landseamask2d = readnc(file_lsm, ['lsm'])
* (landseamask[np.newaxis, :] >= 0.5) \ landseamask = map2scale(landseamask2d[np.newaxis, np.newaxis, :, :], tracer, tracer.domain, region_scale_area=False, region_max_val=True).flatten()
+ sigma_sea * (landseamask[:, np.newaxis] < 0.5) \ import pdb; pdb.set_trace()
* (landseamask[np.newaxis, :] < 0.5)
# Otherwise, isotropic correlation
# Takes sigma_land
else: else:
sigma = sigma_land landseamask = readnc(file_lsm, ['lsm']).flatten()
# Compute matrix of distance # ESO: testing different threshold (0.5) for land value
dx = dist_matrix(zlat, zlon, projection) # sigma = sigma_land * (landseamask[:, np.newaxis] >= 0.5) \
# * (landseamask[np.newaxis, :] >= 0.5) \
# + sigma_sea * (landseamask[:, np.newaxis] < 0.5) \
# * (landseamask[np.newaxis, :] < 0.5)
# Compute the correlation matrix itself sigma = sigma_land * (landseamask[:, np.newaxis] >= 0.01) \
corr = np.exp(old_div(-dx, sigma)) * (landseamask[np.newaxis, :] >= 0.01) \
corr[sigma <= 0] = 0 + sigma_sea * (landseamask[:, np.newaxis] < 0.01) \
* (landseamask[np.newaxis, :] < 0.01)
# Otherwise, isotropic correlation
# Takes sigma_land
else:
sigma = sigma_land
# Compute matrix of distance
dx = dist_matrix(zlat, zlon, projection)
import pdb; pdb.set_trace()
# Compute the correlation matrix itself
corr = np.exp(old_div(-dx, sigma))
corr[sigma <= 0] = 0
# Component analysis # Component analysis
evalues, evectors = np.linalg.eigh(corr) evalues, evectors = np.linalg.eigh(corr)
...@@ -98,7 +120,7 @@ def build_hcorrelations(zlat, zlon, lsm, ...@@ -98,7 +120,7 @@ def build_hcorrelations(zlat, zlon, lsm,
evalues = evalues[index] evalues = evalues[index]
evectors = evectors[:, index] evectors = evectors[:, index]
# Dumping to a txt file # Dumping to a txt file
if dump: if dump:
dump_hcorr(nlon, nlat, sigma_sea, sigma_land, dump_hcorr(nlon, nlat, sigma_sea, sigma_land,
...@@ -108,7 +130,12 @@ def build_hcorrelations(zlat, zlon, lsm, ...@@ -108,7 +130,12 @@ def build_hcorrelations(zlat, zlon, lsm,
raise e raise e
# Truncating values < evalmin # Truncating values < evalmin
mask = evalues >= evalmin # mask = evalues >= evalmin
# ESO: This is how it is done in flexinvert
import pdb; pdb.set_trace()
mask = evalues >= evalmin*evalues.max()
check.verbose("Truncating eigenvalues at " + str(evalmin*evalues.max()))
return evalues[mask] ** 0.5, evectors[:, mask] return evalues[mask] ** 0.5, evectors[:, mask]
...@@ -133,10 +160,16 @@ def dump_hcorr(nlon, nlat, sigma_sea, sigma_land, ...@@ -133,10 +160,16 @@ def dump_hcorr(nlon, nlat, sigma_sea, sigma_land,
"I don't want to overwrite it".format(file_dump)) "I don't want to overwrite it".format(file_dump))
datasave = np.concatenate((evalues[np.newaxis, :], evectors), axis=0) datasave = np.concatenate((evalues[np.newaxis, :], evectors), axis=0)
# ESO: Writing to text file (for testing)
# if isinstance(datasave, np.ma.MaskedArray):
# datasave.filled().tofile(file_dump, sep="\n", format="%11.3E")
# else:
# datasave.tofile(file_dump, sep="\n", format="%11.3E")
if isinstance(datasave, np.ma.MaskedArray): if isinstance(datasave, np.ma.MaskedArray):
datasave.filled().tofile(file_dump) datasave.filled().tofile(file_dump)
else: else:
datasave.tofile(file_dump) datasave.tofile(file_dump)
def read_hcorr(nlon, nlat, sigma_sea, sigma_land, dir_dump, hresoldim=None): def read_hcorr(nlon, nlat, sigma_sea, sigma_land, dir_dump, hresoldim=None):
......
...@@ -50,7 +50,9 @@ def build_tcorrelations(period, dates, sigma_t, ...@@ -50,7 +50,9 @@ def build_tcorrelations(period, dates, sigma_t,
- pd.DatetimeIndex(dates)[np.newaxis, :]), np.timedelta64(sigma_t, 'D')) - pd.DatetimeIndex(dates)[np.newaxis, :]), np.timedelta64(sigma_t, 'D'))
# Compute the correlation matrix itself # Compute the correlation matrix itself
corr = np.exp(-dt ** 2) # corr = np.exp(-dt ** 2)
# ESO:
corr = np.exp(-np.abs(dt))
# Component analysis # Component analysis
evalues, evectors = np.linalg.eigh(corr) evalues, evectors = np.linalg.eigh(corr)
...@@ -72,6 +74,8 @@ def build_tcorrelations(period, dates, sigma_t, ...@@ -72,6 +74,8 @@ def build_tcorrelations(period, dates, sigma_t,
# Truncating values < evalmin # Truncating values < evalmin
mask = evalues >= evalmin mask = evalues >= evalmin
import pdb; pdb.set_trace()
return evalues[mask] ** 0.5, evectors[:, mask] return evalues[mask] ** 0.5, evectors[:, mask]
......
...@@ -60,42 +60,75 @@ def dump(self, cntrl_file, to_netcdf=False, dir_netcdf=None, run_id=None, **kwar ...@@ -60,42 +60,75 @@ def dump(self, cntrl_file, to_netcdf=False, dir_netcdf=None, run_id=None, **kwar
self.x[tracer.xpointer:tracer.xpointer + tracer.dim], self.x[tracer.xpointer:tracer.xpointer + tracer.dim],
(tracer.ndates, -1)) (tracer.ndates, -1))
x = scale2map(x, tracer, tracer.dates, self.domain) x = scale2map(x, tracer, tracer.dates, self.domain)
# store the unaggregated fluxes
# TODO: move this?
arr = self.flxall[:, np.newaxis, :, :]/np.float(self.model.numscale)
xb_grid = xr.DataArray(
arr, coords={'time': tracer.dates[:-1], 'lev': tracer.levels},
dims=('time', 'lev', 'lat', 'lon'))
xb = np.reshape(
self.xb[tracer.xpointer:tracer.xpointer + tracer.dim],
(tracer.ndates, -1))
xb = scale2map(xb, tracer, tracer.dates, self.domain)
xb /= np.float(self.model.numscale)
# Make a copy to write out the original array
xa=copy.deepcopy(x)
xa/=np.float(self.model.numscale)
# If offsets, add prior fluxes # If offsets, add prior fluxes
if getattr(tracer, 'offsets', False): if getattr(tracer, 'offsets', False):
offsets = True
print("Offsets=True") print("Offsets=True")
# Make a copy to write out the original array
xa=copy.deepcopy(x)
xa/=np.float(self.model.numscale)
x[:,0,:,:] = (x[:,0,:,:] + self.flxall)/np.float(self.model.numscale) x[:,0,:,:] = (x[:,0,:,:] + self.flxall)/np.float(self.model.numscale)
else: else:
print("Offsets=False") print("Offsets=False")
x[:,0,:,:] /= np.float(self.model.numscale) offsets = False
xa=copy.deepcopy(x)
xb = np.reshape( # TODO:
self.xb[tracer.xpointer:tracer.xpointer + tracer.dim], # transform the scaling factor (0 - 1) to grid and apply to x. Check
(tracer.ndates, -1)) # that sum of scaling factor is 1 and also check what to do with negative values?
xb = scale2map(xb, tracer, tracer.dates, self.domain)
# Transform scaling factor
scaling_factor = xb_grid / xb
# Rescale using prior flux ratio grid to box
x.values[xb_grid.values > 1.e-15] = x.values[xb_grid.values > 1.e-15]*scaling_factor.values[xb_grid.values > 1.e-15]
#x[xb_grid > 1.e-15] = x*scaling_factor
x[:,:,:,:] /= np.float(self.model.numscale)
xb /= np.float(self.model.numscale)
# store the unaggregated fluxes, for testing
arr = self.flxall[:, np.newaxis, :, :]/np.float(self.model.numscale)
xb_grid = xr.DataArray(arr,