Commit ab34842a authored by Espen Sollum's avatar Espen Sollum
Browse files

Bug in obsparser. Doing merge with master

parent 6a4352d9
...@@ -215,5 +215,6 @@ meteo : ...@@ -215,5 +215,6 @@ meteo :
version : csv version : csv
dirmeteo : /home/eso/repos/CIF/data/dummy_gauss/ dirmeteo : /home/eso/repos/CIF/data/dummy_gauss/
ficmeteo : meteo.csv ficmeteo : meteo.csv
resolution='1H'
##################################################################### #####################################################################
##################################################################### #####################################################################
from builtins import range
from .init_xb import init_xb from .init_xb import init_xb
from .init_bprod import init_bprod from .init_bprod import init_bprod
from .control2native import control2native from .control2native import control2native
...@@ -41,12 +42,8 @@ def ini_data(plugin, **kwargs): ...@@ -41,12 +42,8 @@ def ini_data(plugin, **kwargs):
components = plugin.components components = plugin.components
for comp in components.attributes: for comp in components.attributes:
component = getattr(components, comp) component = getattr(components, comp)
#ESO for trcr in component.parameters.attributes:
print ("component", components) tracer = getattr(component.parameters, trcr)
# for trcr in component.parameters.attributes:
for trcr in component.attributes:
# tracer = getattr(component.parameters, trcr)
tracer = getattr(component, trcr)
tracer.dir = getattr(tracer, 'dir', tracer.dir = getattr(tracer, 'dir',
getattr(component, 'dir', getattr(component, 'dir',
...@@ -54,12 +51,12 @@ def ini_data(plugin, **kwargs): ...@@ -54,12 +51,12 @@ def ini_data(plugin, **kwargs):
getattr(plugin.model, getattr(plugin.model,
comp, None), comp, None),
'dir', ''))) 'dir', '')))
tracer.fic = getattr(tracer, 'fic', tracer.file = getattr(tracer, 'file',
getattr(component, 'fic', getattr(component, 'file',
getattr( getattr(
getattr(plugin.model, getattr(plugin.model,
comp, None), comp, None),
'fic', ''))) 'file', '')))
# Forces the tracer to have an empty read function # Forces the tracer to have an empty read function
if not hasattr(tracer, 'read'): if not hasattr(tracer, 'read'):
...@@ -69,16 +66,21 @@ def ini_data(plugin, **kwargs): ...@@ -69,16 +66,21 @@ def ini_data(plugin, **kwargs):
# Gets read from model if not already defined # Gets read from model if not already defined
try: try:
tracer.read(*range(4)) tracer.read(*range(4))
# except PluginError:
# except TypeError:
# tracer.read(*list(range(4)))
except PluginError: except PluginError:
if comp in ['fluxes', 'inicond', 'latcond', 'topcond']: if comp in ['fluxes', 'inicond', 'latcond', 'topcond']:
tracer.read = getattr(plugin.model, comp).read tracer.read = getattr(plugin.model, comp).read
print "ESO: TypeError"
else: else:
raise Exception('WARNING: {} was not defined in the ' raise Exception('WARNING: {} was not defined in the '
'accepted components types of the ' 'accepted components types of the '
'control vector'.format(comp)) 'control vector'.format(comp))
except Exception: except Exception:
print "ESO: exception"
pass pass
# Gets the domain and # Gets the domain and
......
from __future__ import division
from past.utils import old_div
import numpy as np import numpy as np
import os import os
from itertools import islice from itertools import islice
...@@ -7,7 +9,7 @@ import pycif.utils.check as check ...@@ -7,7 +9,7 @@ import pycif.utils.check as check
def build_hcorrelations(zlat, zlon, lsm, def build_hcorrelations(zlat, zlon, lsm,
sigma_land, sigma_sea, fic_lsm=None, sigma_land, sigma_sea, file_lsm=None,
evalmin=0.5, evalmin=0.5,
dump=False, dir_dump='', projection='gps', dump=False, dir_dump='', projection='gps',
**kwargs): **kwargs):
...@@ -21,7 +23,7 @@ def build_hcorrelations(zlat, zlon, lsm, ...@@ -21,7 +23,7 @@ def build_hcorrelations(zlat, zlon, lsm,
Args: Args:
zlat (np.array): 2D array of latitudes zlat (np.array): 2D array of latitudes
zlon (np.array): 2D array of longitudes zlon (np.array): 2D array of longitudes
fic_lsm (str): path to NetCDF file with land-sea mask (grid must be file_lsm (str): path to NetCDF file with land-sea mask (grid must be
consistent with LMDZ grid); the land-sea mask is assumed to be stored consistent with LMDZ grid); the land-sea mask is assumed to be stored
in the varible 'lsm' in the varible 'lsm'
sigma_land (float): decay distance for correlation between land cells sigma_land (float): decay distance for correlation between land cells
...@@ -52,7 +54,7 @@ def build_hcorrelations(zlat, zlon, lsm, ...@@ -52,7 +54,7 @@ def build_hcorrelations(zlat, zlon, lsm,
check.verbose("Computing hcorr") check.verbose("Computing hcorr")
# No correlation between land and sea if lsm = True # No correlation between land and sea if lsm = True
if lsm: if lsm:
landseamask = readnc(fic_lsm, ['lsm']).flatten() landseamask = readnc(file_lsm, ['lsm']).flatten()
sigma = sigma_land * (landseamask[:, np.newaxis] >= 0.5) \ sigma = sigma_land * (landseamask[:, np.newaxis] >= 0.5) \
* (landseamask[np.newaxis, :] >= 0.5) \ * (landseamask[np.newaxis, :] >= 0.5) \
+ sigma_sea * (landseamask[:, np.newaxis] < 0.5) \ + sigma_sea * (landseamask[:, np.newaxis] < 0.5) \
...@@ -67,7 +69,7 @@ def build_hcorrelations(zlat, zlon, lsm, ...@@ -67,7 +69,7 @@ def build_hcorrelations(zlat, zlon, lsm,
dx = dist_matrix(zlat, zlon, projection) dx = dist_matrix(zlat, zlon, projection)
# Compute the correlation matrix itself # Compute the correlation matrix itself
corr = np.exp(-dx / sigma) corr = np.exp(old_div(-dx, sigma))
corr[sigma <= 0] = 0 corr[sigma <= 0] = 0
# Component analysis # Component analysis
...@@ -85,7 +87,7 @@ def build_hcorrelations(zlat, zlon, lsm, ...@@ -85,7 +87,7 @@ def build_hcorrelations(zlat, zlon, lsm,
dump_hcorr(nlon, nlat, sigma_sea, sigma_land, dump_hcorr(nlon, nlat, sigma_sea, sigma_land,
evalues, evectors, dir_dump) evalues, evectors, dir_dump)
except Exception, e: except Exception as e:
raise e raise e
# Truncating values < evalmin # Truncating values < evalmin
...@@ -105,15 +107,15 @@ def dump_hcorr(nlon, nlat, sigma_sea, sigma_land, ...@@ -105,15 +107,15 @@ def dump_hcorr(nlon, nlat, sigma_sea, sigma_land,
ncell = evalues.size ncell = evalues.size
fic_dump = '{}/horcor_{}x{}_cs{}_cl{}_lmdz.bin'.format( file_dump = '{}/horcor_{}x{}_cs{}_cl{}_lmdz.bin'.format(
dir_dump, nlon, nlat, sigma_sea, sigma_land) dir_dump, nlon, nlat, sigma_sea, sigma_land)
if os.path.isfile(fic_dump): if os.path.isfile(file_dump):
raise IOError("Warning: {} already exists. " raise IOError("Warning: {} already exists. "
"I don't want to overwrite it".format(fic_dump)) "I don't want to overwrite it".format(file_dump))
datasave = np.concatenate((evalues[np.newaxis, :], evectors), axis=0) datasave = np.concatenate((evalues[np.newaxis, :], evectors), axis=0)
datasave.tofile(fic_dump) datasave.tofile(file_dump)
def read_hcorr(nlon, nlat, sigma_sea, sigma_land, dir_dump): def read_hcorr(nlon, nlat, sigma_sea, sigma_land, dir_dump):
...@@ -126,15 +128,15 @@ def read_hcorr(nlon, nlat, sigma_sea, sigma_land, dir_dump): ...@@ -126,15 +128,15 @@ def read_hcorr(nlon, nlat, sigma_sea, sigma_land, dir_dump):
""" """
fic_dump = '{}/horcor_{}x{}_cs{}_cl{}_lmdz.bin'.format( file_dump = '{}/horcor_{}x{}_cs{}_cl{}_lmdz.bin'.format(
dir_dump, nlon, nlat, sigma_sea, sigma_land) dir_dump, nlon, nlat, sigma_sea, sigma_land)
if not os.path.isfile(fic_dump): if not os.path.isfile(file_dump):
raise IOError("{} does not exist. " raise IOError("{} does not exist. "
"Please compute correlations from scratch".format "Please compute correlations from scratch".format
(fic_dump)) (file_dump))
data = np.fromfile(fic_dump).reshape((-1, nlon * nlat)) data = np.fromfile(file_dump).reshape((-1, nlon * nlat))
evalues = data[0] evalues = data[0]
evectors = data[1:] evectors = data[1:]
......
from __future__ import division
from past.utils import old_div
import numpy as np import numpy as np
import os import os
import pandas as pd import pandas as pd
...@@ -43,9 +45,8 @@ def build_tcorrelations(period, dates, sigma_t, ...@@ -43,9 +45,8 @@ def build_tcorrelations(period, dates, sigma_t,
check.verbose("Computing temporal correlations") check.verbose("Computing temporal correlations")
# Compute matrix of distance # Compute matrix of distance
dt = (pd.DatetimeIndex(dates)[:, np.newaxis] dt = old_div((pd.DatetimeIndex(dates)[:, np.newaxis]
- pd.DatetimeIndex(dates)[np.newaxis, :]) \ - pd.DatetimeIndex(dates)[np.newaxis, :]), np.timedelta64(sigma_t, 'h'))
/ np.timedelta64(sigma_t, 'h')
# Compute the correlation matrix itself # Compute the correlation matrix itself
corr = np.exp(-dt ** 2) corr = np.exp(-dt ** 2)
...@@ -65,7 +66,7 @@ def build_tcorrelations(period, dates, sigma_t, ...@@ -65,7 +66,7 @@ def build_tcorrelations(period, dates, sigma_t,
dump_tcorr(period, dates, sigma_t, dump_tcorr(period, dates, sigma_t,
evalues, evectors, dir_dump) evalues, evectors, dir_dump)
except Exception, e: except Exception as e:
raise e raise e
# Truncating values < evalmin # Truncating values < evalmin
...@@ -92,16 +93,16 @@ def dump_tcorr(period, dates, sigma_t, ...@@ -92,16 +93,16 @@ def dump_tcorr(period, dates, sigma_t,
datei = dates[0] datei = dates[0]
datef = dates[-1] datef = dates[-1]
fic_dump = '{}/tempcor_{}_{}_per{}_ct{}_lmdz.bin'.format( file_dump = '{}/tempcor_{}_{}_per{}_ct{}_lmdz.bin'.format(
dir_dump, datei.strftime("%Y%m%d%H%M"), datef.strftime("%Y%m%d%H%M"), dir_dump, datei.strftime("%Y%m%d%H%M"), datef.strftime("%Y%m%d%H%M"),
period, sigma_t) period, sigma_t)
if os.path.isfile(fic_dump): if os.path.isfile(file_dump):
raise IOError("Warning: {} already exists. " raise IOError("Warning: {} already exists. "
"I don't want to overwrite it".format(fic_dump)) "I don't want to overwrite it".format(file_dump))
datasave = np.concatenate((evalues[np.newaxis, :], evectors), axis=0) datasave = np.concatenate((evalues[np.newaxis, :], evectors), axis=0)
datasave.tofile(fic_dump) datasave.tofile(file_dump)
def read_tcorr(period, dates, sigma_t, dir_dump): def read_tcorr(period, dates, sigma_t, dir_dump):
...@@ -119,16 +120,16 @@ def read_tcorr(period, dates, sigma_t, dir_dump): ...@@ -119,16 +120,16 @@ def read_tcorr(period, dates, sigma_t, dir_dump):
datei = dates[0] datei = dates[0]
datef = dates[-1] datef = dates[-1]
fic_dump = '{}/tempcor_{}_{}_per{}_ct{}_lmdz.bin'.format( file_dump = '{}/tempcor_{}_{}_per{}_ct{}_lmdz.bin'.format(
dir_dump, datei.strftime("%Y%m%d%H%M"), datef.strftime("%Y%m%d%H%M"), dir_dump, datei.strftime("%Y%m%d%H%M"), datef.strftime("%Y%m%d%H%M"),
period, sigma_t) period, sigma_t)
if not os.path.isfile(fic_dump): if not os.path.isfile(file_dump):
raise IOError("{} does not exist. " raise IOError("{} does not exist. "
"Please compute correlations from scratch".format "Please compute correlations from scratch".format
(fic_dump)) (file_dump))
data = np.fromfile(fic_dump).reshape((-1, len(dates))) data = np.fromfile(file_dump).reshape((-1, len(dates)))
evalues = data[0] evalues = data[0]
evectors = data[1:] evectors = data[1:]
......
from __future__ import absolute_import
import datetime import datetime
import xarray as xr import xarray as xr
import numpy as np import numpy as np
from pycif.utils.check import verbose from pycif.utils.check import verbose
from pycif.utils.check.errclass import PluginError from pycif.utils.check.errclass import PluginError
from .utils import dateslice, scale2map from .utils.dates import dateslice
from .utils.scalemaps import scale2map
from .utils.reindex import reindex
def control2native(self, mod_input, di, df, mode, def control2native(self, mod_input, di, df, mode,
...@@ -27,7 +30,7 @@ def control2native(self, mod_input, di, df, mode, ...@@ -27,7 +30,7 @@ def control2native(self, mod_input, di, df, mode,
""" """
# TODO: There is a hidden conceptual layer here # TODO: There is a hidden conceptual layer here
# Until now, it is assumed that inputs should have a direct corresponding # So far, it is assumed that inputs should have a direct corresponding
# component in the control vector, whereas one would need a general # component in the control vector, whereas one would need a general
# mapping function between the control vector structure and the inputs # mapping function between the control vector structure and the inputs
# structure # structure
...@@ -37,7 +40,6 @@ def control2native(self, mod_input, di, df, mode, ...@@ -37,7 +40,6 @@ def control2native(self, mod_input, di, df, mode,
ddi = min(di, df) ddi = min(di, df)
ddf = min(di, df) ddf = min(di, df)
# If this type of input is not considered in the control vector, # If this type of input is not considered in the control vector,
# return nothing # return nothing
...@@ -66,9 +68,9 @@ def control2native(self, mod_input, di, df, mode, ...@@ -66,9 +68,9 @@ def control2native(self, mod_input, di, df, mode,
if cdates[0] < ddi: if cdates[0] < ddi:
cdates[0] = ddi cdates[0] = ddi
# Translates only control variables corresponding to the # Translates only control variables corresponding to the
# simulation period # simulation period
for x in variables:
tmp = np.reshape( tmp = np.reshape(
variables[x][tracer.xpointer:tracer.xpointer + tracer.dim], variables[x][tracer.xpointer:tracer.xpointer + tracer.dim],
(tracer.ndates, tracer.nlev, -1))[dslice] (tracer.ndates, tracer.nlev, -1))[dslice]
...@@ -83,7 +85,7 @@ def control2native(self, mod_input, di, df, mode, ...@@ -83,7 +85,7 @@ def control2native(self, mod_input, di, df, mode,
if getattr(tracer, 'type', 'scalar') == 'scalar': if getattr(tracer, 'type', 'scalar') == 'scalar':
# Read the tracer array and apply the present control vector # Read the tracer array and apply the present control vector
# scaling factor # scaling factor
inputs = tracer.read(trcr, tracer.dir, tracer.fic, inputs = tracer.read(trcr, tracer.dir, tracer.file,
self.model.input_dates[ddi], self.model.input_dates[ddi],
comp_type=mod_input, comp_type=mod_input,
model=self.model, model=self.model,
...@@ -118,7 +120,7 @@ def control2native(self, mod_input, di, df, mode, ...@@ -118,7 +120,7 @@ def control2native(self, mod_input, di, df, mode,
xmod[trcr].pop('scale') xmod[trcr].pop('scale')
# Saving reference directories if specified # Saving reference directories if specified
xmod[trcr]['ficorig'] = getattr(tracer, 'fic', None) xmod[trcr]['fileorig'] = getattr(tracer, 'file', None)
xmod[trcr]['dirorig'] = getattr(tracer, 'dir', None) xmod[trcr]['dirorig'] = getattr(tracer, 'dir', None)
return xmod return xmod
import cPickle as pickle from future import standard_library
standard_library.install_aliases()
import pickle as pickle
import numpy as np import numpy as np
import xarray as xr import xarray as xr
from .utils import scale2map from .utils.scalemaps import scale2map
from pycif.utils.path import init_dir from pycif.utils.path import init_dir
def dump(self, fic, to_netcdf=False, dir_netcdf=None, **kwargs): def dump(self, cntrl_file, to_netcdf=False, dir_netcdf=None, **kwargs):
"""Dumps a control vector into a pickle file. """Dumps a control vector into a pickle file.
Does not save large correlations. Does not save large correlations.
Args: Args:
self (pycif.utils.classes.controlvects.ControlVect): self (pycif.utils.classes.controlvects.ControlVect):
the Control Vector to dump the Control Vector to dump
fic (str): path to the file to dump as pickle file (str): path to the file to dump as pickle
to_netcdf (bool): save to netcdf files if True to_netcdf (bool): save to netcdf files if True
dir_netcdf (str): root path for the netcdf directory dir_netcdf (str): root path for the netcdf directory
""" """
...@@ -31,7 +33,7 @@ def dump(self, fic, to_netcdf=False, dir_netcdf=None, **kwargs): ...@@ -31,7 +33,7 @@ def dump(self, fic, to_netcdf=False, dir_netcdf=None, **kwargs):
tosave['xb'] = self.xb tosave['xb'] = self.xb
# Dumping the dictionary to a pickle # Dumping the dictionary to a pickle
with open(fic, 'w') as f: with open(cntrl_file, 'wb') as f:
pickle.dump(tosave, f, pickle.HIGHEST_PROTOCOL) pickle.dump(tosave, f, pickle.HIGHEST_PROTOCOL)
# Dumping to an ensemble of NetCDF files # Dumping to an ensemble of NetCDF files
...@@ -52,24 +54,25 @@ def dump(self, fic, to_netcdf=False, dir_netcdf=None, **kwargs): ...@@ -52,24 +54,25 @@ def dump(self, fic, to_netcdf=False, dir_netcdf=None, **kwargs):
x = np.reshape( x = np.reshape(
self.x[tracer.xpointer:tracer.xpointer + tracer.dim], self.x[tracer.xpointer:tracer.xpointer + tracer.dim],
(tracer.ndates, -1)) (tracer.ndates, -1))
x = scale2map(x, tracer, tracer.dates[:-1], self.domain) x = scale2map(x, tracer, tracer.dates, self.domain)
xb = np.reshape( xb = np.reshape(
self.xb[tracer.xpointer:tracer.xpointer + tracer.dim], self.xb[tracer.xpointer:tracer.xpointer + tracer.dim],
(tracer.ndates, -1)) (tracer.ndates, -1))
xb = scale2map(xb, tracer, tracer.dates[:-1], self.domain) xb = scale2map(xb, tracer, tracer.dates, self.domain)
std = np.reshape( std = np.reshape(
self.std[tracer.xpointer:tracer.xpointer + tracer.dim], self.std[tracer.xpointer:tracer.xpointer + tracer.dim],
(tracer.ndates, -1)) (tracer.ndates, -1))
std = scale2map(std, tracer, tracer.dates[:-1], self.domain) std = scale2map(std, tracer, tracer.dates, self.domain)
ds = xr.Dataset({'x': x, 'xb': xb, 'std': std}) ds = xr.Dataset({'x': x, 'xb': xb, 'std': std, 'lon' : tracer.domain.lon, 'lat' :
tracer.domain.lat})
ds.to_netcdf('{}/controlvect_{}_{}.nc'.format(dir_comp, comp, trcr)) ds.to_netcdf('{}/controlvect_{}_{}.nc'.format(dir_comp, comp, trcr))
def load(self, fic, **kwargs): def load(self, cntrl_file, **kwargs):
with open(fic, 'r') as f: with open(cntrl_file, 'r') as f:
toread = pickle.load(f) toread = pickle.load(f)
out = self.from_dict(toread) out = self.from_dict(toread)
......
...@@ -28,12 +28,11 @@ def init_bprod(cntrlv, options={}, **kwargs): ...@@ -28,12 +28,11 @@ def init_bprod(cntrlv, options={}, **kwargs):
components = cntrlv.components components = cntrlv.components
for comp in components.attributes: for comp in components.attributes:
component = getattr(components, comp) component = getattr(components, comp)
# for trcr in component.parameters.attributes: for trcr in component.parameters.attributes:
# tracer = getattr(component.parameters, trcr) tracer = getattr(component.parameters, trcr)
for trcr in component.attributes:
tracer = getattr(component, trcr)
# Deals with horizontal correlations if any # Deals with horizontal correlations if any
if hasattr(tracer, 'hcorrelations') and tracer.resol == 'hpixels': if hasattr(tracer, 'hcorrelations') and tracer.hresol == 'hpixels':
corr = tracer.hcorrelations corr = tracer.hcorrelations
dump_hcorr = getattr(corr, 'dump_hcorr', False) dump_hcorr = getattr(corr, 'dump_hcorr', False)
dircorrel = getattr(corr, 'dircorrel', '') dircorrel = getattr(corr, 'dircorrel', '')
...@@ -47,13 +46,13 @@ def init_bprod(cntrlv, options={}, **kwargs): ...@@ -47,13 +46,13 @@ def init_bprod(cntrlv, options={}, **kwargs):
if lsm: if lsm:
sigma_land = getattr(corr, 'sigma_land', -1) sigma_land = getattr(corr, 'sigma_land', -1)
sigma_sea = getattr(corr, 'sigma_sea', -1) sigma_sea = getattr(corr, 'sigma_sea', -1)
fic_lsm = getattr(corr, 'ficlsm') file_lsm = getattr(corr, 'filelsm')
else: else:
sigma = getattr(corr, 'sigma', -1) sigma = getattr(corr, 'sigma', -1)
sigma_land = sigma sigma_land = sigma
sigma_sea = -999 sigma_sea = -999
fic_lsm = None file_lsm = None
# Load or compute the horizontal correlations # Load or compute the horizontal correlations
# Checks before whether they were already loaded # Checks before whether they were already loaded
...@@ -70,7 +69,7 @@ def init_bprod(cntrlv, options={}, **kwargs): ...@@ -70,7 +69,7 @@ def init_bprod(cntrlv, options={}, **kwargs):
build_hcorrelations( build_hcorrelations(
grid.zlat, grid.zlon, lsm, grid.zlat, grid.zlon, lsm,
sigma_land, sigma_sea, sigma_land, sigma_sea,
fic_lsm=fic_lsm, evalmin=evalmin, file_lsm=file_lsm, evalmin=evalmin,
dump=dump_hcorr, dir_dump=dircorrel, dump=dump_hcorr, dir_dump=dircorrel,
projection=projection, projection=projection,
**kwargs) **kwargs)
......
import numpy as np import numpy as np
from scipy import ndimage from scipy import ndimage
from pycif.utils import dates from pycif.utils import dates
from .utils import resol2dim from .utils.dimensions import hresol2dim, vresol2dim
from .utils import dateslice, map2scale from .utils.scalemaps import map2scale, vmap2vaggreg
import pdb
def init_xb(cntrlv, **kwargs): def init_xb(cntrlv, **kwargs):
"""Initializes the prior control vector. Loops over all components and """Initializes the prior control vector. Loops over all components and
...@@ -31,13 +30,10 @@ def init_xb(cntrlv, **kwargs): ...@@ -31,13 +30,10 @@ def init_xb(cntrlv, **kwargs):
# Else, carry on initializing