Commit c5094e69 authored by Antoine Berchet's avatar Antoine Berchet
Browse files

Merge branch 'LSCE' into 'master'

Lsce

See merge request !3
parents c5340be7 9c9645aa
Pipeline #38 failed with stage
*.pyc
*.bin
*.o
*.e*
*.o*
*.mod
.idea
pyCIF.egg-info
config_files
......@@ -13,6 +13,7 @@ pages:
stage: deploy
script:
- apk --no-cache add py2-pip python-dev
- apk --no-cache add graphviz
- pip install sphinx
- pip install sphinxcontrib-plantuml
- pip install sphinx-rtd-theme
......
#QSUB -s /bin/tcsh
#PBS -q xlongp
#PBS -l nodes=1:ppn=8
#PBS -l nodes=1:ppn=6
#cd ~/PYVAR/pyvar/trunk
cd ~/CIF
#python -m pyvar examples/config_variational.yml
python -m pycif examples/config_inv_gcp_inca.yml
#python -m pycif config_files/config_inv_gcp_inca_m1qn3.yml
python -m pycif examples/config_lmdz_adjtltest.yml
#####################
# PYVAR config file #
#####################
# PYVAR config file #
#####################
# Define here all parameters for PYVAR following Yaml syntax
......@@ -22,22 +23,23 @@
# Verbose level
# 1 = Basic information
# 2 = Debugging
verbose : 2
verbose : 1
# Log file (to be saved in $wordkir)
logfile: pyvar.logtest
# Execution directory
#workdir : /home/satellites13/msaunois/PYVAR_FWD/GCP_2018/run_00_18_testadtl/
workdir : /home/satellites13/aberchet/msaunois/
#workdir : /home/satellites13/msaunois/INVERSION/GCP_2018/run_00_18_inca_base_network/
#workdir : /home/satellites13/aberchet/msaunois/
workdir : /home/chimereges/aberchet/test_sat/
# Initial date
# Use the following compatible format for the date:
# YYYY-mm-dd or YYYY-mm-dd HH:mm
datei : 2000-01-01
datei : 2010-01-01
# End date (same as for the initial date)
datef : 2001-01-01
datef : 2010-02-01
#####################################################################
#####################################################################
......@@ -46,7 +48,17 @@ datef : 2001-01-01
#####################################################################
# To-Do for initializing PYVAR
# Can be commented if running PYVAR directly
#todo_init: [measurements]
#todo_init: [model]
#####################################################################
#####################################################################
#####################################################################
#####################################################################
platform:
plugin:
name: LSCE
version: obelix
#####################################################################
#####################################################################
......@@ -54,20 +66,42 @@ datef : 2001-01-01
#####################################################################
#####################################################################
# Running mode for PYVAR
#mode:
# plugin:
# name: 4dvar
# version: std
# minimizer:
# plugin:
# name: M1QN3
# version: std
# simulator:
# plugin:
# name: gausscost
# version: std
# reload_from_previous: True
# maxiter: 20
# epsg: 0.02
# df1: 0.01
# chi_min:
# chi_max:
mode:
module:
plugin:
name: adj-tl_test
version: std
increments: 0.2
incrmode: cst
testspace: control
reload_results: True
#####################################################################
#####################################################################
#####################################################################
#####################################################################
# Transport model
# Accepts any model registered in pyvar.models
model :
module:
plugin:
name : LMDZ
version : std
......@@ -75,146 +109,59 @@ model :
periods: 1MS
# Executable
ficexec : /home/users/msaunois/newpyvar/model_sources/DISPERSION_gch/dispersion.e
ficexec : /home/users/aberchet/CIF/model_sources/DISPERSION_gch/dispersion.e
# Definition file (includes some parameters for the simulation)
ficdef : /home/users/msaunois/newpyvar/model_sources/DISPERSION_gch/def/run.def
ficdef : /home/users/aberchet/CIF/model_sources/DISPERSION_gch/def/run.def
# Ap, Bp file
fic_vcoord: /home/inversion/msaunois/LMDZ/39L/restartT2029-2006-1_MCFscaled2CH4scaled.nc
# Initial conditions recovered from:
restart:
inicond:
dir : /home/inversion/msaunois/LMDZ/39L/
fic : restartT2029-2006-1_MCFscaled2CH4scaled.nc
# Include physics and thermals
# Default is True and False respectively
physic : True
thermals : False
# Convection scheme: One of :
# 'TK' = Tiedke
# 'KE' = 'Kerry Emmanuel'
# 'KE-Therm' = 'Kerry Emmanuel + some thermics'
conv_scheme: TK
# Chemistry
chemistry :
kinetic :
dir : /home/inversion/yyin/CHEMFILES/INCA/
fic : inca.an2008.m%m.nc
prescrconcs :
OH :
dir : /home/inversion/yyin/CHEMFILES/INCA/
fic : inca.an2008.m%m.nc
convOH : True
O1D :
dir : /home/inversion/yyin/CHEMFILES/INCA/
fic : inca.an2008.m%m.nc
# Species to be transported in the model
species:
fic_yaml : /home/users/aberchet/CIF/config_files/sacs_chem_scheme_CH4.yml
# Fluxes
emis_species:
CH4:
restart_id: 27
dir: /home/inversion/msaunois/FLUXES/CH4/PYVAR/
fic: sflx_CH4_GCP2018_%Y_LMDZ9696_phy.nc
MCF:
restart_id: 06
dir: /home/inversion/msaunois/FLUXES/CH4/PYVAR/
fic: sflx_MCF_scaled_%Y_LMDZ9696_phy.nc
# Species are read in the file start.nc as qXX with XX the id of the species
restart_ids: [27, 06, 28, 33]
# Dump outputs into a NetCDF file
dump: False
# Flushing trajq.bin files when not needed anymore
autoflush: False
#####################################################################
#####################################################################
#####################################################################
#####################################################################
# Observation operator
# Used to run the model and translates information from model/measurement
# spaces to control/observation spaces
obsoperator:
module:
plugin:
name: standard
version: std
#####################################################################
#####################################################################
#####################################################################
#####################################################################
# Measurements to account for
# Main keys are:
# - infos: infos on tracers to include in the simulation
# - fic_monitor: standard txt file including all observations
measurements :
# File where to save data, if does not exist. Reads from there if exists
fic_monitor : /home/satellites13/msaunois/PYVAR_FWD/GCP_2018/run_00_18_inca/obsoperator/fwd_0000/monitor_selec.nc
dump_type : nc
species :
# List of tracers to include in the monitor file
# For each tracer observations, please specify:
# - provider = list of observation providers
# - format = list of format types
# - dir_obs = directory where to find observation files
# - err_obs (optional) = uncertainty on observations
# - depos (optional) = surface deposition file
# - rescale (optional) = true to change observation scale if necessary
# - na_values (optional) = invalid values to exclude. Default is -999
# - default_unit (optional) = basic unit for reporting the tracer
# everything is then converted to ppm
# - dump (optional) = dump to a monitor file. Default is True
# For tracer fluxes, please specify:
# - dir_flx: directory where reference fluxes are already computed
# - fic_flx: file format to recover fluxes from
CH4 :
dailyPM :
provider : WDCGG
format : std
dir_obs : /home/isomet/msaunois/GCP_2018/Inputs/Obs/CH4/WDCGG_type/*.cn.*
rescale : true
na_values : -999
default_unit : ppb
default_duration: 5
dump : True
err_obs : 0.05
flask :
provider : WDCGG
format : std
dir_obs : /home/isomet/msaunois/GCP_2018/Inputs/Obs/CH4/WDCGG_type/*.fl.*
rescale : true
na_values : -999
default_unit : ppb
default_duration: 1
dump : True
err_obs : 0.05
MCF :
daily_noaa :
provider : WDCGG
format : std
dir_obs : /home/inversion/msaunois/DATA_OBS/MCF/ch3ccl3_20180425_ready/*.aircore.*.cn.*
na_values : -999
err_na_values : -999
default_unit : ppt
default_duration : 24
dump : True
err_obs : 0.1
daily_agage :
provider : WDCGG
format : std
dir_obs : /home/inversion/msaunois/DATA_OBS/MCF/ch3ccl3_20180425_ready/*dmagage.*.cn.*
na_values : 0.
err_na_values : 0.
default_unit : ppt
default_duration : 24
dump : True
err_obs : 0.1
flask :
provider : WDCGG
format : std
dir_obs : /home/inversion/msaunois/DATA_OBS/MCF/ch3ccl3_20180425_ready/*.fl.*
na_values : -999
err_na_values : 0.
default_unit : ppt
default_duration: 1
dump : True
err_obs : 0.1
autorestart: True
#####################################################################
#####################################################################
......@@ -232,69 +179,93 @@ measurements :
# - sigma_sea: spatial correlation length for prior errors over ocean (km)
# - tracers: list of tracers to put in the state vector (with definition arguments):
# - calcstd: calculate global standard deviation
# - resol: resolution at which fields are scaled
# - hresol: resolution at which fields are scaled
# (choice = bands,regions,pixels;
# if regions, provide a netcdf file ficregion
# if bands, define a list of latitudes as band limits (n+1 for n bands)
# - periodflux: period of variation for increments within a month (days)
# - glob_err (optional) = uncertainty on global budget
controlvect:
module:
plugin:
name: standard
version: std
components:
fluxes:
CH4 :
resol : hpixels
type : physical
errtype : max
err : 1
period : 8
dir: /home/inversion/msaunois/FLUXES/CH4/PYVAR/
fic: sflx_CH4_GCP2018_%Y_LMDZ9696_phy.nc
hcorrelations :
ficlsm : /home/inversion/LMDZ/DATA_INV/lmdz9696/lsm.lmdz9696.nc
dump_hcorr : True
dircorrel : /home/inversion/LMDZ/DATA_INV/lmdz9696/CORREL/
sigma_land: 500
sigma_sea: 1000
MCF:
resol : hpixels
period : 8
err : 0.01
hcorrelations :
ficlsm : /home/inversion/LMDZ/DATA_INV/lmdz9696/lsm.lmdz9696.nc
dump_hcorr : True
dircorrel : /home/inversion/LMDZ/DATA_INV/lmdz9696/CORREL/
sigma_land: 500
sigma_sea: 1000
dir: /home/inversion/msaunois/FLUXES/CH4/PYVAR/
fic: sflx_MCF_scaled_%Y_LMDZ9696_phy.nc
prescrconcs :
OH:
resol : bands
period : 8
bands_lat : [-90, -45, 0, 45, 90]
bands_lon : [-180, 180]
err : 0.1
calcstd : False
# fluxes:
# parameters:
# CH4 :
# plugin:
# name: 'LMDZ'
# version: 'sflx'
# hresol : hpixels
# # type : physical
# # errtype : max
# err : 1
# period : 1MS
# subperiod: 10D
# dir: /home/inversion/msaunois/FLUXES/CH4/PYVAR/
# fic: sflx_CH4_GCP2018_%Y_LMDZ9696_phy.nc
# hcorrelations :
# landsea: True
# ficlsm : /home/inversion/LMDZ/DATA_INV/lmdz9696/lsm.lmdz9696.nc
# dump_hcorr : True
# dircorrel : /home/satellites13/aberchet/LMDZ_DATA/CORREL/
# sigma_land: 500
# sigma_sea: 1000
# MCF:
# plugin:
# name: 'LMDZ'
# version: 'sflx'
# hresol : hpixels
# period : 1MS
# subperiod: 10D
# err : 0.01
# hcorrelations :
# landsea: True
# ficlsm : /home/inversion/LMDZ/DATA_INV/lmdz9696/lsm.lmdz9696.nc
# dump_hcorr : True
# dircorrel : /home/satellites13/aberchet/LMDZ_DATA/CORREL/
# sigma_land: 500
# sigma_sea: 1000
# dir: /home/inversion/msaunois/FLUXES/MCF/
# fic: sflx_MCF_scaled_%Y_LMDZ9696_phy.nc
# prescrconcs :
# parameters:
# OH:
# hresol : bands
# period : 1MS
# subperiod: 8D
# bands_lat : [-90, -45, 0, 45, 90]
# bands_lon : [-180, 180]
# err : 0.1
# calcstd : False
inicond :
CH4:
resol : global
err : 0.05
MCF:
resol : global
err : 0.1
parameters:
CH4:
hresol : bands
bands_lat : [-90.1, -60, -30, 0, 30, 60, 90.1]
bands_lon : [-180.1, -150, -120, -90, -60, -30,
0, 30, 60, 90, 120, 150, 180.1]
err : 0.05
calcstd : False
# MCF:
# hresol : bands
# bands_lat : [-90.1, -60, -30, 0, 30, 60, 90.1]
# bands_lon : [-180.1, -150, -120, -90, -60, -30,
# 0, 30, 60, 90, 120, 150, 180.1]
# err : 0.1
# calcstd : False
#hresol : global
#err : 0.1
#####################################################################
#####################################################################
#####################################################################
#####################################################################
# Domain definition
domain :
module :
plugin :
name : LMDZ
version : std
xmin : -180
......@@ -309,16 +280,20 @@ domain :
#####################################################################
#####################################################################
#####################################################################
#####################################################################
# How to build your observation vector and observation uncertainties if needed
# Also projects information from the observation to the model space
# - fic_obsvect: observation vector from previous simulations
obsvect:
module:
plugin:
name: standard
version: std
#fic_obsvect : /home/satellites13/msaunois/PYVAR_FWD/GCP_2018/run_00_18_inca/obsoperator/fwd_0000/monitor_selec1.nc
# fic_obsvect : /home/satellites13/aberchet/LMDZ_DATA/monit/monitor_pycif_test.nc
fic_obsvect : /home/chimereges/aberchet/satellites/monitor_201001.nc
dir_satellites : /home/chimereges/aberchet/satellites/
dump: True
wfunc : False
fic_statlev: /home/users/aberchet/PYVAR/msaunois/pyvar/ReadObs/stations_levels_LMDZ39lev.txt
......@@ -331,32 +306,33 @@ obsvect:
# Definition of meteo parameters
# This can be model dependent, please refer to corresponding documentation for details
meteo:
dirmeteo: /home/comdata1/PYVARLMDZ_files_96x96x39/MASS_FLUXES/TD/
plugin:
name: LMDZ
version: mass-fluxes
dirmeteo: /home/satellites1/fcheval/LMDZ5/
#####################################################################
#####################################################################
##################
# Parameters below are not yet included in PYVAR
# surface geopotential file
ficgeopot : null
# first guess path of the file if wanted
prevanal: null
# LMDz precomputed (1) or not (0)
precalc : 0
# perturb predefined prior/concentrations
perturbdir : null
# seed for random number generation
seed : 3
# diagnostics (0=none; 1=1D(time); 2=2D(lev,time); 3=4D(lat,lon,lev,time) )
diag : 0
#
#
## surface geopotential file
#ficgeopot : null
#
## first guess path of the file if wanted
#prevanal: null
## LMDz precomputed (1) or not (0)
#precalc : 0
## perturb predefined prior/concentrations
#perturbdir : null
## seed for random number generation
#seed : 3
## diagnostics (0=none; 1=1D(time); 2=2D(lev,time); 3=4D(lat,lon,lev,time) )
#diag : 0
#
#
#
#
#
......@@ -35,7 +35,7 @@ extensions = ['sphinx.ext.autodoc',
'sphinx.ext.coverage',
'sphinx.ext.mathjax',
'sphinx.ext.viewcode',
'sphinxcontrib.plantuml']
'sphinx.ext.graphviz']
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
......
#####################
Control vectors
#####################
.. toctree::
:maxdepth: 3
standard
###################################
Standard pyCIF control vector
###################################
.. role:: bash(code)
:language: bash
The control vector plugin includes sub-routines to compute operation
related to uncertainty matrices, and stores the control vector meta-data
and data themselves. Only a standard control vector including most
commonly used control vector shapes is implemented yet.
Configuration
-------------
The control vector is defined with the following parameters:
- :bash:`components`:
the different types of components in the control
vector; accepts: :bash:`fluxes`, :bash:`prescrconcs`, :bash:`prodloss3d` and :bash:`inicond`
- :bash:`fluxes`: fluxes to be optimized
- :bash:`species`: each species is defined through a paragraph with the following parameters:
- :bash:`hresol`: the horizontal resolution; should be one of:
- :bash:`hpixels`: for individual pixels
- :bash:`bands`: for zonal and meridional bands
- :bash:`regions`: for pre-defined regions
- :bash:`type`: (optional) the type of increments to deal with; should be one of:
- :bash:`scalar`: (default) increments are applied to the scaling factor of the prior
- :bash:`physical`: valid only with :bash:`component`=`fluxes` and :bash:`hresol`=`hpixels`
- :bash:`errtype`: (optional) the type of error; if not specified, a scalar is applied to the prior value; if :bash:`max` is given, computes the max of the neighboring cells (spatially and temporally)
- :bash:`err`: error as a proportion of prior fluxes
- :bash:`dir`: directory where data files are stored
- :bash:`fic`: file format of the data
- :bash:`hcorrelations`: (optional) information on horizontal
correlations if any
- :bash:`ficlsm`: path to the land-sea mask file
- :bash:`dump_hcorr`: (optional) dump correlation matrix if True;
default is True
- :bash:`dircorrel`: directory where to save the correlation matrix
- :bash:`sigma_land`: correlation distance over land in km
- :bash:`sigma_sea`: correlation distance over sea in km
- :bash:`tcorrelations` : (optional) information on temporal
correlations if any
- :bash:`sigma_t`: correlation period in hours
- :bash:`dump_tcorr`: (optional) dump correlation matrix if True;
default is True
- :bash:`dircorrel`: directory where to save the correlation matrix
File name format
----------------
.. role:: bash(code)
:language: bash
Various types of files are necessary to run pyCIF. pyCIF accepts generic
formats when file names are dependent on, e.g., the date of the
simulation. In this case, the generic date-dependent file name should be
given as a string compatible with the Python datetime function,
:bash:`strptime` as detailed
`here <https://docs.python.org/2/library/datetime.html#strftime-and-strptime-behavior>`__.
For instance, if your meteo file names are function of the current date,
the following syntax will be required in the Yaml file:
.. code-block:: yaml
mymeteo: /mymeteo_folder/mymeteo_%Y%m%d_%H%M.nc
pyCIF will automatically recognize a date format as YYYYMMDD_HHMM, i.e.
20140327_1212 corresponds to March 27 :sup:`th`, 2014
at 12:12.
Configuration
-------------
.. toctree::
:maxdepth: 2