Commit 3f1b90fc authored by Espen Sollum's avatar Espen Sollum
Browse files

Python 3 compatibility; various changes from master

parent 1f2c5559
......@@ -72,7 +72,6 @@ obsvect:
plugin:
name: standard
version: std
# fic_obsvect : /home/eso/repos/CIF/flextest/monitor_obsvect.nc
# file_obsvect : /home/eso/repos/CIF/flextest/monitor_obsvect.nc
dump: True
dump_type: nc
......@@ -91,7 +90,6 @@ measurements :
version: std
# File where to save data, if does not exist. Reads from there if exists
# fic_monitor : /home/eso/repos/CIF/flextest/monitor_allobs.nc
# file_monit : /home/eso/repos/CIF/flextest/monitor_allobs.nc
# file_monitor : /home/eso/repos/CIF/flextest/monitor_allobs.nc
dump_type : nc
......@@ -222,7 +220,6 @@ controlvect:
dir_netcdf : /home/eso/repos/CIF/flextest/
hresol : regions
inc_ocean : true
ficregions : /home/eso/FLEX_INV/TEST_OUTPUT/regions_ghg.nc
fileregions : /home/eso/FLEX_INV/TEST_OUTPUT/regions_ghg.nc
# errtype : max
errtype : avg
......@@ -236,9 +233,7 @@ controlvect:
# periodflux : 5D
period : 10D
dir : /home/eso/FLEX_INV/TEST_OUTPUT/FLUXES/GHG/
# fic : CH4_TOTAL_%Y_05x05.nc
file : CH4_TOTAL_%Y_05x05.nc
# fic : CH4_TOTAL_%Y_05x05.nc
file_glob : CH4_TOTAL_%Y_10x10.nc
varname_flx: emisch4
......@@ -258,7 +253,6 @@ controlvect:
hcorrelations :
# landsea: True
landsea: False
ficlsm : /home/eso/FLEX_INV/TEST_INPUT/lsm_0.5x0.5_VERIFY.nc
filelsm : /home/eso/FLEX_INV/TEST_INPUT/lsm_0.5x0.5_VERIFY.nc
dircorrel : /home/eso/repos/CIF/flextest/
dump_hcorr : True
......
from __future__ import absolute_import
from .plugins import *
from os.path import dirname, basename, isdir
import glob
modules = glob.glob(dirname(__file__) + "/*")
__all__ = [basename(f) for f in modules
if isdir(f) and not f.endswith('.py')]
import argparse
import pdb
import code
import traceback
from pycif.utils.check.error import handle_except
from pycif.utils.classes.setup import Setup
......@@ -29,13 +25,5 @@ def main():
Setup.run_simu(args)
# TODO:
###### POST PROCESS
############ PRODUCES OUTPUT OBS vs SIMU
############ PRODUCES OUTPUT EMISSIONS
############ PLOT BASIC FIGURES
# These functionalities should probably be included as a running mode
if __name__ == '__main__':
main()
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""config sub-module
This module handles the configuration of the pyCIF runs. It loads a Yaml
configuration file including all paramaters necessary to setting up pyCIF.
"""
import yaml
import os
import pycif.utils.check as check
import pycif.utils.dates as dates
def load(def_file):
"""Generates a dictionary including all pyCIF parameters
Args:
def_file (string) : Path to the definition file
Handles both absolute and relative paths
Returns:
config_dict (dictionary): Dictionary populated with all pyCIF parameters
"""
fic = os.path.abspath(def_file)
try:
with open(fic, 'r') as f:
config_dict = yaml.load(f)
config_dict['def_file'] = fic
# Converting dates to datetime if necessary
config_dict['datei'] = dates.date2datetime(config_dict['datei'])
config_dict['datef'] = dates.date2datetime(config_dict['datef'])
return config_dict
except IOError as e:
print "Couldn't find config file", fic
print "Please check directories"
raise e
except yaml.scanner.ScannerError as e:
print "Error in the syntax of config file", fic
raise e
def config_verbose(config_dict):
"""Prints out main input parameters for pyCIF
"""
verbose_txt = [
"pyCIF has been initialized with the following parameters:",
"Yaml configuration file: {}".format(config_dict['def_file']),
"Log file: {}".format(config_dict['logfile']),
"Start date: {}".format(config_dict['datei']),
"End date: {}".format(config_dict['datef']),
"Working directory: {}".format(config_dict['workdir']),
]
map(lambda v: check.verbose(v), verbose_txt)
......@@ -7,4 +7,4 @@ import glob
modules = glob.glob(dirname(__file__) + "/*")
__all__ = [basename(f) for f in modules
if isdir(f) and not f.endswith('.py')]
if isdir(f) and not f.endswith('.py') and 'pycache' not in f]
......@@ -2,6 +2,7 @@
"""Contains all recognized XXfluxXX formats.
Automatically loads all pre-defined models as sub-modules of pycif.models
"""
from __future__ import absolute_import
from os.path import dirname, basename, isdir
import glob
......@@ -11,4 +12,4 @@ modules = glob.glob(dirname(__file__) + "/*")
__all__ = [basename(f) for f in modules
if isdir(f) and not f.endswith('.py')]
import register
from . import register
from __future__ import absolute_import
import shutil
from distutils.dir_util import copy_tree
from read_chemistry import read_chemicalscheme
from make_chemistry import create_chemicalscheme
from .read_chemistry import read_chemicalscheme
from .make_chemistry import create_chemicalscheme
from pycif.utils.check import verbose
from pycif.utils.path import init_dir
......
......@@ -36,44 +36,44 @@ def create_chemicalscheme(self):
# List of files
finf = '{}/chemical_scheme.nml'.format(dirchem_ref)
ficr = '{}/REACTIONS.{}'.format(dirchem_ref, mecachim)
ficps = '{}/PRESCRIBED_SPECIES.{}'.format(dirchem_ref, mecachim)
ficpl = '{}/PRODLOSS_SPECIES.{}'.format(dirchem_ref, mecachim)
ficdp = '{}/DEPO_SPECIES.{}'.format(dirchem_ref, mecachim)
fica = '{}/ANTHROPIC.{}'.format(dirchem_ref, mecachim)
ficb = '{}/BIOGENIC.{}'.format(dirchem_ref, mecachim)
filer = '{}/REACTIONS.{}'.format(dirchem_ref, mecachim)
fileps = '{}/PRESCRIBED_SPECIES.{}'.format(dirchem_ref, mecachim)
filepl = '{}/PRODLOSS_SPECIES.{}'.format(dirchem_ref, mecachim)
filedp = '{}/DEPO_SPECIES.{}'.format(dirchem_ref, mecachim)
filea = '{}/ANTHROPIC.{}'.format(dirchem_ref, mecachim)
fileb = '{}/BIOGENIC.{}'.format(dirchem_ref, mecachim)
mandatory_files = [ficr, ficps, ficpl, ficdp]
mandatory_files = [filer, fileps, filepl, filedp]
create_mandchem(self, mandatory_files)
fics = '{}/STOICHIOMETRY.{}'.format(dirchem_ref, mecachim)
ficc = '{}/CHEMISTRY.{}'.format(dirchem_ref, mecachim)
ficrr = '{}/REACTION_RATES.{}'.format(dirchem_ref, mecachim)
ficj = '{}/PHOTO_RATES.{}'.format(dirchem_ref, mecachim)
ficf = '{}/FAMILIES.{}'.format(dirchem_ref, mecachim)
ficals = '{}/ALL_SPECIES.{}'.format(dirchem_ref, mecachim)
ficas = '{}/ACTIVE_SPECIES.{}'.format(dirchem_ref, mecachim)
files = '{}/STOICHIOMETRY.{}'.format(dirchem_ref, mecachim)
filec = '{}/CHEMISTRY.{}'.format(dirchem_ref, mecachim)
filerr = '{}/REACTION_RATES.{}'.format(dirchem_ref, mecachim)
filej = '{}/PHOTO_RATES.{}'.format(dirchem_ref, mecachim)
filef = '{}/FAMILIES.{}'.format(dirchem_ref, mecachim)
fileals = '{}/ALL_SPECIES.{}'.format(dirchem_ref, mecachim)
fileas = '{}/ACTIVE_SPECIES.{}'.format(dirchem_ref, mecachim)
cond1 = getsize(ficr) > 0
cond1 = getsize(filer) > 0
if not cond1:
check.verbose('Chemical mechanism not defined')
raise OSError('Chemical mechanism not defined')
nallqmax, nphoto_rates = create_optchem(self, ficr, ficps)
nallqmax, nphoto_rates = create_optchem(self, filer, fileps)
# Create chemical_scheme.nml namelist
os.system('echo "&args" > {}'.format(finf))
os.system('echo "fnacspec = \'{}\'" >> {}'.format(ficas, finf))
os.system('echo "fnallspec = \'{}\'" >> {}'.format(ficals, finf))
os.system('echo "fnprescr = \'{}\'" >> {}'.format(ficps, finf))
os.system('echo "fnprodl = \'{}\'" >> {}'.format(ficpl, finf))
os.system('echo "fndep = \'{}\'" >> {}'.format(ficdp, finf))
os.system('echo "fnchem = \'{}\'" >> {}'.format(ficc, finf))
os.system('echo "fnstoi = \'{}\'" >> {}'.format(fics, finf))
os.system('echo "fnrates = \'{}\'" >> {}'.format(ficrr, finf))
os.system('echo "fnjrates = \'{}\'" >> {}'.format(ficj, finf))
os.system('echo "fnfamilies = \'{}\'" >> {}'.format(ficf, finf))
os.system('echo "fnacspec = \'{}\'" >> {}'.format(fileas, finf))
os.system('echo "fnallspec = \'{}\'" >> {}'.format(fileals, finf))
os.system('echo "fnprescr = \'{}\'" >> {}'.format(fileps, finf))
os.system('echo "fnprodl = \'{}\'" >> {}'.format(filepl, finf))
os.system('echo "fndep = \'{}\'" >> {}'.format(filedp, finf))
os.system('echo "fnchem = \'{}\'" >> {}'.format(filec, finf))
os.system('echo "fnstoi = \'{}\'" >> {}'.format(files, finf))
os.system('echo "fnrates = \'{}\'" >> {}'.format(filerr, finf))
os.system('echo "fnjrates = \'{}\'" >> {}'.format(filej, finf))
os.system('echo "fnfamilies = \'{}\'" >> {}'.format(filef, finf))
os.system('echo "iqmax = {}" >> {}'.format(self.nspecies, finf))
os.system('echo "iallqmax = {}" >> {}'.format(nallqmax, finf))
os.system(
......
......@@ -27,30 +27,30 @@ def read_chemicalscheme(chemistry,
dirchem_ref = '{}/chemical_scheme/{}/'.format(workdir, chemistry.schemeid)
# ACTIVE SPECIES
fic_chem = '{}/ACTIVE_SPECIES.{}'.format(dirchem_ref,
file_chem = '{}/ACTIVE_SPECIES.{}'.format(dirchem_ref,
chemistry.schemeid)
chemistry.species = pd.read_csv(fic_chem, header=None, sep=' ',
chemistry.species = pd.read_csv(file_chem, header=None, sep=' ',
usecols=[0, 1], names=['ID', 'name'])
chemistry.nspec = len(chemistry.species)
# ANTHROPIC
fic_chem = '{}/ANTHROPIC.{}'.format(dirchem_ref,
file_chem = '{}/ANTHROPIC.{}'.format(dirchem_ref,
chemistry.schemeid)
chemistry.anthro_species = pd.read_csv(fic_chem, header=None, sep=' ',
chemistry.anthro_species = pd.read_csv(file_chem, header=None, sep=' ',
usecols=[0, 1], names=['ID', 'name'])
chemistry.nemisa = len(chemistry.anthro_species)
# BIOGENIC
fic_chem = '{}/BIOGENIC.{}'.format(dirchem_ref,
file_chem = '{}/BIOGENIC.{}'.format(dirchem_ref,
chemistry.schemeid)
chemistry.bio_species = pd.read_csv(fic_chem, header=None, sep=' ',
chemistry.bio_species = pd.read_csv(file_chem, header=None, sep=' ',
usecols=[0, 1], names=['ID', 'name'])
chemistry.nemisb = len(chemistry.bio_species)
# DEPO_SPEC
fic_chem = '{}/DEPO_SPEC.{}'.format(dirchem_ref,
file_chem = '{}/DEPO_SPEC.{}'.format(dirchem_ref,
chemistry.schemeid)
chemistry.dep_species = pd.read_csv(fic_chem, header=None, sep=' ',
chemistry.dep_species = pd.read_csv(file_chem, header=None, sep=' ',
usecols=[0, 1], names=['ID', 'name'])
chemistry.ndep = len(chemistry.dep_species)
......
from builtins import str
from builtins import range
import os
from os.path import exists, getsize
import pandas as pd
......@@ -35,7 +37,7 @@ def create_mandchem(chemistry, mandatory_files):
f.write(attr + '\n')
def create_optchem(chemistry, ficr, ficps):
def create_optchem(chemistry, filer, fileps):
"""
Create the chemistry files ALL_SPECIES, CHEMISTRY, REACTION_RATES,
STOICHIOMETRY and ACTIVE_SPECIES
......@@ -45,8 +47,8 @@ def create_optchem(chemistry, ficr, ficps):
Args:
chemistry (pycif.utils.classes.chemistries.Chemistry): chemical scheme
ficr (str) : path to the file with reactions
ficps (str): path to the file with prescribed species
filer (str) : path to the file with reactions
fileps (str): path to the file with prescribed species
"""
......@@ -54,26 +56,26 @@ def create_optchem(chemistry, ficr, ficps):
dirchem_ref = chemistry.dirchem_ref
mecachim = chemistry.schemeid
fics = '{}/STOICHIOMETRY.{}'.format(dirchem_ref, mecachim)
ficc = '{}/CHEMISTRY.{}'.format(dirchem_ref, mecachim)
ficrr = '{}/REACTION_RATES.{}'.format(dirchem_ref, mecachim)
ficj = '{}/PHOTO_RATES.{}'.format(dirchem_ref, mecachim)
ficf = '{}/FAMILIES.{}'.format(dirchem_ref, mecachim)
ficals = '{}/ALL_SPECIES.{}'.format(dirchem_ref, mecachim)
ficas = '{}/ACTIVE_SPECIES.{}'.format(dirchem_ref, mecachim)
ficanth = '{}/ANTHROPIC.{}'.format(dirchem_ref, mecachim)
ficbio = '{}/BIOGENIC.{}'.format(dirchem_ref, mecachim)
files = '{}/STOICHIOMETRY.{}'.format(dirchem_ref, mecachim)
filec = '{}/CHEMISTRY.{}'.format(dirchem_ref, mecachim)
filerr = '{}/REACTION_RATES.{}'.format(dirchem_ref, mecachim)
filej = '{}/PHOTO_RATES.{}'.format(dirchem_ref, mecachim)
filef = '{}/FAMILIES.{}'.format(dirchem_ref, mecachim)
fileals = '{}/ALL_SPECIES.{}'.format(dirchem_ref, mecachim)
fileas = '{}/ACTIVE_SPECIES.{}'.format(dirchem_ref, mecachim)
fileanth = '{}/ANTHROPIC.{}'.format(dirchem_ref, mecachim)
filebio = '{}/BIOGENIC.{}'.format(dirchem_ref, mecachim)
os.system('rm -f {} {} {} {}'.format(fics, ficc, ficrr, ficf))
os.system('rm -f {} {} {} {}'.format(files, filec, filerr, filef))
# Read reactions
df_reac = pd.read_csv(ficr, sep='\s*', index_col=False, header=None,
df_reac = pd.read_csv(filer, sep='\s*', index_col=False, header=None,
comment='#', engine="python")
# Read prescribed species
prescribed_species = np.array([])
if exists(ficps) and getsize(ficps) > 0:
df_prescr = pd.read_csv(ficps, sep='\s*', index_col=False, header=None,
if exists(fileps) and getsize(fileps) > 0:
df_prescr = pd.read_csv(fileps, sep='\s*', index_col=False, header=None,
comment='#', engine="python")
prescribed_species = df_prescr[0]
......@@ -115,7 +117,7 @@ def create_optchem(chemistry, ficr, ficps):
# Create reactions
reactions = df_reac[1].values
reactions_rates, nphoto_rates = read_react(reactions, ficj)
reactions_rates, nphoto_rates = read_react(reactions, filej)
reactions_rates = pd.DataFrame(reactions_rates)
# Active species
......@@ -143,18 +145,18 @@ def create_optchem(chemistry, ficr, ficps):
output_species = pd.DataFrame(output_species)
# Create files to create
stoichiometry.to_csv(fics, sep=' ', header=False, index=False)
file_chemistry.to_csv(ficc, sep=' ', header=False, index=False)
reactions_rates.to_csv(ficrr, sep=' ', header=False, index=False)
all_species.to_csv(ficals, sep=' ', header=False, index=False)
output_species.to_csv(ficas, sep=' ', header=False, index=False)
output_species.to_csv(ficanth, sep=' ', header=False, index=False)
output_species.to_csv(ficbio, sep=' ', header=False, index=False)
stoichiometry.to_csv(files, sep=' ', header=False, index=False)
file_chemistry.to_csv(filec, sep=' ', header=False, index=False)
reactions_rates.to_csv(filerr, sep=' ', header=False, index=False)
all_species.to_csv(fileals, sep=' ', header=False, index=False)
output_species.to_csv(fileas, sep=' ', header=False, index=False)
output_species.to_csv(fileanth, sep=' ', header=False, index=False)
output_species.to_csv(filebio, sep=' ', header=False, index=False)
return all_species.shape[0], nphoto_rates
def read_react(reactions, ficj):
def read_react(reactions, filej):
"""
Parse the types of reactions in the REACTIONS file and extract the data
"""
......@@ -206,6 +208,6 @@ def read_react(reactions, ficj):
nphoto_rates += 1
photo_rates = pd.DataFrame(photo_rates[1:])
photo_rates.to_csv(ficj, sep=' ', header=False, index=False)
photo_rates.to_csv(filej, sep=' ', header=False, index=False)
return reactions_rates, nphoto_rates
......@@ -2,6 +2,7 @@
"""Contains all recognized state vectors.
Automatically loads all pre-defined models as sub-modules of pycif.models
"""
from __future__ import absolute_import
from os.path import dirname, basename, isdir
import glob
......@@ -11,4 +12,4 @@ modules = glob.glob(dirname(__file__) + "/*")
__all__ = [basename(f) for f in modules
if isdir(f) and not f.endswith('.py')]
import register
from . import register
......@@ -66,10 +66,6 @@ def ini_data(plugin, **kwargs):
# Gets read from model if not already defined
try:
tracer.read(*list(range(4)))
# tracer.read(*range(4))
# except PluginError:
# except TypeError:
# tracer.read(*list(range(4)))
except PluginError:
if comp in ['fluxes', 'inicond', 'latcond', 'topcond']:
tracer.read = getattr(plugin.model, comp).read
......
......@@ -18,7 +18,7 @@ def reindex(array, var, levels={}, method='ffill'):
for lev in levels:
loc_levs = list(set(df_levs) - {lev})
lev_data = dataframe.unstack(level=loc_levs)\
.reindex(pd.Index(levels[lev], name=lev), method=method) \
.reindex(pd.Index(levels[lev].data, name=lev), method=method) \
.stack(level=loc_levs)
dataframe = lev_data
......
from builtins import zip
import numpy as np
import xarray as xr
......
......@@ -2,6 +2,7 @@
"""Contains all recognized flux formats.
Automatically loads all pre-defined models as sub-modules of pycif.models
"""
from __future__ import absolute_import
from os.path import dirname, basename, isdir
import glob
......@@ -11,4 +12,4 @@ modules = glob.glob(dirname(__file__) + "/*")
__all__ = [basename(f) for f in modules
if isdir(f) and not f.endswith('.py')]
import register
from . import register
from read_domain import read_grid
from __future__ import absolute_import
from .read_domain import read_grid
......@@ -9,7 +9,7 @@ def read_grid(domain,
Args:
domain (dictionary): dictionary defining the domain. Should include
ficgrid to be able to read the grid from a file
filegrid to be able to read the grid from a file
Return:
Grid dictionary with meshgrids for center lon/lat and corner lon/lat
......@@ -33,17 +33,17 @@ def read_grid(domain,
domain.nlon = nzo
# Reading lat/lon and latc/lonc
fic_hcoord = '{}/HCOORD/COORD_{}'.format(domain.repgrid,
file_hcoord = '{}/HCOORD/COORD_{}'.format(domain.repgrid,
domain.domid)
data = np.genfromtxt(fic_hcoord)
lon = data[:, 0].reshape(nzo, nme, order='F')
lat = data[:, 1].reshape(nzo, nme, order='F')
data = np.genfromtxt(file_hcoord)
lon = data[:, 0].reshape(nme, nzo)
lat = data[:, 1].reshape(nme, nzo)
fic_hcoord = '{}/HCOORD/COORDcorner_{}'.format(domain.repgrid,
file_hcoord = '{}/HCOORD/COORDcorner_{}'.format(domain.repgrid,
domain.domid)
data = np.genfromtxt(fic_hcoord)
lonc = data[:, 0].reshape(nzo + 1, nme + 1, order='F')
latc = data[:, 1].reshape(nzo + 1, nme + 1, order='F')
data = np.genfromtxt(file_hcoord)
lonc = data[:, 0].reshape(nme + 1, nzo + 1)
latc = data[:, 1].reshape(nme + 1, nzo + 1)
# Putting the data into the domain
domain.zlonc = lonc
......@@ -54,8 +54,9 @@ def read_grid(domain,
.format(domain.repgrid, domain.domid)
# Reading vertical coordinates
fic_vcoord = '{}/VCOORD/VCOORD_{}_{}_{}'\
file_vcoord = '{}/VCOORD/VCOORD_{}_{}_{}'\
.format(domain.repgrid, domain.nlev, domain.p1, domain.pmax)
data = np.genfromtxt(fic_vcoord)
data = np.genfromtxt(file_vcoord)
domain.ap = data[:, 0]
domain.bp = data[:, 1]
from create_domain import create_domain
from read_domain import read_grid
from __future__ import absolute_import
from .create_domain import create_domain
from .read_domain import read_grid
from __future__ import division
from past.utils import old_div
import numpy as np
import pycif.utils.check as check
......@@ -8,7 +10,7 @@ def read_grid(domain,
Args:
domain (Plugin): dictionary defining the domain. Should include
ficgrid to be able to read the grid from a file
filegrid to be able to read the grid from a file
Return:
Grid dictionary with meshgrids for center lon/lat and corner lon/lat
......@@ -16,20 +18,20 @@ def read_grid(domain,
Notes: Coordinates are in meters from a reference point
"""
# Tries open ficlon, ficlat
# Tries open filelon, filelat
try:
zlon = np.loadtxt(domain.ficlon)
zlat = np.loadtxt(domain.ficlat)
zlon = np.loadtxt(domain.filelon)
zlat = np.loadtxt(domain.filelat)
nlon = zlon.size
nlat = zlat.size
# Corner coordinates
dlon = np.ptp(zlon) / (nlon - 1) / 2.
dlon = old_div(np.ptp(zlon), (nlon - 1) / 2.)
zlonc = zlon - dlon
zlonc = np.append(zlonc, zlonc[-1] + 2 * dlon)
dlat = np.ptp(zlat) / (nlat - 1) / 2.
dlat = old_div(np.ptp(zlat), (nlat - 1) / 2.)
zlatc = zlat - dlat
zlatc = np.append(zlatc, zlatc[-1] + 2 * dlat)
......@@ -50,3 +52,9 @@ def read_grid(domain,
"Make them from given coordinates")
domain.create_domain()
# Compute areas in m2
domain.areas = np.diff(domain.zlatc, axis=1)[:-1] \
* np.diff(domain.zlonc, axis=0)[:, :-1]
# Projection not as GPS
domain.projection = 'xy'
from create_domain import create_domain
from read_domain import read_grid
from __future__ import absolute_import
from .create_domain import create_domain
from .read_domain import read_grid
domain_flexpart = True
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment