Commit c6aca4c8 authored by Antoine Berchet's avatar Antoine Berchet
Browse files

Merge branch 'new_classes' into 'devel'

Grouping flux/meteo/field into datatream; cleaning plurals in class names;...

See merge request satinv/cif!165
parents 1762bd0a c904fd9c
......@@ -172,20 +172,27 @@ article:
image:
name: pycif/pycif-ubuntu:0.1
entrypoint: [""]
before_script:
# - apt-get update
- pip freeze
# before_script:
## - apt-get update
# - pip freeze
script:
- tox -e py38 -e coverage -- -m "(dummy and article and inversion and not adjtltest and not uncertainties) or (fwd and ref_config)"
after_script:
- mkdir -p coverage
- xmlstarlet sel -t -v "//coverage/@line-rate" reports/coverage.xml > coverage/.current_coverage
- calc() { awk "BEGIN{print $*}"; }
- percent_coverage=`cat coverage/.current_coverage`
- tot_coverage=`calc ${percent_coverage}*100`
- echo 'TOTAL COVERAGE:'" ${tot_coverage}%"
- mv coverage_raw/coverage/.coverage coverage_raw/.coverage.article
coverage: '/^TOTAL COVERAGE: ([0-9\.]+\%)$/'
- echo AAAAAAAAAAAAAAAAAAAAAAAAaaa
- echo ${CI_COMMIT_BRANCH}
- echo AAAAAAAAAAAAAAAAAAAAAAAAaaa
- echo ${CI_MERGE_REQUEST_TARGET_BRANCH_NAME}
- echo AAAAAAAAAAAAAAAAAAAAAAAAaaa
- if [ ${CI_COMMIT_BRANCH} == "new_classes" ]; then echo "TTTTTTTTTTTTTTTTTTTTTTTTT"; fi;
#- tox -e py38 -e coverage -- -m "(dummy and article and inversion and not adjtltest and not uncertainties) or (fwd and ref_config)"
# after_script:
# - mkdir -p coverage
# - xmlstarlet sel -t -v "//coverage/@line-rate" reports/coverage.xml > coverage/.current_coverage
# - calc() { awk "BEGIN{print $*}"; }
# - percent_coverage=`cat coverage/.current_coverage`
# - tot_coverage=`calc ${percent_coverage}*100`
# - echo 'TOTAL COVERAGE:'" ${tot_coverage}%"
# - mv coverage_raw/coverage/.coverage coverage_raw/.coverage.article
# coverage: '/^TOTAL COVERAGE: ([0-9\.]+\%)$/'
artifacts:
when: always
paths:
......@@ -195,8 +202,8 @@ article:
- coverage_raw
- examples_artifact
- figures_artifact
only:
- LSCE
# only:
# - LSCE
article_uncertainties:
stage: test
......@@ -297,6 +304,7 @@ tests_chimere:
# Run the tests for flexpart (include downloading data)
tests_flexpart:
stage: test
retry: 2
image:
name: pycif/pycif-ubuntu:0.1
entrypoint: [""]
......
......@@ -318,7 +318,6 @@ def process_pycif_keywords(app, what, obj_name, obj, options, lines):
- default_values
- mandatory_values
"""
ref_lines = copy.deepcopy(lines)
# Adding bash highlight by default
......@@ -403,9 +402,30 @@ def process_pycif_keywords(app, what, obj_name, obj, options, lines):
preftree = key_req.get("preftree", "")
empty = key_req.get("empty", False)
name = key_req.get("name", None)
version = key_req.get("version", "")
plg_type = Plugin.plugin_types[key_req.get("type", req)][1]
plg_path = Plugin.plugin_types[key_req.get("type", req)][0][1:]
version = key_req.get("version", None)
req_type = key_req.get("type", req)
req_subtype = key_req.get("subtype", "")
# Load required plugin to deal with types and sub-types
plg_req = Plugin.from_dict({
"plugin": {
"name": name,
"version": version,
"type": req_type,
"subtype": req_subtype
}
})
plg_req._load_plugin_type(req)
plg_type = \
Plugin.plugin_types[plg_req.plugin.type][1]
plg_path = \
Plugin.plugin_types[plg_req.plugin.type][0][1:]
plg_subtype = \
Plugin.plugin_subtypes[
plg_req.plugin.type][
plg_req.plugin.subtype][1:]
# String to dump
newplg = key_req.get("newplg", False)
towrite.extend((
" * - {}\n"
......@@ -534,7 +554,7 @@ def build_rst_from_plugins(app):
init_dir(plg_dir)
# Initialize index
towrite = [
towrite_overall_index = [
"##################",
"Plugins in pyCIF",
"##################",
......@@ -544,12 +564,7 @@ def build_rst_from_plugins(app):
"",
" ../plugin_description",
" ../dependencies"
] + [
" {}/index".format(Plugin.plugin_types[plg_type][0][1:])
for plg_type in Plugin.plugin_types
]
with open("{}/index.rst".format(plg_dir), "w") as f:
f.write("\n".join(towrite))
# Loop on all plugin types
for plg_type in Plugin.plugin_types:
......@@ -567,10 +582,32 @@ def build_rst_from_plugins(app):
# Loop over modules of this class
package_path = "pycif.plugins{}".format(Plugin.plugin_types[plg_type][0])
if pkgutil.importlib.util.find_spec(package_path) is None:
continue
# Update overall index
towrite_overall_index.append(
" {}/index".format(Plugin.plugin_types[plg_type][0][1:]))
# Loop over sub-types
import_package = pkgutil.importlib.import_module(package_path)
package_index = []
for mod in pkgutil.walk_packages(import_package.__path__,
prefix=import_package.__name__ + "."):
for subtype in Plugin.plugin_subtypes[plg_type]:
local_subpackage = "{}{}".format(
package_path,
Plugin.plugin_subtypes[plg_type][subtype])
import_subpackage = pkgutil.importlib.import_module(local_subpackage)
# Create directory
plg_subtype_dir = "{}/{}".format(
plg_type_dir,
Plugin.plugin_subtypes[plg_type][subtype][1:])
init_dir(plg_subtype_dir)
# Loop over modules in the sub-type
package_subindex = []
for mod in pkgutil.walk_packages(import_subpackage.__path__,
prefix=import_subpackage.__name__ + "."):
if not mod.ispkg:
continue
......@@ -582,7 +619,7 @@ def build_rst_from_plugins(app):
# Create corresponding rst file
file_name = "{}/{}.rst".format(
plg_type_dir, loc_mod.__name__.split(".")[-1])
plg_subtype_dir, loc_mod.__name__.split(".")[-1])
title = ":bash:`{}` / :bash:`{}`".format(
loc_mod._name, getattr(loc_mod, "_version", "std"))
......@@ -606,10 +643,47 @@ def build_rst_from_plugins(app):
f.write("\n".join(towrite))
# Append name for plugin type index
package_index.append(loc_mod.__name__.split(".")[-1])
package_subindex.append(loc_mod.__name__.split(".")[-1])
# Sort names
package_index.sort()
package_subindex.sort()
# Write the plugin type index
if subtype == "":
continue
title = list(subtype)
title[0] = title[0].upper()
title = "".join(title)
towrite = [
".. role:: bash(code)",
" :language: bash",
"",
"",
len(title) * "#",
title,
len(title) * "#",
""] + ([".. contents:: Contents", " :local:", ""]
if import_subpackage.__doc__ is not None else []) + [
"Available {}".format(title),
(len(title) + 11) * "=",
"",
"The following :bash:`{}` of sub-type {} "
"are implemented in pyCIF so far:".format(
Plugin.plugin_types[plg_type][0][1:],
subtype),
"",
".. toctree::",
"",
] + [
" {}".format(plg) for plg in package_subindex
] + (
import_subpackage.__doc__.split('\n')
if import_subpackage.__doc__ is not None
else []
)
with open("{}/index.rst".format(plg_subtype_dir), "w") as f:
f.write("\n".join(towrite))
# Write the plugin type index
title = list(Plugin.plugin_types[plg_type][0][1:])
......@@ -630,22 +704,45 @@ def build_rst_from_plugins(app):
] if import_package.__doc__ is not None else []) + [
"Available {}".format(title),
(len(title) + 11) * "=",
"",
""]
# If only one sub-type, just create an index of all available plugins
if len(Plugin.plugin_subtypes[plg_type]) == 1:
towrite.extend([
"The following :bash:`{}` are implemented in pyCIF so far:".format(
Plugin.plugin_types[plg_type][0][1:]),
"",
".. toctree::",
"",
] + [
" {}".format(plg) for plg in package_index
] + (
] + [" {}".format(plg) for plg in package_subindex])
# If sub-types create an index pointing to sub-types and plugins
else:
towrite.extend([
"The following sub-types and :bash:`{}` are implemented "
"in pyCIF so far:".format(
Plugin.plugin_types[plg_type][0][1:]),
"",
".. toctree::",
"",
] + [" {}/index".format(Plugin.plugin_subtypes[plg_type][subtype][1:])
for subtype in Plugin.plugin_subtypes[plg_type]
])
# Append overall type description
towrite.extend(
import_package.__doc__.split('\n')
if import_package.__doc__ is not None
else []
)
else [])
# Dump the string to the rst file
with open("{}/index.rst".format(plg_type_dir), "w") as f:
f.write("\n".join(towrite))
# Dump the overall index
with open("{}/index.rst".format(plg_dir), "w") as f:
f.write("\n".join(towrite_overall_index))
# Generate available list
s = StringIO()
Plugin.print_registered(print_rst=True, print_requirement=True, stream=s)
......
#############################
Developments around CHIMERE
############################
#############################
.. role:: bash(code)
:language: bash
......
......@@ -27,7 +27,7 @@ Example: for a CTM with emitted species,
.. code-block:: python
emis = {
("fluxes", s): dict_surface
("flux", s): dict_surface
for s in model.chemistry.emis_species.attributes
}
......
.. role:: bash(code)
:language: bash
.. currentmodule:: pycif.plugins.fields.bc_plugin_template
.. currentmodule:: pycif.plugins.datastreams.fields.bc_plugin_template
Run pycif with this yaml: the new plugin will simply perform what is in the template i.e. print some instructions on what you have to do where. The following codes must be developped in the places matching the instructions - and checked. To check that each new code works as intended, run the CIF with the yaml using the new plugin and with the same yaml but using a known plugin with print statements. The scripts have to be developped in this order:
......
......@@ -2,9 +2,6 @@
:language: bash
Have a yaml file ready with a simulation that works with known plugins.
For the :doc:`obsoperator</documentation/plugins/obsoperators/index>`,
choose the optional argument :bash:`onlyinit` so that only the inputs are computed
XXXX CHECK THIS OPTION ACTUALLY DOES THISXXXX, not the whole simulation.
.. code-block:: yaml
......
......@@ -47,7 +47,7 @@ XXXXXXX what about the input arguments? Ils demandent une partie dediee!?XXXXXXX
Template plugin for BCs
########################
.. automodule:: pycif.plugins.fields.bc_plugin_template
.. automodule:: pycif.plugins.datastreams.fields.bc_plugin_template
c) add the reference to the rst file in docs/source/documentation/plugins/fields/index.rst:
......
......@@ -5,17 +5,104 @@ How to add a new type of flux data to be processed by the CIF into a model's inp
.. role:: bash(code)
:language: bash
0. .. include:: ../newBCdata/knownplugin.rst
Pre-requisites
================
Before starting to implement a new flux plugin, you must have:
1. In directory :bash:`plugins/fluxes`, copy the directory containing the template for a flux plugin :bash:`flux_plugin_template` in the directory for your new plugin.
- a yaml file ready with a simulation that works with known plugins.
- a folder where the data you need to implement is stored
- basic information about the data you need to implement (licensing, format, etc.)
We help you below to navigate through different documentation pages to implement your plugin.
The main reference pages are :doc:`the datastream documentation page </documentation/plugins/datastreams/index>`
and :doc:`the flux template documentation page</documentation/plugins/datastreams/fluxes/flux_plugin_template>`.
Switch from working fluxes to the reference template
=====================================================
The :bash:`datavect` paragraph of your working yaml should look like that:
.. container:: toggle
.. container:: header
Show/Hide Code
.. code-block:: yaml
:linenos:
datavect:
plugin:
name: standard
version: std
components:
flux:
parameters:
CO2:
plugin:
name: CHIMERE
type: flux
version: AEMISSIONS
file_freq: 120H
dir: some_dir
file: some_file
1. follow the initial steps in :doc:`the flux template documentation page</documentation/plugins/datastreams/fluxes/flux_plugin_template>`
to initialize your new plugin and register it.
It includes copying the template folder to a new path and changing the variables
:bash:`_name`,:bash:`_fullname` and :bash:`_version` in the file :bash:`__init__.py`
2. update your Yaml to use the template flux (renamed with your preference). It should now look like that:
.. container:: toggle
.. container:: header
Show/Hide Code
.. code-block:: yaml
:linenos:
datavect:
plugin:
name: standard
version: std
components:
flux:
parameters:
CO2:
plugin:
name: your_new_name
type: flux
version: your_version
3. Test running again your test case. It should generate fluxes with random values
#####################################
0. .. include:: ../newBCdata/knownplugin.rst
1. In directory :bash:`plugins/fluxes`, copy the directory containing the template
for a flux plugin :bash:`flux_plugin_template` in the directory for your new plugin.
.. include:: ../newBCdata/register.rst
2. Modify the yaml file to use the new plugin: the minimum input arguments are :bash:`dir`, :bash:`file`, :bash:`varname` and :bash:`unit_conversion`. The default space and time interpolations will be applied (see XXXX doc sur premiere simu directe avec exmeple yaml quand mise a jourXXXXX).
2. Modify the yaml file to use the new plugin: the minimum input arguments are
:bash:`dir`, :bash:`file`, :bash:`varname` and :bash:`unit_conversion`.
The default space and time interpolations will be applied
(see XXXX doc sur premiere simu directe avec exmeple yaml quand mise a jourXXXXX).
.. code-block:: yaml
.. code-block:: yaml
components:
fluxes:
......@@ -29,9 +116,9 @@ How to add a new type of flux data to be processed by the CIF into a model's inp
unit_conversion:
scale: 1.
3. .. include:: ../newBCdata/devplugin.rst
3. .. include:: ../newBCdata/devplugin.rst
XXXXXXX what about the input arguements? Ils demandent une partie dediee!?XXXXXXXXXX
XXXXXXX what about the input arguements? Ils demandent une partie dediee!?XXXXXXXXXX
4. Document the new plugin:
......@@ -49,7 +136,7 @@ XXXXXXX what about the input arguements? Ils demandent une partie dediee!?XXXXXX
Template plugin for fluxes
###########################
.. automodule:: pycif.plugins.fluxes.flux_plugin_template
.. automodule:: pycif.plugins.datastreams.fluxes.flux_plugin_template
c) add the reference to the rst file in docs/source/documentation/plugins/fluxes/index.rst:
......
......@@ -26,16 +26,27 @@ You then need to create an empty file called :bash:`__init__.py`, so python inte
touch __init__.py
.. note::
Although the :bash:`__init__.py` is not strictly needed by the newest version of python, pycif fetches information
directly from this file, for initializing the plugin and automatically documenting it.
Therefore, please include it anyway
Registering your new :bash:`plugin` to pyCIF
--------------------------------------------
pyCIF attaches plugin functions as defined in the corresponding python module in :bash:`pycif/plugins/` automatically
from the yaml configuration file.
When the new plugin is created, it must be registered, so it can be called by other routines of pyCIF.
:bash:`plugins` in pyCIF are identified with:
- a name
- optional: a version (default: std)
When the new plugin is created, it must be registered, so it can be called by other routines of pyCIF.
This can be done by providing the name and (if relevant) version in :bash:`__init__.py` :
To register a new plugin, one must define the name and (if relevant) version in the file :bash:`__init__.py` :
.. code-block:: python
......@@ -50,6 +61,7 @@ You can check that your :bash:`plugin` is correctly registered by using the foll
from pycif.utils.classes.baseclass import Plugin
Plugin.print_registered()
Adding requirements to your :bash:`plugin`
------------------------------------------
......@@ -75,7 +87,7 @@ Below is an example of requirements for the :bash:`model` CHIMERE:
"empty": False,
"any": False,
},
"fluxes": {
"flux": {
"name": "CHIMERE",
"version": "AEMISSIONS",
"empty": True,
......
......@@ -253,7 +253,7 @@ _name = "standard"
requirements = {
"domain": {"any": True, "empty": False},
"model": {"any": True, "empty": False},
# "components": {"any": True, "empty": True, "type": "fields"},
# "components": {"any": True, "empty": True, "type": "field"},
# "obsvect": {
# "any": True,
# "empty": True,
......
"""
Description
============
The :bash:`datastream` Plugin type include interfaces to input data for pycif,
with the exception of observations.
It includes the sub-types :bash:`flux`, :bash:`meteo` and :bash:`field`.
It is used for the following purposes:
i) fetching relevant input files for direct use by, e.g, CTMs, only linking to the
original file
ii) reading relevant input files when data manipulation is required, for, e.g.,
defining the control vector, or auxiliary transformations, such as temporal
interpolation or horizontal regridding
iii) writing data from pycif to the corresponding format; this can either be used
when data from pycif needs to be read as input for a CTM, or for sharing data
from pycif with a known standard data format
Required parameters, dependencies and functions
===============================================
"""
\ No newline at end of file
......@@ -3,7 +3,7 @@ import os
import numpy as np
import pandas as pd
import datetime
from ....utils import path
from .....utils import path
def fetch(
......
......@@ -18,7 +18,7 @@ Returns:
import numpy as np
from ....utils.classes.setup import Setup
from .....utils.classes.setup import Setup
def get_domain(ref_dir, ref_file, input_dates, target_dir, tracer=None):
......
......@@ -24,7 +24,7 @@ import numpy as np
import xarray as xr
from netCDF4 import Dataset
from ....utils.netcdf import readnc
from .....utils.netcdf import readnc
def read(
......
......@@ -3,7 +3,7 @@ import os
import numpy as np
import pandas as pd
import datetime
from ....utils import path
from .....utils import path
def fetch(
......
......@@ -6,7 +6,7 @@ import xarray as xr
from netCDF4 import Dataset
from logging import debug
from ....utils.netcdf import readnc
from .....utils.netcdf import readnc
def read(
......
......@@ -5,7 +5,7 @@ import pandas as pd
from netCDF4 import Dataset
import xarray as xr
from ....utils.netcdf import save_nc
from .....utils.netcdf import save_nc
def write(self, name, lbc_file, data, mode="a", comp_type=None):
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment