Merge branch 'devel' into 'LSCE'

Grouping flux/meteo/field into datatream; cleaning plurals in class names;...

See merge request satinv/cif!167

.gitlab-ci.yml

@@ -176,7 +176,12 @@ article:
 #    - apt-get update
     - pip freeze
   script:
-    - tox -e py38 -e coverage -- -m "(dummy and article and inversion and not adjtltest and not uncertainties) or (fwd and ref_config)"
+    - |
+      if [ ${CI_COMMIT_BRANCH} == "LSCE" ]; then
+        tox -e py38 -e coverage -- -m "(dummy and article and inversion and not adjtltest and not uncertainties) or (fwd and ref_config) or (allsimulations)";
+      else
+        tox -e py38 -e coverage -- -m "(dummy and article and inversion and not adjtltest and not uncertainties and bands) or (fwd and ref_config)";
+      fi;
   after_script:
     - mkdir -p coverage
     - xmlstarlet sel -t -v "//coverage/@line-rate" reports/coverage.xml > coverage/.current_coverage
@@ -195,8 +200,6 @@ article:
       - coverage_raw
       - examples_artifact
       - figures_artifact
-  only:
-    - LSCE
 
 article_uncertainties:
   stage: test
@@ -207,7 +210,12 @@ article_uncertainties:
 #    - apt-get update
     - pip freeze
   script:
-    - tox -e py38 -e coverage -- -m "(dummy and article and inversion and not adjtltest and uncertainties) or (fwd and ref_config)"
+    - |
+      if [ ${CI_COMMIT_BRANCH} == "LSCE" ]; then
+        tox -e py38 -e coverage -- -m "(dummy and article and inversion and not adjtltest and uncertainties) or (fwd and ref_config) or (allsimulations)";
+      else
+        tox -e py38 -e coverage -- -m "(dummy and article and inversion and not adjtltest and uncertainties and bands) or (fwd and ref_config)";
+      fi;
   after_script:
     - mkdir -p coverage
     - xmlstarlet sel -t -v "//coverage/@line-rate" reports/coverage.xml > coverage/.current_coverage
@@ -226,8 +234,6 @@ article_uncertainties:
       - coverage_raw
       - examples_artifact
       - figures_artifact
-  only:
-    - LSCE
 
 # Run the tests for the dummy model
 tests_dummy:
@@ -297,6 +303,7 @@ tests_chimere:
 # Run the tests for flexpart (include downloading data)
 tests_flexpart:
   stage: test
+  retry: 2
   image:
     name: pycif/pycif-ubuntu:0.1
     entrypoint: [""]

bin/pycif_test.sh

@@ -16,10 +16,10 @@ export PYCIF_DATATEST=/home/chimereges/PYCIF_TEST_DATA/
 export PYCIF_PLATFORM=LSCE
 ###
 # select a subset of tests to run by using the tags ("mark")
-#mark="(dummy and article and inversion and not adjtltest and not uncertainties) or (fwd and ref_config)"
+mark="(dummy and article and inversion and not adjtltest and not uncertainties) or (fwd and ref_config)"
 #mark="(dummy and article and inversion and not adjtltest and uncertainties) or (fwd and ref_config)"
 #mark="(fwd and ref_config)"
-mark="test_in_ci and dummy"
+#mark="test_in_ci and dummy"
 #mark="test_in_ci and chimere"
 #mark="chimere and argfrsd and fwd"
 #mark="tm5 and test_in_ci and fwd"

docs/source/conf.py

@@ -318,7 +318,6 @@ def process_pycif_keywords(app, what, obj_name, obj, options, lines):
     - default_values
     - mandatory_values
     """
-    
     ref_lines = copy.deepcopy(lines)
     
     # Adding bash highlight by default
@@ -403,9 +402,30 @@ def process_pycif_keywords(app, what, obj_name, obj, options, lines):
             preftree = key_req.get("preftree", "")
             empty = key_req.get("empty", False)
             name = key_req.get("name", None)
-            version = key_req.get("version", "")
-            plg_type = Plugin.plugin_types[key_req.get("type", req)][1]
-            plg_path = Plugin.plugin_types[key_req.get("type", req)][0][1:]
+            version = key_req.get("version", None)
+            req_type = key_req.get("type", req)
+            req_subtype = key_req.get("subtype", "")
+
+            # Load required plugin to deal with types and sub-types
+            plg_req = Plugin.from_dict({
+                "plugin": {
+                    "name": name,
+                    "version": version,
+                    "type": req_type,
+                    "subtype": req_subtype
+                }
+            })
+            plg_req._load_plugin_type(req)
+            plg_type = \
+                Plugin.plugin_types[plg_req.plugin.type][1]
+            plg_path = \
+                Plugin.plugin_types[plg_req.plugin.type][0][1:]
+            plg_subtype = \
+                Plugin.plugin_subtypes[
+                    plg_req.plugin.type][
+                    plg_req.plugin.subtype][1:]
+            
+            # String to dump
             newplg = key_req.get("newplg", False)
             towrite.extend((
                 "   * - {}\n"
@@ -534,7 +554,7 @@ def build_rst_from_plugins(app):
     init_dir(plg_dir)
     
     # Initialize index
-    towrite = [
+    towrite_overall_index = [
         "##################",
         "Plugins in pyCIF",
         "##################",
@@ -544,12 +564,7 @@ def build_rst_from_plugins(app):
         "",
         "    ../plugin_description",
         "    ../dependencies"
-    ] + [
-        "    {}/index".format(Plugin.plugin_types[plg_type][0][1:])
-        for plg_type in Plugin.plugin_types
     ]
-    with open("{}/index.rst".format(plg_dir), "w") as f:
-        f.write("\n".join(towrite))
     
     # Loop on all plugin types
     for plg_type in Plugin.plugin_types:
@@ -559,7 +574,7 @@ def build_rst_from_plugins(app):
             Plugin.plugin_types[plg_type][0])
         class_module = pkgutil.importlib.import_module(class_path)
         local_class = getattr(class_module, class_name)
-        
+
         # Create directory
         plg_type_dir = "{}/{}".format(
             plg_dir, Plugin.plugin_types[plg_type][0][1:])
@@ -567,29 +582,79 @@ def build_rst_from_plugins(app):
         
         # Loop over modules of this class
         package_path = "pycif.plugins{}".format(Plugin.plugin_types[plg_type][0])
+    
+        if pkgutil.importlib.util.find_spec(package_path) is None:
+            continue
+        
+        # Update overall index
+        towrite_overall_index.append(
+            "    {}/index".format(Plugin.plugin_types[plg_type][0][1:]))
+        
+        # Loop over sub-types
         import_package = pkgutil.importlib.import_module(package_path)
-        package_index = []
-        for mod in pkgutil.walk_packages(import_package.__path__,
-                                         prefix=import_package.__name__ + "."):
-            if not mod.ispkg:
-                continue
+        for subtype in Plugin.plugin_subtypes[plg_type]:
+            local_subpackage = "{}{}".format(
+                package_path,
+                Plugin.plugin_subtypes[plg_type][subtype])
+            import_subpackage = pkgutil.importlib.import_module(local_subpackage)
+
+            # Create directory
+            plg_subtype_dir = "{}/{}".format(
+                plg_type_dir,
+                Plugin.plugin_subtypes[plg_type][subtype][1:])
+            init_dir(plg_subtype_dir)
             
-            loc_mod = pkgutil.importlib.import_module(mod.name)
-
-            # Register modules only when a name is given
-            if not hasattr(loc_mod, "_name"):
-                continue
+            # Loop over modules in the sub-type
+            package_subindex = []
+            for mod in pkgutil.walk_packages(import_subpackage.__path__,
+                                             prefix=import_subpackage.__name__ + "."):
+                if not mod.ispkg:
+                    continue
+
+                loc_mod = pkgutil.importlib.import_module(mod.name)
+    
+                # Register modules only when a name is given
+                if not hasattr(loc_mod, "_name"):
+                    continue
+                    
+                # Create corresponding rst file
+                file_name = "{}/{}.rst".format(
+                    plg_subtype_dir, loc_mod.__name__.split(".")[-1])
                 
-            # Create corresponding rst file
-            file_name = "{}/{}.rst".format(
-                plg_type_dir, loc_mod.__name__.split(".")[-1])
-            
-            title = ":bash:`{}` / :bash:`{}`".format(
-                loc_mod._name, getattr(loc_mod, "_version", "std"))
+                title = ":bash:`{}` / :bash:`{}`".format(
+                    loc_mod._name, getattr(loc_mod, "_version", "std"))
+                
+                if hasattr(loc_mod, "_fullname"):
+                    title = "{} ({})".format(loc_mod._fullname, title)
+                    
+                towrite = [
+                    ".. role:: bash(code)",
+                    "   :language: bash",
+                    "",
+                    "",
+                    len(title) * "#",
+                    title,
+                    len(title) * "#",
+                    "",
+                    ".. automodule:: {}".format(loc_mod.__name__)
+                ]
+                
+                with open(file_name, "w") as f:
+                    f.write("\n".join(towrite))
+                
+                # Append name for plugin type index
+                package_subindex.append(loc_mod.__name__.split(".")[-1])
+        
+            # Sort names
+            package_subindex.sort()
             
-            if hasattr(loc_mod, "_fullname"):
-                title = "{} ({})".format(loc_mod._fullname, title)
+            # Write the plugin type index
+            if subtype == "":
+                continue
                 
+            title = list(subtype)
+            title[0] = title[0].upper()
+            title = "".join(title)
             towrite = [
                 ".. role:: bash(code)",
                 "   :language: bash",
@@ -598,54 +663,86 @@ def build_rst_from_plugins(app):
                 len(title) * "#",
                 title,
                 len(title) * "#",
+                ""] + ([".. contents:: Contents", "    :local:", ""]
+                       if import_subpackage.__doc__ is not None else []) + [
+                "Available {}".format(title),
+                (len(title) + 11) * "=",
                 "",
-                ".. automodule:: {}".format(loc_mod.__name__)
-            ]
-            
-            with open(file_name, "w") as f:
+                "The following :bash:`{}` of sub-type {} "
+                "are implemented in pyCIF so far:".format(
+                    Plugin.plugin_types[plg_type][0][1:],
+                    subtype),
+                "",
+                ".. toctree::",
+                "",
+            ] + [
+                "  {}".format(plg) for plg in package_subindex
+            ] + (
+                import_subpackage.__doc__.split('\n')
+                if import_subpackage.__doc__ is not None
+                else []
+            )
+            with open("{}/index.rst".format(plg_subtype_dir), "w") as f:
                 f.write("\n".join(towrite))
-            
-            # Append name for plugin type index
-            package_index.append(loc_mod.__name__.split(".")[-1])
-        
-        # Sort names
-        package_index.sort()
-        
+
         # Write the plugin type index
         title = list(Plugin.plugin_types[plg_type][0][1:])
         title[0] = title[0].upper()
         title = "".join(title) + " (:bash:`{}`)".format(plg_type)
         towrite = [
-            ".. role:: bash(code)",
-            "   :language: bash",
-            "",
-            "",
-            len(title) * "#",
-            title,
-            len(title) * "#",
-            ""] + ([
-            ".. contents:: Contents",
-            "    :local:",
-            ""
-            ] if import_package.__doc__ is not None else []) + [
-            "Available {}".format(title),
-            (len(title) + 11) * "=",
-            "",
-            "The following :bash:`{}` are implemented in pyCIF so far:".format(
-                Plugin.plugin_types[plg_type][0][1:]),
-            "",
-            ".. toctree::",
-            "",
-        ] + [
-            "  {}".format(plg) for plg in package_index
-        ] + (
+                      ".. role:: bash(code)",
+                      "   :language: bash",
+                      "",
+                      "",
+                      len(title) * "#",
+                      title,
+                      len(title) * "#",
+                      ""] + ([
+                                 ".. contents:: Contents",
+                                 "    :local:",
+                                 ""
+                             ] if import_package.__doc__ is not None else []) + [
+                      "Available {}".format(title),
+                      (len(title) + 11) * "=",
+                      ""]
+
+        # If only one sub-type, just create an index of all available plugins
+        if len(Plugin.plugin_subtypes[plg_type]) == 1:
+            towrite.extend([
+                "The following :bash:`{}` are implemented in pyCIF so far:".format(
+                    Plugin.plugin_types[plg_type][0][1:]),
+                "",
+                ".. toctree::",
+                "",
+            ] + ["  {}".format(plg) for plg in package_subindex])
+        
+        # If sub-types create an index pointing to sub-types and plugins
+        else:
+            towrite.extend([
+                "The following sub-types and :bash:`{}` are implemented "
+                "in pyCIF so far:".format(
+                    Plugin.plugin_types[plg_type][0][1:]),
+                "",
+                ".. toctree::",
+                "",
+            ] + ["  {}/index".format(Plugin.plugin_subtypes[plg_type][subtype][1:])
+                for subtype in Plugin.plugin_subtypes[plg_type]
+            ])
+
+        # Append overall type description
+        towrite.extend(
             import_package.__doc__.split('\n')
             if import_package.__doc__ is not None
-            else []
-        )
+            else [])
+
+        # Dump the string to the rst file
         with open("{}/index.rst".format(plg_type_dir), "w") as f:
             f.write("\n".join(towrite))
-
+    
+    # Dump the overall index
+    with open("{}/index.rst".format(plg_dir), "w") as f:
+        f.write("\n".join(towrite_overall_index))
+    
     # Generate available list
     s = StringIO()
     Plugin.print_registered(print_rst=True, print_requirement=True, stream=s)

docs/source/devtutos/chimere/index.rst

 #############################
 Developments around CHIMERE
-############################
+#############################
 
 .. role:: bash(code)
    :language: bash

docs/source/devtutos/miscellanea.rst

@@ -27,7 +27,7 @@ Example: for a CTM with emitted species,
   .. code-block:: python
 
      emis = {
-        ("fluxes", s): dict_surface
+        ("flux", s): dict_surface
         for s in model.chemistry.emis_species.attributes
       }
 

docs/source/devtutos/newBCdata/devplugin.rst

 .. role:: bash(code)
    :language: bash
 
-.. currentmodule:: pycif.plugins.fields.bc_plugin_template
+.. currentmodule:: pycif.plugins.datastreams.fields.bc_plugin_template
 
 Run pycif with this yaml: the new plugin will simply perform what is in the template i.e. print some instructions on what you have to do where. The following codes must be developped in the places matching the instructions - and checked. To check that each new code works as intended, run the CIF with the yaml using the new plugin and with the same yaml but using a known plugin with print statements. The scripts have to be developped in this order:
 

docs/source/devtutos/newBCdata/knownplugin.rst

@@ -2,9 +2,6 @@
    :language: bash
 
 Have a yaml file ready with a simulation that works with known plugins.
-For the :doc:`obsoperator</documentation/plugins/obsoperators/index>`,
-choose the optional argument :bash:`onlyinit` so that only the inputs are computed
-XXXX CHECK THIS OPTION ACTUALLY DOES THISXXXX, not the whole simulation.
 
 .. code-block:: yaml
     

docs/source/devtutos/newBCdata/newBCdata.rst

@@ -47,7 +47,7 @@ XXXXXXX what about the input arguments? Ils demandent une partie dediee!?XXXXXXX
        Template plugin for BCs
        ########################
 
-       .. automodule:: pycif.plugins.fields.bc_plugin_template
+       .. automodule:: pycif.plugins.datastreams.fields.bc_plugin_template
 
    c) add the reference to the rst file in docs/source/documentation/plugins/fields/index.rst:
 

docs/source/devtutos/newfluxdata/newfluxdata.rst

@@ -5,33 +5,142 @@ How to add a new type of flux data to be processed by the CIF into a model's inp
 .. role:: bash(code)
    :language: bash
 
-0. .. include:: ../newBCdata/knownplugin.rst
+Pre-requisites
+================
 
+Before starting to implement a new flux plugin, you must have:
 
-1. In directory :bash:`plugins/fluxes`, copy the directory containing the template for a flux plugin :bash:`flux_plugin_template` in the directory for your new plugin. 
+    - a yaml file ready with a simulation that works with known plugins.
+    - a folder where the data you need to implement is stored
+    - basic information about the data you need to implement (licensing, format, etc.)
 
-   .. include:: ../newBCdata/register.rst
+We help you below to navigate through different documentation pages to implement your plugin.
+The main reference pages are :doc:`the datastream documentation page </documentation/plugins/datastreams/index>`
+and :doc:`the flux template documentation page</documentation/plugins/datastreams/fluxes/flux_plugin_template>`.
 
+Switch from working fluxes to the reference template
+=====================================================
 
-2. Modify the yaml file to use the new plugin: the minimum input arguments are :bash:`dir`, :bash:`file`, :bash:`varname` and :bash:`unit_conversion`. The default space and time interpolations will be applied (see XXXX doc sur premiere simu directe avec exmeple yaml quand mise a jourXXXXX). 
+The :bash:`datavect` paragraph of your working yaml should look like that:
 
-.. code-block:: yaml
+.. container:: toggle
 
- components:
-    fluxes:
+  .. container:: header
+
+     Show/Hide Code
+
+  .. code-block:: yaml
+    :linenos:
+
+    datavect:
       plugin:
-         name: fluxes
-         version: template
-         type: fluxes
-      dir: dir_with_original_files/
-      file: file_with_new_fluxes_to_use_as_inputs
-      varname: NAMEORIG
-      unit_conversion:
-            scale: 1.
+        name: standard
+        version: std
+      components:
+        flux:
+          parameters:
+            CO2:
+              plugin:
+                name: CHIMERE
+                type: flux
+                version: AEMISSIONS
+              file_freq: 120H
+              dir: some_dir
+              file: some_file
+
+
+  1. follow the initial steps in :doc:`the flux template documentation page</documentation/plugins/datastreams/fluxes/flux_plugin_template>`
+     to initialize your new plugin and register it.
+
+     It includes copying the template folder to a new path and changing the variables
+     :bash:`_name`,:bash:`_fullname` and :bash:`_version` in the file :bash:`__init__.py`
+
+  2. update your Yaml to use the template flux (renamed with your preference). It should now look like that:
+
+     .. container:: toggle
+
+       .. container:: header
+
+         Show/Hide Code
+
+       .. code-block:: yaml
+         :linenos:
+
+           datavect:
+             plugin:
+               name: standard
+               version: std
+             components:
+               flux:
+                 parameters:
+                   CO2:
+                     plugin:
+                       name: your_new_name
+                       type: flux
+                       version: your_version
+
+  3. Test running again your test case. It should generate fluxes with random values as in the template
+
+Document your plugin
+====================
+
+Before going further, be sure to document your plugin properly.
+
+To do so, please replace the docstring header in the file :bash:`__init__.py`.
+
+Include the following information:
+
+    - licensing information
+    - permanent link to download the data (or a contact person if no link is publicly available)
+    - data format (temporal and horizontal resolution, names and shape of the data files)
+    - any specific treatment that prevents the plugin from working with another type of files.
+
+
+Updating functions and data to implement your flux data
+=======================================================
+
+
+
+
+
+
+
+#####################################
+
+
+
+
+  0. .. include:: ../newBCdata/knownplugin.rst
+
+
+  1. In directory :bash:`plugins/fluxes`, copy the directory containing the template
+     for a flux plugin :bash:`flux_plugin_template` in the directory for your new plugin.
+
+     .. include:: ../newBCdata/register.rst
+
+
+  2. Modify the yaml file to use the new plugin: the minimum input arguments are
+     :bash:`dir`, :bash:`file`, :bash:`varname` and :bash:`unit_conversion`.
+     The default space and time interpolations will be applied
+     (see XXXX doc sur premiere simu directe avec exmeple yaml quand mise a jourXXXXX).
+
+     .. code-block:: yaml
+
+       components:
+         fluxes:
+           plugin:
+              name: fluxes
+              version: template
+              type: fluxes
+           dir: dir_with_original_files/
+           file: file_with_new_fluxes_to_use_as_inputs
+           varname: NAMEORIG
+           unit_conversion:
+                 scale: 1.
 
-3. .. include:: ../newBCdata/devplugin.rst
+  3. .. include:: ../newBCdata/devplugin.rst
 
-XXXXXXX what about the input arguements? Ils demandent une partie dediee!?XXXXXXXXXX
+     XXXXXXX what about the input arguements? Ils demandent une partie dediee!?XXXXXXXXXX
 
 4. Document the new plugin:
    
@@ -49,15 +158,15 @@ XXXXXXX what about the input arguements? Ils demandent une partie dediee!?XXXXXX
        Template plugin for fluxes
        ###########################
 
-       .. automodule:: pycif.plugins.fluxes.flux_plugin_template
+       .. automodule:: pycif.plugins.datastreams.fluxes.flux_plugin_template
 
    c) add the reference to the rst file in docs/source/documentation/plugins/fluxes/index.rst:
 
      .. code-block:: rest
 
-        ####################
+        #####################
         Fluxes :bash:`fluxes`
-        ####################
+        #####################
 
         Available Fluxes
         =========================

docs/source/devtutos/newplugin/newplugin.rst

@@ -26,16 +26,27 @@ You then need to create an empty file called :bash:`__init__.py`, so python inte
     touch __init__.py
 
 
+.. note::
+
+   Although the :bash:`__init__.py` is not strictly needed by the newest version of python, pycif fetches information
+   directly from this file, for initializing the plugin and automatically documenting it.
+   Therefore, please include it anyway
+
+
+
 Registering your new :bash:`plugin` to pyCIF
 --------------------------------------------
 
+pyCIF attaches plugin functions as defined in the corresponding python module in :bash:`pycif/plugins/` automatically
+from the yaml configuration file.
+When the new plugin is created, it must be registered, so it can be called by other routines of pyCIF.
+
 :bash:`plugins` in pyCIF are identified with:
 
     - a name
     - optional: a version (default: std)
 
-When the new plugin is created, it must be registered, so it can be called by other routines of pyCIF.
-This can be done by providing the name and (if relevant) version in :bash:`__init__.py` :
+To register a new plugin, one must define the name and (if relevant) version in the file :bash:`__init__.py` :
 
 .. code-block:: python
 
@@ -50,6 +61,7 @@ You can check that your :bash:`plugin` is correctly registered by using the foll
     from pycif.utils.classes.baseclass import Plugin
     Plugin.print_registered()
 
+
 Adding requirements to your :bash:`plugin`