Skip to content
GitLab
Commit aaf8938c authored by Elise Potier's avatar Elise Potier
Browse files

Merge branch 'devel' of gitlab.in2p3.fr:satinv/cif into devel 

to check : input read, TNO, VPRM ?
parents 6de65783 e6a4b3a3
Hide whitespace changes
Inline Side-by-side
.gitlab-ci.yml
View file @ aaf8938c
......@@ -126,6 +126,45 @@ release_tagging:
only:
- devel
# Check that coverage did not decreased
improved_coverage:
stage: coverage
image:
name: pycif/pycif-ubuntu:0.1
entrypoint: [""]
before_script:
- pip freeze
- pip install coverage
script:
- python3 -m coverage combine coverage_raw/.coverage*
- coverage html -d reports/coverage
- coverage xml -o reports/coverage.xml
after_script:
- mkdir -p coverage
- xmlstarlet sel -t -v "//coverage/@line-rate" reports/coverage.xml > coverage/.current_coverage
- calc() { awk "BEGIN{print $*}"; }
- percent_coverage=`cat coverage/.current_coverage`
- tot_coverage=`calc ${percent_coverage}*100`
- echo 'TOTAL COVERAGE:'" ${tot_coverage}%"
coverage: '/^TOTAL COVERAGE: ([0-9\.]+\%)$/'
# - MAX=`if [ -f coverage/.master_cov ] ; then cat coverage/.master_cov ; else echo 0.00 ; fi`
# - CURRENT=`cat coverage/.current_coverage`
# - echo $CURRENT
# - echo $MAX
# - if [[ $CURRENT < $MAX ]] ; then echo "Coverage decreased!!!"; exit 1 ; else echo "Coverage did not decrease, good job!"; exit 0 ; fi;
artifacts:
when: always
paths:
- reports/coverage/
- coverage/.current_coverage
# - coverage_raw/
# cache:
# paths:
# - coverage/.master_cov
# Generate figures and artifacts for the website
article:
......@@ -137,11 +176,20 @@ article:
# - apt-get update
- pip freeze
script:
- tox -e py38 -e coverage -- -m "(dummy and article and inversion and not adjtltest and not uncertainties) or (fwd and ref_config)"
- |
if [ ${CI_COMMIT_BRANCH} == "LSCE" ]; then
tox -e py38 -e coverage -- -m "(dummy and article and inversion and not adjtltest and not uncertainties) or (fwd and ref_config) or (allsimulations)";
else
tox -e py38 -e coverage -- -m "(dummy and article and inversion and not adjtltest and not uncertainties and bands) or (fwd and ref_config)";
fi;
after_script:
- mkdir -p coverage
- xmlstarlet sel -t -v "//coverage/@line-rate" reports/coverage.xml > coverage/.current_coverage
- echo 'TOTAL COVERAGE:'" $(cat coverage/.current_coverage)%"
- calc() { awk "BEGIN{print $*}"; }
- percent_coverage=`cat coverage/.current_coverage`
- tot_coverage=`calc ${percent_coverage}*100`
- echo 'TOTAL COVERAGE:'" ${tot_coverage}%"
- mv coverage_raw/coverage/.coverage coverage_raw/.coverage.article
coverage: '/^TOTAL COVERAGE: ([0-9\.]+\%)$/'
artifacts:
when: always
......@@ -149,10 +197,9 @@ article:
- reports/pytest*.html
- reports/coverage/
- coverage/.current_coverage
- coverage_raw
- examples_artifact
- figures_artifact
only:
- LSCE
article_uncertainties:
stage: test
......@@ -163,11 +210,20 @@ article_uncertainties:
# - apt-get update
- pip freeze
script:
- tox -e py38 -e coverage -- -m "(dummy and article and inversion and not adjtltest and uncertainties) or (fwd and ref_config)"
- |
if [ ${CI_COMMIT_BRANCH} == "LSCE" ]; then
tox -e py38 -e coverage -- -m "(dummy and article and inversion and not adjtltest and uncertainties) or (fwd and ref_config) or (allsimulations)";
else
tox -e py38 -e coverage -- -m "(dummy and article and inversion and not adjtltest and uncertainties and bands) or (fwd and ref_config)";
fi;
after_script:
- mkdir -p coverage
- xmlstarlet sel -t -v "//coverage/@line-rate" reports/coverage.xml > coverage/.current_coverage
- echo 'TOTAL COVERAGE:'" $(cat coverage/.current_coverage)%"
- calc() { awk "BEGIN{print $*}"; }
- percent_coverage=`cat coverage/.current_coverage`
- tot_coverage=`calc ${percent_coverage}*100`
- echo 'TOTAL COVERAGE:'" ${tot_coverage}%"
- mv coverage_raw/coverage/.coverage coverage_raw/.coverage.article_uncertainties
coverage: '/^TOTAL COVERAGE: ([0-9\.]+\%)$/'
artifacts:
when: always
......@@ -175,10 +231,9 @@ article_uncertainties:
- reports/pytest*.html
- reports/coverage/
- coverage/.current_coverage
- coverage_raw
- examples_artifact
- figures_artifact
only:
- LSCE
# Run the tests for the dummy model
tests_dummy:
......@@ -194,7 +249,11 @@ tests_dummy:
after_script:
- mkdir -p coverage
- xmlstarlet sel -t -v "//coverage/@line-rate" reports/coverage.xml > coverage/.current_coverage
- echo 'TOTAL COVERAGE:'" $(cat coverage/.current_coverage)%"
- calc() { awk "BEGIN{print $*}"; }
- percent_coverage=`cat coverage/.current_coverage`
- tot_coverage=`calc ${percent_coverage}*100`
- echo 'TOTAL COVERAGE:'" ${tot_coverage}%"
- mv coverage_raw/coverage/.coverage coverage_raw/.coverage.dummy
coverage: '/^TOTAL COVERAGE: ([0-9\.]+\%)$/'
artifacts:
when: always
......@@ -202,6 +261,7 @@ tests_dummy:
- reports/pytest*.html
- reports/coverage/
- coverage/.current_coverage
- coverage_raw
- examples_artifact
# Run the tests for chimere (include downloading data)
......@@ -225,7 +285,11 @@ tests_chimere:
after_script:
- mkdir -p coverage
- xmlstarlet sel -t -v "//coverage/@line-rate" reports/coverage.xml > coverage/.current_coverage
- echo 'TOTAL COVERAGE:'" $(cat coverage/.current_coverage)%"
- calc() { awk "BEGIN{print $*}"; }
- percent_coverage=`cat coverage/.current_coverage`
- tot_coverage=`calc ${percent_coverage}*100`
- echo 'TOTAL COVERAGE:'" ${tot_coverage}%"
- mv coverage_raw/coverage/.coverage coverage_raw/.coverage.chimere
coverage: '/^TOTAL COVERAGE: ([0-9\.]+\%)$/'
artifacts:
when: always
......@@ -233,11 +297,13 @@ tests_chimere:
- reports/pytest*.html
- reports/coverage/
- coverage/.current_coverage
- coverage_raw
- examples_artifact
# Run the tests for flexpart (include downloading data)
tests_flexpart:
stage: test
retry: 2
image:
name: pycif/pycif-ubuntu:0.1
entrypoint: [""]
......@@ -254,7 +320,11 @@ tests_flexpart:
after_script:
- mkdir -p coverage
- xmlstarlet sel -t -v "//coverage/@line-rate" reports/coverage.xml > coverage/.current_coverage
- echo 'TOTAL COVERAGE:'" $(cat coverage/.current_coverage)%"
- calc() { awk "BEGIN{print $*}"; }
- percent_coverage=`cat coverage/.current_coverage`
- tot_coverage=`calc ${percent_coverage}*100`
- echo 'TOTAL COVERAGE:'" ${tot_coverage}%"
- mv coverage_raw/coverage/.coverage coverage_raw/.coverage.flexpart
coverage: '/^TOTAL COVERAGE: ([0-9\.]+\%)$/'
artifacts:
when: always
......@@ -262,6 +332,7 @@ tests_flexpart:
- reports/pytest*.html
- reports/coverage/
- coverage/.current_coverage
- coverage_raw
- examples_artifact
# Run the tests for flexpart (include downloading data)
......@@ -283,7 +354,11 @@ tests_tm5:
after_script:
- mkdir -p coverage
- xmlstarlet sel -t -v "//coverage/@line-rate" reports/coverage.xml > coverage/.current_coverage
- echo 'TOTAL COVERAGE:'" $(cat coverage/.current_coverage)%"
- calc() { awk "BEGIN{print $*}"; }
- percent_coverage=`cat coverage/.current_coverage`
- tot_coverage=`calc ${percent_coverage}*100`
- echo 'TOTAL COVERAGE:'" ${tot_coverage}%"
- mv coverage_raw/coverage/.coverage coverage_raw/.coverage.tm5
coverage: '/^TOTAL COVERAGE: ([0-9\.]+\%)$/'
artifacts:
when: always
......@@ -291,6 +366,7 @@ tests_tm5:
- reports/pytest*.html
- reports/coverage/
- coverage/.current_coverage
- coverage_raw
- examples_artifact
#
......@@ -351,22 +427,7 @@ tests_tm5:
# paths:
# - reports/flake8/
# Check that coverage did not decreased
# improved_coverage:
# stage: coverage
# image: bashell/alpine-bash
# script:
# - MAX=`if [ -f coverage/.master_cov ] ; then cat coverage/.master_cov ; else echo 0.00 ; fi`
# - CURRENT=`cat coverage/.current_coverage`
# - echo $CURRENT
# - echo $MAX
# - if [[ $CURRENT < $MAX ]] ; then echo "Coverage decreased!!!"; exit 1 ; else echo "Coverage did not decrease, good job!"; exit 0 ; fi;
# artifacts:
# paths:
# - coverage/.current_coverage
# cache:
# paths:
# - coverage/.master_cov
# Store the new coverage value only for the master and devel branches
# store_coverage:
......
bin/docker_debug.sh
View file @ aaf8938c
......@@ -10,13 +10,17 @@ config_file=`echo "$(cd "$(dirname "$config_file")"; pwd)/$(basename "$config_fi
pycif_root_dir=`echo "$(cd "$pycif_root_dir"; pwd)"`
# Fetch workdir to mount it in the container for saving outputs
workdir=`python3 -c \
"from pycif.utils.yml import ordered_load;
workdir=`python3 -W ignore -c \
"
from pycif.utils.yml import ordered_load;
with open('$config_file', 'r') as f:
config = ordered_load(f)
print('AAAAAAAAAAAA')
print(config['workdir']); "`
workdir=`echo $workdir | awk -F "AAAAAAAAAAAA" '{print $2}'`
mkdir -p $workdir
# Run the configuration into the container and writing outputs to workdir
......@@ -24,9 +28,9 @@ docker run -it -v $config_file:/config/config.yml \
-v $pycif_root_dir:/tmp/CIF/ \
-v $workdir:/workdir/ \
$extra_volumns \
--entrypoint /bin/bash pycif/pycif-tm5:0.2
pycif/pycif-tm5:0.2
# --entrypoint /bin/bash pycif/pycif-tm5:0.2
# --entrypoint /bin/bash pycif/pycif-ubuntu:0.1
# pycif/pycif-tm5:latest
bin/docker_tox.sh
View file @ aaf8938c
......@@ -8,7 +8,7 @@ extra_volumns="-v /home/aberchet/Projects/PYCIF_DATA_TEST/:/tmp/PYCIF_DATA_TEST/
docker run \
-v $pycif_root_dir:/tmp/CIF/ \
$extra_volumns \
-it --entrypoint /tmp/CIF/bin/tox_command.sh pycif/pycif-tm5:0.2
-it --entrypoint /tmp/CIF/bin/tox_command.sh pycif/pycif-ubuntu:0.1
......
bin/job_pycif
View file @ aaf8938c
#QSUB -s /bin/tcsh
#PBS -q longp
#PBS -l nodes=1:ppn=8
#PBS -q xlongp
#PBS -l nodes=1:ppn=10
python -m pycif /home/users/aberchet/CIF/config_files/tuto_chimere/config_chimere_argonaut_n2o_sat_inv_corr.yml
#python -m pycif /homen/users/aberchet/CIF/config_files/tuto_chimere/config_chimere_EUROCOM_satOMI.yml
python -m pycif /home/users/aberchet/CIF/config_files/RECAPP/config_chimere_fwd_EUROCOM_CO2_AB_6mois_TNO_2008_1.yml
bin/job_pycif_recapp_2008_1 0 → 100644
View file @ aaf8938c
#QSUB -s /bin/tcsh
#PBS -q xlongp
#PBS -l nodes=1:ppn=10
#PBS -N RECAPP_2008_1
python -m pycif /home/users/aberchet/CIF/config_files/RECAPP/config_chimere_fwd_EUROCOM_CO2_AB_6mois_TNO_2008_1.yml
bin/pycif_test.sh
View file @ aaf8938c
......@@ -13,11 +13,11 @@ dirout=/home/chimereges/aberchet/debugchimere/pytest/
export PYCIF_DATATEST=/home/chimereges/PYCIF_TEST_DATA/
# Set up the platform to be run at LSCE, otherwise, use default docker parameters
#export PYCIF_PLATFORM=LSCE
export PYCIF_PLATFORM=LSCE
###
# select a subset of tests to run by using the tags ("mark")
#mark="(dummy and article and inversion and not adjtltest and not uncertainties) or (fwd and ref_config)"
mark="(dummy and article and inversion and not adjtltest and uncertainties) or (fwd and ref_config)"
mark="(dummy and article and inversion and not adjtltest and not uncertainties) or (fwd and ref_config)"
#mark="(dummy and article and inversion and not adjtltest and uncertainties) or (fwd and ref_config)"
#mark="(fwd and ref_config)"
#mark="test_in_ci and dummy"
#mark="test_in_ci and chimere"
......
bin/tox_command.sh
View file @ aaf8938c
......@@ -2,4 +2,4 @@
cd /tmp/CIF/
pip freeze
tox -e py38 -e coverage -- -m 'test_in_ci and tm5 and fwd'
tox -e py38 -e coverage -- -m 'test_in_ci and dummy and fwd'
docs/source/conf.py
View file @ aaf8938c
......@@ -318,7 +318,6 @@ def process_pycif_keywords(app, what, obj_name, obj, options, lines):
- default_values
- mandatory_values
"""
ref_lines = copy.deepcopy(lines)
# Adding bash highlight by default
......@@ -403,9 +402,30 @@ def process_pycif_keywords(app, what, obj_name, obj, options, lines):
preftree = key_req.get("preftree", "")
empty = key_req.get("empty", False)
name = key_req.get("name", None)
version = key_req.get("version", "")
plg_type = Plugin.plugin_types[key_req.get("type", req)][1]
plg_path = Plugin.plugin_types[key_req.get("type", req)][0][1:]
version = key_req.get("version", None)
req_type = key_req.get("type", req)
req_subtype = key_req.get("subtype", "")
# Load required plugin to deal with types and sub-types
plg_req = Plugin.from_dict({
"plugin": {
"name": name,
"version": version,
"type": req_type,
"subtype": req_subtype
}
})
plg_req._load_plugin_type(req)
plg_type = \
Plugin.plugin_types[plg_req.plugin.type][1]
plg_path = \
Plugin.plugin_types[plg_req.plugin.type][0][1:]
plg_subtype = \
Plugin.plugin_subtypes[
plg_req.plugin.type][
plg_req.plugin.subtype][1:]
# String to dump
newplg = key_req.get("newplg", False)
towrite.extend((
" * - {}\n"
......@@ -534,7 +554,7 @@ def build_rst_from_plugins(app):
init_dir(plg_dir)
# Initialize index
towrite = [
towrite_overall_index = [
"##################",
"Plugins in pyCIF",
"##################",
......@@ -542,13 +562,9 @@ def build_rst_from_plugins(app):
".. toctree::",
" :maxdepth: 3",
"",
" ../plugin_description",
" ../dependencies"
] + [
" {}/index".format(Plugin.plugin_types[plg_type][0][1:])
for plg_type in Plugin.plugin_types
]
with open("{}/index.rst".format(plg_dir), "w") as f:
f.write("\n".join(towrite))
# Loop on all plugin types
for plg_type in Plugin.plugin_types:
......@@ -558,7 +574,7 @@ def build_rst_from_plugins(app):
Plugin.plugin_types[plg_type][0])
class_module = pkgutil.importlib.import_module(class_path)
local_class = getattr(class_module, class_name)
# Create directory
plg_type_dir = "{}/{}".format(
plg_dir, Plugin.plugin_types[plg_type][0][1:])
......@@ -566,29 +582,79 @@ def build_rst_from_plugins(app):
# Loop over modules of this class
package_path = "pycif.plugins{}".format(Plugin.plugin_types[plg_type][0])
if pkgutil.importlib.util.find_spec(package_path) is None:
continue
# Update overall index
towrite_overall_index.append(
" {}/index".format(Plugin.plugin_types[plg_type][0][1:]))
# Loop over sub-types
import_package = pkgutil.importlib.import_module(package_path)
package_index = []
for mod in pkgutil.walk_packages(import_package.__path__,
prefix=import_package.__name__ + "."):
if not mod.ispkg:
continue
for subtype in Plugin.plugin_subtypes[plg_type]:
local_subpackage = "{}{}".format(
package_path,
Plugin.plugin_subtypes[plg_type][subtype])
import_subpackage = pkgutil.importlib.import_module(local_subpackage)
# Create directory
plg_subtype_dir = "{}/{}".format(
plg_type_dir,
Plugin.plugin_subtypes[plg_type][subtype][1:])
init_dir(plg_subtype_dir)
loc_mod = pkgutil.importlib.import_module(mod.name)
# Register modules only when a name is given
if not hasattr(loc_mod, "_name"):
continue
# Loop over modules in the sub-type
package_subindex = []
for mod in pkgutil.walk_packages(import_subpackage.__path__,
prefix=import_subpackage.__name__ + "."):
if not mod.ispkg:
continue
loc_mod = pkgutil.importlib.import_module(mod.name)
# Register modules only when a name is given
if not hasattr(loc_mod, "_name"):
continue
# Create corresponding rst file
file_name = "{}/{}.rst".format(
plg_subtype_dir, loc_mod.__name__.split(".")[-1])
# Create corresponding rst file
file_name = "{}/{}.rst".format(
plg_type_dir, loc_mod.__name__.split(".")[-1])
title = ":bash:`{}` / :bash:`{}`".format(
loc_mod._name, getattr(loc_mod, "_version", "std"))
title = ":bash:`{}` / :bash:`{}`".format(
loc_mod._name, getattr(loc_mod, "_version", "std"))
if hasattr(loc_mod, "_fullname"):
title = "{} ({})".format(loc_mod._fullname, title)
towrite = [
".. role:: bash(code)",
" :language: bash",
"",
"",
len(title) * "#",
title,
len(title) * "#",
"",
".. automodule:: {}".format(loc_mod.__name__)
]
with open(file_name, "w") as f:
f.write("\n".join(towrite))
# Append name for plugin type index
package_subindex.append(loc_mod.__name__.split(".")[-1])
# Sort names
package_subindex.sort()
if hasattr(loc_mod, "_fullname"):
title = "{} ({})".format(loc_mod._fullname, title)
# Write the plugin type index
if subtype == "":
continue
title = list(subtype)
title[0] = title[0].upper()
title = "".join(title)
towrite = [
".. role:: bash(code)",
" :language: bash",
......@@ -597,54 +663,86 @@ def build_rst_from_plugins(app):
len(title) * "#",
title,
len(title) * "#",
""] + ([".. contents:: Contents", " :local:", ""]
if import_subpackage.__doc__ is not None else []) + [
"Available {}".format(title),
(len(title) + 11) * "=",
"",
".. automodule:: {}".format(loc_mod.__name__)
]
with open(file_name, "w") as f:
"The following :bash:`{}` of sub-type {} "
"are implemented in pyCIF so far:".format(
Plugin.plugin_types[plg_type][0][1:],
subtype),
"",
".. toctree::",
"",
] + [
" {}".format(plg) for plg in package_subindex
] + (
import_subpackage.__doc__.split('\n')
if import_subpackage.__doc__ is not None
else []
)
with open("{}/index.rst".format(plg_subtype_dir), "w") as f:
f.write("\n".join(towrite))
# Append name for plugin type index
package_index.append(loc_mod.__name__.split(".")[-1])
# Sort names
package_index.sort()
# Write the plugin type index
title = list(Plugin.plugin_types[plg_type][0][1:])
title[0] = title[0].upper()
title = "".join(title) + " (:bash:`{}`)".format(plg_type)
towrite = [
".. role:: bash(code)",
" :language: bash",
"",
"",
len(title) * "#",
title,
len(title) * "#",
""] + ([
".. contents:: Contents",
" :local:",
""
] if import_package.__doc__ is not None else []) + [
"Available {}".format(title),
(len(title) + 11) * "=",
"",
"The following :bash:`{}` are implemented in pyCIF so far:".format(
Plugin.plugin_types[plg_type][0][1:]),
"",
".. toctree::",
"",
] + [
" {}".format(plg) for plg in package_index
] + (
".. role:: bash(code)",
" :language: bash",
"",
"",
len(title) * "#",
title,
len(title) * "#",
""] + ([
".. contents:: Contents",
" :local:",
""
] if import_package.__doc__ is not None else []) + [
"Available {}".format(title),
(len(title) + 11) * "=",
""]
# If only one sub-type, just create an index of all available plugins
if len(Plugin.plugin_subtypes[plg_type]) == 1:
towrite.extend([
"The following :bash:`{}` are implemented in pyCIF so far:".format(
Plugin.plugin_types[plg_type][0][1:]),
"",
".. toctree::",
"",
] + [" {}".format(plg) for plg in package_subindex])
# If sub-types create an index pointing to sub-types and plugins
else:
towrite.extend([
"The following sub-types and :bash:`{}` are implemented "
"in pyCIF so far:".format(
Plugin.plugin_types[plg_type][0][1:]),
"",
".. toctree::",
"",
] + [" {}/index".format(Plugin.plugin_subtypes[plg_type][subtype][1:])
for subtype in Plugin.plugin_subtypes[plg_type]
])
# Append overall type description
towrite.extend(
import_package.__doc__.split('\n')
if import_package.__doc__ is not None
else []
)
else [])
# Dump the string to the rst file
with open("{}/index.rst".format(plg_type_dir), "w") as f:
f.write("\n".join(towrite))
# Dump the overall index
with open("{}/index.rst".format(plg_dir), "w") as f:
f.write("\n".join(towrite_overall_index))
# Generate available list
s = StringIO()
Plugin.print_registered(print_rst=True, print_requirement=True, stream=s)
......
docs/source/contrib_doc.rst
View file @ aaf8938c
......@@ -17,9 +17,9 @@ Requirements
To contribute to the documentation the following softwares need to be install on your system:
* `Sphinx <https://www.sphinx-doc.org/en/master/>`__ : automatic generation of html pages from rst files
* `Graphviz <https://www.graphviz.org/>`__ : embed graphics in the documentation
* pycif : the CIF python package needs to be properly installed on your system to automatically build from modules
* `Sphinx <https://www.sphinx-doc.org/en/master/>`__: automatic generation of html pages from rst files
* `Graphviz <https://www.graphviz.org/>`__: embed graphics in the documentation
* :doc:`pycif<installation>`: the CIF python package needs to be properly installed on your system to automatically build from modules
Compiling commands
==================
......@@ -34,6 +34,13 @@ To do so, type:
Then, you can simply open html files generated in :bash:`CIF_ROOT/docs/build/html/` in any web browser.
It is recommended to re-compile the documentation from scratch time to time:
.. code-block:: bash
cd CIF_ROOT/docs
make clean
Adding new files in the documentation
-------------------------------------
......@@ -43,21 +50,22 @@ Tables of contents are generally placed in the :bash:`index.rst` file in each su