Fixing GitLab CI for CHIMERE

bin/pycif_test.sh

@@ -7,5 +7,5 @@ export PYCIF_DATATEST=/tmp/PYCIF_DATA_TEST/
 #    --html reports/pytest.html --self-contained-html --junitxml=reports/pytest.xml --collect-only ../tests/dummy/
 cd ../
 #pytest -s -m "test_in_ci" --html reports/pytest.html --self-contained-html
-pytest -s -m "test_in_ci and dummy" / #--basetemp=/home/satellites13/aberchet/pytest/ \
+pytest -s -m "test_in_ci and chimere" / #--basetemp=/home/satellites13/aberchet/pytest/ \
     --html reports/pytest.html --self-contained-html

pycif-tests.Dockerfile

@@ -5,20 +5,21 @@ FROM pycif-test
 RUN apt-get install -y python-yaml \
     && pip install pytest pytest-html pyyaml
 
-COPY ./ /tmp/CIF
-
 RUN cd /tmp/CIF/ \
     && python setup.py develop --user -u \
     && python setup.py develop --user
 
-#RUN cd /tmp/CIF/bin \
-#    && ./pycif_test.sh
+COPY ./ /tmp/CIF
+
+RUN cd /tmp/CIF/ \
+    && export PYCIF_DATATEST=/tmp/PYCIF_DATA_TEST/ \
+    && pytest -s -m "test_in_ci and chimere"
 
-RUN cd /tmp/CIF/config_files/tuto_gauss \
-    && python -m pycif config_dummy_fwd.yml
+#RUN cd /tmp/CIF/config_files/tuto_gauss \
+#    && python -m pycif config_dummy_fwd.yml
 
 
-RUN ls -R /tmp/PYCIF_DATA_TEST
+#RUN ls -R /tmp/PYCIF_DATA_TEST
 
 #RUN cd /tmp/CIF/config_files/tuto_chimere/ \
 #    && python -m pycif config_chimere_fwd_academic_onespec.yml;

pycif/plugins/models/chimere/compile.py

@@ -24,11 +24,13 @@ def compile(self):
         copy(source, comp_dir)
         return
     
-    except IOError:
+    except IOError as e:
         if not getattr(self, "auto-recompile"):
-            return Exception(
-                "CHIMERE could not find executables and was not asked to "
-                "compile them; specify auto-recompile = True in Yaml to do so")
+            raise Exception(
+                "CHIMERE could not find executables ({}) and was not asked to "
+                "compile them; specify auto-recompile = True in Yaml to do so"
+                .format(source)
+            )
         
     # Otherwise, re-compile
     # Copying sources locally; overwrite folder if exists
@@ -100,7 +102,9 @@ def compile(self):
                 stderr=subprocess.PIPE,
             )
             _, stderr = process.communicate()
-            debug(stderr)
+            
+            for ln in stderr.split("\n"):
+                debug(ln)
         
         source = "{}/sources/src{}/{}chimere.e".format(comp_dir, suff, pref)
         copy(source, comp_dir)

setup.py

@@ -44,7 +44,7 @@ requirements = [
     NUMPY,
     CFTIME,
     'scipy',
-    'matplotlib',
+    'matplotlib<3.4',
     'pandas',
     'netCDF4',
     'pytz',

tests/chimere/fixtures/__init__.py

@@ -19,7 +19,7 @@ def ref_chimere(tmpdir):
     tmpdir_str = tmpdir.strpath
 
     config = {
-        "verbose": 2,
+        "verbose": 1,
         "logfile": "pycif.logtest",
         "workdir": tmpdir_str,
         "datei": datetime.datetime(2011, 3, 22, 0),
@@ -141,21 +141,23 @@ def ref_chimere(tmpdir):
     Path("{}/src".format(model_dir)).mkdir(parents=True, exist_ok=True)
     Path("{}/src_tl".format(model_dir)).mkdir(parents=True, exist_ok=True)
     Path("{}/src_ad".format(model_dir)).mkdir(parents=True, exist_ok=True)
+
+    # Copy and replace if not symlinks
+    if os.path.islink("{}/fwdchimere.e".format(model_dir)):
+        return
     
-    #
-    #
-    # shutil.copy(
-    #     "{}/model/fwdchimere.e".format(tmpdir_str),
-    #     "{}/src/fwdchimere.e".format(model_dir),
-    #     follow_symlinks=True
-    # )
-    # shutil.copy(
-    #     "{}/model/tlchimere.e".format(tmpdir_str),
-    #     "{}/src_tl/tlchimere.e".format(model_dir),
-    #     follow_symlinks=True
-    # )
-    # shutil.copy(
-    #     "{}/model/achimere.e".format(tmpdir_str),
-    #     "{}/src_ad/achimere.e".format(model_dir),
-    #     follow_symlinks=True
-    # )
+    shutil.copy(
+        "{}/model/fwdchimere.e".format(tmpdir_str),
+        "{}/src/fwdchimere.e".format(model_dir),
+        follow_symlinks=True
+    )
+    shutil.copy(
+        "{}/model/tlchimere.e".format(tmpdir_str),
+        "{}/src_tl/tlchimere.e".format(model_dir),
+        follow_symlinks=True
+    )
+    shutil.copy(
+        "{}/model/achimere.e".format(tmpdir_str),
+        "{}/src_ad/achimere.e".format(model_dir),
+        follow_symlinks=True
+    )

tests/chimere/test_integration_1fwd.py

@@ -11,7 +11,7 @@ from pycif.utils.datastores.dump import read_datastore
 @pytest.mark.chimere
 @pytest.mark.short
 @pytest.mark.long
-# @pytest.mark.test_in_ci
+@pytest.mark.test_in_ci
 def test_integration_fwd(chimere_config_fwd):
     """
     Integration test that runs the chimere_forward model.