add gen_batch_jobs and run_batch

Makefile

-options:
+default:
+	./gen_cases.sh
+#hust the options of the first group (gridded output)
+options_1A:
+	./gen_options_group1_gridout.sh
+cases_1A: 
+	./gen_cases.sh gen_options_group1_gridout.sh  
+define:
 	./define_default_run.sh	
 slfiles: 
 	./loop_on_pathnames.sh
 run:
 	./run_all_sl
 clean:
-	rm -r options* pathnames* output* batch_dry_pathnames_*
+	rm -r options* pathnames* output* *.sl *.cl
+cleanSLURM:
+	rm output-prod-cn* error-prod-cn*
 archive_dir := archive_$(shell date "+%y-%m-%d_%H%M%S")
 archive:
 	@echo "archive generated files"

README

@@ -12,9 +12,11 @@ The cases are derived from reference options dir and pathnames file (working exa
 
 (currently focuses on ecmwf winds: pathanmes can be edited at a later stage to accomodate different winds with the same options IMPLEMENT)
 files 
+- export*: copy define* to a different directory to build the run cases there.
+
 - define_<runs>: Generates flexpart run cases (groups depending on _<runs>) 
-- export*: copy define* to a ifferent directory to build the run cases there.
 - (pathanmes could be edited here)
+
 - loop_on_pathnames*.sh: for a fived options-pathnames pair, create a run case (adding an executable and an output directory)
 - run_all_sl*: execute the run cases generating a set of flexpart outputs
 - read_joblist*.sh: verify that the runs finished properly.
@@ -24,6 +26,7 @@ input:
    default located repo in the same directory 
    i.e. ../flexpart/options/
  pathname: ../flexpart/pathnames
+
  destinantion dir: 
   default "../flex_cases" alongside in the same directory level
   it is created every time. it it already exists from a previous run it will stop 
@@ -87,48 +90,74 @@ the stdout of flexpart is tee'd to stdoutbatch_dry_pathnames*.sl
 
 the output* directory(es) is(are) ready to be read by flex_read
 
+running in SLURM
+1) generate the cases
+2) export the files together with the run generating scripts
+3) run the run generator 
+    specify the executable and other parameters needed by SLURM
+4) run the run runner: IT HAS TO BE IN THE SAME DIR AS ABOvE OTHERWISE SLURM MAY BE CONFUSED
+
+
 2018-08-22
 decouple:
 
-gen_options_*.sh
-input: base options directory (preferrably a working one)
-output: derived options directories defining the corresponding test or use cases  
+** gen_options_*.sh
+input: 
+  base options directory (preferrably a working one)
+output: 
+  derived options directories defining the corresponding test or use cases  
+e.g.:
+ gen_options_default.sh creates one run (copies the reference)
+ gen_options_group1_gridout.sh (gridded output)
+ gen_options_group2.sh (backward runs)
+   generate group 1 (gridded output) of test cases from default options in flexpart local repository
+
 
-once options are created, consistent output and patnames are needed
-in addition winds need to be available and consistent with a corresponding AVAILABLE file
+once options are created, corresponding output dir and pathnames are needed (mutually consistent)
+On the other hand, a well formed, operational pathnames needs in addition 
+- wind field files
+- a corresponding AVAILABLE file referring to those winds 
 
-gen_pathnames_*.sh
+** gen_pathnames_*.sh
 input: 
 - base pathnames (template -- sets wind dirs and available file path)
-- set of options (output from gen_options_*.sh) located in the current directory
- e.g: gen_pathnames_default.sh  creates pathnames and output in local dir from repos default  
-
-
+- set of options (output from gen_options_*.sh) located in the current directory 
 output: corresponding set of pathnames
+e.g.: 
+ gen_pathnames_default.sh  creates pathnames and output in local dir from repos default  
+ gen_pathnames.sh loop on list of option dirs and generate corresponding pathnames
 
-gen_cases_*.sh
+** gen_cases_*.sh
 calls gen_options_*.sh and gen_pathnames_*.sh
 input: input from gen_options_*.sh and gen_pathnames_*.sh combined 
 output: 
 - output from gen_options_*.sh and gen_pathnames_*.sh combined 
 - output dirs (in the local directory by default) 
+e.g. 
+ gen_cases_default.sh 
+ gen_cases.sh
 
-gen_runs_*.sh
-
+** gen_runs_*.sh
+   gen_batch_jobs_*.sh 
 cretes cases and defines a running string for each case
 the running string is part of a file that depends on the system
-e.g. can be executed directly in the command line or using a workload manager (LoadLeveler, SLURM, etc.) 
+can be executed directly in the command line or using a workload manager (LoadLeveler, SLURM, etc.) 
+e.g. 
+gen_runs.sh 
 
-run_runs
-
-check_runs
-verifies that the runs have finished without errors
+** gen_batch_jobs_*.sh
+gen_batch_jobs_cl.sh
+gen_batch_jobs_sl.sh
+needs some pathanmes to exist in the directory
 
+** run_batch_cl.sh run runs
+depends on how the previous batch scripts have been created
+run_batch_cl.sh
+run_batch_sl.sh to be copied 
 
-example:
-generate group 1 (gridded output) of test cases from default options in flexpart local repository
+** check_runs*
+verifies that the runs have finished without errors
 
-gen_options_group1_gridout.sh
 
 
 

gen_batch_jobs_cl.sh

+#!/bin/bash
+
+OUTPUTdefault="$(ls -1 pathnames*)"
+
+
+OUTPUT=${OUTPUTvar:-$OUTPUTdefault}
+
+#OUTPUTpwd="$(pwd)"
+
+#FLEXPARTdefault=FLEXPART_8d70e43
+# if FLEXPART is in $PATH
+FLEXPARTdefault=FLEXPART
+
+#FLEXPART=${FLEXPARTvar:-$FLEXPARTdefault}
+# $FLEXPART is the command line in or default defined above
+FLEXPART=${1:-$FLEXPARTdefault}
+
+argsdefault=' ' # just a blank
+# if 2nd arg
+args=${2:-$argsdefault}
+
+# echo $OUTPUT
+
+for i in ${OUTPUT}
+do
+
+suffix=${i#*patnames}
+
+#FP_slurm_batch_sl=slurm_batch_$i.sl
+#FP_slurm_batch_sl=batch_dry_$i.sl
+#FP_slurm_batch_sl=clrun_full_$i.sl
+#FP_slurm_batch_sl=fork_cl_$i.sl
+FP_slurm_batch_sl=batch_job_$suffix.cl
+
+        echo  '                                          ' >> $FP_slurm_batch_sl
+        #FLEXPART_run_string="$FLEXPART  $i -i"
+        FLEXPART_run_string="$FLEXPART  $i $args"
+        echo $FLEXPART_run_string   >> $FP_slurm_batch_sl
+	echo $FLEXPART_run_string
+done
+

gen_pathnames.sh

 #!/bin/bash
+# loop on list of option dirs and generate corresponding pathnames
+# snippet from 
+# /Users/ignacio/todo/doing/flexpart/2018-05-25/loop_on_ls.sh
+# /Users/ignacio/todo/doing/flexpart/2018-05-24/loop_on_ls.sh
+# there the aim was to convert the format
 
 pathnames_template=~/repos/flexpart/pathnames
+echo pathnames_template = $pathnames_template 
 
-
-defaultdir=/Users/Shared/Jenkins/Home/jobs/FLEXPART-dev-tests-run2/workspace/
-defaultdir=
+#defaultdir=
 #echo $defaultdir
 
-OUTPUT="$(ls -d1 ${1:-$defaultdir}options*)"
-#OUTPUT="$(ls -d1 ${1:-$defaultdir}options_*)"
-#OUTPUT="$(find ${1:-$defaultdir}options_*)"
-OUTPUTpwd="$(pwd)"
-
-echo OUTPUT=$OUTPUT
-
+#OUTPUT="$(ls -d1 ${1:-$defaultdir}options*)"
+option_dir_list="$(ls -d1 options*)"
 
 
-#mkdir output
-#cp $pathnames_local pathnames
-
-for i in ${OUTPUT}
+for i in ${option_dir_list}
 do
-
-echo $i
-  echo expand:
+  # echo $i
+  # echo expand:
+  # extract suffix from options dirs
   suffix=${i#*options}
   echo suffix=$suffix
-
-options_new=$i
-
-output_new=output$suffix
-mkdir $output_new
-
-pathnames_new=pathnames$suffix
-
-echo pathnames_new = $pathnames_new
-echo pathnames_template = $pathnames_template 
-
-# copy template (contains valid AVAILABLE file and winds)
-cp $pathnames_template $pathnames_new
-# replace options line
-sed -i "s/\boptions\b/$options_new/g" $pathnames_new
-# replace output line
-sed -i "s/\boutput\b/$output_new/g" $pathnames_new
-
+  options_new=$i
+
+  # define name of new output and create it
+  output_new=output$suffix
+  mkdir $output_new
+
+  # define name of new pathnames
+  pathnames_new=pathnames$suffix
+  echo pathnames_new = $pathnames_new
+  # copy template (contains valid AVAILABLE file and winds)
+  cp $pathnames_template $pathnames_new
+  # replace options line
+  sed -i "s/\boptions\b/$options_new/g" $pathnames_new
+  # replace output line
+  sed -i "s/\boutput\b/$output_new/g" $pathnames_new
 done
 

gen_runs.sh

@@ -5,7 +5,7 @@
 # run generate cases
 ./gen_cases.sh $1
 
-./loop_on_pathnames_laptop.sh
-
+#./loop_on_pathnames_laptop.sh
+./gen_batch_jobs_cl.sh
 
 

run_batch_cl.sh

+#!/bin/bash
+
+OUTPUTdefault="$(ls -1 *.cl)"
+#OUTPUT=${OUTPUTvar:-$OUTPUTdefault}
+# could input a subset of jobs to run "$(ls -1 *.cl)"
+OUTPUT=${1:-$OUTPUTdefault}
+
+#rm joblist
+
+for i in ${OUTPUT}
+do
+   echo batch job $i #| tee -a joblist1$group
+   #source $i | tee -a joblist2$group
+   source $i | tee stdout.$i  # -a joblist2$group
+   echo '##########################################' 
+done
+
+
+