Commit 46fc297d authored by Ignacio Pisso's avatar Ignacio Pisso
Browse files

Merge branch 'master' of git.nilu.no:ignacio/flex_gen_input

parents ecaae4f0 d3ae8702
......@@ -4,7 +4,17 @@ slfiles:
./loop_on_pathnames.sh
run:
./run_all_sl
clean: this_is_a_run
rm -r options*
clean:
rm -r options* pathnames* output* batch_dry_pathnames_*
archive_dir := archive_$(shell date "+%y-%m-%d_%H%M%S")
archive:
@echo "archive generated files"
mkdir $(archive_dir)
mv options* pathnames* output* stdout* clrun* $(archive_dir)
cleanrun:
rm *.out rm *.err *.sl
logfile := file_$(shell date +%FT%T%Z).log
flush_log:
@echo "==> flush logs to $(logfile)"
echo ‘my_log’ > $(logfile)
echo `wc -l $(logfile)`
flex gen cases
Generates flexpart run cases based on the current default options dir and pathnames file.
(pathanmes can be edited at a later stage to accomodate different winds with the same options IMPLEMENT
)
1)define and generate cases (options, pathnames and output dirs)
2)prepare for run (enforce conditions of existence and define execution files and instructions)
3)run (generate output) depends on system ()
3bis) set reference output
4)read generated output
5)compare to reference
Generates standard flexpart run cases.
The cases are derived from reference options dir and pathnames file (working examples by default from the current repository head /release? decide on the reference FIXME).
(currently focuses on ecmwf winds: pathanmes can be edited at a later stage to accomodate different winds with the same options IMPLEMENT)
files
- define_<runs>: Generates flexpart run cases (groups depending on _<runs>)
- export*: copy define* to a ifferent directory to build the run cases there.
......@@ -22,6 +30,21 @@ input:
(add --force option ENHANCE)
history
1-
1bis-postprocessing: eg change format Legras/Fast or prepare CTBTO XML descriptor
eg
/Users/ignacio/todo/doing/flexpart/2018-05-23/loop_on_ls.sh
2- run
/Users/ignacio/todo/doing/flexpart/2018-04-03/run_generated_tests.sh
/Users/ignacio/todo/doing/flexpart/2018-04-04/run_generated_tests_serial.sh
/Users/ignacio/todo/doing/flexpart/2018-04-04/run_in_njord.sh
3bis-
/Users/ignacio/todo/doing/flexpart/2018-04-04/generate2
4,5- subsequent task
/Users/ignacio/Desktop/flexpart/read_cases/logbook
repo flex_gen_input cloned to ~/repos
how to run
......@@ -64,6 +87,57 @@ the stdout of flexpart is tee'd to stdoutbatch_dry_pathnames*.sl
the output* directory(es) is(are) ready to be read by flex_read
2018-08-22
decouple:
gen_options_*.sh
input: base options directory (preferrably a working one)
output: derived options directories defining the corresponding test or use cases
once options are created, consistent output and patnames are needed
in addition winds need to be available and consistent with a corresponding AVAILABLE file
gen_pathnames_*.sh
input:
- base pathnames (template -- sets wind dirs and available file path)
- set of options (output from gen_options_*.sh) located in the current directory
e.g: gen_pathnames_default.sh creates pathnames and output in local dir from repos default
output: corresponding set of pathnames
gen_cases_*.sh
calls gen_options_*.sh and gen_pathnames_*.sh
input: input from gen_options_*.sh and gen_pathnames_*.sh combined
output:
- output from gen_options_*.sh and gen_pathnames_*.sh combined
- output dirs (in the local directory by default)
gen_runs_*.sh
cretes cases and defines a running string for each case
the running string is part of a file that depends on the system
e.g. can be executed directly in the command line or using a workload manager (LoadLeveler, SLURM, etc.)
run_runs
check_runs
verifies that the runs have finished without errors
example:
generate group 1 (gridded output) of test cases from default options in flexpart local repository
gen_options_group1_gridout.sh
......
#!/bin/bash
# Ignacio Pisso 2018-03-29
default_gen_options=gen_options_default.sh
gen_options=${1:-$default_gen_options} # $1
echo gen_options=$gen_options
#execute gen options
./$gen_options
options_list="$(ls -d options*)"
# generate pathnames
# generate outputs
for i in ${options_list}
do
echo $i
done
./gen_pathnames.sh
# list available suffixes
pathnames_local=~/repos/flexpart/pathnames
# mkdir output
# cp $pathnames_local pathnames
gen_pathnames_default.sh
#!/bin/bash
# Ignacio Pisso 2018-03-29
options_repo=~/repos/flexpart/options
cp -r $options_repo options
#!/bin/bash
# Ignacio Pisso 2018-03-29
# generate FLEXPART variations on the default for parameter sweep hello world tests
options_template=~/repos/flexpart/options
# group 1 gridded output
#
suffix=_1
options_new=options$suffix
cp -r $options_template $options_new
sed -i '/IOUT=/c\ IOUT=1,' $options_new/COMMAND
suffix=_2
options_new=options$suffix
cp -r $options_template $options_new
sed -i '/IOUT=/c\ IOUT=2,' $options_new/COMMAND
suffix=_3
options_new=options$suffix
cp -r $options_template $options_new
sed -i '/IOUT=/c\ IOUT=3,' $options_new/COMMAND
suffix=_4
options_new=options$suffix
cp -r $options_template $options_new
sed -i '/IOUT=/c\ IOUT=4,' $options_new/COMMAND
suffix=_5
options_new=options$suffix
cp -r $options_template $options_new
sed -i '/IOUT=/c\ IOUT=5,' $options_new/COMMAND
#!/bin/bash
# Ignacio Pisso 2018-03-29
# generate FLEXPART variations on the default for parameter sweep hello world tests
options_template=~/repos/flexpart/options
# group 1 gridded output
#
suffix=_bwd
options_new=options$suffix
cp -r $options_template $options_new
sed -i '/LDIRECT=/c\ LDIRECT= -1,' $options_new/COMMAND
sed -i '/IOUTPUTFOREACHRELEASE=/c\ IOUTPUTFOREACHRELEASE= 1,' $options_new/COMMAND
sed -i '/IOUT=/c\ IOUT= 1,' $options_new/COMMAND #should not be needed
suffix=_bwd5
options_new=options$suffix
cp -r $options_template $options_new
sed -i '/LDIRECT=/c\ LDIRECT= -1,' $options_new/COMMAND
sed -i '/IOUTPUTFOREACHRELEASE=/c\ IOUTPUTFOREACHRELEASE= 1,' $options_new/COMMAND
sed -i '/IOUT=/c\ IOUT= 5,' $options_new/COMMAND
suffix=_bwd_nc
options_new=options$suffix
cp -r $options_template $options_new
sed -i '/LDIRECT=/c\ LDIRECT= -1,' $options_new/COMMAND
sed -i '/IOUTPUTFOREACHRELEASE=/c\ IOUTPUTFOREACHRELEASE= 1,' $options_new/COMMAND
sed -i '/IOUT=/c\ IOUT= 9,' $options_new/COMMAND
#!/bin/bash
pathnames_template=~/repos/flexpart/pathnames
defaultdir=/Users/Shared/Jenkins/Home/jobs/FLEXPART-dev-tests-run2/workspace/
defaultdir=
#echo $defaultdir
OUTPUT="$(ls -d1 ${1:-$defaultdir}options*)"
#OUTPUT="$(ls -d1 ${1:-$defaultdir}options_*)"
#OUTPUT="$(find ${1:-$defaultdir}options_*)"
OUTPUTpwd="$(pwd)"
echo OUTPUT=$OUTPUT
#mkdir output
#cp $pathnames_local pathnames
for i in ${OUTPUT}
do
echo $i
echo expand:
suffix=${i#*options}
echo suffix=$suffix
options_new=$i
output_new=output$suffix
mkdir $output_new
pathnames_new=pathnames$suffix
echo pathnames_new = $pathnames_new
echo pathnames_template = $pathnames_template
# copy template (contains valid AVAILABLE file and winds)
cp $pathnames_template $pathnames_new
# replace options line
sed -i "s/\boptions\b/$options_new/g" $pathnames_new
# replace output line
sed -i "s/\boutput\b/$output_new/g" $pathnames_new
done
pathnames_local=~/repos/flexpart/pathnames
mkdir output
cp $pathnames_local pathnames
#!/bin/bash
#FLEXPART=FLEXPART_8d70e43
# run generate cases
./gen_cases.sh $1
./loop_on_pathnames_laptop.sh
#!/bin/bash
OUTPUTdefault="$(ls -1 pathnames*)"
OUTPUT=${OUTPUTvar:-$OUTPUTdefault}
OUTPUTpwd="$(pwd)"
FLEXPARTdefault=FLEXPART_8d70e43
FLEXPART=${FLEXPARTvar:-$FLEXPARTdefault}
echo $OUTPUT
for i in ${OUTPUT}
do
#FP_slurm_batch_sl=slurm_batch_$i.sl
#FP_slurm_batch_sl=batch_dry_$i.sl
FP_slurm_batch_sl=clrun_full_$i.sl
echo ' ' >> $FP_slurm_batch_sl
#FLEXPART_run_string="$FLEXPART $i -i"
FLEXPART_run_string="$FLEXPART $i"
echo $FLEXPART_run_string >> $FP_slurm_batch_sl
echo $FLEXPART_run_string
done
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment