Commit af65de2b authored by Jean-Marie Lepioufle's avatar Jean-Marie Lepioufle
Browse files

clean code

parent 2513eec9
......@@ -6,13 +6,11 @@ CLI client to get MET timeseries at SPO from backend Frost blob
## Usage
```shell
./frostb.R -w path/to/workdir -c frost.json -s session.json
./frost.R -w path/to/workdir -c frost.json -s session.json
```
## Installation
frost requires frost blob as backend (See folder 'data' in repository 'backend' [documentation](https://git.nilu.no/blob/backend/-/tree/master/data)).
frost CLI requires R v4.1.2
Instruction for installation
......@@ -20,8 +18,8 @@ Instruction for installation
cd ~/
mkdir tmp_frost
cd tmp_frost
git clone git@git.nilu.no:blob/core-client/frostblobs.git
git clone git@git.nilu.no:blob/cli/frost.git
git clone git@git.nilu.no:aqdl/blobs/core-client/frostblobs.git
git clone git@git.nilu.no:aqdl/blobs/cli/frost.git
sudo apt-get install libsodium-dev
......
......@@ -4,12 +4,12 @@
# NILU - nilu.no
# 2022-02-10
'frostb
'frost
Usage:
frostb -w <workdir> -c <config> -s <sessions>
frostb (-h | --help)
frostb --version
frost -w <workdir> -c <config> -s <sessions> [-j <join>]
frost (-h | --help)
frost --version
Options:
-h --help Show this screen.
......@@ -21,7 +21,7 @@ Outcomes:
' -> doc
library(docopt)
arguments <- docopt::docopt(doc, version = 'frostb v0.0.1')
arguments <- docopt::docopt(doc, version = 'frost v0.0.1')
###################
......@@ -56,13 +56,26 @@ sessions <- tryCatch({
if (file.exists(file.path(confdir,as.character(arguments$sessions)))){
res <- jsonlite::fromJSON(file.path(confdir,as.character(arguments$sessions)), simplifyVector = TRUE)
} else {
res <- list(frostb="")
res <- list(frost="")
}
res
}, error = function(err) {
stop(paste("Error in argument 'sessions': ",arguments$config,err))
})
JOIN <- tryCatch({
join <- as.character(arguments$join)
if (length(join) == 0) {join <- "False"}
res <- switch(join,
"True" = TRUE,
"False" = FALSE,
stop("'argument' need either to be 'True' or 'False'"))
res
}, error = function(err) {
stop(paste("Error in argument 'join': ",arguments$join,err))
})
# session directory to store the result
key <- paste0(sodium::random(10),collapse="")
sessiondir <- normalizePath(file.path(as.character(arguments$workdir),"session", key,"tab"),mustWork = FALSE)
......@@ -70,17 +83,17 @@ dir.create(sessiondir, showWarnings = FALSE, recursive = TRUE)
# init parameters
fromDateTime <- conf$frostb$fromDateTime
toDateTime <- conf$frostb$toDateTime
elementIDs <- conf$frostb$elementIDs
timeAggregation <- conf$frostb$timeAggregation
west <- conf$frostb$west
south <- conf$frostb$south
east <- conf$frostb$east
north <- conf$frostb$north
stationIDs <- conf$frostb$stationIDs
path_metadata <- conf$frostb$path_metadata
path_metadata_url <- conf$frostb$path_metadata_url
fromDateTime <- conf$frost$fromDateTime
toDateTime <- conf$frost$toDateTime
elementIDs <- conf$frost$elementIDs
timeAggregation <- conf$frost$timeAggregation
west <- conf$frost$west
south <- conf$frost$south
east <- conf$frost$east
north <- conf$frost$north
stationIDs <- conf$frost$stationIDs
path_metadata <- conf$frost$path_metadata
path_metadata_url <- conf$frost$path_metadata_url
##################################
##################################
......@@ -95,8 +108,6 @@ cat("Frost data \n")
cat("-- Get data from frost blobs \n")
hello <- frostblob::frostblob(path_metadata,path_metadata_url)
getDTthreads(verbose = getOption("datatable.verbose"))
meta <- hello$meta(stationIDs=stationIDs,west=west,south=south,east=east,north=north,elementIDs=elementIDs)
frost <- hello$ts(fromDateTime=fromDateTime,
......@@ -113,10 +124,10 @@ cat("-- SPO timeseries preparation \n")
frost <- cbind(frost,frost[,.(data.table::IDateTime(date))])
frost_aggreg <- switch(timeAggregation,
"hourly" = frost[, .(value = mean(value,na.rm=TRUE)), by = list(ID = ID, year = data.table::year(idate),month = data.table::month(idate), dom = data.table::mday(idate),hour = data.table::hour(itime))],
"daily" = frost[, .(value = mean(value,na.rm=TRUE)), by = list(ID = ID, year = data.table::year(idate),month = data.table::month(idate), dom = data.table::mday(idate))],
"monthly" = frost[, .(value = mean(value,na.rm=TRUE)), by = list(ID = ID, year = data.table::year(idate),month = data.table::month(idate))],
"yearly" = frost[, .(value = mean(value,na.rm=TRUE)), by = list(ID = ID, year = data.table::year(idate))],
"hourly" = frost[, .(value = mean(value,na.rm=TRUE)), by = list(ID = ID, variable = CO_NAME, year = data.table::year(idate),month = data.table::month(idate), dom = data.table::mday(idate),hour = data.table::hour(itime))],
"daily" = frost[, .(value = mean(value,na.rm=TRUE)), by = list(ID = ID, variable = CO_NAME, year = data.table::year(idate),month = data.table::month(idate), dom = data.table::mday(idate))],
"monthly" = frost[, .(value = mean(value,na.rm=TRUE)), by = list(ID = ID, variable = CO_NAME, year = data.table::year(idate),month = data.table::month(idate))],
"yearly" = frost[, .(value = mean(value,na.rm=TRUE)), by = list(ID = ID, variable = CO_NAME, year = data.table::year(idate))],
stop("unknown parameter 'timeAggregation'")
)
......@@ -140,14 +151,26 @@ if (timeAggregation == "hourly" || timeAggregation == "daily" ) {
}
frost_aggreg[,c("date"):=NULL]
frost_aggreg[,c("time"):=NULL]
if (timeAggregation == "hourly") frost_aggreg[,c("time"):=NULL]
##############################################
# replace column called value by element_id #
##############################################
cat("-- replace column called value by element_id \n")
names(frost_aggreg)[names(frost_aggreg)=="value"] <- elementIDs
##############################################################
# spread: format [ID , year , month , dom , dow , hour , ... ,$variable] #
##############################################################
cat("-- format [ID , year , month , dom , dow , hour , ... ,$variable] \n")
frost_aggreg <- switch(timeAggregation,
"hourly" = data.table::dcast(frost_aggreg, ID + year + month + dom + hour ~ variable , value.var = "value"),
"daily" = data.table::dcast(frost_aggreg, ID + year + month + dom ~ variable, value.var = "value"),
"monthly" = data.table::dcast(frost_aggreg, ID + year + month ~ variable, value.var = "value"),
"yearly" = data.table::dcast(frost_aggreg, ID + year ~ variable, value.var = "value"),
stop("unknown parameter 'timeAggregation'")
)
#################################################
#################################################
## join timeserie if sessions["data"] not null ##
#################################################
#################################################
# join by date elements
switch(timeAggregation,
"hourly" = data.table::setkeyv(frost_aggreg, c("ID","year", "month","dom","hour")),
"daily" = data.table::setkeyv(frost_aggreg, c("ID","year", "month","dom")),
......@@ -156,25 +179,37 @@ switch(timeAggregation,
stop("unknown parameter 'timeAggregation'")
)
if (JOIN && !is.null(sessions$data)){
cat("-- join timeserie with existing 'data' \n")
filepath <- normalizePath(file.path(as.character(arguments$workdir),"session", sessions["data"],"tab","dataset.csv"),mustWork = TRUE)
extra <- data.table::fread(filepath)
frost_aggreg <- frost_aggreg[extra]
}
#########################
#########################
## update session file ##
#########################
#########################
cat("-- update session file \n")
if (JOIN && !is.null(sessions$data)){
sessions["frost"] <- sessions["data"] <- as.character(key)
sessions_json <- jsonlite::toJSON(sessions)
write(sessions_json, file.path(confdir,"sessions.json"))
} else{
new_session <- list(frost=key,data=key)
sessions_json <- jsonlite::toJSON(new_session)
write(sessions_json, file.path(confdir,"sessions.json"))
}
sessions["frostb"] <- sessions["data"] <- as.character(key)
sessions_json <- jsonlite::toJSON(sessions)
write(sessions_json, file.path(confdir,"sessions.json"))
#######################################
# SAVE dataset as a RDA and CSV file #
#######################################
cat("-- saving dataset \n")
save(frost_aggreg,file=file.path(sessiondir,"dataset.rda"),compress="xz")
utils::write.csv(frost_aggreg,file=file.path(sessiondir,"dataset.csv"))
save(meta,file=file.path(sessiondir,"metadata.rda"),compress="xz")
utils::write.csv(meta,file=file.path(sessiondir,"metadata.csv"))
save(meta,file=file.path(sessiondir,"metadata.rda"),compress="xz")
utils::write.csv(meta,file=file.path(sessiondir,"metadata.csv"))
cat("-- frostb completed \n")
utils::write.csv(frost_aggreg,file=file.path(sessiondir,"dataset.csv"),row.names = FALSE)
utils::write.csv(meta,file=file.path(sessiondir,"metadata.csv"),row.names = FALSE)
utils::write.csv(meta,file=file.path(sessiondir,"metadata.csv"),row.names = FALSE)
cat("-- frost completed \n")
{"frostb":{
{"frost":{
"fromDateTime" : "2019-01-01 00:00:00",
"toDateTime" : "2019-12-31 23:00:00",
"timeAggregation" : "hourly",
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment