Commit a0b98bea authored by Jean-Marie Lepioufle's avatar Jean-Marie Lepioufle
Browse files

reduce friednlyr to its core

parent 8b1fd66f
Package: friendlyr
Type: Package
Version: 0.1.2
Version: 0.1.4
Authors@R: c(person("Jean-Marie", "Lepioufle", , "jml@nilu.no", role=c("aut","cre")))
Title: friendlyr
Description: Make remote timeserie dataset easy to get and to work with.
The package focuses on:
1. importing key libraries related to a specific remote dataset.
2. authentication to the related database/db/api.
3. having access to the related database/db/api/embedded dataset.
4. having timeserie dataset in a friendly format.
5. checking the date timeserie.
6. having metadata.
Description: Make the use of algorithms for getting timeseries/metadata easy.
The package aims at making easy the use of diverse libraries from remote or local machines.
One simple syntax for every libraries.
Depends:
R (>= 3.2.0)
Imports:
utils,
timemanip,
osmdata,
tibble
osmdata
Remotes: git::https://git.nilu.no/rspacetimemanip/timemanip.git,
Suggests:
knitr
......
# Generated by roxygen2: do not edit by hand
export(friendlyr)
export(meta)
export(qa)
export(ts)
#' meta
#' meta
#' @param service service
#' @param username username
#' @param password password
#' @param element_id element ids "NO2" "O3", c(...),...
#' @param timeResolution timeResolution time resolution second, minute, houly, daily, monthly, yearly
#' @param station_id station ids "665", c(...),...
#' @param location location
#' @keywords friendlyr
#' @export
#' @examples
#' \dontrun{
#' meta()
#' }
meta <- function(service, username=NULL,password=NULL,element_id=NULL,timeResolution=NULL,station_id=NULL,location=NULL) {
friend <- friendlyr(service=service, username=username,password=password)
res <- friend$meta(element_id=element_id,timeResolution=timeResolution,station_id=station_id,location=location)
res <- tibble::as_tibble(res)
return(res)
}
#' qa
#' qa
#' @param service service
#' @param username username
#' @param password password
#' @param fromDate fromDate from "2016/11/01 00:00:00"
#' @param toDate toDate to "2017/05/01 00:00:00"
#' @param precision precision second, minute, houly, daily, monthly, yearly
#' @param timeResolution timeResolution time resolution second, minute, houly, daily, monthly, yearly
#' @param element_id element ids "NO2" "O3", c(...),...
#' @param station_id station ids "665", c(...),...
#' @param location location
#' @keywords friendlyr
#' @export
#' @examples
#' \dontrun{
#' qa()
#' }
qa <- function(service=NULL, username=NULL,password=NULL,fromDate=NULL,toDate=NULL,precision=NULL,timeResolution=NULL,element_id=NULL,station_id=NULL,location=NULL) {
friend <- friendlyr(service=service, username=username,password=password)
res <- friend$qa(fromDate = fromDate, toDate = toDate, precision=precision,timeResolution=timeResolution,element_id=element_id,station_id=station_id,location=location,COL_DATE=FALSE)
return(res)
}
#' ts
#' ts
#' @param service service
#' @param username username
#' @param password password
#' @param fromDate fromDate from "2016/11/01 00:00:00"
#' @param toDate toDate to "2017/05/01 00:00:00"
#' @param precision precision second, minute, houly, daily, monthly, yearly
#' @param timeResolution timeResolution time resolution second, minute, houly, daily, monthly, yearly
#' @param element_id element ids "NO2" "O3", c(...),...
#' @param station_id station ids "665", c(...),...
#' @param location location
#' @keywords friendlyr
#' @export
#' @examples
#' \dontrun{
#' ts()
#' }
ts <- function(service=NULL, username=NULL,password=NULL,fromDate=NULL,toDate=NULL,precision=NULL,timeResolution=NULL,element_id=NULL,station_id=NULL,location=NULL) {
friend <- friendlyr(service=service, username=username,password=password)
res <- friend$data(fromDate = fromDate, toDate = toDate, precision=precision,timeResolution=timeResolution,element_id=element_id,station_id=station_id,location=location,COL_DATE=FALSE)
return(res)
}
#!/usr/lib/R/bin/Rscript
# Jean-Marie Lepioufle
# NILU - nilu.no
# 2019-02-21
# Get timeseries from a service by using friendlyr
'friendlyr
Usage:
friendlyr.R service <service> fromDate <fromDate> toDate <toDate> precision <precision> timeResolution <timeResolution> element_id <element_id> station_id <station_id> username <username> password <password> result <result> name <name>
conv.R (-h | --help)
conv.R --version
Options:
-h --help Show this screen.
--version Show version.
Outcomes:
timeserie: Timeserie of as a friendlyts data.frame.
' -> doc
#library(docopt)
arguments <- docopt::docopt(doc, version = 'conv version 0.0.1')
###################
# requirement #
###################
suppressPackageStartupMessages(require("friendlyr"))
###################
# init parameters #
###################
cat("Initialising parameters... \n")
service <- tryCatch({
res <- as.character(arguments$service)
if(!(res %in% c("luftlyr","innoly","frostlyr","d.luft.oslyr","d.eipalyr","d.eimetlyr"))) stop("'service' need to be one of these: 'luftlyr','innoly','frostlyr','d.luft.oslyr','d.eipalyr','d.eimetlyr'")
res
}, error = function(err) {
print(paste("Error in argument 'service': ",err))
print("It will crash.")
})
fromDate <- tryCatch({
res <- as.character(arguments$fromDate)
# add some filter
res
}, error = function(err) {
print(paste("Error in argument 'fromDate': ",err))
print("It will crash.")
})
toDate <- tryCatch({
res <- as.character(arguments$toDate)
# add some filter
res
}, error = function(err) {
print(paste("Error in argument 'toDate': ",err))
print("It will crash.")
})
precision <- tryCatch({
res <- as.character(arguments$precision)
if(!(res %in% c("second","minute","hourly","daily","monthly","yearly"))) stop("'service' need to be one of these: 'second','minute','hourly','daily','monthly','yearly'")
res
}, error = function(err) {
print(paste("Error in argument 'precision': ",err))
print("It will crash.")
})
timeResolution <- tryCatch({
res <- as.character(arguments$timeResolution)
if(!(res %in% c("second","minute","hourly","daily","monthly","yearly"))) stop("'service' need to be one of these: 'second','minute','hourly','daily','monthly','yearly'")
res
}, error = function(err) {
print(paste("Error in argument 'timeResolution': ",err))
print("It will crash.")
})
element_id <- tryCatch({
res <- as.character(arguments$element_id)
# add some filter
res
}, error = function(err) {
print(paste("Error in argument 'element_id': ",err))
print("It will crash.")
})
station_id <- tryCatch({
res <- as.character(arguments$station_id)
# add some filter
res
}, error = function(err) {
print(paste("Error in argument 'station_id': ",err))
print("It will crash.")
})
username <- tryCatch({
res <- as.character(arguments$username)
# add some filter
res
}, error = function(err) {
print(paste("Error in argument 'username': ",err))
print("It will crash.")
})
password <- tryCatch({
res <- as.character(arguments$password)
# add some filter
res
}, error = function(err) {
print(paste("Error in argument 'password': ",err))
print("It will crash.")
})
# get path of where to store the result
result <- tryCatch({
res <- normalizePath(file.path(as.character(arguments$result),"friendlyr"),mustWork = FALSE)
dir.create(res, showWarnings = FALSE, recursive = TRUE)
res
}, error = function(err) {
print(paste("Error in argument 'result': ",err))
print("'result' get its default value: '~'")
res <- normalizePath(file.path(path.expand('~'),"friendlyr"),mustWork = FALSE)
dir.create(res, showWarnings = FALSE, recursive = TRUE)
return(res)
})
name <- tryCatch({
res <- as.character(arguments$name)
# add some filter
res
}, error = function(err) {
print(paste("Error in argument 'name': ",err))
print("It will crash.")
})
#######################
# get timeseries data #
#######################
df <- friendlyr::ts(service=service,username=username,password=password,
fromDate=fromDate,toDate=toDate,precision=precision,timeResolution=timeResolution,
element_id=element_id,station_id=station_id)
#save
pathfile <- normalizePath(file.path(path, paste0(name,".csv")),mustWork = FALSE)
write.csv(df,file=pathfile,row.names=FALSE)
## Friendlyr in bash
Author: Jean-Marie Lepioufle, [NILU](https://nilu.no).
Date: 2019-02-21.
**Purpose** :
- Gathering long timeseries from different internal and external services.
**Available services** :
- luftkval (NILU): "luftlyr"
- innosense (NILU): "innolyr"
- frost (MET): "frostlyr"
- embedded dataset luftkval.oslo10: "d.luft.oslyr"
- embedded dataset EPA Ireland: "d.eipalyr"
- embedded dataset MetOffice Ireland: "d.eimetlyr"
**Component** :
- friendlyr.sh: run friendlyr.R with possibility of asynchronous loop.
- friendlyr.R: Gather data
**Remarks** :
Friendlyr required installation of libraries from NILU's gitlab.
A full frontend version of friendlyr is available through [rØya](https://git.nilu.no/oya/roya) service
### Pre-requisities
#### Get easily command-line interface through Rscript.
In order to get a command-line interface through Rscript, you need to install docopt packages in R.
```R
install.packages("docopt")
```
#### Get the right packages to run the case study
The package batqa and its dependencies enable to optimize a model.
For our case study, and as suggested in batqa DESCRIPTION, need one to install other packages: metrics, friendlyr, d.luft.oslyr, friendlyts and ranger.
```R
# devtools Version: 2.0.1.9000
# devtools::install_github("r-lib/devtools")
library("devtools")
devtools::install_git("https://git.nilu.no/rfriendlyr/friendlyr.git")
```
#### R script as executables
The path to Rscript listed in the shebang line of the file friendlyr.R needs to be in your PATH variable.
Check the path of your Rscript.
```shell
which Rscript
```
Then paste it into your .bashrc
```shell
echo 'export PATH="$PATH:/usr/lib/R/bin/Rscript"' >> ~/.bashrc
```
The script friendlyr.R needs then to be executable.
```shell
cd </path/to/friendlyr.R>
chmod +x friendlyr.R
```
Then the script friendlyr.R is executable by writing:
```shell
./friendlyr.R -service <service> -fromDate <fromDate> -toDate <toDate> -precision <precision> -timeResolution <timeResolution> -element_id <element_id> -station_id <station_id> -username <username> -password <password> -result <result> -name <name>
```
#### shell script as executable
In order to gather long timeseries of data, the request is splitted. The script friendly.sh focuses on this.
The script friendlyr.sh needs to be executable.
```shell
cd </path/to/friendlyr.sh>
chmod +x friendlyr.sh
```
Then the script friendlyr.sh is executable by writing:
```shell
. ./friendlyr.sh <service> <fromDate> <toDate> <precision> <timeResolution> <element_id> <station_id> <username> <password> <result>
```
### Usage
Parameters <station> and <batea_type> are always arrays.
```shell
# example 1
cd </path/to/bat_oslo.R>
./friendlyr.R -service "frostlyr" -fromDate "2016/11/6 00:00:00" -toDate "2016/11/10 00:00:00" -precision "hourly" -timeResolution "hourly" -element_id "air_temperature" -station_id "SN18700" -username "****" -password "*****" -result "~/" -name "1"
# example 2
cd </path/to/bat_oslo.sh>
element_id=("7")
b_type=("r")
./friendlyr.R -service "frostlyr" -fromDate "2016/11/6 00:00:00" -toDate "2016/11/10 00:00:00" -precision "hourly" -timeResolution "hourly" -element_id "air_temperature" -station_id "SN18700" -username "****" -password "*****" -result "~/" -name "1"
# example 2
station=("7" "11" "9")
b_type=("r")
. ./bat_oslo.sh "1" station b_type "mh_pw" 10 "~/"
# example 3
station=("7" "11" "9")
b_type=("r" "or" "sor")
. ./bat_oslo.sh "1" station b_type "mh_pw" 10 "~/"
```
```R
# dates
timeResolution <- "hourly"
precision <- "hourly"
fromDate <- "2013010100"
toDate <- "2018060100"
tmp <- timemanip::timeserie(timeResolution="monthly",fromPeriod=fromDate,toPeriod=toDate,precision=precision)
from <- tmp$seqPeriod[1:(tmp$nbStep-1)]
to <- tmp$seqPeriod[2:(tmp$nbStep)]
if (length(to)>1) {
to[1:(length(to)-1)] <- timemanip::addition_nonsec(to[1:(length(to)-1)],timeResolution=timeResolution,v=-1)
}
# elements
element_id <- c("sum(precipitation_amount PT1H)",
"wind_from_direction",
"wind_speed",
"surface_air_pressure",
"mean(surface_downwelling_shortwave_flux_in_air PT1H)",
"air_temperature",
"relative_humidity")
# stations
location <- "Oslo, Norway"
frost_meta <- friendlyr::meta(service="frostlyr",element_id=element_id,location=location,username=username,password=password)
station_id <- as.character(frost_meta$ID)
```
% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/meta.R
\name{meta}
\alias{meta}
\title{meta
meta}
\usage{
meta(service, username = NULL, password = NULL, element_id = NULL,
timeResolution = NULL, station_id = NULL, location = NULL)
}
\arguments{
\item{service}{service}
\item{username}{username}
\item{password}{password}
\item{element_id}{element ids "NO2" "O3", c(...),...}
\item{timeResolution}{timeResolution time resolution second, minute, houly, daily, monthly, yearly}
\item{station_id}{station ids "665", c(...),...}
\item{location}{location}
}
\description{
meta
meta
}
\examples{
\dontrun{
meta()
}
}
\keyword{friendlyr}
% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/qa.R
\name{qa}
\alias{qa}
\title{qa
qa}
\usage{
qa(service = NULL, username = NULL, password = NULL,
fromDate = NULL, toDate = NULL, precision = NULL,
timeResolution = NULL, element_id = NULL, station_id = NULL,
location = NULL)
}
\arguments{
\item{service}{service}
\item{username}{username}
\item{password}{password}
\item{fromDate}{fromDate from "2016/11/01 00:00:00"}
\item{toDate}{toDate to "2017/05/01 00:00:00"}
\item{precision}{precision second, minute, houly, daily, monthly, yearly}
\item{timeResolution}{timeResolution time resolution second, minute, houly, daily, monthly, yearly}
\item{element_id}{element ids "NO2" "O3", c(...),...}
\item{station_id}{station ids "665", c(...),...}
\item{location}{location}
}
\description{
qa
qa
}
\examples{
\dontrun{
qa()
}
}
\keyword{friendlyr}
% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/ts.R
\name{ts}
\alias{ts}
\title{ts
ts}
\usage{
ts(service = NULL, username = NULL, password = NULL,
fromDate = NULL, toDate = NULL, precision = NULL,
timeResolution = NULL, element_id = NULL, station_id = NULL,
location = NULL)
}
\arguments{
\item{service}{service}
\item{username}{username}
\item{password}{password}
\item{fromDate}{fromDate from "2016/11/01 00:00:00"}
\item{toDate}{toDate to "2017/05/01 00:00:00"}
\item{precision}{precision second, minute, houly, daily, monthly, yearly}
\item{timeResolution}{timeResolution time resolution second, minute, houly, daily, monthly, yearly}
\item{element_id}{element ids "NO2" "O3", c(...),...}
\item{station_id}{station ids "665", c(...),...}
\item{location}{location}
}
\description{
ts
ts
}
\examples{
\dontrun{
ts()
}
}
\keyword{friendlyr}
---
title: "data.eimet"
output: rmarkdown::html_vignette
vignette: >
%\VignetteIndexEntry{data.eimet}
%\VignetteEngine{knitr::rmarkdown}
%\VignetteEncoding{UTF-8}
---
```{r setup, include = FALSE}
library(friendlyr)
knitr::opts_chunk$set(collapse = T, comment = "#>")
```
## install
```R
library("devtools")
devtools::install_git("https://git.nilu.no/rfriendlyr/friendlyr.git")
devtools::install_git("https://git.nilu.no/rfriendlyr/d.eimetlyr.git")
```
## Get timeseries data
Get data from eimet embedded dataset.
```R
fromDate="2008/03/01 02:00:00"
toDate="2008/03/01 09:00:00"
timeResolution="hourly"
station_id=c(1475,2175)
element_id=c("rain","temp")
location <- NULL