Commit 8f64c75a authored by Rune Åvar Ødegård's avatar Rune Åvar Ødegård
Browse files

Merge branch 'master' of https://git.nilu.no/eea-tools/raven

parents a989c875 481918ac
from flask import Blueprint, request, abort
from flask_login import login_required
from web.helpers.responses import Responses
from .timeseries_handler import TimeseriesHandler
from .observation_handler import ObservationHandler
from web.helpers.model_binder import ModelBinder as Binder
from werkzeug.exceptions import InternalServerError
historical = Blueprint("historical", __name__)
@historical.route("/api/viewer/historical/timeseries", methods=['GET'])
@login_required
def timeseries():
try:
timeseries = TimeseriesHandler.handle()
return Responses.json(timeseries)
except Exception as e:
raise InternalServerError(description=str(e))
@historical.route("/api/viewer/historical/observations", methods=['POST'])
@login_required
def observations():
p = Binder.bind_and_validate(ObservationHandler.get_validation_rules())
try:
observation = ObservationHandler.handle(p)
return Responses.json(observation)
except Exception as e:
raise InternalServerError(description=str(e))
\ No newline at end of file
from io import StringIO
import csv
from flask import Blueprint, request, abort, make_response
from flask_login import login_required
from web.helpers.responses import Responses
from web.api.viewer.historical.timeseries_handler import TimeseriesHandler
from web.api.viewer.historical.observation_handler import ObservationHandler
from web.helpers.model_binder import ModelBinder as Binder
from werkzeug.exceptions import InternalServerError
historical = Blueprint("historical", __name__)
@historical.route("/api/viewer/historical/timeseries", methods=['GET'])
@login_required
def timeseries():
try:
timeseries = TimeseriesHandler.handle()
return Responses.json(timeseries)
except Exception as e:
raise InternalServerError(description=str(e))
@historical.route("/api/viewer/historical/observations", methods=['POST'])
@login_required
def observations():
p = Binder.bind_and_validate(ObservationHandler.get_validation_rules())
try:
observations = ObservationHandler.handle(p)
return Responses.json(observations)
except Exception as e:
raise InternalServerError(description=str(e))
@historical.route("/api/viewer/historical/csv/timeseries", methods=['POST'])
@login_required
def timeseriesCsv():
try:
p = Binder.bind_and_validate(ObservationHandler.get_validation_rules())
observations = ObservationHandler.handleCsv(p)
si = StringIO()
cw = csv.writer(si)
cw.writerows(observations)
output = make_response(si.getvalue())
output.headers["Content-Disposition"] = "attachment; filename=export.csv"
output.headers["Content-type"] = "text/csv"
return output
except Exception as e:
raise InternalServerError(description=str(e))
@historical.route("/api/viewer/historical/csv/pivot/timeseries", methods=['POST'])
@login_required
def timeseriesCsvPivot():
try:
p = Binder.bind_and_validate(ObservationHandler.get_validation_rules())
observations = ObservationHandler.handleCsvPivot(p)
si = StringIO()
cw = csv.writer(si)
cw.writerows(observations)
output = make_response(si.getvalue())
output.headers["Content-Disposition"] = "attachment; filename=export.csv"
output.headers["Content-type"] = "text/csv"
return output
except Exception as e:
raise InternalServerError(description=str(e))
from web.helpers.db import Db
from web.helpers.utils import *
class ObservationHandler:
@staticmethod
def get_validation_rules():
rules = [
{"name": "oc_id", "required": True, "type": str},
{"name": "from", "required": True, "type": str},
{"name": "to", "required": True, "type": str},
{"name": "onlyValidValues", "required": True, "type": bool},
{"name": "viewAsBar", "required": True, "type": bool}
]
return rules
@staticmethod
def handle(p):
sql = """
SELECT
CASE WHEN %(viewAsBar)s THEN 'bar' ELSE 'line' END as type,
s.name || ' - ' || po.notation as name,
array_agg(array[(extract(epoch from (aa.to_time))*1000)::double PRECISION, aa.value::double PRECISION]
order by aa.to_time asc) as data
FROM stations s, sampling_points sp, observing_capabilities oc, eea_pollutants po,
(
SELECT o.from_time, o.to_time,
CASE WHEN (o.validation_flag < 1 OR o.value = -9900) AND %(onlyValidValues)s THEN
NULL ELSE o.value END as value, oc.id
FROM observations o, observing_capabilities oc
WHERE 1=1
AND oc.sampling_point_id = o.sampling_point_id
AND oc.id in %(oc_id_tup)s
AND o.from_time >= %(from)s
AND o.to_time < %(to)s
) aa
WHERE 1=1
AND aa.id = oc.id
AND oc.sampling_point_id = sp.id
AND sp.station_id = s.id
AND oc.pollutant = po.uri
GROUP by s.name,sp.id, oc.pollutant, oc.id, po.notation
"""
# Make sure its an array, not just a string
if not isinstance(p["oc_id"], list):
p["oc_id"] = p["oc_id"].split(',')
p["oc_id_tup"] = tuple(p["oc_id"])
observation = Db.fetchall(sql, p)
return observation
from web.helpers.db import Db
from web.helpers.utils import *
from datetime import *
from dateutil.relativedelta import *
from dateutil.rrule import *
from collections import OrderedDict
from web.helpers.mean import Mean
from web.helpers.mean import MeanType
from web.helpers.mean import MeanValue
from web.helpers.mapper import Mapper
class ObservationHandler:
@staticmethod
def get_validation_rules():
rules = [
{"name": "oc_id", "required": True, "type": str},
{"name": "from", "required": True, "type": str},
{"name": "to", "required": True, "type": str},
{"name": "onlyValidValues", "required": True, "type": bool},
{"name": "meantype", "required": True, "type": int},
{"name": "coverage", "required": True, "type": int}
]
return rules
@staticmethod
def handle(p):
# Make sure its an array, not just a string
if not isinstance(p["oc_id"], list):
p["oc_id"] = p["oc_id"].split(',')
p["oc_id_tup"] = tuple(p["oc_id"])
observations = ObservationHandler.get_observations(p)
values = ObservationHandler.group_observations(p["oc_id"], observations)
return sorted(values, key=lambda i: i['station'])
@staticmethod
def group_observations(ids, observations):
values = []
for oc in ids:
v = list(filter(lambda x: x.OCId == oc, observations))
t = v[0]
data = [{"datetime": d.DateTime, "val": d.Val} for d in v]
o = {"station": t.Station, "component": t.Component, "unit": t.Unit, "timestep": t.Timestep, "values": data}
values.append(o)
return values
@staticmethod
def get_observations(p):
observations = []
if p["meantype"] == -1:
observations = ObservationHandler.get_originals(p)
else:
observations = Mean.Aggregate(MeanType(p["meantype"]), p["oc_id_tup"], p["from"], p["to"], p["coverage"], 3, 3, True)
return sorted(observations, key=lambda i: i.DateTime)
@staticmethod
def get_originals(p):
sql = """
SELECT
sta.name "Station",
po.notation "Component",
ti.timestep "Timestep",
con.notation "Unit",
oc.sampling_point_id "SamplingPointId",
oc.id "OCId",
100 "Coverage",
1 "Cnt",
o.to_time "DateTime",
CASE
WHEN (o.validation_flag < 1 OR o.value = -9900) AND %(onlyValidValues)s THEN NULL
ELSE ROUND(o.value,3)::double PRECISION
END "Val"
FROM observations o, stations sta, sampling_points spo, observing_capabilities oc, eea_pollutants po, eea_times ti, eea_concentrations con
WHERE 1=1
and sta.id = spo.station_id
and spo.id = oc.sampling_point_id
and oc.pollutant = po.uri
and oc.timestep = ti.id
and oc.concentration = con.id
AND oc.sampling_point_id = o.sampling_point_id
AND oc.id in %(oc_id_tup)s
AND o.from_time >= %(from)s
AND o.from_time < %(to)s
"""
rows = Db.fetchall(sql, p)
return Mapper.map_list_of_dict(rows, MeanValue)
@staticmethod
def handleCsv(p):
# Make sure its an array, not just a string
if not isinstance(p["oc_id"], list):
p["oc_id"] = p["oc_id"].split(',')
p["oc_id_tup"] = tuple(p["oc_id"])
observations = ObservationHandler.get_observations(p)
result = []
keys = ["SamplingPointId", "Station", "Component", "Unit", "Timestep", "DateTime", "Value", "Coverage", "Count"]
result.append(keys)
for o in observations:
values = [o.SamplingPointId, o.Station, o.Component, o.Unit, o.Timestep, o.DateTime, o.Val, o.Coverage, o.Cnt]
result.append(values)
return result
@staticmethod
def handleCsvPivot(p):
# Make sure its an array, not just a string
if not isinstance(p["oc_id"], list):
p["oc_id"] = p["oc_id"].split(',')
p["oc_id_tup"] = tuple(p["oc_id"])
observations = ObservationHandler.get_observations(p)
series = ObservationHandler.group_observations(p["oc_id"], observations)
dates = list(set([o.DateTime for o in observations]))
dates.sort()
result = [{"datetime": o} for o in dates]
for serie in series:
if serie:
for observation in serie["values"]:
for row in result:
if row["datetime"] == observation["datetime"]:
row[serie["station"] + " " + serie["component"]] = None if observation["val"] == None else str(observation["val"])
break
csvresult = []
csvkeys = []
firstrow = result[0]
for key in firstrow.keys():
csvkeys.append(key)
csvresult.append(csvkeys)
for items in result:
values = []
for val in items.values():
values.append(val)
csvresult.append(values)
return csvresult
from web.helpers.db import Db
class TimeseriesHandler:
@staticmethod
def handle():
sql = """
SELECT
aa.value,
CONCAT(aa.networkname, ', ', aa.name,', ', aa.pollutant) as label,
to_char(aa.fromtime, 'YYYY-MM-DD"T"HH24:MI:SS') as fromtime,
to_char(aa.totime, 'YYYY-MM-DD"T"HH24:MI:SS') as totime
FROM
(
SELECT sp.id as sp, oc.id as value, s.name, po.notation pollutant, n.name networkname, oc.from_time as fromtime, oc.to_time as totime
FROM
stations s,
sampling_points sp,
observing_capabilities oc,
eea_pollutants po,
networks n
WHERE 1=1
and s.id = sp.station_id
and n.id = s.network_id
and sp.id = oc.sampling_point_id
and oc.pollutant = po.uri
and oc.from_time is not null
and oc.to_time is not null
"""
sql = sql + Db.add_network_ids_requirement()
sql = sql + """
GROUP by s.name, sp.id, oc.pollutant, n.name,
oc.id, po.notation, oc.from_time, oc.to_time
) aa
"""
timeseries = Db.fetchall(sql)
return timeseries
from web.helpers.db import Db
class TimeseriesHandler:
@staticmethod
def handle():
sql = """
SELECT
aa.value,
CONCAT(aa.networkname, ', ', aa.name,', ', aa.pollutant) as label,
to_char(aa.fromtime, 'YYYY-MM-DD"T"HH24:MI:SS') as fromtime,
to_char(aa.totime, 'YYYY-MM-DD"T"HH24:MI:SS') as totime
FROM
(
SELECT sp.id as sp, oc.id as value, s.name, po.notation pollutant, n.name networkname, oc.from_time as fromtime, oc.to_time as totime
FROM
stations s,
sampling_points sp,
observing_capabilities oc,
eea_pollutants po,
networks n
WHERE 1=1
and s.id = sp.station_id
and n.id = s.network_id
and sp.id = oc.sampling_point_id
and oc.pollutant = po.uri
and oc.from_time is not null
and oc.to_time is not null
"""
sql = sql + Db.add_network_ids_requirement()
sql = sql + """
GROUP by s.name, sp.id, oc.pollutant, n.name,
oc.id, po.notation, oc.from_time, oc.to_time
) aa
"""
timeseries = Db.fetchall(sql)
return timeseries
\ No newline at end of file
{
"name": "client",
"version": "0.1.0",
"private": true,
"scripts": {
"serve": "vue-cli-service serve",
"build": "vue-cli-service build"
},
"dependencies": {
"@fortawesome/fontawesome-svg-core": "^1.2.19",
"@fortawesome/free-solid-svg-icons": "^5.9.0",
"@fortawesome/vue-fontawesome": "^0.1.6",
"@handsontable/vue": "^4.1.0",
"apexcharts": "^3.8.5",
"axios": "^0.19.0",
"core-js": "^2.6.5",
"d3": "^5.9.7",
"handsontable": "^7.1.1",
"js-file-download": "^0.4.8",
"popper.js": "^1.15.0",
"pretty-checkbox-vue": "^1.1.9",
"v-click-outside": "^2.1.3",
"v-tooltip": "^2.0.2",
"vue": "^2.6.10",
"vue-apexcharts": "^1.4.0",
"vue-ctk-date-time-picker": "^2.1.1",
"vue-property-decorator": "^8.2.1",
"vue-router": "^3.0.6",
"vue-select": "^3.9.5",
"vue-toastify": "^0.4.4",
"xml2js": "^0.4.19"
},
"devDependencies": {
"@fullhuman/postcss-purgecss": "^1.2.0",
"@ky-is/vue-cli-plugin-tailwind": "^2.0.0",
"@vue/cli-plugin-babel": "^3.8.0",
"@vue/cli-service": "^3.8.0",
"node-sass": "^4.9.0",
"postcss-preset-env": "^6.6.0",
"sass-loader": "^7.1.0",
"tailwindcss": "^1.0.1",
"vue-template-compiler": "^2.6.10"
}
}
{
"name": "client",
"version": "0.1.0",
"private": true,
"scripts": {
"serve": "vue-cli-service serve",
"build": "vue-cli-service build"
},
"dependencies": {
"@fortawesome/fontawesome-svg-core": "^1.2.19",
"@fortawesome/free-solid-svg-icons": "^5.9.0",
"@fortawesome/vue-fontawesome": "^0.1.6",
"@handsontable/vue": "^4.1.1",
"apexcharts": "^3.23.1",
"axios": "^0.19.0",
"core-js": "^2.6.5",
"d3": "^5.9.7",
"handsontable": "^7.4.2",
"js-file-download": "^0.4.8",
"popper.js": "^1.15.0",
"pretty-checkbox-vue": "^1.1.9",
"v-click-outside": "^2.1.3",
"v-tooltip": "^2.0.2",
"vue": "^2.6.10",
"vue-apexcharts": "^1.6.0",
"vue-ctk-date-time-picker": "^2.1.1",
"vue-property-decorator": "^8.2.1",
"vue-router": "^3.0.6",
"vue-select": "^3.9.5",
"vue-toastify": "^0.4.4",
"xml2js": "^0.4.19"
},
"devDependencies": {
"@fullhuman/postcss-purgecss": "^1.2.0",
"@ky-is/vue-cli-plugin-tailwind": "^2.0.0",
"@vue/cli-plugin-babel": "^3.8.0",
"@vue/cli-service": "^3.8.0",
"node-sass": "^5.0.0",
"postcss-preset-env": "^6.6.0",
"sass-loader": "^7.1.0",
"tailwindcss": "^1.0.1",
"vue-template-compiler": "^2.6.10"
}
}
<template>
<div
class="w-full bg-white pl-4 py-4 font-medium shadow-sm select-none flex justify-between px-4"
>
<div>
<router-link to="/">
<font-awesome-icon icon="crow" class="text-xl font-medium text-pastel-green-600"/>
<h1 class="ml-2 tracking-widest">Raven</h1>
</router-link>
</div>
<div class="text-xs self-center">
<div
class="font-medium text-gray-600 flex hover:cursor-pointer hover:text-gray-700 text-center"
@click="handleSubmit"
>
<div v-if="me" class="self-center mr-2">
<font-awesome-icon
v-if="me.role == 'Administrator'"
icon="shield-alt"
class="text-lg text-pastel-red self-center pt-1"
/>
<div class="w-full bg-white pl-4 py-4 font-medium border-b select-none flex justify-between px-4">
<div>
<router-link to="/">
<font-awesome-icon icon="crow" class="text-xl font-medium text-pastel-green-600" />
<h1 class="ml-2 tracking-widest">Raven</h1>
</router-link>
</div>
<div class="text-xs self-center">
<div class="font-medium text-gray-600 flex hover:cursor-pointer hover:text-gray-700 text-center" @click="handleSubmit">
<div v-if="me" class="self-center mr-2">
<font-awesome-icon v-if="me.role == 'Administrator'" icon="shield-alt" class="text-lg text-pastel-red self-center pt-1" />
{{ me.name }}
{{ me.name }}
</div>
<font-awesome-icon icon="sign-out-alt" class="text-lg self-center" />
</div>
</div>
<font-awesome-icon icon="sign-out-alt" class="text-lg self-center"/>
</div>
</div>
</div>
</template>
<script>
import UsersService from "@/helpers/services/users.service";
export default {
name: "LHeader",
data() {
return {
me: undefined
};
},
methods: {
async handleSubmit() {
await UsersService.signOut();
window.location.href = "/login";
}
},
async created() {
try {
this.me = await UsersService.me();
} catch (e) {}
}
name: "LHeader",
data() {
return {
me: undefined,
};
},
methods: {
async handleSubmit() {
await UsersService.signOut();
window.location.href = "/login";
},
},
async created() {
try {
this.me = await UsersService.me();
} catch (e) {}
},
};
</script>
<style lang="postcss" scoped>
h1 {
@apply text-lg font-bold inline-block;
@apply text-lg font-bold inline-block;
}
a {
text-decoration: none;
@apply text-pastel-black;
text-decoration: none;
@apply text-pastel-black;
}
</style>
<template>
<div class="flex w-full p-2 bg-yellow-200 border border-yellow-400" v-if="active">
<slot></slot>
</div>
</template>
<script>
export default {
name: "LInfoBox",
props: ["active"]
};
</script>
\ No newline at end of file
<template>
<div class="flex w-full p-2 bg-yellow-200 border border-yellow-400" v-if="active">
<slot></slot>
</div>
</template>
<script>
export default {
name: "LInfoBox",