Maintenance is scheduled between 16:00 and 23:59 CEST (14:00 and 21:59 UTC) on Thursday 2021-10-28. The system may be unavailable at any time during this timeframe. Please plan accordingly.

Commit 0742876e authored by Jean-Marie Lepioufle's avatar Jean-Marie Lepioufle
Browse files

DNN_on_edge

parent cfae629b
{ "sessiondir": {
"path": ""
},
"data" : {
"class" : "class_ts_none",
"path": "",
"date": ["Year","Month","Day","Hour","Minute","Second"],
"id_start": null,
"id_stop": null,
"scaling": "MinMaxScaler",
"target" : {"target": null},
"features" : {"features": ["NO2mod_I_act","NO2mod_I_m128"], "features_cyclic": ["Minute","Second"], "history_length": 0}
},
"pred_param" : {
"model" : {"path":"", "name":""},
"target" : {"target": ["NO2"], "gap_length": 0, "horizon_length": 0}
}
}
{ "sessiondir": {
"path": ""
},
"data" : {
"class" : "class_ts_none",
"path": "",
"date": ["Year","Month","Day","Hour","Minute","Second"],
"id_start": null,
"id_stop": null,
"scaling": "MinMaxScaler",
"target" : {"target": ["NO2"], "gap_length": 0, "horizon_length": 0},
"features" : {"features": ["NO2mod_I_act","NO2mod_I_m128"], "features_cyclic": ["Minute","Second"], "history_length": 0}
},
"model" : {
"name": "mlp",
"param":{"mlp_input_size":6,
"mlp_hidden_size":21,
"mlp_output_size":1
}
},
"training_param" : {
"criterion":["smooth_l1_loss"],
"optimizer": {"name": "adam","param":{"lr": 1.0e-5}},
"takes_target": false,
"forward_params": {},
"kfolds": 2,
"epochs": 3,
"batch_size":32
},
"eval_param" : {
"criterion":["rmse","r2","bias"],
"batch_size":32
}
}
,Year,Month,Day,Wday,Hour,Minute,Second,SN,Lat,Lon,Main_I_act,Main_I_m128,Main_I_max,Main_I_mea,Main_I_min,Main_V_act,Main_V_m128,Main_V_max,Main_V_mea,Main_V_min,MeasMod_I_act,MeasMod_I_m128,MeasMod_I_max,MeasMod_I_mea,MeasMod_I_min,NO2,NO2mod_I_act,NO2mod_I_m128,NO2mod_I_max,NO2mod_I_mea,NO2mod_I_min,P,PM0.5_cnt,PM1.0_cnt,PM1.0_conc,PM10_cnt,PM10_conc,PM2.5_cnt,PM2.5_conc,PM4_cnt,PM4_conc,PMmod_I_act,PMmod_I_m128,PMmod_I_max,PMmod_I_mea,PMmod_I_min,RH,Radio_I_act,Radio_I_m128,Radio_I_max,Radio_I_mea,Radio_I_min,Radio_V_act,Radio_V_m128,Radio_V_max,Radio_V_mea,Radio_V_min,System_V_act,System_V_m128,System_V_max,System_V_mea,System_V_min,TPS,T_NO2,T_P,USBsocket_I_act,USBsocket_I_m128,USBsocket_I_max,USBsocket_I_mea,USBsocket_I_min,timestamp_act,timestamp_m128,timestamp_max,timestamp_mea,timestamp_min
0,2021,7,8,4,17,19,2,51820010245,5422.252808,1836.837387,629582.0,593187.0,727954.0,587129.0,543877.0,4984.0,4996.0,5016.0,4998.0,4953.0,3692.0,3703.0,3759.0,3671.0,3358.0,621.0,50.0,931.0,5647.0,405.0,0.0,1016.99,22.0,26.0,3.0,26.0,3.0,26.0,3.0,26.0,3.0,451.0,8500.0,101947.0,3509.0,334.0,46.0,88362.0,92753.0,427709.0,92157.0,82463.0,5011.0,5013.0,5030.0,5015.0,4941.0,3270.0,3271.0,3274.0,3271.0,3268.0,349.0,29.0,31.08,156823.0,156095.0,160014.0,156866.0,154701.0,2697.0,2697.0,2697.0,2697.0,2697.0
HAPADS sensors measurement. Press [CTRL+C] twice to stop.
Day,Month,Year,Hour,Minute,Second,Wday[Sun=0,Mon=1,...],SN,Element,Unit,Value,Lat[ddmm.mmmmmm,dd 00-89 (degree), mm.mmmmmm 00.000000-59.999999 (minute)],Lon[ddmm.mmmmmm,dd 00-89 (degree), mm.mmmmmm 00.000000-59.999999 (minute)]
08,07,2021,17,19,02,4,051820010245,NO2,ppb,621,5422.252808,01836.837387
08,07,2021,17,19,02,4,051820010245,T_NO2,C,29,5422.252808,01836.837387
08,07,2021,17,19,02,4,051820010245,RH,%,46,5422.252808,01836.837387
08,07,2021,17,19,02,4,051820010245,P,mbar,1016.99,5422.252808,01836.837387
08,07,2021,17,19,02,4,051820010245,T_P,C,31.08,5422.252808,01836.837387
08,07,2021,17,19,02,4,051820010245,PM1.0_conc,ug/m3,3,5422.252808,01836.837387
08,07,2021,17,19,02,4,051820010245,PM2.5_conc,ug/m3,3,5422.252808,01836.837387
08,07,2021,17,19,02,4,051820010245,PM4_conc,ug/m3,3,5422.252808,01836.837387
08,07,2021,17,19,02,4,051820010245,PM10_conc,ug/m3,3,5422.252808,01836.837387
08,07,2021,17,19,02,4,051820010245,PM0.5_cnt,#/cm3,22,5422.252808,01836.837387
08,07,2021,17,19,02,4,051820010245,PM1.0_cnt,#/cm3,26,5422.252808,01836.837387
08,07,2021,17,19,02,4,051820010245,PM2.5_cnt,#/cm3,26,5422.252808,01836.837387
08,07,2021,17,19,02,4,051820010245,PM4_cnt,#/cm3,26,5422.252808,01836.837387
08,07,2021,17,19,02,4,051820010245,PM10_cnt,#cm3,26,5422.252808,01836.837387
08,07,2021,17,19,02,4,051820010245,TPS,nm,349,5422.252808,01836.837387
08,07,2021,17,19,02,4,051820010245,timestamp_act,12.8sec,2697,5422.252808,01836.837387
08,07,2021,17,19,02,4,051820010245,Radio_V_act,mV,5011,5422.252808,01836.837387
08,07,2021,17,19,02,4,051820010245,Radio_I_act,uA,88362,5422.252808,01836.837387
08,07,2021,17,19,02,4,051820010245,Main_V_act,mV,4984,5422.252808,01836.837387
08,07,2021,17,19,02,4,051820010245,Main_I_act,uA,629582,5422.252808,01836.837387
08,07,2021,17,19,02,4,051820010245,System_V_act,mV,3270,5422.252808,01836.837387
08,07,2021,17,19,02,4,051820010245,MeasMod_I_act,uA,3692,5422.252808,01836.837387
08,07,2021,17,19,02,4,051820010245,USBsocket_I_act,uA,156823,5422.252808,01836.837387
08,07,2021,17,19,02,4,051820010245,PMmod_I_act,uA,451,5422.252808,01836.837387
08,07,2021,17,19,02,4,051820010245,NO2mod_I_act,uA,50,5422.252808,01836.837387
08,07,2021,17,19,02,4,051820010245,timestamp_min,12.8sec,2697,5422.252808,01836.837387
08,07,2021,17,19,02,4,051820010245,Radio_V_min,mV,4941,5422.252808,01836.837387
08,07,2021,17,19,02,4,051820010245,Radio_I_min,uA,82463,5422.252808,01836.837387
08,07,2021,17,19,02,4,051820010245,Main_V_min,mV,4953,5422.252808,01836.837387
08,07,2021,17,19,02,4,051820010245,Main_I_min,uA,543877,5422.252808,01836.837387
08,07,2021,17,19,02,4,051820010245,System_V_min,mV,3268,5422.252808,01836.837387
08,07,2021,17,19,02,4,051820010245,MeasMod_I_min,uA,3358,5422.252808,01836.837387
08,07,2021,17,19,02,4,051820010245,USBsocket_I_min,uA,154701,5422.252808,01836.837387
08,07,2021,17,19,02,4,051820010245,PMmod_I_min,uA,334,5422.252808,01836.837387
08,07,2021,17,19,02,4,051820010245,NO2mod_I_min,uA,0,5422.252808,01836.837387
08,07,2021,17,19,02,4,051820010245,timestamp_m128,12.8sec,2697,5422.252808,01836.837387
08,07,2021,17,19,02,4,051820010245,Radio_V_m128,mV,5013,5422.252808,01836.837387
08,07,2021,17,19,02,4,051820010245,Radio_I_m128,uA,92753,5422.252808,01836.837387
08,07,2021,17,19,02,4,051820010245,Main_V_m128,mV,4996,5422.252808,01836.837387
08,07,2021,17,19,02,4,051820010245,Main_I_m128,uA,593187,5422.252808,01836.837387
08,07,2021,17,19,02,4,051820010245,System_V_m128,mV,3271,5422.252808,01836.837387
08,07,2021,17,19,02,4,051820010245,MeasMod_I_m128,uA,3703,5422.252808,01836.837387
08,07,2021,17,19,02,4,051820010245,USBsocket_I_m128,uA,156095,5422.252808,01836.837387
08,07,2021,17,19,02,4,051820010245,PMmod_I_m128,uA,8500,5422.252808,01836.837387
08,07,2021,17,19,02,4,051820010245,NO2mod_I_m128,uA,931,5422.252808,01836.837387
08,07,2021,17,19,02,4,051820010245,timestamp_mea,12.8sec,2697,5422.252808,01836.837387
08,07,2021,17,19,02,4,051820010245,Radio_V_mea,mV,5015,5422.252808,01836.837387
08,07,2021,17,19,02,4,051820010245,Radio_I_mea,uA,92157,5422.252808,01836.837387
08,07,2021,17,19,02,4,051820010245,Main_V_mea,mV,4998,5422.252808,01836.837387
08,07,2021,17,19,02,4,051820010245,Main_I_mea,uA,587129,5422.252808,01836.837387
08,07,2021,17,19,02,4,051820010245,System_V_mea,mV,3271,5422.252808,01836.837387
08,07,2021,17,19,02,4,051820010245,MeasMod_I_mea,uA,3671,5422.252808,01836.837387
08,07,2021,17,19,02,4,051820010245,USBsocket_I_mea,uA,156866,5422.252808,01836.837387
08,07,2021,17,19,02,4,051820010245,PMmod_I_mea,uA,3509,5422.252808,01836.837387
08,07,2021,17,19,02,4,051820010245,NO2mod_I_mea,uA,405,5422.252808,01836.837387
08,07,2021,17,19,02,4,051820010245,timestamp_max,12.8sec,2697,5422.252808,01836.837387
08,07,2021,17,19,02,4,051820010245,Radio_V_max,mV,5030,5422.252808,01836.837387
08,07,2021,17,19,02,4,051820010245,Radio_I_max,uA,427709,5422.252808,01836.837387
08,07,2021,17,19,02,4,051820010245,Main_V_max,mV,5016,5422.252808,01836.837387
08,07,2021,17,19,02,4,051820010245,Main_I_max,uA,727954,5422.252808,01836.837387
08,07,2021,17,19,02,4,051820010245,System_V_max,mV,3274,5422.252808,01836.837387
08,07,2021,17,19,02,4,051820010245,MeasMod_I_max,uA,3759,5422.252808,01836.837387
08,07,2021,17,19,02,4,051820010245,USBsocket_I_max,uA,160014,5422.252808,01836.837387
08,07,2021,17,19,02,4,051820010245,PMmod_I_max,uA,101947,5422.252808,01836.837387
08,07,2021,17,19,02,4,051820010245,NO2mod_I_max,uA,5647,5422.252808,01836.837387
\ No newline at end of file
This source diff could not be displayed because it is too large. You can view the blob instead.
This source diff could not be displayed because it is too large. You can view the blob instead.
## Purpose
Build up a pipeline from training a DNN model to its optimization and conversion into an Intermediate Representation (IR) format.
Then infer it on a Intel Movidius Myriad stick
## installation
You will require Openvino, for the conversion and optimization of a DNN model.
Please, follow the instruction in their [webpage](https://docs.openvinotoolkit.org/latest/installation_guides.html)
Openvino does not work for python 3.9. We used python 3.8.
```bash
cd /tmp
mkdir local_pkg
cd local_pkg
git clone https://git.nilu.no/aqdl/mtsaq_pkg.git
cd mtsaq_pkg
pip3 install -e .
cd /tmp
git clone https://git.nilu.no/aqdl/case_studies.git
```
## train/eval on the cloud side
### jupyter notebook
```bash
cd /tmp/case_studies/hapads_wp3
py -3.8 -m notebook
```
### in bash
```bash
cd "script"
py -3.8 measure_rect_preproc.py -d "/tmp/case_studies/hapads_wp3/data/measures.txt" -o "/tmp/case_studies/hapads_wp3/data/measure_post.csv"
py -3.8 measure_rect_preproc.py -d "/tmp/case_studies/hapads_wp3/data/last_measures.txt" -o "/tmp/case_studies/hapads_wp3/data/last_measure_post.csv"
cd "/tmp/local_pkg/mtsaq_dev_pkg"
py -3.8 mtsaq/k_fold_train.py -w "/tmp/case_studies/hapads_wp3" -c "train_eval_mlp.json" -d "/tmp/case_studies/hapads_wp3/data/measure_post.csv"
py -3.8 mtsaq/k_fold_eval.py -w "/tmp/case_studies/hapads_wp3" -c "train_eval_mlp.json" -s "c2sjw6hurbpjrfurr7jr" -d "/tmp/case_studies/hapads_wp3/data/measure_post.csv"
py -3.8 mtsaq/prediction.py -w "/tmp/case_studies/hapads_wp3" -c "predict_mlp.json" -s "c2sjw6hurbpjrfurr7jr" -m "fold_1" -d "/tmp/case_studies/hapads_wp3/data/last_measure_post.csv"
cd "path/to/Intel/openvino_2021/deployment_tools/model_optimizer"
py -3.8 mo.py --input_model "/tmp/case_studies/hapads_wp3/session/c2sjw6hurbpjrfurr7jr/train/fold_1_model.onnx" --output_dir "/tmp/case_studies/hapads_wp3/session/c2sjw6hurbpjrfurr7jr/train/"
```
## Inference on Movidius Myriad stick
```bash
cd "script"
py -3.8 measure_rect_preproc.py -d "/tmp/case_studies/hapads_wp3/data/last_measures.txt" -o "/tmp/case_studies/hapads_wp3/data/last_measure_post.csv"
py -3.8 infer_myriad.py -w "/tmp/case_studies/hapads_wp3" -c "predict_mlp.json" -s "c2sjw6hurbpjrfurr7jr" -m "fold_1" -device "MYRIAD"-d "/tmp/case_studies/hapads_wp3/data/last_measure_post.csv"
```
import argparse
from mtsaq.classes.class_ts_x import class_ts_x
from openvino.inference_engine import IECore, IENetwork
def main():
"""
Main inerence on the Myriad
"""
parser = argparse.ArgumentParser(description="Argument parsing for data pre-proc")
parser.add_argument("-w", help="path of the workdir")
parser.add_argument("-c", help="json configuration file providing the whole set of parameter")
parser.add_argument("-s", help="session with trained model to look at.")
parser.add_argument("-m", help="name of the model to use.")
parser.add_argument("-device", help="name of the device.")
parser.add_argument("-d", help="path and filename of the dataset")
args = parser.parse_args()
if not os.path.exists(args.w):
raise ValueError("path "+args.w+" does not exist")
with open(os.path.join(args.w, "config",args.c),encoding='utf8') as f:
get_mtsaq_config = json.load(f)
key = get_random_alphanumeric_string(20)
get_mtsaq_config['sessiondir']['path'] = os.path.join(args.w, "session",key)
train_data_filename = os.path.join(args.w, "session",args.s,'train','data_class_ts_x.pkl')
data_filename = os.path.join(parent_dir,"session",args.s,'pred_myr','data_class_ts_x.pkl')
f_train = open(train_data_filename, 'rb')
train_data = pickle.load(f_train)
params_data["path"] = args.d
data = class_ts_x(params_data)
data.scale_param_features, data.scale_param_target = train_data.get_scale_param_target()
data.scale()
ie = IECore()
# Load any network from file
model_xml = os.path.joint(parent_dir,"session",args.s,"train",args.m+".xml")
model_bin = os.path.joint(parent_dir,"session",args.s,"train",args.m+".bin")
net = IENetwork(model=model_xml, weights=model_bin)
input_blob = next(iter(net.inputs))
out_blob = next(iter(net.outputs))
exec_net = ie.load_network(network=net, device_name=args.device)
res = exec_net.infer(inputs={input_blob: dataset[get_mtsaq_config["data"]["features"]["history_length"]+1,get_mtsaq_config["data"]["features"]["features"]]})
res = res[out_blob]
output_unscaled = train_data.unscale(res,"target")
prediction_filename = os.path.join(parent_dir,"session",args.s,'pred_myr','prediction.pkl')
#save pred
f_pred = open(prediction_filename, 'wb')
pickle.dump(output_unscaled.numpy(), f_pred)
# save the class_ts_x data
f_data = open(data_filename, 'wb')
pickle.dump(data, f_data)
print("inference completed.")
if __name__ == "__main__":
main()
import argparse
import pandas as pd
def main():
"""
Main pre-proc for dataset got from 'measure_rect'
"""
parser = argparse.ArgumentParser(description="Argument parsing for data pre-proc")
parser.add_argument("-d", help="path and filename of the dataset")
parser.add_argument("-o", help="path and filename of the csv output")
args = parser.parse_args()
df = pd.read_csv(args.d,skiprows=2,header=None)
df = df.drop_duplicates()
df.columns=["Day","Month","Year","Hour","Minute","Second","Wday","SN","Element","Unit","Value","Lat","Lon"]
df = df.pivot(index=['Year','Month','Day','Wday','Hour','Minute','Second','SN','Lat','Lon'],columns='Element',values=['Value'])
dataset = df["Value"].reset_index(level=['Year','Month','Day','Wday','Hour','Minute','Second','SN','Lat','Lon'])
dataset.to_csv(args.o)
print("measure_rect pre-proc completed.")
if __name__ == "__main__":
main()
{
"cells": [
{
"cell_type": "markdown",
"id": "6e6866b8",
"metadata": {},
"source": [
"# Train/eval on the cloud side (py -3.8)"
]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "719d7be0",
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"import os\n",
"import json"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "72dc9819",
"metadata": {},
"outputs": [],
"source": [
"parent_dir = os.path.abspath(os.path.join(os.path.abspath(os.getcwd()), os.pardir))"
]
},
{
"cell_type": "markdown",
"id": "32aeaaa6",
"metadata": {},
"source": [
"## pre-process data"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "3df0901f",
"metadata": {},
"outputs": [],
"source": [
"df = pd.read_csv(os.path.join(parent_dir,\"data\",\"measures.txt\"),skiprows=2,header=None)\n",
"df = df.drop_duplicates()"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "b40da7ac",
"metadata": {},
"outputs": [],
"source": [
"df.columns=[\"Day\",\"Month\",\"Year\",\"Hour\",\"Minute\",\"Second\",\"Wday\",\"SN\",\"Element\",\"Unit\",\"Value\",\"Lat\",\"Lon\"]\n",
"df = df.pivot(index=['Year','Month','Day','Wday','Hour','Minute','Second','SN','Lat','Lon'],columns='Element',values=['Value'])\n",
"dataset = df[\"Value\"].reset_index(level=['Year','Month','Day','Wday','Hour','Minute','Second','SN','Lat','Lon'])\n",
"measure_filename = os.path.join(parent_dir,'data','measure_post.csv')\n",
"dataset.to_csv(measure_filename)"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "b8888b19",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"False"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# no missing value\n",
"dataset.isnull().values.any()"
]
},
{
"cell_type": "markdown",
"id": "6f4e0bb7",
"metadata": {},
"source": [
"## k-fold training"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "fe0eb31e",
"metadata": {},
"outputs": [],
"source": [
"from mtsaq.train_eval_pred.k_fold import k_fold_train_function"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "8f0c3707",
"metadata": {},
"outputs": [],
"source": [
"config_dir = os.path.join(parent_dir,'config')"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "fff41fa4",
"metadata": {},
"outputs": [],
"source": [
"f_config = open(os.path.join(config_dir,'train_eval_mlp.json'),encoding='utf8')\n",
"params = json.load(f_config)"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "1f8915e7",
"metadata": {},
"outputs": [],
"source": [
"params[\"data\"][\"path\"] = measure_filename"
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "ce59c2a6",
"metadata": {},
"outputs": [],
"source": [
"from mtsaq.utils.utils import get_random_alphanumeric_string\n",
"key = get_random_alphanumeric_string(20)\n",
"params['sessiondir']['path'] = os.path.join(parent_dir, \"session\",key)"
]
},
{
"cell_type": "code",
"execution_count": 11,
"id": "cf0623e4",
"metadata": {
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Torch is using cpu\n",
"Torch is using cpu\n"
]
}
],
"source": [
"k_fold_train_function(params)"
]
},
{
"cell_type": "code",
"execution_count": 12,
"id": "fc155ad8",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"c2sjw6hurbpjrfurr7jr\n"
]
}
],
"source": [
"print(key)"
]
},
{
"cell_type": "markdown",
"id": "c6ceb85e",
"metadata": {},
"source": [
"## k-fold evaluation"
]
},
{
"cell_type": "code",
"execution_count": 13,
"id": "b9b61966",
"metadata": {},
"outputs": [],
"source": [
"from mtsaq.train_eval_pred.k_fold import k_fold_train_function, k_fold_eval_function"
]
},
{
"cell_type": "code",
"execution_count": 14,
"id": "0c6aeb9a",
"metadata": {},
"outputs": [],
"source": [
"config_dir = os.path.join(parent_dir,'config')"
]
},
{
"cell_type": "code",
"execution_count": 15,
"id": "60832ef2",
"metadata": {},
"outputs": [],
"source": [
"f_config = open(os.path.join(config_dir,'train_eval_mlp.json'),encoding='utf8')\n",
"params = json.load(f_config)"
]
},
{
"cell_type": "code",
"execution_count": 16,
"id": "33576839",
"metadata": {},
"outputs": [],
"source": [
"params[\"data\"][\"path\"] = measure_filename"
]
},
{
"cell_type": "code",
"execution_count": 17,
"id": "68f65597",
"metadata": {},
"outputs": [],
"source": [
"params['sessiondir']['path'] = os.path.join(parent_dir,'session',key)"
]
},
{
"cell_type": "code",
"execution_count": 18,
"id": "2110efd6",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Torch is using cpu\n",
"Torch is using cpu\n"
]
}
],
"source": [
"k_fold_eval_function(params)"
]
},
{
"cell_type": "markdown",
"id": "47e598ec",
"metadata": {},
"source": [
"## prediction (for double-checking)"
]
},
{
"cell_type": "code",
"execution_count": 19,
"id": "285fd96e",
"metadata": {},
"outputs": [],
"source": [
"from mtsaq.train_eval_pred.prediction import prediction_function"
]
},
{
"cell_type": "code",
"execution_count": 20,
"id": "bb942d67",
"metadata": {},
"outputs": [],
"source": [
"f_config = open(os.path.join(config_dir,'predict_mlp.json'),encoding='utf8')\n",
"params = json.load(f_config)"
]
},
{
"cell_type": "code",
"execution_count": 21,
"id": "a58013c4",
"metadata": {},
"outputs": [],
"source": [
"df = pd.read_csv(os.path.join(parent_dir,\"data\",\"last_measures.txt\"),skiprows=2,header=None)\n",
"df = df.drop_duplicates()\n",
"df.columns=[\"Day\",\"Month\",\"Year\",\"Hour\",\"Minute\",\"Second\",\"Wday\",\"SN\",\"Element\",\"Unit\",\"Value\",\"Lat\",\"Lon\"]\n",
"df = df.pivot(index=['Year','Month','Day','Wday','Hour','Minute','Second','SN','Lat','Lon'],columns='Element',values=['Value'])\n",
"dataset = df[\"Value\"].reset_index(level=['Year','Month','Day','Wday','Hour','Minute','Second','SN','Lat','Lon'])"
]
},
{
"cell_type": "code",
"execution_count": 22,
"id": "440e30e7",
"metadata": {},
"outputs": [],
"source": [
"measure_filename = os.path.join(parent_dir,'data','last_measure_post.csv')\n",
"dataset.to_csv(measure_filename)\n",
"params[\"data\"][\"path\"] = measure_filename"
]
},
{
"cell_type": "code",
"execution_count": 23,
"id": "b3c9ca3a",
"metadata": {},
"outputs": [],
"source": [
"from mtsaq.utils.utils import get_random_alphanumeric_string\n",
"key2 = get_random_alphanumeric_string(20)\n",
"params['sessiondir']['path'] = os.path.join(parent_dir,\"session\", key2)\n",
"params['pred_param']['model']['path'] = os.path.join(parent_dir,'session',key)\n",
"params['pred_param']['model']['name'] = \"fold_1\""
]
},
{
"cell_type": "code",
"execution_count": 24,
"id": "51a32e5e",
"metadata": {},
"outputs": [
{