Maintenance is scheduled between 16:00 and 23:59 CEST (14:00 and 21:59 UTC) on Thursday 2021-10-28. The system may be unavailable at any time during this timeframe. Please plan accordingly.

Commit f018cbd0 authored by Jean-Marie Lepioufle's avatar Jean-Marie Lepioufle
Browse files

adapted for CUDA

parent 8b006121
......@@ -15,11 +15,17 @@ pip3 install -e .
Package 'mtsaq' has been tested on python 3.8 and 3.9
Running 'mtsaq' with CUDA (v11.1), requires a manual installation of pytorch:
```bash
pip3 install torch==1.8.1+cu111
```
## Usage
```python
py -3.9 mtsaq/k_fold_train.py -w "/path/to/workdir" -c "config_train.json" -d "/path/to/data.csv"
py -3.9 mtsaq/k_fold_eval.py -w "/path/to/workdir" -c "config_eval.json" -s "key_of_a_session" -d "/path/to/data.csv"
py -3.9 mtsaq/prediction.py -w "/path/to/workdir" -c "config_pred.json" -s "key_of_a_session" -m "core_name_model" -d "path/to/last_data.csv"
py -3.8 mtsaq/k_fold_train.py -w "/path/to/workdir" -c "config_train.json" -d "/path/to/data.csv"
py -3.8 mtsaq/k_fold_eval.py -w "/path/to/workdir" -c "config_eval.json" -s "key_of_a_session" -d "/path/to/data.csv"
py -3.8 mtsaq/prediction.py -w "/path/to/workdir" -c "config_pred.json" -s "key_of_a_session" -m "core_name_model" -d "path/to/last_data.csv"
```
## Examples
......
......@@ -6,8 +6,14 @@ import os
import json
class class_model():
def __init__(self):
self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
def __init__(self, params: dict):
# parameters related to the device (cpu,cuda)
if params['cuda'] is not None and torch.cuda.is_available():
self.device = torch.device('cuda'+":"+params['cuda']['name'])
elif params['cpu'] is not None:
self.device = torch.device('cpu')
else:
raise Exception("device not specified.")
print("Torch is using " + str(self.device))
def build(self,params:dict):
......
......@@ -12,7 +12,6 @@ from mtsaq.dictionnary.dictionnary_scaling import scaling_dict
class class_ts_img():
def __init__(self, params: dict,subset_indice=None ):
path= params['path']
self.date = params['date']
features=params['features']['features']
......@@ -87,7 +86,7 @@ class class_ts_img():
filename = self.df_images.loc[self.indice[indice_i],k]
np_images.append(img_as_float(io.imread(os.path.join(self.imgdir,filename))))
image_data = torch.from_numpy(np.array(np_images))
image_data = image_data.permute((0, 3, 1, 2)).float() # channels, row, columns
image_data = image_data.permute((0, 3, 1, 2)).float() # x, channels, row, columns
#target
if self.target is not None:
if (self.history_length>0 or self.gap_length>0):
......@@ -148,7 +147,7 @@ class class_ts_img():
def unscale(self, newdata=None, datatype = None):
if newdata is not None:
if isinstance(newdata, torch.Tensor):
tmp = newdata.numpy()
tmp = newdata.cpu().numpy() # newdata.numpy() does not work on cuda
elif isinstance(newdata, pd.Series) or isinstance(newdata, pd.DataFrame):
tmp = np.asarray(newdata)
elif isinstance(newdata, np.ndarray):
......
......@@ -4,13 +4,14 @@ class mlp_cnn(torch.nn.Module):
'''
MLP CNN
'''
def __init__(self,mlp_input_size, mlp_hidden_size, mlp_output_size, cnn_input_size, cnn_output_size, output_size):
def __init__(self,mlp_input_size, mlp_hidden_size, mlp_output_size, cnn_input_size, cnn_output_size, fc_output_size, output_size):
super().__init__()
self.mlp_input_size = mlp_input_size
self.mlp_hidden_size = mlp_hidden_size
self.mlp_output_size = mlp_output_size
self.cnn_input_size = cnn_input_size
self.cnn_output_size = cnn_output_size
self.fc_output_size = fc_output_size
self.output_size = output_size
# mlp block
self.mlp = torch.nn.Sequential(
......@@ -76,6 +77,11 @@ class mlp_cnn(torch.nn.Module):
torch.nn.BatchNorm2d(num_features=self.cnn_output_size),
torch.nn.ReLU(),
torch.nn.MaxPool2d(kernel_size=2, stride=2))
# fully connection
self.fc = torch.nn.Sequential(
torch.nn.Linear((mlp_input_size+self.cnn_input_size*self.cnn_output_size*3), self.fc_output_size), # 3 becasue R,G,B
torch.nn.ReLU(),
torch.nn.Linear(self.fc_output_size, self.output_size))
def cnn_block(self,x_img):
out = self.cnn_layer1(x_img)
......@@ -99,9 +105,5 @@ class mlp_cnn(torch.nn.Module):
x_img = self.cnn_block(x_img)
x_img = torch.flatten(x_img,start_dim=1)
z = torch.cat((x_f, x_img), 1)
# fully connected
z = torch.nn.Sequential(
torch.nn.Linear(z.shape[1], 128),
torch.nn.ReLU())(z)
z = torch.nn.Linear(128, self.output_size)(z)
return(z)
res = self.fc(z)
return(res)
......@@ -7,9 +7,11 @@ from sklearn.model_selection import KFold
import pandas as pd
import os
import pickle
import torch
def k_fold_train_function(params:dict):
params_sessiondir=params['sessiondir']
params_device = params['device']
params_data=params['data']
params_model=params['model']
params_train=params['training_param']
......@@ -26,6 +28,16 @@ def k_fold_train_function(params:dict):
data_filename = os.path.join(resdir,'data_class_ts_x.pkl')
loss_filename = os.path.join(resdir,'loss.csv')
folds_filename = os.path.join(resdir,'folds.pkl')
# parameters for dataloader according to the device (cpu,cuda)
if params_device['cuda'] is not None and torch.cuda.is_available():
num_workers = params_device['cuda']['worker']
pin_memory = True
elif params_device['cpu'] is not None:
num_workers = params_device['cpu']['worker']
pin_memory = False
else:
num_workers = 0
pin_memory = False
# Define the K-fold Cross Validator
kfold = KFold(n_splits=kfolds, shuffle=True)
data = class_ts_x(params_data)
......@@ -33,15 +45,15 @@ def k_fold_train_function(params:dict):
folds_index = []
# loop on the folds
for fold, (train_index, valid_index) in enumerate(kfold.split(data)):
model_ = class_model(params_device)
model_.build(params_model)
train_fold = class_ts_x(params_data,train_index)
valid_fold = class_ts_x(params_data,valid_index)
folds_index.append({'fold': fold, 'train_index': train_index, 'valid_index': valid_index})
train_fold.scale()
valid_fold.scale()
train_data_loader = DataLoader(train_fold,batch_size=batch_size, shuffle=False,sampler=None,batch_sampler=None,num_workers=0,collate_fn=None,pin_memory=False,drop_last=False,timeout=0,worker_init_fn=None)
valid_data_loader = DataLoader(valid_fold,batch_size=batch_size, shuffle=False,sampler=None,batch_sampler=None,num_workers=0,collate_fn=None,pin_memory=False,drop_last=False,timeout=0,worker_init_fn=None)
model_ = class_model()
model_.build(params_model)
train_data_loader = DataLoader(train_fold,batch_size=batch_size, shuffle=False,sampler=None,batch_sampler=None,num_workers=num_workers,pin_memory=pin_memory)
valid_data_loader = DataLoader(valid_fold,batch_size=batch_size, shuffle=False,sampler=None,batch_sampler=None,num_workers=num_workers,pin_memory=pin_memory)
for epoch in range(epochs):
res = pd.DataFrame(columns=['fold', 'epoch', 'batch_size'])
res = res.append({'fold': fold, 'epoch': epoch, 'batch_size':batch_size},ignore_index=True)
......@@ -63,6 +75,7 @@ def k_fold_train_function(params:dict):
def k_fold_eval_function(params:dict):
params_sessiondir = params['sessiondir']
params_device = params['device']
params_data=params['data']
params_train=params['training_param']
params_eval=params['eval_param']
......@@ -74,16 +87,28 @@ def k_fold_eval_function(params:dict):
resdir = os.path.join(sessiondir,"eval")
if not os.path.exists(resdir):
os.makedirs(resdir, exist_ok=True)
# parameters for dataloader according to the device (cpu,cuda)
if params_device['cuda'] is not None and torch.cuda.is_available():
num_workers = params_device['cuda']['worker']
pin_memory = True
elif params_device['cpu'] is not None:
num_workers = params_device['cpu']['worker']
pin_memory = False
else:
num_workers = 0
pin_memory = False
data = class_ts_x(params_data)
params["eval_param"]["batch_size"] = len(data)
metrics_filename = os.path.join(resdir,'metrics.csv')
metrics_res = pd.DataFrame(columns=['fold', 'batch_size'])
folds_filename = os.path.join(sessiondir,'train','folds.pkl')
f = open(folds_filename, 'rb')
folds_index = pickle.load(f)
for fold in folds_index:
fold_i = folds_index.index(fold)
#print('\n Fold: ',str(fold_i))
print('\n Fold: ',str(fold_i))
metrics_filename = os.path.join(resdir,'metrics'+"_fold_"+str(fold_i)+'.csv')
metrics_res = pd.DataFrame(columns=['fold', 'batch_size'])
model_ = class_model(params_device)
model_.load(os.path.join(sessiondir,"train"),"fold_"+str(fold_i))
res = pd.DataFrame(columns=['fold', 'batch_size'])
for i in range(len(params_data['target']['target'])):
res = res.append({'fold': fold_i, 'batch_size':batch_size},ignore_index=True)
......@@ -91,11 +116,8 @@ def k_fold_eval_function(params:dict):
valid_index = fold.get('valid_index')
valid_fold = class_ts_x(params_data,valid_index)
valid_fold.scale()
valid_data_loader = DataLoader(valid_fold,batch_size=batch_size, shuffle=False,sampler=None,batch_sampler=None,num_workers=0,collate_fn=None,pin_memory=False,drop_last=False,timeout=0,worker_init_fn=None)
model_ = class_model()
model_.load(os.path.join(sessiondir,"train"),"fold_"+str(fold_i))
valid_data_loader = DataLoader(valid_fold,batch_size=batch_size, shuffle=False,sampler=None,batch_sampler=None,num_workers=num_workers,pin_memory=pin_memory)
eval_valid_tmp = evaluation_by_target(model_, params_eval, valid_data_loader,valid_fold)
res = pd.concat([res,eval_valid_tmp],axis=1)
metrics_res = pd.concat([metrics_res,res],axis=0)
# Save the train validation history for further visualization
metrics_res.to_csv(metrics_filename)
metrics_res.to_csv(metrics_filename)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment