Commit c64eda08 authored by Christoffer Stoll's avatar Christoffer Stoll
Browse files

first commit

parent f246b1d5
#Dat2Csv
Dat2Csv converts dat files to observations.csv.
The csv is used for import of observation data to Raven.
##Usage
```
python dat2csv.py -d <datfolder> -m <mapfile> -o <outputfile> -z <timezone>
```
\ No newline at end of file
import sys
import getopt
import os
from collections import OrderedDict
import csv
from datetime import datetime, timedelta
def get_unit(line):
comp = line[10:]
unit = ""
if comp == "Carbon monoxide (air)":
unit = "mg.m-3"
elif comp == "Nitrogen monoxide (air)":
unit = "ug.m-3"
elif comp == "Ozone (air)":
unit = "ug.m-3"
elif comp == "Nitrogen dioxide (air)":
unit = "ug.m-3"
elif comp == "PM2.5 (aerosol)":
unit = "ug.m-3"
elif comp == "PM10 (aerosol)":
unit = "ug.m-3"
elif comp == "Sulphur dioxide (air)":
unit = "ug.m-3"
else:
unit = "ug.m-3"
return unit
def nlstrip(s):
return s.replace('\n', '').replace('\r', '').lstrip()
def get_files(folder_name):
d = []
try:
for f in os.listdir(folder_name):
if f.endswith(".dat"):
d.append(f)
return d
except:
print "Error: Could not find/open ", folder_name
sys.exit(2)
def parse_map(file_name):
d = {}
try:
for line in open(file_name, 'r'):
cols = line.split(",")
if len(cols) == 2:
if cols[1].startswith("SPO"):
d[cols[0]] = nlstrip(cols[1])
return d
except:
print "Error: Could not find/open ", file_name
sys.exit(2)
def parse_dat(file_name, id, timezone):
a = []
timestep = ""
unit = ""
try:
with open(file_name, 'r') as f:
l = f.readlines()
if len(l) > 2:
first_line = l[0].split(",")
if len(first_line) == 2:
unit = get_unit(first_line[0])
timestep = nlstrip(first_line[1])
if (timestep == "day" or timestep == "hour") and unit:
for c in range(2, len(l)):
cols = l[c].split(",")
if len(cols) == 3:
d = OrderedDict()
d["sampling_point_id"] = id
d["begin_position"] = parse_date(cols[0], timezone)
d["end_position"] = parse_date(cols[0], timezone, 24 if timestep == "day" else 1)
d["value"] = cols[1]
d["verification_flag"] = 1
d["validation_flag"] = nlstrip(cols[2])
d["concentration"] = "http://dd.eionet.europa.eu/vocabulary/uom/concentration/" + unit
d["timestep"] = "http://dd.eionet.europa.eu/vocabulary/aq/primaryObservation/" + timestep
a.append(d)
return a
except IOError:
print "Error: Could not find/open ", file_name
sys.exit(2)
except ValueError as e:
print "Error: Could not parse file ", e.message
sys.exit(2)
def parse_date(s, z, h=0):
ns = datetime.strptime(s, '%Y%m%d %H:%M')
ns = ns + timedelta(hours=h)
s = str(ns)
s = s.replace(" ", "T")
s += "+0" + z + ":00"
return s
def write_output(file_name, data):
try:
parsed_headers = False
with open(file_name, 'wb') as f:
for d in data:
w = csv.DictWriter(f, d.keys(), quotechar='\'', quoting=csv.QUOTE_MINIMAL)
if not parsed_headers:
w.writeheader()
parsed_headers = True
w.writerow(d)
except:
print 'Error: Could not write to file ' + file_name
sys.exit(2)
def main(argv):
datfolder = ""
mapfile = ""
outputfile = ""
timezone = 1
imported = 0
no_data = ""
no_map = ""
# Read input arguments
try:
opts, args = getopt.getopt(argv, "h:d:m:o:z:")
except getopt.GetoptError:
print 'Error: Could not get arguments'
sys.exit(2)
# Get input arguments
for opt, arg in opts:
if opt == '-h':
print 'dat2csv.py -d <datfolder> -m <mapfile> -o <outputfile> -z <timezone>'
sys.exit()
elif opt in ("-d"):
datfolder = arg
elif opt in ("-m"):
mapfile = arg
elif opt in ("-o"):
outputfile = arg
elif opt in ("-z"):
timezone = arg
folder_end = "" if datfolder.endswith("/") else "/"
datfolder += folder_end
map = parse_map(mapfile)
files = get_files(datfolder)
print "Found ", len(files), " data files"
file_data = []
for f in files:
if f in map:
arr = parse_dat(datfolder + f, map[f], timezone)
if len(arr) > 0:
file_data.extend(arr)
imported += 1
print ".",
else:
no_data += ", " + f
print "o",
else:
no_map += ", " + f
print "x",
write_output(outputfile, file_data)
print "\nImported ", imported, " files"
print "No data imported in files: ", no_data[1:]
print "No match in map file: ", no_map[1:]
if __name__ == "__main__":
main(sys.argv[1:])
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment