Commit 2cb8dc90 authored by christoffer.stoll's avatar christoffer.stoll
Browse files

first commit

parent 2c5133ab
.DS_Store
*.xml
*.csv
# local env files
.env.local
.env.*.local
# Log files
npm-debug.log*
yarn-debug.log*
yarn-error.log*
# Editor directories and files
.idea
.vscode
*.suo
*.ntvs*
*.njsproj
*.sln
*.sw*
# User-specific stuff:
.idea/workspace.xml
.idea/tasks.xml
.idea/dictionaries
.idea/vcs.xml
.idea/jsLibraryMappings.xml
.idea/misc.xml
.idea/modules.xml
profiles_settings.xml
Project_Default.xml
.idea/raven.iml
# Sensitive or high-churn files:
.idea/dataSources.ids
.idea/dataSources.xml
.idea/dataSources.local.xml
.idea/sqlDataSources.xml
.idea/dynamic.xml
.idea/uiDesigner.xml
# Gradle:
.idea/gradle.xml
.idea/libraries
# Mongo Explorer plugin:
.idea/mongoSettings.xml
## File-based project format:
*.iws
## Plugin-specific files:
# IntelliJ
/out/
# mpeltonen/sbt-idea plugin
.idea_modules/
# JIRA plugin
atlassian-ide-plugin.xml
# Crashlytics plugin (for Android Studio and IntelliJ)
com_crashlytics_export_strings.xml
crashlytics.properties
crashlytics-build.properties
fabric.properties
### Python template
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
env/
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
*.egg-info/
.installed.cfg
*.egg
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*,cover
.hypothesis/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
target/
# IPython Notebook
.ipynb_checkpoints
# pyenv
.python-version
# celery beat schedule file
celerybeat-schedule
# dotenv
.env
# virtualenv
venv/
ENV/
# Spyder project settings
.spyderproject
# Rope project settings
.ropeproject
# albania-observation
Retrieves data for Albania from external server and converts it to Raven friendly observation csv
## Requirements
This script requires **Raven** to be installed.
## Dependencies
* requests (pip install requests)
* python
## Arguments
**--output** Name of resulting csv file
## Usage
```
python al2obs.py --output observations.csv
```
## Notes
Please look inside the script to see if url and usr:psw needs to change.
import requests
import json
from datetime import date, timedelta
import os.path
import csv
import io
from contextlib import closing
import codecs
import re
import datetime
import argparse
def get_args():
parser = argparse.ArgumentParser()
parser.add_argument(
'--output', help='Name of output file', default='observations.csv')
return parser.parse_args()
args = get_args()
# HARD-CODED PARAMETERS
url = "http://easkunde3.eu/cgi-bin/leineweb_cgi.cgi"
usr = "Albania"
pwd = "MoEonly"
fmt = "CSV"
data = []
columns = ["sampling_point_id", "begin_position", "end_position", "value",
"verification_flag", "validation_flag", "concentration", "timestep"]
print("Setting up variables")
# Compute the date to request: since yesterday at 00 to today at 00 (we might add other behaviours later on)
dateTo = date.today()
dateFrom = dateTo - timedelta(days=1)
pms = {}
hds = {}
mappings = {}
# Parameters used to get data from external server
with open("./params.json", "r") as fp:
pms = json.loads(fp.read())
# Headers used when requesting data from external server
with open("./headers.json", "r") as fp:
hds = json.loads(fp.read())
# Mappings used to map external ids to eea ids (sampling_point and unit)
with open("./mappings.json", "r") as fp:
mappings = json.loads(fp.read())
# SET THINGS THAT DEPEND ON EXTERNAL PARAMETERS
# =============================================================
# Hour and minute are already set at 00 from the template
pms["D_VON"] = dateFrom.strftime("%d-%m-%Y")
pms["D_BIS"] = dateTo.strftime("%d-%m-%Y")
# Set format
pms["GRAFTAB"] = fmt
outputFname = dateTo.strftime("%Y%m%d") + ".csv"
print("Loading data from external server")
resp = requests.post(url, data=pms, headers=hds, auth=(usr, pwd))
# Convert data
print("Converting external data to Raven observation format")
pattern = '[^\[\]]* \([^\(]+\)'
reader = csv.DictReader(resp.text.strip().split('\n'), delimiter=';')
for row in reader:
if len(row["Datum"]) > 10:
end_pos = datetime.datetime.strptime(row["Datum"], '%d.%m.%Y %H:%M:%S')
else:
end_pos = datetime.datetime.strptime(row["Datum"], '%d.%m.%Y')
begin_pos = end_pos + timedelta(hours=-1)
for (k, v) in row.items():
row = {}
if k == 'AK' or k == 'Datum' or k == None or k == "":
continue
spo_unit = [part.strip()
for part in re.split('[\[\]]', k) if part.strip()]
if len(spo_unit) != 2:
continue
try:
spo = mappings["stations"][spo_unit[0]]
unit = mappings["units"][spo_unit[1].lower()]
except:
continue
if len(v) == 0:
continue
row["value"] = float(v.replace(",", "."))
row["begin_position"] = begin_pos.strftime("%Y-%m-%dT%H:%M:%S+01:00")
row["end_position"] = end_pos.strftime("%Y-%m-%dT%H:%M:%S+01:00")
row["sampling_point_id"] = spo
row["concentration"] = unit
row["timestep"] = "http://dd.eionet.europa.eu/vocabulary/aq/primaryObservation/hour"
row["verification_flag"] = 1
row["validation_flag"] = 1
data.append(row)
print("Saving data fo file")
try:
with open(args.output, 'w') as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=columns)
writer.writeheader()
for d in data:
writer.writerow(d)
except IOError:
print("I/O error")
print("Done")
{
"Host":"easkunde3.eu"
, "User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:67.0) Gecko/20100101 Firefox/67.0"
, "Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
, "Accept-Language":"es-ES,es;q=0.8,en-US;q=0.5,en;q=0.3"
, "Accept-Encoding":"gzip, deflate"
, "Referer":"http://easkunde3.eu/CGI-BIN/leineweb_CGI.CGI"
, "Content-Type":"application/x-www-form-urlencoded"
}
\ No newline at end of file
{
"stations": {
"Durres SO2": "SPO-AL0205A_00001_100",
"Durres O3": "SPO-AL0205A_00007_100",
"Durres PM10": "SPO-AL0205A_00005_100",
"Durres PM2_5nc": "SPO-AL0205A_06001_100",
"Durres NO2": "SPO-AL0205A_00008_100",
"Durres NOx": "SPO-AL0205A_00009_100",
"Durres CO": "SPO-AL0205A_00010_100",
"Durres Benzene": "SPO-AL0205A_00020_100"
},
"units": {
"ug/m3": "http://dd.eionet.europa.eu/vocabularyconcept/uom/concentration/ug.m-3",
"µg/m³": "http://dd.eionet.europa.eu/vocabularyconcept/uom/concentration/ug.m-3",
"ppb": "http://dd.eionet.europa.eu/vocabularyconcept/uom/concentration/ppbv",
"ppm": "http://dd.eionet.europa.eu/vocabularyconcept/uom/concentration/ppmv"
}
}
\ No newline at end of file
{
"FORM":"DATENB"
, "WIDTH": "1436"
, "HEIGHT": "849"
, "Sorter": "Province"
, "ANZEIGE": ["30001!30001!1!SO2!", "30002!30002!2!O3!", "30003!30003!3!CO!", "30004!30004!4!NO!", "30005!30005!5!NOx!", "30006!30006!6!NO2!", "30010!30010!10!Temp!", "30011!30011!11!Rh!", "30012!30012!12!Pressure!", "30015!30015!15!PM10!", "30016!30016!16!PM2_5nc!", "30017!30017!17!PM1!", "30021!30021!21!Benzene!", "30022!30022!22!Toluene!", "30023!30023!23!p_Xylene!", "30024!30024!24!o_Xylene!", "30025!30025!25!Ethylb.!", "30041!30041!41!PM2_5!", "30042!30042!42!GlobRad!", "30043!30043!43!Roomtemp!", "30044!30044!44!Rain!", "30080!30080!80!WS!", "30081!30081!81!WD!", "40001!40001!1!SO2!", "40002!40002!2!O3!", "40003!40003!3!CO!", "40004!40004!4!NO!", "40005!40005!5!NOx!", "40006!40006!6!NO2!", "40010!40010!10!Temp.!", "40011!40011!11!Humidity!", "40012!40012!12!Pressure!", "40015!40015!15!PM10!", "40016!40016!16!PM2.5!", "40021!40021!21!Benzene!", "40022!40022!22!Toulene!", "40023!40023!23!p_Xylene!", "40024!40024!24!o_Xylene!", "40025!40025!25!Ethylb.!", "40027!40027!27!Humidity!", "40028!40028!28!Pressure!", "40041!40041!41!PM2_5!", "40042!40042!42!GlobRad!", "40043!40043!43!RoomTemp!", "40044!40044!44!Rain!", "40080!40080!80!W.Spd+Gi!", "40081!40081!81!W.Dir+Gi!", "50001!50001!1!SO2!", "50002!50002!2!O3!", "50003!50003!3!CO!", "50004!50004!4!NO!", "50005!50005!5!NOx!", "50006!50006!6!NO2!", "50010!50010!10!Temp!", "50011!50011!11!Rh!", "50012!50012!12!Pressure!", "50015!50015!15!PM10!", "50016!50016!16!PM2_5nc!", "50021!50021!21!Benzene!", "50022!50022!22!Toluene!", "50023!50023!23!p_Xylene!", "50024!50024!24!o_Xylene!", "50025!50025!25!Ethylb.!", "50041!50041!41!PM2_5!", "50042!50042!42!GlobRad!", "50043!50043!43!Roomtemp!", "50044!50044!44!Rain!", "50080!50080!80!WS!", "50081!50081!81!WD!", "60001!60001!1!SO2!", "60002!60002!2!O3!", "60003!60003!3!CO!", "60004!60004!4!NO!", "60005!60005!5!NOx!", "60006!60006!6!NO2!", "60010!60010!10!Temp!", "60011!60011!11!Rh!", "60012!60012!12!Pressure!", "60015!60015!15!PM10!", "60016!60016!16!PM2_5nc!", "60021!60021!21!Benzene!", "60022!60022!22!Toluene!", "60023!60023!23!p_Xylene!", "60024!60024!24!o_Xylene!", "60025!60025!25!Ethylb.!", "60041!60041!41!PM2_5!", "60042!60042!42!GlobRad!", "60043!60043!43!Roomtemp!", "60044!60044!44!Rain!", "60080!60080!80!WS!", "60081!60081!81!WD!", "80001!80001!1!SO2!", "80002!80002!2!O3!", "80003!80003!3!CO!", "80004!80004!4!NO!", "80005!80005!5!NOx!", "80006!80006!6!NO2!", "80021!80021!21!Benzene!", "80022!80022!22!Toluene!", "80023!80023!23!p_Xylene!", "80024!80024!24!o_Xylene!", "80025!80025!25!Ethylb.!", "80043!80043!43!Roomtemp!", "90001!90001!1!SO2!", "90002!90002!2!O3!", "90003!90003!3!CO!", "90004!90004!4!NO!", "90005!90005!5!NOx!", "90006!90006!6!NO2!", "90010!90010!10!Temp!", "90011!90011!11!Rh!", "90012!90012!12!Pressure!", "90019!90019!19!Precip!", "90021!90021!21!Benzene!", "90022!90022!22!Toluene!", "90023!90023!23!p_Xylene!", "90024!90024!24!o_Xylene!", "90025!90025!25!Ethylb.!", "90042!90042!42!GlobRad!", "90080!90080!80!WS!", "90081!90081!81!WD!"]
, "T1": ""
, "W1": ""
, "T2": ""
, "W2": ""
, "T3": ""
, "W3": ""
, "T4": ""
, "W4": ""
, "T5": ""
, "W5": ""
, "T6": ""
, "W6": ""
, "T7": ""
, "W7": ""
, "DATENTYP":"DBS"
, "BEREICH":"SERVER"
, "GRAFTAB":"CSV"
, "GRAFTYP":"Y-T"
, "VONHOUR":"00"
, "VONMINUTE":"00"
, "BISHOUR":"00"
, "BISMINUTE":"00"
, "SEND":"request"
, "SVG":"DBS"
}
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment