From 44ca09243c9fbea03c25b4817ed2954c05e75c27 Mon Sep 17 00:00:00 2001
From: mye <mye@nilu.no>
Date: Mon, 11 May 2020 12:28:01 +0200
Subject: [PATCH] added ipynb file for ozone
---
Examples/ebas_ozon_template.ipynb | 420 ++++++++++++++++++++++++++++++
1 file changed, 420 insertions(+)
create mode 100644 Examples/ebas_ozon_template.ipynb
diff --git a/Examples/ebas_ozon_template.ipynb b/Examples/ebas_ozon_template.ipynb
new file mode 100644
index 0000000..8e30903
--- /dev/null
+++ b/Examples/ebas_ozon_template.ipynb
@@ -0,0 +1,420 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from ebas.io.file import nasa_ames\n",
+ "from nilutility.datatypes import DataObject\n",
+ "from ebas.domain.time_period import estimate_period_code, \\\n",
+ " estimate_resolution_code, estimate_sample_duration_code\n",
+ "import datetime\n",
+ "\n",
+ "__version__ = '1.00.00'"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def set_fileglobal_metadata(nas):\n",
+ " \"\"\"\n",
+ " Set file global metadata for the EbasNasaAmes file object\n",
+ "\n",
+ " Parameters:\n",
+ " nas EbasNasaAmes file object\n",
+ " Returns:\n",
+ " None\n",
+ " \"\"\"\n",
+ " # All times reported to EBAS need to be in UTC!\n",
+ " # Setting the timezone here explicitly should remind you to check your data\n",
+ " nas.metadata.timezone = 'UTC'\n",
+ "\n",
+ " # Revision information\n",
+ " nas.metadata.revdate = datetime.datetime(2020, 05, 07, 12, 54, 21)\n",
+ " nas.metadata.revision = '1.1a'\n",
+ " nas.metadata.revdesc = \\\n",
+ " 'initiol revision to ebas, generated with MyDataTool 1.22'\n",
+ "\n",
+ " # Data Originator Organisation\n",
+ " nas.metadata.org = DataObject(\n",
+ " OR_CODE='NO01L',\n",
+ " OR_NAME='Norwegian Institute for Air Research',\n",
+ " OR_ACRONYM='NILU', OR_UNIT='Atmosphere and Climate Department',\n",
+ " OR_ADDR_LINE1='Instituttveien 18', OR_ADDR_LINE2=None,\n",
+ " OR_ADDR_ZIP='2007', OR_ADDR_CITY='Kjeller', OR_ADDR_COUNTRY='Norway')\n",
+ "\n",
+ " # Projects the data are associated to\n",
+ " nas.metadata.projects = ['EMEP', 'NILU']\n",
+ "\n",
+ " # Data Originators (PIs)\n",
+ " nas.metadata.originator = []\n",
+ " nas.metadata.originator.append(\n",
+ " DataObject(\n",
+ " PS_LAST_NAME='Ytre-Eide', PS_FIRST_NAME='Martin', PS_EMAIL='mye@nilu.no',\n",
+ " PS_ORG_NAME='Norwegian Institute for Air Research',\n",
+ " PS_ORG_ACR='NILU', PS_ORG_UNIT='Atmosphere and Climate Department',\n",
+ " PS_ADDR_LINE1='Instituttveien 18', PS_ADDR_LINE2=None,\n",
+ " PS_ADDR_ZIP='2007', PS_ADDR_CITY='Kjeller',\n",
+ " PS_ADDR_COUNTRY='Norway',\n",
+ " PS_ORCID=None,\n",
+ " ))\n",
+ " nas.metadata.originator.append(\n",
+ " DataObject(\n",
+ " PS_LAST_NAME=u'Rud', PS_FIRST_NAME='Richard', PS_EMAIL='ror@nilu.no',\n",
+ " PS_ORG_NAME='Norwegian Institute for Air Research',\n",
+ " PS_ORG_ACR='NILU', PS_ORG_UNIT='Atmosphere and Climate Department',\n",
+ " PS_ADDR_LINE1='Instituttveien 18', PS_ADDR_LINE2=None,\n",
+ " PS_ADDR_ZIP='2007', PS_ADDR_CITY='Kjeller',\n",
+ " PS_ADDR_COUNTRY='Norway',\n",
+ " PS_ORCID=None,\n",
+ " ))\n",
+ "\n",
+ " # Data Submitters (contact for data technical issues)\n",
+ " nas.metadata.submitter = []\n",
+ " nas.metadata.submitter.append(\n",
+ " DataObject(\n",
+ " PS_LAST_NAME=u'Ytre-Eide', PS_FIRST_NAME='Martin', PS_EMAIL='mye@nilu.no',\n",
+ " PS_ORG_NAME='Norwegian Institute for Air Research',\n",
+ " PS_ORG_ACR='NILU', PS_ORG_UNIT='Atmosphere and Climate Department',\n",
+ " PS_ADDR_LINE1='Instituttveien 18', PS_ADDR_LINE2=None,\n",
+ " PS_ADDR_ZIP='2007', PS_ADDR_CITY='Kjeller',\n",
+ " PS_ADDR_COUNTRY='Norway',\n",
+ " PS_ORCID=None,\n",
+ " ))\n",
+ "\n",
+ " # Station metadata\n",
+ " nas.metadata.station_code = 'NO0489R' #NO00450R'\n",
+ " nas.metadata.platform_code = 'NO0489R' #'NO00450S'\n",
+ " nas.metadata.station_name = u'Kjeller (NILU)'\n",
+ "\n",
+ " # nas.metadata.station_wdca_id = 'GAWANO__ZEP'\n",
+ " # nas.metadata.station_gaw_id = 'ZEP'\n",
+ " # nas.metadata.station_gaw_name = u'Zeppelin Mountain (Ny Ã…lesund)'\n",
+ " # nas.metadata.station_airs_id = # N/A\n",
+ " # nas.metadata.station_other_ids = '721 (NILUDB)'\n",
+ " # nas.metadata.station_state_code = # N/A\n",
+ " # nas.metadata.station_landuse = 'Gravel and stone'\n",
+ " # nas.metadata.station_setting = 'Polar'\n",
+ " # nas.metadata.station_gaw_type = 'G'\n",
+ " # nas.metadata.station_wmo_region = 6\n",
+ " nas.metadata.station_latitude = 59.966667\n",
+ " nas.metadata.station_longitude = 11.033333\n",
+ " nas.metadata.station_altitude = 110.0\n",
+ "\n",
+ " # More file global metadata, but those can be overridden per variable\n",
+ " # See set_variables for examples\n",
+ " nas.metadata.instr_type = 'uv_abs'\n",
+ " nas.metadata.lab_code = 'NO01L'\n",
+ " nas.metadata.instr_name = 'NO42_uv_abs_42'\n",
+ " nas.metadata.method = 'NO01L_uv_ab'\n",
+ " #nas.metadata.regime = 'IMG'\n",
+ " nas.metadata.matrix = 'air'\n",
+ " #nas.metadata.comp_name will be set on variable level\n",
+ " #nas.metadata.unit will be set on variable level\n",
+ " nas.metadata.statistics = 'arithmetic mean'\n",
+ " nas.metadata.datalevel = '2'"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def set_time_axes(nas):\n",
+ " \"\"\"\n",
+ " Set the time axes and related metadata for the EbasNasaAmes file object.\n",
+ "\n",
+ " Parameters:\n",
+ " nas EbasNasaAmes file object\n",
+ " Returns:\n",
+ " None\n",
+ " \"\"\"\n",
+ " # define start and end times for all samples\n",
+ " nas.sample_times = \\\n",
+ " [(datetime.datetime(2014, 1, 1, 11, 0), datetime.datetime(2014, 3, 7, 13, 22)),\n",
+ " (datetime.datetime(2014, 3, 7, 13, 57), datetime.datetime(2014, 6, 12, 11, 32)),\n",
+ " (datetime.datetime(2014, 6, 12, 11, 47), datetime.datetime(2014, 9, 15, 17, 3)),\n",
+ " (datetime.datetime(2014, 9, 15, 17, 3), datetime.datetime(2014, 12, 15, 14, 42))]\n",
+ "\n",
+ " #\n",
+ " # Generate metadata that are related to the time axes:\n",
+ " #\n",
+ "\n",
+ " # period code is an estimate of the current submissions period, so it should\n",
+ " # always be calculated from the actual time axes, like this:\n",
+ " nas.metadata.period = estimate_period_code(nas.sample_times[0][0],\n",
+ " nas.sample_times[-1][1])\n",
+ "\n",
+ " # Sample duration can be set automatically\n",
+ " nas.metadata.duration = estimate_sample_duration_code(nas.sample_times)\n",
+ " # or set it hardcoded:\n",
+ " # nas.metadata.duration = '3mo'\n",
+ "\n",
+ " # Resolution code can be set automatically\n",
+ " # But be aware that resolution code is an identifying metadata element.\n",
+ " # That means, several submissions of data (multiple years) will\n",
+ " # only be stored as the same dataset if the resolution code is the same\n",
+ " # for all submissions!\n",
+ " # That might be a problem for time series with varying resolution code\n",
+ " # (sometimes 2 months, sometimes 3 months, sometimes 9 weeks, ...). You\n",
+ " # might consider using a fixed resolution code for those time series.\n",
+ " # Automatic calculation (will work from ebas.io V.3.0.7):\n",
+ " nas.metadata.resolution = estimate_resolution_code(nas.sample_times)\n",
+ " # or set it hardcoded:\n",
+ " # nas.metadata.resolution = '3mo'\n",
+ "\n",
+ " # It's a good practice to use Jan 1st of the year of the first sample\n",
+ " # endtime as the file reference date (zero point of time axes).\n",
+ " nas.metadata.reference_date = \\\n",
+ " datetime.datetime(nas.sample_times[0][1].year, 1, 1)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def set_variables(nas):\n",
+ " \"\"\"\n",
+ " Set metadata and data for all variables for the EbasNasaAmes file object.\n",
+ "\n",
+ " Parameters:\n",
+ " nas EbasNasaAmes file object\n",
+ " Returns:\n",
+ " None\n",
+ " \"\"\"\n",
+ " # variable 1: examples for missing values and flagging\n",
+ " #values = [1.22, 2.33, None, 4.55] # missing value is None!\n",
+ " values = [73,103,106,9999.9]\n",
+ " flags = [[],[], [],[999]]\n",
+ " # [] means no flags for this measurement\n",
+ " # [999] missing or invalid flag needed because of missing value (None)\n",
+ " # [632, 665] multiple flags per measurement possible\n",
+ " metadata = DataObject()\n",
+ " metadata.comp_name = 'ozone'\n",
+ " metadata.unit = 'nmol/mol'\n",
+ " # alternatively, you could set all metadata at once:\n",
+ " # metadata = DataObject(comp_name='HCB', unit = 'pg/m3')\n",
+ " #metadata.statistics = 'arithmetic mean'\n",
+ " nas.variables.append(DataObject(values_=values, flags=flags, flagcol=True,\n",
+ " metadata=metadata))\n",
+ "\n",
+ " # variable 2: examples for overridden metadata, uncertainty and detection\n",
+ " # limit\n",
+ " values = [1.22, 2.33, 3.44, 4.55]\n",
+ " flags = [[], [], [], []]\n",
+ " metadata = DataObject()\n",
+ " metadata.comp_name = 'ozone'\n",
+ " metadata.unit = 'nmol/mol'\n",
+ " metadata.statistics = 'arithmetic mean'\n",
+ " # matrix is different for this variable. Generally, you can override most\n",
+ " # elements of nas.metadata on a per-variable basis by just setting the\n",
+ " # according nas.variables[i].metadata element.\n",
+ " #metadata.matrix = 'air+aerosol'\n",
+ " # additionally, we also specify uncertainty and detection limit for this\n",
+ " # variable:\n",
+ " #metadata.detection_limit = [0.10, 'nmol/mol']\n",
+ " # detection limit unit must always be the same as the variable's unit!\n",
+ " #metadata.uncertainty = [0.12, 'nmol/mol']\n",
+ " # uncertainty unit is either the same as the variable's unit, ot '%' for\n",
+ " # relative uncertainty:\n",
+ " # metadata.uncertainty = [10.0, '%']\n",
+ " nas.variables.append(DataObject(values_=values, flags=flags, flagcol=True,\n",
+ " metadata=metadata))\n",
+ "\n",
+ " # variable 3: uncertainty will be specified for each sample (see variable 4)\n",
+ " values = [1.22, 2.33, 3.44, 4.55]\n",
+ " flags = [[], [], [], []]\n",
+ " metadata = DataObject()\n",
+ " metadata.comp_name = 'ozone'\n",
+ " metadata.unit = 'nmol/mol'\n",
+ " nas.variables.append(DataObject(values_=values, flags=flags, flagcol=True,\n",
+ " metadata=metadata))\n",
+ "\n",
+ " # variable 4: this variable contains the uncertainties for varable 3\n",
+ " values = [0.22, 0.33, 0.44, 0.55]\n",
+ " flags = [[], [], [], []]\n",
+ " metadata = DataObject()\n",
+ " metadata.comp_name = 'ozone'\n",
+ " metadata.unit = 'nmol/mol'\n",
+ " \n",
+ " #metadata.statistics = 'arithmetic mean'\n",
+ " # this is what makes this variable the uncetainty time series:\n",
+ " #metadata.statistics = 'uncertainty'\n",
+ " nas.variables.append(DataObject(values_=values, flags=flags, flagcol=True,\n",
+ " metadata=metadata))\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def ebas_genfile():\n",
+ " \"\"\"\n",
+ " Main program for ebas_flatcsv\n",
+ " Created for lexical scoping.\n",
+ "\n",
+ " Parameters:\n",
+ " None\n",
+ " Returns:\n",
+ " none\n",
+ " \"\"\"\n",
+ "\n",
+ " # Create an EbasNasaAmes file object\n",
+ " nas = nasa_ames.EbasNasaAmes()\n",
+ "\n",
+ " # Set file global metadata\n",
+ " set_fileglobal_metadata(nas)\n",
+ "\n",
+ " # Set the time axes and related metadata\n",
+ " set_time_axes(nas)\n",
+ "\n",
+ " # Set metadata and data for all variables\n",
+ " set_variables(nas)\n",
+ "\n",
+ " # write the file:\n",
+ " nas.write(createfiles=True)\n",
+ " # createfiles=True\n",
+ " # Actually creates output files, else the output would go to STDOUT.\n",
+ " # You can also specify:\n",
+ " # destdir='path/to/directory'\n",
+ " # Specify a specific relative or absolute path to a directory the\n",
+ " # files should be written to\n",
+ " # flags=FLAGS_COMPRESS\n",
+ " # Compresses the file size by reducing flag columns.\n",
+ " # Flag columns will be less explicit and thus less intuitive for\n",
+ " # humans to read.\n",
+ " # flags=FLAGS_ALL\n",
+ " # Always generate one flag column per variable. Very intuitive to\n",
+ " # read, but increases filesize.\n",
+ " # The default for flags is: Generate one flag column per file if the\n",
+ " # flags are the same for all variables in the file. Else generate one\n",
+ " # flag column per variable.\n",
+ " # This is a trade-off between the advantages and disadvantages of the\n",
+ " # above mentioned approaches.\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "52 1001\n",
+ "Ytre-Eide, Martin; Rud, Richard\n",
+ "NO01L, Norwegian Institute for Air Research, NILU, Atmosphere and Climate Department, Instituttveien 18, , 2007, Kjeller, Norway\n",
+ "Ytre-Eide, Martin\n",
+ "EMEP NILU\n",
+ "1 1\n",
+ "2014 01 01 2020 05 07\n",
+ "0\n",
+ "days from file reference point\n",
+ "6\n",
+ "1 1 1 1 1 1\n",
+ "999.999999 99999.9 99.99 99.99 9.99 9.999\n",
+ "end_time of measurement, days from the file reference point\n",
+ "ozone, nmol/mol\n",
+ "ozone, nmol/mol\n",
+ "ozone, nmol/mol\n",
+ "ozone, nmol/mol\n",
+ "numflag, no unit\n",
+ "0\n",
+ "32\n",
+ "Data definition: EBAS_1.1\n",
+ "Set type code: TI\n",
+ "Timezone: UTC\n",
+ "File name: NO0489R.20140101110000.20200507125421.uv_abs.ozone.air.11mo.3mo.NO01L_NO42_uv_abs_42.NO01L_uv_ab.lev2.nas\n",
+ "File creation: 20200511083805\n",
+ "Startdate: 20140101110000\n",
+ "Revision date: 20200507125421\n",
+ "Version: 1.1a\n",
+ "Version description: initiol revision to ebas, generated with MyDataTool 1.22\n",
+ "Statistics: arithmetic mean\n",
+ "Data level: 2\n",
+ "Period code: 11mo\n",
+ "Resolution code: 3mo\n",
+ "Sample duration: 3mo\n",
+ "Station code: NO0489R\n",
+ "Platform code: NO0489R\n",
+ "Station name: Kjeller (NILU)\n",
+ "Station latitude: 59.966667\n",
+ "Station longitude: 11.033333\n",
+ "Station altitude: 110.0 m\n",
+ "Regime: IMG\n",
+ "Component: ozone\n",
+ "Unit: nmol/mol\n",
+ "Matrix: air\n",
+ "Laboratory code: NO01L\n",
+ "Instrument type: uv_abs\n",
+ "Instrument name: NO42_uv_abs_42\n",
+ "Method ref: NO01L_uv_ab\n",
+ "Originator: Ytre-Eide, Martin, mye@nilu.no, Norwegian Institute for Air Research, NILU, Atmosphere and Climate Department, Instituttveien 18, , 2007, Kjeller, Norway\n",
+ "Originator: Rud, Richard, ror@nilu.no, Norwegian Institute for Air Research, NILU, Atmosphere and Climate Department, Instituttveien 18, , 2007, Kjeller, Norway\n",
+ "Submitter: Ytre-Eide, Martin, mye@nilu.no, Norwegian Institute for Air Research, NILU, Atmosphere and Climate Department, Instituttveien 18, , 2007, Kjeller, Norway\n",
+ "starttime endtime ozone ozone ozone ozone flag\n",
+ " 0.458333 65.556944 73.0 1.22 1.22 0.22 0.000\n",
+ " 65.581250 162.480556 103.0 2.33 2.33 0.33 0.000\n",
+ "162.490972 257.710417 106.0 3.44 3.44 0.44 0.000\n",
+ "257.710417 348.612500 9999.9 4.55 4.55 0.55 0.000\n"
+ ]
+ }
+ ],
+ "source": [
+ "nas = nasa_ames.EbasNasaAmes()\n",
+ "set_fileglobal_metadata(nas)\n",
+ "set_time_axes(nas)\n",
+ "set_variables(nas)\n",
+ "nas.write()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 62,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "ebas_genfile()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 2",
+ "language": "python",
+ "name": "python2"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 2
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython2",
+ "version": "2.7.18"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
--
GitLab