dev: dataflows reportnet3

api/config.py

@@ -2,6 +2,7 @@ import os
 from dotenv import load_dotenv
 from datetime import datetime
 from datetime import timedelta
+from typing import Optional
 
 basedir = os.path.abspath(os.path.dirname(__file__))
 
@@ -12,9 +13,22 @@ path = os.path.join(basedir, '.env') if env_in_api else os.path.join(basedir, '.
 
 load_dotenv(path)
 
+def get_variable(name: str, default_value: Optional[bool] = None) -> bool:
+    true_ = ('true', '1', 't')
+    false_ = ('false', '0', 'f')
+    value: str | None = os.getenv(name, None)
+    if value is None:
+        if default_value is None:
+            raise ValueError(f'Variable `{name}` not set!')
+        else:
+            value = str(default_value)
+    if value.lower() not in true_ + false_:
+        raise ValueError(f'Invalid value `{value}` for variable `{name}`')
+    return value in true_
 
 class Config(object):
     DB_URI = os.environ.get('DB_URI') or "postgresql://user:psw@localhost/db"
     JWT_SECRET_KEY = os.environ.get('JWT_SECRET_KEY')
     JWT_ACCESS_TOKEN_EXPIRES_SECONDS = int(os.environ.get('JWT_ACCESS_TOKEN_EXPIRES_SECONDS')) or 3600
     JWT_ACCESS_TOKEN_EXPIRES = timedelta(seconds=JWT_ACCESS_TOKEN_EXPIRES_SECONDS)
+    SHOWREPORTNET3 = get_variable('SHOWREPORTNET3', False)
\ No newline at end of file

api/core/eea/dataflows_reportnet3.py

+from core.database import CursorFromPool
+from core.eea.zone import Zone
+from core.eea.dataflows import Dataflows
+
+class Dataflows_reportnet3:
+    
+    @staticmethod
+    def get_dataflow(type: str, year: int, timezone: str, description: str):
+        if type.upper() == "B":
+            return Dataflows_reportnet3.get_dataflowB()
+        # if type.upper() == "C":
+        #     return Dataflows.get_dataflowC(year, timezone, description)
+        if type.upper() == "D":
+            return Dataflows_reportnet3.get_dataflowD(year, timezone, description)
+        # if type.upper() == "E1A":
+        #     return Dataflows.get_dataflowE1A(year, timezone, description)
+        # if type.upper() == "G":
+        #     return Dataflows.get_dataflowG(year, timezone, description)
+        return None
+
+    @staticmethod
+    def get_dataflowB():
+        with CursorFromPool() as cursor:
+            cursor.execute("""
+              SELECT z."id",
+                    z."name" AS zone_name,
+                    z.code   AS zone_id,
+                    z.area   AS zone_area,
+                    to_timestamp(z.year::text || '-01-01 00:00:00', 'YYYY-MM-DD HH24:MI:SS') AT TIME ZONE 'UTC' AS start_time,
+                    (
+                        SELECT json_build_object(
+                                        'type', 'FeatureCollection',
+                                        'features', json_agg(
+                                                json_build_object(
+                                                        'type', 'Feature',
+                                                        'properties', json_build_object(
+                                                                'srid', '4326'
+                                                            ),
+                                                        'geometry', ST_AsGeoJSON(z2.geom)::json
+                                                    )
+                                            )
+                                    ) AS geometry
+                        FROM zones z2
+                        WHERE z2.id = z.id
+                    )        AS geometry
+              FROM zones z
+              ORDER BY z.code;
+
+              """)
+            rows = cursor.fetchall()
+            return rows
+        
+    @staticmethod
+    def get_dataflowD(year: int, timezone: int, description: str):
+        processes = Dataflows_reportnet3.get_processes()
+        sampling_points = Dataflows_reportnet3.get_samplingpoints()
+
+        return {            
+            "processes.csv": processes,
+            "sampling_points.csv": sampling_points
+        } 
+
+    
+    @staticmethod
+    def get_processes():
+        with CursorFromPool() as cursor:
+            cursor.execute("""
+                select p.id as process_id,
+                      p.id as process_name,
+                      equiv_demonstration as equivalence_demo,
+                      measurement_type,
+                      measurement_method,
+                      sampling_method,
+                      analytical_tech as analytical_technique,
+                      sampling_equipment,
+                      measurement_equipment,
+                      detection_limit,
+                      detection_limit_uom as detection_limit_unit,
+                      uncertainty_estimate,
+                      documentation,
+                      qa_report,
+                      duration_number as measurement_duration_value,
+                      duration_unit as measurement_duration_unit,
+                      cadence_number as measurement_interval_value,
+                      cadence_unit as measurement_interval_unit
+                from processes p
+            """)
+            rows = cursor.fetchall()
+            return rows
+
+    @staticmethod
+    def get_samplingpoints():
+        with CursorFromPool() as cursor:
+            cursor.execute("""
+              select sp.id                     as samplingpoint_local_id,
+                    sp.pollutant,
+                    sp.id                     as samplingpoint_name,
+                    sp.from_time              as start_date_time,
+                    sp.to_time                as end_date_time,
+                    sp.station_id             as station_local_id,
+                    s.name                    as station_name,
+                    s.city,
+                    s.eoi_code                as eoi_site_id,
+                    s.national_station_code,
+                    ST_Y(s.geom)              as latitude,
+                    ST_X(s.geom)              as longitude,
+                    ST_Z(s.geom)              as altitude,
+                    s.area_classification,
+                    sp.station_classification as station_type,
+                    s.network_id              as network_local_id,
+                    n.name                    as network_name,
+                    n.organisational          as organisationallevel,
+                    n.aggregation_timezone    as aggregation_time_zone,
+                    sp.assessment_type        as intended_assessment_type,
+                    --sp. as process
+                    sp.distance_source,
+                    sp.main_emission_sources
+              from sampling_points sp
+                      left join stations s on sp.station_id = s.id
+                      left join networks n on s.network_id = n.id
+              """)
+            rows = cursor.fetchall()
+            return rows
\ No newline at end of file

api/core/utils.py

@@ -4,7 +4,8 @@ import xml.etree.cElementTree as ET
 from datetime import timedelta, datetime
 import calendar
 import csv
-
+import pandas as pd
+from collections import OrderedDict
 
 class U:
     # Function is needed to make dictionary hashable/comparable
@@ -30,8 +31,12 @@ class U:
             return resp
 
     @staticmethod
-    def dataframe_to_csv_response(df, name):
-        return U.csv_response(df.to_csv(index=False, quoting=csv.QUOTE_ALL), name)
+    def dataframe_to_csv_response(data, name):
+        if isinstance(data, list):
+          data = pd.DataFrame(data)
+        elif isinstance(data, OrderedDict):
+          data = pd.DataFrame([data])
+        return U.csv_response(data.to_csv(index=False, quoting=csv.QUOTE_ALL), name)
 
     @staticmethod
     def csv_response(csv_file, name):

api/endpoints/data/dataflow/routes.py

-from flask import jsonify, Blueprint, request
+from flask import jsonify, Blueprint, request, current_app, Response
 from flask_jwt_extended import jwt_required
 from werkzeug.exceptions import BadRequest
 from core.query import Q
 from core.utils import U
 from endpoints.data.dataflow.models import DataflowModel, DataflowModelE2a
 from core.eea.dataflows import Dataflows
+from core.eea.dataflows_reportnet3 import Dataflows_reportnet3
 from core.jwt_ext_custom import jwt_required_with_exporting_claim
+import zipfile
+from io import BytesIO
+import pandas as pd
 
 dataflow_endpoint = Blueprint('dataflow', __name__)
 
@@ -14,7 +18,7 @@ dataflow_endpoint = Blueprint('dataflow', __name__)
 @jwt_required_with_exporting_claim()
 def dataflows():
     m = DataflowModel(**request.args.to_dict())
-    xml = Dataflows.get_xml(m.type, m.year, m.timezone, m.description)
+    xml = Dataflows.get_dataflow(m.type, m.year, m.timezone, m.description)
     if xml == None:
         raise BadRequest("Could not create xml for dataflow " + m.type)
 
@@ -30,3 +34,51 @@ def dataflows_e2a():
         raise BadRequest("Could not create xml for dataflow E2a")
 
     return U.xmlify(xml)
+
+
+@dataflow_endpoint.route('/api/dataflow/reportnet3', methods=['GET'])
+@jwt_required_with_exporting_claim()
+def dataflows_reportnet3():
+    if not current_app.config['SHOWREPORTNET3']:
+        return jsonify({"error": "Reportnet3 dataflow is not enabled"}), 403
+    
+    m = DataflowModel(**request.args.to_dict())
+    data = Dataflows_reportnet3.get_dataflow(m.type, m.year, m.timezone, m.description)
+    if not data:
+        raise BadRequest("Could not create csv for dataflow "+m.type+", no data found")
+
+    # Create a zip file in memory
+    zip_buffer = BytesIO()
+    with zipfile.ZipFile(zip_buffer, "w", zipfile.ZIP_DEFLATED) as zip_file:
+      if isinstance(data, list):
+          # Convert data to a DataFrame and write it to a single CSV file
+          df = pd.DataFrame(data)
+          csv_buffer = BytesIO()
+          df.to_csv(csv_buffer, index=False)
+          csv_buffer.seek(0)
+          zip_file.writestr("reportnet3_dataflow.csv", csv_buffer.getvalue())
+      else:
+          for filename, df in data.items():
+              # Convert df to a DataFrame if it's a list
+              if isinstance(df, list):
+                  df = pd.DataFrame(df)
+              # Convert each DataFrame to a CSV and add it to the zip file
+              csv_buffer = BytesIO()
+              df.to_csv(csv_buffer, index=False)
+              csv_buffer.seek(0)
+              zip_file.writestr(filename, csv_buffer.getvalue())
+
+    # Get the bytes of the zip file
+    zip_buffer.seek(0)
+
+    # Create a Flask response and set the appropriate headers
+    response = Response(zip_buffer.read(), mimetype='application/zip')
+    response.headers['Content-Disposition'] = 'attachment; filename=reportnet3_dataflow.zip'
+    zip_buffer.close()
+
+    return response
+
+@dataflow_endpoint.route('/api/dataflow/showreportnet3', methods=['GET'])
+@jwt_required()
+def show_reportnet3():
+    return jsonify({"showreportnet3": current_app.config['SHOWREPORTNET3']})
\ No newline at end of file

client/src/views/data/dataflow/Dataflow.vue

@@ -8,6 +8,16 @@ const timezoneId = ref();
 const description = ref("update");
 const lastRequest = ref("");
 
+const r3_year = ref();
+const r3_timezoneId = ref();
+const r3_description = ref("update");
+
+const showReportnet3 = ref(false);
+
+onMounted(async () => {
+  showReportnet3.value = await getShowReportnet3();
+});
+
 // METHODS
 const years = () => {
   const start = 2013;
@@ -32,12 +42,25 @@ const downloadDataflow = async (type) => {
   Eventy.hideMessage();
 };
 
+const downloadReportnet3Dataflow = async (type) => {
+  Eventy.showMessage("Creating csv. Please wait", "loading");
+  const response = await Service.dataflowReportnet3(type, year.value, parseInt(timezoneId.value), description.value);
+  const blob = new Blob([response], { type: "text/csv" });
+  FileDownload(blob, `dataflow-${type.toLowerCase()}.zip`);
+  Eventy.hideMessage();
+};
+
 const downloadDataflowE2A = async () => {
   Eventy.showMessage("Creating xml. Please wait", "loading");
   const xml = await Service.dataflowE2A(lastRequest.value);
   FileDownload(xml, `dataflow-e2a.xml`);
   Eventy.hideMessage();
 };
+
+const getShowReportnet3 = async () => {
+  const json = await Service.showReportnet3();
+  return json.showreportnet3;
+};
 </script>
 
 <template>
@@ -86,6 +109,38 @@ const downloadDataflowE2A = async () => {
         </div>
       </div>
     </div>
+
+    <div v-if="showReportnet3">
+      <tool-bar title="Reportnet3" :show-filter="false" :show-add="false" :show-download="false" :show-column-picker="false" class="mt-4" />
+      <div class="border border-nord4 bg-gray-50 p-2 flex flex-col gap-5">
+        <!--   <div class="flex gap-3">
+          <div>
+            <div class="font-bold">Year</div>
+            <n-select class="!w-40" v-model="r3_year">
+              <n-option v-for="opt in years()" :key="opt" :value="opt" :label="opt" />
+            </n-select>
+          </div>
+          <div>
+            <div class="font-bold">Timezone</div>
+            <n-select class="!w-40" v-model="r3_timezoneId">
+              <n-option v-for="opt in timezones()" :key="opt" :value="opt" :label="'UTC +0' + opt" />
+            </n-select>
+          </div>
+          <div>
+            <div class="font-bold">Description</div>
+            <input class="n-input !w-56" v-model="r3_description" />
+          </div>
+        </div>
+         -->
+        <div class="flex gap-4">
+          <button class="n-button" @click="downloadReportnet3Dataflow('B')" :disabled="false">Dataflow B</button>
+          <button class="n-button" disabled @click="downloadReportnet3Dataflow('C')" :disabled="false">Dataflow C</button>
+          <button class="n-button" @click="downloadReportnet3Dataflow('D')" :disabled="false">Dataflow D</button>
+          <button class="n-button" disabled @click="downloadReportnet3Dataflow('E1A')" :disabled="false">Dataflow E1A</button>
+          <button class="n-button" disabled @click="downloadReportnet3Dataflow('G')" :disabled="false">Dataflow G</button>
+        </div>
+      </div>
+    </div>
   </common-layout>
 </template>
 

client/src/views/data/dataflow/service.js

 import { Get, Post } from "../../../helpers/request";
+import axios from "axios";
 
 const Service = {
   dataflow: async (type, year, timezone, description) => Get(`/api/dataflow?type=${type}&year=${year}&timezone=${timezone}&description=${description}`),
-  dataflowE2A: async (lastRequest) => Get(`/api/dataflow/e2a?last_request=${lastRequest}`)
+  dataflowE2A: async (lastRequest) => Get(`/api/dataflow/e2a?last_request=${lastRequest}`),
+  dataflowReportnet3: async (type, year, timezone, description) => {
+    const response = await axios.get(`/api/dataflow/reportnet3?type=${type}&year=${year}&timezone=${timezone}&description=${description}`, { responseType: "arraybuffer" });
+    return new Blob([response.data], { type: "application/zip" });
+  },
+  showReportnet3: async () => Get(`/api/dataflow/showreportnet3`)
 };
 
 export default Service;