From 92c36cafc286caff9621a2d75da7b6610070503d Mon Sep 17 00:00:00 2001
From: Riccardo Boero <ribo@nilu.no>
Date: Mon, 6 Jan 2025 13:55:28 +0100
Subject: [PATCH] add indexes and columns to support faster selection using
 state-country information

---
 inputs/modify_tables.sh                       |  9 +++++++++
 .../sql/modify_tables_for_state_country.sql   | 19 +++++++++++++++++++
 prepare_database.sh                           |  3 +++
 3 files changed, 31 insertions(+)
 create mode 100755 inputs/modify_tables.sh
 create mode 100644 inputs/sql/modify_tables_for_state_country.sql

diff --git a/inputs/modify_tables.sh b/inputs/modify_tables.sh
new file mode 100755
index 0000000..ac43745
--- /dev/null
+++ b/inputs/modify_tables.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+# Assign parameters to variables
+HOST=$1
+USER=$2
+PASSWORD=$3
+
+echo "*** Fix tables to include states and countries data and to be indexed on them:"
+mariadb -h "$HOST" -u "$USER" -p"$PASSWORD"  < inputs/sql/modify_tables_for_state_country.sql
diff --git a/inputs/sql/modify_tables_for_state_country.sql b/inputs/sql/modify_tables_for_state_country.sql
new file mode 100644
index 0000000..8d93b75
--- /dev/null
+++ b/inputs/sql/modify_tables_for_state_country.sql
@@ -0,0 +1,19 @@
+-- US data
+-- add index to LODES8 table, query already optimized
+ALTER TABLE LODES8
+ADD INDEX idx_geoid (GeoID);
+-- no need to modify QCEW because there is agg level already to select geographical layer
+
+-- EU data
+-- LFS at the country level only
+-- SBS at the country level only
+
+-- Step 1: Add the country column to the table
+ALTER TABLE REA
+ADD COLUMN eu_country CHAR(2);
+-- Step 2: Add the new column to the table index
+ALTER TABLE REA
+ADD INDEX idx_eu_country (eu_country);
+-- Step 3: Update the country column based on the cntr_code column
+UPDATE REA
+SET eu_country = LEFT(GeoID, 2);
diff --git a/prepare_database.sh b/prepare_database.sh
index 40fe907..64d1b49 100755
--- a/prepare_database.sh
+++ b/prepare_database.sh
@@ -45,5 +45,8 @@ PASSWORD=$3
 ./inputs/load_eurostat_lfs.sh "$HOST" "$USER" "$PASSWORD"
 ./inputs/load_eurostat_sbs.sh "$HOST" "$USER" "$PASSWORD"
 
+# modify tables to include states and countries data and to be indexed on them
+./inputs/modify_tables.sh "$HOST" "$USER" "$PASSWORD"
+
 # dump data and drop database on 'local' instance
 ./inputs/dump_and_drop.sh "$HOST" "$USER" "$PASSWORD"
-- 
GitLab