Switched to initial setup of partitioned table.

inputs/load_vector_data.sh

@@ -18,9 +18,6 @@ fi
 # Maximum number of parallel ogr2ogr instances
 max_parallel=15
 
-# Process the first file separately
-first_file=true
-
 # Array to hold the PIDs of the background processes
 declare -a pids
 
@@ -30,23 +27,11 @@ processed_files=0
 remaining_files=$total_files
 
 for file in "$dir"/*; do
-  # Skip if not a file
-  if [ ! -f "$file" ]; then
-  	continue
-  fi
+	# Skip if not a file
+	if [ ! -f "$file" ]; then
+	continue
+	fi
   
-  if [ "$first_file" = true ]; then
-	echo "Processing first file: $file"
-	# create table wioth first file
-	ogr2ogr -f MySQL MySQL:$DB_NAME,host=$HOST,user=$USER,password=$PASSWORD "$file" -nln footprints -update -overwrite -lco engine=InnoDB
-	# partition the table
-	mariadb -h "$HOST" -u "$USER" -p"$PASSWORD" < inputs/sql/partition_db.sql
-	# go on with updating counters
-	first_file=false
-	((processed_files++))
-	((remaining_files--))
-	echo "Processed files: $processed_files, Remaining: $remaining_files"
-  else
 	# Start processing the file in the background
 	echo "Processing file in parallel: $file"
 	ogr2ogr -f MySQL MySQL:$DB_NAME,host=$HOST,user=$USER,password=$PASSWORD  "$file" -nln footprints -update -append &
@@ -67,7 +52,6 @@ for file in "$dir"/*; do
 		pids=("${pids[@]}")
 		sleep 1 # A short delay to prevent the loop from consuming too much CPU
 	done
-  fi
 done
 
 # Wait for any remaining background processes to finish

inputs/setup_db.sh

@@ -7,3 +7,6 @@ PASSWORD=$3
 
 echo "*** Create database:"
 mariadb -h "$HOST" -u "$USER" -p"$PASSWORD" < inputs/sql/create_db.sql
+
+echo "*** Create partitioned table:"
+mariadb -h "$HOST" -u "$USER" -p"$PASSWORD" < inputs/sql/create_table.sql

inputs/sql/create_table.sql

-CREATE DATABASE IF NOT EXISTS FACT_bldgs;
+CREATE TABLE FACT_bldgs.`footprints` (
+  `OGR_FID` int(11) NOT NULL AUTO_INCREMENT,
+  `SHAPE` geometry NOT NULL,
+  `type` varchar(255) DEFAULT NULL,
+  `properties` varchar(255) DEFAULT NULL,
+  `id` varchar(2) DEFAULT NULL,
+  PRIMARY KEY (`OGR_FID`, `id`),  -- Primary key includes the partition key
+  SPATIAL KEY `SHAPE` (`SHAPE`)
+) ENGINE=InnoDB
+PARTITION BY LIST COLUMNS (id) (
+    PARTITION p_unknown VALUES IN (''),
+    PARTITION p_01 VALUES IN ('01'),
+    PARTITION p_02 VALUES IN ('02'),
+    PARTITION p_04 VALUES IN ('04'),
+    PARTITION p_05 VALUES IN ('05'),
+    PARTITION p_06 VALUES IN ('06'),
+    PARTITION p_08 VALUES IN ('08'),
+    PARTITION p_09 VALUES IN ('09'),
+    PARTITION p_10 VALUES IN ('10'),
+    PARTITION p_11 VALUES IN ('11'),
+    PARTITION p_12 VALUES IN ('12'),
+    PARTITION p_13 VALUES IN ('13'),
+    PARTITION p_15 VALUES IN ('15'),
+    PARTITION p_16 VALUES IN ('16'),
+    PARTITION p_17 VALUES IN ('17'),
+    PARTITION p_18 VALUES IN ('18'),
+    PARTITION p_19 VALUES IN ('19'),
+    PARTITION p_20 VALUES IN ('20'),
+    PARTITION p_21 VALUES IN ('21'),
+    PARTITION p_22 VALUES IN ('22'),
+    PARTITION p_23 VALUES IN ('23'),
+    PARTITION p_24 VALUES IN ('24'),
+    PARTITION p_25 VALUES IN ('25'),
+    PARTITION p_26 VALUES IN ('26'),
+    PARTITION p_27 VALUES IN ('27'),
+    PARTITION p_28 VALUES IN ('28'),
+    PARTITION p_29 VALUES IN ('29'),
+    PARTITION p_30 VALUES IN ('30'),
+    PARTITION p_31 VALUES IN ('31'),
+    PARTITION p_32 VALUES IN ('32'),
+    PARTITION p_33 VALUES IN ('33'),
+    PARTITION p_34 VALUES IN ('34'),
+    PARTITION p_35 VALUES IN ('35'),
+    PARTITION p_36 VALUES IN ('36'),
+    PARTITION p_37 VALUES IN ('37'),
+    PARTITION p_38 VALUES IN ('38'),
+    PARTITION p_39 VALUES IN ('39'),
+    PARTITION p_40 VALUES IN ('40'),
+    PARTITION p_41 VALUES IN ('41'),
+    PARTITION p_42 VALUES IN ('42'),
+    PARTITION p_44 VALUES IN ('44'),
+    PARTITION p_45 VALUES IN ('45'),
+    PARTITION p_46 VALUES IN ('46'),
+    PARTITION p_47 VALUES IN ('47'),
+    PARTITION p_48 VALUES IN ('48'),
+    PARTITION p_49 VALUES IN ('49'),
+    PARTITION p_50 VALUES IN ('50'),
+    PARTITION p_51 VALUES IN ('51'),
+    PARTITION p_53 VALUES IN ('53'),
+    PARTITION p_54 VALUES IN ('54'),
+    PARTITION p_55 VALUES IN ('55'),
+    PARTITION p_56 VALUES IN ('56'),
+    PARTITION p_AL VALUES IN ('AL'),
+    PARTITION p_AT VALUES IN ('AT'),
+    PARTITION p_BE VALUES IN ('BE'),
+    PARTITION p_BG VALUES IN ('BG'),
+    PARTITION p_CH VALUES IN ('CH'),
+    PARTITION p_CY VALUES IN ('CY'),
+    PARTITION p_CZ VALUES IN ('CZ'),
+    PARTITION p_DE VALUES IN ('DE'),
+    PARTITION p_DK VALUES IN ('DK'),
+    PARTITION p_EE VALUES IN ('EE'),
+    PARTITION p_EL VALUES IN ('EL'),
+    PARTITION p_ES VALUES IN ('ES'),
+    PARTITION p_FI VALUES IN ('FI'),
+    PARTITION p_FR VALUES IN ('FR'),
+    PARTITION p_GB VALUES IN ('GB'),
+    PARTITION p_GR VALUES IN ('GR'),
+    PARTITION p_HR VALUES IN ('HR'),
+    PARTITION p_HU VALUES IN ('HU'),
+    PARTITION p_IE VALUES IN ('IE'),
+    PARTITION p_IS VALUES IN ('IS'),
+    PARTITION p_IT VALUES IN ('IT'),
+    PARTITION p_LI VALUES IN ('LI'),
+    PARTITION p_LT VALUES IN ('LT'),
+    PARTITION p_LU VALUES IN ('LU'),
+    PARTITION p_LV VALUES IN ('LV'),
+    PARTITION p_ME VALUES IN ('ME'),
+    PARTITION p_MK VALUES IN ('MK'),
+    PARTITION p_MT VALUES IN ('MT'),
+    PARTITION p_NL VALUES IN ('NL'),
+    PARTITION p_NO VALUES IN ('NO'),
+    PARTITION p_PL VALUES IN ('PL'),
+    PARTITION p_PT VALUES IN ('PT'),
+    PARTITION p_RO VALUES IN ('RO'),
+    PARTITION p_RS VALUES IN ('RS'),
+    PARTITION p_SE VALUES IN ('SE'),
+    PARTITION p_SI VALUES IN ('SI'),
+    PARTITION p_SK VALUES IN ('SK'),
+    PARTITION p_TR VALUES IN ('TR'),
+    PARTITION p_UA VALUES IN ('UA'),
+    PARTITION p_UK VALUES IN ('UK')
+);

inputs/sql/partition_db.sql

-ALTER TABLE FACT_bldgs.footprints
-MODIFY COLUMN id VARCHAR(2);
-ALTER TABLE FACT_bldgs.footprints
-PARTITION BY LIST COLUMNS (id) (
-    PARTITION p_unknown VALUES IN (''),
-    PARTITION p_01 VALUES IN ('01'),
-    PARTITION p_02 VALUES IN ('02'),
-    PARTITION p_04 VALUES IN ('04'),
-    PARTITION p_05 VALUES IN ('05'),
-    PARTITION p_06 VALUES IN ('06'),
-    PARTITION p_08 VALUES IN ('08'),
-    PARTITION p_09 VALUES IN ('09'),
-    PARTITION p_10 VALUES IN ('10'),
-    PARTITION p_11 VALUES IN ('11'),
-    PARTITION p_12 VALUES IN ('12'),
-    PARTITION p_13 VALUES IN ('13'),
-    PARTITION p_15 VALUES IN ('15'),
-    PARTITION p_16 VALUES IN ('16'),
-    PARTITION p_17 VALUES IN ('17'),
-    PARTITION p_18 VALUES IN ('18'),
-    PARTITION p_19 VALUES IN ('19'),
-    PARTITION p_20 VALUES IN ('20'),
-    PARTITION p_21 VALUES IN ('21'),
-    PARTITION p_22 VALUES IN ('22'),
-    PARTITION p_23 VALUES IN ('23'),
-    PARTITION p_24 VALUES IN ('24'),
-    PARTITION p_25 VALUES IN ('25'),
-    PARTITION p_26 VALUES IN ('26'),
-    PARTITION p_27 VALUES IN ('27'),
-    PARTITION p_28 VALUES IN ('28'),
-    PARTITION p_29 VALUES IN ('29'),
-    PARTITION p_30 VALUES IN ('30'),
-    PARTITION p_31 VALUES IN ('31'),
-    PARTITION p_32 VALUES IN ('32'),
-    PARTITION p_33 VALUES IN ('33'),
-    PARTITION p_34 VALUES IN ('34'),
-    PARTITION p_35 VALUES IN ('35'),
-    PARTITION p_36 VALUES IN ('36'),
-    PARTITION p_37 VALUES IN ('37'),
-    PARTITION p_38 VALUES IN ('38'),
-    PARTITION p_39 VALUES IN ('39'),
-    PARTITION p_40 VALUES IN ('40'),
-    PARTITION p_41 VALUES IN ('41'),
-    PARTITION p_42 VALUES IN ('42'),
-    PARTITION p_44 VALUES IN ('44'),
-    PARTITION p_45 VALUES IN ('45'),
-    PARTITION p_46 VALUES IN ('46'),
-    PARTITION p_47 VALUES IN ('47'),
-    PARTITION p_48 VALUES IN ('48'),
-    PARTITION p_49 VALUES IN ('49'),
-    PARTITION p_50 VALUES IN ('50'),
-    PARTITION p_51 VALUES IN ('51'),
-    PARTITION p_53 VALUES IN ('53'),
-    PARTITION p_54 VALUES IN ('54'),
-    PARTITION p_55 VALUES IN ('55'),
-    PARTITION p_56 VALUES IN ('56'),
-    PARTITION p_AL VALUES IN ('AL'),
-    PARTITION p_AT VALUES IN ('AT'),
-    PARTITION p_BE VALUES IN ('BE'),
-    PARTITION p_BG VALUES IN ('BG'),
-    PARTITION p_CH VALUES IN ('CH'),
-    PARTITION p_CY VALUES IN ('CY'),
-    PARTITION p_CZ VALUES IN ('CZ'),
-    PARTITION p_DE VALUES IN ('DE'),
-    PARTITION p_DK VALUES IN ('DK'),
-    PARTITION p_EE VALUES IN ('EE'),
-    PARTITION p_EL VALUES IN ('EL'),
-    PARTITION p_ES VALUES IN ('ES'),
-    PARTITION p_FI VALUES IN ('FI'),
-    PARTITION p_FR VALUES IN ('FR'),
-    PARTITION p_GB VALUES IN ('GB'),
-    PARTITION p_GR VALUES IN ('GR'),
-    PARTITION p_HR VALUES IN ('HR'),
-    PARTITION p_HU VALUES IN ('HU'),
-    PARTITION p_IE VALUES IN ('IE'),
-    PARTITION p_IS VALUES IN ('IS'),
-    PARTITION p_IT VALUES IN ('IT'),
-    PARTITION p_LI VALUES IN ('LI'),
-    PARTITION p_LT VALUES IN ('LT'),
-    PARTITION p_LU VALUES IN ('LU'),
-    PARTITION p_LV VALUES IN ('LV'),
-    PARTITION p_ME VALUES IN ('ME'),
-    PARTITION p_MK VALUES IN ('MK'),
-    PARTITION p_MT VALUES IN ('MT'),
-    PARTITION p_NL VALUES IN ('NL'),
-    PARTITION p_NO VALUES IN ('NO'),
-    PARTITION p_PL VALUES IN ('PL'),
-    PARTITION p_PT VALUES IN ('PT'),
-    PARTITION p_RO VALUES IN ('RO'),
-    PARTITION p_RS VALUES IN ('RS'),
-    PARTITION p_SE VALUES IN ('SE'),
-    PARTITION p_SI VALUES IN ('SI'),
-    PARTITION p_SK VALUES IN ('SK'),
-    PARTITION p_TR VALUES IN ('TR'),
-    PARTITION p_UA VALUES IN ('UA'),
-    PARTITION p_UK VALUES IN ('UK')
-);