From 045ca0bf1475396d16c6e581047e2c0307b8e8bb Mon Sep 17 00:00:00 2001
From: Riccardo Boero <ribo@nilu.no>
Date: Thu, 11 Apr 2024 09:48:59 +0200
Subject: [PATCH] Pumped up size and duration parameters of MySQL queries.
---
src/FACT_air.jl | 17 +++++++++++++---
src/FACT_bldgs.jl | 17 ++++++++++++++--
src/FACT_jobs.jl | 40 +++++++++++++++++++++++++++++++++-----
src/FACT_mine.jl | 21 ++++++++++++++------
src/FACT_population.jl | 40 +++++++++++++++++++++++++++++++++-----
src/FACT_ports.jl | 25 +++++++++++++++++++++---
src/FACT_power.jl | 17 ++++++++++++++--
src/FACT_rail.jl | 34 ++++++++++++++++++++++++++++----
src/utils_DB_connection.jl | 27 +++++++++++++++++++++++++
9 files changed, 208 insertions(+), 30 deletions(-)
diff --git a/src/FACT_air.jl b/src/FACT_air.jl
index 5263424..108447f 100644
--- a/src/FACT_air.jl
+++ b/src/FACT_air.jl
@@ -32,7 +32,13 @@ function eu_air(conn_dict::Dict, selection::Dict)
# create connection
conn = establish_connection(conn_dict["host"], conn_dict["user"], conn_dict["password"], conn_dict["database"]; port=conn_dict["port"])
# execute
- result = query_connection(conn, query)
+ result = DataFrame()
+ try
+ # Execute session variable adjustments
+ result = query_connection(conn, query)
+ catch ex
+ @error "Failed to set session variables for query $query with expection " exception=(ex, catch_backtrace())
+ end
# close connection
close_connection(conn)
# Check if the result is empty
@@ -72,7 +78,6 @@ This function queries a database for air traffic information, such as the number
This example queries air traffic data for specified U.S. regions and returns the results in a DataFrame.
"""
function us_air(conn_dict::Dict, selection::Dict)
-
# getting valuable info from dictionary: acceptable years are 1961, 71, 81, 91, 2001, 2011
geo_id = selection["geo_id"]
shape_obj = selection["shape_obj"]
@@ -85,7 +90,13 @@ function us_air(conn_dict::Dict, selection::Dict)
# create connection
conn = establish_connection(conn_dict["host"], conn_dict["user"], conn_dict["password"], conn_dict["database"]; port=conn_dict["port"])
# execute
- result = query_connection(conn, query)
+ result = DataFrame()
+ try
+ # Execute session variable adjustments
+ result = query_connection(conn, query)
+ catch ex
+ @error "Failed to set session variables for query $query with expection " exception=(ex, catch_backtrace())
+ end
# close connection
close_connection(conn)
# Check if the result is empty
diff --git a/src/FACT_bldgs.jl b/src/FACT_bldgs.jl
index ae8ad28..6b76ac0 100644
--- a/src/FACT_bldgs.jl
+++ b/src/FACT_bldgs.jl
@@ -34,8 +34,21 @@ function footprint_avg_height(conn_dict::Dict, selection::Dict)
# create connection
conn = establish_connection(conn_dict["host"], conn_dict["user"], conn_dict["password"], conn_dict["database"]; port=conn_dict["port"])
# execute
- result = query_connection(conn, query)
- result2 = query_connection(conn, query2)
+ result = DataFrame()
+ try
+ # Execute session variable adjustments
+ result = query_connection(conn, query)
+ catch ex
+ @error "Failed to set session variables for query $query with expection " exception=(ex, catch_backtrace())
+ end
+ # execute
+ result2 = DataFrame()
+ try
+ # Execute session variable adjustments
+ result2 = query_connection(conn, query2)
+ catch ex
+ @error "Failed to set session variables for query $query2 with expection " exception=(ex, catch_backtrace())
+ end
# close connection
close_connection(conn)
# execute
diff --git a/src/FACT_jobs.jl b/src/FACT_jobs.jl
index 91be5cf..070dd5e 100644
--- a/src/FACT_jobs.jl
+++ b/src/FACT_jobs.jl
@@ -37,7 +37,13 @@ function us_lodes(conn_dict::Dict, selection::Dict)
# create connection
conn = establish_connection(conn_dict["host"], conn_dict["user"], conn_dict["password"], conn_dict["database"]; port=conn_dict["port"])
# execute
- result = query_connection(conn, query)
+ result = DataFrame()
+ try
+ # Execute session variable adjustments
+ result = query_connection(conn, query)
+ catch ex
+ @error "Failed to set session variables for query $query with expection " exception=(ex, catch_backtrace())
+ end
# close connection
close_connection(conn)
long_df = stack(result, names(result), variable_name=:industry, value_name=:jobs)
@@ -103,7 +109,13 @@ function us_qcew(conn_dict::Dict, selection::Dict)
# create connection
conn = establish_connection(conn_dict["host"], conn_dict["user"], conn_dict["password"], conn_dict["database"]; port=conn_dict["port"])
# execute
- result = query_connection(conn, query)
+ result = DataFrame()
+ try
+ # Execute session variable adjustments
+ result = query_connection(conn, query)
+ catch ex
+ @error "Failed to set session variables for query $query with expection " exception=(ex, catch_backtrace())
+ end
# close connection
close_connection(conn)
# Check if the 'industry' column exists
@@ -148,7 +160,13 @@ function eu_lfs(conn_dict::Dict, selection::Dict)
# create connection
conn = establish_connection(conn_dict["host"], conn_dict["user"], conn_dict["password"], conn_dict["database"]; port=conn_dict["port"])
# execute
- result = query_connection(conn, query)
+ result = DataFrame()
+ try
+ # Execute session variable adjustments
+ result = query_connection(conn, query)
+ catch ex
+ @error "Failed to set session variables for query $query with expection " exception=(ex, catch_backtrace())
+ end
# close connection
close_connection(conn)
if !("industry" in names(result))
@@ -192,7 +210,13 @@ function eu_sbs(conn_dict::Dict, selection::Dict)
# create connection
conn = establish_connection(conn_dict["host"], conn_dict["user"], conn_dict["password"], conn_dict["database"]; port=conn_dict["port"])
# execute
- result = query_connection(conn, query)
+ result = DataFrame()
+ try
+ # Execute session variable adjustments
+ result = query_connection(conn, query)
+ catch ex
+ @error "Failed to set session variables for query $query with expection " exception=(ex, catch_backtrace())
+ end
# close connection
close_connection(conn)
if !("industry" in names(result))
@@ -236,7 +260,13 @@ function eu_rea(conn_dict::Dict, selection::Dict)
# create connection
conn = establish_connection(conn_dict["host"], conn_dict["user"], conn_dict["password"], conn_dict["database"]; port=conn_dict["port"])
# execute
- result = query_connection(conn, query)
+ result = DataFrame()
+ try
+ # Execute session variable adjustments
+ result = query_connection(conn, query)
+ catch ex
+ @error "Failed to set session variables for query $query with expection " exception=(ex, catch_backtrace())
+ end
# close connection
close_connection(conn)
if !("industry" in names(result))
diff --git a/src/FACT_mine.jl b/src/FACT_mine.jl
index 795e447..64ee8f4 100644
--- a/src/FACT_mine.jl
+++ b/src/FACT_mine.jl
@@ -21,9 +21,6 @@ This example retrieves geology data for two geographic areas in the "US" region
"""
function geology_full(conn_dict::Dict, selection::Dict)
- # temporarily increase max_allowed_packet
- size_query = "SET GLOBAL max_allowed_packet=1073741824;" #Sets max_allowed_packet to 1GB
-
# getting valuable info from dictionary
geo_id = selection["geo_id"]
shape_obj = selection["shape_obj"]
@@ -66,8 +63,13 @@ function geology_full(conn_dict::Dict, selection::Dict)
# create connection
conn = establish_connection(conn_dict["host"], conn_dict["user"], conn_dict["password"], conn_dict["database"]; port=conn_dict["port"])
# execute
- DBInterface.execute(conn, size_query)
- result = query_connection(conn, query)
+ result = DataFrame()
+ try
+ # Execute session variable adjustments
+ result = query_connection(conn, query)
+ catch ex
+ @error "Failed to set session variables for query $query with expection " exception=(ex, catch_backtrace())
+ end
# close connection
close_connection(conn)
# Check if the result is empty
@@ -123,7 +125,14 @@ function minerals_full(conn_dict::Dict, selection::Dict)
for table in tables
# prepare query
query = "SELECT SUM(ST_Area(ST_Intersection(SHAPE, ST_GeomFromText('$shape')))) / $ar AS $table FROM $table WHERE ST_Intersects(SHAPE, ST_GeomFromText('$shape'));"
- result2 = query_connection(conn, query)
+ # execute
+ result2 = DataFrame()
+ try
+ # Execute session variable adjustments
+ result2 = query_connection(conn, query)
+ catch ex
+ @error "Failed to set session variables for query $query with expection " exception=(ex, catch_backtrace())
+ end
result = hcat(result, result2)
end
# close connection
diff --git a/src/FACT_population.jl b/src/FACT_population.jl
index dc45a05..554b5a0 100644
--- a/src/FACT_population.jl
+++ b/src/FACT_population.jl
@@ -34,7 +34,13 @@ function eu_lau(conn_dict::Dict, selection::Dict)
# create connection
conn = establish_connection(conn_dict["host"], conn_dict["user"], conn_dict["password"], conn_dict["database"]; port=conn_dict["port"])
# execute
- result = query_connection(conn, query)
+ result = DataFrame()
+ try
+ # Execute session variable adjustments
+ result = query_connection(conn, query)
+ catch ex
+ @error "Failed to set session variables for query $query with expection " exception=(ex, catch_backtrace())
+ end
# close connection
close_connection(conn)
# Check if the result is empty
@@ -86,7 +92,13 @@ function eu_nuts(conn_dict::Dict, selection::Dict)
# create connection
conn = establish_connection(conn_dict["host"], conn_dict["user"], conn_dict["password"], conn_dict["database"]; port=conn_dict["port"])
# execute
- result = query_connection(conn, query)
+ result = DataFrame()
+ try
+ # Execute session variable adjustments
+ result = query_connection(conn, query)
+ catch ex
+ @error "Failed to set session variables for query $query with expection " exception=(ex, catch_backtrace())
+ end
# close connection
close_connection(conn)
# Check if the result is empty
@@ -143,7 +155,13 @@ function us_block(conn_dict::Dict, selection::Dict)
# create connection
conn = establish_connection(conn_dict["host"], conn_dict["user"], conn_dict["password"], conn_dict["database"]; port=conn_dict["port"])
# execute
- result = query_connection(conn, query)
+ result = DataFrame()
+ try
+ # Execute session variable adjustments
+ result = query_connection(conn, query)
+ catch ex
+ @error "Failed to set session variables for query $query with expection " exception=(ex, catch_backtrace())
+ end
# close connection
close_connection(conn)
# Check if the result is empty
@@ -164,7 +182,13 @@ function us_block(conn_dict::Dict, selection::Dict)
# create connection
conn = establish_connection(conn_dict["host"], conn_dict["user"], conn_dict["password"], conn_dict["database"]; port=conn_dict["port"])
# execute
- result = query_connection(conn, query)
+ result = DataFrame()
+ try
+ # Execute session variable adjustments
+ result = query_connection(conn, query)
+ catch ex
+ @error "Failed to set session variables for query $query with expection " exception=(ex, catch_backtrace())
+ end
# close connection
close_connection(conn)
# Check if the result is empty
@@ -217,7 +241,13 @@ function us_county(conn_dict::Dict, selection::Dict)
# create connection
conn = establish_connection(conn_dict["host"], conn_dict["user"], conn_dict["password"], conn_dict["database"]; port=conn_dict["port"])
# execute
- result = query_connection(conn, query)
+ result = DataFrame()
+ try
+ # Execute session variable adjustments
+ result = query_connection(conn, query)
+ catch ex
+ @error "Failed to set session variables for query $query with expection " exception=(ex, catch_backtrace())
+ end
# close connection
close_connection(conn)
# Check if the result is empty
diff --git a/src/FACT_ports.jl b/src/FACT_ports.jl
index 0e30d5c..a39d569 100644
--- a/src/FACT_ports.jl
+++ b/src/FACT_ports.jl
@@ -33,8 +33,21 @@ function eu_ports(conn_dict::Dict, selection::Dict)
# create connection
conn = establish_connection(conn_dict["host"], conn_dict["user"], conn_dict["password"], conn_dict["database"]; port=conn_dict["port"])
# execute
- result = query_connection(conn, query)
- result2 = query_connection(conn, query2)
+ result = DataFrame()
+ try
+ # Execute session variable adjustments
+ result = query_connection(conn, query)
+ catch ex
+ @error "Failed to set session variables for query $query with expection " exception=(ex, catch_backtrace())
+ end
+ # execute
+ result2 = DataFrame()
+ try
+ # Execute session variable adjustments
+ result2 = query_connection(conn, query2)
+ catch ex
+ @error "Failed to set session variables for query $query2 with expection " exception=(ex, catch_backtrace())
+ end
# close connection
close_connection(conn)
result = hcat(result, result2)
@@ -87,7 +100,13 @@ function us_ports(conn_dict::Dict, selection::Dict)
# create connection
conn = establish_connection(conn_dict["host"], conn_dict["user"], conn_dict["password"], conn_dict["database"]; port=conn_dict["port"])
# execute
- result = query_connection(conn, query)
+ result = DataFrame()
+ try
+ # Execute session variable adjustments
+ result = query_connection(conn, query)
+ catch ex
+ @error "Failed to set session variables for query $query with expection " exception=(ex, catch_backtrace())
+ end
# close connection
close_connection(conn)
# Check if the result is empty
diff --git a/src/FACT_power.jl b/src/FACT_power.jl
index f6d169d..ad2ef75 100644
--- a/src/FACT_power.jl
+++ b/src/FACT_power.jl
@@ -89,8 +89,21 @@ function us_power(conn_dict::Dict, selection::Dict)
# create connection
conn = establish_connection(conn_dict["host"], conn_dict["user"], conn_dict["password"], conn_dict["database"]; port=conn_dict["port"])
# execute
- result = query_connection(conn, query)
- result2 = query_connection(conn, query2)
+ result = DataFrame()
+ try
+ # Execute session variable adjustments
+ result = query_connection(conn, query)
+ catch ex
+ @error "Failed to set session variables for query $query with expection " exception=(ex, catch_backtrace())
+ end
+ # execute
+ result2 = DataFrame()
+ try
+ # Execute session variable adjustments
+ result2 = query_connection(conn, query2)
+ catch ex
+ @error "Failed to set session variables for query $query2 with expection " exception=(ex, catch_backtrace())
+ end
# close connection
close_connection(conn)
result = hcat(result, result2)
diff --git a/src/FACT_rail.jl b/src/FACT_rail.jl
index 2c615ac..2138048 100644
--- a/src/FACT_rail.jl
+++ b/src/FACT_rail.jl
@@ -33,8 +33,21 @@ function eu_rail(conn_dict::Dict, selection::Dict)
# create connection
conn = establish_connection(conn_dict["host"], conn_dict["user"], conn_dict["password"], conn_dict["database"]; port=conn_dict["port"])
# execute
- result = query_connection(conn, query)
- result2 = query_connection(conn, query2)
+ result = DataFrame()
+ try
+ # Execute session variable adjustments
+ result = query_connection(conn, query)
+ catch ex
+ @error "Failed to set session variables for query $query with expection " exception=(ex, catch_backtrace())
+ end
+ # execute
+ result2 = DataFrame()
+ try
+ # Execute session variable adjustments
+ result2 = query_connection(conn, query2)
+ catch ex
+ @error "Failed to set session variables for query $query2 with expection " exception=(ex, catch_backtrace())
+ end
# close connection
close_connection(conn)
result = hcat(result, result2)
@@ -88,8 +101,21 @@ function us_rail(conn_dict::Dict, selection::Dict)
# create connection
conn = establish_connection(conn_dict["host"], conn_dict["user"], conn_dict["password"], conn_dict["database"]; port=conn_dict["port"])
# execute
- result = query_connection(conn, query)
- result2 = query_connection(conn, query2)
+ result = DataFrame()
+ try
+ # Execute session variable adjustments
+ result = query_connection(conn, query)
+ catch ex
+ @error "Failed to set session variables for query $query with expection " exception=(ex, catch_backtrace())
+ end
+ # execute
+ result2 = DataFrame()
+ try
+ # Execute session variable adjustments
+ result2 = query_connection(conn, query2)
+ catch ex
+ @error "Failed to set session variables for query $query2 with expection " exception=(ex, catch_backtrace())
+ end
# close connection
close_connection(conn)
result = hcat(result, result2)
diff --git a/src/utils_DB_connection.jl b/src/utils_DB_connection.jl
index 58d0e12..d951005 100644
--- a/src/utils_DB_connection.jl
+++ b/src/utils_DB_connection.jl
@@ -14,7 +14,33 @@ Establish a connection to a MySQL/MariaDB database.
- `MySQL.Connection`: A connection object to the database.
"""
function establish_connection(host::String, user::String, password::String, dbname::String; port::Int=3306)
+ # Establish connection
conn = MySQL.Connection(host, user, password, dbname, port, "nothing")
+ # Temporarily increase max_allowed_packet
+ size_query = "SET SESSION max_allowed_packet=1073741824;" # Sets max_allowed_packet to 1GB
+ timeout_query = "SET SESSION wait_timeout = 31536000;" # Sets wait_timeout to 1 year
+ maxtime_query = "SET SESSION max_execution_time = 604800000;" # Sets max_execution_time to 1 week
+ try
+ # Execute session variable adjustments
+ DBInterface.execute(conn, size_query)
+ catch ex
+ @error "Failed to set session variables for query $size_query with expection " exception=(ex, catch_backtrace())
+ # Handle error, possibly rethrow or clean up
+ end
+ try
+ # Execute session variable adjustments
+ DBInterface.execute(conn, timeout_query)
+ catch ex
+ @error "Failed to set session variables for query $timeout_query with expection " exception=(ex, catch_backtrace())
+ # Handle error, possibly rethrow or clean up
+ end
+ try
+ # Execute session variable adjustments
+ DBInterface.execute(conn, maxtime_query)
+ catch ex
+ @error "Failed to set session variables for query $maxtime_query with expection " exception=(ex, catch_backtrace())
+ # Handle error, possibly rethrow or clean up
+ end
return conn
end
@@ -39,6 +65,7 @@ Each column in the DataFrame is of type `String`.
- It converts all columns in the query result to `String` to avoid type interpretation issues.
"""
function query_connection(conn::MySQL.Connection, query::String)
+ #println("Executing query: ", query)
cursor = DBInterface.execute(conn, query)
cursor.types = [String for _ in cursor.types]
rows = Tables.rows(cursor)
--
GitLab