Completed OSM filtering, geojson export and DB import.

gen_rail_data.sh

@@ -6,8 +6,5 @@
 # load maps and filter them
 ./inputs/load_maps.sh
 
-# load NUTS objects into DB
-./inputs/load_table_data.sh
-
 # dump data and drop database on 'local' instance
 ./inputs/dump_and_drop.sh

inputs/dump_and_drop.sh

+echo "*** Dump database:"
+mkdir -p ./dump
+mariadb-dump -h "plazablanca.nilu.no" -u "root" -pimpadminPSWD! --databases FACT_rail > ./dump/db_rail.sql
+
+echo "*** Drop database:"
+mariadb -h "plazablanca.nilu.no" -u "root" -pimpadminPSWD!  < inputs/sql/drop_db.sql

inputs/load_maps.sh

@@ -3,12 +3,33 @@
 # create a temp dir
 mkdir -p temp
 
+# Europe
 # download maps from http://download.geofabrik.de/
 wget http://download.geofabrik.de/europe-latest.osm.pbf -P temp
-wget http://download.geofabrik.de/north-america-latest.osm.pbf -P temp
-
-
-
+# export rail lines
+osmium tags-filter temp/europe-latest.osm.pbf w/railway=rail -o temp/line_europe.pbf --overwrite
+osmium export temp/line_europe.pbf --geometry-types=linestring -o temp/line_europe.geojson --overwrite
+# export stations
+osmium tags-filter temp/europe-latest.osm.pbf n/railway=station -R -o temp/station_europe.pbf --overwrite
+osmium tags-filter temp/station_europe.pbf n/railway:traffic_mode -R -o temp/station_europe.pbf --overwrite
+osmium export temp/station_europe.pbf -o temp/station_europe.geojson --overwrite
+# load in DB
+ogr2ogr -f MySQL MySQL:FACT_rail,host=plazablanca.nilu.no,user=root,password=impadminPSWD! ./temp/station_europe.geojson -nln eu_stations -update -overwrite -lco engine=Aria
+ogr2ogr -f MySQL MySQL:FACT_rail,host=plazablanca.nilu.no,user=root,password=impadminPSWD! ./temp/line_europe.geojson -nln eu_lines -update -overwrite -lco engine=Aria
+#clean
+rm temp/*.*
 
+# North America
+# download maps from http://download.geofabrik.de/
+wget http://download.geofabrik.de/north-america-latest.osm.pbf -P temp
+osmium tags-filter temp/north-america-latest.osm.pbf w/railway=rail -o temp/line_north-america.pbf --overwrite
+osmium export temp/line_north-america.pbf --geometry-types=linestring -o temp/line_north-america.geojson --overwrite
+# export stations
+osmium tags-filter temp/north-america-latest.osm.pbf n/railway=station -R -o temp/station_north-america.pbf --overwrite
+osmium tags-filter temp/station_north-america.pbf n/railway:traffic_mode -R -o temp/station_north-america.pbf --overwrite
+osmium export temp/station_north-america.pbf -o temp/station_north-america.geojson --overwrite
+# load in DB
+ogr2ogr -f MySQL MySQL:FACT_rail,host=plazablanca.nilu.no,user=root,password=impadminPSWD! ./temp/station_north-america.geojson -nln na_stations -update -overwrite -lco engine=Aria
+ogr2ogr -f MySQL MySQL:FACT_rail,host=plazablanca.nilu.no,user=root,password=impadminPSWD! ./temp/line_north-america.geojson -nln na_lines -update -overwrite -lco engine=Aria
 #clean up
 rm -r temp

inputs/setup_db.sh

 echo "*** Create database:"
 mariadb -h "plazablanca.nilu.no" -u "root" -pimpadminPSWD!  < inputs/sql/create_db.sql
-echo "*** Create tables:"
-mariadb -h "plazablanca.nilu.no" -u "root" -pimpadminPSWD!  < inputs/sql/create_tables.sql

inputs/sql/create_tables.sql

-CREATE TABLE FACT_population.eu_grid_2006_pop (
-  grd_id varchar(50) NOT NULL,
-  pop_tot INT NULL,
-  year INT NOT NULL,
-  methd_cl varchar(10) NULL,
-  cntr_code varchar(30) NULL,
-  data_src varchar(10) NULL,
-  PRIMARY KEY (grd_id, cntr_code, year)
-)
-ENGINE=Aria
-DEFAULT CHARSET=utf8mb4
-COLLATE=utf8mb4_general_ci;
-
-CREATE TABLE FACT_population.eu_grid_2011_pop (
-  pop_tot INT NULL,
-  grd_id varchar(50) NOT NULL,
-  cntr_code varchar(30) NULL,
-  methd_cl varchar(10) NULL,
-  year INT NOT NULL,
-  data_src varchar(10) NULL,
-  tot_p_con_dt varchar(10) NULL,
-  PRIMARY KEY (grd_id, cntr_code)
-)
-ENGINE=Aria
-DEFAULT CHARSET=utf8mb4
-COLLATE=utf8mb4_general_ci;
-
-CREATE TABLE FACT_population.eu_lau (
-	CNTR_CODE varchar(2) NOT NULL,
-	CNTR_LAU_CODE varchar(20) NOT NULL,
-	LAU_LABEL varchar(100) NULL,
-	POP_1961_01_01 DOUBLE NULL,
-	POP_1971_01_01 DOUBLE NULL,
-	POP_1981_01_01 DOUBLE NULL,
-	POP_1991_01_01 DOUBLE NULL,
-	POP_2001_01_01 DOUBLE NULL,
-	POP_2011_01_01 DOUBLE NULL,
-  	PRIMARY KEY (CNTR_LAU_CODE)
-)
-ENGINE=Aria
-DEFAULT CHARSET=utf8mb4
-COLLATE=utf8mb4_general_ci;
-
-CREATE TABLE FACT_population.eu_nuts (
-  GeoID varchar(10) NOT NULL,
-  Year INT NOT NULL,
-  AgeClass varchar(6) NOT NULL,
-	Pop_M INT NULL,
-	Pop_F INT NULL,
-  PRIMARY KEY (GeoID,Year,AgeClass)
-)
-ENGINE=Aria
-DEFAULT CHARSET=utf8mb4
-COLLATE=utf8mb4_general_ci;
-
-CREATE TABLE FACT_population.us_county (
-  GeoID varchar(10) NOT NULL,
-  Year INT NOT NULL,
-  AgeClass varchar(6) NOT NULL,
-	Pop_M INT NULL,
-	Pop_F INT NULL,
-  PRIMARY KEY (GeoID,Year,AgeClass)
-)
-ENGINE=Aria
-DEFAULT CHARSET=utf8mb4
-COLLATE=utf8mb4_general_ci;
-
-CREATE TABLE FACT_population.us_block_2020 (
-  H1_001N INT,
-  H9_001N INT,
-  P1_001N INT,
-  P12_003N INT,
-  P12_004N INT,
-  P12_005N INT,
-  P12_006N INT,
-  P12_007N INT,
-  P12_008N INT,
-  P12_009N INT,
-  P12_010N INT,
-  P12_011N INT,
-  P12_012N INT,
-  P12_013N INT,
-  P12_014N INT,
-  P12_015N INT,
-  P12_016N INT,
-  P12_017N INT,
-  P12_018N INT,
-  P12_019N INT,
-  P12_020N INT,
-  P12_021N INT,
-  P12_022N INT,
-  P12_023N INT,
-  P12_024N INT,
-  P12_025N INT,
-  P12_027N INT,
-  P12_028N INT,
-  P12_029N INT,
-  P12_030N INT,
-  P12_031N INT,
-  P12_032N INT,
-  P12_033N INT,
-  P12_034N INT,
-  P12_035N INT,
-  P12_036N INT,
-  P12_037N INT,
-  P12_038N INT,
-  P12_039N INT,
-  P12_040N INT,
-  P12_041N INT,
-  P12_042N INT,
-  P12_043N INT,
-  P12_044N INT,
-  P12_045N INT,
-  P12_046N INT,
-  P12_047N INT,
-  P12_048N INT,
-  P12_049N INT,
-  state INT NOT NULL,
-  county INT NOT NULL,
-  tract INT NOT NULL,
-  block INT NOT NULL,
-  PRIMARY KEY (state,county,tract,block)
-)
-ENGINE=Aria
-DEFAULT CHARSET=utf8mb4
-COLLATE=utf8mb4_general_ci;