diff --git a/inputs/download_transform.py b/inputs/download_transform.py index 531363776ba94b9decea3afa8b6c4504e93ac7fc..49a5d0be43c31a6fe4eb52eed48e83e1516221c1 100644 --- a/inputs/download_transform.py +++ b/inputs/download_transform.py @@ -1,3 +1,4 @@ +import os import pandas as pd import geopandas as gpd from shapely.geometry import shape @@ -7,15 +8,19 @@ from concurrent.futures import ThreadPoolExecutor interested_locations = ['Austria', 'Belgium', 'Bulgaria', 'Croatia', 'Cyprus', 'CzechRepublic', 'Denmark', 'Estonia', 'Finland', 'France', 'Germany', 'Greece', 'Hungary', 'Ireland', 'Italy', 'Latvia', 'Lithuania', 'Luxembourg', 'Malta', 'Netherlands', 'Poland', 'Portugal', 'Romania', 'Slovakia', 'Slovenia', 'Spain', 'Sweden', 'Iceland', 'Switzerland', 'Liechtenstein', 'Norway', 'UnitedKingdom', 'Ukraine', 'UnitedStates'] def process_file(row): - if row['Location'] in interested_locations: + file_path = f"data/{row['QuadKey']}.geojson" + if row['Location'] in interested_locations and not os.path.exists(file_path): df = pd.read_json(row['Url'], lines=True) df['geometry'] = df['geometry'].apply(shape) gdf = gpd.GeoDataFrame(df, crs=4326) - gdf.to_file(f"data/{row['QuadKey']}.geojson", driver="GeoJSON") + gdf.to_file(file_path, driver="GeoJSON") def main(): dataset_links = pd.read_csv("https://minedbuildings.blob.core.windows.net/global-buildings/dataset-links.csv") + if not os.path.exists('data'): + os.makedirs('data') + # Use ThreadPoolExecutor to process files in parallel with ThreadPoolExecutor(max_workers=20) as executor: executor.map(process_file, dataset_links.to_dict(orient='records'))