From 9a571132c8226cdcf47de2feb3cc76d571c07a64 Mon Sep 17 00:00:00 2001 From: Davide Passoni Date: Fri, 23 Feb 2024 15:54:49 +0100 Subject: [PATCH] Refined automatic map generation script --- .gitignore | 1 + scripts/python/generateMaps/capture_screen.py | 81 ------- .../configs/LasVegas/LasVegas.yml | 4 - scripts/python/generateMaps/generate_tiles.py | 49 ---- .../.vscode/launch.json | 2 +- .../configs/LasVegas/LasVegas.kml | 47 +++- .../configs/LasVegas/LasVegas.yml | 4 + .../configs/screen_properties.yml | 0 .../{generateMaps => map_generator}/main.py | 17 +- scripts/python/map_generator/map_generator.py | 224 ++++++++++++++++++ .../requirements.txt | 0 11 files changed, 274 insertions(+), 155 deletions(-) delete mode 100644 scripts/python/generateMaps/capture_screen.py delete mode 100644 scripts/python/generateMaps/configs/LasVegas/LasVegas.yml delete mode 100644 scripts/python/generateMaps/generate_tiles.py rename scripts/python/{generateMaps => map_generator}/.vscode/launch.json (93%) rename scripts/python/{generateMaps => map_generator}/configs/LasVegas/LasVegas.kml (52%) create mode 100644 scripts/python/map_generator/configs/LasVegas/LasVegas.yml rename scripts/python/{generateMaps => map_generator}/configs/screen_properties.yml (100%) rename scripts/python/{generateMaps => map_generator}/main.py (87%) create mode 100644 scripts/python/map_generator/map_generator.py rename scripts/python/{generateMaps => map_generator}/requirements.txt (100%) diff --git a/.gitignore b/.gitignore index 088a483d..c49b6812 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,4 @@ frontend/server/public/plugins/controltipsplugin/index.js frontend/website/plugins/controltips/index.js /frontend/server/public/maps *.pyc +/scripts/**/*.jpg diff --git a/scripts/python/generateMaps/capture_screen.py b/scripts/python/generateMaps/capture_screen.py deleted file mode 100644 index 6cc4ef1f..00000000 --- a/scripts/python/generateMaps/capture_screen.py +++ /dev/null @@ -1,81 +0,0 @@ -import math -import requests -import json -import pyautogui -import time -import os - -from pyproj import Geod -from fastkml import kml -from shapely import wkt - -def deg_to_num(lat_deg, lon_deg, zoom): - lat_rad = math.radians(lat_deg) - n = 1 << zoom - xtile = int((lon_deg + 180.0) / 360.0 * n) - ytile = int((1.0 - math.asinh(math.tan(lat_rad)) / math.pi) / 2.0 * n) - return xtile, ytile - -def num_to_deg(xtile, ytile, zoom): - n = 1 << zoom - lon_deg = xtile / n * 360.0 - 180.0 - lat_rad = math.atan(math.sinh(math.pi * (1 - 2 * ytile / n))) - lat_deg = math.degrees(lat_rad) - return lat_deg, lon_deg - -def run(map_config): - with open(map_config["boundary_file"], 'rt', encoding="utf-8") as bp: - # Read the config file - doc = bp.read() - k = kml.KML() - k.from_string(doc) - - geod = Geod(ellps="WGS84") - features = [f for f in list(k.features()) if not f.isopen] - print(f"Found {len(features)} closed features in the provided kml file") - - area = 0 - for feature in features: - for sub_feature in list(feature.features()): - geo = sub_feature.geometry - - start_lat = geo.bounds[1] - start_lng = geo.bounds[0] - end_lat = geo.bounds[3] - end_lng = geo.bound[2] - - # Find the starting and ending points - start_X, start_Y = deg_to_num(start_lat, start_lng, zoom) - end_X, end_Y = deg_to_num(end_lat, end_lng, zoom) - - time.sleep(2) - - # Create output folder - if not os.path.exists("output"): - os.mkdir("output") - - # Start looping - n = 1 - total = math.floor((end_X - start_X) / 2) * math.floor((end_Y - start_Y) / 2) - for X in range(start_X, end_X, 2): - for Y in range(start_Y, end_Y, 2): - # Find the center of the screen - center_lat, center_lng = num_to_deg(X + 1, Y + 1, zoom) - center_alt = camera_altitude(center_lat) - - # Making PUT request - data = json.dumps({'lat': center_lat, 'lng': center_lng, 'alt': center_alt}) - r = requests.put('http://localhost:8080', data = data) - - # Take and save screenshot - screenshot = pyautogui.screenshot() - screenshot.save(f"output/{X + 1}_{Y + 1}_{zoom}.png") - - time.sleep(0.5) - - print(f"Shot {n} of {total}") - n = n + 1 - - - - diff --git a/scripts/python/generateMaps/configs/LasVegas/LasVegas.yml b/scripts/python/generateMaps/configs/LasVegas/LasVegas.yml deleted file mode 100644 index b4f8eb93..00000000 --- a/scripts/python/generateMaps/configs/LasVegas/LasVegas.yml +++ /dev/null @@ -1,4 +0,0 @@ -{ - 'output_directory': './LasVegas', # Where to save the output files - 'boundary_file': './configs/LasVegas/LasVegas.kml' -} \ No newline at end of file diff --git a/scripts/python/generateMaps/generate_tiles.py b/scripts/python/generateMaps/generate_tiles.py deleted file mode 100644 index 05302ec9..00000000 --- a/scripts/python/generateMaps/generate_tiles.py +++ /dev/null @@ -1,49 +0,0 @@ -import os -from PIL import Image -import concurrent.futures -import math - -# correction parameters - -# NTTR -rotation = math.degrees(0.01895) -scale = 0.973384 - -zoom = 16 -path = "output" - -def crop_image(filename): - img = Image.open(os.path.join(path, filename)).rotate(-rotation) - img = img.resize((math.floor(img.width * scale), math.floor(img.height * scale))) - center_X, center_Y = filename.removesuffix(".png").split("_") - center_X = int(center_X) - center_Y = int(center_Y) - w, h = img.size - - box_top_left = (w / 2 - 256, h / 2 - 256, w / 2, h / 2) - box_top_right = (w / 2, h / 2 - 256, w / 2 + 256, h / 2) - box_bottom_left = (w / 2 - 256, h / 2, w / 2, h / 2 + 256) - box_bottom_right = (w / 2, h / 2, w / 2 + 256, h / 2 + 256) - - if not os.path.exists(f"output/{zoom}/{center_X - 1}"): - os.mkdir(f"output/{zoom}/{center_X - 1}") - - if not os.path.exists(f"output/{zoom}/{center_X}"): - os.mkdir(f"output/{zoom}/{center_X}") - - img.crop(box_top_left).save(f"output/{zoom}/{center_X - 1}/{center_Y - 1}.png") - img.crop(box_top_right).save(f"output/{zoom}/{center_X}/{center_Y - 1}.png") - img.crop(box_bottom_left).save(f"output/{zoom}/{center_X - 1}/{center_Y}.png") - img.crop(box_bottom_right).save(f"output/{zoom}/{center_X}/{center_Y}.png") - - return True - -filenames = [f for f in os.listdir(path) if os.path.isfile(os.path.join(path, f))] - -# Create output folder -if not os.path.exists(f"output/{zoom}"): - os.mkdir(f"output/{zoom}") - -with concurrent.futures.ThreadPoolExecutor() as executor: - futures = [executor.submit(crop_image, filename) for filename in filenames] - results = [future.result() for future in concurrent.futures.as_completed(futures)] \ No newline at end of file diff --git a/scripts/python/generateMaps/.vscode/launch.json b/scripts/python/map_generator/.vscode/launch.json similarity index 93% rename from scripts/python/generateMaps/.vscode/launch.json rename to scripts/python/map_generator/.vscode/launch.json index 01d13453..c5fdb6ca 100644 --- a/scripts/python/generateMaps/.vscode/launch.json +++ b/scripts/python/map_generator/.vscode/launch.json @@ -8,7 +8,7 @@ "name": "Python: Current File", "type": "python", "request": "launch", - "program": "${file}", + "program": "main.py", "console": "integratedTerminal", "args": ["./configs/LasVegas/LasVegas.yml"] } diff --git a/scripts/python/generateMaps/configs/LasVegas/LasVegas.kml b/scripts/python/map_generator/configs/LasVegas/LasVegas.kml similarity index 52% rename from scripts/python/generateMaps/configs/LasVegas/LasVegas.kml rename to scripts/python/map_generator/configs/LasVegas/LasVegas.kml index a401dd8c..b2da4f16 100644 --- a/scripts/python/generateMaps/configs/LasVegas/LasVegas.kml +++ b/scripts/python/map_generator/configs/LasVegas/LasVegas.kml @@ -2,7 +2,7 @@ Senza titolo - + - + - + normal - #__managed_style_1E6AF60F852F06CED14C + #__managed_style_1FB08D70372F0ACE3361 highlight - #__managed_style_29D7120C702F06CED14C + #__managed_style_2F9039BE1B2F0ACE3361 - + Poligono senza titolo - -115.7575513617584 - 36.45909683572987 - 1668.83938821393 + -115.0437621195802 + 36.2404454323581 + 568.1069300877758 0 0 35 - 382627.9679017514 + 21582.08160380367 absolute - #__managed_style_05AC9C65832F06CED14C + #__managed_style_0152B588AD2F0ACE3361 - -115.5765603362792,35.92113103850846,0 -114.7667743850415,35.914903046417,0 -114.7800419428627,36.39467581209903,0 -115.6198023719551,36.39886214564519,0 -115.5765603362792,35.92113103850846,0 + -115.0657741984423,36.20908708202413,0 -115.0064821223275,36.20969233542438,0 -115.0077003574054,36.25251471885595,0 -115.0604905801644,36.25236266770626,0 -115.0657741984423,36.20908708202413,0 + + + + + + + Poligono senza titolo + + -115.144039076036 + 36.07599823986274 + 636.6854074835677 + 0 + 0 + 35 + 14114.14487087633 + absolute + + #__managed_style_0152B588AD2F0ACE3361 + + + + + -115.1866576903507,36.06787728722109,0 -115.1107872218999,36.06765965614163,0 -115.1145241760975,36.10294242539007,0 -115.1779590799479,36.10179027153036,0 -115.1866576903507,36.06787728722109,0 diff --git a/scripts/python/map_generator/configs/LasVegas/LasVegas.yml b/scripts/python/map_generator/configs/LasVegas/LasVegas.yml new file mode 100644 index 00000000..404a661b --- /dev/null +++ b/scripts/python/map_generator/configs/LasVegas/LasVegas.yml @@ -0,0 +1,4 @@ +{ + 'output_directory': '.\LasVegas', # Where to save the output files + 'boundary_file': '.\configs\LasVegas\LasVegas.kml' +} \ No newline at end of file diff --git a/scripts/python/generateMaps/configs/screen_properties.yml b/scripts/python/map_generator/configs/screen_properties.yml similarity index 100% rename from scripts/python/generateMaps/configs/screen_properties.yml rename to scripts/python/map_generator/configs/screen_properties.yml diff --git a/scripts/python/generateMaps/main.py b/scripts/python/map_generator/main.py similarity index 87% rename from scripts/python/generateMaps/main.py rename to scripts/python/map_generator/main.py index d2f19ae7..79cac4a6 100644 --- a/scripts/python/generateMaps/main.py +++ b/scripts/python/map_generator/main.py @@ -5,7 +5,7 @@ from fastkml import kml from shapely import wkt from datetime import timedelta -import capture_screen +import map_generator if len(sys.argv) == 1: print("Please provide a configuration file as first argument. You can also drop the configuration file on this script to run it.") @@ -38,14 +38,15 @@ else: k.from_string(doc) geod = Geod(ellps="WGS84") - features = [f for f in list(k.features()) if not f.isopen] - print(f"Found {len(features)} closed features in the provided kml file") - + features = [] area = 0 - for feature in features: + for feature in k.features(): for sub_feature in list(feature.features()): geo = sub_feature.geometry area += abs(geod.geometry_area_perimeter(wkt.loads(geo.wkt))[0]) + features.append(sub_feature) + + print(f"Found {len(features)} features in the provided kml file") tile_size = 256 * screen_config["geo_resolution"] # meters tiles_per_screenshot = int(screen_config["width"] / 256) * int(screen_config["height"] / 256) @@ -57,10 +58,10 @@ else: print(f"Estimated number of tiles: {tiles_num}") print(f"Estimated number of screenshots: {screenshots_num}") print(f"Estimated time to complete: {timedelta(seconds=total_time)} (hh:mm:ss)") - print("The script is ready to go. After you press any key, it will wait for 5 seconds, and then it will start.") + print("The script is ready to go. After you press enter, it will wait for 5 seconds, then it will start.") - input("Press any key to continue...") - capture_screen.run(map_config) + input("Press enter to continue...") + map_generator.run(map_config) diff --git a/scripts/python/map_generator/map_generator.py b/scripts/python/map_generator/map_generator.py new file mode 100644 index 00000000..f5adaff2 --- /dev/null +++ b/scripts/python/map_generator/map_generator.py @@ -0,0 +1,224 @@ +import math +import requests +import pyautogui +import time +import os +import yaml + +from fastkml import kml +from shapely import wkt, Point +from PIL import Image +from concurrent import futures + +# global counters +fut_counter = 0 +tot_futs = 0 + +# constants +C = 40075016.686 # meters, Earth equatorial circumference + +def deg_to_num(lat_deg, lon_deg, zoom): + lat_rad = math.radians(lat_deg) + n = 1 << zoom + xtile = int((lon_deg + 180.0) / 360.0 * n) + ytile = int((1.0 - math.asinh(math.tan(lat_rad)) / math.pi) / 2.0 * n) + return xtile, ytile + +def num_to_deg(xtile, ytile, zoom): + n = 1 << zoom + lon_deg = xtile / n * 360.0 - 180.0 + lat_rad = math.atan(math.sinh(math.pi * (1 - 2 * ytile / n))) + lat_deg = math.degrees(lat_rad) + return lat_deg, lon_deg + +def compute_mpps(lat, z): + return C * math.cos(math.radians(lat)) / math.pow(2, z + 8) + +def printProgressBar(iteration, total, prefix = '', suffix = '', decimals = 1, length = 100, fill = '█', printEnd = "\r"): + percent = ("{0:." + str(decimals) + "f}").format(100 * (iteration / float(total))) + filledLength = int(length * iteration // total) + bar = fill * filledLength + '-' * (length - filledLength) + print(f'\r{prefix} |{bar}| {percent}% {suffix}', end = printEnd) + # Print New Line on Complete + if iteration == total: + print() + +def done_callback(fut): + global fut_counter, tot_futs + fut_counter += 1 + printProgressBar(fut_counter, tot_futs) + +def extract_tiles(n, screenshots_coordinates, params): + f = params["f"] + zoom = params["zoom"] + output_directory = params["output_directory"] + n_width = params["n_width"] + n_height = params["n_height"] + screen_resolution = params["screen_resolution"] + mpps = params["mpps"] + + coords = screenshots_coordinates[n] + if (os.path.exists(os.path.join(output_directory, "screenshots", f"{f}_{n}.jpg"))): + # Open the source screenshot + img = Image.open(os.path.join(output_directory, "screenshots", f"{f}_{n}.jpg")) + + # Scale the image so that tiles are 256x256 + scale = screen_resolution / mpps + w, h = img.size + img = img.resize((int(w * scale), int(h * scale))) + + # Compute the Web Mercator Projection position of the top left corner of the most centered tile + lat = coords[0] + lng = coords[1] + X_center, Y_center = deg_to_num(lat, lng, zoom) + + # Compute the position of the top left corner of the top left tile + start_x = w / 2 - n_width / 2 * 256 + start_y = h / 2 - n_height / 2 * 256 + + # Iterate on the grid + for column in range(0, n_width): + for row in range(0, n_height): + # Crop the tile and compute its Web Mercator Projection position + box = (start_x + column * 256, start_y + row * 256, start_x + (column + 1) * 256, start_y + (row + 1) * 256) + X = X_center - math.floor(n_width / 2) + column + Y = Y_center - math.floor(n_height / 2) + row + + # Save the tile + if not os.path.exists(os.path.join(output_directory, "tiles", str(zoom), str(X))): + try: + os.mkdir(os.path.join(output_directory, "tiles", str(zoom), str(X))) + except FileExistsError: + continue + except Exception as e: + raise e + img.crop(box).save(os.path.join(output_directory, "tiles", str(zoom), str(X), f"{Y}.jpg")) + n += 1 + + else: + raise Exception(f"{os.path.join(output_directory, "screenshots", f"{f}_{n}.jpg")} missing") + +def run(map_config): + with open('configs/screen_properties.yml', 'r') as sp: + screen_config = yaml.safe_load(sp) + + # Create output folders + output_directory = map_config["output_directory"] + if not os.path.exists(output_directory): + os.mkdir(output_directory) + + if not os.path.exists(os.path.join(output_directory, "screenshots")): + os.mkdir(os.path.join(output_directory, "screenshots")) + else: + skip_screenshots = (input("Raw screenshots already found for this config, do you want to skip directly to tiles extraction? Enter y to skip: ") == "y") + + if not os.path.exists(os.path.join(output_directory, "tiles")): + os.mkdir(os.path.join(output_directory, "tiles")) + + # Compute the optimal zoom level + usable_width = screen_config["width"] - 200 # Keep a margin around the center + usable_height = screen_config["height"] - 200 # Keep a margin around the center + + with open(map_config["boundary_file"], 'rt', encoding="utf-8") as bp: + # Read the config file + doc = bp.read() + k = kml.KML() + k.from_string(doc) + + # Extract the features + features = [] + for feature in k.features(): + for sub_feature in list(feature.features()): + features.append(sub_feature) + + # Iterate over all the closed features in the kml file + f = 1 + for feature in features: + geo = sub_feature.geometry + + # Define the boundary rect around the area + start_lat = geo.bounds[3] + start_lng = geo.bounds[0] + end_lat = geo.bounds[1] + end_lng = geo.bounds[2] + + # Find the zoom level that better approximates the provided resolution + screen_resolution = screen_config['geo_resolution'] + mpps_delta = [abs(compute_mpps((start_lat + end_lat) / 2, z) - screen_resolution) for z in range(0, 21)] + zoom = mpps_delta.index(min(mpps_delta)) + + print(f"Feature {f} of {len(features)}, using zoom level {zoom}") + + # Find the maximum dimension of the tiles at the given resolution + mpps = compute_mpps(end_lat, zoom) + d = 256 * mpps / screen_resolution + + n_height = math.floor(usable_height / d) + n_width = math.floor(usable_width / d) + + print(f"Feature {f} of {len(features)}, each screenshot will provide {n_height} tiles in height and {n_width} tiles in width") + + # Find the starting and ending points + start_X, start_Y = deg_to_num(start_lat, start_lng, zoom) + end_X, end_Y = deg_to_num(end_lat, end_lng, zoom) + + # Find all the X, Y coordinates inside of the provided area + screenshots_coordinates = [] + for X in range(start_X, end_X, n_width): + for Y in range(start_Y, end_Y, n_height): + lat, lng = num_to_deg(X, Y, zoom) + p = Point(lng, lat) + if p.within(wkt.loads(geo.wkt)): + screenshots_coordinates.append((lat, lng)) + + print(f"Feature {f} of {len(features)}, {len(screenshots_coordinates)} screenshots will be taken") + + # Start looping + if not skip_screenshots: + print(f"Feature {f} of {len(features)}, taking screenshots...") + n = 0 + for coords in screenshots_coordinates: + # Making PUT request + #data = json.dumps({'lat': coords[0], 'lng': coords[1]}) + #r = requests.put('http://localhost:8080', data = data) + + time.sleep(0.1) + + ## Take and save screenshot + screenshot = pyautogui.screenshot() + screenshot.save(os.path.join(output_directory, "screenshots", f"{f}_{n}.jpg")) + + printProgressBar(n + 1, len(screenshots_coordinates)) + n += 1 + + # Extract the tiles + if not os.path.exists(os.path.join(output_directory, "tiles", str(zoom))): + os.mkdir(os.path.join(output_directory, "tiles", str(zoom))) + + params = { + "f": f, + "zoom": zoom, + "output_directory": output_directory, + "n_width": n_width, + "n_height": n_height, + "screen_resolution": screen_resolution, + "mpps": mpps + } + + # Extract the tiles with parallel thread execution + with futures.ThreadPoolExecutor() as executor: + print(f"Feature {f} of {len(features)}, extracting tiles...") + global tot_futs, fut_counter + futs = [executor.submit(extract_tiles, n, screenshots_coordinates, params) for n in range(0, len(screenshots_coordinates))] + tot_futs = len(futs) + fut_counter = 0 + [fut.add_done_callback(done_callback) for fut in futs] + [fut.result() for fut in futures.as_completed(futs)] + + # Increase the feature counter + print(f"Feature {f} of {len(features)} completed!") + f += 1 + + + + diff --git a/scripts/python/generateMaps/requirements.txt b/scripts/python/map_generator/requirements.txt similarity index 100% rename from scripts/python/generateMaps/requirements.txt rename to scripts/python/map_generator/requirements.txt