From 61dc9c8b31c4ea6e0f0167f8f6aa0d1f4bf1704a Mon Sep 17 00:00:00 2001 From: Pax1601 Date: Sat, 23 Mar 2024 13:18:00 +0100 Subject: [PATCH] Added compression step to image generation --- .../python/map_generator/.vscode/launch.json | 3 +- scripts/python/map_generator/main.py | 5 +- scripts/python/map_generator/map_generator.py | 101 ++++++++++++++---- scripts/python/map_generator/requirements.txt | Bin 376 -> 854 bytes 4 files changed, 85 insertions(+), 24 deletions(-) diff --git a/scripts/python/map_generator/.vscode/launch.json b/scripts/python/map_generator/.vscode/launch.json index d5e2882c..1138bd0a 100644 --- a/scripts/python/map_generator/.vscode/launch.json +++ b/scripts/python/map_generator/.vscode/launch.json @@ -10,7 +10,8 @@ "request": "launch", "program": "main.py", "console": "integratedTerminal", - "args": ["-s", "-l", "1", "./configs/Test/MediumResolution.yml"] + "args": ["./configs/Test/MediumResolution.yml"], + "justMyCode": false }, { "name": "Convert", diff --git a/scripts/python/map_generator/main.py b/scripts/python/map_generator/main.py index c263b01b..6e44ec79 100644 --- a/scripts/python/map_generator/main.py +++ b/scripts/python/map_generator/main.py @@ -24,6 +24,9 @@ parser.add_argument('-t', '--tiles_folder', help='if provided, will force the sc parser.add_argument('-o', '--screenshots_only', action='store_true', help='if provided, the script will only run the screenshot acquisition algorithm.') parser.add_argument('-e', '--extraction_only', action='store_true', help='if provided, the script will only run the tiles extraction algorithm.') parser.add_argument('-m', '--merging_only', action='store_true', help='if provided, the script will only run the tiles merging algorithm.') +parser.add_argument('-c', '--compression_only', action='store_true', help='if provided, the script will only run the compression algorithm.') +parser.add_argument('-n', '--colors_number', type=int, default=256, help='number of colors used by the png quantization algorithm. By default, 256. Must be less than 256.') + args = parser.parse_args() @@ -81,7 +84,7 @@ with open('configs/screen_properties.yml', 'r') as sp: # Let the user input the size of the screen to compute resolution data = json.dumps({'lat': features[0].geometry.bounds[1], 'lng': features[0].geometry.bounds[0], 'alt': 1350 + map_config['zoom_factor'] * (25000 - 1350), 'mode': 'map'}) try: - r = requests.put(f'http://127.0.0.1:{port}', data = data) + r = requests.post(f'http://127.0.0.1:{port}', data = data) print("The F10 map in your DCS installation was setup. Please, use the measure tool and measure the width of the screen in Nautical Miles") except: print("No running DCS instance detected. You can still run the algorithm if you already took the screenshots, otherwise you will not be able to produce a map.") diff --git a/scripts/python/map_generator/map_generator.py b/scripts/python/map_generator/map_generator.py index c2040693..4a887b23 100644 --- a/scripts/python/map_generator/map_generator.py +++ b/scripts/python/map_generator/map_generator.py @@ -8,6 +8,7 @@ import json import numpy import datetime +from geopy import distance from fastkml import kml from shapely import wkt, Point from PIL import Image @@ -18,11 +19,14 @@ from os.path import isfile, isdir, join # global counters fut_counter = 0 tot_futs = 0 +start_time = None +last_screenshot_position = None # constants C = 40075016.686 # meters, Earth equatorial circumference R = C / (2 * math.pi) # meters, Earth equatorial radius PUT_RETRIES = 10 # allowable number of retries for the PUT request +SLEEP_DISTANCE = 30 # distance in kms. If a screenshot is taken with a distance from the previous screenshot longer than this value, the algorithm waits 5s for DCS to load the textures def deg_to_num(lat_deg, lon_deg, zoom): lat_rad = math.radians(lat_deg) @@ -41,19 +45,22 @@ def num_to_deg(xtile, ytile, zoom): def compute_mpps(lat, z): return C * math.cos(math.radians(lat)) / math.pow(2, z + 8) -def print_progress_bar(iteration, total, prefix = '', suffix = '', decimals = 1, length = 100, fill = '█', printEnd = "\r"): +def print_progress_bar(iteration, total, start_time, prefix = '', suffix = '', decimals = 1, length = 80, fill = '█', printEnd = "\r"): + now = datetime.datetime.now() + diff = (now - start_time).total_seconds() percent = ("{0:." + str(decimals) + "f}").format(100 * (iteration / float(total))) filledLength = int(length * iteration // total) bar = fill * filledLength + '-' * (length - filledLength) - print(f'\r{prefix} |{bar}| {percent}% {suffix}', end = printEnd) + print(f'\r{prefix} |{bar}| {percent}% {suffix} {iteration / diff:.3f} ops/s', end = printEnd) + # Print New Line on Complete if iteration == total: print() def done_callback(fut): - global fut_counter, tot_futs + global fut_counter, tot_futs, start_time fut_counter += 1 - print_progress_bar(fut_counter, tot_futs) + print_progress_bar(fut_counter, tot_futs, start_time) def extract_tiles(n, screenshots_XY, params): f = params['f'] @@ -104,7 +111,7 @@ def merge_tiles(base_path, zoom, tile): # If the image already exists, open it so we can paste the higher quality data in it if os.path.exists(os.path.join(base_path, str(zoom - 1), str(X), f"{Y}.png")): - dst = Image.open(os.path.join(base_path, str(zoom - 1), str(X), f"{Y}.png")) + dst = Image.open(os.path.join(base_path, str(zoom - 1), str(X), f"{Y}.png")).convert('RGBA') else: dst = Image.new('RGBA', (256, 256), (0, 0, 0, 0)) @@ -113,7 +120,7 @@ def merge_tiles(base_path, zoom, tile): for i in range(0, 4): # Open the subtile, if it exists, and resize it down to 128x128 if os.path.exists(os.path.join(base_path, str(zoom), str(2*X + positions[i][0]), f"{2*Y + positions[i][1]}.png")): - im = Image.open(os.path.join(base_path, str(zoom), str(2*X + positions[i][0]), f"{2*Y + positions[i][1]}.png")).resize((128, 128)) + im = Image.open(os.path.join(base_path, str(zoom), str(2*X + positions[i][0]), f"{2*Y + positions[i][1]}.png")).convert('RGBA').resize((128, 128)) dst.paste(im, (positions[i][0] * 128, positions[i][1] * 128), im) # Create the output folder if it exists @@ -127,7 +134,17 @@ def merge_tiles(base_path, zoom, tile): raise e # Save the image - dst.save(os.path.join(base_path, str(zoom - 1), str(X), f"{Y}.png"), quality=98) + dst.save(os.path.join(base_path, str(zoom - 1), str(X), f"{Y}.png")) + +def compress_tiles(base_path, zoom, tile, colors_number): + X = tile[0] + Y = tile[1] + path = os.path.join(base_path, str(zoom), str(X), f"{Y}.png") + initial_size = os.path.getsize(path) + im = Image.open(path) + im = im.quantize(colors_number) + im.save(path) + return initial_size, os.path.getsize(path) def compute_correction_factor(XY, n_width, n_height, map_config, zoom, screenshots_folder, port): # Take screenshots at the given position @@ -191,7 +208,9 @@ def compute_variation(imageA): return max - min def take_screenshot(XY, n_width, n_height, map_config, zoom, screenshots_folder, f, n, port, correction = (0, 0)): - # Making PUT request + global last_screenshot_position + + # Making POST request # If the number of rows or columns is odd, we need to take the picture at the CENTER of the tile! lat, lng = num_to_deg(XY[0] + (n_width % 2) / 2, XY[1] + (n_height % 2) / 2, zoom) data = json.dumps({'lat': lat, 'lng': lng, 'alt': 1350 + map_config['zoom_factor'] * (25000 - 1350), 'mode': 'map'}) @@ -201,7 +220,7 @@ def take_screenshot(XY, n_width, n_height, map_config, zoom, screenshots_folder, success = False while not success and retries > 0: try: - r = requests.put(f'http://127.0.0.1:{port}', data = data) + r = requests.post(f'http://127.0.0.1:{port}', data = data) success = True except: retries -= 1 @@ -212,7 +231,12 @@ def take_screenshot(XY, n_width, n_height, map_config, zoom, screenshots_folder, geo_data = json.loads(r.text) - time.sleep(0.2) + if last_screenshot_position is None or distance.geodesic(last_screenshot_position, (lat, lng)).km > SLEEP_DISTANCE: + time.sleep(5.0) + else: + time.sleep(0.2) + + last_screenshot_position = (lat, lng) # Take and save screenshot. The response to the put request contains data, among which there is the north rotation at that point. screenshot = pyautogui.screenshot() @@ -240,7 +264,7 @@ def take_screenshot(XY, n_width, n_height, map_config, zoom, screenshots_folder, screenshot.rotate(math.degrees(geo_data['northRotation'])).resize((int(sx * screenshot.width) + correction[0], int(sy * screenshot.height)+ correction[1] )).save(os.path.join(screenshots_folder, f"{f}_{n}_{zoom}.jpg"), quality=98) def run(map_config, port): - global tot_futs, fut_counter + global tot_futs, fut_counter, start_time print("Script start time: ", datetime.datetime.now()) with open('configs/screen_properties.yml', 'r') as sp: @@ -315,13 +339,13 @@ def run(map_config, port): screenshots_XY.append((X, Y)) ########### Take screenshots - if not map_config["extraction_only"] and not map_config["merging_only"]: - print("Screenshots taking starting at: ", datetime.datetime.now()) - print(f"Feature {f} of {len(features)}, {len(screenshots_XY)} screenshots will be taken") + if not map_config["extraction_only"] and not map_config["merging_only"] and not map_config["compression_only"]: # Start looping correction = None if not skip_screenshots: - print(f"Feature {f} of {len(features)}, taking screenshots...") + start_time = datetime.datetime.now() + print("Screenshots taking starting at: ", start_time) + print(f"Feature {f} of {len(features)}, {len(screenshots_XY)} screenshots will be taken") n = 0 for XY in screenshots_XY: if not os.path.exists(os.path.join(map_config['screenshots_folder'], f"{f}_{n}_{zoom}.jpg")) or replace_screenshots: @@ -331,12 +355,15 @@ def run(map_config, port): correction = new_correction take_screenshot(XY, n_width, n_height, map_config, zoom, map_config['screenshots_folder'], f, n, port, correction if correction is not None else (0, 0)) - print_progress_bar(n + 1, len(screenshots_XY)) + print_progress_bar(n + 1, len(screenshots_XY), start_time) n += 1 + print(f"Taken {n} screenshots in {datetime.datetime.now() - start_time}s") ########### Extract the tiles - if not map_config["screenshots_only"] and not map_config["merging_only"]: - print("Tiles extraction starting at: ", datetime.datetime.now()) + if not map_config["screenshots_only"] and not map_config["merging_only"] and not map_config["compression_only"]: + start_time = datetime.datetime.now() + res = [] + print("Tiles extraction starting at: ", start_time) if not os.path.exists(os.path.join(map_config["tiles_folder"], str(zoom))): os.mkdir(os.path.join(map_config["tiles_folder"], str(zoom))) @@ -356,15 +383,18 @@ def run(map_config, port): tot_futs = len(futs) fut_counter = 0 [fut.add_done_callback(done_callback) for fut in futs] - [fut.result() for fut in futures.as_completed(futs)] + res.extend([fut.result() for fut in futures.as_completed(futs)]) + print(f"Extracted {len(res) * n_width * n_height} images in {datetime.datetime.now() - start_time}s") # Increase the feature counter print(f"Feature {f} of {len(features)} completed!") f += 1 ########### Assemble tiles to get lower zoom levels - if not map_config["screenshots_only"] and not map_config["extraction_only"]: - print("Tiles merging start time: ", datetime.datetime.now()) + if not map_config["screenshots_only"] and not map_config["extraction_only"] and not map_config["compression_only"]: + start_time = datetime.datetime.now() + res = [] + print("Tiles merging start time: ", start_time) for current_zoom in range(zoom, map_config["final_level"], -1): Xs = [int(d) for d in listdir(os.path.join(map_config["tiles_folder"], str(current_zoom))) if isdir(join(map_config["tiles_folder"], str(current_zoom), d))] existing_tiles = [] @@ -389,7 +419,34 @@ def run(map_config, port): tot_futs = len(futs) fut_counter = 0 [fut.add_done_callback(done_callback) for fut in futs] - [fut.result() for fut in futures.as_completed(futs)] + res.extend([fut.result() for fut in futures.as_completed(futs)]) + print(f"Merged {len(res)} images in {datetime.datetime.now() - start_time}s") + + ########### Assemble tiles to get lower zoom levels + if not map_config["screenshots_only"] and not map_config["extraction_only"] and not map_config["merging_only"]: + start_time = datetime.datetime.now() + res = [] + print("Tiles compression start time: ", start_time) + for current_zoom in range(zoom, map_config["final_level"], -1): + Xs = [int(d) for d in listdir(os.path.join(map_config["tiles_folder"], str(current_zoom))) if isdir(join(map_config["tiles_folder"], str(current_zoom), d))] + existing_tiles = [] + for X in Xs: + Ys = [int(f.removesuffix(".png")) for f in listdir(os.path.join(map_config["tiles_folder"], str(current_zoom), str(X))) if isfile(join(map_config["tiles_folder"], str(current_zoom), str(X), f))] + for Y in Ys: + existing_tiles.append((X, Y)) + + # Compress the tiles with parallel thread execution + with futures.ThreadPoolExecutor() as executor: + print(f"Compressing tiles for zoom level {current_zoom }...") + + futs = [executor.submit(compress_tiles, os.path.join(map_config["tiles_folder"]), current_zoom, tile, map_config['colors_number']) for tile in existing_tiles] + tot_futs = len(futs) + fut_counter = 0 + [fut.add_done_callback(done_callback) for fut in futs] + res.extend([fut.result() for fut in futures.as_completed(futs)]) + total_initial_size = numpy.sum([r[0] for r in res]) / 1024 / 1024 + total_final_size = numpy.sum([r[1] for r in res]) / 1024 / 1024 + print(f"Compressed {len(res)} images in {datetime.datetime.now() - start_time}, inizial size {total_initial_size:.3f}MB, final size {total_final_size:.3f}MB, compression ratio {(1 - total_final_size / total_initial_size )* 100:.3f}%") print("Script end time: ", datetime.datetime.now()) diff --git a/scripts/python/map_generator/requirements.txt b/scripts/python/map_generator/requirements.txt index 43e52dfdfcbf8763d79afe749ed1fa3944d2687f..ce43e8dab3402d72e7a61b82ba3be04828519fd4 100644 GIT binary patch literal 854 zcmY+CK~KU^5QO(^;!lB4L6n0By$}*fh{PC=hy~HoR$8L?WVLU3)=E0)*QqZ-N2f-=OOQbS4(_kJcWIDZ{pp`BY2w8cMU@gVr=&|X%$9b&Rdp4 z<6h}T4;QW=hOwvlfL6{NQA<8`MlEj2ENi;~@!7cfkI)RIZy@AXwwOy;!X^S{2g9pZ zLsl_GdAWmO$vn~J@vL)dI}oYwI%1Sszez+2HCrs0rNcA}zEV#}NvcDP$Bt^!6=&`V ztQ{3L@jvHBlAhYFSGA$AN9|CK-qA??mQ}4vpW6``D!;kPC&OX%A&zhlDzyJnO}Ig^ abB^YWM?KM1=-H#J+o8MKZ#gwzxRt+0;dxE~ literal 376 zcmXX?O;5ux487;SEK1T**dd4QGEJfq0wKgPv~C+nlaM$X-LJT8{<8`SnuW2YyF}b%J2WitkB21HM^5)DgkSLR6QhQ~0Tg6~{F~ z>Ya_1c%WiLr0}_w;gamn&|@9`Tldl@fE!do@