diff --git a/cut_and_georeference_Sectional_insets.pl b/cut_and_georeference_Sectional_insets.pl index 28ed40a..e99ec4d 100755 --- a/cut_and_georeference_Sectional_insets.pl +++ b/cut_and_georeference_Sectional_insets.pl @@ -309,7 +309,7 @@ sub cutOutInsetFromSourceRaster { say ""; } - # "python ./memoize.py -d $linkedRastersDirectory -d $clippedRastersDirectory + # "python ./memoize.py -d $linkedRastersDirectory -d $clippedRastersDirectory #Run gdal_translate my $gdal_translateoutput = qx($gdal_translateCommand); @@ -365,7 +365,8 @@ sub warpRaster { say $gdalWarpCommand; say ""; } -# "python ./memoize.py -d $warpedRastersDirectory -d $clippedRastersDirectory + + # "python ./memoize.py -d $warpedRastersDirectory -d $clippedRastersDirectory #Run gdalwarp my $gdalWarpOutput = qx($gdalWarpCommand); diff --git a/make_seamless_charts.sh b/make_seamless_charts.sh index 6d7f2f5..ac710fe 100755 --- a/make_seamless_charts.sh +++ b/make_seamless_charts.sh @@ -377,31 +377,33 @@ warp_and_clip(){ echo "expanded source: $EXPANDED_FILE_DIRECTORY/${fbname}.vrt" echo "clipped destination: $CLIPPED_FILE_DIRECTORY/${fbname}.vrt" - # BUG TODO crop_to_cutline results in a resampled image with non-square pixels + # BUG TODO crop_to_cutline results in a resampled image with non-square + # pixels # How to best handle this? One fix is an additional warp to EPSG:3857 - # Do I need -dstalpha here? That adds a band, I just want to re-use the existing one + # Do I need -dstalpha here? That adds a band, I just want to re-use the + # existing one time \ nice -10 \ "${PROGDIR}/memoize.py" -t \ -d "$CHARTS_BASE_DIRECTORY" \ - gdalwarp \ - -of vrt \ - -overwrite \ + gdalwarp \ + -of vrt \ + -overwrite \ -cutline "${CLIPPING_SHAPES_DIRECTORY}/${fbname}.shp" \ - -crop_to_cutline \ - -cblend 10 \ - -r lanczos \ - -dstalpha \ - -co ALPHA=YES \ - -co TILED=YES \ - -multi \ - -wo NUM_THREADS=ALL_CPUS \ - -wm 1024 \ + -crop_to_cutline \ + -cblend 10 \ + -r lanczos \ + -dstalpha \ + -co ALPHA=YES \ + -co TILED=YES \ + -multi \ + -wo NUM_THREADS=ALL_CPUS \ + -wm 1024 \ --config GDAL_CACHEMAX 1024 \ "${EXPANDED_FILE_DIRECTORY}/${fbname}.vrt" \ "${CLIPPED_FILE_DIRECTORY}/${fbname}.vrt" - #Warp the expanded file + # Warp the expanded file echo "*** Warp to vrt --- gdalwarp $sourceChartName" echo "clipped source: ${CLIPPED_FILE_DIRECTORY}/${fbname}.vrt" echo "warped destination: ${WARPED_RASTERS_DIRECTORY}/${fbname}.vrt" @@ -410,16 +412,16 @@ warp_and_clip(){ nice -10 \ "${PROGDIR}/memoize.py" -t \ -d "$CHARTS_BASE_DIRECTORY" \ - gdalwarp \ - -of vrt \ - -t_srs EPSG:3857 \ - -r lanczos \ - -overwrite \ - -multi \ - -wo NUM_THREADS=ALL_CPUS \ - -wm 1024 \ + gdalwarp \ + -of vrt \ + -t_srs EPSG:3857 \ + -r lanczos \ + -overwrite \ + -multi \ + -wo NUM_THREADS=ALL_CPUS \ + -wm 1024 \ --config GDAL_CACHEMAX 1024 \ - -co TILED=YES \ + -co TILED=YES \ "${CLIPPED_FILE_DIRECTORY}/${fbname}.vrt" \ "${WARPED_RASTERS_DIRECTORY}/${fbname}.vrt" @@ -432,14 +434,14 @@ warp_and_clip(){ # -co COMPRESS=LZW \ time \ nice -10 \ - "${PROGDIR}/memoize.py" -t \ + "${PROGDIR}/memoize.py" -t \ -d "$CHARTS_BASE_DIRECTORY" \ - gdal_translate \ - -strict \ - -co TILED=YES \ - -co COMPRESS=DEFLATE \ - -co PREDICTOR=1 \ - -co ZLEVEL=9 \ + gdal_translate \ + -strict \ + -co TILED=YES \ + -co COMPRESS=DEFLATE \ + -co PREDICTOR=1 \ + -co ZLEVEL=9 \ --config GDAL_CACHEMAX 1024 \ "${WARPED_RASTERS_DIRECTORY}/${fbname}.vrt" \ "${WARPED_RASTERS_DIRECTORY}/${fbname}.tif" @@ -447,17 +449,17 @@ warp_and_clip(){ #Create external overviews to make display faster in QGIS echo "*** Overviews --- gdaladdo $sourceChartName" - time \ - nice -10 \ - "${PROGDIR}/memoize.py" -t \ - -d "$CHARTS_BASE_DIRECTORY" \ - gdaladdo \ - -ro \ - -r average \ - --config INTERLEAVE_OVERVIEW PIXEL \ - --config COMPRESS_OVERVIEW JPEG \ - --config BIGTIFF_OVERVIEW IF_NEEDED \ - "$WARPED_RASTERS_DIRECTORY/${fbname}.tif" \ + time \ + nice -10 \ + "${PROGDIR}/memoize.py" -t \ + -d "$CHARTS_BASE_DIRECTORY" \ + gdaladdo \ + -ro \ + -r average \ + --config INTERLEAVE_OVERVIEW PIXEL \ + --config COMPRESS_OVERVIEW JPEG \ + --config BIGTIFF_OVERVIEW IF_NEEDED \ + "$WARPED_RASTERS_DIRECTORY/${fbname}.tif" \ 2 4 8 16 32 64 done } diff --git a/memoize.py b/memoize.py index a753cc1..0727d94 100755 --- a/memoize.py +++ b/memoize.py @@ -72,20 +72,26 @@ def files_up_to_date(files): if files == None: if args.verbose: - print('No files yet') + print('Memoize: No files yet') return False for (fname, hash_digest, mtime) in files: + if not os.path.isfile(fname): + if args.verbose: + print ('Memoize: File does not exist: ', fname) + return False if opt_use_modtime: if modtime(fname) != mtime: if args.verbose: - print ('MEMOIZE File modtime changed: ', fname) + print ('Memoize: File modtime changed: ', fname) return False else: if hash_file(fname) != hash_digest: if args.verbose: - print ('MEMOIZE File hash changed: ', fname) + print ('Memoize: File hash changed: ', fname) return False + if args.verbose: + print ('Memoize: File up to date: ', fname) return True @@ -117,12 +123,14 @@ def is_relevant(fname): def generate_deps(cmd): """ Gather dependencies for a command and store their hash and modtime """ - #print ('Memoize running: ', cmd) + + if args.verbose: + print ('Memoize: running: ', cmd) strace_output_filename = tempfile.mktemp() if args.verbose: - print("strace output saved in ",strace_output_filename) + print("Memoize: strace output saved in ",strace_output_filename) wholecmd = \ 'strace -f -o %s -e trace=open,rename,stat64,exit_group %s' \ @@ -172,7 +180,7 @@ def generate_deps(cmd): if args.verbose: print ('Memoize: Is relevant: ', fname) - # Add this file's MD5 and datestamp to our dictionary + # Add this file's hash and datestamp to our dictionary # and mark that we've seen it already files.append((fname, hash_file(fname), modtime(fname))) files_dict[fname] = True @@ -220,31 +228,34 @@ def memoize_with_deps(depsname, deps, cmd): files = deps.get(cmd) if args.verbose: - print ('Files used:', files) + print ('Memoize: Files used:', files) + if opt_unconditional: + print ('Memoize: Forcing command execution with unconditional flag') + # Check the status of all of this command's files - if files and files_up_to_date(files): + if files and files_up_to_date(files) and not opt_unconditional: if args.verbose: - print ('Up to date:', cmd) + print ('Memoize: Up to date:', cmd) return 0 else: # Run the command and collect list of files that it opens if args.verbose: - print ('Not up to date:', cmd) + print ('Memoize: Not up to date:', cmd) (status, files) = generate_deps(cmd) # If the command was successful.. if status == 0: if args.verbose: - print ('Success!', files) + print ('Memoize: Success!', files) # Add the files list to the dictionary deps[cmd] = files elif cmd in deps: if args.verbose: - print ('Failure!', files) + print ('Memoize: Failure!', files) # Delete the key if the command was unsuccessful del deps[cmd] @@ -254,9 +265,9 @@ def memoize_with_deps(depsname, deps, cmd): if __name__ == '__main__': - + # Parse the command line options - + parser = argparse.ArgumentParser(description='memoize any program or command') parser.add_argument('-t' , '--timestamps' @@ -283,6 +294,11 @@ def memoize_with_deps(depsname, deps, cmd): , help='More output' , action='store_true' , required=False) + parser.add_argument('-u' + ,'--unconditional' + , help='Force command to execute, even if up to date' + , action='store_true' + , required=False) parser.add_argument('-f' , '--filename' , default='.deps3' @@ -293,7 +309,11 @@ def memoize_with_deps(depsname, deps, cmd): , help='The command to memoize') args = parser.parse_args() - + # Print usage if no actual command was provided + if not args.command: + parser.print_help(file=None) + sys.exit(1) + # Sort the individual elements of the command. # Good idea or not?: Consider using this as the key # so the command could be rearranged without being considered out of date @@ -312,10 +332,12 @@ def memoize_with_deps(depsname, deps, cmd): print('Ignoring these directory trees: ', args.ignore) print('Monitoring these directory trees: ', args.directory) print('Using timestamps: ', args.timestamps) + print('Unconditional execution of command: ', args.unconditional) print('Dependencies file: ', args.filename) opt_use_modtime = args.timestamps + opt_unconditional = args.unconditional opt_dirs = args.directory ignore_dirs = args.ignore diff --git a/rasterize_Caribbean_charts.sh b/rasterize_Caribbean_charts.sh index 5c03370..5e0acbf 100755 --- a/rasterize_Caribbean_charts.sh +++ b/rasterize_Caribbean_charts.sh @@ -74,7 +74,7 @@ do echo "--------------------------------------------" # Needs to point to where memoize is - "$installedDirectory/memoize.py" -t \ + "$installedDirectory/memoize.py" -t \ gdal_translate \ -strict \ -co TILED=YES \ @@ -97,7 +97,7 @@ do "$output_raster_path/$f.tif" \ 2 4 8 16 32 64 - rm "$output_raster_path/$f-untiled.tif" +# rm "$output_raster_path/$f-untiled.tif" done diff --git a/tile_charts.sh b/tile_charts.sh index 6b5443d..0f86b2c 100755 --- a/tile_charts.sh +++ b/tile_charts.sh @@ -77,7 +77,7 @@ main() { ) local -r heli_chart_array_1000000=( - U.S._Gulf_Coast_HEL + U_S_Gulf_Coast_HEL ) local -r heli_chart_array_250000=( @@ -115,7 +115,6 @@ main() { local -r grand_canyon_chart_array=( Grand_Canyon_General_Aviation Grand_Canyon_Air_Tour_Operators - U.S._VFR_Wall_Planning_Chart ) local -r enroute_chart_array_2000000=(