Skip to content

Commit

Permalink
Minor fixes
Browse files Browse the repository at this point in the history
  • Loading branch information
@jlmcgraw
jlmcgraw committed May 27, 2017
1 parent 4a8eefb commit dbdf9f1
Show file tree
Hide file tree
Showing 5 changed files with 87 additions and 63 deletions.
5 changes: 3 additions & 2 deletions cut_and_georeference_Sectional_insets.pl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -309,7 +309,7 @@ sub cutOutInsetFromSourceRaster {
say "";
}

# "python ./memoize.py -d $linkedRastersDirectory -d $clippedRastersDirectory
# "python ./memoize.py -d $linkedRastersDirectory -d $clippedRastersDirectory
#Run gdal_translate
my $gdal_translateoutput = qx($gdal_translateCommand);

Expand Down Expand Up @@ -365,7 +365,8 @@ sub warpRaster {
say $gdalWarpCommand;
say "";
}
# "python ./memoize.py -d $warpedRastersDirectory -d $clippedRastersDirectory

# "python ./memoize.py -d $warpedRastersDirectory -d $clippedRastersDirectory
#Run gdalwarp
my $gdalWarpOutput = qx($gdalWarpCommand);

Expand Down
86 changes: 44 additions & 42 deletions make_seamless_charts.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -377,31 +377,33 @@ warp_and_clip(){

echo "expanded source: $EXPANDED_FILE_DIRECTORY/${fbname}.vrt"
echo "clipped destination: $CLIPPED_FILE_DIRECTORY/${fbname}.vrt"
# BUG TODO crop_to_cutline results in a resampled image with non-square pixels
# BUG TODO crop_to_cutline results in a resampled image with non-square
# pixels
# How to best handle this? One fix is an additional warp to EPSG:3857
# Do I need -dstalpha here? That adds a band, I just want to re-use the existing one
# Do I need -dstalpha here? That adds a band, I just want to re-use the
# existing one
time \
nice -10 \
"${PROGDIR}/memoize.py" -t \
-d "$CHARTS_BASE_DIRECTORY" \
gdalwarp \
-of vrt \
-overwrite \
gdalwarp \
-of vrt \
-overwrite \
-cutline "${CLIPPING_SHAPES_DIRECTORY}/${fbname}.shp" \
-crop_to_cutline \
-cblend 10 \
-r lanczos \
-dstalpha \
-co ALPHA=YES \
-co TILED=YES \
-multi \
-wo NUM_THREADS=ALL_CPUS \
-wm 1024 \
-crop_to_cutline \
-cblend 10 \
-r lanczos \
-dstalpha \
-co ALPHA=YES \
-co TILED=YES \
-multi \
-wo NUM_THREADS=ALL_CPUS \
-wm 1024 \
--config GDAL_CACHEMAX 1024 \
"${EXPANDED_FILE_DIRECTORY}/${fbname}.vrt" \
"${CLIPPED_FILE_DIRECTORY}/${fbname}.vrt"

#Warp the expanded file
# Warp the expanded file
echo "*** Warp to vrt --- gdalwarp $sourceChartName"
echo "clipped source: ${CLIPPED_FILE_DIRECTORY}/${fbname}.vrt"
echo "warped destination: ${WARPED_RASTERS_DIRECTORY}/${fbname}.vrt"
Expand All @@ -410,16 +412,16 @@ warp_and_clip(){
nice -10 \
"${PROGDIR}/memoize.py" -t \
-d "$CHARTS_BASE_DIRECTORY" \
gdalwarp \
-of vrt \
-t_srs EPSG:3857 \
-r lanczos \
-overwrite \
-multi \
-wo NUM_THREADS=ALL_CPUS \
-wm 1024 \
gdalwarp \
-of vrt \
-t_srs EPSG:3857 \
-r lanczos \
-overwrite \
-multi \
-wo NUM_THREADS=ALL_CPUS \
-wm 1024 \
--config GDAL_CACHEMAX 1024 \
-co TILED=YES \
-co TILED=YES \
"${CLIPPED_FILE_DIRECTORY}/${fbname}.vrt" \
"${WARPED_RASTERS_DIRECTORY}/${fbname}.vrt"

Expand All @@ -432,32 +434,32 @@ warp_and_clip(){
# -co COMPRESS=LZW \
time \
nice -10 \
"${PROGDIR}/memoize.py" -t \
"${PROGDIR}/memoize.py" -t \
-d "$CHARTS_BASE_DIRECTORY" \
gdal_translate \
-strict \
-co TILED=YES \
-co COMPRESS=DEFLATE \
-co PREDICTOR=1 \
-co ZLEVEL=9 \
gdal_translate \
-strict \
-co TILED=YES \
-co COMPRESS=DEFLATE \
-co PREDICTOR=1 \
-co ZLEVEL=9 \
--config GDAL_CACHEMAX 1024 \
"${WARPED_RASTERS_DIRECTORY}/${fbname}.vrt" \
"${WARPED_RASTERS_DIRECTORY}/${fbname}.tif"


#Create external overviews to make display faster in QGIS
echo "*** Overviews --- gdaladdo $sourceChartName"
time \
nice -10 \
"${PROGDIR}/memoize.py" -t \
-d "$CHARTS_BASE_DIRECTORY" \
gdaladdo \
-ro \
-r average \
--config INTERLEAVE_OVERVIEW PIXEL \
--config COMPRESS_OVERVIEW JPEG \
--config BIGTIFF_OVERVIEW IF_NEEDED \
"$WARPED_RASTERS_DIRECTORY/${fbname}.tif" \
time \
nice -10 \
"${PROGDIR}/memoize.py" -t \
-d "$CHARTS_BASE_DIRECTORY" \
gdaladdo \
-ro \
-r average \
--config INTERLEAVE_OVERVIEW PIXEL \
--config COMPRESS_OVERVIEW JPEG \
--config BIGTIFF_OVERVIEW IF_NEEDED \
"$WARPED_RASTERS_DIRECTORY/${fbname}.tif" \
2 4 8 16 32 64
done
}
Expand Down
52 changes: 37 additions & 15 deletions memoize.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,20 +72,26 @@ def files_up_to_date(files):

if files == None:
if args.verbose:
print('No files yet')
print('Memoize: No files yet')
return False

for (fname, hash_digest, mtime) in files:
if not os.path.isfile(fname):
if args.verbose:
print ('Memoize: File does not exist: ', fname)
return False
if opt_use_modtime:
if modtime(fname) != mtime:
if args.verbose:
print ('MEMOIZE File modtime changed: ', fname)
print ('Memoize: File modtime changed: ', fname)
return False
else:
if hash_file(fname) != hash_digest:
if args.verbose:
print ('MEMOIZE File hash changed: ', fname)
print ('Memoize: File hash changed: ', fname)
return False
if args.verbose:
print ('Memoize: File up to date: ', fname)
return True


Expand Down Expand Up @@ -117,12 +123,14 @@ def is_relevant(fname):

def generate_deps(cmd):
""" Gather dependencies for a command and store their hash and modtime """
#print ('Memoize running: ', cmd)

if args.verbose:
print ('Memoize: running: ', cmd)

strace_output_filename = tempfile.mktemp()

if args.verbose:
print("strace output saved in ",strace_output_filename)
print("Memoize: strace output saved in ",strace_output_filename)

wholecmd = \
'strace -f -o %s -e trace=open,rename,stat64,exit_group %s' \
Expand Down Expand Up @@ -172,7 +180,7 @@ def generate_deps(cmd):
if args.verbose:
print ('Memoize: Is relevant: ', fname)

# Add this file's MD5 and datestamp to our dictionary
# Add this file's hash and datestamp to our dictionary
# and mark that we've seen it already
files.append((fname, hash_file(fname), modtime(fname)))
files_dict[fname] = True
Expand Down Expand Up @@ -220,31 +228,34 @@ def memoize_with_deps(depsname, deps, cmd):
files = deps.get(cmd)

if args.verbose:
print ('Files used:', files)
print ('Memoize: Files used:', files)

if opt_unconditional:
print ('Memoize: Forcing command execution with unconditional flag')

# Check the status of all of this command's files
if files and files_up_to_date(files):
if files and files_up_to_date(files) and not opt_unconditional:
if args.verbose:
print ('Up to date:', cmd)
print ('Memoize: Up to date:', cmd)
return 0
else:

# Run the command and collect list of files that it opens
if args.verbose:
print ('Not up to date:', cmd)
print ('Memoize: Not up to date:', cmd)
(status, files) = generate_deps(cmd)

# If the command was successful..
if status == 0:
if args.verbose:
print ('Success!', files)
print ('Memoize: Success!', files)

# Add the files list to the dictionary
deps[cmd] = files

elif cmd in deps:
if args.verbose:
print ('Failure!', files)
print ('Memoize: Failure!', files)
# Delete the key if the command was unsuccessful
del deps[cmd]

Expand All @@ -254,9 +265,9 @@ def memoize_with_deps(depsname, deps, cmd):


if __name__ == '__main__':

# Parse the command line options

parser = argparse.ArgumentParser(description='memoize any program or command')
parser.add_argument('-t'
, '--timestamps'
Expand All @@ -283,6 +294,11 @@ def memoize_with_deps(depsname, deps, cmd):
, help='More output'
, action='store_true'
, required=False)
parser.add_argument('-u'
,'--unconditional'
, help='Force command to execute, even if up to date'
, action='store_true'
, required=False)
parser.add_argument('-f'
, '--filename'
, default='.deps3'
Expand All @@ -293,7 +309,11 @@ def memoize_with_deps(depsname, deps, cmd):
, help='The command to memoize')
args = parser.parse_args()


# Print usage if no actual command was provided
if not args.command:
parser.print_help(file=None)
sys.exit(1)

# Sort the individual elements of the command.
# Good idea or not?: Consider using this as the key
# so the command could be rearranged without being considered out of date
Expand All @@ -312,10 +332,12 @@ def memoize_with_deps(depsname, deps, cmd):
print('Ignoring these directory trees: ', args.ignore)
print('Monitoring these directory trees: ', args.directory)
print('Using timestamps: ', args.timestamps)
print('Unconditional execution of command: ', args.unconditional)
print('Dependencies file: ', args.filename)


opt_use_modtime = args.timestamps
opt_unconditional = args.unconditional
opt_dirs = args.directory
ignore_dirs = args.ignore

Expand Down
4 changes: 2 additions & 2 deletions rasterize_Caribbean_charts.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ do
echo "--------------------------------------------"

# Needs to point to where memoize is
"$installedDirectory/memoize.py" -t \
"$installedDirectory/memoize.py" -t \
gdal_translate \
-strict \
-co TILED=YES \
Expand All @@ -97,7 +97,7 @@ do
"$output_raster_path/$f.tif" \
2 4 8 16 32 64

rm "$output_raster_path/$f-untiled.tif"
# rm "$output_raster_path/$f-untiled.tif"

done

Expand Down
3 changes: 1 addition & 2 deletions tile_charts.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ main() {
)

local -r heli_chart_array_1000000=(
U.S._Gulf_Coast_HEL
U_S_Gulf_Coast_HEL
)

local -r heli_chart_array_250000=(
Expand Down Expand Up @@ -115,7 +115,6 @@ main() {
local -r grand_canyon_chart_array=(
Grand_Canyon_General_Aviation
Grand_Canyon_Air_Tour_Operators
U.S._VFR_Wall_Planning_Chart
)

local -r enroute_chart_array_2000000=(
Expand Down

0 comments on commit dbdf9f1

Please sign in to comment.