[WIP] New feature: add AGF sources

.gitignore

@@ -2,9 +2,11 @@
 AGFGlassCat.jl
 HOYA.jl
 NIKON.jl
+NHG.jl
 OHARA.jl
 SCHOTT.jl
 Sumita.jl
+*.agf
 
 # Don't track sysimage file: dll on windows, so on linux
 *.dll

deps/build.jl

@@ -20,24 +20,18 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE
 
-const AGF_DIR = joinpath(@__DIR__, "downloads", "glasscat") # contains SCHOTT.agf, Sumita.agf, etc.
-const GLASSCAT_DIR = joinpath(@__DIR__, "..", "src", "GlassCat") # contains GlassCat.jl (pre-existing)
-const JL_DIR = joinpath(GLASSCAT_DIR, "data") # contains AGFGlasscat.jl, SCHOTT.jl, etc.
-
-const SOURCES_PATH = joinpath(@__DIR__, "sources.txt")
-const AGFGLASSCAT_NAME = "AGFGlassCat.jl"
-
+include(joinpath("..", "src", "GlassCat", "constants.jl"))
 include(joinpath(GLASSCAT_DIR, "GlassTypes.jl"))
-include("sources.jl")
-include("generate.jl")
+include(joinpath(GLASSCAT_DIR, "sources.jl"))
+include(joinpath(GLASSCAT_DIR, "generate.jl"))
 
 mkpath(AGF_DIR)
 mkpath(JL_DIR)
 
 # Build/verify a source directory using information from sources.txt
-sources = [split(line, " ") for line in readlines(SOURCES_PATH)]
+sources = split.(readlines(SOURCES_PATH))
 verify_sources!(sources, AGF_DIR)
-verified_source_names = [source[1] for source in sources]
+verified_source_names = first.(sources)
 
 # Use verified sources to generate required .jl files
 @info "Using sources: $(join(verified_source_names, ", ", " and "))"

docs/src/glasscat.md

@@ -1,12 +1,38 @@
 # GlassCat
 
-Julia module for importing and using AGF glass specifications.
+This submodule is used to download, parse, install and manage AGF glass specifications for use in OpticSim.
 
-The entire AGF glass catalog is specified in `AGFGlassCat.jl`. This Julia source file is generated automatically when `] build OpticSim` is called. The build script downloads AGF files to `deps/downloads/glasscat/` and then uses these to generate corresponding Julia source files at `src/GlassCat/data/`. These steps are run automatically on setup when the package is first installed using `] add OpticSim`, creating a sufficient working environment for our example/test code.
+The central configuration file for GlassCat is located at `src/GlassCat/data/sources.txt`, which ships with the
+following default entries.
 
-Adding new AGF sources is done by editing `deps/sources.txt`. Minimally, you must provide a name (e.g. SCHOTT) and sha256sum for the AGF file, which can then be placed manually into `deps/downloads/glasscat/[NAME].agf`. Instead of manually sourcing the AGF file, you can also provide a download link for the build script. `deps/sources.txt` already contains examples of all possible use cases. After updating the file, execute `] build OpticSim` to rebuild `AGFGlassCat.jl`.
+```
+NIKON a49714470fa875ad4fd8d11cbc0edbf1adfe877f42926d7612c1649dd9441e75 https://www.nikon.com/products/components/assets/pdf/nikon_zemax_data.zip
+OHARA 0c9021bf11b8d4e660012191818685ad3110d4f9490699cabdc89aae1fd26d2e https://www.oharacorp.com/xls/OHARA_201130_CATALOG.zip
+HOYA b02c203e5a5b7a8918cc786badf0a9db1fe2572372c1c163dc306b0a8a908854 http://www.hoya-opticalworld.com/common/agf/HOYA20210105.agf
+SCHOTT e9aabbb8ebff116ba0c106a71afd86e72f2a397ac9bc447469129e325e795f4e https://www.schott.com/d/advanced_optics/6959f9a4-0e4f-4ef2-a302-2347468a82f5/1.31/schott-optical-glass-overview-zemax-format.zip
+Sumita c1093e42a1d08acbe30698aba730161e3b43f8f0d50533f65de8b6b11100fdc8 https://www.sumita-opt.co.jp/en/wp/wp-content/themes/sumita-opt-en/download-files.php files%5B%5D=new_sumita.agf&category=data
+```
+
+Each line corresponds to one AGF source, which is described by 2 to 4 space-delimited columns. The first column provides
+the installed module name for the catalog, e.g. `GlassCat.NIKON`. The second column is the expected SHA256 checksum for
+the AGF file.
+
+The final two columns are optional, specifying download instructions for acquiring the zipped AGF files
+automatically from the web. The fourth column allows us to use POST requests to acquire files from interactive sites.
+
+When `] build OpticSim` is run, the sources are verified and parsed into corresponding Julia files. These are then
+included in OpticSim via `AGFGlassCat.jl`. These steps are run automatically when the package is first installed using
+`] add OpticSim`, creating a sufficient working environment for our examples and tests.
+
+## Adding glass catalogs
+`sources.txt` can be edited freely to add more glass catalogs. However, this is a somewhat tedious process, so we have a
+convenience function for adding a locally downloaded AGF file to the source list.
+
+```@docs
+OpticSim.GlassCat.add_agf
+```
 
-Source names will be used as module names, so follow the standard convections: alphanumeric, no leading numbers, begin with an uppercase letter. Furthermore, optical systems in the examples file expect glass catalogs to have specific names. The default setup includes HOYA, NIKON, OHARA, SCHOTT and Sumita; changing these names could break some examples.
+## Using installed glasses
 
 Glass types are accessed like so: `OpticSim.GlassCat.CATALOG_NAME.GLASS_NAME`, e.g.
 
@@ -88,3 +114,11 @@ OpticSim.GlassCat.glassforid
 OpticSim.GlassCat.polyfit_indices
 OpticSim.GlassCat.plot_indices
 ```
+
+---
+
+```@docs
+OpticSim.GlassCat.verify_sources!
+OpticSim.GlassCat.verify_source
+OpticSim.GlassCat.download_source
+```

src/GlassCat/GlassCat.jl

@@ -38,8 +38,8 @@ include("Air.jl")
 export Air, isair
 
 # include built glass cat source files
-include("data/AGFGlassCat.jl")
-include("data/CARGILLE.jl")
+include("data/jl/AGFGlassCat.jl")
+include("data/jl/CARGILLE.jl")
 
 # include functionality for managing runtime (dynamic) glass cats: MIL_GLASSES and MODEL_GLASSES
 include("runtime.jl")
@@ -52,5 +52,9 @@ export glasscatalogs, glassnames, findglass
 include("utilities.jl")
 export plot_indices, index, polyfit_indices, absairindex, absorption
 
+# include utility functions for maintaining the AGF source list
+include("sources.jl")
+export add_agf
+
 end # module
 export GlassCat

src/GlassCat/constants.jl

@@ -20,10 +20,20 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE
 
+using Unitful
+
 const TEMP_REF = 20.0
 const PRESSURE_REF = 1.0
 const TEMP_REF_UNITFUL = TEMP_REF * u"°C"
 export TEMP_REF, PRESSURE_REF
 
 const DISPFORM_NAMES = ["Schott", "Sellmeier1", "Herzberger", "Sellmeier2", "Conrady", "Sellmeier3", "HandbookOfOptics1", "HandbookOfOptics2", "Sellmeier4", "Extended1", "Sellmeier5", "Extended2", "Extended3"]
 const STATUS = ["Standard", "Preferred", "Obsolete", "Special", "Melt"]
+
+# paths for GlassCat source file builds
+const GLASSCAT_DIR = @__DIR__ # contains GlassCat.jl (pre-existing)
+const AGF_DIR = joinpath(GLASSCAT_DIR, "data", "agf") # contains SCHOTT.agf, Sumita.agf, etc.
+const JL_DIR = joinpath(GLASSCAT_DIR, "data", "jl") # contains AGFGlasscat.jl, SCHOTT.jl, etc.
+
+const SOURCES_PATH = joinpath(GLASSCAT_DIR, "data", "sources.txt")
+const AGFGLASSCAT_NAME = "AGFGlassCat.jl"

deps/sources.txt → src/GlassCat/data/sources.txt

@@ -3,4 +3,3 @@ OHARA 0c9021bf11b8d4e660012191818685ad3110d4f9490699cabdc89aae1fd26d2e https://w
 HOYA b02c203e5a5b7a8918cc786badf0a9db1fe2572372c1c163dc306b0a8a908854 http://www.hoya-opticalworld.com/common/agf/HOYA20210105.agf
 SCHOTT e9aabbb8ebff116ba0c106a71afd86e72f2a397ac9bc447469129e325e795f4e https://www.schott.com/d/advanced_optics/6959f9a4-0e4f-4ef2-a302-2347468a82f5/1.31/schott-optical-glass-overview-zemax-format.zip
 Sumita c1093e42a1d08acbe30698aba730161e3b43f8f0d50533f65de8b6b11100fdc8 https://www.sumita-opt.co.jp/en/wp/wp-content/themes/sumita-opt-en/download-files.php files%5B%5D=new_sumita.agf&category=data
-NHG b3db4c677b1ac67cd9cce5be2abff88d13bcf24e4349dc789b2da12fdfe1a280

src/GlassCat/search.jl

@@ -48,7 +48,7 @@ Returns the glass names available from a given catalog.
 
 # Example
 ```julia-repl
-julia> glasses(GlassCat.CARGILLE)
+julia> glassnames(GlassCat.CARGILLE)
 3-element Array{Any,1}:
  "OG0607"
  "OG0608"
@@ -74,7 +74,7 @@ Returns the glass names available from all catalogs.
 
 # Example
 ```julia-repl
-julia> glasses()
+julia> glassnames()
 6-element Array{Pair{Module,Array{Any,1}},1}:
  OpticSim.GlassCat.CARGILLE => ["OG0607", "OG0608", "OG081160"]
      OpticSim.GlassCat.HOYA => ["BAC4", "BACD11"  …  "TAFD65"]

deps/sources.jl → src/GlassCat/sources.jl

@@ -27,6 +27,48 @@ import ZipFile
 const Maybe{T} = Union{T, Nothing}
 
 """
+    add_agf(sourcefile::AbstractString; name::Maybe{AbstractString} = nothing, rebuild::Bool = true)
+
+Adds an already downloaded AGF file to the sourcelist at data/sources.txt, generating the SHA256 checksum automatically.
+
+Optionally provide a `name` for the corresponding module, and `rebuild` AGFGlassCat.jl by default.
+"""
+function add_agf(sourcefile::AbstractString; name::Maybe{AbstractString} = nothing, rebuild::Bool = true)
+    if !isfile(sourcefile)
+        @error "AGF file not found at $sourcefile"
+        return
+    end
+
+    # infer catalog name from sourcefile basename (alphabetical only)
+    if name === nothing
+        name = uppercase(match(r"^([a-zA-Z]+)\.agf$"i, basename(sourcefile))[1])
+    end
+
+    # avoid duplicate catalog names
+    if name ∈ first.(split.(readlines(SOURCES_PATH)))
+        @error "Adding the catalog name \"$name\" would create a duplicate entry in sources.txt"
+        return
+    end
+
+    # copy sourcefile to correct location
+    mkpath(AGF_DIR)
+    cp(sourcefile, joinpath(AGF_DIR, name * ".agf"))
+
+    # append a corresponding entry to sources.txt
+    sha256sum = SHA.bytes2hex(SHA.sha256(read(sourcefile)))
+    open(SOURCES_PATH, "a") do io
+        write(io, join([name, sha256sum], ' '))
+    end
+
+    # re-build AGFGlassCat.jl
+    if rebuild
+        include(joinpath(@__DIR__, "..", "..", "deps", "build.jl"))
+    end
+end
+
+"""
+    verify_sources!(sources::AbstractVector{<:AbstractVector{<:AbstractString}}, sourcedir::AbstractString)
+
 Verify a list of `sources` located in `sourcedir`. If AGF files are missing or invalid, try to download them using the
 information provided in `sources`.
 
@@ -58,6 +100,8 @@ function verify_sources!(sources::AbstractVector{<:AbstractVector{<:AbstractStri
 end
 
 """
+    verify_source(sourcefile::AbstractString, sha256sum::AbstractString)
+
 Verify a source file using SHA256, returning true if successful. Otherwise, remove the file and return false.
 """
 function verify_source(sourcefile::AbstractString, sha256sum::AbstractString)
@@ -74,6 +118,8 @@ function verify_source(sourcefile::AbstractString, sha256sum::AbstractString)
 end
 
 """
+    download_source(sourcefile::AbstractString, url::AbstractString, POST_data::Maybe{AbstractString} = nothing)
+
 Download and unzip an AGF glass catalog from a publicly available source. Supports POST requests.
 """
 function download_source(sourcefile::AbstractString, url::AbstractString, POST_data::Maybe{AbstractString} = nothing)

test/Project.toml

@@ -2,6 +2,7 @@
 BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
 DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
 FiniteDifferences = "26cc04aa-876d-5657-8c51-4c34ba976000"
+HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
 Images = "916415d5-f1e6-5110-898d-aaa5f9f070e0"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"

test/testsets/GlassCat.jl

@@ -25,12 +25,14 @@
     @test isempty(detect_ambiguities(GlassCat))
     @test isempty(detect_unbound_args(GlassCat))
 
-    include(joinpath("..", "..", "src", "GlassCat", "GlassTypes.jl"))
-    include(joinpath(@__DIR__, "..", "..", "deps", "generate.jl"))
+    GLASSCAT_DIR = joinpath(@__DIR__, "..", "..", "src", "GlassCat")
+    include(joinpath(GLASSCAT_DIR, "GlassTypes.jl"))
+    include(joinpath(GLASSCAT_DIR, "generate.jl"))
 
     CATALOG_NAME = "TEST_CAT"
     SOURCE_DIR = joinpath(@__DIR__, "..")
     SOURCE_FILE = joinpath(SOURCE_DIR, "$(CATALOG_NAME).agf")
+
     tmpdir = mktempdir()
     MAIN_FILE = joinpath(tmpdir, "AGF_TEST_CAT.jl")
     JL_DIR = tmpdir