Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

TileJSON support for pmtiles serve #55

Merged
merged 2 commits into from
Jun 23, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 7 additions & 6 deletions main.go
Original file line number Diff line number Diff line change
Expand Up @@ -58,11 +58,12 @@ var cli struct {
} `cmd:"" help:"Verifies that a local archive is valid."`

Serve struct {
Path string `arg:"" help:"Local path or bucket prefix"`
Port int `default:8080`
Cors string `help:"Value of HTTP CORS header."`
CacheSize int `default:64 help:"Size of cache in Megabytes."`
Bucket string `help:"Remote bucket"`
Path string `arg:"" help:"Local path or bucket prefix"`
Port int `default:8080`
Cors string `help:"Value of HTTP CORS header."`
CacheSize int `default:64 help:"Size of cache in Megabytes."`
Bucket string `help:"Remote bucket"`
PublicHostname string `help:"Public hostname of tile endpoint e.g. https://example.com"`
} `cmd:"" help:"Run an HTTP proxy server for Z/X/Y tiles."`

Upload struct {
Expand Down Expand Up @@ -96,7 +97,7 @@ func main() {
logger.Fatalf("Failed to show database, %v", err)
}
case "serve <path>":
server, err := pmtiles.NewServer(cli.Serve.Bucket, cli.Serve.Path, logger, cli.Serve.CacheSize, cli.Serve.Cors)
server, err := pmtiles.NewServer(cli.Serve.Bucket, cli.Serve.Path, logger, cli.Serve.CacheSize, cli.Serve.Cors, cli.Serve.PublicHostname)

if err != nil {
logger.Fatalf("Failed to create new server, %v", err)
Expand Down
17 changes: 17 additions & 0 deletions pmtiles/directory.go
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,23 @@ func headerContentType(header HeaderV3) (string, bool) {
}
}

func headerExt(header HeaderV3) string {
switch header.TileType {
case Mvt:
return ".mvt"
case Png:
return ".png"
case Jpeg:
return ".jpg"
case Webp:
return ".webp"
case Avif:
return ".avif"
default:
return ""
}
}

func headerContentEncoding(compression Compression) (string, bool) {
switch compression {
case Gzip:
Expand Down
111 changes: 93 additions & 18 deletions pmtiles/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,11 @@ package pmtiles

import (
"bytes"
"compress/gzip"
"container/list"
"context"
"encoding/json"
"errors"
"fmt"
"gocloud.dev/blob"
"io"
Expand Down Expand Up @@ -61,14 +64,15 @@ type Response struct {
// }

type Server struct {
reqs chan Request
bucket *blob.Bucket
logger *log.Logger
cacheSize int
cors string
reqs chan Request
bucket *blob.Bucket
logger *log.Logger
cacheSize int
cors string
publicHostname string
}

func NewServer(bucketURL string, prefix string, logger *log.Logger, cacheSize int, cors string) (*Server, error) {
func NewServer(bucketURL string, prefix string, logger *log.Logger, cacheSize int, cors string, publicHostname string) (*Server, error) {
if bucketURL == "" {
if strings.HasPrefix(prefix, "/") {
bucketURL = "file:///"
Expand All @@ -92,11 +96,12 @@ func NewServer(bucketURL string, prefix string, logger *log.Logger, cacheSize in
}

l := &Server{
reqs: reqs,
bucket: bucket,
logger: logger,
cacheSize: cacheSize,
cors: cors,
reqs: reqs,
bucket: bucket,
logger: logger,
cacheSize: cacheSize,
cors: cors,
publicHostname: publicHostname,
}

return l, nil
Expand Down Expand Up @@ -210,32 +215,90 @@ func (server *Server) Start() {
}()
}

func (server *Server) get_metadata(ctx context.Context, http_headers map[string]string, name string) (int, map[string]string, []byte) {
func (server *Server) get_header_metadata(ctx context.Context, name string) (error, bool, HeaderV3, []byte) {
root_req := Request{key: CacheKey{name: name, offset: 0, length: 0}, value: make(chan CachedValue, 1)}
server.reqs <- root_req
root_value := <-root_req.value
header := root_value.header

if !root_value.ok {
return 404, http_headers, []byte("Archive not found")
return nil, false, HeaderV3{}, nil
}

r, err := server.bucket.NewRangeReader(ctx, name+".pmtiles", int64(header.MetadataOffset), int64(header.MetadataLength), nil)
if err != nil {
return 404, http_headers, []byte("Archive not found")
return nil, false, HeaderV3{}, nil
}
defer r.Close()
b, err := io.ReadAll(r)

var metadata_bytes []byte
if header.InternalCompression == Gzip {
metadata_reader, _ := gzip.NewReader(r)
defer metadata_reader.Close()
metadata_bytes, err = io.ReadAll(metadata_reader)
} else if header.InternalCompression == NoCompression {
metadata_bytes, err = io.ReadAll(r)
} else {
return errors.New("Unknown compression"), true, HeaderV3{}, nil
}

return nil, true, header, metadata_bytes
}

func (server *Server) get_tilejson(ctx context.Context, http_headers map[string]string, name string) (int, map[string]string, []byte) {
err, found, header, metadata_bytes := server.get_header_metadata(ctx, name)

if err != nil {
return 500, http_headers, []byte("I/O Error")
}

if header_val, ok := headerContentEncoding(header.InternalCompression); ok {
http_headers["Content-Encoding"] = header_val
if !found {
return 404, http_headers, []byte("Archive not found")
}

var metadata_map map[string]interface{}
json.Unmarshal(metadata_bytes, &metadata_map)

tilejson := make(map[string]interface{})

if server.publicHostname == "" {
return 501, http_headers, []byte("PUBLIC_HOSTNAME must be set for TileJSON")
}

http_headers["Content-Type"] = "application/json"
tilejson["tilejson"] = "3.0.0"
tilejson["scheme"] = "xyz"
tilejson["tiles"] = []string{server.publicHostname + "/" + name + "/{z}/{x}/{y}" + headerExt(header)}
tilejson["vector_layers"] = metadata_map["vector_layers"]
tilejson["attribution"] = metadata_map["attribution"]
tilejson["description"] = metadata_map["description"]
tilejson["name"] = metadata_map["name"]
tilejson["version"] = metadata_map["version"]

E7 := 10000000.0
tilejson["bounds"] = []float64{float64(header.MinLonE7) / E7, float64(header.MinLatE7) / E7, float64(header.MaxLonE7) / E7, float64(header.MaxLatE7) / E7}
tilejson["center"] = []interface{}{float64(header.CenterLonE7) / E7, float64(header.CenterLatE7) / E7, header.CenterZoom}
tilejson["minzoom"] = header.MinZoom
tilejson["maxzoom"] = header.MaxZoom

tilejson_bytes, err := json.Marshal(tilejson)

return 200, http_headers, tilejson_bytes
}

return 200, http_headers, b
func (server *Server) get_metadata(ctx context.Context, http_headers map[string]string, name string) (int, map[string]string, []byte) {
err, found, _, metadata_bytes := server.get_header_metadata(ctx, name)

if err != nil {
return 500, http_headers, []byte("I/O Error")
}

if !found {
return 404, http_headers, []byte("Archive not found")
}

http_headers["Content-Type"] = "application/json"
return 200, http_headers, metadata_bytes
}

func (server *Server) get_tile(ctx context.Context, http_headers map[string]string, name string, z uint8, x uint32, y uint32, ext string) (int, map[string]string, []byte) {
Expand Down Expand Up @@ -318,6 +381,7 @@ func (server *Server) get_tile(ctx context.Context, http_headers map[string]stri

var tilePattern = regexp.MustCompile(`^\/([-A-Za-z0-9_\/!-_\.\*'\(\)']+)\/(\d+)\/(\d+)\/(\d+)\.([a-z]+)$`)
var metadataPattern = regexp.MustCompile(`^\/([-A-Za-z0-9_\/!-_\.\*'\(\)']+)\/metadata$`)
var tileJSONPattern = regexp.MustCompile(`^\/([-A-Za-z0-9_\/!-_\.\*'\(\)']+)\.json$`)

func parse_tile_path(path string) (bool, string, uint8, uint32, uint32, string) {
if res := tilePattern.FindStringSubmatch(path); res != nil {
Expand All @@ -331,6 +395,14 @@ func parse_tile_path(path string) (bool, string, uint8, uint32, uint32, string)
return false, "", 0, 0, 0, ""
}

func parse_tilejson_path(path string) (bool, string) {
if res := tileJSONPattern.FindStringSubmatch(path); res != nil {
name := res[1]
return true, name
}
return false, ""
}

func parse_metadata_path(path string) (bool, string) {
if res := metadataPattern.FindStringSubmatch(path); res != nil {
name := res[1]
Expand All @@ -348,6 +420,9 @@ func (server *Server) Get(ctx context.Context, path string) (int, map[string]str
if ok, key, z, x, y, ext := parse_tile_path(path); ok {
return server.get_tile(ctx, http_headers, key, z, x, y, ext)
}
if ok, key := parse_tilejson_path(path); ok {
return server.get_tilejson(ctx, http_headers, key)
}
if ok, key := parse_metadata_path(path); ok {
return server.get_metadata(ctx, http_headers, key)
}
Expand Down
3 changes: 3 additions & 0 deletions pmtiles/server_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,4 +33,7 @@ func TestRegex(t *testing.T) {
ok, key = parse_metadata_path("/!-_.*'()/metadata")
assert.True(t, ok)
assert.Equal(t, key, "!-_.*'()")
ok, key = parse_tilejson_path("/!-_.*'().json")
assert.True(t, ok)
assert.Equal(t, key, "!-_.*'()")
}