diff --git a/app/app.go b/app/app.go index 1e68dfc..716a5a6 100644 --- a/app/app.go +++ b/app/app.go @@ -15,7 +15,7 @@ import ( "github.com/sfomuseum/go-timings" "github.com/whosonfirst/go-reader" "github.com/whosonfirst/go-whosonfirst-iterate/v2/iterator" - "github.com/whosonfirst/go-whosonfirst-spatial/database" + "github.com/whosonfirst/go-whosonfirst-spatial/database" ) type SpatialApplication struct { diff --git a/app/database.go b/app/database.go index f42485a..c7bce5b 100644 --- a/app/database.go +++ b/app/database.go @@ -3,7 +3,7 @@ package app import ( "context" "flag" - + "github.com/sfomuseum/go-flags/lookup" "github.com/whosonfirst/go-whosonfirst-spatial/database" "github.com/whosonfirst/go-whosonfirst-spatial/flags" diff --git a/app/iterator.go b/app/iterator.go index c2be970..c0148ad 100644 --- a/app/iterator.go +++ b/app/iterator.go @@ -12,7 +12,7 @@ import ( "github.com/whosonfirst/go-whosonfirst-iterate/v2/iterator" "github.com/whosonfirst/go-whosonfirst-spatial/database" "github.com/whosonfirst/go-whosonfirst-spatial/flags" - "github.com/whosonfirst/warning" + "github.com/whosonfirst/warning" ) func NewIteratorWithFlagSet(ctx context.Context, fl *flag.FlagSet, spatial_db database.SpatialDatabase) (*iterator.Iterator, error) { diff --git a/app/placetypes.go b/app/placetypes.go index a4b6bf2..b3943e5 100644 --- a/app/placetypes.go +++ b/app/placetypes.go @@ -8,7 +8,7 @@ import ( "github.com/sfomuseum/go-flags/lookup" "github.com/whosonfirst/go-whosonfirst-placetypes" - "github.com/whosonfirst/go-whosonfirst-spatial/flags" + "github.com/whosonfirst/go-whosonfirst-spatial/flags" ) func AppendCustomPlacetypesWithFlagSet(ctx context.Context, fs *flag.FlagSet) error { diff --git a/app/properties.go b/app/properties.go index 7da2415..7297432 100644 --- a/app/properties.go +++ b/app/properties.go @@ -4,7 +4,7 @@ import ( "context" "flag" "fmt" - + "github.com/sfomuseum/go-flags/lookup" "github.com/whosonfirst/go-reader" "github.com/whosonfirst/go-whosonfirst-spatial/flags" diff --git a/database/database.go b/database/database.go index 080a0e7..892b102 100644 --- a/database/database.go +++ b/database/database.go @@ -6,11 +6,11 @@ import ( "net/url" "sort" "strings" - + "github.com/aaronland/go-roster" "github.com/whosonfirst/go-reader" "github.com/whosonfirst/go-whosonfirst-spatial" - "github.com/whosonfirst/go-writer/v3" + "github.com/whosonfirst/go-writer/v3" ) type SpatialDatabase interface { diff --git a/database/fs.go b/database/fs.go new file mode 100644 index 0000000..ca0b3e4 --- /dev/null +++ b/database/fs.go @@ -0,0 +1,50 @@ +package database + +import ( + "context" + "fmt" + "io" + "io/fs" + + "github.com/whosonfirst/go-whosonfirst-feature/geometry" +) + +func IndexDatabaseWithFS(ctx context.Context, db SpatialDatabase, index_fs fs.FS) error { + + walk_func := func(path string, d fs.DirEntry, err error) error { + + if d.IsDir() { + return nil + } + + r, err := index_fs.Open(path) + + if err != nil { + return fmt.Errorf("Failed to open %s for reading, %w", path, err) + } + + defer r.Close() + + body, err := io.ReadAll(r) + + if err != nil { + return fmt.Errorf("Failed to read %s, %w", path, err) + } + + geom_type, err := geometry.Type(body) + + if err != nil { + return fmt.Errorf("Failed to derive geometry type for %s, %w", path, err) + } + + switch geom_type { + case "Polygon", "MultiPolygon": + return db.IndexFeature(ctx, body) + default: + return nil + } + return nil + } + + return fs.WalkDir(index_fs, ".", walk_func) +} diff --git a/database/iterator.go b/database/iterator.go index 6ff977b..835f60a 100644 --- a/database/iterator.go +++ b/database/iterator.go @@ -6,7 +6,7 @@ import ( "io" "github.com/whosonfirst/go-whosonfirst-feature/geometry" - "github.com/whosonfirst/go-whosonfirst-iterate/v2/iterator" + "github.com/whosonfirst/go-whosonfirst-iterate/v2/iterator" ) // IndexDatabaseWithIterator is a general-purpose method for indexing a `database.Spatial.Database` instance with a diff --git a/filter/filter.go b/filter/filter.go index 92cec61..a35176d 100644 --- a/filter/filter.go +++ b/filter/filter.go @@ -5,12 +5,12 @@ import ( "fmt" "log" "runtime" - + "github.com/whosonfirst/go-whosonfirst-flags/date" "github.com/whosonfirst/go-whosonfirst-flags/geometry" "github.com/whosonfirst/go-whosonfirst-flags/placetypes" "github.com/whosonfirst/go-whosonfirst-spatial" - "github.com/whosonfirst/go-whosonfirst-spr/v2" + "github.com/whosonfirst/go-whosonfirst-spr/v2" ) func FilterSPR(filters spatial.Filter, s spr.StandardPlacesResult) error { @@ -91,29 +91,29 @@ func FilterSPR(filters spatial.Filter, s spr.StandardPlacesResult) error { case "js": // This will always fail under JS (WASM) default: - - af, err := geometry.NewAlternateGeometryFlag(s.Path()) - if err != nil { + af, err := geometry.NewAlternateGeometryFlag(s.Path()) - msg := fmt.Sprintf("Unable to parse alternate geometry (%s) for ID %s, because '%s' - skipping alternate geometry filters", s.Path(), s.Id(), err) - log.Println(msg) + if err != nil { - } else { + msg := fmt.Sprintf("Unable to parse alternate geometry (%s) for ID %s, because '%s' - skipping alternate geometry filters", s.Path(), s.Id(), err) + log.Println(msg) - ok = filters.IsAlternateGeometry(af) + } else { - if !ok { - return errors.New("Failed 'is alternate geometry' test") - } + ok = filters.IsAlternateGeometry(af) - ok = filters.HasAlternateGeometry(af) + if !ok { + return errors.New("Failed 'is alternate geometry' test") + } - if !ok { - return errors.New("Failed 'has alternate geometry' test") + ok = filters.HasAlternateGeometry(af) + + if !ok { + return errors.New("Failed 'has alternate geometry' test") + } } } - } - + return nil } diff --git a/filter/flagset.go b/filter/flagset.go index 872538a..25d976d 100644 --- a/filter/flagset.go +++ b/filter/flagset.go @@ -2,7 +2,7 @@ package filter import ( "flag" - + "github.com/sfomuseum/go-flags/lookup" "github.com/whosonfirst/go-whosonfirst-spatial" "github.com/whosonfirst/go-whosonfirst-spatial/flags" diff --git a/filter/query.go b/filter/query.go index 982be4b..9c6e979 100644 --- a/filter/query.go +++ b/filter/query.go @@ -4,7 +4,7 @@ import ( "net/url" "strconv" - "github.com/whosonfirst/go-whosonfirst-spatial" + "github.com/whosonfirst/go-whosonfirst-spatial" ) func NewSPRFilterFromQuery(query url.Values) (spatial.Filter, error) { diff --git a/filter/spr.go b/filter/spr.go index e7e11cb..87c6150 100644 --- a/filter/spr.go +++ b/filter/spr.go @@ -12,7 +12,7 @@ import ( "github.com/whosonfirst/go-whosonfirst-flags/existential" "github.com/whosonfirst/go-whosonfirst-flags/geometry" "github.com/whosonfirst/go-whosonfirst-flags/placetypes" - "github.com/whosonfirst/go-whosonfirst-spatial" + "github.com/whosonfirst/go-whosonfirst-spatial" ) var sanitizeOpts *sanitize.Options diff --git a/flags/common.go b/flags/common.go index 3d92dea..84de111 100644 --- a/flags/common.go +++ b/flags/common.go @@ -3,7 +3,7 @@ package flags import ( "flag" "fmt" - + "github.com/sfomuseum/go-flags/flagset" "github.com/sfomuseum/go-flags/lookup" "github.com/whosonfirst/go-reader" diff --git a/flags/iterator.go b/flags/iterator.go index ccae8e6..13d7f78 100644 --- a/flags/iterator.go +++ b/flags/iterator.go @@ -7,7 +7,7 @@ import ( "strings" "github.com/sfomuseum/go-flags/lookup" - "github.com/whosonfirst/go-whosonfirst-iterate/v2/emitter" + "github.com/whosonfirst/go-whosonfirst-iterate/v2/emitter" ) func AppendIndexingFlags(fs *flag.FlagSet) error { diff --git a/flags/query.go b/flags/query.go index e84bd7f..044cf13 100644 --- a/flags/query.go +++ b/flags/query.go @@ -3,7 +3,7 @@ package flags import ( "flag" "fmt" - + "github.com/sfomuseum/go-flags/lookup" "github.com/sfomuseum/go-flags/multi" "github.com/whosonfirst/go-whosonfirst-spatial/geo" diff --git a/geo/boundingbox.go b/geo/boundingbox.go index 8657a37..d52066e 100644 --- a/geo/boundingbox.go +++ b/geo/boundingbox.go @@ -2,7 +2,7 @@ package geo import ( "errors" - + "github.com/paulmach/orb" ) diff --git a/geo/coord.go b/geo/coord.go index 4c3cc1c..e6d6b94 100644 --- a/geo/coord.go +++ b/geo/coord.go @@ -2,7 +2,7 @@ package geo import ( "errors" - + "github.com/paulmach/orb" ) diff --git a/go.mod b/go.mod index c963509..bb2d17e 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/paulmach/orb v0.10.0 github.com/sfomuseum/go-flags v0.10.0 github.com/sfomuseum/go-timings v1.2.1 - github.com/tidwall/gjson v1.16.0 + github.com/tidwall/gjson v1.17.0 github.com/tidwall/sjson v1.2.5 github.com/whosonfirst/go-reader v1.0.2 github.com/whosonfirst/go-sanitize v0.1.0 diff --git a/go.sum b/go.sum index 1368fc2..ccf0518 100644 --- a/go.sum +++ b/go.sum @@ -49,8 +49,8 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/gjson v1.16.0 h1:SyXa+dsSPpUlcwEDuKuEBJEz5vzTvOea+9rjyYodQFg= -github.com/tidwall/gjson v1.16.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.17.0 h1:/Jocvlh98kcTfpN2+JzGQWQcqrPQwDrVEMApx/M5ZwM= +github.com/tidwall/gjson v1.17.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= diff --git a/properties.go b/properties.go index a3badce..caa0e61 100644 --- a/properties.go +++ b/properties.go @@ -11,7 +11,7 @@ import ( "github.com/tidwall/gjson" "github.com/tidwall/sjson" "github.com/whosonfirst/go-reader" - "github.com/whosonfirst/go-whosonfirst-spr/v2" + "github.com/whosonfirst/go-whosonfirst-spr/v2" ) type PropertiesResponseOptions struct { diff --git a/spatial.go b/spatial.go index 5e06c6f..0dfca9a 100644 --- a/spatial.go +++ b/spatial.go @@ -2,7 +2,7 @@ package spatial import ( "context" - + "github.com/paulmach/orb" "github.com/whosonfirst/go-whosonfirst-flags" "github.com/whosonfirst/go-whosonfirst-spr/v2" diff --git a/vendor/github.com/tidwall/gjson/README.md b/vendor/github.com/tidwall/gjson/README.md index da2bad4..96b2e4d 100644 --- a/vendor/github.com/tidwall/gjson/README.md +++ b/vendor/github.com/tidwall/gjson/README.md @@ -427,16 +427,6 @@ if result.Index > 0 { This is a best-effort no allocation sub slice of the original json. This method utilizes the `result.Index` field, which is the position of the raw data in the original json. It's possible that the value of `result.Index` equals zero, in which case the `result.Raw` is converted to a `[]byte`. -## Get multiple values at once - -The `GetMany` function can be used to get multiple values at the same time. - -```go -results := gjson.GetMany(json, "name.first", "name.last", "age") -``` - -The return value is a `[]Result`, which will always contain exactly the same number of items as the input paths. - ## Performance Benchmarks of GJSON alongside [encoding/json](https://golang.org/pkg/encoding/json/), diff --git a/vendor/github.com/tidwall/gjson/gjson.go b/vendor/github.com/tidwall/gjson/gjson.go index a1633be..7949825 100644 --- a/vendor/github.com/tidwall/gjson/gjson.go +++ b/vendor/github.com/tidwall/gjson/gjson.go @@ -3410,7 +3410,7 @@ func (t Result) Path(json string) string { if !rcomp.Exists() { goto fail } - comp := escapeComp(rcomp.String()) + comp := Escape(rcomp.String()) path = append(path, '.') path = append(path, comp...) } @@ -3425,17 +3425,31 @@ fail: // isSafePathKeyChar returns true if the input character is safe for not // needing escaping. func isSafePathKeyChar(c byte) bool { - return c <= ' ' || c > '~' || c == '_' || c == '-' || c == ':' || - (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || - (c >= '0' && c <= '9') + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || + (c >= '0' && c <= '9') || c <= ' ' || c > '~' || c == '_' || + c == '-' || c == ':' } -// escapeComp escaped a path compontent, making it safe for generating a -// path for later use. -func escapeComp(comp string) string { +// Escape returns an escaped path component. +// +// json := `{ +// "user":{ +// "first.name": "Janet", +// "last.name": "Prichard" +// } +// }` +// user := gjson.Get(json, "user") +// println(user.Get(gjson.Escape("first.name")) +// println(user.Get(gjson.Escape("last.name")) +// // Output: +// // Janet +// // Prichard +func Escape(comp string) string { for i := 0; i < len(comp); i++ { if !isSafePathKeyChar(comp[i]) { - ncomp := []byte(comp[:i]) + ncomp := make([]byte, len(comp)+1) + copy(ncomp, comp[:i]) + ncomp = ncomp[:i] for ; i < len(comp); i++ { if !isSafePathKeyChar(comp[i]) { ncomp = append(ncomp, '\\') diff --git a/vendor/modules.txt b/vendor/modules.txt index c226cdd..0729d98 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -46,7 +46,7 @@ github.com/sfomuseum/go-timings # github.com/sfomuseum/iso8601duration v1.1.0 ## explicit; go 1.18 github.com/sfomuseum/iso8601duration -# github.com/tidwall/gjson v1.16.0 +# github.com/tidwall/gjson v1.17.0 ## explicit; go 1.12 github.com/tidwall/gjson # github.com/tidwall/match v1.1.1