add database.IndexDatabaseWithFS

whosonfirst · Nov 3, 2023 · 1f0507c · 1f0507c
1 parent 4fcf7af
commit 1f0507c
Show file tree

Hide file tree

Showing 24 changed files with 111 additions and 57 deletions.
diff --git a/app/app.go b/app/app.go
@@ -15,7 +15,7 @@ import (
 	"github.com/sfomuseum/go-timings"
 	"github.com/whosonfirst/go-reader"
 	"github.com/whosonfirst/go-whosonfirst-iterate/v2/iterator"
-	"github.com/whosonfirst/go-whosonfirst-spatial/database"	
+	"github.com/whosonfirst/go-whosonfirst-spatial/database"
 )
 
 type SpatialApplication struct {

diff --git a/app/database.go b/app/database.go
@@ -3,7 +3,7 @@ package app
 import (
 	"context"
 	"flag"
-	
+
 	"github.com/sfomuseum/go-flags/lookup"
 	"github.com/whosonfirst/go-whosonfirst-spatial/database"
 	"github.com/whosonfirst/go-whosonfirst-spatial/flags"

diff --git a/app/iterator.go b/app/iterator.go
@@ -12,7 +12,7 @@ import (
 	"github.com/whosonfirst/go-whosonfirst-iterate/v2/iterator"
 	"github.com/whosonfirst/go-whosonfirst-spatial/database"
 	"github.com/whosonfirst/go-whosonfirst-spatial/flags"
-	"github.com/whosonfirst/warning"	
+	"github.com/whosonfirst/warning"
 )
 
 func NewIteratorWithFlagSet(ctx context.Context, fl *flag.FlagSet, spatial_db database.SpatialDatabase) (*iterator.Iterator, error) {

diff --git a/app/placetypes.go b/app/placetypes.go
@@ -8,7 +8,7 @@ import (
 
 	"github.com/sfomuseum/go-flags/lookup"
 	"github.com/whosonfirst/go-whosonfirst-placetypes"
-	"github.com/whosonfirst/go-whosonfirst-spatial/flags"	
+	"github.com/whosonfirst/go-whosonfirst-spatial/flags"
 )
 
 func AppendCustomPlacetypesWithFlagSet(ctx context.Context, fs *flag.FlagSet) error {

diff --git a/app/properties.go b/app/properties.go
@@ -4,7 +4,7 @@ import (
 	"context"
 	"flag"
 	"fmt"
-	
+
 	"github.com/sfomuseum/go-flags/lookup"
 	"github.com/whosonfirst/go-reader"
 	"github.com/whosonfirst/go-whosonfirst-spatial/flags"

diff --git a/database/database.go b/database/database.go
@@ -6,11 +6,11 @@ import (
 	"net/url"
 	"sort"
 	"strings"
-	
+
 	"github.com/aaronland/go-roster"
 	"github.com/whosonfirst/go-reader"
 	"github.com/whosonfirst/go-whosonfirst-spatial"
-	"github.com/whosonfirst/go-writer/v3"	
+	"github.com/whosonfirst/go-writer/v3"
 )
 
 type SpatialDatabase interface {

diff --git a/database/fs.go b/database/fs.go
@@ -0,0 +1,50 @@
+package database
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"io/fs"
+
+	"github.com/whosonfirst/go-whosonfirst-feature/geometry"
+)
+
+func IndexDatabaseWithFS(ctx context.Context, db SpatialDatabase, index_fs fs.FS) error {
+
+	walk_func := func(path string, d fs.DirEntry, err error) error {
+
+		if d.IsDir() {
+			return nil
+		}
+
+		r, err := index_fs.Open(path)
+
+		if err != nil {
+			return fmt.Errorf("Failed to open %s for reading, %w", path, err)
+		}
+
+		defer r.Close()
+
+		body, err := io.ReadAll(r)
+
+		if err != nil {
+			return fmt.Errorf("Failed to read %s, %w", path, err)
+		}
+
+		geom_type, err := geometry.Type(body)
+
+		if err != nil {
+			return fmt.Errorf("Failed to derive geometry type for %s, %w", path, err)
+		}
+
+		switch geom_type {
+		case "Polygon", "MultiPolygon":
+			return db.IndexFeature(ctx, body)
+		default:
+			return nil
+		}
+		return nil
+	}
+
+	return fs.WalkDir(index_fs, ".", walk_func)
+}
diff --git a/database/iterator.go b/database/iterator.go
@@ -6,7 +6,7 @@ import (
 	"io"
 
 	"github.com/whosonfirst/go-whosonfirst-feature/geometry"
-	"github.com/whosonfirst/go-whosonfirst-iterate/v2/iterator"	
+	"github.com/whosonfirst/go-whosonfirst-iterate/v2/iterator"
 )
 
 // IndexDatabaseWithIterator is a general-purpose method for indexing a `database.Spatial.Database` instance with a

diff --git a/filter/filter.go b/filter/filter.go
@@ -5,12 +5,12 @@ import (
 	"fmt"
 	"log"
 	"runtime"
-	
+
 	"github.com/whosonfirst/go-whosonfirst-flags/date"
 	"github.com/whosonfirst/go-whosonfirst-flags/geometry"
 	"github.com/whosonfirst/go-whosonfirst-flags/placetypes"
 	"github.com/whosonfirst/go-whosonfirst-spatial"
-	"github.com/whosonfirst/go-whosonfirst-spr/v2"	
+	"github.com/whosonfirst/go-whosonfirst-spr/v2"
 )
 
 func FilterSPR(filters spatial.Filter, s spr.StandardPlacesResult) error {
@@ -91,29 +91,29 @@ func FilterSPR(filters spatial.Filter, s spr.StandardPlacesResult) error {
 	case "js":
 		// This will always fail under JS (WASM)
 	default:
-
-	af, err := geometry.NewAlternateGeometryFlag(s.Path())
 
-	if err != nil {
+		af, err := geometry.NewAlternateGeometryFlag(s.Path())
 
-		msg := fmt.Sprintf("Unable to parse alternate geometry (%s) for ID %s, because '%s' - skipping alternate geometry filters", s.Path(), s.Id(), err)
-		log.Println(msg)
+		if err != nil {
 
-	} else {
+			msg := fmt.Sprintf("Unable to parse alternate geometry (%s) for ID %s, because '%s' - skipping alternate geometry filters", s.Path(), s.Id(), err)
+			log.Println(msg)
 
-		ok = filters.IsAlternateGeometry(af)
+		} else {
 
-		if !ok {
-			return errors.New("Failed 'is alternate geometry' test")
-		}
+			ok = filters.IsAlternateGeometry(af)
 
-		ok = filters.HasAlternateGeometry(af)
+			if !ok {
+				return errors.New("Failed 'is alternate geometry' test")
+			}
 
-		if !ok {
-			return errors.New("Failed 'has alternate geometry' test")
+			ok = filters.HasAlternateGeometry(af)
+
+			if !ok {
+				return errors.New("Failed 'has alternate geometry' test")
+			}
 		}
 	}
-	}
-
+
 	return nil
 }
diff --git a/filter/flagset.go b/filter/flagset.go
@@ -2,7 +2,7 @@ package filter
 
 import (
 	"flag"
-	
+
 	"github.com/sfomuseum/go-flags/lookup"
 	"github.com/whosonfirst/go-whosonfirst-spatial"
 	"github.com/whosonfirst/go-whosonfirst-spatial/flags"

diff --git a/filter/query.go b/filter/query.go
@@ -4,7 +4,7 @@ import (
 	"net/url"
 	"strconv"
 
-	"github.com/whosonfirst/go-whosonfirst-spatial"	
+	"github.com/whosonfirst/go-whosonfirst-spatial"
 )
 
 func NewSPRFilterFromQuery(query url.Values) (spatial.Filter, error) {

diff --git a/filter/spr.go b/filter/spr.go
@@ -12,7 +12,7 @@ import (
 	"github.com/whosonfirst/go-whosonfirst-flags/existential"
 	"github.com/whosonfirst/go-whosonfirst-flags/geometry"
 	"github.com/whosonfirst/go-whosonfirst-flags/placetypes"
-	"github.com/whosonfirst/go-whosonfirst-spatial"	
+	"github.com/whosonfirst/go-whosonfirst-spatial"
 )
 
 var sanitizeOpts *sanitize.Options

diff --git a/flags/common.go b/flags/common.go
@@ -3,7 +3,7 @@ package flags
 import (
 	"flag"
 	"fmt"
-	
+
 	"github.com/sfomuseum/go-flags/flagset"
 	"github.com/sfomuseum/go-flags/lookup"
 	"github.com/whosonfirst/go-reader"

diff --git a/flags/iterator.go b/flags/iterator.go
@@ -7,7 +7,7 @@ import (
 	"strings"
 
 	"github.com/sfomuseum/go-flags/lookup"
-	"github.com/whosonfirst/go-whosonfirst-iterate/v2/emitter"	
+	"github.com/whosonfirst/go-whosonfirst-iterate/v2/emitter"
 )
 
 func AppendIndexingFlags(fs *flag.FlagSet) error {

diff --git a/flags/query.go b/flags/query.go
@@ -3,7 +3,7 @@ package flags
 import (
 	"flag"
 	"fmt"
-	
+
 	"github.com/sfomuseum/go-flags/lookup"
 	"github.com/sfomuseum/go-flags/multi"
 	"github.com/whosonfirst/go-whosonfirst-spatial/geo"

diff --git a/geo/boundingbox.go b/geo/boundingbox.go
@@ -2,7 +2,7 @@ package geo
 
 import (
 	"errors"
-	
+
 	"github.com/paulmach/orb"
 )
 

diff --git a/geo/coord.go b/geo/coord.go
@@ -2,7 +2,7 @@ package geo
 
 import (
 	"errors"
-	
+
 	"github.com/paulmach/orb"
 )
 

diff --git a/go.mod b/go.mod
@@ -7,7 +7,7 @@ require (
 	github.com/paulmach/orb v0.10.0
 	github.com/sfomuseum/go-flags v0.10.0
 	github.com/sfomuseum/go-timings v1.2.1
-	github.com/tidwall/gjson v1.16.0
+	github.com/tidwall/gjson v1.17.0
 	github.com/tidwall/sjson v1.2.5
 	github.com/whosonfirst/go-reader v1.0.2
 	github.com/whosonfirst/go-sanitize v0.1.0

diff --git a/go.sum b/go.sum
@@ -49,8 +49,8 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
 github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
-github.com/tidwall/gjson v1.16.0 h1:SyXa+dsSPpUlcwEDuKuEBJEz5vzTvOea+9rjyYodQFg=
-github.com/tidwall/gjson v1.16.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/gjson v1.17.0 h1:/Jocvlh98kcTfpN2+JzGQWQcqrPQwDrVEMApx/M5ZwM=
+github.com/tidwall/gjson v1.17.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
 github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
 github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=

diff --git a/properties.go b/properties.go
@@ -11,7 +11,7 @@ import (
 	"github.com/tidwall/gjson"
 	"github.com/tidwall/sjson"
 	"github.com/whosonfirst/go-reader"
-	"github.com/whosonfirst/go-whosonfirst-spr/v2"	
+	"github.com/whosonfirst/go-whosonfirst-spr/v2"
 )
 
 type PropertiesResponseOptions struct {

diff --git a/spatial.go b/spatial.go
@@ -2,7 +2,7 @@ package spatial
 
 import (
 	"context"
-	
+
 	"github.com/paulmach/orb"
 	"github.com/whosonfirst/go-whosonfirst-flags"
 	"github.com/whosonfirst/go-whosonfirst-spr/v2"

diff --git a/vendor/github.com/tidwall/gjson/README.md b/vendor/github.com/tidwall/gjson/README.md
diff --git a/vendor/github.com/tidwall/gjson/gjson.go b/vendor/github.com/tidwall/gjson/gjson.go
diff --git a/vendor/modules.txt b/vendor/modules.txt
@@ -46,7 +46,7 @@ github.com/sfomuseum/go-timings
 # github.com/sfomuseum/iso8601duration v1.1.0
 ## explicit; go 1.18
 github.com/sfomuseum/iso8601duration
-# github.com/tidwall/gjson v1.16.0
+# github.com/tidwall/gjson v1.17.0
 ## explicit; go 1.12
 github.com/tidwall/gjson
 # github.com/tidwall/match v1.1.1
-Original file line number
+Diff line change
@@ Expand Up / @@ -2,7 +2,7 @@ package geo @@
     import (
     	"errors"
     	"github.com/paulmach/orb"
     )
@@ Expand Down @@