Skip to content

Commit

Permalink
storage: bug fixes for postgres (#16)
Browse files Browse the repository at this point in the history
- postgres bugs w multiple feeds
- test for storage w multiple feeds 
- move gtfs_test to gtfs/testutils
matslina authored Jan 2, 2024

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
1 parent cdba1e2 commit 04e9e8c
Showing 10 changed files with 553 additions and 101 deletions.
54 changes: 19 additions & 35 deletions manager_test.go
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
package gtfs_test

import (
"archive/zip"
"bytes"
"context"
"errors"
"fmt"
@@ -11,7 +9,6 @@ import (
"net/http/httptest"
"os"
"sort"
"strings"
"testing"
"time"

@@ -23,6 +20,7 @@ import (
"tidbyt.dev/gtfs"
"tidbyt.dev/gtfs/downloader"
"tidbyt.dev/gtfs/storage"
"tidbyt.dev/gtfs/testutil"
)

type MockGTFSServer struct {
@@ -133,27 +131,13 @@ func validRealtimeFeed(t *testing.T, timestamp time.Time) []byte {
return data
}

func buildZip(t *testing.T, files map[string][]string) []byte {
buf := &bytes.Buffer{}
w := zip.NewWriter(buf)
for filename, content := range files {
f, err := w.Create(filename)
require.NoError(t, err)
_, err = f.Write([]byte(strings.Join(content, "\n")))
require.NoError(t, err)
}
require.NoError(t, w.Close())

return buf.Bytes()
}

func testManagerLoadSingleFeed(t *testing.T, strg storage.Storage) {
m := gtfs.NewManager(strg)

server := managerFixture()
defer server.Server.Close()

server.Feeds["/static.zip"] = buildZip(t, validFeed())
server.Feeds["/static.zip"] = testutil.BuildZip(t, validFeed())

when := time.Date(2019, 2, 1, 0, 0, 0, 0, time.UTC)

@@ -183,7 +167,7 @@ func testManagerLoadMultipleURLs(t *testing.T, strg storage.Storage) {

// Two different static feeds, served on separate URLs.
files := validFeed()
server.Feeds["/static1.zip"] = buildZip(t, validFeed())
server.Feeds["/static1.zip"] = testutil.BuildZip(t, validFeed())
files["stops.txt"] = []string{
"stop_id,stop_name,stop_lat,stop_lon",
"s2,S2,12,34",
@@ -192,7 +176,7 @@ func testManagerLoadMultipleURLs(t *testing.T, strg storage.Storage) {
"trip_id,arrival_time,departure_time,stop_id,stop_sequence",
"t,12:00:00,12:00:00,s2,1",
}
server.Feeds["/static2.zip"] = buildZip(t, files)
server.Feeds["/static2.zip"] = testutil.BuildZip(t, files)

// First request for each will fail, but create requests.
when := time.Date(2019, 2, 1, 0, 0, 0, 0, time.UTC)
@@ -232,7 +216,7 @@ func testManagerLoadWithHeaders(t *testing.T, strg storage.Storage) {

// Three feeds, on separate URLs.
files := validFeed()
server.Feeds["/static1.zip"] = buildZip(t, validFeed())
server.Feeds["/static1.zip"] = testutil.BuildZip(t, validFeed())
files["stops.txt"] = []string{
"stop_id,stop_name,stop_lat,stop_lon",
"s2,S2,12,34",
@@ -241,7 +225,7 @@ func testManagerLoadWithHeaders(t *testing.T, strg storage.Storage) {
"trip_id,arrival_time,departure_time,stop_id,stop_sequence",
"t,12:00:00,12:00:00,s2,1",
}
server.Feeds["/static2.zip"] = buildZip(t, files)
server.Feeds["/static2.zip"] = testutil.BuildZip(t, files)
files["stops.txt"] = []string{
"stop_id,stop_name,stop_lat,stop_lon",
"s3,S3,12,34",
@@ -250,7 +234,7 @@ func testManagerLoadWithHeaders(t *testing.T, strg storage.Storage) {
"trip_id,arrival_time,departure_time,stop_id,stop_sequence",
"t,12:00:00,12:00:00,s3,1",
}
server.Feeds["/static3.zip"] = buildZip(t, files)
server.Feeds["/static3.zip"] = testutil.BuildZip(t, files)

// First and second requires different headers. Third requires
// no headers.
@@ -326,7 +310,7 @@ func testManagerMultipleConsumers(t *testing.T, strg storage.Storage) {

// Three feeds, on separate URLs.
files := validFeed()
server.Feeds["/static1.zip"] = buildZip(t, validFeed())
server.Feeds["/static1.zip"] = testutil.BuildZip(t, validFeed())
files["stops.txt"] = []string{
"stop_id,stop_name,stop_lat,stop_lon",
"s2,S2,12,34",
@@ -335,7 +319,7 @@ func testManagerMultipleConsumers(t *testing.T, strg storage.Storage) {
"trip_id,arrival_time,departure_time,stop_id,stop_sequence",
"t,12:00:00,12:00:00,s2,1",
}
server.Feeds["/static2.zip"] = buildZip(t, files)
server.Feeds["/static2.zip"] = testutil.BuildZip(t, files)
files["stops.txt"] = []string{
"stop_id,stop_name,stop_lat,stop_lon",
"s3,S3,12,34",
@@ -344,7 +328,7 @@ func testManagerMultipleConsumers(t *testing.T, strg storage.Storage) {
"trip_id,arrival_time,departure_time,stop_id,stop_sequence",
"t,12:00:00,12:00:00,s3,1",
}
server.Feeds["/static3.zip"] = buildZip(t, files)
server.Feeds["/static3.zip"] = testutil.BuildZip(t, files)

// First and second requires different headers. Third requires
// no headers.
@@ -481,7 +465,7 @@ func testManagerLoadWithRefresh(t *testing.T, strg storage.Storage) {

// Three versions of a feed, each differing in stops.txt.
files := validFeed()
feed1Zip := buildZip(t, files)
feed1Zip := testutil.BuildZip(t, files)
files["stops.txt"] = []string{
"stop_id,stop_name,stop_lat,stop_lon",
"s2,S,12,34",
@@ -490,7 +474,7 @@ func testManagerLoadWithRefresh(t *testing.T, strg storage.Storage) {
"trip_id,arrival_time,departure_time,stop_id,stop_sequence",
"t,12:00:00,12:00:00,s2,1",
}
feed2Zip := buildZip(t, files)
feed2Zip := testutil.BuildZip(t, files)
files["stops.txt"] = []string{
"stop_id,stop_name,stop_lat,stop_lon",
"s3,S,12,34",
@@ -499,7 +483,7 @@ func testManagerLoadWithRefresh(t *testing.T, strg storage.Storage) {
"trip_id,arrival_time,departure_time,stop_id,stop_sequence",
"t,12:00:00,12:00:00,s3,1",
}
feed3Zip := buildZip(t, files)
feed3Zip := testutil.BuildZip(t, files)

when := time.Date(2019, 2, 1, 0, 0, 0, 0, time.UTC)

@@ -613,8 +597,8 @@ func testManagerBrokenData(t *testing.T, strg storage.Storage) {
server := managerFixture()
defer server.Server.Close()

goodZip := buildZip(t, validFeed())
badZip := buildZip(t, map[string][]string{"parse": []string{"fail"}})
goodZip := testutil.BuildZip(t, validFeed())
badZip := testutil.BuildZip(t, map[string][]string{"parse": []string{"fail"}})

when := time.Date(2019, 2, 1, 0, 0, 0, 0, time.UTC)

@@ -699,7 +683,7 @@ func testManagerAsyncLoad(t *testing.T, strg storage.Storage) {
server := managerFixture()
defer server.Server.Close()

server.Feeds["/static.zip"] = buildZip(t, validFeed())
server.Feeds["/static.zip"] = testutil.BuildZip(t, validFeed())

when := time.Date(2019, 2, 1, 0, 0, 0, 0, time.UTC)

@@ -749,7 +733,7 @@ func testManagerLoadRealtime(t *testing.T, strg storage.Storage) {
server := managerFixture()
defer server.Server.Close()

server.Feeds["/static.zip"] = buildZip(t, validFeed())
server.Feeds["/static.zip"] = testutil.BuildZip(t, validFeed())
server.Feeds["/realtime.pb"] = validRealtimeFeed(t, time.Unix(12345, 0))

when := time.Date(2019, 2, 1, 0, 0, 0, 0, time.UTC)
@@ -885,9 +869,9 @@ func TestManager(t *testing.T) {
test.Test(t, s)

})
if PostgresConnStr != "" {
if testutil.PostgresConnStr != "" {
t.Run(fmt.Sprintf("%s_Postgres", test.Name), func(t *testing.T) {
s, err := storage.NewPSQLStorage(PostgresConnStr, true)
s, err := storage.NewPSQLStorage(testutil.PostgresConnStr, true)
require.NoError(t, err)
test.Test(t, s)
})
3 changes: 2 additions & 1 deletion realtime_integration_test.go
Original file line number Diff line number Diff line change
@@ -17,10 +17,11 @@ import (
"github.com/stretchr/testify/require"

"tidbyt.dev/gtfs"
"tidbyt.dev/gtfs/testutil"
)

func loadNYCFerryRealtime(t *testing.T, suffix string) *gtfs.Realtime {
static := GTFSTest_LoadStaticFile(t, "sqlite", "testdata/nycferry_static.zip")
static := testutil.LoadStaticFile(t, "sqlite", "testdata/nycferry_static.zip")

buf, err := ioutil.ReadFile(fmt.Sprintf("testdata/nycferry_realtime_%s", suffix))
require.NoError(t, err)
13 changes: 7 additions & 6 deletions realtime_test.go
Original file line number Diff line number Diff line change
@@ -13,6 +13,7 @@ import (

"tidbyt.dev/gtfs"
"tidbyt.dev/gtfs/storage"
"tidbyt.dev/gtfs/testutil"
)

// Helpers for building gtfs-realtime feeds
@@ -122,7 +123,7 @@ func buildFeed(t *testing.T, tripUpdates []TripUpdate) [][]byte {
// A simple Static fixture. Trips t1 and t2 cover the same three
// stops s1-s3. Trip t3 covers z1-z2. Full service all days of 2020.
func SimpleStaticFixture(t *testing.T) *gtfs.Static {
static := GTFSTest_BuildStatic(t, "sqlite", map[string][]string{
static := testutil.BuildStatic(t, "sqlite", map[string][]string{
"calendar.txt": {
"service_id,start_date,end_date,monday,tuesday,wednesday,thursday,friday,saturday,sunday",
"everyday,20200101,20210101,1,1,1,1,1,1,1",
@@ -811,7 +812,7 @@ func TestRealtimeTimeWindowing(t *testing.T) {
func TestRealtimeTripWithLoop(t *testing.T) {
// This static schedule has t1 running from s1 to s2, and then
// 3 loops s3-s5, and finally end of the trip at s3.
static := GTFSTest_BuildStatic(t, "sqlite", map[string][]string{
static := testutil.BuildStatic(t, "sqlite", map[string][]string{
// A weekdays only schedule
"calendar.txt": {
"service_id,start_date,end_date,monday,tuesday,wednesday,thursday,friday,saturday,sunday",
@@ -986,7 +987,7 @@ func TestRealtimeDepartureFiltering(t *testing.T) {
// Two routes: bus going center to south and rail going center
// to east. Each route has two trips: one heading out, one
// heading in.
static := GTFSTest_BuildStatic(t, "sqlite", map[string][]string{
static := testutil.BuildStatic(t, "sqlite", map[string][]string{
// A weekdays only schedule
"calendar.txt": {
"service_id,start_date,end_date,monday,tuesday,wednesday,thursday,friday,saturday,sunday",
@@ -1446,7 +1447,7 @@ func TestRealtimeArrivalRecovery(t *testing.T) {

func TestRealtimeDelayCrossingMidnight(t *testing.T) {
// A single trip passing through s1...s4 over midnight.
static := GTFSTest_BuildStatic(t, "sqlite", map[string][]string{
static := testutil.BuildStatic(t, "sqlite", map[string][]string{
"calendar.txt": {
"service_id,start_date,end_date,monday,tuesday,wednesday,thursday,friday,saturday,sunday",
"everyday,20200101,20210101,1,1,1,1,1,1,1",
@@ -1576,7 +1577,7 @@ func TestRealtimeDelayCrossingMidnight(t *testing.T) {

func TestRealtimeDelayCrossingDSTBoundaryOnCurrentDay(t *testing.T) {
// Delays on trips crossing a DST boundary
static := GTFSTest_BuildStatic(t, "sqlite", map[string][]string{
static := testutil.BuildStatic(t, "sqlite", map[string][]string{
"agency.txt": {
"agency_id,agency_name,agency_url,agency_timezone",
"a,A,http://a.com/,America/New_York",
@@ -1814,7 +1815,7 @@ func TestRealtimeDelayCrossingDSTBoundaryOnCurrentDay(t *testing.T) {
func TestRealtimeDelayCrossingDSTBoundaryFromPreviousDay(t *testing.T) {
// Delays on trips crossing a DST boundary on the _following
// day_ (via scheduled trips running past 24:00).
static := GTFSTest_BuildStatic(t, "sqlite", map[string][]string{
static := testutil.BuildStatic(t, "sqlite", map[string][]string{
"agency.txt": {
"agency_id,agency_name,agency_url,agency_timezone",
"a,A,http://a.com/,America/New_York",
8 changes: 5 additions & 3 deletions static_bench_test.go
Original file line number Diff line number Diff line change
@@ -3,10 +3,12 @@ package gtfs_test
import (
"testing"
"time"

"tidbyt.dev/gtfs/testutil"
)

func benchNearbyStops(b *testing.B, backend string) {
static := GTFSTest_LoadStaticFile(b, backend, "testdata/caltrain_20160406.zip")
static := testutil.LoadStaticFile(b, backend, "testdata/caltrain_20160406.zip")

b.ResetTimer()

@@ -20,7 +22,7 @@ func benchNearbyStops(b *testing.B, backend string) {
}

func benchDepartures(b *testing.B, backend string) {
static := GTFSTest_LoadStaticFile(b, backend, "testdata/caltrain_20160406.zip")
static := testutil.LoadStaticFile(b, backend, "testdata/caltrain_20160406.zip")

tz, err := time.LoadLocation("America/Los_Angeles")
if err != nil {
@@ -59,7 +61,7 @@ func BenchmarkGTFSStatic(b *testing.B) {
b.Run(test.Name+"_sqlite", func(b *testing.B) {
test.Bench(b, "sqlite")
})
if PostgresConnStr != "" {
if testutil.PostgresConnStr != "" {
b.Run(test.Name+"_postgres", func(b *testing.B) {
test.Bench(b, "postgres")
})
7 changes: 4 additions & 3 deletions static_integration_test.go
Original file line number Diff line number Diff line change
@@ -9,6 +9,7 @@ import (

"tidbyt.dev/gtfs"
"tidbyt.dev/gtfs/storage"
"tidbyt.dev/gtfs/testutil"
)

func testGTFSStaticIntegrationNearbyStops(t *testing.T, backend string) {
@@ -17,7 +18,7 @@ func testGTFSStaticIntegrationNearbyStops(t *testing.T, backend string) {
}

// This is a giant GTFS file from the MTA
g := GTFSTest_LoadStaticFile(t, backend, "testdata/mta_static.zip")
g := testutil.LoadStaticFile(t, backend, "testdata/mta_static.zip")

// The 4 nearest stops for 544 Park Ave, BK. There are other
// stops with the same coordinates, but they all have
@@ -71,7 +72,7 @@ func testGTFSStaticIntegrationDepartures(t *testing.T, backend string) {
}

// This is a giant GTFS file from the MTA
g := GTFSTest_LoadStaticFile(t, backend, "testdata/mta_static.zip")
g := testutil.LoadStaticFile(t, backend, "testdata/mta_static.zip")

// Let's look at the G33S stop, also known as "Bedford -
// Nostrand Avs". Between 22:50 and 23:10 there are are 6
@@ -201,7 +202,7 @@ func TestStaticIntegration(t *testing.T) {
t.Run(test.Name+"_SQLite", func(t *testing.T) {
test.Test(t, "sqlite")
})
if PostgresConnStr != "" {
if testutil.PostgresConnStr != "" {
t.Run(test.Name+"_postgres", func(t *testing.T) {
test.Test(t, "postgres")
})
23 changes: 12 additions & 11 deletions static_test.go
Original file line number Diff line number Diff line change
@@ -9,14 +9,15 @@ import (
"github.com/stretchr/testify/require"

"tidbyt.dev/gtfs"
"tidbyt.dev/gtfs/testutil"
)

func testStaticDeparturesWindowing(t *testing.T, backend string) {
duration := func(h, m, s int) time.Duration {
return time.Duration(h)*time.Hour + time.Duration(m)*time.Minute + time.Duration(s)*time.Second
}

g := GTFSTest_BuildStatic(t, backend, map[string][]string{
g := testutil.BuildStatic(t, backend, map[string][]string{
// A weekdays only schedule
"calendar.txt": {
"service_id,start_date,end_date,monday,tuesday,wednesday,thursday,friday,saturday,sunday",
@@ -225,7 +226,7 @@ func testStaticDeparturesWindowing(t *testing.T, backend string) {
}

func testStaticDeparturesWeekendSchedule(t *testing.T, backend string) {
g := GTFSTest_BuildStatic(t, backend, map[string][]string{
g := testutil.BuildStatic(t, backend, map[string][]string{
// A weekend and a weekday schedules
"calendar.txt": {
"service_id,start_date,end_date,monday,tuesday,wednesday,thursday,friday,saturday,sunday",
@@ -320,7 +321,7 @@ func testStaticDeparturesWeekendSchedule(t *testing.T, backend string) {
}

func testStaticDeparturesTimezones(t *testing.T, backend string) {
g := GTFSTest_BuildStatic(t, backend, map[string][]string{
g := testutil.BuildStatic(t, backend, map[string][]string{
// Eastern Time
"agency.txt": {"agency_timezone,agency_name,agency_url", "America/New_York,MTA,http://example.com"},
// Mondays only!
@@ -397,7 +398,7 @@ func testStaticDeparturesTimezones(t *testing.T, backend string) {
}

func testStaticDeparturesOvernightTrip(t *testing.T, backend string) {
g := GTFSTest_BuildStatic(t, backend, map[string][]string{
g := testutil.BuildStatic(t, backend, map[string][]string{
"agency.txt": {"agency_timezone,agency_name,agency_url", "America/New_York,MTA,http://example.com"},
"calendar.txt": {
"service_id,start_date,end_date,monday,tuesday,wednesday,thursday,friday,saturday,sunday",
@@ -487,7 +488,7 @@ func testStaticDeparturesOvernightTrip(t *testing.T, backend string) {
}

func testStaticDeparturesCalendarDateOverride(t *testing.T, backend string) {
g := GTFSTest_BuildStatic(t, backend, map[string][]string{
g := testutil.BuildStatic(t, backend, map[string][]string{
"agency.txt": {"agency_timezone,agency_name,agency_url", "America/New_York,MTA,http://example.com"},
"calendar.txt": {
"service_id,start_date,end_date,monday,tuesday,wednesday,thursday,friday,saturday,sunday",
@@ -618,7 +619,7 @@ func testStaticDeparturesCalendarDateOverride(t *testing.T, backend string) {
// using at the time of this writing).
func testStaticDeparturesNoDepartureFromFinalStop(t *testing.T, backend string) {

g := GTFSTest_BuildStatic(t, backend, map[string][]string{
g := testutil.BuildStatic(t, backend, map[string][]string{
"agency.txt": {"agency_timezone,agency_name,agency_url", "America/New_York,MTA,http://example.com"},
"calendar.txt": {
"service_id,start_date,end_date,monday,tuesday,wednesday,thursday,friday,saturday,sunday",
@@ -670,7 +671,7 @@ func testStaticDeparturesFiltering(t *testing.T, backend string) {
// This weekend schedule has RouteA running alpha-beta-gamma
// and gamma-beta-alpha a few times per day. Route B does a
// single run beta-epsilon-gamma.
g := GTFSTest_BuildStatic(t, backend, map[string][]string{
g := testutil.BuildStatic(t, backend, map[string][]string{
"agency.txt": {"agency_timezone,agency_name,agency_url", "America/New_York,MTA,http://example.com"},
"calendar.txt": {
"service_id,start_date,end_date,monday,tuesday,wednesday,thursday,friday,saturday,sunday",
@@ -818,7 +819,7 @@ func testStaticDeparturesFiltering(t *testing.T, backend string) {
// overrides the former.
func testStaticDeparturesStopTimeWithHeadsignOverride(t *testing.T, backend string) {
// A single trip on Mondays, going through the alphabet
g := GTFSTest_BuildStatic(t, backend, map[string][]string{
g := testutil.BuildStatic(t, backend, map[string][]string{
"calendar.txt": {
"service_id,start_date,end_date,monday,tuesday,wednesday,thursday,friday,saturday,sunday",
"mondays,20200101,20201231,1,0,0,0,0,0,0",
@@ -883,7 +884,7 @@ func testStaticDeparturesStopTimeWithHeadsignOverride(t *testing.T, backend stri
// Verifies that departures can be retrieved both for individual stops
// and for their parent stations (if any)
func testStaticDeparturesWithParentStations(t *testing.T, backend string) {
g := GTFSTest_BuildStatic(t, backend, map[string][]string{
g := testutil.BuildStatic(t, backend, map[string][]string{
"calendar.txt": {
"service_id,start_date,end_date,monday,tuesday,wednesday,thursday,friday,saturday,sunday",
"mondays,20200101,20201231,1,0,0,0,0,0,0",
@@ -940,7 +941,7 @@ func testStaticDeparturesWithParentStations(t *testing.T, backend string) {
func testStaticDeparturesDaylightsSavings(t *testing.T, backend string) {
// Two trips, one early in morning, one so late that it
// overflows into next day. Each on a separate route.
g := GTFSTest_BuildStatic(t, backend, map[string][]string{
g := testutil.BuildStatic(t, backend, map[string][]string{
"agency.txt": {
"agency_id,agency_name,agency_url,agency_timezone",
"1,MTA,http://mta.com/,America/New_York",
@@ -1120,7 +1121,7 @@ func TestStatic(t *testing.T) {
t.Run(fmt.Sprintf("%s SQLite", test.Name), func(t *testing.T) {
test.Test(t, "sqlite")
})
if PostgresConnStr != "" {
if testutil.PostgresConnStr != "" {
t.Run(fmt.Sprintf("%s Postgres", test.Name), func(t *testing.T) {
test.Test(t, "postgres")
})
46 changes: 29 additions & 17 deletions storage/postgres.go
Original file line number Diff line number Diff line change
@@ -994,10 +994,10 @@ Exceptions AS (
Regular AS (
SELECT service_id
FROM calendar
WHERE hash = $3 AND
WHERE hash = $1 AND
`+weekday+` = 1 AND
start_date <= $4 AND
end_date >= $5
start_date <= $2 AND
end_date >= $2
)
SELECT service_id FROM Regular
WHERE service_id NOT IN (
@@ -1006,7 +1006,7 @@ WHERE service_id NOT IN (
UNION
SELECT service_id FROM Exceptions
WHERE exception_type = 1
`, r.id, date, r.id, date, date)
`, r.id, date)
if err != nil {
return nil, fmt.Errorf("querying for active services: %w", err)
}
@@ -1090,8 +1090,8 @@ FROM stop_times
INNER JOIN stops ON stop_times.stop_id = stops.id
INNER JOIN trips ON stop_times.trip_id = trips.id
INNER JOIN routes ON trips.route_id = routes.id
WHERE stops.hash = $1 AND
stop_times.hash = $1 AND
WHERE stop_times.hash = $1 AND
stops.hash = $1 AND
trips.hash = $1 AND
routes.hash = $1
`
@@ -1273,8 +1273,7 @@ WHERE stops.hash = $1 AND
}

placeholders := []string{}
parentIDs := []interface{}{}
parentIDs = append(parentIDs, r.id)
parentIDs := []interface{}{r.id}
i := 2
for id := range parents {
parentIDs = append(parentIDs, id)
@@ -1398,7 +1397,10 @@ SELECT
FROM
stops
WHERE
stops.location_type = 0 AND parent_station IS NULL OR stops.location_type = 1`)
stops.hash = $1 AND
(stops.location_type = 0 AND parent_station IS NULL OR stops.location_type = 1)`,
r.id)

if err != nil {
return nil, fmt.Errorf("querying for nearby stops: %w", err)
}
@@ -1434,16 +1436,16 @@ WHERE
}

func (r *PSQLFeedReader) getStopsByRouteType(routeTypes []RouteType) ([]*Stop, error) {
queryValues := []interface{}{}
queryValues := []interface{}{r.id}
for _, rt := range routeTypes {
queryValues = append(queryValues, rt)
}
routeTypePlaceholders := []string{}
for i := range routeTypes {
routeTypePlaceholders = append(routeTypePlaceholders, fmt.Sprintf("$%d", i+1))
routeTypePlaceholders = append(routeTypePlaceholders, fmt.Sprintf("$%d", i+2))
}

rows, err := r.db.Query(`
query := `
SELECT
stops.id,
stops.code,
@@ -1470,18 +1472,24 @@ INNER JOIN routes ON trips.route_id = routes.id
INNER JOIN stops ON stop_times.stop_id = stops.id
LEFT OUTER JOIN stops AS parent ON stops.parent_station = parent.id
WHERE
stop_times.hash = $1 AND
trips.hash = $1 AND
routes.hash = $1 AND
stops.hash = $1 AND
stops.location_type = 0 AND
routes.type IN (`+strings.Join(routeTypePlaceholders, ", ")+`)
`, queryValues...)
routes.type IN (` + strings.Join(routeTypePlaceholders, ", ") + `)
`

rows, err := r.db.Query(query, queryValues...)
if err != nil {
return nil, fmt.Errorf("querying for stops by route type: %w", err)
return nil, fmt.Errorf("querying: %w", err)
}
defer rows.Close()

allStops := map[string]*Stop{}
for rows.Next() {
s := &Stop{}
//stopParentStation := sql.NullString{}
stopParentStation := sql.NullString{}
parentID := sql.NullString{}
parentCode := sql.NullString{}
parentName := sql.NullString{}
@@ -1500,7 +1508,7 @@ WHERE
&s.Lon,
&s.URL,
&s.LocationType,
&s.ParentStation,
&stopParentStation,
&s.PlatformCode,
&parentID,
&parentCode,
@@ -1516,6 +1524,10 @@ WHERE
return nil, fmt.Errorf("scanning stop: %w", err)
}

if stopParentStation.Valid {
s.ParentStation = stopParentStation.String
}

if parentID.Valid {
allStops[parentID.String] = &Stop{
ID: parentID.String,
6 changes: 5 additions & 1 deletion storage/storage.go
Original file line number Diff line number Diff line change
@@ -120,8 +120,12 @@ type FeedReader interface {
//
// Currently, stations are returned when available. Stops that
// lack a parent_station are also included, to accommodate
// feeds without stations. This behavior should probable be
// feeds without stations. This behavior should probably be
// configurable/optional.
//
// TODO: This feels really stupid. Should probably return only
// stops, and include parent stations if it's available. Let
// the caller decide what to do with that.
NearbyStops(lat float64, lng float64, limit int, routeTypes []RouteType) ([]Stop, error)
}

461 changes: 451 additions & 10 deletions storage/storage_test.go

Large diffs are not rendered by default.

33 changes: 19 additions & 14 deletions gtfs_test.go → testutil/testutil.go
Original file line number Diff line number Diff line change
@@ -1,10 +1,6 @@
package gtfs_test
package testutil

// Helpers and configuration for tests.
//
// Many tests in this package run against the in-memory and sqlite
// backends by default. If PostgresConnStr is set, they'll also run
// against postgres.

import (
"archive/zip"
@@ -24,7 +20,7 @@ const (
PostgresConnStr = "" // "postgres://postgres:mysecretpassword@localhost:5432/gtfs?sslmode=disable"
)

func GFSTest_BuildStorage(t testing.TB, backend string) storage.Storage {
func BuildStorage(t testing.TB, backend string) storage.Storage {
var s storage.Storage
var err error
if backend == "sqlite" {
@@ -39,14 +35,14 @@ func GFSTest_BuildStorage(t testing.TB, backend string) storage.Storage {
return s
}

func GTFSTest_LoadStatic(t testing.TB, backend string, buf *bytes.Buffer) *gtfs.Static {
s := GFSTest_BuildStorage(t, backend)
func LoadStatic(t testing.TB, backend string, buf []byte) *gtfs.Static {
s := BuildStorage(t, backend)

// Parse buf into storage
feedWriter, err := s.GetWriter("test")
require.NoError(t, err)

metadata, err := parse.ParseStatic(feedWriter, buf.Bytes())
metadata, err := parse.ParseStatic(feedWriter, buf)
require.NoError(t, err)

require.NoError(t, feedWriter.Close())
@@ -61,14 +57,14 @@ func GTFSTest_LoadStatic(t testing.TB, backend string, buf *bytes.Buffer) *gtfs.
return static
}

func GTFSTest_LoadStaticFile(t testing.TB, backend string, filename string) *gtfs.Static {
func LoadStaticFile(t testing.TB, backend string, filename string) *gtfs.Static {
buf, err := ioutil.ReadFile(filename)
require.NoError(t, err)

return GTFSTest_LoadStatic(t, backend, bytes.NewBuffer(buf))
return LoadStatic(t, backend, buf)
}

func GTFSTest_BuildStatic(
func BuildStatic(
t testing.TB,
backend string,
files map[string][]string,
@@ -94,7 +90,16 @@ func GTFSTest_BuildStatic(
files["stop_times.txt"] = []string{"stop_id"}
}

// Create zip
buf := BuildZip(t, files)

return LoadStatic(t, backend, buf)
}

func BuildZip(
t testing.TB,
files map[string][]string,
) []byte {

buf := &bytes.Buffer{}
w := zip.NewWriter(buf)
for filename, content := range files {
@@ -105,5 +110,5 @@ func GTFSTest_BuildStatic(
}
require.NoError(t, w.Close())

return GTFSTest_LoadStatic(t, backend, buf)
return buf.Bytes()
}

0 comments on commit 04e9e8c

Please sign in to comment.