From ccee771edf5ea3cf821193a796fbd9303c0d3f83 Mon Sep 17 00:00:00 2001 From: bruce Date: Wed, 15 Jan 2025 13:25:17 +0800 Subject: [PATCH] feat: Query readings without pagination offset Query readings without pagination offset to prevent large dataset query and count. Signed-off-by: bruce (cherry picked from commit 1a89671151ac64faeeb06f70b556460da8f76616) --- internal/core/data/application/reading.go | 138 +++++++++++------- internal/core/data/controller/http/reading.go | 18 ++- 2 files changed, 94 insertions(+), 62 deletions(-) diff --git a/internal/core/data/application/reading.go b/internal/core/data/application/reading.go index cfb46dcbfc..7ea6428823 100644 --- a/internal/core/data/application/reading.go +++ b/internal/core/data/application/reading.go @@ -38,6 +38,16 @@ func ReadingTotalCount(dic *di.Container) (uint32, errors.EdgeX) { // AllReadings query events by offset, and limit func AllReadings(offset int, limit int, dic *di.Container) (readings []dtos.BaseReading, totalCount uint32, err errors.EdgeX) { dbClient := container.DBClientFrom(dic.Get) + + readingModels, err := dbClient.AllReadings(offset, limit) + if err != nil { + return readings, totalCount, errors.NewCommonEdgeXWrapper(err) + } + readings, err = convertReadingModelsToDTOs(readingModels) + + if offset < 0 { + return readings, 0, err // skip total count + } totalCount, err = dbClient.ReadingTotalCount() if err != nil { return readings, totalCount, errors.NewCommonEdgeXWrapper(err) @@ -46,12 +56,6 @@ func AllReadings(offset int, limit int, dic *di.Container) (readings []dtos.Base if !cont { return []dtos.BaseReading{}, totalCount, err } - - readingModels, err := dbClient.AllReadings(offset, limit) - if err != nil { - return readings, totalCount, errors.NewCommonEdgeXWrapper(err) - } - readings, err = convertReadingModelsToDTOs(readingModels) return readings, totalCount, err } @@ -62,6 +66,15 @@ func ReadingsByResourceName(offset int, limit int, resourceName string, dic *di. } dbClient := container.DBClientFrom(dic.Get) + readingModels, err := dbClient.ReadingsByResourceName(offset, limit, resourceName) + if err != nil { + return readings, totalCount, errors.NewCommonEdgeXWrapper(err) + } + readings, err = convertReadingModelsToDTOs(readingModels) + + if offset < 0 { + return readings, 0, err // skip total count + } totalCount, err = dbClient.ReadingCountByResourceName(resourceName) if err != nil { return readings, totalCount, errors.NewCommonEdgeXWrapper(err) @@ -70,12 +83,6 @@ func ReadingsByResourceName(offset int, limit int, resourceName string, dic *di. if !cont { return []dtos.BaseReading{}, totalCount, err } - - readingModels, err := dbClient.ReadingsByResourceName(offset, limit, resourceName) - if err != nil { - return readings, totalCount, errors.NewCommonEdgeXWrapper(err) - } - readings, err = convertReadingModelsToDTOs(readingModels) return readings, totalCount, err } @@ -86,6 +93,15 @@ func ReadingsByDeviceName(offset int, limit int, name string, dic *di.Container) } dbClient := container.DBClientFrom(dic.Get) + readingModels, err := dbClient.ReadingsByDeviceName(offset, limit, name) + if err != nil { + return readings, totalCount, errors.NewCommonEdgeXWrapper(err) + } + readings, err = convertReadingModelsToDTOs(readingModels) + + if offset < 0 { + return readings, 0, err // skip total count + } totalCount, err = dbClient.ReadingCountByDeviceName(name) if err != nil { return readings, totalCount, errors.NewCommonEdgeXWrapper(err) @@ -94,18 +110,22 @@ func ReadingsByDeviceName(offset int, limit int, name string, dic *di.Container) if !cont { return []dtos.BaseReading{}, totalCount, err } - - readingModels, err := dbClient.ReadingsByDeviceName(offset, limit, name) - if err != nil { - return readings, totalCount, errors.NewCommonEdgeXWrapper(err) - } - readings, err = convertReadingModelsToDTOs(readingModels) return readings, totalCount, err } // ReadingsByTimeRange query readings with offset, limit and time range func ReadingsByTimeRange(start int64, end int64, offset int, limit int, dic *di.Container) (readings []dtos.BaseReading, totalCount uint32, err errors.EdgeX) { dbClient := container.DBClientFrom(dic.Get) + + readingModels, err := dbClient.ReadingsByTimeRange(start, end, offset, limit) + if err != nil { + return readings, totalCount, errors.NewCommonEdgeXWrapper(err) + } + readings, err = convertReadingModelsToDTOs(readingModels) + + if offset < 0 { + return readings, 0, err // skip total count + } totalCount, err = dbClient.ReadingCountByTimeRange(start, end) if err != nil { return readings, totalCount, errors.NewCommonEdgeXWrapper(err) @@ -114,12 +134,6 @@ func ReadingsByTimeRange(start int64, end int64, offset int, limit int, dic *di. if !cont { return []dtos.BaseReading{}, totalCount, err } - - readingModels, err := dbClient.ReadingsByTimeRange(start, end, offset, limit) - if err != nil { - return readings, totalCount, errors.NewCommonEdgeXWrapper(err) - } - readings, err = convertReadingModelsToDTOs(readingModels) return readings, totalCount, err } @@ -151,6 +165,16 @@ func ReadingsByResourceNameAndTimeRange(resourceName string, start int64, end in return readings, totalCount, errors.NewCommonEdgeX(errors.KindContractInvalid, "resourceName is empty", nil) } dbClient := container.DBClientFrom(dic.Get) + + readingModels, err := dbClient.ReadingsByResourceNameAndTimeRange(resourceName, start, end, offset, limit) + if err != nil { + return readings, totalCount, errors.NewCommonEdgeXWrapper(err) + } + readings, err = convertReadingModelsToDTOs(readingModels) + + if offset < 0 { + return readings, 0, err // skip total count + } totalCount, err = dbClient.ReadingCountByResourceNameAndTimeRange(resourceName, start, end) if err != nil { return readings, totalCount, errors.NewCommonEdgeXWrapper(err) @@ -159,12 +183,6 @@ func ReadingsByResourceNameAndTimeRange(resourceName string, start int64, end in if !cont { return []dtos.BaseReading{}, totalCount, err } - - readingModels, err := dbClient.ReadingsByResourceNameAndTimeRange(resourceName, start, end, offset, limit) - if err != nil { - return readings, totalCount, errors.NewCommonEdgeXWrapper(err) - } - readings, err = convertReadingModelsToDTOs(readingModels) return readings, totalCount, err } @@ -178,6 +196,16 @@ func ReadingsByDeviceNameAndResourceName(deviceName string, resourceName string, } dbClient := container.DBClientFrom(dic.Get) + + readingModels, err := dbClient.ReadingsByDeviceNameAndResourceName(deviceName, resourceName, offset, limit) + if err != nil { + return readings, totalCount, errors.NewCommonEdgeXWrapper(err) + } + readings, err = convertReadingModelsToDTOs(readingModels) + + if offset < 0 { + return readings, 0, err // skip total count + } totalCount, err = dbClient.ReadingCountByDeviceNameAndResourceName(deviceName, resourceName) if err != nil { return readings, totalCount, errors.NewCommonEdgeXWrapper(err) @@ -186,12 +214,6 @@ func ReadingsByDeviceNameAndResourceName(deviceName string, resourceName string, if !cont { return []dtos.BaseReading{}, totalCount, err } - - readingModels, err := dbClient.ReadingsByDeviceNameAndResourceName(deviceName, resourceName, offset, limit) - if err != nil { - return readings, totalCount, errors.NewCommonEdgeXWrapper(err) - } - readings, err = convertReadingModelsToDTOs(readingModels) return readings, totalCount, err } @@ -205,6 +227,16 @@ func ReadingsByDeviceNameAndResourceNameAndTimeRange(deviceName string, resource } dbClient := container.DBClientFrom(dic.Get) + + readingModels, err := dbClient.ReadingsByDeviceNameAndResourceNameAndTimeRange(deviceName, resourceName, start, end, offset, limit) + if err != nil { + return readings, totalCount, errors.NewCommonEdgeXWrapper(err) + } + readings, err = convertReadingModelsToDTOs(readingModels) + + if offset < 0 { + return readings, 0, err // skip total count + } totalCount, err = dbClient.ReadingCountByDeviceNameAndResourceNameAndTimeRange(deviceName, resourceName, start, end) if err != nil { return readings, totalCount, errors.NewCommonEdgeXWrapper(err) @@ -213,12 +245,6 @@ func ReadingsByDeviceNameAndResourceNameAndTimeRange(deviceName string, resource if !cont { return []dtos.BaseReading{}, totalCount, err } - - readingModels, err := dbClient.ReadingsByDeviceNameAndResourceNameAndTimeRange(deviceName, resourceName, start, end, offset, limit) - if err != nil { - return readings, totalCount, errors.NewCommonEdgeXWrapper(err) - } - readings, err = convertReadingModelsToDTOs(readingModels) return readings, totalCount, err } @@ -231,21 +257,25 @@ func ReadingsByDeviceNameAndResourceNamesAndTimeRange(deviceName string, resourc dbClient := container.DBClientFrom(dic.Get) var readingModels []models.Reading if len(resourceNames) > 0 { - totalCount, err = dbClient.ReadingCountByDeviceNameAndResourceNamesAndTimeRange(deviceName, resourceNames, start, end) - if err != nil { - return readings, totalCount, errors.NewCommonEdgeXWrapper(err) - } - if cont, err := utils.CheckCountRange(totalCount, offset, limit); !cont { - return []dtos.BaseReading{}, totalCount, err + if offset >= 0 { + totalCount, err = dbClient.ReadingCountByDeviceNameAndResourceNamesAndTimeRange(deviceName, resourceNames, start, end) + if err != nil { + return readings, totalCount, errors.NewCommonEdgeXWrapper(err) + } + if cont, err := utils.CheckCountRange(totalCount, offset, limit); !cont { + return []dtos.BaseReading{}, totalCount, err + } } readingModels, err = dbClient.ReadingsByDeviceNameAndResourceNamesAndTimeRange(deviceName, resourceNames, start, end, offset, limit) } else { - totalCount, err = dbClient.ReadingCountByDeviceNameAndTimeRange(deviceName, start, end) - if err != nil { - return readings, totalCount, errors.NewCommonEdgeXWrapper(err) - } - if cont, err := utils.CheckCountRange(totalCount, offset, limit); !cont { - return []dtos.BaseReading{}, totalCount, err + if offset >= 0 { + totalCount, err = dbClient.ReadingCountByDeviceNameAndTimeRange(deviceName, start, end) + if err != nil { + return readings, totalCount, errors.NewCommonEdgeXWrapper(err) + } + if cont, err := utils.CheckCountRange(totalCount, offset, limit); !cont { + return []dtos.BaseReading{}, totalCount, err + } } readingModels, err = dbClient.ReadingsByDeviceNameAndTimeRange(deviceName, start, end, offset, limit) } diff --git a/internal/core/data/controller/http/reading.go b/internal/core/data/controller/http/reading.go index f2b5d43a17..449f1741f7 100644 --- a/internal/core/data/controller/http/reading.go +++ b/internal/core/data/controller/http/reading.go @@ -25,6 +25,8 @@ import ( "github.com/labstack/echo/v4" ) +const minOffset = -1 // allow using -1 to query reading data and skip the total count for pagination + type ReadingController struct { reader io.DtoReader dic *di.Container @@ -65,7 +67,7 @@ func (rc *ReadingController) AllReadings(c echo.Context) error { config := dataContainer.ConfigurationFrom(rc.dic.Get) // parse URL query string for offset, and limit, and labels - offset, limit, _, err := utils.ParseGetAllObjectsRequestQueryString(c, 0, math.MaxInt32, -1, config.Service.MaxResultCount) + offset, limit, _, err := utils.ParseGetAllObjectsRequestQueryString(c, minOffset, math.MaxInt32, -1, config.Service.MaxResultCount) if err != nil { return utils.WriteErrorResponse(w, ctx, lc, err, "") } @@ -87,7 +89,7 @@ func (rc *ReadingController) ReadingsByTimeRange(c echo.Context) error { config := dataContainer.ConfigurationFrom(rc.dic.Get) // parse time range (start, end), offset, and limit from incoming request - start, end, offset, limit, err := utils.ParseTimeRangeOffsetLimit(c, 0, math.MaxInt32, -1, config.Service.MaxResultCount) + start, end, offset, limit, err := utils.ParseTimeRangeOffsetLimit(c, minOffset, math.MaxInt32, -1, config.Service.MaxResultCount) if err != nil { return utils.WriteErrorResponse(w, ctx, lc, err, "") } @@ -111,7 +113,7 @@ func (rc *ReadingController) ReadingsByResourceName(c echo.Context) error { resourceName := c.Param(common.ResourceName) // parse URL query string for offset, limit - offset, limit, _, err := utils.ParseGetAllObjectsRequestQueryString(c, 0, math.MaxInt32, -1, config.Service.MaxResultCount) + offset, limit, _, err := utils.ParseGetAllObjectsRequestQueryString(c, minOffset, math.MaxInt32, -1, config.Service.MaxResultCount) if err != nil { return utils.WriteErrorResponse(w, ctx, lc, err, "") } @@ -135,7 +137,7 @@ func (rc *ReadingController) ReadingsByDeviceName(c echo.Context) error { name := c.Param(common.Name) // parse URL query string for offset, limit - offset, limit, _, err := utils.ParseGetAllObjectsRequestQueryString(c, 0, math.MaxInt32, -1, config.Service.MaxResultCount) + offset, limit, _, err := utils.ParseGetAllObjectsRequestQueryString(c, minOffset, math.MaxInt32, -1, config.Service.MaxResultCount) if err != nil { return utils.WriteErrorResponse(w, ctx, lc, err, "") } @@ -181,7 +183,7 @@ func (rc *ReadingController) ReadingsByResourceNameAndTimeRange(c echo.Context) resourceName := c.Param(common.ResourceName) // parse time range (start, end), offset, and limit from incoming request - start, end, offset, limit, err := utils.ParseTimeRangeOffsetLimit(c, 0, math.MaxInt32, -1, config.Service.MaxResultCount) + start, end, offset, limit, err := utils.ParseTimeRangeOffsetLimit(c, minOffset, math.MaxInt32, -1, config.Service.MaxResultCount) if err != nil { return utils.WriteErrorResponse(w, ctx, lc, err, "") } @@ -206,7 +208,7 @@ func (rc *ReadingController) ReadingsByDeviceNameAndResourceName(c echo.Context) resourceName := c.Param(common.ResourceName) // parse URL query string for offset, limit - offset, limit, _, err := utils.ParseGetAllObjectsRequestQueryString(c, 0, math.MaxInt32, -1, config.Service.MaxResultCount) + offset, limit, _, err := utils.ParseGetAllObjectsRequestQueryString(c, minOffset, math.MaxInt32, -1, config.Service.MaxResultCount) if err != nil { return utils.WriteErrorResponse(w, ctx, lc, err, "") } @@ -231,7 +233,7 @@ func (rc *ReadingController) ReadingsByDeviceNameAndResourceNameAndTimeRange(c e resourceName := c.Param(common.ResourceName) // parse time range (start, end), offset, and limit from incoming request - start, end, offset, limit, err := utils.ParseTimeRangeOffsetLimit(c, 0, math.MaxInt32, -1, config.Service.MaxResultCount) + start, end, offset, limit, err := utils.ParseTimeRangeOffsetLimit(c, minOffset, math.MaxInt32, -1, config.Service.MaxResultCount) if err != nil { return utils.WriteErrorResponse(w, ctx, lc, err, "") } @@ -256,7 +258,7 @@ func (rc *ReadingController) ReadingsByDeviceNameAndResourceNamesAndTimeRange(c deviceName := c.Param(common.Name) // parse time range (start, end), offset, and limit from incoming request - start, end, offset, limit, err := utils.ParseTimeRangeOffsetLimit(c, 0, math.MaxInt32, -1, config.Service.MaxResultCount) + start, end, offset, limit, err := utils.ParseTimeRangeOffsetLimit(c, minOffset, math.MaxInt32, -1, config.Service.MaxResultCount) if err != nil { return utils.WriteErrorResponse(w, ctx, lc, err, "") }