From 157131646b888d2cf74a1c5b14c1dd20fd6adc85 Mon Sep 17 00:00:00 2001 From: Max Blazejewski Date: Mon, 29 Jul 2024 19:07:04 +0200 Subject: [PATCH] Add a route with a list of all cached requests (#22) --- Dockerfile | 2 +- README.md | 5 +- cache/cache.go | 5 ++ docs/openapi.json | 135 ++++++++++++++++++++++++++++++++ go.mod | 2 + go.sum | 4 + handlers/GetAdventurerSearch.go | 7 +- handlers/GetCache.go | 57 ++++++++++++++ handlers/GetStatus.go | 2 +- httpServer/BuildServer.go | 1 + 10 files changed, 213 insertions(+), 7 deletions(-) create mode 100644 handlers/GetCache.go diff --git a/Dockerfile b/Dockerfile index 6f6a637..3c58c97 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.21-alpine AS build +FROM golang:1.22-alpine AS build RUN apk add --no-cache git WORKDIR /src/bdo-rest-api COPY . . diff --git a/README.md b/README.md index b95835c..1b1f24d 100644 --- a/README.md +++ b/README.md @@ -3,8 +3,9 @@ A scraper for Black Desert Online player in-game data with a REST API. It currently supports European, North American, and South American servers (Korean server support is in progress). ## Projects using this API -- [BDO Leaderboards](https://bdo.hemlo.cc/leaderboards) ([Source](https://github.com/man90es/BDO-Leaderboards)): web-based leaderboards for Black Desert Online. -- [Ikusa](https://ikusa.site) ([Source](https://github.com/sch-28/ikusa_api)): a powerful tool that allows you to analyze your game logs and gain valuable insights into your combat performance. +- BDO Leaderboards ([Website](https://bdo.hemlo.cc/leaderboards), [sources](https://github.com/man90es/BDO-Leaderboards)): web-based leaderboards for Black Desert Online. +- Ikusa ([Website](https://ikusa.site), [sources](https://github.com/sch-28/ikusa_api)): powerful tool that allows you to analyze your game logs and gain valuable insights into your combat performance. +- GuildYapper ([Discord server](https://discord.gg/x2nKYuu2Z2)): Discord bot with various features for BDO guilds such as guild and player history logging, and automatic trial Discord management (more features TBA). ## How to start using it There are two ways to use this scraper for your needs: diff --git a/cache/cache.go b/cache/cache.go index 3a32af9..82ffd25 100644 --- a/cache/cache.go +++ b/cache/cache.go @@ -5,6 +5,7 @@ import ( "time" goCache "github.com/patrickmn/go-cache" + "golang.org/x/exp/maps" "bdo-rest-api/config" "bdo-rest-api/utils" @@ -64,3 +65,7 @@ func (c *cache[T]) GetRecord(keys []string) (data T, status int, date string, ex func (c *cache[T]) GetItemCount() int { return c.internalCache.ItemCount() } + +func (c *cache[T]) GetKeys() []string { + return maps.Keys(c.internalCache.Items()) +} diff --git a/docs/openapi.json b/docs/openapi.json index 1a302ce..cb3ade5 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -77,6 +77,7 @@ "type": "number" }, "/guild/search": { + "deprecated": true, "type": "number" } } @@ -96,6 +97,10 @@ } } }, + "docs": { + "type": "string", + "example": "https://man90es.github.io/BDO-REST-API" + }, "proxies": { "type": "number" }, @@ -404,6 +409,7 @@ "name": "page", "in": "query", "description": "This parameter is understood by the API, but you should either omit it or set to 1. Because of how search currently works, there is never more than one page.", + "deprecated": true, "schema": { "type": "number", "default": 1 @@ -600,6 +606,7 @@ "summary": "Search for a guild.", "description": "Search for a guild by combination of its region and name.", "operationId": "getGuildSearch", + "deprecated": true, "parameters": [ { "name": "query", @@ -691,6 +698,134 @@ } } } + }, + "/v1/cache": { + "get": { + "summary": "Retrieve cached routes", + "operationId": "getCache", + "responses": { + "200": { + "description": "OK.", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "/adventurer": { + "type": "array", + "items": { + "type": "object", + "properties": { + "profileTarget": { + "nullable": false, + "type": "string", + "example": "XXX" + }, + "region": { + "nullable": false, + "type": "string", + "enum": [ + "EU", + "NA", + "SA" + ] + } + } + } + }, + "/adventurer/search": { + "type": "array", + "items": { + "type": "object", + "properties": { + "page": { + "deprecated": true, + "nullable": false, + "type": "number", + "example": 1 + }, + "query": { + "nullable": false, + "type": "string", + "example": "Apple" + }, + "region": { + "nullable": false, + "type": "string", + "enum": [ + "EU", + "NA", + "SA" + ] + }, + "searchType": { + "nullable": false, + "type": "string", + "enum": [ + "familyName", + "characterName" + ] + } + } + } + }, + "/guild": { + "type": "array", + "items": { + "type": "object", + "properties": { + "guildName": { + "nullable": false, + "type": "string", + "example": "TumblrGirls" + }, + "region": { + "nullable": false, + "type": "string", + "enum": [ + "EU", + "NA", + "SA" + ] + } + } + } + }, + "/guild/search": { + "deprecated": true, + "type": "array", + "items": { + "type": "object", + "properties": { + "page": { + "nullable": false, + "type": "number", + "example": 1 + }, + "query": { + "nullable": false, + "type": "string", + "example": "TumblrGirls" + }, + "region": { + "nullable": false, + "type": "string", + "enum": [ + "EU", + "NA", + "SA" + ] + } + } + } + } + } + } + } + } + } + } + } } } } diff --git a/go.mod b/go.mod index 2c6500f..6bae13a 100644 --- a/go.mod +++ b/go.mod @@ -19,8 +19,10 @@ require ( github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/kennygrant/sanitize v1.2.4 // indirect + github.com/sa-/slicefunk v0.1.4 // indirect github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect github.com/temoto/robotstxt v1.1.2 // indirect + golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect golang.org/x/net v0.19.0 // indirect golang.org/x/text v0.14.0 // indirect google.golang.org/appengine v1.6.8 // indirect diff --git a/go.sum b/go.sum index 74d49b3..c13014d 100644 --- a/go.sum +++ b/go.sum @@ -67,6 +67,8 @@ github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTK github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/sa-/slicefunk v0.1.4 h1:fCgDllo0nYVywdREyJm53BQ5rfMW8pin57yNVpyPxNU= +github.com/sa-/slicefunk v0.1.4/go.mod h1:k0abNpV9EW8LIPl2+Hc9RiKsojKmsUhNNGFyMpjMTCI= github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA= github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= @@ -82,6 +84,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= diff --git a/handlers/GetAdventurerSearch.go b/handlers/GetAdventurerSearch.go index bf73991..61f8f9d 100644 --- a/handlers/GetAdventurerSearch.go +++ b/handlers/GetAdventurerSearch.go @@ -18,7 +18,8 @@ func GetAdventurerSearch(w http.ResponseWriter, r *http.Request) { page := validators.ValidatePageQueryParam(r.URL.Query()["page"]) query, queryOk := validators.ValidateAdventurerNameQueryParam(r.URL.Query()["query"]) region, regionOk := validators.ValidateRegionQueryParam(r.URL.Query()["region"]) - searchType := validators.ValidateSearchTypeQueryParam(r.URL.Query()["searchType"]) + searchTypeQueryParam := r.URL.Query()["searchType"] + searchType := validators.ValidateSearchTypeQueryParam(searchTypeQueryParam) if !queryOk || !regionOk { giveBadRequestResponse(w) @@ -33,7 +34,7 @@ func GetAdventurerSearch(w http.ResponseWriter, r *http.Request) { query = strings.ToLower(query) // Look for cached data, then run the scraper if needed - data, status, date, expires, found := profileSearchCache.GetRecord([]string{region, query, fmt.Sprint(searchType), fmt.Sprint(page)}) + data, status, date, expires, found := profileSearchCache.GetRecord([]string{region, query, searchTypeQueryParam[0], fmt.Sprint(page)}) if !found { data, status = scrapers.ScrapeAdventurerSearch(region, query, searchType, page) @@ -46,7 +47,7 @@ func GetAdventurerSearch(w http.ResponseWriter, r *http.Request) { return } - date, expires = profileSearchCache.AddRecord([]string{region, query, fmt.Sprint(searchType), fmt.Sprint(page)}, data, status) + date, expires = profileSearchCache.AddRecord([]string{region, query, searchTypeQueryParam[0], fmt.Sprint(page)}, data, status) } w.Header().Set("Date", date) diff --git a/handlers/GetCache.go b/handlers/GetCache.go new file mode 100644 index 0000000..ca29ae7 --- /dev/null +++ b/handlers/GetCache.go @@ -0,0 +1,57 @@ +package handlers + +import ( + "encoding/json" + "net/http" + "strconv" + "strings" + + sf "github.com/sa-/slicefunk" +) + +func getParseCacheKey(cacheType string) func(string) map[string]interface{} { + return func(key string) map[string]interface{} { + parts := strings.Split(key, ",") + + switch cacheType { + case "/adventurer": + return map[string]interface{}{ + "region": parts[0], + "profileTarget": parts[1], + } + case "/adventurer/search": + page, _ := strconv.Atoi(parts[3]) + + return map[string]interface{}{ + "region": parts[0], + "query": parts[1], + "searhType": parts[2], + "page": page, + } + case "/guild": + return map[string]interface{}{ + "region": parts[0], + "guildName": parts[1], + } + case "/guild/search": + page, _ := strconv.Atoi(parts[2]) + + return map[string]interface{}{ + "region": parts[0], + "query": parts[1], + "page": page, + } + default: + return nil + } + } +} + +func GetCache(w http.ResponseWriter, r *http.Request) { + json.NewEncoder(w).Encode(map[string]interface{}{ + "/adventurer": sf.Map(profilesCache.GetKeys(), getParseCacheKey("/adventurer")), + "/adventurer/search": sf.Map(profileSearchCache.GetKeys(), getParseCacheKey("/adventurer/search")), + "/guild": sf.Map(guildProfilesCache.GetKeys(), getParseCacheKey("/guild")), + "/guild/search": sf.Map(guildSearchCache.GetKeys(), getParseCacheKey("/guild/search")), + }) +} diff --git a/handlers/GetStatus.go b/handlers/GetStatus.go index aff0962..9b53f70 100644 --- a/handlers/GetStatus.go +++ b/handlers/GetStatus.go @@ -10,7 +10,7 @@ import ( ) var initTime = time.Now() -var version = "1.8.4" +var version = "1.9.0" func GetStatus(w http.ResponseWriter, r *http.Request) { json.NewEncoder(w).Encode(map[string]interface{}{ diff --git a/httpServer/BuildServer.go b/httpServer/BuildServer.go index 3e4515a..a44df5c 100644 --- a/httpServer/BuildServer.go +++ b/httpServer/BuildServer.go @@ -16,6 +16,7 @@ func BuildServer() *http.Server { "/v1": handlers.GetStatus, "/v1/adventurer": handlers.GetAdventurer, "/v1/adventurer/search": handlers.GetAdventurerSearch, + "/v1/cache": handlers.GetCache, "/v1/guild": handlers.GetGuild, "/v1/guild/search": handlers.GetGuildSearch, }, handlers.Catchall)