Merge pull request #353 from systemli/Extract-Websites-from-Ticker

✨ Extract Websites from Ticker

internal/api/api.go

@@ -67,6 +67,8 @@ func API(config config.Config, store storage.Storage, log *logrus.Logger) *gin.E
 		admin.GET(`/tickers/:tickerID`, ticker.PrefetchTicker(store, storage.WithPreload()), handler.GetTicker)
 		admin.POST(`/tickers`, user.NeedAdmin(), handler.PostTicker)
 		admin.PUT(`/tickers/:tickerID`, ticker.PrefetchTicker(store, storage.WithPreload()), handler.PutTicker)
+		admin.DELETE(`/tickers/:tickerID/websites`, ticker.PrefetchTicker(store, storage.WithPreload()), handler.DeleteTickerWebsite)
+		admin.POST(`/tickers/:tickerID/websites`, ticker.PrefetchTicker(store, storage.WithPreload()), handler.PostTickerWebsite)
 		admin.PUT(`/tickers/:tickerID/telegram`, ticker.PrefetchTicker(store, storage.WithPreload()), handler.PutTickerTelegram)
 		admin.DELETE(`/tickers/:tickerID/telegram`, ticker.PrefetchTicker(store, storage.WithPreload()), handler.DeleteTickerTelegram)
 		admin.PUT(`/tickers/:tickerID/mastodon`, ticker.PrefetchTicker(store, storage.WithPreload()), handler.PutTickerMastodon)

internal/api/helper/util.go

@@ -2,14 +2,13 @@ package helper
 
 import (
 	"errors"
-	"net/url"
-	"strings"
-
+	"fmt"
 	"github.com/gin-gonic/gin"
 	"github.com/systemli/ticker/internal/storage"
+	"net/url"
 )
 
-func GetDomain(c *gin.Context) (string, error) {
+func GetOrigin(c *gin.Context) (string, error) {
 	origin := c.Request.URL.Query().Get("origin")
 	if origin != "" {
 		return origin, nil
@@ -25,13 +24,11 @@ func GetDomain(c *gin.Context) (string, error) {
 		return "", err
 	}
 
-	domain := strings.TrimPrefix(u.Host, "www.")
-	if strings.Contains(domain, ":") {
-		parts := strings.Split(domain, ":")
-		domain = parts[0]
+	if u.Scheme == "" || u.Host == "" {
+		return "", errors.New("invalid origin")
 	}
 
-	return domain, nil
+	return fmt.Sprintf("%s://%s", u.Scheme, u.Host), nil
 }
 
 func Me(c *gin.Context) (storage.User, error) {

internal/api/helper/util_test.go

@@ -14,47 +14,56 @@ type UtilTestSuite struct {
 	suite.Suite
 }
 
-func (s *UtilTestSuite) TestGetDomain() {
+func (s *UtilTestSuite) TestGetOrigin() {
 	s.Run("when origin is empty", func() {
 		c := s.buildContext(url.URL{}, http.Header{})
-		domain, err := GetDomain(c)
-		s.Equal("", domain)
+		origin, err := GetOrigin(c)
+		s.Equal("", origin)
 		s.Equal("origin header not found", err.Error())
 	})
 
+	s.Run("when origin is not a valid URL", func() {
+		c := s.buildContext(url.URL{}, http.Header{
+			"Origin": []string{"localhost"},
+		})
+		origin, err := GetOrigin(c)
+		s.Equal("", origin)
+		s.Error(err)
+	})
+
 	s.Run("when origin is localhost", func() {
 		c := s.buildContext(url.URL{}, http.Header{
-			"Origin": []string{"http://localhost/"},
+			"Origin": []string{"http://localhost"},
 		})
-		domain, err := GetDomain(c)
-		s.Equal("localhost", domain)
+		origin, err := GetOrigin(c)
+		s.Equal("http://localhost", origin)
 		s.NoError(err)
 	})
 
 	s.Run("when origin is localhost with port", func() {
 		c := s.buildContext(url.URL{}, http.Header{
-			"Origin": []string{"http://localhost:3000/"},
+			"Origin": []string{"http://localhost:3000"},
 		})
-		domain, err := GetDomain(c)
-		s.Equal("localhost", domain)
+		origin, err := GetOrigin(c)
+		s.Equal("http://localhost:3000", origin)
 		s.NoError(err)
 	})
 
 	s.Run("when origin has subdomain", func() {
 		c := s.buildContext(url.URL{}, http.Header{
 			"Origin": []string{"http://www.demoticker.org/"},
 		})
-		domain, err := GetDomain(c)
-		s.Equal("demoticker.org", domain)
+		origin, err := GetOrigin(c)
+		s.Equal("http://www.demoticker.org", origin)
 		s.NoError(err)
 	})
 
 	s.Run("when query param is set", func() {
-		c := s.buildContext(url.URL{RawQuery: "origin=another.demoticker.org"}, http.Header{
+		c := s.buildContext(url.URL{RawQuery: "origin=http://another.demoticker.org"}, http.Header{
 			"Origin": []string{"http://www.demoticker.org/"},
 		})
-		domain, err := GetDomain(c)
-		s.Equal("another.demoticker.org", domain)
+		domain, err := GetOrigin(c)
+		s.Equal("http://another.demoticker.org", domain)
 		s.NoError(err)
 	})
 }

internal/api/init.go

@@ -12,13 +12,13 @@ func (h *handler) GetInit(c *gin.Context) {
 	settings := response.Settings{
 		RefreshInterval: h.storage.GetRefreshIntervalSettings().RefreshInterval,
 	}
-	domain, err := helper.GetDomain(c)
+	origin, err := helper.GetOrigin(c)
 	if err != nil {
 		c.JSON(http.StatusOK, response.SuccessResponse(map[string]interface{}{"ticker": nil, "settings": settings}))
 		return
 	}
 
-	ticker, err := h.storage.FindTickerByDomain(domain)
+	ticker, err := h.storage.FindTickerByOrigin(origin)
 	if err != nil || !ticker.Active {
 		settings.InactiveSettings = h.storage.GetInactiveSettings()
 		c.JSON(http.StatusOK, response.SuccessResponse(map[string]interface{}{"ticker": nil, "settings": settings}))

internal/api/init_test.go

@@ -39,48 +39,48 @@ func (s *InitTestSuite) TestGetInit() {
 
 		s.Equal(http.StatusOK, s.w.Code)
 		s.Equal(`{"data":{"settings":{"refreshInterval":10000},"ticker":null},"status":"success","error":{}}`, s.w.Body.String())
-		s.store.AssertNotCalled(s.T(), "FindTickerByDomain", mock.AnythingOfType("string"), mock.Anything)
+		s.store.AssertNotCalled(s.T(), "FindTickerByOrigin", "", mock.Anything)
 		s.store.AssertExpectations(s.T())
 	})
 
 	s.Run("when database returns error", func() {
-		s.ctx.Request = httptest.NewRequest(http.MethodGet, "/v1/init?origin=demoticker.org", nil)
-		s.store.On("FindTickerByDomain", mock.AnythingOfType("string"), mock.Anything).Return(storage.Ticker{}, errors.New("storage error")).Once()
+		s.ctx.Request = httptest.NewRequest(http.MethodGet, "/v1/init?origin=https://demoticker.org", nil)
+		s.store.On("FindTickerByOrigin", "https://demoticker.org", mock.Anything).Return(storage.Ticker{}, errors.New("storage error")).Once()
 		s.store.On("GetInactiveSettings").Return(storage.DefaultInactiveSettings()).Once()
 		h := s.handler()
 		h.GetInit(s.ctx)
 
 		s.Equal(http.StatusOK, s.w.Code)
 		s.Contains(s.w.Body.String(), `"ticker":null`)
-		s.store.AssertCalled(s.T(), "FindTickerByDomain", "demoticker.org", mock.Anything)
+		s.store.AssertCalled(s.T(), "FindTickerByOrigin", "https://demoticker.org", mock.Anything)
 		s.store.AssertExpectations(s.T())
 	})
 
 	s.Run("when database returns an inactive ticker", func() {
-		s.ctx.Request = httptest.NewRequest(http.MethodGet, "/v1/init?origin=demoticker.org", nil)
+		s.ctx.Request = httptest.NewRequest(http.MethodGet, "/v1/init?origin=https://demoticker.org", nil)
 		ticker := storage.NewTicker()
 		ticker.Active = false
-		s.store.On("FindTickerByDomain", mock.AnythingOfType("string"), mock.Anything).Return(ticker, nil).Once()
+		s.store.On("FindTickerByOrigin", "https://demoticker.org", mock.Anything).Return(ticker, nil).Once()
 		s.store.On("GetInactiveSettings").Return(storage.DefaultInactiveSettings()).Once()
 		h := s.handler()
 		h.GetInit(s.ctx)
 
 		s.Equal(http.StatusOK, s.w.Code)
 		s.Contains(s.w.Body.String(), `"ticker":null`)
-		s.store.AssertCalled(s.T(), "FindTickerByDomain", "demoticker.org", mock.Anything)
+		s.store.AssertCalled(s.T(), "FindTickerByOrigin", "https://demoticker.org", mock.Anything)
 	})
 
 	s.Run("when database returns an active ticker", func() {
-		s.ctx.Request = httptest.NewRequest(http.MethodGet, "/v1/init?origin=demoticker.org", nil)
+		s.ctx.Request = httptest.NewRequest(http.MethodGet, "/v1/init?origin=https://demoticker.org", nil)
 		ticker := storage.NewTicker()
 		ticker.Active = true
-		s.store.On("FindTickerByDomain", mock.AnythingOfType("string"), mock.Anything).Return(ticker, nil).Once()
+		s.store.On("FindTickerByOrigin", "https://demoticker.org", mock.Anything).Return(ticker, nil).Once()
 		s.store.On("GetInactiveSettings").Return(storage.DefaultInactiveSettings()).Once()
 		h := s.handler()
 		h.GetInit(s.ctx)
 
 		s.Equal(http.StatusOK, s.w.Code)
-		s.store.AssertCalled(s.T(), "FindTickerByDomain", "demoticker.org", mock.Anything)
+		s.store.AssertCalled(s.T(), "FindTickerByOrigin", "https://demoticker.org", mock.Anything)
 	})
 }
 

internal/api/middleware/prometheus/prometheus.go

@@ -47,7 +47,7 @@ func prepareHandler(h string) string {
 }
 
 func prepareOrigin(c *gin.Context) string {
-	domain, err := helper.GetDomain(c)
+	domain, err := helper.GetOrigin(c)
 	if err != nil {
 		return ""
 	}

internal/api/middleware/response_cache/response_cache.go

@@ -116,7 +116,7 @@ func CachePage(cache *cache.Cache, expires time.Duration, handle gin.HandlerFunc
 }
 
 func CreateKey(c *gin.Context) string {
-	domain, err := helper.GetDomain(c)
+	domain, err := helper.GetOrigin(c)
 	if err != nil {
 		domain = "unknown"
 	}

internal/api/middleware/response_cache/response_cache_test.go

@@ -85,7 +85,7 @@ func (s *ResponseCacheTestSuite) TestCachePage() {
 
 		inMemoryCache := cache.NewCache(time.Minute)
 		defer inMemoryCache.Close()
-		inMemoryCache.Set("response:localhost:/ping:", responseCache{
+		inMemoryCache.Set("response:http://localhost:/ping:", responseCache{
 			Status: http.StatusOK,
 			Header: http.Header{
 				"DNT": []string{"1"},

internal/api/middleware/ticker/ticker.go

@@ -32,13 +32,13 @@ func PrefetchTicker(s storage.Storage, opts ...func(*gorm.DB) *gorm.DB) gin.Hand
 
 func PrefetchTickerFromRequest(s storage.Storage, opts ...func(*gorm.DB) *gorm.DB) gin.HandlerFunc {
 	return func(c *gin.Context) {
-		domain, err := helper.GetDomain(c)
+		origin, err := helper.GetOrigin(c)
 		if err != nil {
 			c.JSON(http.StatusOK, response.ErrorResponse(response.CodeDefault, response.TickerNotFound))
 			return
 		}
 
-		ticker, err := s.FindTickerByDomain(domain, opts...)
+		ticker, err := s.FindTickerByOrigin(origin, opts...)
 		if err != nil {
 			c.JSON(http.StatusOK, response.ErrorResponse(response.CodeDefault, response.TickerNotFound))
 			return

internal/api/middleware/ticker/ticker_test.go

@@ -84,7 +84,7 @@ func (s *TickerTestSuite) TestPrefetchTickerFromRequest() {
 		c.Request = httptest.NewRequest(http.MethodGet, "/v1/timeline", nil)
 		c.Request.Header.Set("Origin", "https://demoticker.org")
 		store := &storage.MockStorage{}
-		store.On("FindTickerByDomain", mock.Anything).Return(storage.Ticker{}, errors.New("not found"))
+		store.On("FindTickerByOrigin", mock.Anything).Return(storage.Ticker{}, errors.New("not found"))
 		mw := PrefetchTickerFromRequest(store)
 
 		mw(c)
@@ -101,7 +101,7 @@ func (s *TickerTestSuite) TestPrefetchTickerFromRequest() {
 		c.Request = httptest.NewRequest(http.MethodGet, "/v1/timeline", nil)
 		c.Request.Header.Set("Origin", "https://demoticker.org")
 		store := &storage.MockStorage{}
-		store.On("FindTickerByDomain", mock.Anything).Return(storage.Ticker{}, nil)
+		store.On("FindTickerByOrigin", mock.Anything).Return(storage.Ticker{}, nil)
 		mw := PrefetchTickerFromRequest(store)
 
 		mw(c)

internal/api/response/init.go

@@ -9,7 +9,6 @@ import (
 type InitTicker struct {
 	ID          int                   `json:"id"`
 	CreatedAt   time.Time             `json:"createdAt"`
-	Domain      string                `json:"domain"`
 	Title       string                `json:"title"`
 	Description string                `json:"description"`
 	Information InitTickerInformation `json:"information"`
@@ -30,7 +29,6 @@ func InitTickerResponse(ticker storage.Ticker) InitTicker {
 	return InitTicker{
 		ID:          ticker.ID,
 		CreatedAt:   ticker.CreatedAt,
-		Domain:      ticker.Domain,
 		Title:       ticker.Title,
 		Description: ticker.Description,
 		Information: InitTickerInformation{

internal/api/response/init_test.go

@@ -35,7 +35,6 @@ func (s *InitTickerResponseTestSuite) TestInitTickerResponse() {
 
 	s.Equal(ticker.ID, response.ID)
 	s.Equal(ticker.CreatedAt, response.CreatedAt)
-	s.Equal(ticker.Domain, response.Domain)
 	s.Equal(ticker.Title, response.Title)
 	s.Equal(ticker.Description, response.Description)
 	s.Equal(ticker.Information.Author, response.Information.Author)

internal/api/response/ticker.go

@@ -10,11 +10,11 @@ import (
 type Ticker struct {
 	ID          int         `json:"id"`
 	CreatedAt   time.Time   `json:"createdAt"`
-	Domain      string      `json:"domain"`
 	Title       string      `json:"title"`
 	Description string      `json:"description"`
 	Active      bool        `json:"active"`
 	Information Information `json:"information"`
+	Websites    []Website   `json:"websites"`
 	Telegram    Telegram    `json:"telegram"`
 	Mastodon    Mastodon    `json:"mastodon"`
 	Bluesky     Bluesky     `json:"bluesky"`
@@ -33,6 +33,12 @@ type Information struct {
 	Bluesky  string `json:"bluesky"`
 }
 
+type Website struct {
+	ID        int       `json:"id"`
+	CreatedAt time.Time `json:"createdAt"`
+	Origin    string    `json:"origin"`
+}
+
 type Telegram struct {
 	Active      bool   `json:"active"`
 	Connected   bool   `json:"connected"`
@@ -69,10 +75,18 @@ type Location struct {
 }
 
 func TickerResponse(t storage.Ticker, config config.Config) Ticker {
+	websites := make([]Website, 0)
+	for _, website := range t.Websites {
+		websites = append(websites, Website{
+			ID:        website.ID,
+			CreatedAt: website.CreatedAt,
+			Origin:    website.Origin,
+		})
+	}
+
 	return Ticker{
 		ID:          t.ID,
 		CreatedAt:   t.CreatedAt,
-		Domain:      t.Domain,
 		Title:       t.Title,
 		Description: t.Description,
 		Active:      t.Active,
@@ -86,6 +100,7 @@ func TickerResponse(t storage.Ticker, config config.Config) Ticker {
 			Mastodon: t.Information.Mastodon,
 			Bluesky:  t.Information.Bluesky,
 		},
+		Websites: websites,
 		Telegram: Telegram{
 			Active:      t.Telegram.Active,
 			Connected:   t.Telegram.Connected(),

internal/api/response/ticker_test.go

@@ -32,6 +32,11 @@ func (s *TickersResponseTestSuite) TestTickersResponse() {
 			Mastodon: "https://systemli.social/@example",
 			Bluesky:  "https://example.com",
 		},
+		Websites: []storage.TickerWebsite{
+			{
+				Origin: "example.com",
+			},
+		},
 		Telegram: storage.TickerTelegram{
 			Active:      true,
 			ChannelName: "example",
@@ -69,7 +74,6 @@ func (s *TickersResponseTestSuite) TestTickersResponse() {
 	s.Equal(1, len(tickerResponse))
 	s.Equal(ticker.ID, tickerResponse[0].ID)
 	s.Equal(ticker.CreatedAt, tickerResponse[0].CreatedAt)
-	s.Equal(ticker.Domain, tickerResponse[0].Domain)
 	s.Equal(ticker.Title, tickerResponse[0].Title)
 	s.Equal(ticker.Description, tickerResponse[0].Description)
 	s.Equal(ticker.Active, tickerResponse[0].Active)
@@ -81,6 +85,8 @@ func (s *TickersResponseTestSuite) TestTickersResponse() {
 	s.Equal(ticker.Information.Telegram, tickerResponse[0].Information.Telegram)
 	s.Equal(ticker.Information.Mastodon, tickerResponse[0].Information.Mastodon)
 	s.Equal(ticker.Information.Bluesky, tickerResponse[0].Information.Bluesky)
+	s.Equal(1, len(ticker.Websites))
+	s.Equal(ticker.Websites[0].Origin, tickerResponse[0].Websites[0].Origin)
 	s.Equal(ticker.Telegram.Active, tickerResponse[0].Telegram.Active)
 	s.Equal(ticker.Telegram.Connected(), tickerResponse[0].Telegram.Connected)
 	s.Equal(config.Telegram.User.UserName, tickerResponse[0].Telegram.BotUsername)