add exclusive order data (#122)

* add exclusive order data

* add tradelog mview

* fix lint

* add worker for refresh

* add worker for refresh

* add worker for refresh

v2/cmd/migrations/00017_add_exclusive_order_data.up.sql

@@ -0,0 +1,311 @@
+ALTER TABLE tradelogs_uniswapx
+ADD COLUMN expiration BIGINT NOT NULL DEFAULT 0,
+ADD COLUMN decay_start_time BIGINT DEFAULT NULL,
+ADD COLUMN decay_end_time BIGINT DEFAULT NULL,
+ADD COLUMN exclusive_filler TEXT NOT NULL DEFAULT '';
+
+ALTER TABLE tradelogs_pancakeswap
+ADD COLUMN expiration BIGINT NOT NULL DEFAULT 0;
+
+CREATE MATERIALIZED VIEW mview_tradelogs AS
+WITH all_trades AS (
+  SELECT
+      maker,
+      taker,
+      maker_token,
+      taker_token,
+      CASE 
+        WHEN maker_token_amount ~ '^[0-9]+(\.[0-9]+)?$' THEN REPLACE(maker_token_amount, ',', '.')::DECIMAL
+        ELSE NULL
+      END AS maker_token_amount,
+      CASE 
+          WHEN taker_token_amount ~ '^[0-9]+(\.[0-9]+)?$' THEN REPLACE(taker_token_amount, ',', '.')::DECIMAL
+          ELSE NULL
+      END AS taker_token_amount,
+      contract_address,
+      block_number,
+      tx_hash,
+      timestamp,
+      event_hash,
+      maker_token_price,
+      taker_token_price,
+      maker_usd_amount,
+      taker_usd_amount,
+      'zerox' AS contract,
+      '' AS type
+  FROM public.tradelogs_zerox
+  WHERE timestamp >= EXTRACT(EPOCH FROM CURRENT_TIMESTAMP - INTERVAL '3 months') * 1000
+  AND maker_usd_amount > 0 AND taker_usd_amount > 0
+
+
+  UNION ALL
+
+  SELECT
+      maker,
+      taker,
+      maker_token,
+      taker_token,
+      CASE 
+        WHEN maker_token_amount ~ '^[0-9]+(\.[0-9]+)?$' THEN REPLACE(maker_token_amount, ',', '.')::DECIMAL
+        ELSE NULL
+      END AS maker_token_amount,
+      CASE 
+          WHEN taker_token_amount ~ '^[0-9]+(\.[0-9]+)?$' THEN REPLACE(taker_token_amount, ',', '.')::DECIMAL
+          ELSE NULL
+      END AS taker_token_amount,
+      contract_address,
+      block_number,
+      tx_hash,
+      timestamp,
+      event_hash,
+      maker_token_price,
+      taker_token_price,
+      maker_usd_amount,
+      taker_usd_amount,
+      'kyberswap' AS contract,
+      '' AS type
+  FROM public.tradelogs_kyberswap 
+  WHERE timestamp >= EXTRACT(EPOCH FROM CURRENT_TIMESTAMP - INTERVAL '3 months') * 1000
+  AND maker_usd_amount > 0 AND taker_usd_amount > 0
+
+  UNION ALL
+
+  SELECT
+      maker,
+      taker,
+      maker_token,
+      taker_token,
+      CASE 
+        WHEN maker_token_amount ~ '^[0-9]+(\.[0-9]+)?$' THEN REPLACE(maker_token_amount, ',', '.')::DECIMAL
+        ELSE NULL
+      END AS maker_token_amount,
+      CASE 
+          WHEN taker_token_amount ~ '^[0-9]+(\.[0-9]+)?$' THEN REPLACE(taker_token_amount, ',', '.')::DECIMAL
+          ELSE NULL
+      END AS taker_token_amount,
+      contract_address,
+      block_number,
+      tx_hash,
+      timestamp,
+      event_hash,
+      maker_token_price,
+      taker_token_price,
+      maker_usd_amount,
+      taker_usd_amount,
+      'kyberswap_rfq' AS contract,
+      '' AS type
+  FROM public.tradelogs_kyberswap_rfq 
+  WHERE timestamp >= EXTRACT(EPOCH FROM CURRENT_TIMESTAMP - INTERVAL '3 months') * 1000
+  AND maker_usd_amount > 0 AND taker_usd_amount > 0
+
+  UNION ALL
+
+  SELECT
+      maker,
+      taker,
+      maker_token,
+      taker_token,
+      CASE 
+        WHEN maker_token_amount ~ '^[0-9]+(\.[0-9]+)?$' THEN REPLACE(maker_token_amount, ',', '.')::DECIMAL
+        ELSE NULL
+      END AS maker_token_amount,
+      CASE 
+          WHEN taker_token_amount ~ '^[0-9]+(\.[0-9]+)?$' THEN REPLACE(taker_token_amount, ',', '.')::DECIMAL
+          ELSE NULL
+      END AS taker_token_amount,
+      contract_address,
+      block_number,
+      tx_hash,
+      timestamp,
+      event_hash,
+      maker_token_price,
+      taker_token_price,
+      maker_usd_amount,
+      taker_usd_amount,
+      'paraswap' AS contract,
+      '' AS type
+  FROM public.tradelogs_paraswap 
+  WHERE timestamp >= EXTRACT(EPOCH FROM CURRENT_TIMESTAMP - INTERVAL '3 months') * 1000
+  AND maker_usd_amount > 0 AND taker_usd_amount > 0
+
+  UNION ALL
+
+  SELECT
+      maker,
+      taker,
+      maker_token,
+      taker_token,
+      CASE 
+        WHEN maker_token_amount ~ '^[0-9]+(\.[0-9]+)?$' THEN REPLACE(maker_token_amount, ',', '.')::DECIMAL
+        ELSE NULL
+      END AS maker_token_amount,
+      CASE 
+          WHEN taker_token_amount ~ '^[0-9]+(\.[0-9]+)?$' THEN REPLACE(taker_token_amount, ',', '.')::DECIMAL
+          ELSE NULL
+      END AS taker_token_amount,
+      contract_address,
+      block_number,
+      tx_hash,
+      timestamp,
+      event_hash,
+      maker_token_price,
+      taker_token_price,
+      maker_usd_amount,
+      taker_usd_amount,
+      'hashflow_v3' AS contract,
+      '' AS type
+  FROM public.tradelogs_hashflow_v3 
+  WHERE timestamp >= EXTRACT(EPOCH FROM CURRENT_TIMESTAMP - INTERVAL '3 months') * 1000
+  AND maker_usd_amount > 0 AND taker_usd_amount > 0
+
+  UNION ALL
+
+  SELECT
+      maker,
+      taker,
+      maker_token,
+      taker_token,
+      CASE 
+        WHEN maker_token_amount ~ '^[0-9]+(\.[0-9]+)?$' THEN REPLACE(maker_token_amount, ',', '.')::DECIMAL
+        ELSE NULL
+      END AS maker_token_amount,
+      CASE 
+          WHEN taker_token_amount ~ '^[0-9]+(\.[0-9]+)?$' THEN REPLACE(taker_token_amount, ',', '.')::DECIMAL
+          ELSE NULL
+      END AS taker_token_amount,
+      contract_address,
+      block_number,
+      tx_hash,
+      timestamp,
+      event_hash,
+      maker_token_price,
+      taker_token_price,
+      maker_usd_amount,
+      taker_usd_amount,
+      'oneinch_v6' AS contract,
+      type
+  FROM public.tradelogs_oneinch_v6 
+  WHERE timestamp >= EXTRACT(EPOCH FROM CURRENT_TIMESTAMP - INTERVAL '3 months') * 1000
+  AND maker_usd_amount > 0 AND taker_usd_amount > 0
+
+  UNION ALL
+
+  SELECT
+      maker,
+      taker,
+      maker_token,
+      taker_token,
+      CASE 
+        WHEN maker_token_amount ~ '^[0-9]+(\.[0-9]+)?$' THEN REPLACE(maker_token_amount, ',', '.')::DECIMAL
+        ELSE NULL
+      END AS maker_token_amount,
+      CASE 
+          WHEN taker_token_amount ~ '^[0-9]+(\.[0-9]+)?$' THEN REPLACE(taker_token_amount, ',', '.')::DECIMAL
+          ELSE NULL
+      END AS taker_token_amount,
+      contract_address,
+      block_number,
+      tx_hash,
+      timestamp,
+      event_hash,
+      maker_token_price,
+      taker_token_price,
+      maker_usd_amount,
+      taker_usd_amount,
+      'uniswapx' AS contract,
+      '' AS type
+  FROM public.tradelogs_uniswapx 
+  WHERE timestamp >= EXTRACT(EPOCH FROM CURRENT_TIMESTAMP - INTERVAL '3 months') * 1000
+  AND maker_usd_amount > 0 AND taker_usd_amount > 0
+
+  UNION ALL
+
+  SELECT
+      maker,
+      taker,
+      maker_token,
+      taker_token,
+      CASE 
+        WHEN maker_token_amount ~ '^[0-9]+(\.[0-9]+)?$' THEN REPLACE(maker_token_amount, ',', '.')::DECIMAL
+        ELSE NULL
+      END AS maker_token_amount,
+      CASE 
+          WHEN taker_token_amount ~ '^[0-9]+(\.[0-9]+)?$' THEN REPLACE(taker_token_amount, ',', '.')::DECIMAL
+          ELSE NULL
+      END AS taker_token_amount,
+      contract_address,
+      block_number,
+      tx_hash,
+      timestamp,
+      event_hash,
+      maker_token_price,
+      taker_token_price,
+      maker_usd_amount,
+      taker_usd_amount,
+      'bebop' AS contract,
+      '' AS type
+  FROM public.tradelogs_bebop 
+  WHERE timestamp >= EXTRACT(EPOCH FROM CURRENT_TIMESTAMP - INTERVAL '3 months') * 1000
+  AND maker_usd_amount > 0 AND taker_usd_amount > 0
+
+  UNION ALL
+
+  SELECT
+      maker,
+      taker,
+      maker_token,
+      taker_token,
+      CASE 
+        WHEN maker_token_amount ~ '^[0-9]+(\.[0-9]+)?$' THEN REPLACE(maker_token_amount, ',', '.')::DECIMAL
+        ELSE NULL
+      END AS maker_token_amount,
+      CASE 
+          WHEN taker_token_amount ~ '^[0-9]+(\.[0-9]+)?$' THEN REPLACE(taker_token_amount, ',', '.')::DECIMAL
+          ELSE NULL
+      END AS taker_token_amount,
+      contract_address,
+      block_number,
+      tx_hash,
+      timestamp,
+      event_hash,
+      maker_token_price,
+      taker_token_price,
+      maker_usd_amount,
+      taker_usd_amount,
+      'zerox_v3' AS contract,
+      '' AS type
+  FROM public.tradelogs_zerox_v3
+  WHERE timestamp >= EXTRACT(EPOCH FROM CURRENT_TIMESTAMP - INTERVAL '3 months') * 1000
+  AND maker_usd_amount > 0 AND taker_usd_amount > 0
+
+  UNION ALL
+
+  SELECT
+      maker,
+      taker,
+      maker_token,
+      taker_token,
+      CASE 
+        WHEN maker_token_amount ~ '^[0-9]+(\.[0-9]+)?$' THEN REPLACE(maker_token_amount, ',', '.')::DECIMAL
+        ELSE NULL
+      END AS maker_token_amount,
+      CASE 
+          WHEN taker_token_amount ~ '^[0-9]+(\.[0-9]+)?$' THEN REPLACE(taker_token_amount, ',', '.')::DECIMAL
+          ELSE NULL
+      END AS taker_token_amount,
+      contract_address,
+      block_number,
+      tx_hash,
+      timestamp,
+      event_hash,
+      maker_token_price,
+      taker_token_price,
+      maker_usd_amount,
+      taker_usd_amount,
+      'pancakeswap' AS contract,
+      '' AS type
+  FROM public.tradelogs_pancakeswap 
+  WHERE timestamp >= EXTRACT(EPOCH FROM CURRENT_TIMESTAMP - INTERVAL '3 months') * 1000
+  AND maker_usd_amount > 0 AND taker_usd_amount > 0
+)
+SELECT * FROM all_trades ORDER BY timestamp DESC;

v2/cmd/tradelogs/main.go

@@ -6,6 +6,7 @@ import (
 	"os"
 
 	"github.com/KyberNetwork/tradelogs/v2/internal/server"
+	"github.com/KyberNetwork/tradelogs/v2/internal/worker"
 	libapp "github.com/KyberNetwork/tradelogs/v2/pkg/app"
 	dashboardStorage "github.com/KyberNetwork/tradelogs/v2/pkg/storage/dashboard"
 	bebopStorage "github.com/KyberNetwork/tradelogs/v2/pkg/storage/tradelogs/bebop"
@@ -69,6 +70,8 @@ func run(c *cli.Context) error {
 		pancakeswapStorage.New(l, db),
 	}
 	dashStorage := dashboardStorage.New(l, db)
+	mviewRefresher := worker.NewRefresher(dashStorage, l)
+	mviewRefresher.Run()
 	s := server.NewTradeLogs(l, storage, dashStorage, c.String(libapp.HTTPTradeLogsServerFlag.Name))
 	return s.Run()
 }

v2/internal/worker/mview_refresher.go

@@ -0,0 +1,37 @@
+package worker
+
+import (
+	"time"
+
+	dashboardStorage "github.com/KyberNetwork/tradelogs/v2/pkg/storage/dashboard"
+	"go.uber.org/zap"
+)
+
+type Refresher struct {
+	DashStorage *dashboardStorage.Storage
+	l           *zap.SugaredLogger
+}
+
+func NewRefresher(dashStorage *dashboardStorage.Storage, l *zap.SugaredLogger) *Refresher {
+	return &Refresher{
+		DashStorage: dashStorage,
+		l:           l,
+	}
+}
+
+func (r *Refresher) Run() {
+	go r.ScheduleMViewRefresh()
+}
+
+func (r *Refresher) ScheduleMViewRefresh() {
+	ticker := time.NewTicker(24 * time.Hour)
+	defer ticker.Stop()
+	for range ticker.C {
+		err := r.DashStorage.RefreshTradelogsMView()
+		if err != nil {
+			r.l.Errorf("Error refreshing materialized view: %v", err)
+		} else {
+			r.l.Info("Materialized view refreshed successfully.")
+		}
+	}
+}

v2/pkg/parser/uniswapx/parser.go

@@ -249,6 +249,15 @@ func (p *Parser) updateOrder(tradeLog storageTypes.TradeLog, call types.CallFram
 	if order.Info.Deadline != nil {
 		tradeLog.Expiry = order.Info.Deadline.Uint64()
 	}
+	if order.CosignerData.DecayStartTime != nil {
+		var decayStartTime uint64 = order.CosignerData.DecayStartTime.Uint64()
+		tradeLog.DecayStartTime = &decayStartTime
+	}
+	if order.CosignerData.DecayEndTime != nil {
+		var decayEndTime uint64 = order.CosignerData.DecayEndTime.Uint64()
+		tradeLog.DecayEndTime = &decayEndTime
+	}
+	tradeLog.ExclusiveFiller = order.CosignerData.ExclusiveFiller.Hex()
 	return tradeLog, nil
 }
 

v2/pkg/storage/dashboard/storage.go

@@ -196,3 +196,8 @@ func (s *Storage) GetTxOrigin(query types.TxOriginQuery) ([]types.TxOrigin, erro
 
 	return txOrigins, nil
 }
+
+func (s *Storage) RefreshTradelogsMView() error {
+	_, err := s.db.Exec("REFRESH MATERIALIZED VIEW mview_tradelogs WITH DATA")
+	return err
+}