Skip to content

Commit

Permalink
Add backend supports for tso and scheduling (#1648)
Browse files Browse the repository at this point in the history 
* add backend supports for tso and scheduling

Signed-off-by: Ryan Leung <[email protected]>

* fix: fix api error and handle compatibility with lower version tidb cluster

---------

Signed-off-by: Ryan Leung <[email protected]>
Co-authored-by: baurine <[email protected]>
  • Loading branch information
@rleungx @baurine
rleungx and baurine authored Mar 14, 2024
1 parent abee03e commit 68517df
Show file tree
Hide file tree
Showing 53 changed files with 1,391 additions and 89 deletions.
10 changes: 10 additions & 0 deletions pkg/apiserver/apiserver.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,17 +32,21 @@ import (
"github.com/pingcap/tidb-dashboard/pkg/apiserver/user/sso"
"github.com/pingcap/tidb-dashboard/pkg/apiserver/user/sso/ssoauth"
"github.com/pingcap/tidb-dashboard/pkg/apiserver/visualplan"
"github.com/pingcap/tidb-dashboard/pkg/scheduling"
"github.com/pingcap/tidb-dashboard/pkg/ticdc"
"github.com/pingcap/tidb-dashboard/pkg/tiflash"
"github.com/pingcap/tidb-dashboard/pkg/tiproxy"
"github.com/pingcap/tidb-dashboard/pkg/tso"
"github.com/pingcap/tidb-dashboard/pkg/utils/version"
"github.com/pingcap/tidb-dashboard/util/client/httpclient"
"github.com/pingcap/tidb-dashboard/util/client/pdclient"
"github.com/pingcap/tidb-dashboard/util/client/schedulingclient"
"github.com/pingcap/tidb-dashboard/util/client/ticdcclient"
"github.com/pingcap/tidb-dashboard/util/client/tidbclient"
"github.com/pingcap/tidb-dashboard/util/client/tiflashclient"
"github.com/pingcap/tidb-dashboard/util/client/tikvclient"
"github.com/pingcap/tidb-dashboard/util/client/tiproxyclient"
"github.com/pingcap/tidb-dashboard/util/client/tsoclient"
"github.com/pingcap/tidb-dashboard/util/featureflag"
"github.com/pingcap/tidb-dashboard/util/rest"

Expand Down Expand Up @@ -113,6 +117,8 @@ var Modules = fx.Options(
httpc.NewHTTPClient,
pd.NewEtcdClient,
pd.NewPDClient,
tso.NewTSOClient,
scheduling.NewSchedulingClient,
config.NewDynamicConfigManager,
tidb.NewTiDBClient,
tikv.NewTiKVClient,
Expand Down Expand Up @@ -199,6 +205,8 @@ func newClients(lc fx.Lifecycle, config *config.Config) (
pdClient *pdclient.APIClient,
ticdcClient *ticdcclient.StatusClient,
tiproxyClient *tiproxyclient.StatusClient,
tsoClient *tsoclient.StatusClient,
schedulingClient *schedulingclient.StatusClient,
) {
httpConfig := httpclient.Config{
TLSConfig: config.ClusterTLSConfig,
Expand All @@ -209,6 +217,8 @@ func newClients(lc fx.Lifecycle, config *config.Config) (
pdClient = pdclient.NewAPIClient(httpConfig)
ticdcClient = ticdcclient.NewStatusClient(httpConfig)
tiproxyClient = tiproxyclient.NewStatusClient(httpConfig)
tsoClient = tsoclient.NewStatusClient(httpConfig)
schedulingClient = schedulingclient.NewStatusClient(httpConfig)
lc.Append(fx.Hook{
OnStart: func(ctx context.Context) error {
dbClient.SetDefaultCtx(ctx)
Expand Down
16 changes: 16 additions & 0 deletions pkg/apiserver/clusterinfo/host.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,22 @@ func (s *Service) fetchAllInstanceHosts() ([]string, error) {
allHostsMap[i.IP] = struct{}{}
}

tsoInfo, err := topology.FetchTSOTopology(s.lifecycleCtx, s.params.PDClient)
if err != nil {
return nil, err
}
for _, i := range tsoInfo {
allHostsMap[i.IP] = struct{}{}
}

schedulingInfo, err := topology.FetchSchedulingTopology(s.lifecycleCtx, s.params.PDClient)
if err != nil {
return nil, err
}
for _, i := range schedulingInfo {
allHostsMap[i.IP] = struct{}{}
}

allHosts := lo.Keys(allHostsMap)
sort.Strings(allHosts)

Expand Down
2 changes: 1 addition & 1 deletion pkg/apiserver/clusterinfo/hostinfo/hostinfo.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ type InfoMap = map[string]*Info
var clusterTableQueryTemplate = template.Must(template.New("").Parse(`
SELECT
*,
FIELD(LOWER(A.TYPE), 'tiflash', 'tikv', 'pd', 'tidb', 'tiproxy') AS _ORDER
FIELD(LOWER(A.TYPE), 'tiflash', 'tikv', 'pd', 'tidb', 'tiproxy', 'tso', 'scheduling') AS _ORDER
FROM (
SELECT
TYPE, INSTANCE, DEVICE_TYPE, DEVICE_NAME, JSON_OBJECTAGG(NAME, VALUE) AS JSON_VALUE
Expand Down
43 changes: 43 additions & 0 deletions pkg/apiserver/clusterinfo/service.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"context"
"fmt"
"net/http"
"strings"
"sync"
"time"

Expand Down Expand Up @@ -59,6 +60,8 @@ func RegisterRouter(r *gin.RouterGroup, auth *user.AuthService, s *Service) {
endpoint.DELETE("/tidb/:address", s.deleteTiDBTopology)
endpoint.GET("/store", s.getStoreTopology)
endpoint.GET("/pd", s.getPDTopology)
endpoint.GET("/tso", s.getTSOTopology)
endpoint.GET("/scheduling", s.getSchedulingTopology)
endpoint.GET("/alertmanager", s.getAlertManagerTopology)
endpoint.GET("/alertmanager/:address/count", s.getAlertManagerCounts)
endpoint.GET("/grafana", s.getGrafanaTopology)
Expand Down Expand Up @@ -156,6 +159,46 @@ func (s *Service) getTiProxyTopology(c *gin.Context) {
c.JSON(http.StatusOK, instances)
}

// @ID getTSOTopology
// @Summary Get all TSO instances
// @Success 200 {array} topology.TSOInfo
// @Router /topology/tso [get]
// @Security JwtAuth
// @Failure 401 {object} rest.ErrorResponse
func (s *Service) getTSOTopology(c *gin.Context) {
instances, err := topology.FetchTSOTopology(s.lifecycleCtx, s.params.PDClient)
if err != nil {
// TODO: refine later
if strings.Contains(err.Error(), "status code 404") {
rest.Error(c, rest.ErrNotFound.Wrap(err, "api not found"))
} else {
rest.Error(c, err)
}
return
}
c.JSON(http.StatusOK, instances)
}

// @ID getSchedulingTopology
// @Summary Get all Scheduling instances
// @Success 200 {array} topology.SchedulingInfo
// @Router /topology/scheduling [get]
// @Security JwtAuth
// @Failure 401 {object} rest.ErrorResponse
func (s *Service) getSchedulingTopology(c *gin.Context) {
instances, err := topology.FetchSchedulingTopology(s.lifecycleCtx, s.params.PDClient)
if err != nil {
// TODO: refine later
if strings.Contains(err.Error(), "status code 404") {
rest.Error(c, rest.ErrNotFound.Wrap(err, "api not found"))
} else {
rest.Error(c, err)
}
return
}
c.JSON(http.StatusOK, instances)
}

type StoreTopologyResponse struct {
TiKV []topology.StoreInfo `json:"tikv"`
TiFlash []topology.StoreInfo `json:"tiflash"`
Expand Down
36 changes: 36 additions & 0 deletions pkg/apiserver/clusterinfo/statistics.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,8 @@ func (s *Service) calculateStatistics(db *gorm.DB) (*ClusterStatistics, error) {
infoByIk["tiflash"] = newInstanceKindImmediateInfo()
infoByIk["ticdc"] = newInstanceKindImmediateInfo()
infoByIk["tiproxy"] = newInstanceKindImmediateInfo()
infoByIk["tso"] = newInstanceKindImmediateInfo()
infoByIk["scheduling"] = newInstanceKindImmediateInfo()

// Fill from topology info
pdInfo, err := topology.FetchPDTopology(s.params.PDClient)
Expand Down Expand Up @@ -135,6 +137,26 @@ func (s *Service) calculateStatistics(db *gorm.DB) (*ClusterStatistics, error) {
globalInfo.instances[net.JoinHostPort(i.IP, strconv.Itoa(int(i.Port)))] = struct{}{}
infoByIk["tiproxy"].instances[net.JoinHostPort(i.IP, strconv.Itoa(int(i.Port)))] = struct{}{}
}
tsoInfo, err := topology.FetchTSOTopology(s.lifecycleCtx, s.params.PDClient)
if err != nil {
return nil, err
}
for _, i := range tsoInfo {
globalHostsSet[i.IP] = struct{}{}
globalVersionsSet[i.Version] = struct{}{}
globalInfo.instances[net.JoinHostPort(i.IP, strconv.Itoa(int(i.Port)))] = struct{}{}
infoByIk["tso"].instances[net.JoinHostPort(i.IP, strconv.Itoa(int(i.Port)))] = struct{}{}
}
schedulingInfo, err := topology.FetchSchedulingTopology(s.lifecycleCtx, s.params.PDClient)
if err != nil {
return nil, err
}
for _, i := range schedulingInfo {
globalHostsSet[i.IP] = struct{}{}
globalVersionsSet[i.Version] = struct{}{}
globalInfo.instances[net.JoinHostPort(i.IP, strconv.Itoa(int(i.Port)))] = struct{}{}
infoByIk["scheduling"].instances[net.JoinHostPort(i.IP, strconv.Itoa(int(i.Port)))] = struct{}{}
}

// Fill from hardware info
allHostsInfoMap := make(map[string]*hostinfo.Info)
Expand Down Expand Up @@ -198,6 +220,20 @@ func (s *Service) calculateStatistics(db *gorm.DB) (*ClusterStatistics, error) {
globalFailureHostsSet[i.IP] = struct{}{}
}
}
for _, i := range tsoInfo {
if v, ok := globalInfo.hosts[i.IP]; ok {
infoByIk["tso"].hosts[i.IP] = v
} else {
globalFailureHostsSet[i.IP] = struct{}{}
}
}
for _, i := range schedulingInfo {
if v, ok := globalInfo.hosts[i.IP]; ok {
infoByIk["scheduling"].hosts[i.IP] = v
} else {
globalFailureHostsSet[i.IP] = struct{}{}
}
}

// Generate result..
versions := lo.Keys(globalVersionsSet)
Expand Down
46 changes: 40 additions & 6 deletions pkg/apiserver/debugapi/endpoint/payload.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,13 @@ import (
"github.com/pingcap/tidb-dashboard/pkg/utils/topology"
"github.com/pingcap/tidb-dashboard/util/client/httpclient"
"github.com/pingcap/tidb-dashboard/util/client/pdclient"
"github.com/pingcap/tidb-dashboard/util/client/schedulingclient"
"github.com/pingcap/tidb-dashboard/util/client/ticdcclient"
"github.com/pingcap/tidb-dashboard/util/client/tidbclient"
"github.com/pingcap/tidb-dashboard/util/client/tiflashclient"
"github.com/pingcap/tidb-dashboard/util/client/tikvclient"
"github.com/pingcap/tidb-dashboard/util/client/tiproxyclient"
"github.com/pingcap/tidb-dashboard/util/client/tsoclient"
"github.com/pingcap/tidb-dashboard/util/rest"
"github.com/pingcap/tidb-dashboard/util/topo"
)
Expand All @@ -38,12 +40,14 @@ type RequestPayload struct {
}

type HTTPClients struct {
PDAPIClient *pdclient.APIClient
TiDBStatusClient *tidbclient.StatusClient
TiKVStatusClient *tikvclient.StatusClient
TiFlashStatusClient *tiflashclient.StatusClient
TiCDCStatusClient *ticdcclient.StatusClient
TiProxyStatusClient *tiproxyclient.StatusClient
PDAPIClient *pdclient.APIClient
TiDBStatusClient *tidbclient.StatusClient
TiKVStatusClient *tikvclient.StatusClient
TiFlashStatusClient *tiflashclient.StatusClient
TiCDCStatusClient *ticdcclient.StatusClient
TiProxyStatusClient *tiproxyclient.StatusClient
TSOStatusClient *tsoclient.StatusClient
SchedulingStatusClient *schedulingclient.StatusClient
}

func (c HTTPClients) GetHTTPClientByNodeKind(kind topo.Kind) *httpclient.Client {
Expand Down Expand Up @@ -306,6 +310,36 @@ func (p *ResolvedRequestPayload) verifyEndpoint(ctx context.Context, etcdClient
if !matched {
return ErrInvalidEndpoint.New("invalid endpoint '%s:%d'", p.host, p.port)
}
case topo.KindTSO:
infos, err := topology.FetchTSOTopology(ctx, pdClient)
if err != nil {
return ErrInvalidEndpoint.Wrap(err, "failed to fetch tso topology")
}
matched := false
for _, info := range infos {
if info.IP == p.host && info.Port == uint(p.port) {
matched = true
break
}
}
if !matched {
return ErrInvalidEndpoint.New("invalid endpoint '%s:%d'", p.host, p.port)
}
case topo.KindScheduling:
infos, err := topology.FetchSchedulingTopology(ctx, pdClient)
if err != nil {
return ErrInvalidEndpoint.Wrap(err, "failed to fetch scheduling topology")
}
matched := false
for _, info := range infos {
if info.IP == p.host && info.Port == uint(p.port) {
matched = true
break
}
}
if !matched {
return ErrInvalidEndpoint.New("invalid endpoint '%s:%d'", p.host, p.port)
}
default:
return ErrUnknownComponent.New("Unknown component '%s'", p.api.Component)
}
Expand Down
34 changes: 20 additions & 14 deletions pkg/apiserver/debugapi/service.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,13 @@ import (
"github.com/pingcap/tidb-dashboard/pkg/apiserver/user"
"github.com/pingcap/tidb-dashboard/pkg/pd"
"github.com/pingcap/tidb-dashboard/util/client/pdclient"
"github.com/pingcap/tidb-dashboard/util/client/schedulingclient"
"github.com/pingcap/tidb-dashboard/util/client/ticdcclient"
"github.com/pingcap/tidb-dashboard/util/client/tidbclient"
"github.com/pingcap/tidb-dashboard/util/client/tiflashclient"
"github.com/pingcap/tidb-dashboard/util/client/tikvclient"
"github.com/pingcap/tidb-dashboard/util/client/tiproxyclient"
"github.com/pingcap/tidb-dashboard/util/client/tsoclient"
"github.com/pingcap/tidb-dashboard/util/rest"
"github.com/pingcap/tidb-dashboard/util/rest/fileswap"
)
Expand All @@ -37,14 +39,16 @@ func registerRouter(r *gin.RouterGroup, auth *user.AuthService, s *Service) {

type ServiceParams struct {
fx.In
PDAPIClient *pdclient.APIClient
TiDBStatusClient *tidbclient.StatusClient
TiKVStatusClient *tikvclient.StatusClient
TiFlashStatusClient *tiflashclient.StatusClient
TiCDCStatusClient *ticdcclient.StatusClient
TiProxyStatusClient *tiproxyclient.StatusClient
EtcdClient *clientv3.Client
PDClient *pd.Client
PDAPIClient *pdclient.APIClient
TiDBStatusClient *tidbclient.StatusClient
TiKVStatusClient *tikvclient.StatusClient
TiFlashStatusClient *tiflashclient.StatusClient
TiCDCStatusClient *ticdcclient.StatusClient
TiProxyStatusClient *tiproxyclient.StatusClient
EtcdClient *clientv3.Client
PDClient *pd.Client
TSOStatusClient *tsoclient.StatusClient
SchedulingStatusClient *schedulingclient.StatusClient
}

type Service struct {
Expand All @@ -57,12 +61,14 @@ type Service struct {

func newService(p ServiceParams) *Service {
httpClients := endpoint.HTTPClients{
PDAPIClient: p.PDAPIClient,
TiDBStatusClient: p.TiDBStatusClient,
TiKVStatusClient: p.TiKVStatusClient,
TiFlashStatusClient: p.TiFlashStatusClient,
TiCDCStatusClient: p.TiCDCStatusClient,
TiProxyStatusClient: p.TiProxyStatusClient,
PDAPIClient: p.PDAPIClient,
TiDBStatusClient: p.TiDBStatusClient,
TiKVStatusClient: p.TiKVStatusClient,
TiFlashStatusClient: p.TiFlashStatusClient,
TiCDCStatusClient: p.TiCDCStatusClient,
TiProxyStatusClient: p.TiProxyStatusClient,
TSOStatusClient: p.TSOStatusClient,
SchedulingStatusClient: p.SchedulingStatusClient,
}
return &Service{
httpClients: httpClients,
Expand Down
32 changes: 20 additions & 12 deletions pkg/apiserver/model/common_models.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,14 @@ import (
type NodeKind string

const (
NodeKindTiDB NodeKind = "tidb"
NodeKindTiKV NodeKind = "tikv"
NodeKindPD NodeKind = "pd"
NodeKindTiFlash NodeKind = "tiflash"
NodeKindTiCDC NodeKind = "ticdc"
NodeKindTiProxy NodeKind = "tiproxy"
NodeKindTiDB NodeKind = "tidb"
NodeKindTiKV NodeKind = "tikv"
NodeKindPD NodeKind = "pd"
NodeKindTiFlash NodeKind = "tiflash"
NodeKindTiCDC NodeKind = "ticdc"
NodeKindTiProxy NodeKind = "tiproxy"
NodeKindTSO NodeKind = "tso"
NodeKindScheduling NodeKind = "scheduling"
)

type RequestTargetNode struct {
Expand All @@ -35,12 +37,14 @@ func (n *RequestTargetNode) FileName() string {
}

type RequestTargetStatistics struct {
NumTiKVNodes int `json:"num_tikv_nodes"`
NumTiDBNodes int `json:"num_tidb_nodes"`
NumPDNodes int `json:"num_pd_nodes"`
NumTiFlashNodes int `json:"num_tiflash_nodes"`
NumTiCDCNodes int `json:"num_ticdc_nodes"`
NumTiProxyNodes int `json:"num_tiproxy_nodes"`
NumTiKVNodes int `json:"num_tikv_nodes"`
NumTiDBNodes int `json:"num_tidb_nodes"`
NumPDNodes int `json:"num_pd_nodes"`
NumTiFlashNodes int `json:"num_tiflash_nodes"`
NumTiCDCNodes int `json:"num_ticdc_nodes"`
NumTiProxyNodes int `json:"num_tiproxy_nodes"`
NumTSONodes int `json:"num_tso_nodes"`
NumSchedulingNodes int `json:"num_scheduling_nodes"`
}

func NewRequestTargetStatisticsFromArray(arr *[]RequestTargetNode) RequestTargetStatistics {
Expand All @@ -59,6 +63,10 @@ func NewRequestTargetStatisticsFromArray(arr *[]RequestTargetNode) RequestTarget
stats.NumTiCDCNodes++
case NodeKindTiProxy:
stats.NumTiProxyNodes++
case NodeKindTSO:
stats.NumTSONodes++
case NodeKindScheduling:
stats.NumSchedulingNodes++
}
}
return stats
Expand Down
Loading

0 comments on commit 68517df

Please sign in to comment.