Merge branch 'master' into mysql_comments

.github/workflows/lint.yaml

@@ -0,0 +1,22 @@
+name: lint
+
+on:
+  pull_request:
+
+jobs:
+  install:
+    name: Lint
+    runs-on: ubuntu-latest
+    env:
+      working-directory: ${{ github.workspace }}/go/src/github.com/${{ github.repository }}
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          path: go/src/github.com/${{ github.repository }}
+
+      - name: Lint
+        working-directory: ${{ env.working-directory }}
+        run: make lint
+

CHANGELOG.md

@@ -1,5 +1,35 @@
 TiUP Changelog
 
+## [1.14.0] 2023-11-17
+
+### New Features
+
+- Support use different component versions in `tiup-cluster` (#2010 #2264 #2306, @nexustar)
+- Add global listen_host config in `tiup-cluster` and `tiup-dm` (#2303, @nexustar)
+- Add gloabl component_sources config in `tiup-cluster` and `tiup-dm` (#2320, @nexustar)
+- Support TiDB upgrade API to automatically pause DDL when upgrade in `tiup-cluster`(#2287 #2289, @nexustar)
+- Support TiProxy in `tiup-cluster` (#2271, @xhebox)
+- Support scheduling service in `tiup-playground` (#2273, @rleungx)
+- Support to show numa node when display in `tiup-cluster` and `tiup-dm` (#2295 #2312, @KanShiori)
+
+### Fixes
+
+- Make sure to signal tiproxy to stop in `tiup-playground` (#2318, @dveeden)
+- Fix "clone" operation may be missing packages without throwing an error in `tiup` (#2309 #2311, @nexustar)
+- Not cache the output of ss -tln command when checking wether components are started/stopped in `tiup-cluster` and `tiup-dm` (#2298, @KanShiori)
+- Fix tidb-dashboard listen_host in `tiup-cluster` (#2304, @nexustar)
+- Fix tiproxy metrics addr in `tiup-playground` (#2299, @xhebox)
+
+### Improvements
+
+- Upgrade go-sql-driver/mysql version to v1.7.1 (#2246, @srstack)
+- Use release version of TiProxy instead of nightly (#2305, @nexustar)
+- Use port to probe TiDB in `tiup-playground` (#2296, @KanShiori)
+- Add mcs name for pd log in `tiup-playground` (#2310, @HuSharp)
+- Not overwrite real error in retry function (#2290, @nexustar)
+- Fix alignment of output in `tiup-playground` (#2313, @dveeden)
+- ci improve (#2301 #2308 #2316, @nexustar @dveeden)
+
 ## [1.13.1] 2023-09-25
 
 ### Fixes

components/dm/ansible/import.go

@@ -273,6 +273,8 @@ func instancDeployDir(comp string, port int, hostDir string, globalDir string) s
 }
 
 // ImportFromAnsibleDir generate the metadata from ansible deployed cluster.
+//
+//revive:disable
 func (im *Importer) ImportFromAnsibleDir(ctx context.Context) (clusterName string, meta *spec.Metadata, err error) {
 	dir := im.dir
 	inventoryFileName := im.inventoryFileName

components/dm/spec/logic.go

@@ -68,6 +68,15 @@ func (c *DMMasterComponent) Role() string {
 	return ComponentDMMaster
 }
 
+// Source implements Component interface.
+func (c *DMMasterComponent) Source() string {
+	source := c.Topology.ComponentSources.Master
+	if source != "" {
+		return source
+	}
+	return ComponentDMMaster
+}
+
 // CalculateVersion implements the Component interface
 func (c *DMMasterComponent) CalculateVersion(clusterVersion string) string {
 	return clusterVersion
@@ -93,7 +102,7 @@ func (c *DMMasterComponent) Instances() []Instance {
 				ListenHost:   c.Topology.BaseTopo().GlobalOptions.ListenHost,
 				Port:         s.Port,
 				SSHP:         s.SSHPort,
-				Source:       s.GetSource(),
+				Source:       s.Source,
 
 				Ports: []int{
 					s.Port,
@@ -283,6 +292,15 @@ func (c *DMWorkerComponent) Role() string {
 	return ComponentDMWorker
 }
 
+// Source implements Component interface.
+func (c *DMWorkerComponent) Source() string {
+	source := c.Topology.ComponentSources.Worker
+	if source != "" {
+		return source
+	}
+	return ComponentDMWorker
+}
+
 // CalculateVersion implements the Component interface
 func (c *DMWorkerComponent) CalculateVersion(clusterVersion string) string {
 	return clusterVersion
@@ -308,7 +326,7 @@ func (c *DMWorkerComponent) Instances() []Instance {
 				ListenHost:   c.Topology.BaseTopo().GlobalOptions.ListenHost,
 				Port:         s.Port,
 				SSHP:         s.SSHPort,
-				Source:       s.GetSource(),
+				Source:       s.Source,
 
 				Ports: []int{
 					s.Port,

components/dm/spec/topology_dm.go

@@ -36,9 +36,10 @@ const (
 )
 
 var (
-	globalOptionTypeName  = reflect.TypeOf(GlobalOptions{}).Name()
-	monitorOptionTypeName = reflect.TypeOf(MonitoredOptions{}).Name()
-	serverConfigsTypeName = reflect.TypeOf(DMServerConfigs{}).Name()
+	globalOptionTypeName     = reflect.TypeOf(GlobalOptions{}).Name()
+	monitorOptionTypeName    = reflect.TypeOf(MonitoredOptions{}).Name()
+	serverConfigsTypeName    = reflect.TypeOf(DMServerConfigs{}).Name()
+	componentSourcesTypeName = reflect.TypeOf(ComponentSources{}).Name()
 )
 
 func setDefaultDir(parent, role, port string, field reflect.Value) {
@@ -69,7 +70,7 @@ func isSkipField(field reflect.Value) bool {
 		field = field.Elem()
 	}
 	tp := field.Type().Name()
-	return tp == globalOptionTypeName || tp == monitorOptionTypeName || tp == serverConfigsTypeName
+	return tp == globalOptionTypeName || tp == monitorOptionTypeName || tp == serverConfigsTypeName || tp == componentSourcesTypeName
 }
 
 type (
@@ -95,10 +96,17 @@ type (
 		Grafana map[string]string `yaml:"grafana"`
 	}
 
+	// ComponentSources represents the source of components
+	ComponentSources struct {
+		Master string `yaml:"master,omitempty"`
+		Worker string `yaml:"worker,omitempty"`
+	}
+
 	// Specification represents the specification of topology.yaml
 	Specification struct {
 		GlobalOptions    GlobalOptions            `yaml:"global,omitempty" validate:"global:editable"`
 		MonitoredOptions *MonitoredOptions        `yaml:"monitored,omitempty" validate:"monitored:editable"`
+		ComponentSources ComponentSources         `yaml:"component_sources,omitempty" validate:"component_sources:editable"`
 		ServerConfigs    DMServerConfigs          `yaml:"server_configs,omitempty" validate:"server_configs:ignore"`
 		Masters          []*MasterSpec            `yaml:"master_servers"`
 		Workers          []*WorkerSpec            `yaml:"worker_servers"`
@@ -203,14 +211,6 @@ func (s *MasterSpec) GetAdvertisePeerURL(enableTLS bool) string {
 	return fmt.Sprintf("%s://%s", scheme, utils.JoinHostPort(s.Host, s.PeerPort))
 }
 
-// GetSource returns source to download the component
-func (s *MasterSpec) GetSource() string {
-	if s.Source == "" {
-		return ComponentDMMaster
-	}
-	return s.Source
-}
-
 // WorkerSpec represents the Master topology specification in topology.yaml
 type WorkerSpec struct {
 	Host           string `yaml:"host"`
@@ -282,14 +282,6 @@ func (s *WorkerSpec) IgnoreMonitorAgent() bool {
 	return s.IgnoreExporter
 }
 
-// GetSource returns source to download the component
-func (s *WorkerSpec) GetSource() string {
-	if s.Source == "" {
-		return ComponentDMWorker
-	}
-	return s.Source
-}
-
 // UnmarshalYAML sets default values when unmarshaling the topology file
 func (s *Specification) UnmarshalYAML(unmarshal func(any) error) error {
 	type topology Specification

components/playground/playground.go

@@ -1304,7 +1304,12 @@ func (p *Playground) renderSDFile() error {
 	cid2targets := make(map[string]instance.MetricAddr)
 
 	_ = p.WalkInstances(func(cid string, inst instance.Instance) error {
-		cid2targets[cid] = inst.MetricAddr()
+		v := inst.MetricAddr()
+		t, ok := cid2targets[cid]
+		if ok {
+			v.Targets = append(v.Targets, t.Targets...)
+		}
+		cid2targets[cid] = v
 		return nil
 	})
 

doc/rfcs/0001-separate-component-version-in-cluster.md

@@ -0,0 +1,55 @@
+# Separate Component Version in Cluster
+
+## Summary
+
+Add a version field to each component on topology file. Allow user to upgrade single component to given version.
+
+## Motivation
+
+- New component TiDB Dashboard need to deployed with TiDB but not release with TiDB. Need a way to specify Dashboard version when deployment and upgrade.
+- User want to upgrade node_exporter without upgrade TiDB cluster
+- Maybe it could replace tiup cluster patch function to provide patch version of single component
+
+## Detailed design
+
+1. ~~add "latest" alias to tiup download function.It cloud be used to download latest release package of component.~~ just use "" as latest alias
+
+2. add ComponentVersions struct to topology
+
+```
+// ComponentVersions represents the versions of components
+ComponentVersions struct {
+	TiDB         string `yaml:"tidb,omitempty"`
+	TiKV         string `yaml:"tikv,omitempty"`
+	TiFlash      string `yaml:"tiflash,omitempty"`
+	PD           string `yaml:"pd,omitempty"`
+	Dashboard    string `yaml:"tidb_dashboard,omitempty"`
+	Pump         string `yaml:"pump,omitempty"`
+	Drainer      string `yaml:"drainer,omitempty"`
+	CDC          string `yaml:"cdc,omitempty"`
+	TiKVCDC      string `yaml:"kvcdc,omitempty"`
+	TiProxy      string `yaml:"tiproxy,omitempty"`
+	Prometheus   string `yaml:"prometheus,omitempty"`
+	Grafana      string `yaml:"grafana,omitempty"`
+	AlertManager string `yaml:"alertmanager,omitempty"`
+}
+```
+
+3. add node_exporter and blackbox_exporter version to MonitoredOptions struct
+
+```
+MonitoredOptions struct {
+    ...
+	NodeExporterVersion     string    `yaml:"node_exporter_version,omitempty" default:"latest"`
+	BlackboxExporterVersion string    `yaml:"blackbox_exporter_version,omitempty" default:"latest"`
+    ...
+	}
+```
+
+4. Add CalculateVersion for each component struct. It returns cluster version if component version is not set for components like pd, tikv. It returns "" by default for components like alertmanager
+
+5. Add flags to specify component versions
+
+6. Merge ComponentVersion struct when scale-out
+
+7. apply those version to deploy,scale-out and upgrade command

pkg/cluster/ansible/inventory.go

@@ -133,6 +133,7 @@ func ParseAndImportInventory(ctx context.Context, dir, ansCfgFile string, clsMet
 	return defaults.Set(clsMeta)
 }
 
+//revive:disable
 func parseGroupVars(ctx context.Context, dir, ansCfgFile string, clsMeta *spec.ClusterMeta, inv *aini.InventoryData) error {
 	logger := ctx.Value(logprinter.ContextKeyLogger).(*logprinter.Logger)
 

pkg/cluster/ansible/service.go

@@ -34,6 +34,8 @@ var (
 )
 
 // parseDirs sets values of directories of component
+//
+//revive:disable
 func parseDirs(ctx context.Context, user string, ins spec.InstanceSpec, sshTimeout uint64, sshType executor.SSHType) (spec.InstanceSpec, error) {
 	logger := ctx.Value(logprinter.ContextKeyLogger).(*logprinter.Logger)
 	hostName, sshPort := ins.SSH()

pkg/cluster/operation/scale_in.go

@@ -83,6 +83,8 @@ func ScaleIn(
 }
 
 // ScaleInCluster scales in the cluster
+//
+//revive:disable
 func ScaleInCluster(
 	ctx context.Context,
 	cluster *spec.Specification,

pkg/cluster/spec/alertmanager.go

@@ -98,6 +98,11 @@ func (c *AlertManagerComponent) Role() string {
 	return RoleMonitor
 }
 
+// Source implements Component interface.
+func (c *AlertManagerComponent) Source() string {
+	return ComponentAlertmanager
+}
+
 // CalculateVersion implements the Component interface
 func (c *AlertManagerComponent) CalculateVersion(_ string) string {
 	// always not follow cluster version, use ""(latest) by default

pkg/cluster/spec/cdc.go

@@ -92,14 +92,6 @@ func (s *CDCSpec) IgnoreMonitorAgent() bool {
 	return s.IgnoreExporter
 }
 
-// GetSource returns source to download the component
-func (s *CDCSpec) GetSource() string {
-	if s.Source == "" {
-		return ComponentCDC
-	}
-	return s.Source
-}
-
 // CDCComponent represents CDC component.
 type CDCComponent struct{ Topology *Specification }
 
@@ -113,6 +105,15 @@ func (c *CDCComponent) Role() string {
 	return ComponentCDC
 }
 
+// Source implements Component interface.
+func (c *CDCComponent) Source() string {
+	source := c.Topology.ComponentSources.CDC
+	if source != "" {
+		return source
+	}
+	return ComponentCDC
+}
+
 // CalculateVersion implements the Component interface
 func (c *CDCComponent) CalculateVersion(clusterVersion string) string {
 	version := c.Topology.ComponentVersions.CDC
@@ -140,7 +141,7 @@ func (c *CDCComponent) Instances() []Instance {
 			ListenHost:   c.Topology.BaseTopo().GlobalOptions.ListenHost,
 			Port:         s.Port,
 			SSHP:         s.SSHPort,
-			Source:       s.GetSource(),
+			Source:       s.Source,
 			NumaNode:     s.NumaNode,
 			NumaCores:    "",
 

pkg/cluster/spec/dashboard.go

@@ -94,14 +94,6 @@ func (s *DashboardSpec) IgnoreMonitorAgent() bool {
 	return s.IgnoreExporter
 }
 
-// GetSource returns source to download the component
-func (s *DashboardSpec) GetSource() string {
-	if s.Source == "" {
-		return ComponentDashboard
-	}
-	return s.Source
-}
-
 // DashboardComponent represents Drainer component.
 type DashboardComponent struct{ Topology *Specification }
 
@@ -115,6 +107,15 @@ func (c *DashboardComponent) Role() string {
 	return ComponentDashboard
 }
 
+// Source implements Component interface.
+func (c *DashboardComponent) Source() string {
+	source := c.Topology.ComponentSources.Dashboard
+	if source != "" {
+		return source
+	}
+	return ComponentDashboard
+}
+
 // CalculateVersion implements the Component interface
 func (c *DashboardComponent) CalculateVersion(clusterVersion string) string {
 	version := c.Topology.ComponentVersions.Dashboard