diff --git a/docs/api-references/docs.md b/docs/api-references/docs.md
index fe93a06fef6..c40614dd118 100644
--- a/docs/api-references/docs.md
+++ b/docs/api-references/docs.md
@@ -18040,7 +18040,7 @@ string
|
(Optional)
EvictLeaderTimeout indicates the timeout to evict tikv leader, in the format of Go Duration.
-Defaults to 3m
+Defaults to 10m
|
diff --git a/go.mod b/go.mod
index ced3d9913bf..49baabfcc35 100644
--- a/go.mod
+++ b/go.mod
@@ -56,8 +56,10 @@ require (
github.com/pingcap/errors v0.11.0
github.com/pingcap/kvproto v0.0.0-20200927054727-1290113160f0
github.com/pingcap/tidb v2.1.0-beta+incompatible
- github.com/prometheus/client_golang v0.9.2
- github.com/prometheus/common v0.0.0-20181126121408-4724e9255275
+ github.com/prometheus/client_golang v1.0.0
+ github.com/prometheus/client_model v0.1.0
+ github.com/prometheus/common v0.7.0
+ github.com/prometheus/prom2json v1.3.0
github.com/prometheus/prometheus v1.8.2
github.com/robfig/cron v1.1.0
github.com/sirupsen/logrus v1.5.0
@@ -152,3 +154,5 @@ replace k8s.io/node-api => k8s.io/node-api v0.0.0-20190918163711-2299658ad911
replace github.com/uber-go/atomic => go.uber.org/atomic v1.5.0
replace github.com/Azure/go-autorest => github.com/Azure/go-autorest v12.2.0+incompatible
+
+replace github.com/prometheus/client_golang => github.com/prometheus/client_golang v0.9.4
diff --git a/go.sum b/go.sum
index 95a913e4220..7afb3fe27c4 100644
--- a/go.sum
+++ b/go.sum
@@ -67,6 +67,11 @@ github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
github.com/Rican7/retry v0.1.0/go.mod h1:FgOROf8P5bebcC1DS0PdOQiqGUridaZvikzUmkFW6gg=
github.com/ajg/form v0.0.0-20160822230020-523a5da1a92f/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY=
+github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q=
github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
@@ -83,6 +88,8 @@ github.com/bazelbuild/bazel-gazelle v0.0.0-20181012220611-c728ce9f663e/go.mod h1
github.com/bazelbuild/buildtools v0.0.0-20180226164855-80c7f0d45d7e/go.mod h1:5JP0TXzWDHXv8qvxRC4InIazwdyDseBDbzESUMKk1yU=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0=
+github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/bifurcation/mint v0.0.0-20180715133206-93c51c6ce115/go.mod h1:zVt7zX3K/aDCk9Tj+VM7YymsX66ERvzCJzw8rFCX2JU=
github.com/blang/semver v3.5.0+incompatible h1:CGxCgetQ64DKk7rdZ++Vfnb1+ogGNnB17OJKJXD2Cfs=
github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
@@ -193,6 +200,10 @@ github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0
github.com/go-acme/lego v2.5.0+incompatible/go.mod h1:yzMNe9CasVUhkquNvti5nAtPmG94USbYxYrZfTkIn0M=
github.com/go-bindata/go-bindata v3.1.1+incompatible/go.mod h1:xK8Dsgwmeed+BBsSy2XTopBn/8uK2HWuGSnA11C3Joo=
github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
+github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
+github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
github.com/go-logr/logr v0.1.0 h1:M1Tv3VzNlEHg6uyACnRdtrploV2P7wZqH8BoQMtz0cg=
github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
github.com/go-logr/zapr v0.1.0 h1:h+WVe9j6HAA01niTJPA/kKH0i7e0rLZBCwauQFcRE54=
@@ -243,6 +254,7 @@ github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG
github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/gobuffalo/buffalo v0.12.8-0.20181004233540-fac9bb505aa8/go.mod h1:sLyT7/dceRXJUxSsE813JTQtA3Eb1vjxWfo/N//vXIY=
github.com/gobuffalo/buffalo v0.13.0/go.mod h1:Mjn1Ba9wpIbpbrD+lIDMy99pQ0H0LiddMIIDGse7qT4=
github.com/gobuffalo/buffalo-plugins v1.0.2/go.mod h1:pOp/uF7X3IShFHyobahTkTLZaeUXwb0GrUTb9ngJWTs=
@@ -363,6 +375,7 @@ github.com/gobuffalo/x v0.0.0-20181003152136-452098b06085/go.mod h1:WevpGD+5YOre
github.com/gobuffalo/x v0.0.0-20181007152206-913e47c59ca7/go.mod h1:9rDPXaB3kXdKWzMc4odGQQdG2e2DIEmANy5aSJ9yesY=
github.com/godbus/dbus v4.1.0+incompatible/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw=
github.com/gofrs/uuid v3.1.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
+github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
@@ -494,6 +507,7 @@ github.com/juju/loggo v0.0.0-20180524022052-584905176618 h1:MK144iBQF9hTSwBW/9eJ
github.com/juju/loggo v0.0.0-20180524022052-584905176618/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U=
github.com/juju/testing v0.0.0-20180920084828-472a3e8b2073 h1:WQM1NildKThwdP7qWrNAFGzp4ijNLw8RlgENkaI4MJs=
github.com/juju/testing v0.0.0-20180920084828-472a3e8b2073/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA=
+github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/karrick/godirwalk v1.7.5/go.mod h1:2c9FRhkDxdIbgkOnCEvnSWs71Bhugbl46shStcFDJ34=
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
@@ -503,6 +517,9 @@ github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgo
github.com/konsorten/go-windows-terminal-sequences v0.0.0-20180402223658-b729f2633dfe/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
+github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@@ -599,6 +616,7 @@ github.com/mrunalp/fileutils v0.0.0-20160930181131-4ee1cc9a8058/go.mod h1:x8F1gn
github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d h1:7PxY7LVfSZm7PEeBTyK1rj1gABdCO2mbri6GKO1cMDs=
github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/mvdan/xurls v1.1.0/go.mod h1:tQlNn3BED8bE/15hnSL2HLkDeLWpNPAwtw7wkEq44oU=
+github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0=
github.com/naoina/toml v0.1.1/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E=
@@ -657,14 +675,25 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA=
github.com/pquerna/ffjson v0.0.0-20180717144149-af8b230fcd20/go.mod h1:YARuvh7BUWHNhzDq2OM5tzR2RiCcN2D7sapiKyCel/M=
-github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740=
-github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM=
+github.com/prometheus/client_golang v0.9.4 h1:Y8E/JaaPbmFSW2V81Ab/d8yZFYQQGbni1b1jPcG9Y6A=
+github.com/prometheus/client_golang v0.9.4/go.mod h1:oCXIBxdI62A4cR6aTRJCgetEjecSIYzOEaeAn4iYEpM=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.1.0 h1:ElTg5tNp4DqfV7UQjDqv2+RJlNzsDtvNAWccbItceIE=
+github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 h1:PnBWHBf+6L0jOqq0gIVUe6Yk0/QMZ640k6NvkxcBf+8=
github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
+github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.7.0 h1:L+1lyG48J1zAQXA3RBX/nG/B3gjlHq0zTt2tlbJLyCY=
+github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA=
+github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nLJdBg+pBmGgkJlSaKC2KaQmTCk1XDtE=
github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs=
+github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/prom2json v1.3.0 h1:BlqrtbT9lLH3ZsOVhXPsHzFrApCTKRifB7gjJuypu6Y=
+github.com/prometheus/prom2json v1.3.0/go.mod h1:rMN7m0ApCowcoDlypBHlkNbp5eJQf/+1isKykIP5ZnM=
github.com/prometheus/prometheus v1.8.2 h1:PAL466mnJw1VolZPm1OarpdUpqukUy/eX4tagia17DM=
github.com/prometheus/prometheus v1.8.2/go.mod h1:oAIUtOny2rjMX0OWN5vPR5/q/twIROJvdqnQKDdil/s=
github.com/quobyte/api v0.1.2/go.mod h1:jL7lIHrmqQ7yh05OJ+eEEdHr0u/kmT1Ff9iHd+4H6VI=
@@ -737,6 +766,7 @@ github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoH
github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/syndtr/gocapability v0.0.0-20160928074757-e7cb7fa329f4/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
@@ -837,7 +867,6 @@ golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73r
golang.org/x/net v0.0.0-20181102091132-c10e9556a7bc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -889,6 +918,7 @@ golang.org/x/sys v0.0.0-20181026064943-731415f00dce/go.mod h1:STP8DvDyc/dI5b8T5h
golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181106135930-3a76605856fd/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190124100055-b90733256f2e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -905,6 +935,8 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190620070143-6f217b454f45 h1:Dl2hc890lrizvUppGbRWhnIh2f8jOTCQpY5IKWRS0oM=
golang.org/x/sys v0.0.0-20190620070143-6f217b454f45/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191220142924-d4481acd189f h1:68K/z8GLUxV76xGSqwTWw2gyk/jwn79LUL43rES2g8o=
+golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20171227012246-e19ae1496984/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -982,6 +1014,7 @@ google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac
google.golang.org/grpc v1.24.0 h1:vb/1TCsVn3DcJlQ0Gs1yB1pKI6Do2/QNwxdKqmc/b0s=
google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA=
gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
diff --git a/pkg/apis/pingcap/v1alpha1/openapi_generated.go b/pkg/apis/pingcap/v1alpha1/openapi_generated.go
index 4bbb019a846..7a8aef4ded0 100644
--- a/pkg/apis/pingcap/v1alpha1/openapi_generated.go
+++ b/pkg/apis/pingcap/v1alpha1/openapi_generated.go
@@ -9785,7 +9785,7 @@ func schema_pkg_apis_pingcap_v1alpha1_TiKVSpec(ref common.ReferenceCallback) com
},
"evictLeaderTimeout": {
SchemaProps: spec.SchemaProps{
- Description: "EvictLeaderTimeout indicates the timeout to evict tikv leader, in the format of Go Duration. Defaults to 3m",
+ Description: "EvictLeaderTimeout indicates the timeout to evict tikv leader, in the format of Go Duration. Defaults to 10m",
Type: []string{"string"},
Format: "",
},
diff --git a/pkg/apis/pingcap/v1alpha1/tidbcluster.go b/pkg/apis/pingcap/v1alpha1/tidbcluster.go
index a4cde740e2e..fb2036a38bd 100644
--- a/pkg/apis/pingcap/v1alpha1/tidbcluster.go
+++ b/pkg/apis/pingcap/v1alpha1/tidbcluster.go
@@ -35,7 +35,7 @@ const (
defaultSeparateRaftLog = false
defaultEnablePVReclaim = false
// defaultEvictLeaderTimeout is the timeout limit of evict leader
- defaultEvictLeaderTimeout = 3 * time.Minute
+ defaultEvictLeaderTimeout = 10 * time.Minute
)
var (
diff --git a/pkg/apis/pingcap/v1alpha1/types.go b/pkg/apis/pingcap/v1alpha1/types.go
index 618582dc08d..bdb9964ac1e 100644
--- a/pkg/apis/pingcap/v1alpha1/types.go
+++ b/pkg/apis/pingcap/v1alpha1/types.go
@@ -461,7 +461,7 @@ type TiKVSpec struct {
MountClusterClientSecret *bool `json:"mountClusterClientSecret,omitempty"`
// EvictLeaderTimeout indicates the timeout to evict tikv leader, in the format of Go Duration.
- // Defaults to 3m
+ // Defaults to 10m
// +optional
EvictLeaderTimeout *string `json:"evictLeaderTimeout,omitempty"`
diff --git a/pkg/controller/dependences.go b/pkg/controller/dependences.go
index 78ed79f7713..dcaf9100529 100644
--- a/pkg/controller/dependences.go
+++ b/pkg/controller/dependences.go
@@ -25,6 +25,7 @@ import (
"github.com/pingcap/tidb-operator/pkg/dmapi"
"github.com/pingcap/tidb-operator/pkg/pdapi"
"github.com/pingcap/tidb-operator/pkg/scheme"
+ "github.com/pingcap/tidb-operator/pkg/tikvapi"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
kubeinformers "k8s.io/client-go/informers"
@@ -140,6 +141,7 @@ type Controls struct {
PodControl PodControlInterface
TypedControl TypedControlInterface
PDControl pdapi.PDControlInterface
+ TiKVControl tikvapi.TiKVControlInterface
DMMasterControl dmapi.MasterControlInterface
TiDBClusterControl TidbClusterControlInterface
DMClusterControl DMClusterControlInterface
@@ -200,6 +202,7 @@ func newRealControls(
// Shared variables to construct `Dependencies` and some of its fields
var (
pdControl = pdapi.NewDefaultPDControl(kubeClientset)
+ tikvControl = tikvapi.NewDefaultTiKVControl(kubeClientset)
masterControl = dmapi.NewDefaultMasterControl(kubeClientset)
genericCtrl = NewRealGenericControl(genericCli, recorder)
tidbClusterLister = informerFactory.Pingcap().V1alpha1().TidbClusters().Lister()
@@ -222,6 +225,7 @@ func newRealControls(
PodControl: NewRealPodControl(kubeClientset, pdControl, podLister, recorder),
TypedControl: NewTypedControl(genericCtrl),
PDControl: pdControl,
+ TiKVControl: tikvControl,
DMMasterControl: masterControl,
TiDBClusterControl: NewRealTidbClusterControl(clientset, tidbClusterLister, recorder),
DMClusterControl: NewRealDMClusterControl(clientset, dmClusterLister, recorder),
@@ -331,6 +335,7 @@ func newFakeControl(kubeClientset kubernetes.Interface, informerFactory informer
PodControl: NewFakePodControl(kubeInformerFactory.Core().V1().Pods()),
TypedControl: NewTypedControl(genericCtrl),
PDControl: pdapi.NewFakePDControl(kubeClientset),
+ TiKVControl: tikvapi.NewFakeTiKVControl(kubeClientset),
DMMasterControl: dmapi.NewFakeMasterControl(kubeClientset),
TiDBClusterControl: NewFakeTidbClusterControl(informerFactory.Pingcap().V1alpha1().TidbClusters()),
CDCControl: NewDefaultTiCDCControl(kubeClientset), // TODO: no fake control?
diff --git a/pkg/controller/tikv_control.go b/pkg/controller/tikv_control.go
new file mode 100644
index 00000000000..1f1de2cbae6
--- /dev/null
+++ b/pkg/controller/tikv_control.go
@@ -0,0 +1,26 @@
+// Copyright 2021 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package controller
+
+import (
+ "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1"
+ "github.com/pingcap/tidb-operator/pkg/tikvapi"
+)
+
+// NewFakeTiKVClient creates a fake tikvclient that is set as the tikv client
+func NewFakeTiKVClient(tikvControl *tikvapi.FakeTiKVControl, tc *v1alpha1.TidbCluster, podName string) *tikvapi.FakeTiKVClient {
+ tikvClient := tikvapi.NewFakeTiKVClient()
+ tikvControl.SetTiKVPodClient(tc.Namespace, tc.Name, podName, tikvClient)
+ return tikvClient
+}
diff --git a/pkg/manager/member/tikv_upgrader.go b/pkg/manager/member/tikv_upgrader.go
index d462f544077..cefc55dc63e 100644
--- a/pkg/manager/member/tikv_upgrader.go
+++ b/pkg/manager/member/tikv_upgrader.go
@@ -169,7 +169,7 @@ func (u *tikvUpgrader) upgradeTiKVPod(tc *v1alpha1.TidbCluster, ordinal int32, n
return u.beginEvictLeader(tc, storeID, upgradePod)
}
- if u.readyToUpgrade(upgradePod, store, tc.TiKVEvictLeaderTimeout()) {
+ if u.readyToUpgrade(upgradePod, tc) {
setUpgradePartition(newSet, ordinal)
return nil
}
@@ -181,10 +181,22 @@ func (u *tikvUpgrader) upgradeTiKVPod(tc *v1alpha1.TidbCluster, ordinal int32, n
return controller.RequeueErrorf("tidbcluster: [%s/%s] no store status found for tikv pod: [%s]", ns, tcName, upgradePodName)
}
-func (u *tikvUpgrader) readyToUpgrade(upgradePod *corev1.Pod, store v1alpha1.TiKVStore, evictLeaderTimeout time.Duration) bool {
- if store.LeaderCount == 0 {
+func (u *tikvUpgrader) readyToUpgrade(upgradePod *corev1.Pod, tc *v1alpha1.TidbCluster) bool {
+ evictLeaderTimeout := tc.TiKVEvictLeaderTimeout()
+ tlsEnabled := tc.IsTLSClusterEnabled()
+ leaderCount, err := u.deps.TiKVControl.GetTiKVPodClient(tc.Namespace, tc.Name, upgradePod.Name, tlsEnabled).GetLeaderCount()
+ if err != nil {
+ klog.Warningf("Fail to get region leader count for Pod %s/%s, error: %v", upgradePod.Namespace, upgradePod.Name, err)
+ return false
+ }
+
+ if leaderCount == 0 {
+ klog.Infof("Region leader count is 0 for Pod %s/%s", upgradePod.Namespace, upgradePod.Name)
return true
}
+
+ klog.Infof("Region leader count is %d for Pod %s/%s", leaderCount, upgradePod.Namespace, upgradePod.Name)
+
if evictLeaderBeginTimeStr, evicting := upgradePod.Annotations[EvictLeaderBeginTime]; evicting {
evictLeaderBeginTime, err := time.Parse(time.RFC3339, evictLeaderBeginTimeStr)
if err != nil {
@@ -192,6 +204,7 @@ func (u *tikvUpgrader) readyToUpgrade(upgradePod *corev1.Pod, store v1alpha1.TiK
return false
}
if time.Now().After(evictLeaderBeginTime.Add(evictLeaderTimeout)) {
+ klog.Infof("Evict region leader timeout (threshold: %v) for Pod %s/%s", evictLeaderTimeout, upgradePod.Namespace, upgradePod.Name)
return true
}
}
diff --git a/pkg/manager/member/tikv_upgrader_test.go b/pkg/manager/member/tikv_upgrader_test.go
index 2cc93535184..173db3ac8d6 100644
--- a/pkg/manager/member/tikv_upgrader_test.go
+++ b/pkg/manager/member/tikv_upgrader_test.go
@@ -24,6 +24,7 @@ import (
"github.com/pingcap/tidb-operator/pkg/controller"
"github.com/pingcap/tidb-operator/pkg/label"
"github.com/pingcap/tidb-operator/pkg/pdapi"
+ "github.com/pingcap/tidb-operator/pkg/tikvapi"
apps "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -47,6 +48,9 @@ func TestTiKVUpgraderUpgrade(t *testing.T) {
changePods func([]*corev1.Pod)
beginEvictLeaderErr bool
endEvictLeaderErr bool
+ getLeaderCountErr bool
+ leaderCount int
+ podName string
updatePodErr bool
errExpectFn func(*GomegaWithT, error)
expectFn func(*GomegaWithT, *v1alpha1.TidbCluster, *apps.StatefulSet, map[string]*corev1.Pod)
@@ -54,7 +58,7 @@ func TestTiKVUpgraderUpgrade(t *testing.T) {
testFn := func(test *testcase, t *testing.T) {
t.Log(test.name)
- upgrader, pdControl, podControl, podInformer := newTiKVUpgrader()
+ upgrader, pdControl, podControl, podInformer, tikvControl := newTiKVUpgrader()
tc := newTidbClusterForTiKVUpgrader()
if test.changeFn != nil {
@@ -77,6 +81,20 @@ func TestTiKVUpgraderUpgrade(t *testing.T) {
return nil, nil
})
}
+
+ tikvClient := controller.NewFakeTiKVClient(tikvControl, tc, "upgrader-tikv-2")
+ if len(test.podName) > 0 {
+ tikvClient = controller.NewFakeTiKVClient(tikvControl, tc, test.podName)
+ }
+ if test.getLeaderCountErr {
+ tikvClient.AddReaction(tikvapi.GetLeaderCountActionType, func(action *tikvapi.Action) (interface{}, error) {
+ return 0, fmt.Errorf("failed to begin evict leader")
+ })
+ } else {
+ tikvClient.AddReaction(tikvapi.GetLeaderCountActionType, func(action *tikvapi.Action) (interface{}, error) {
+ return test.leaderCount, nil
+ })
+ }
if test.endEvictLeaderErr {
pdClient.AddReaction(pdapi.EndEvictLeaderActionType, func(action *pdapi.Action) (interface{}, error) {
return nil, fmt.Errorf("failed to end evict leader")
@@ -216,6 +234,7 @@ func TestTiKVUpgraderUpgrade(t *testing.T) {
beginEvictLeaderErr: false,
endEvictLeaderErr: false,
updatePodErr: false,
+ podName: "upgrader-tikv-1",
errExpectFn: func(g *GomegaWithT, err error) {
g.Expect(err).NotTo(HaveOccurred())
},
@@ -385,6 +404,44 @@ func TestTiKVUpgraderUpgrade(t *testing.T) {
beginEvictLeaderErr: false,
endEvictLeaderErr: false,
updatePodErr: false,
+ podName: "upgrader-tikv-1",
+ leaderCount: 10,
+ errExpectFn: func(g *GomegaWithT, err error) {
+ g.Expect(err).To(HaveOccurred())
+ g.Expect(err.Error()).To(Equal("tidbcluster: [default/upgrader]'s tikv pod: [upgrader-tikv-1] is evicting leader"))
+ },
+ expectFn: func(g *GomegaWithT, tc *v1alpha1.TidbCluster, newSet *apps.StatefulSet, pods map[string]*corev1.Pod) {
+ g.Expect(*newSet.Spec.UpdateStrategy.RollingUpdate.Partition).To(Equal(int32(2)))
+ },
+ },
+ {
+ name: "get leader count error",
+ getLeaderCountErr: true,
+ changeFn: func(tc *v1alpha1.TidbCluster) {
+ tc.Status.PD.Phase = v1alpha1.NormalPhase
+ tc.Status.TiKV.Phase = v1alpha1.UpgradePhase
+ tc.Status.TiKV.Synced = true
+ tc.Status.TiKV.StatefulSet.CurrentReplicas = 2
+ tc.Status.TiKV.StatefulSet.UpdatedReplicas = 1
+ },
+ changeOldSet: func(oldSet *apps.StatefulSet) {
+ SetStatefulSetLastAppliedConfigAnnotation(oldSet)
+ oldSet.Status.CurrentReplicas = 2
+ oldSet.Status.UpdatedReplicas = 1
+ oldSet.Spec.UpdateStrategy.RollingUpdate.Partition = pointer.Int32Ptr(2)
+ },
+ changePods: func(pods []*corev1.Pod) {
+ for _, pod := range pods {
+ if pod.GetName() == TikvPodName(upgradeTcName, 1) {
+ pod.Annotations = map[string]string{EvictLeaderBeginTime: time.Now().Format(time.RFC3339)}
+ }
+ }
+ },
+ beginEvictLeaderErr: false,
+ endEvictLeaderErr: false,
+ updatePodErr: false,
+ podName: "upgrader-tikv-1",
+ leaderCount: 10,
errExpectFn: func(g *GomegaWithT, err error) {
g.Expect(err).To(HaveOccurred())
g.Expect(err.Error()).To(Equal("tidbcluster: [default/upgrader]'s tikv pod: [upgrader-tikv-1] is evicting leader"))
@@ -440,13 +497,15 @@ func TestTiKVUpgraderUpgrade(t *testing.T) {
changePods: func(pods []*corev1.Pod) {
for _, pod := range pods {
if pod.GetName() == TikvPodName(upgradeTcName, 1) {
- pod.Annotations = map[string]string{EvictLeaderBeginTime: time.Now().Add(-5 * time.Minute).Format(time.RFC3339)}
+ pod.Annotations = map[string]string{EvictLeaderBeginTime: time.Now().Add(-15 * time.Minute).Format(time.RFC3339)}
}
}
},
beginEvictLeaderErr: false,
endEvictLeaderErr: false,
updatePodErr: false,
+ podName: "upgrader-tikv-1",
+ leaderCount: 10,
errExpectFn: func(g *GomegaWithT, err error) {
g.Expect(err).NotTo(HaveOccurred())
},
@@ -551,12 +610,13 @@ func TestTiKVUpgraderUpgrade(t *testing.T) {
}
}
-func newTiKVUpgrader() (TiKVUpgrader, *pdapi.FakePDControl, *controller.FakePodControl, podinformers.PodInformer) {
+func newTiKVUpgrader() (TiKVUpgrader, *pdapi.FakePDControl, *controller.FakePodControl, podinformers.PodInformer, *tikvapi.FakeTiKVControl) {
fakeDeps := controller.NewFakeDependencies()
pdControl := fakeDeps.PDControl.(*pdapi.FakePDControl)
+ tikvControl := fakeDeps.TiKVControl.(*tikvapi.FakeTiKVControl)
podControl := fakeDeps.PodControl.(*controller.FakePodControl)
podInformer := fakeDeps.KubeInformerFactory.Core().V1().Pods()
- return &tikvUpgrader{deps: fakeDeps}, pdControl, podControl, podInformer
+ return &tikvUpgrader{deps: fakeDeps}, pdControl, podControl, podInformer, tikvControl
}
func newStatefulSetForTiKVUpgrader() *apps.StatefulSet {
diff --git a/pkg/tikvapi/fake_tikvapi.go b/pkg/tikvapi/fake_tikvapi.go
new file mode 100644
index 00000000000..7cde1087e61
--- /dev/null
+++ b/pkg/tikvapi/fake_tikvapi.go
@@ -0,0 +1,74 @@
+// Copyright 2021 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tikvapi
+
+import (
+ "fmt"
+)
+
+type ActionType string
+
+const (
+ GetLeaderCountActionType ActionType = "GetLeaderCount"
+)
+
+type NotFoundReaction struct {
+ actionType ActionType
+}
+
+func (nfr *NotFoundReaction) Error() string {
+ return fmt.Sprintf("not found %s reaction. Please add the reaction", nfr.actionType)
+}
+
+type Action struct {
+ ID uint64
+ Name string
+ Labels map[string]string
+}
+
+type Reaction func(action *Action) (interface{}, error)
+
+// FakeTiKVClient implements a fake version of TiKVClient.
+type FakeTiKVClient struct {
+ reactions map[ActionType]Reaction
+}
+
+func NewFakeTiKVClient() *FakeTiKVClient {
+ return &FakeTiKVClient{reactions: map[ActionType]Reaction{}}
+}
+
+func (c *FakeTiKVClient) AddReaction(actionType ActionType, reaction Reaction) {
+ c.reactions[actionType] = reaction
+}
+
+// fakeAPI is a small helper for fake API calls
+func (c *FakeTiKVClient) fakeAPI(actionType ActionType, action *Action) (interface{}, error) {
+ if reaction, ok := c.reactions[actionType]; ok {
+ result, err := reaction(action)
+ if err != nil {
+ return nil, err
+ }
+ return result, nil
+ }
+ return nil, &NotFoundReaction{actionType}
+}
+
+func (c *FakeTiKVClient) GetLeaderCount() (int, error) {
+ action := &Action{}
+ result, err := c.fakeAPI(GetLeaderCountActionType, action)
+ if err != nil {
+ return 0, err
+ }
+ return result.(int), nil
+}
diff --git a/pkg/tikvapi/tikv_control.go b/pkg/tikvapi/tikv_control.go
new file mode 100644
index 00000000000..439f13e7801
--- /dev/null
+++ b/pkg/tikvapi/tikv_control.go
@@ -0,0 +1,95 @@
+// Copyright 2021 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tikvapi
+
+import (
+ "crypto/tls"
+ "fmt"
+ "sync"
+
+ "github.com/pingcap/tidb-operator/pkg/pdapi"
+ "github.com/pingcap/tidb-operator/pkg/util"
+ "k8s.io/client-go/kubernetes"
+ "k8s.io/klog"
+)
+
+// TiKVControlInterface is an interface that knows how to manage and get client for TiKV
+type TiKVControlInterface interface {
+ // GetTiKVPodClient provides TiKVClient of the TiKV cluster.
+ GetTiKVPodClient(namespace string, tcName string, podName string, tlsEnabled bool) TiKVClient
+}
+
+// defaultTiKVControl is the default implementation of TiKVControlInterface.
+type defaultTiKVControl struct {
+ mutex sync.Mutex
+ kubeCli kubernetes.Interface
+ tikvClients map[string]TiKVClient
+}
+
+// NewDefaultTiKVControl returns a defaultTiKVControl instance
+func NewDefaultTiKVControl(kubeCli kubernetes.Interface) TiKVControlInterface {
+ return &defaultTiKVControl{kubeCli: kubeCli, tikvClients: map[string]TiKVClient{}}
+}
+
+func (tc *defaultTiKVControl) GetTiKVPodClient(namespace string, tcName string, podName string, tlsEnabled bool) TiKVClient {
+ tc.mutex.Lock()
+ defer tc.mutex.Unlock()
+
+ var tlsConfig *tls.Config
+ var err error
+ var scheme = "http"
+
+ if tlsEnabled {
+ scheme = "https"
+ tlsConfig, err = pdapi.GetTLSConfig(tc.kubeCli, pdapi.Namespace(namespace), tcName, util.ClusterClientTLSSecretName(tcName))
+ if err != nil {
+ klog.Errorf("Unable to get tls config for TiKV cluster %q, tikv client may not work: %v", tcName, err)
+ return NewTiKVClient(TiKVPodClientURL(namespace, tcName, podName, scheme), DefaultTimeout, tlsConfig, true)
+ }
+
+ return NewTiKVClient(TiKVPodClientURL(namespace, tcName, podName, scheme), DefaultTimeout, tlsConfig, true)
+ }
+
+ return NewTiKVClient(TiKVPodClientURL(namespace, tcName, podName, scheme), DefaultTimeout, tlsConfig, true)
+}
+
+func tikvPodClientKey(schema, namespace, clusterName, podName string) string {
+ return fmt.Sprintf("%s.%s.%s.%s", schema, clusterName, namespace, podName)
+}
+
+// TiKVPodClientURL builds the url of tikv pod client
+func TiKVPodClientURL(namespace, clusterName, podName, scheme string) string {
+ return fmt.Sprintf("%s://%s.%s-tikv-peer.%s:20180", scheme, podName, clusterName, namespace)
+}
+
+// FakeTiKVControl implements a fake version of TiKVControlInterface.
+type FakeTiKVControl struct {
+ defaultTiKVControl
+ tikvPodClients map[string]TiKVClient
+}
+
+func NewFakeTiKVControl(kubeCli kubernetes.Interface) *FakeTiKVControl {
+ return &FakeTiKVControl{
+ defaultTiKVControl: defaultTiKVControl{kubeCli: kubeCli, tikvClients: map[string]TiKVClient{}},
+ tikvPodClients: map[string]TiKVClient{},
+ }
+}
+
+func (ftc *FakeTiKVControl) SetTiKVPodClient(namespace, tcName, podName string, tikvPodClient TiKVClient) {
+ ftc.tikvPodClients[tikvPodClientKey("http", namespace, tcName, podName)] = tikvPodClient
+}
+
+func (ftc *FakeTiKVControl) GetTiKVPodClient(namespace, tcName, podName string, tlsEnabled bool) TiKVClient {
+ return ftc.tikvPodClients[tikvPodClientKey("http", namespace, tcName, podName)]
+}
diff --git a/pkg/tikvapi/tikvapi.go b/pkg/tikvapi/tikvapi.go
new file mode 100644
index 00000000000..77868c75d86
--- /dev/null
+++ b/pkg/tikvapi/tikvapi.go
@@ -0,0 +1,90 @@
+// Copyright 2021 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tikvapi
+
+import (
+ "crypto/tls"
+ "fmt"
+ "net"
+ "net/http"
+ "strconv"
+ "time"
+
+ dto "github.com/prometheus/client_model/go"
+ "github.com/prometheus/prom2json"
+ "k8s.io/klog"
+)
+
+const (
+ DefaultTimeout = 5 * time.Second
+ metricNameRegionCount = "tikv_raftstore_region_count"
+ labelNameLeaderCount = "leader"
+ metricsPrefix = "metrics"
+)
+
+// TiKVClient provides tikv server's api
+type TiKVClient interface {
+ GetLeaderCount() (int, error)
+}
+
+// tikvClient is default implementation of TiKVClient
+type tikvClient struct {
+ url string
+ httpClient *http.Client
+}
+
+// GetLeaderCount gets region leader count from the URL
+func (c *tikvClient) GetLeaderCount() (int, error) {
+ apiURL := fmt.Sprintf("%s/%s", c.url, metricsPrefix)
+ transport := c.httpClient.Transport
+ mfChan := make(chan *dto.MetricFamily, 1024)
+
+ go func() {
+ if err := prom2json.FetchMetricFamilies(apiURL, mfChan, transport); err != nil {
+ klog.Errorf("Fail to get region leader count from %s, error: %v", apiURL, err)
+ }
+ }()
+
+ for mf := range mfChan {
+ fm := prom2json.NewFamily(mf)
+ if fm.Name == metricNameRegionCount {
+ for _, m := range fm.Metrics {
+ if m, ok := m.(prom2json.Metric); ok && m.Labels["type"] == labelNameLeaderCount {
+ return strconv.Atoi(m.Value)
+ }
+ }
+ }
+ }
+
+ return 0, fmt.Errorf("metric %s{type=\"%s\"} not found for %s", metricNameRegionCount, labelNameLeaderCount, apiURL)
+}
+
+// NewTiKVClient returns a new TiKVClient
+func NewTiKVClient(url string, timeout time.Duration, tlsConfig *tls.Config, disableKeepalive bool) TiKVClient {
+ return &tikvClient{
+ url: url,
+ httpClient: &http.Client{
+ Timeout: timeout,
+ Transport: &http.Transport{
+ TLSClientConfig: tlsConfig,
+ DisableKeepAlives: disableKeepalive,
+ ResponseHeaderTimeout: 10 * time.Second,
+ TLSHandshakeTimeout: 10 * time.Second,
+ DialContext: (&net.Dialer{
+ Timeout: 10 * time.Second,
+ }).DialContext,
+ },
+ },
+ }
+}
diff --git a/tests/actions.go b/tests/actions.go
index 13ff3ae243b..b4869b74d0e 100644
--- a/tests/actions.go
+++ b/tests/actions.go
@@ -1014,7 +1014,7 @@ func (oa *operatorActions) CheckTidbClusterStatus(info *TidbClusterConfig) error
ns := info.Namespace
tcName := info.ClusterName
// TODO: remove redundant checks already in WaitForTidbClusterReady
- if err := wait.Poll(oa.pollInterval, 10*time.Minute, func() (bool, error) {
+ if err := wait.Poll(oa.pollInterval, 20*time.Minute, func() (bool, error) {
var tc *v1alpha1.TidbCluster
var err error
if tc, err = oa.cli.PingcapV1alpha1().TidbClusters(ns).Get(tcName, metav1.GetOptions{}); err != nil {
diff --git a/tests/pkg/fixture/fixture.go b/tests/pkg/fixture/fixture.go
index 8eed0f40e94..e80d688a524 100644
--- a/tests/pkg/fixture/fixture.go
+++ b/tests/pkg/fixture/fixture.go
@@ -126,6 +126,7 @@ func GetTidbCluster(ns, name, version string) *v1alpha1.TidbCluster {
ComponentSpec: v1alpha1.ComponentSpec{
Affinity: buildAffinity(name, ns, v1alpha1.TiKVMemberType),
},
+ EvictLeaderTimeout: pointer.StringPtr("3m"),
},
TiDB: &v1alpha1.TiDBSpec{