rebase on master

Signed-off-by: Wenqi Mou <[email protected]>

br/cmd/br/cmd.go

@@ -37,7 +37,7 @@ var (
 	tidbGlue        = gluetidb.New()
 	envLogToTermKey = "BR_LOG_TO_TERM"
 
-	filterOutSysAndMemTables = []string{
+	filterOutSysAndMemKeepAuthAndBind = []string{
 		"*.*",
 		fmt.Sprintf("!%s.*", utils.TemporaryDBName("*")),
 		"!mysql.*",

br/cmd/br/restore.go

@@ -186,7 +186,7 @@ func newFullRestoreCommand() *cobra.Command {
 			return runRestoreCommand(cmd, task.FullRestoreCmd)
 		},
 	}
-	task.DefineFilterFlags(command, filterOutSysAndMemTables, false)
+	task.DefineFilterFlags(command, filterOutSysAndMemKeepAuthAndBind, false)
 	task.DefineRestoreSnapshotFlags(command)
 	return command
 }
@@ -254,7 +254,7 @@ func newStreamRestoreCommand() *cobra.Command {
 			return runRestoreCommand(command, task.PointRestoreCmd)
 		},
 	}
-	task.DefineFilterFlags(command, filterOutSysAndMemTables, true)
+	task.DefineFilterFlags(command, filterOutSysAndMemKeepAuthAndBind, true)
 	task.DefineStreamRestoreFlags(command)
 	return command
 }

br/pkg/backup/client.go

@@ -692,7 +692,7 @@ func (bc *Client) SetApiVersion(v kvrpcpb.APIVersion) {
 	bc.apiVersion = v
 }
 
-// Client.BuildBackupRangeAndSchema calls BuildBackupRangeAndSchema,
+// BuildBackupRangeAndSchema calls BuildBackupRangeAndInitSchema,
 // if the checkpoint mode is used, return the ranges from checkpoint meta
 func (bc *Client) BuildBackupRangeAndSchema(
 	storage kv.Storage,
@@ -731,7 +731,7 @@ func CheckBackupStorageIsLocked(ctx context.Context, s storage.ExternalStorage)
 	return nil
 }
 
-// BuildBackupRangeAndSchema gets KV range and schema of tables.
+// BuildBackupRangeAndInitSchema gets KV range and schema of tables.
 // KV ranges are separated by Table IDs.
 // Also, KV ranges are separated by Index IDs in the same table.
 func BuildBackupRangeAndInitSchema(

br/pkg/checkpoint/checkpoint_test.go

@@ -105,12 +105,12 @@ func TestCheckpointMetaForRestore(t *testing.T) {
 	exists := checkpoint.ExistsCheckpointProgress(ctx, dom)
 	require.False(t, exists)
 	err = checkpoint.SaveCheckpointProgress(ctx, se, &checkpoint.CheckpointProgress{
-		Progress: checkpoint.InLogRestoreAndIdMapPersist,
+		Progress: checkpoint.InLogRestoreAndIdMapPersisted,
 	})
 	require.NoError(t, err)
 	progress, err := checkpoint.LoadCheckpointProgress(ctx, se.GetSessionCtx().GetRestrictedSQLExecutor())
 	require.NoError(t, err)
-	require.Equal(t, checkpoint.InLogRestoreAndIdMapPersist, progress.Progress)
+	require.Equal(t, checkpoint.InLogRestoreAndIdMapPersisted, progress.Progress)
 
 	taskInfo, err := checkpoint.TryToGetCheckpointTaskInfo(ctx, s.Mock.Domain, se.GetSessionCtx().GetRestrictedSQLExecutor())
 	require.NoError(t, err)
@@ -120,7 +120,7 @@ func TestCheckpointMetaForRestore(t *testing.T) {
 	require.Equal(t, uint64(333), taskInfo.Metadata.RewriteTS)
 	require.Equal(t, "1.0", taskInfo.Metadata.GcRatio)
 	require.Equal(t, true, taskInfo.HasSnapshotMetadata)
-	require.Equal(t, checkpoint.InLogRestoreAndIdMapPersist, taskInfo.Progress)
+	require.Equal(t, checkpoint.InLogRestoreAndIdMapPersisted, taskInfo.Progress)
 
 	exists = checkpoint.ExistsCheckpointIngestIndexRepairSQLs(ctx, dom)
 	require.False(t, exists)

br/pkg/checkpoint/log_restore.go

@@ -194,22 +194,22 @@ func ExistsLogRestoreCheckpointMetadata(
 		TableExists(ast.NewCIStr(LogRestoreCheckpointDatabaseName), ast.NewCIStr(checkpointMetaTableName))
 }
 
-// A progress type for snapshot + log restore.
+// RestoreProgress is a progress type for snapshot + log restore.
 //
-// Before the id-maps is persist into external storage, the snapshot restore and
-// id-maps constructure can be repeated. So if the progress is in `InSnapshotRestore`,
+// Before the id-maps is persisted into external storage, the snapshot restore and
+// id-maps building can be retried. So if the progress is in `InSnapshotRestore`,
 // it can retry from snapshot restore.
 //
-// After the id-maps is persist into external storage, there are some meta-kvs has
-// been restored into the cluster, such as `rename ddl`. Where would be a situation:
+// After the id-maps is persisted into external storage, there are some meta-kvs has
+// been restored into the cluster, such as `rename ddl`. A situation could be:
 //
 // the first execution:
 //
 //	table A created in snapshot restore is renamed to table B in log restore
 //	     table A (id 80)       -------------->        table B (id 80)
 //	  ( snapshot restore )                            ( log restore )
 //
-// the second execution if don't skip snasphot restore:
+// the second execution if don't skip snapshot restore:
 //
 //	table A is created again in snapshot restore, because there is no table named A
 //	     table A (id 81)       -------------->   [not in id-maps, so ignored]
@@ -221,8 +221,8 @@ type RestoreProgress int
 
 const (
 	InSnapshotRestore RestoreProgress = iota
-	// Only when the id-maps is persist, status turns into it.
-	InLogRestoreAndIdMapPersist
+	// Only when the id-maps is persisted, status turns into it.
+	InLogRestoreAndIdMapPersisted
 )
 
 type CheckpointProgress struct {
@@ -254,8 +254,8 @@ func ExistsCheckpointProgress(
 		TableExists(ast.NewCIStr(LogRestoreCheckpointDatabaseName), ast.NewCIStr(checkpointProgressTableName))
 }
 
-// CheckpointTaskInfo is unique information within the same cluster id. It represents the last
-// restore task executed for this cluster.
+// CheckpointTaskInfoForLogRestore is tied to a specific cluster.
+// It represents the last restore task executed in this cluster.
 type CheckpointTaskInfoForLogRestore struct {
 	Metadata            *CheckpointMetadataForLogRestore
 	HasSnapshotMetadata bool

br/pkg/restore/import_mode_switcher.go

@@ -176,7 +176,7 @@ func (switcher *ImportModeSwitcher) GoSwitchToImportMode(
 	return nil
 }
 
-// RestorePreWork executes some prepare work before restore.
+// RestorePreWork switches to import mode and removes pd schedulers if needed
 // TODO make this function returns a restore post work.
 func RestorePreWork(
 	ctx context.Context,

br/pkg/restore/internal/prealloc_db/db.go

@@ -287,7 +287,7 @@ func (db *DB) canReuseTableID(ti *model.TableInfo) bool {
 	}
 	prealloced := db.preallocedIDs.PreallocedFor(ti)
 	if prealloced {
-		log.Info("reusing table ID", zap.Stringer("table", ti.Name))
+		log.Info("reusing table ID", zap.Stringer("table", ti.Name), zap.Int64("tableID", ti.ID))
 	}
 	return prealloced
 }

br/pkg/restore/log_client/BUILD.bazel

@@ -3,6 +3,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
 go_library(
     name = "log_client",
     srcs = [
+        "batch_meta_processor.go",
         "client.go",
         "compacted_file_strategy.go",
         "import.go",
@@ -37,6 +38,7 @@ go_library(
         "//br/pkg/stream",
         "//br/pkg/summary",
         "//br/pkg/utils",
+        "//br/pkg/utils/consts",
         "//br/pkg/utils/iter",
         "//br/pkg/version",
         "//pkg/ddl/util",
@@ -49,7 +51,6 @@ go_library(
         "//pkg/util/codec",
         "//pkg/util/redact",
         "//pkg/util/sqlexec",
-        "//pkg/util/table-filter",
         "@com_github_docker_go_units//:go-units",
         "@com_github_fatih_color//:color",
         "@com_github_gogo_protobuf//proto",
@@ -107,6 +108,7 @@ go_test(
         "//br/pkg/storage",
         "//br/pkg/stream",
         "//br/pkg/utils",
+        "//br/pkg/utils/consts",
         "//br/pkg/utils/iter",
         "//br/pkg/utiltest",
         "//pkg/domain",
@@ -121,7 +123,6 @@ go_test(
         "//pkg/util/chunk",
         "//pkg/util/codec",
         "//pkg/util/sqlexec",
-        "//pkg/util/table-filter",
         "@com_github_docker_go_units//:go-units",
         "@com_github_google_uuid//:uuid",
         "@com_github_pingcap_errors//:errors",