Skip to content

Commit

Permalink
save storage policy to jobinfo instead of BackupMeta
Browse files Browse the repository at this point in the history
  • Loading branch information
justfortaste committed Dec 12, 2024
1 parent 626b7d6 commit 8a6c454
Show file tree
Hide file tree
Showing 4 changed files with 12 additions and 46 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -103,10 +103,8 @@ public final class FeMetaVersion {

public static final int VERSION_140 = 140;

// For BackupMeta storage policy
public static final int VERSION_141 = 141;
// note: when increment meta version, should assign the latest version to VERSION_CURRENT
public static final int VERSION_CURRENT = VERSION_141;
public static final int VERSION_CURRENT = VERSION_140;

// all logs meta version should >= the minimum version, so that we could remove many if clause, for example
// if (FE_METAVERSION < VERSION_94) ...
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -340,7 +340,7 @@ public static class BriefBackupJobInfo {
@SerializedName("s3_resource_list")
public List<BackupS3ResourceInfo> s3ResourceList = Lists.newArrayList();
@SerializedName("storage_policy_list")
public List<BackupStoragePolicyInfo> storagePolicyList = Lists.newArrayList();
public List<StoragePolicy> storagePolicyList = Lists.newArrayList();

public static BriefBackupJobInfo fromBackupJobInfo(BackupJobInfo backupJobInfo) {
BriefBackupJobInfo briefBackupJobInfo = new BriefBackupJobInfo();
Expand Down Expand Up @@ -381,7 +381,7 @@ public static class NewBackupObjects {
@SerializedName("s3_resources")
public List<BackupS3ResourceInfo> s3Resources = Lists.newArrayList();
@SerializedName("storage_policy")
public List<BackupStoragePolicyInfo> storagePolicies = Lists.newArrayList();
public List<StoragePolicy> storagePolicies = Lists.newArrayList();
}

public static class BackupOlapTableInfo {
Expand Down Expand Up @@ -505,11 +505,6 @@ public static class BackupS3ResourceInfo {
public String name;
}

public static class BackupStoragePolicyInfo {
@SerializedName("name")
public String name;
}

// eg: __db_10001/__tbl_10002/__part_10003/__idx_10002/__10004
public String getFilePath(String db, String tbl, String part, String idx, long tabletId) {
if (!db.equalsIgnoreCase(dbName)) {
Expand Down Expand Up @@ -721,9 +716,7 @@ public static BackupJobInfo fromCatalog(long backupTime, String label, String db
// storage policies
Collection<StoragePolicy> storagePolicies = backupMeta.getStoragePolicyNameMap().values();
for (StoragePolicy storagePolicy : storagePolicies) {
BackupStoragePolicyInfo backupStoragePolicyInfo = new BackupStoragePolicyInfo();
backupStoragePolicyInfo.name = storagePolicy.getName();
jobInfo.newBackupObjects.storagePolicies.add(backupStoragePolicyInfo);
jobInfo.newBackupObjects.storagePolicies.add(storagePolicy.clone());
}

return jobInfo;
Expand Down
17 changes: 0 additions & 17 deletions fe/fe-core/src/main/java/org/apache/doris/backup/BackupMeta.java
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,6 @@ public class BackupMeta implements Writable, GsonPostProcessable {
@SerializedName(value = "resourceNameMap")
private Map<String, Resource> resourceNameMap = Maps.newHashMap();
// storagePolicy name -> resource
@SerializedName(value = "storagePolicyNameMap")
private Map<String, StoragePolicy> storagePolicyNameMap = Maps.newHashMap();

private BackupMeta() {
Expand Down Expand Up @@ -108,10 +107,6 @@ public Resource getResource(String resourceName) {
return resourceNameMap.get(resourceName);
}

public StoragePolicy getStoragePolicy(String policyName) {
return storagePolicyNameMap.get(policyName);
}

public Table getTable(Long tblId) {
return tblIdMap.get(tblId);
}
Expand Down Expand Up @@ -168,10 +163,6 @@ public void write(DataOutput out) throws IOException {
for (Resource resource : resourceNameMap.values()) {
resource.write(out);
}
out.writeInt(storagePolicyNameMap.size());
for (StoragePolicy storagePolicy : storagePolicyNameMap.values()) {
storagePolicy.write(out);
}
}

@Deprecated
Expand All @@ -187,14 +178,6 @@ public void readFields(DataInput in) throws IOException {
Resource resource = Resource.read(in);
resourceNameMap.put(resource.getName(), resource);
}

if (Env.getCurrentEnvJournalVersion() >= FeMetaVersion.VERSION_141) {
size = in.readInt();
for (int i = 0; i < size; i++) {
StoragePolicy policy = StoragePolicy.read(in);
storagePolicyNameMap.put(policy.getName(), policy);
}
}
}

@Override
Expand Down
24 changes: 8 additions & 16 deletions fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
Original file line number Diff line number Diff line change
Expand Up @@ -700,8 +700,8 @@ private void checkAndPrepareMeta() {
}
}

for (BackupJobInfo.BackupStoragePolicyInfo backupStoragePolicyInfo : jobInfo.newBackupObjects.storagePolicies) {
String backupStoragePoliceName = backupStoragePolicyInfo.name;
for (StoragePolicy backupStoragePolicy : jobInfo.newBackupObjects.storagePolicies) {
String backupStoragePoliceName = backupStoragePolicy.getName();
Optional<Policy> localPolicy = Env.getCurrentEnv().getPolicyMgr().findPolicy(backupStoragePoliceName,
PolicyTypeEnum.STORAGE);
if (localPolicy.isPresent() && localPolicy.get().getType() != PolicyTypeEnum.STORAGE) {
Expand Down Expand Up @@ -1390,21 +1390,19 @@ private void checkAndRestoreStoragePolicies() {
return;
}
PolicyMgr policyMgr = Env.getCurrentEnv().getPolicyMgr();
for (BackupJobInfo.BackupStoragePolicyInfo backupStoragePolicyInfo : jobInfo.newBackupObjects.storagePolicies) {
String backupStoragePoliceName = backupStoragePolicyInfo.name;
for (StoragePolicy backupStoragePolicy : jobInfo.newBackupObjects.storagePolicies) {
String backupStoragePoliceName = backupStoragePolicy.getName();
Optional<Policy> localPolicy = policyMgr.findPolicy(backupStoragePoliceName,
PolicyTypeEnum.STORAGE);
StoragePolicy backupStoargePolicy = backupMeta.getStoragePolicy(backupStoragePoliceName);

// use specified storageResource
if (storageResource != null) {
backupStoargePolicy.setStorageResource(storageResource);
backupStoragePolicy.setStorageResource(storageResource);
}
if (localPolicy.isPresent()) {
StoragePolicy localStoargePolicy = (StoragePolicy) localPolicy.get();
// storage policy name and resource name should be same
if (localStoargePolicy.getSignature(BackupHandler.SIGNATURE_VERSION)
!= backupStoargePolicy.getSignature(BackupHandler.SIGNATURE_VERSION)) {
!= backupStoragePolicy.getSignature(BackupHandler.SIGNATURE_VERSION)) {
status = new Status(ErrCode.COMMON_ERROR, "Storage policy "
+ jobInfo.getAliasByOriginNameIfSet(backupStoragePoliceName)
+ " already exist but with different properties");
Expand All @@ -1414,11 +1412,11 @@ private void checkAndRestoreStoragePolicies() {
} else {
// restore storage policy
try {
policyMgr.createStoragePolicy(backupStoargePolicy);
policyMgr.createStoragePolicy(backupStoragePolicy);
} catch (Exception e) {
LOG.error("restore storage policy fail should not happen", e);
}
storagePolicies.add(backupStoargePolicy);
storagePolicies.add(backupStoragePolicy);
}
}
}
Expand Down Expand Up @@ -2818,12 +2816,6 @@ private void readOthers(DataInput in) throws IOException {
restoredResources.add(Resource.read(in));
}

// restored storage policy
size = in.readInt();
for (int i = 0; i < size; i++) {
storagePolicies.add(StoragePolicy.read(in));
}

// read properties
size = in.readInt();
for (int i = 0; i < size; i++) {
Expand Down

0 comments on commit 8a6c454

Please sign in to comment.