diff --git a/fe/fe-core/src/main/java/com/starrocks/meta/SqlBlackList.java b/fe/fe-core/src/main/java/com/starrocks/meta/SqlBlackList.java index d8d84003b136f..92e7cf61b5883 100644 --- a/fe/fe-core/src/main/java/com/starrocks/meta/SqlBlackList.java +++ b/fe/fe-core/src/main/java/com/starrocks/meta/SqlBlackList.java @@ -15,57 +15,130 @@ package com.starrocks.meta; +import com.staros.util.LockCloseable; import com.starrocks.common.AnalysisException; import com.starrocks.common.ErrorCode; import com.starrocks.common.ErrorReport; +import com.starrocks.persist.ImageWriter; +import com.starrocks.persist.SqlBlackListPersistInfo; +import com.starrocks.persist.metablock.SRMetaBlockEOFException; +import com.starrocks.persist.metablock.SRMetaBlockException; +import com.starrocks.persist.metablock.SRMetaBlockID; +import com.starrocks.persist.metablock.SRMetaBlockReader; +import com.starrocks.persist.metablock.SRMetaBlockWriter; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import java.io.IOException; +import java.util.Comparator; +import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; // Used by sql's blacklist public class SqlBlackList { - private static final SqlBlackList INSTANCE = new SqlBlackList(); - public static SqlBlackList getInstance() { - return INSTANCE; - } + private static final Logger LOG = LogManager.getLogger(SqlBlackList.class); - public static void verifying(String sql) throws AnalysisException { + public void verifying(String sql) throws AnalysisException { String formatSql = sql.replace("\r", " ").replace("\n", " ").replaceAll("\\s+", " "); - for (BlackListSql patternAndId : getInstance().sqlBlackListMap.values()) { - Matcher m = patternAndId.pattern.matcher(formatSql); - if (m.find()) { - ErrorReport.reportAnalysisException(ErrorCode.ERR_SQL_IN_BLACKLIST_ERROR); + try (LockCloseable ignored = new LockCloseable(rwLock.readLock())) { + for (BlackListSql patternAndId : sqlBlackListMap.values()) { + Matcher m = patternAndId.pattern.matcher(formatSql); + if (m.find()) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_SQL_IN_BLACKLIST_ERROR); + } + } + } + } + + public void load(SRMetaBlockReader reader) throws IOException, SRMetaBlockException, SRMetaBlockEOFException { + try (LockCloseable ignored = new LockCloseable(rwLock.writeLock())) { + int cnt = reader.readInt(); + for (int i = 0; i < cnt; i++) { + SqlBlackListPersistInfo sqlBlackListPersistInfo = reader.readJson(SqlBlackListPersistInfo.class); + put(sqlBlackListPersistInfo.id, Pattern.compile(sqlBlackListPersistInfo.pattern)); } + LOG.info("loaded {} SQL blacklist patterns", sqlBlackListMap.size()); } } // we use string of sql as key, and (pattern, id) as value. - public void put(Pattern pattern) { - if (!sqlBlackListMap.containsKey(pattern.toString())) { - long id = ids.getAndIncrement(); - sqlBlackListMap.putIfAbsent(pattern.toString(), new BlackListSql(pattern, id)); + public long put(Pattern pattern) { + try (LockCloseable ignored = new LockCloseable(rwLock.writeLock())) { + BlackListSql blackListSql = sqlBlackListMap.get(pattern.toString()); + if (blackListSql == null) { + long id = ids.getAndIncrement(); + sqlBlackListMap.put(pattern.toString(), new BlackListSql(pattern, id)); + return id; + } else { + return blackListSql.id; + } + } + } + + public void put(long id, Pattern pattern) { + try (LockCloseable ignored = new LockCloseable(rwLock.writeLock())) { + BlackListSql blackListSql = sqlBlackListMap.get(pattern.toString()); + if (blackListSql == null) { + ids.set(Math.max(ids.get(), id + 1)); + sqlBlackListMap.put(pattern.toString(), new BlackListSql(pattern, id)); + } } } // we delete sql's regular expression use id, so we iterate this map. public void delete(long id) { - for (Map.Entry entry : sqlBlackListMap.entrySet()) { - if (entry.getValue().id == id) { - sqlBlackListMap.remove(entry.getKey()); + try (LockCloseable ignored = new LockCloseable(rwLock.writeLock())) { + for (Map.Entry entry : sqlBlackListMap.entrySet()) { + if (entry.getValue().id == id) { + sqlBlackListMap.remove(entry.getKey()); + } + } + } + } + + public void delete(List ids) { + for (Long id : ids) { + this.delete(id); + } + } + + public void save(ImageWriter imageWriter) throws IOException, SRMetaBlockException { + try (LockCloseable ignored = new LockCloseable(rwLock.readLock())) { + // one for self and N for patterns + final int cnt = 1 + sqlBlackListMap.size(); + SRMetaBlockWriter writer = imageWriter.getBlockWriter(SRMetaBlockID.BLACKLIST_MGR, cnt); + + // write patterns + writer.writeInt(sqlBlackListMap.size()); + for (BlackListSql p : sqlBlackListMap.values()) { + writer.writeJson(new SqlBlackListPersistInfo(p.id, p.pattern.pattern())); } + writer.close(); } } + public List getBlackLists() { + try (LockCloseable ignored = new LockCloseable(rwLock.readLock())) { + return this.sqlBlackListMap.values().stream().sorted(Comparator.comparing(x -> x.id)).collect(Collectors.toList()); + } + } + + private final ReadWriteLock rwLock = new ReentrantReadWriteLock(); + // sqlBlackListMap: key is String(sql), value is BlackListSql. // BlackListSql is (Pattern, id). Pattern is the regular expression, id marks this sql, and is show with "show sqlblacklist"; - public ConcurrentMap sqlBlackListMap = new ConcurrentHashMap<>(); + private final ConcurrentMap sqlBlackListMap = new ConcurrentHashMap<>(); // ids used in sql blacklist - public AtomicLong ids = new AtomicLong(); + private final AtomicLong ids = new AtomicLong(); } diff --git a/fe/fe-core/src/main/java/com/starrocks/persist/DeleteSqlBlackLists.java b/fe/fe-core/src/main/java/com/starrocks/persist/DeleteSqlBlackLists.java new file mode 100644 index 0000000000000..b30c4fe554ec4 --- /dev/null +++ b/fe/fe-core/src/main/java/com/starrocks/persist/DeleteSqlBlackLists.java @@ -0,0 +1,29 @@ +// Copyright 2021-present StarRocks, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.starrocks.persist; + +import com.google.gson.annotations.SerializedName; +import com.starrocks.common.io.JsonWriter; + +import java.util.List; + +public class DeleteSqlBlackLists extends JsonWriter { + public DeleteSqlBlackLists(List ids) { + this.ids = ids; + } + + @SerializedName("ids") + public final List ids; +} diff --git a/fe/fe-core/src/main/java/com/starrocks/persist/EditLog.java b/fe/fe-core/src/main/java/com/starrocks/persist/EditLog.java index 4f627fb8ec9a2..e8230bf721932 100644 --- a/fe/fe-core/src/main/java/com/starrocks/persist/EditLog.java +++ b/fe/fe-core/src/main/java/com/starrocks/persist/EditLog.java @@ -124,6 +124,7 @@ import java.util.Set; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ExecutionException; +import java.util.regex.Pattern; /** * EditLog maintains a log of the memory modifications. @@ -1111,6 +1112,17 @@ public void loadJournal(GlobalStateMgr globalStateMgr, JournalEntity journal) globalStateMgr.getClusterSnapshotMgr().replayLog(log); break; } + case OperationType.OP_ADD_SQL_QUERY_BLACK_LIST: { + SqlBlackListPersistInfo addBlacklistRequest = (SqlBlackListPersistInfo) journal.data(); + GlobalStateMgr.getCurrentState().getSqlBlackList() + .put(addBlacklistRequest.id, Pattern.compile(addBlacklistRequest.pattern)); + break; + } + case OperationType.OP_DELETE_SQL_QUERY_BLACK_LIST: { + DeleteSqlBlackLists deleteBlackListsRequest = (DeleteSqlBlackLists) journal.data(); + GlobalStateMgr.getCurrentState().getSqlBlackList().delete(deleteBlackListsRequest.ids); + break; + } default: { if (Config.metadata_ignore_unknown_operation_type) { LOG.warn("UNKNOWN Operation Type {}", opCode); @@ -1795,6 +1807,15 @@ public void logAlterMaterializedViewProperties(ModifyTablePropertyOperationLog l logEdit(OperationType.OP_ALTER_MATERIALIZED_VIEW_PROPERTIES, log); } + public void logAddSQLBlackList(SqlBlackListPersistInfo addBlackList) { + logEdit(OperationType.OP_ADD_SQL_QUERY_BLACK_LIST, addBlackList); + } + + public void logDeleteSQLBlackList(DeleteSqlBlackLists deleteBlacklists) { + logEdit(OperationType.OP_DELETE_SQL_QUERY_BLACK_LIST, deleteBlacklists); + } + + public void logStarMgrOperation(StarMgrJournal journal) { logEdit(OperationType.OP_STARMGR, journal); } diff --git a/fe/fe-core/src/main/java/com/starrocks/persist/EditLogDeserializer.java b/fe/fe-core/src/main/java/com/starrocks/persist/EditLogDeserializer.java index 598255874ea45..40359c2a9b056 100644 --- a/fe/fe-core/src/main/java/com/starrocks/persist/EditLogDeserializer.java +++ b/fe/fe-core/src/main/java/com/starrocks/persist/EditLogDeserializer.java @@ -243,6 +243,8 @@ public class EditLogDeserializer { .put(OperationType.OP_ALTER_WAREHOUSE, Warehouse.class) .put(OperationType.OP_DROP_WAREHOUSE, DropWarehouseLog.class) .put(OperationType.OP_CLUSTER_SNAPSHOT_LOG, ClusterSnapshotLog.class) + .put(OperationType.OP_ADD_SQL_QUERY_BLACK_LIST, SqlBlackListPersistInfo.class) + .put(OperationType.OP_DELETE_SQL_QUERY_BLACK_LIST, DeleteSqlBlackLists.class) .build(); public static Writable deserialize(Short opCode, DataInput in) throws IOException { diff --git a/fe/fe-core/src/main/java/com/starrocks/persist/OperationType.java b/fe/fe-core/src/main/java/com/starrocks/persist/OperationType.java index 892150ae323d5..b64b4600f5d59 100644 --- a/fe/fe-core/src/main/java/com/starrocks/persist/OperationType.java +++ b/fe/fe-core/src/main/java/com/starrocks/persist/OperationType.java @@ -591,6 +591,11 @@ public class OperationType { @IgnorableOnReplayFailed public static final short OP_CLUSTER_SNAPSHOT_LOG = 13513; + @IgnorableOnReplayFailed + public static final short OP_ADD_SQL_QUERY_BLACK_LIST = 13520; + @IgnorableOnReplayFailed + public static final short OP_DELETE_SQL_QUERY_BLACK_LIST = 13521; + /** * NOTICE: OperationType cannot use a value exceeding 20000, please follow the above sequence number */ diff --git a/fe/fe-core/src/main/java/com/starrocks/persist/SqlBlackListPersistInfo.java b/fe/fe-core/src/main/java/com/starrocks/persist/SqlBlackListPersistInfo.java new file mode 100644 index 0000000000000..7bb7c0b783dda --- /dev/null +++ b/fe/fe-core/src/main/java/com/starrocks/persist/SqlBlackListPersistInfo.java @@ -0,0 +1,31 @@ +// Copyright 2021-present StarRocks, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.starrocks.persist; + +import com.google.gson.annotations.SerializedName; +import com.starrocks.common.io.JsonWriter; + +public class SqlBlackListPersistInfo extends JsonWriter { + public SqlBlackListPersistInfo(long id, String pattern) { + this.id = id; + this.pattern = pattern; + } + + @SerializedName("id") + public final long id; + + @SerializedName("pattern") + public final String pattern; +} diff --git a/fe/fe-core/src/main/java/com/starrocks/persist/metablock/SRMetaBlockID.java b/fe/fe-core/src/main/java/com/starrocks/persist/metablock/SRMetaBlockID.java index f9a7505b9f31a..7a8f87c720330 100644 --- a/fe/fe-core/src/main/java/com/starrocks/persist/metablock/SRMetaBlockID.java +++ b/fe/fe-core/src/main/java/com/starrocks/persist/metablock/SRMetaBlockID.java @@ -99,6 +99,8 @@ public int getId() { public static final SRMetaBlockID CLUSTER_SNAPSHOT_MGR = new SRMetaBlockID(33); + public static final SRMetaBlockID BLACKLIST_MGR = new SRMetaBlockID(34); + /** * NOTICE: SRMetaBlockID cannot use a value exceeding 20000, please follow the above sequence number */ diff --git a/fe/fe-core/src/main/java/com/starrocks/qe/ShowExecutor.java b/fe/fe-core/src/main/java/com/starrocks/qe/ShowExecutor.java index 04d58ac7ec5fa..1fa5c5e05c26b 100644 --- a/fe/fe-core/src/main/java/com/starrocks/qe/ShowExecutor.java +++ b/fe/fe-core/src/main/java/com/starrocks/qe/ShowExecutor.java @@ -129,7 +129,6 @@ import com.starrocks.load.streamload.StreamLoadFunctionalExprProvider; import com.starrocks.load.streamload.StreamLoadTask; import com.starrocks.meta.BlackListSql; -import com.starrocks.meta.SqlBlackList; import com.starrocks.proto.FailPointTriggerModeType; import com.starrocks.proto.PFailPointInfo; import com.starrocks.proto.PFailPointTriggerMode; @@ -2234,10 +2233,10 @@ public ShowResultSet visitShowPluginsStatement(ShowPluginsStmt statement, Connec @Override public ShowResultSet visitShowSqlBlackListStatement(ShowSqlBlackListStmt statement, ConnectContext context) { List> rows = new ArrayList<>(); - for (Map.Entry entry : SqlBlackList.getInstance().sqlBlackListMap.entrySet()) { + for (BlackListSql entry : GlobalStateMgr.getCurrentState().getSqlBlackList().getBlackLists()) { List oneSql = new ArrayList<>(); - oneSql.add(String.valueOf(entry.getValue().id)); - oneSql.add(entry.getKey()); + oneSql.add(String.valueOf(entry.id)); + oneSql.add(entry.pattern.toString()); rows.add(oneSql); } return new ShowResultSet(statement.getMetaData(), rows); diff --git a/fe/fe-core/src/main/java/com/starrocks/qe/StmtExecutor.java b/fe/fe-core/src/main/java/com/starrocks/qe/StmtExecutor.java index 76763ce6713b9..b20c5fabf0ac9 100644 --- a/fe/fe-core/src/main/java/com/starrocks/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/com/starrocks/qe/StmtExecutor.java @@ -99,7 +99,6 @@ import com.starrocks.load.InsertOverwriteJobMgr; import com.starrocks.load.loadv2.InsertLoadJob; import com.starrocks.load.loadv2.LoadJob; -import com.starrocks.meta.SqlBlackList; import com.starrocks.metric.MetricRepo; import com.starrocks.metric.TableMetricsEntity; import com.starrocks.metric.TableMetricsRegistry; @@ -108,6 +107,8 @@ import com.starrocks.mysql.MysqlEofPacket; import com.starrocks.mysql.MysqlSerializer; import com.starrocks.persist.CreateInsertOverwriteJobLog; +import com.starrocks.persist.DeleteSqlBlackLists; +import com.starrocks.persist.SqlBlackListPersistInfo; import com.starrocks.persist.gson.GsonUtils; import com.starrocks.planner.FileScanNode; import com.starrocks.planner.HiveTableSink; @@ -631,7 +632,7 @@ public void execute() throws Exception { String originSql = origStmt.originStmt.trim() .toLowerCase().replaceAll(" +", " "); // If this sql is in blacklist, show message. - SqlBlackList.verifying(originSql); + GlobalStateMgr.getCurrentState().getSqlBlackList().verifying(originSql); } } @@ -1691,7 +1692,9 @@ public void checkPrivilegeForKillAnalyzeStmt(ConnectContext context, long analyz private void handleAddSqlBlackListStmt() { AddSqlBlackListStmt addSqlBlackListStmt = (AddSqlBlackListStmt) parsedStmt; - SqlBlackList.getInstance().put(addSqlBlackListStmt.getSqlPattern()); + long id = GlobalStateMgr.getCurrentState().getSqlBlackList().put(addSqlBlackListStmt.getSqlPattern()); + GlobalStateMgr.getCurrentState().getEditLog() + .logAddSQLBlackList(new SqlBlackListPersistInfo(id, addSqlBlackListStmt.getSqlPattern().pattern())); } private void handleDelSqlBlackListStmt() { @@ -1699,8 +1702,10 @@ private void handleDelSqlBlackListStmt() { List indexs = delSqlBlackListStmt.getIndexs(); if (indexs != null) { for (long id : indexs) { - SqlBlackList.getInstance().delete(id); + GlobalStateMgr.getCurrentState().getSqlBlackList().delete(id); } + GlobalStateMgr.getCurrentState().getEditLog() + .logDeleteSQLBlackList(new DeleteSqlBlackLists(indexs)); } } diff --git a/fe/fe-core/src/main/java/com/starrocks/server/GlobalStateMgr.java b/fe/fe-core/src/main/java/com/starrocks/server/GlobalStateMgr.java index 4b74586c334a5..1738230bacf2f 100644 --- a/fe/fe-core/src/main/java/com/starrocks/server/GlobalStateMgr.java +++ b/fe/fe-core/src/main/java/com/starrocks/server/GlobalStateMgr.java @@ -166,6 +166,7 @@ import com.starrocks.load.streamload.StreamLoadMgr; import com.starrocks.memory.MemoryUsageTracker; import com.starrocks.memory.ProcProfileCollector; +import com.starrocks.meta.SqlBlackList; import com.starrocks.metric.MetricRepo; import com.starrocks.persist.BackendIdsUpdateInfo; import com.starrocks.persist.EditLog; @@ -521,6 +522,8 @@ public class GlobalStateMgr { private final ClusterSnapshotMgr clusterSnapshotMgr; + private final SqlBlackList sqlBlackList; + public NodeMgr getNodeMgr() { return nodeMgr; } @@ -819,6 +822,7 @@ public void transferToNonLeader(FrontendNodeType newType) { this.authorizer = new Authorizer(accessControlProvider); this.ddlStmtExecutor = new DDLStmtExecutor(DDLStmtExecutor.StmtExecutorVisitor.getInstance()); this.showExecutor = new ShowExecutor(ShowExecutor.ShowExecutorVisitor.getInstance()); + this.sqlBlackList = new SqlBlackList(); this.temporaryTableCleaner = new TemporaryTableCleaner(); this.queryDeployExecutor = ThreadPoolManager.newDaemonFixedThreadPool(Config.query_deploy_threadpool_size, Integer.MAX_VALUE, @@ -1563,6 +1567,7 @@ public void loadImage(String imageDir) throws IOException { .put(SRMetaBlockID.PIPE_MGR, pipeManager.getRepo()::load) .put(SRMetaBlockID.WAREHOUSE_MGR, warehouseMgr::load) .put(SRMetaBlockID.CLUSTER_SNAPSHOT_MGR, clusterSnapshotMgr::load) + .put(SRMetaBlockID.BLACKLIST_MGR, sqlBlackList::load) .build(); Set metaMgrMustExists = new HashSet<>(loadImages.keySet()); @@ -1762,6 +1767,7 @@ public void saveImage(ImageWriter imageWriter, File curFile) throws IOException keyMgr.save(imageWriter); pipeManager.getRepo().save(imageWriter); warehouseMgr.save(imageWriter); + sqlBlackList.save(imageWriter); } catch (SRMetaBlockException e) { LOG.error("Save meta block failed ", e); throw new IOException("Save meta block failed ", e); @@ -2182,6 +2188,10 @@ public ExportMgr getExportMgr() { return this.exportMgr; } + public SqlBlackList getSqlBlackList() { + return this.sqlBlackList; + } + public MaterializedViewMgr getMaterializedViewMgr() { return this.materializedViewMgr; } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/ast/AddSqlBlackListStmt.java b/fe/fe-core/src/main/java/com/starrocks/sql/ast/AddSqlBlackListStmt.java index 5b9720b885164..078a31cb58ad6 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/ast/AddSqlBlackListStmt.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/ast/AddSqlBlackListStmt.java @@ -65,7 +65,7 @@ public R accept(AstVisitor visitor, C context) { @Override public RedirectStatus getRedirectStatus() { - return RedirectStatus.NO_FORWARD; + return RedirectStatus.FORWARD_NO_SYNC; } } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/ast/DelSqlBlackListStmt.java b/fe/fe-core/src/main/java/com/starrocks/sql/ast/DelSqlBlackListStmt.java index 51c3b04e218f3..34d4c367b57b5 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/ast/DelSqlBlackListStmt.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/ast/DelSqlBlackListStmt.java @@ -46,7 +46,7 @@ public R accept(AstVisitor visitor, C context) { @Override public RedirectStatus getRedirectStatus() { - return RedirectStatus.NO_FORWARD; + return RedirectStatus.FORWARD_NO_SYNC; } } diff --git a/fe/fe-core/src/test/java/com/starrocks/analysis/SqlBlacklistAndWhitelistTest.java b/fe/fe-core/src/test/java/com/starrocks/analysis/SqlBlacklistAndWhitelistTest.java index f4e308c87d43f..93c1fc1f95ad5 100644 --- a/fe/fe-core/src/test/java/com/starrocks/analysis/SqlBlacklistAndWhitelistTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/analysis/SqlBlacklistAndWhitelistTest.java @@ -39,7 +39,7 @@ public void testAddSqlBlacklist() { @Test public void testDelSqlBlacklist() { DelSqlBlackListStmt stmt = (DelSqlBlackListStmt) analyzeSuccess("delete sqlblacklist 2, 6;"); - Assert.assertEquals(Lists.asList(2L, new Long[] {6L}), stmt.getIndexs()); + Assert.assertEquals(Lists.asList(2L, new Long[]{6L}), stmt.getIndexs()); Assert.assertNotNull(stmt.getRedirectStatus()); // bad cases analyzeFail("DELETE SQLBLACKLIST"); diff --git a/fe/fe-core/src/test/java/com/starrocks/backup/RestoreJobTest.java b/fe/fe-core/src/test/java/com/starrocks/backup/RestoreJobTest.java index daf600a4ab8a0..02302cc293378 100644 --- a/fe/fe-core/src/test/java/com/starrocks/backup/RestoreJobTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/backup/RestoreJobTest.java @@ -557,7 +557,7 @@ public void testRunBackupListTable() { minTimes = 0; result = id.incrementAndGet(); - GlobalStateMgr.getCurrentState().getNodeMgr().getClusterInfo(); + globalStateMgr.getNodeMgr().getClusterInfo(); minTimes = 0; result = systemInfoService; } diff --git a/fe/fe-core/src/test/java/com/starrocks/persist/OperationTypeTest.java b/fe/fe-core/src/test/java/com/starrocks/persist/OperationTypeTest.java index 01f6e411a2a21..1da0c84a9a509 100644 --- a/fe/fe-core/src/test/java/com/starrocks/persist/OperationTypeTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/persist/OperationTypeTest.java @@ -151,6 +151,8 @@ public void testRecoverableOperations() { Assert.assertTrue(OperationType.IGNORABLE_OPERATIONS.contains(OperationType.OP_DELETE_REPLICATION_JOB)); Assert.assertTrue(OperationType.IGNORABLE_OPERATIONS.contains(OperationType.OP_RESET_FRONTENDS)); Assert.assertTrue(OperationType.IGNORABLE_OPERATIONS.contains(OperationType.OP_CLUSTER_SNAPSHOT_LOG)); + Assert.assertTrue(OperationType.IGNORABLE_OPERATIONS.contains(OperationType.OP_ADD_SQL_QUERY_BLACK_LIST)); + Assert.assertTrue(OperationType.IGNORABLE_OPERATIONS.contains(OperationType.OP_DELETE_SQL_QUERY_BLACK_LIST)); } @Test diff --git a/fe/fe-core/src/test/java/com/starrocks/planner/QueryPlannerTest.java b/fe/fe-core/src/test/java/com/starrocks/planner/QueryPlannerTest.java index 816e93f56a185..41da313ebd57b 100644 --- a/fe/fe-core/src/test/java/com/starrocks/planner/QueryPlannerTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/planner/QueryPlannerTest.java @@ -38,7 +38,6 @@ import com.starrocks.common.util.UUIDUtil; import com.starrocks.ha.FrontendNodeType; import com.starrocks.meta.BlackListSql; -import com.starrocks.meta.SqlBlackList; import com.starrocks.qe.ConnectContext; import com.starrocks.qe.QueryState; import com.starrocks.qe.StmtExecutor; @@ -161,11 +160,11 @@ public void testSqlBlackList() throws Exception { StmtExecutor stmtExecutor1 = new StmtExecutor(connectContext, statement); stmtExecutor1.execute(); - Assert.assertEquals(SqlBlackList.getInstance().sqlBlackListMap.entrySet().size(), 1); + Assert.assertEquals(GlobalStateMgr.getCurrentState().getSqlBlackList().getBlackLists().size(), 1); long id = -1; - for (Map.Entry entry : SqlBlackList.getInstance().sqlBlackListMap.entrySet()) { - id = entry.getValue().id; - Assert.assertEquals("select k1 from .+", entry.getKey()); + for (BlackListSql entry : GlobalStateMgr.getCurrentState().getSqlBlackList().getBlackLists()) { + id = entry.id; + Assert.assertEquals("select k1 from .+", entry.pattern.pattern()); } String sql = "select k1 from test.baseall"; @@ -193,7 +192,7 @@ public void testSqlBlackList() throws Exception { connectContext.getSessionVariable().getSqlMode()); StmtExecutor stmtExecutor3 = new StmtExecutor(connectContext, statement); stmtExecutor3.execute(); - Assert.assertEquals(0, SqlBlackList.getInstance().sqlBlackListMap.entrySet().size()); + Assert.assertEquals(0, GlobalStateMgr.getCurrentState().getSqlBlackList().getBlackLists().size()); } @Test @@ -210,11 +209,11 @@ public void testSqlBlackListWithLineSeparators() throws Exception { StmtExecutor stmtExecutor1 = new StmtExecutor(connectContext, statement); stmtExecutor1.execute(); - Assert.assertEquals(1, SqlBlackList.getInstance().sqlBlackListMap.entrySet().size()); + Assert.assertEquals(1, GlobalStateMgr.getCurrentState().getSqlBlackList().getBlackLists().size()); long id = -1; - for (Map.Entry entry : SqlBlackList.getInstance().sqlBlackListMap.entrySet()) { - id = entry.getValue().id; - Assert.assertEquals("select k1 from .+", entry.getKey()); + for (BlackListSql entry : GlobalStateMgr.getCurrentState().getSqlBlackList().getBlackLists()) { + id = entry.id; + Assert.assertEquals("select k1 from .+", entry.pattern.pattern()); } String sql = "select k1 from test.baseall"; @@ -241,7 +240,7 @@ public void testSqlBlackListWithLineSeparators() throws Exception { connectContext.getSessionVariable().getSqlMode()); StmtExecutor stmtExecutor3 = new StmtExecutor(connectContext, statement); stmtExecutor3.execute(); - Assert.assertEquals(0, SqlBlackList.getInstance().sqlBlackListMap.entrySet().size()); + Assert.assertEquals(0, GlobalStateMgr.getCurrentState().getSqlBlackList().getBlackLists().size()); } @Test @@ -258,11 +257,11 @@ public void testSqlBlackListUseWhere() throws Exception { StmtExecutor stmtExecutor1 = new StmtExecutor(connectContext, statement); stmtExecutor1.execute(); - Assert.assertEquals(SqlBlackList.getInstance().sqlBlackListMap.entrySet().size(), 1); + Assert.assertEquals(GlobalStateMgr.getCurrentState().getSqlBlackList().getBlackLists().size(), 1); long id = -1; - for (Map.Entry entry : SqlBlackList.getInstance().sqlBlackListMap.entrySet()) { - id = entry.getValue().id; - Assert.assertEquals("( where )", entry.getKey()); + for (BlackListSql entry : GlobalStateMgr.getCurrentState().getSqlBlackList().getBlackLists()) { + id = entry.id; + Assert.assertEquals("( where )", entry.pattern.pattern()); } String sql4 = "select k1 as awhere from test.baseall"; @@ -283,7 +282,7 @@ public void testSqlBlackListUseWhere() throws Exception { connectContext.getSessionVariable().getSqlMode()); StmtExecutor stmtExecutor3 = new StmtExecutor(connectContext, statement); stmtExecutor3.execute(); - Assert.assertEquals(0, SqlBlackList.getInstance().sqlBlackListMap.entrySet().size()); + Assert.assertEquals(0, GlobalStateMgr.getCurrentState().getSqlBlackList().getBlackLists().size()); } @Test public void testSqlBlackListWithInsert() throws Exception { @@ -299,11 +298,11 @@ public void testSqlBlackListWithInsert() throws Exception { StmtExecutor stmtExecutor1 = new StmtExecutor(connectContext, statement); stmtExecutor1.execute(); - Assert.assertEquals(SqlBlackList.getInstance().sqlBlackListMap.entrySet().size(), 1); + Assert.assertEquals(GlobalStateMgr.getCurrentState().getSqlBlackList().getBlackLists().size(), 1); long id = -1; - for (Map.Entry entry : SqlBlackList.getInstance().sqlBlackListMap.entrySet()) { - id = entry.getValue().id; - Assert.assertEquals("insert into .+ values.+", entry.getKey()); + for (BlackListSql entry : GlobalStateMgr.getCurrentState().getSqlBlackList().getBlackLists()) { + id = entry.id; + Assert.assertEquals("insert into .+ values.+", entry.pattern.pattern()); } String sql = @@ -323,7 +322,7 @@ public void testSqlBlackListWithInsert() throws Exception { connectContext.getSessionVariable().getSqlMode()); StmtExecutor stmtExecutor3 = new StmtExecutor(connectContext, statement); stmtExecutor3.execute(); - Assert.assertEquals(0, SqlBlackList.getInstance().sqlBlackListMap.entrySet().size()); + Assert.assertEquals(0, GlobalStateMgr.getCurrentState().getSqlBlackList().getBlackLists().size()); } @Test diff --git a/fe/fe-core/src/test/java/com/starrocks/server/SqlBlacklistTest.java b/fe/fe-core/src/test/java/com/starrocks/server/SqlBlacklistTest.java new file mode 100644 index 0000000000000..12244274f7d3d --- /dev/null +++ b/fe/fe-core/src/test/java/com/starrocks/server/SqlBlacklistTest.java @@ -0,0 +1,228 @@ +// Copyright 2021-present StarRocks, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.starrocks.server; + +import com.starrocks.analysis.RedirectStatus; +import com.starrocks.common.jmockit.Deencapsulation; +import com.starrocks.meta.BlackListSql; +import com.starrocks.meta.SqlBlackList; +import com.starrocks.persist.DeleteSqlBlackLists; +import com.starrocks.persist.EditLog; +import com.starrocks.persist.SqlBlackListPersistInfo; +import com.starrocks.qe.ConnectContext; +import com.starrocks.qe.ShowExecutor; +import com.starrocks.qe.ShowResultSet; +import com.starrocks.qe.StmtExecutor; +import com.starrocks.sql.analyzer.AnalyzeTestUtil; +import com.starrocks.sql.ast.AddSqlBlackListStmt; +import com.starrocks.sql.ast.DelSqlBlackListStmt; +import com.starrocks.sql.ast.ShowSqlBlackListStmt; +import com.starrocks.utframe.UtFrameUtils; +import mockit.Mock; +import mockit.MockUp; +import org.junit.Assert; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; + +import java.util.List; +import java.util.UUID; +import java.util.regex.Pattern; + +import static com.starrocks.sql.analyzer.AnalyzeTestUtil.parseSql; + +public class SqlBlacklistTest { + GlobalStateMgr state; + SqlBlackList sqlBlackList; + EditLog editLog; + ConnectContext connectContext; + + @BeforeClass + public static void beforeClass() throws Exception { + AnalyzeTestUtil.init(); + } + + @Before + public void beforeEach() { + state = Deencapsulation.newInstance(GlobalStateMgr.class); + sqlBlackList = new SqlBlackList(); + connectContext = UtFrameUtils.createDefaultCtx(); + editLog = Mockito.mock(EditLog.class); + connectContext.setQueryId(UUID.randomUUID()); + } + + @Test + public void testAddSQLBlacklist() throws Exception { + mockupGlobalState(); + + ArgumentCaptor addBlacklistEditLogArgument = ArgumentCaptor + .forClass(SqlBlackListPersistInfo.class); + + AddSqlBlackListStmt addStatement = (AddSqlBlackListStmt) parseSql("ADD SQLBLACKLIST \".+\";"); + Assert.assertEquals(addStatement.getSql(), ".+"); + + StmtExecutor addStatementExecutor = new StmtExecutor(connectContext, addStatement); + addStatementExecutor.execute(); + List blackLists = sqlBlackList.getBlackLists(); + Assert.assertEquals(1, blackLists.size()); + Assert.assertEquals(0, blackLists.get(0).id); + Assert.assertEquals(".+", blackLists.get(0).pattern.pattern()); + + Mockito.verify(editLog).logAddSQLBlackList(addBlacklistEditLogArgument.capture()); + + Assert.assertEquals(0, addBlacklistEditLogArgument.getValue().id); + Assert.assertEquals(".+", addBlacklistEditLogArgument.getValue().pattern); + } + + @Test + public void testShowBlacklist() { + mockupGlobalState(); + sqlBlackList.put(Pattern.compile("qwert")); + sqlBlackList.put(Pattern.compile("abcde")); + + ShowSqlBlackListStmt showSqlStatement = (ShowSqlBlackListStmt) parseSql("SHOW SQLBLACKLIST"); + + ShowResultSet resultSet = ShowExecutor.execute(showSqlStatement, connectContext); + Assert.assertTrue(resultSet.next()); + Assert.assertEquals(0L, resultSet.getLong(0)); + Assert.assertEquals("qwert", resultSet.getString(1)); + Assert.assertTrue(resultSet.next()); + Assert.assertEquals(1L, resultSet.getLong(0)); + Assert.assertEquals("abcde", resultSet.getString(1)); + Assert.assertFalse(resultSet.next()); + } + + @Test + public void testBlackListReturnsSameIdIfPatternAlreadyExists() { + mockupGlobalState(); + Pattern p = Pattern.compile("qwert"); + long id = sqlBlackList.put(p); + + Assert.assertEquals(id, sqlBlackList.put(p)); + } + + @Test + public void testDeleteSqlBlacklist() throws Exception { + mockupGlobalState(); + long id1 = sqlBlackList.put(Pattern.compile("qwert")); + long id2 = sqlBlackList.put(Pattern.compile("abcde")); + + ArgumentCaptor deleteBlacklistsEditLogArgument = + ArgumentCaptor.forClass(DeleteSqlBlackLists.class); + + StmtExecutor deleteStatementExecutor = new StmtExecutor(connectContext, new DelSqlBlackListStmt(List.of(id1, id2))); + deleteStatementExecutor.execute(); + Assert.assertTrue(sqlBlackList + .getBlackLists().stream().noneMatch(x -> x.id == id1 || x.id != id2)); + + Mockito.verify(editLog).logDeleteSQLBlackList(deleteBlacklistsEditLogArgument.capture()); + + Assert.assertEquals(List.of(id1, id2), deleteBlacklistsEditLogArgument.getValue().ids); + } + + @Test + public void testRedirectStatus() { + Assert.assertEquals( + new AddSqlBlackListStmt("ADD SQLBLACKLIST \".+\";").getRedirectStatus(), + RedirectStatus.FORWARD_NO_SYNC + ); + Assert.assertEquals( + new DelSqlBlackListStmt(List.of(1L, 2L)).getRedirectStatus(), + RedirectStatus.FORWARD_NO_SYNC + ); + } + + @Test + public void testSaveLoadBlackListImage() throws Exception { + SqlBlackList originalBlacklist = new SqlBlackList(); + originalBlacklist.put(Pattern.compile("zxcvbqwert")); + originalBlacklist.put(Pattern.compile("qwdsad")); + + UtFrameUtils.PseudoImage testImage = new UtFrameUtils.PseudoImage(); + originalBlacklist.save(testImage.getImageWriter()); + + SqlBlackList recoveredBlackList = new SqlBlackList(); + recoveredBlackList.load(testImage.getMetaBlockReader()); + + Assert.assertEquals(originalBlacklist.getBlackLists().size(), recoveredBlackList.getBlackLists().size()); + Assert.assertEquals(originalBlacklist.getBlackLists().get(0).id, recoveredBlackList.getBlackLists().get(0).id); + Assert.assertEquals( + originalBlacklist.getBlackLists().get(0).pattern.pattern(), + recoveredBlackList.getBlackLists().get(0).pattern.pattern() + ); + Assert.assertEquals(originalBlacklist.getBlackLists().get(1).id, recoveredBlackList.getBlackLists().get(1).id); + Assert.assertEquals( + originalBlacklist.getBlackLists().get(1).pattern.pattern(), + recoveredBlackList.getBlackLists().get(1).pattern.pattern() + ); + } + + @Test + public void testSqlBlacklistJournalOperations() throws Exception { + UtFrameUtils.createMinStarRocksCluster(); + UtFrameUtils.setUpForPersistTest(); + UtFrameUtils.PseudoJournalReplayer.resetFollowerJournalQueue(); + + // add blacklists + + GlobalStateMgr.getCurrentState().getEditLog().logAddSQLBlackList(new SqlBlackListPersistInfo(123, "p1")); + GlobalStateMgr.getCurrentState().getEditLog().logAddSQLBlackList(new SqlBlackListPersistInfo(1234, "p2")); + UtFrameUtils.PseudoJournalReplayer.replayJournalToEnd(); + + List resultBlackLists = GlobalStateMgr.getCurrentState().getSqlBlackList().getBlackLists(); + Assert.assertEquals(2, resultBlackLists.size()); + Assert.assertEquals(123L, resultBlackLists.get(0).id); + Assert.assertEquals("p1", resultBlackLists.get(0).pattern.pattern()); + Assert.assertEquals(1234L, resultBlackLists.get(1).id); + Assert.assertEquals("p2", resultBlackLists.get(1).pattern.pattern()); + + // delete blacklists + + GlobalStateMgr.getCurrentState().getEditLog().logDeleteSQLBlackList(new DeleteSqlBlackLists(List.of(123L, 1234L))); + UtFrameUtils.PseudoJournalReplayer.replayJournalToEnd(); + + Assert.assertTrue( + sqlBlackList.getBlackLists().stream() + .noneMatch(x -> x.id == 123L || x.id == 1234L) + ); + + } + + private void mockupGlobalState() { + MockUp mockUp = new MockUp() { + @Mock + GlobalStateMgr getCurrentState() { + return state; + } + + @Mock + public SqlBlackList getSqlBlackList() { + return sqlBlackList; + } + + @Mock + public boolean isLeader() { + return true; + } + + @Mock + public EditLog getEditLog() { + return editLog; + } + }; + } +}