Skip to content

Commit

Permalink
[Fix][Common] rewrite code generate,fix bit shift (#6914)
Browse files Browse the repository at this point in the history
* rewrite code generate,fix bit shift

* fix ut

* add algorithm from licenses file

* add algorithm from licenses file

* add algorithm from licenses file

* add algorithm from licenses file

* add algorithm from licenses file

* fix ut
  • Loading branch information
brave-lee authored Nov 19, 2021
1 parent 595e484 commit 4bec792
Show file tree
Hide file tree
Showing 17 changed files with 142 additions and 139 deletions.
1 change: 1 addition & 0 deletions .licenserc.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ header:
- LICENSE
- DISCLAIMER
- dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/ScriptRunner.java
- dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CodeGenerateUtils.java
- mvnw.cmd
- dolphinscheduler-dao/src/main/resources/sql/soft_version
- .mvn
Expand Down
2 changes: 1 addition & 1 deletion LICENSE
Original file line number Diff line number Diff line change
Expand Up @@ -219,4 +219,4 @@ The text of each license is the standard Apache 2.0 license.
DolphinPluginClassLoader from https://github.com/prestosql/presto Apache 2.0
DolphinPluginDiscovery from https://github.com/prestosql/presto Apache 2.0
DolphinPluginLoader from https://github.com/prestosql/presto Apache 2.0

CodeGenerateUtils from https://github.com/twitter-archive/snowflake/tree/snowflake-2010 Apache 2.0
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,9 @@
import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils;
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils.CodeGenerateException;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils;
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils.SnowFlakeException;
import org.apache.dolphinscheduler.dao.entity.Environment;
import org.apache.dolphinscheduler.dao.entity.EnvironmentWorkerGroupRelation;
import org.apache.dolphinscheduler.dao.entity.TaskDefinition;
Expand Down Expand Up @@ -115,9 +115,9 @@ public Map<String, Object> createEnvironment(User loginUser, String name, String
env.setUpdateTime(new Date());
long code = 0L;
try {
code = SnowFlakeUtils.getInstance().nextId();
code = CodeGenerateUtils.getInstance().genCode();
env.setCode(code);
} catch (SnowFlakeException e) {
} catch (CodeGenerateException e) {
logger.error("Environment code get error, ", e);
}
if (code == 0L) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,10 +40,10 @@
import org.apache.dolphinscheduler.common.model.TaskNode;
import org.apache.dolphinscheduler.common.model.TaskNodeRelation;
import org.apache.dolphinscheduler.common.thread.Stopper;
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils;
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils.CodeGenerateException;
import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils;
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils.SnowFlakeException;
import org.apache.dolphinscheduler.dao.entity.DagData;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinitionLog;
Expand Down Expand Up @@ -222,8 +222,8 @@ public Map<String, Object> createProcessDefinition(User loginUser,
}
long processDefinitionCode;
try {
processDefinitionCode = SnowFlakeUtils.getInstance().nextId();
} catch (SnowFlakeException e) {
processDefinitionCode = CodeGenerateUtils.getInstance().genCode();
} catch (CodeGenerateException e) {
putMsg(result, Status.INTERNAL_SERVER_ERROR_ARGS);
return result;
}
Expand Down Expand Up @@ -874,8 +874,8 @@ private boolean checkAndImport(User loginUser, long projectCode, Map<String, Obj
processDefinition.setProjectCode(projectCode);
processDefinition.setUserId(loginUser.getId());
try {
processDefinition.setCode(SnowFlakeUtils.getInstance().nextId());
} catch (SnowFlakeException e) {
processDefinition.setCode(CodeGenerateUtils.getInstance().genCode());
} catch (CodeGenerateException e) {
putMsg(result, Status.CREATE_PROCESS_DEFINITION_ERROR);
return false;
}
Expand All @@ -894,10 +894,10 @@ private boolean checkAndImport(User loginUser, long projectCode, Map<String, Obj
taskDefinitionLog.setOperator(loginUser.getId());
taskDefinitionLog.setOperateTime(now);
try {
long code = SnowFlakeUtils.getInstance().nextId();
long code = CodeGenerateUtils.getInstance().genCode();
taskCodeMap.put(taskDefinitionLog.getCode(), code);
taskDefinitionLog.setCode(code);
} catch (SnowFlakeException e) {
} catch (CodeGenerateException e) {
logger.error("Task code get error, ", e);
putMsg(result, Status.INTERNAL_SERVER_ERROR_ARGS, "Error generating task definition code");
return false;
Expand Down Expand Up @@ -1363,8 +1363,8 @@ private void doBatchOperateProcessDefinition(User loginUser,
processDefinition.setProjectCode(targetProjectCode);
if (isCopy) {
try {
processDefinition.setCode(SnowFlakeUtils.getInstance().nextId());
} catch (SnowFlakeException e) {
processDefinition.setCode(CodeGenerateUtils.getInstance().genCode());
} catch (CodeGenerateException e) {
putMsg(result, Status.INTERNAL_SERVER_ERROR_ARGS);
throw new ServiceException(Status.INTERNAL_SERVER_ERROR_ARGS);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,8 @@
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils;
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils.SnowFlakeException;
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils;
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils.CodeGenerateException;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
import org.apache.dolphinscheduler.dao.entity.Project;
import org.apache.dolphinscheduler.dao.entity.ProjectUser;
Expand Down Expand Up @@ -97,14 +97,14 @@ public Map<String, Object> createProject(User loginUser, String name, String des
project = Project
.newBuilder()
.name(name)
.code(SnowFlakeUtils.getInstance().nextId())
.code(CodeGenerateUtils.getInstance().genCode())
.description(desc)
.userId(loginUser.getId())
.userName(loginUser.getUserName())
.createTime(now)
.updateTime(now)
.build();
} catch (SnowFlakeException e) {
} catch (CodeGenerateException e) {
putMsg(result, Status.CREATE_PROJECT_ERROR);
return result;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,9 @@
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.Flag;
import org.apache.dolphinscheduler.common.enums.ReleaseState;
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils;
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils.CodeGenerateException;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils;
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils.SnowFlakeException;
import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelation;
import org.apache.dolphinscheduler.dao.entity.Project;
import org.apache.dolphinscheduler.dao.entity.TaskDefinition;
Expand Down Expand Up @@ -431,9 +431,9 @@ public Map<String, Object> genTaskCodeList(Integer genNum) {
List<Long> taskCodes = new ArrayList<>();
try {
for (int i = 0; i < genNum; i++) {
taskCodes.add(SnowFlakeUtils.getInstance().nextId());
taskCodes.add(CodeGenerateUtils.getInstance().genCode());
}
} catch (SnowFlakeException e) {
} catch (CodeGenerateException e) {
logger.error("Task code get error, ", e);
putMsg(result, Status.INTERNAL_SERVER_ERROR_ARGS, "Error generating task definition code");
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
/** Copyright 2010-2012 Twitter, Inc.*/

package org.apache.dolphinscheduler.common.utils;

import java.net.InetAddress;
import java.net.UnknownHostException;
import java.util.Objects;

/**
* Rewriting based on Twitter snowflake algorithm
*/
public class CodeGenerateUtils {
// start timestamp
private static final long START_TIMESTAMP = 1609430400000L; //2021-01-01 00:00:00
// Each machine generates 32 in the same millisecond
private static final long LOW_DIGIT_BIT = 5L;
private static final long MIDDLE_BIT = 2L;
private static final long MAX_LOW_DIGIT = ~(-1L << LOW_DIGIT_BIT);
// The displacement to the left
private static final long MIDDLE_LEFT = LOW_DIGIT_BIT;
private static final long HIGH_DIGIT_LEFT = LOW_DIGIT_BIT + MIDDLE_BIT;
private final long machineHash;
private long lowDigit = 0L;
private long recordMillisecond = -1L;

private static final long SYSTEM_TIMESTAMP = System.currentTimeMillis();
private static final long SYSTEM_NANOTIME = System.nanoTime();

private CodeGenerateUtils() throws CodeGenerateException {
try {
this.machineHash = Math.abs(Objects.hash(InetAddress.getLocalHost().getHostName())) % (2 << (MIDDLE_BIT - 1));
} catch (UnknownHostException e) {
throw new CodeGenerateException(e.getMessage());
}
}

private static CodeGenerateUtils instance = null;

public static synchronized CodeGenerateUtils getInstance() throws CodeGenerateException {
if (instance == null) {
instance = new CodeGenerateUtils();
}
return instance;
}

public synchronized long genCode() throws CodeGenerateException {
long nowtMillisecond = systemMillisecond();
if (nowtMillisecond < recordMillisecond) {
throw new CodeGenerateException("New code exception because time is set back.");
}
if (nowtMillisecond == recordMillisecond) {
lowDigit = (lowDigit + 1) & MAX_LOW_DIGIT;
if (lowDigit == 0L) {
while (nowtMillisecond <= recordMillisecond) {
nowtMillisecond = systemMillisecond();
}
}
} else {
lowDigit = 0L;
}
recordMillisecond = nowtMillisecond;
return (nowtMillisecond - START_TIMESTAMP) << HIGH_DIGIT_LEFT | machineHash << MIDDLE_LEFT | lowDigit;
}

private long systemMillisecond() {
return SYSTEM_TIMESTAMP + (System.nanoTime() - SYSTEM_NANOTIME) / 1000000;
}

public static class CodeGenerateException extends Exception {
public CodeGenerateException(String message) {
super(message);
}
}
}

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -22,14 +22,14 @@
import org.junit.Assert;
import org.junit.Test;

public class SnowFlakeUtilsTest {
public class CodeGenerateUtilsTest {
@Test
public void testNoGenerateDuplicateId() throws SnowFlakeUtils.SnowFlakeException {
HashSet<Long> existsSnowFlakeId = new HashSet<>();
public void testNoGenerateDuplicateCode() throws CodeGenerateUtils.CodeGenerateException {
HashSet<Long> existsCode = new HashSet<>();
for (int i = 0; i < 100; i++) {
Long currentId = SnowFlakeUtils.getInstance().nextId();
Assert.assertFalse(existsSnowFlakeId.contains(currentId));
existsSnowFlakeId.add(currentId);
Long currentCode = CodeGenerateUtils.getInstance().genCode();
Assert.assertFalse(existsCode.contains(currentCode));
existsCode.add(currentCode);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.Flag;
import org.apache.dolphinscheduler.common.enums.ReleaseState;
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils;
import org.apache.dolphinscheduler.common.utils.ConnectionUtils;
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;

import java.sql.Connection;
Expand Down Expand Up @@ -110,7 +110,7 @@ public List<ProcessDefinition> queryProcessDefinition(Connection conn) {
processDefinition.setId(rs.getInt(1));
long code = rs.getLong(2);
if (code == 0L) {
code = SnowFlakeUtils.getInstance().nextId();
code = CodeGenerateUtils.getInstance().genCode();
}
processDefinition.setCode(code);
processDefinition.setVersion(Constants.VERSION_FIRST);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,8 @@

package org.apache.dolphinscheduler.dao.upgrade;

import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils;
import org.apache.dolphinscheduler.common.utils.ConnectionUtils;
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils;

import java.sql.Connection;
import java.sql.PreparedStatement;
Expand Down Expand Up @@ -51,7 +51,7 @@ public Map<Integer, Long> queryAllProject(Connection conn) {
Integer id = rs.getInt(1);
long code = rs.getLong(2);
if (code == 0L) {
code = SnowFlakeUtils.getInstance().nextId();
code = CodeGenerateUtils.getInstance().genCode();
}
projectMap.put(id, code);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,11 @@
import org.apache.dolphinscheduler.common.enums.TimeoutFlag;
import org.apache.dolphinscheduler.common.process.ResourceInfo;
import org.apache.dolphinscheduler.common.task.TaskTimeoutParameter;
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils;
import org.apache.dolphinscheduler.common.utils.ConnectionUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.SchemaUtils;
import org.apache.dolphinscheduler.common.utils.ScriptRunner;
import org.apache.dolphinscheduler.common.utils.SnowFlakeUtils;
import org.apache.dolphinscheduler.dao.datasource.ConnectionFactory;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinitionLog;
Expand Down Expand Up @@ -673,7 +673,7 @@ private void splitProcessDefinitionJson(List<ProcessDefinition> processDefinitio
String name = task.get("name").asText();
taskDefinitionLog.setName(name);
taskDefinitionLog.setWorkerGroup(task.get("workerGroup").asText());
long taskCode = SnowFlakeUtils.getInstance().nextId();
long taskCode = CodeGenerateUtils.getInstance().genCode();
taskDefinitionLog.setCode(taskCode);
taskDefinitionLog.setVersion(Constants.VERSION_FIRST);
taskDefinitionLog.setProjectCode(processDefinition.getProjectCode());
Expand Down
Loading

0 comments on commit 4bec792

Please sign in to comment.