Skip to content

Commit

Permalink
Merge pull request #110 from wolfboys/master
Browse files Browse the repository at this point in the history
v1.0.0-beta.2
  • Loading branch information
wolfboys authored Apr 22, 2021
2 parents fdeae1a + 8190df1 commit b95440e
Show file tree
Hide file tree
Showing 33 changed files with 1,017 additions and 140 deletions.
12 changes: 12 additions & 0 deletions streamx-console/streamx-console-service/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -202,6 +202,18 @@
<version>1.7.2</version>
</dependency>

<dependency>
<groupId>org.freemarker</groupId>
<artifactId>freemarker</artifactId>
<version>2.3.30</version>
</dependency>

<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-email</artifactId>
<version>1.5</version>
</dependency>

<!-- jwt -->
<dependency>
<groupId>com.auth0</groupId>
Expand Down
198 changes: 103 additions & 95 deletions streamx-console/streamx-console-service/src/assembly/conf/streamx.sql

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@
*/
package com.streamxhub.streamx.console.core.controller;

import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.streamxhub.streamx.console.base.controller.BaseController;
import com.streamxhub.streamx.console.base.domain.RestResponse;
import com.streamxhub.streamx.console.core.entity.Setting;
Expand Down Expand Up @@ -49,7 +51,8 @@ public class SettingController extends BaseController {
@PostMapping("all")
@RequiresPermissions("setting:view")
public RestResponse all() {
List<Setting> setting = settingService.list();
LambdaQueryWrapper<Setting> query = new QueryWrapper<Setting>().lambda().orderByAsc(Setting::getNum);
List<Setting> setting = settingService.list(query);
return RestResponse.create().data(setting);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,14 @@ public class Application implements Serializable {

private Integer optionState;

/**
* checkPoint 保留的最大条数
*/
private Integer cpThreshold;

//失败告警的通知邮箱
private String alertEmail;

private String args;
/**
* 应用程序模块
Expand All @@ -104,6 +112,7 @@ public class Application implements Serializable {
private String dynamicOptions;
private Integer appType;
private Boolean flameGraph;

/**
* 是否需要跟踪监控状态
*/
Expand All @@ -127,7 +136,6 @@ public class Application implements Serializable {
@TableField("TOTAL_TM")
private Integer totalTM;


private Integer totalSlot;
private Integer availableSlot;
private Integer jmMemory;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -52,4 +52,6 @@ public class SavePoint {
private Date triggerTime;

private Date createTime;

private transient Integer cpThreshold;
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
/*
* Copyright (c) 2019 The StreamX Project
* <p>
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package com.streamxhub.streamx.console.core.entity;

import lombok.Data;

@Data
public class SenderEmail {

private String smtpHost;
private Integer smtpPort;
private String email;
private String password;
private boolean ssl;

}
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,10 @@
@TableName("t_setting")
@Slf4j
public class Setting implements Serializable {

@TableField("`NUM`")
private Integer num;

@TableField("`TITLE`")
private String title;

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
/*
* Copyright (c) 2019 The StreamX Project
* <p>
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package com.streamxhub.streamx.console.core.service;

import com.streamxhub.streamx.console.core.entity.Application;
import com.streamxhub.streamx.console.core.enums.FlinkAppState;

/**
* @author benjobs
*/
public interface AlertService {

/**
* alert
*
* @param application
*/
void alert(Application application, FlinkAppState appState);

}
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@

import com.baomidou.mybatisplus.extension.service.IService;
import com.streamxhub.streamx.console.base.domain.RestResponse;
import com.streamxhub.streamx.console.core.entity.SenderEmail;
import com.streamxhub.streamx.console.core.entity.Setting;

/**
Expand All @@ -32,8 +33,14 @@ public interface SettingService extends IService<Setting> {
String KEY_MAVEN_REPOSITORY = "maven.central.repository";
String KEY_STREAMX_ADDRESS = "streamx.console.webapp.address";
String KEY_STREAMX_WORKSPACE = "streamx.console.workspace";

String KEY_ALERT_EMAIL_HOST = "alert.email.host";
String KEY_ALERT_EMAIL_PORT = "alert.email.port";
String KEY_ALERT_EMAIL_ADDRESS = "alert.email.address";
String KEY_ALERT_EMAIL_PASSWORD = "alert.email.password";
String KEY_ALERT_EMAIL_SSL = "alert.email.ssl";

String KEY_ENV_FLINK_HOME = "env.flink.home";
String KEY_CHECKPOINT_THRESHOLD = "streamx.console.checkpoint.threshold";

/**
* @param key
Expand All @@ -57,6 +64,5 @@ public interface SettingService extends IService<Setting> {

boolean checkWorkspace();

Integer getCheckpointThreshold();

SenderEmail getSenderEmail();
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,141 @@
/*
* Copyright (c) 2019 The StreamX Project
* <p>
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package com.streamxhub.streamx.console.core.service.impl;

import com.streamxhub.streamx.common.util.DateUtils;
import com.streamxhub.streamx.common.util.HadoopUtils;
import com.streamxhub.streamx.common.util.Utils;
import com.streamxhub.streamx.console.core.entity.Application;
import com.streamxhub.streamx.console.core.entity.SenderEmail;
import com.streamxhub.streamx.console.core.enums.FlinkAppState;
import com.streamxhub.streamx.console.core.service.AlertService;
import com.streamxhub.streamx.console.core.service.SettingService;
import freemarker.template.Configuration;
import freemarker.template.Template;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.mail.HtmlEmail;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;

import javax.annotation.PostConstruct;
import java.io.File;
import java.io.StringWriter;
import java.net.URL;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.Map;
import java.util.TimeZone;

/**
* @author benjobs
*/
@Slf4j
@Service
public class AlertServiceImpl implements AlertService {

private Template template;

@Autowired
private SettingService settingService;

private SenderEmail senderEmail;

@PostConstruct
public void initConfig() throws Exception {
Configuration configuration = new Configuration(Configuration.VERSION_2_3_28);
String template = "email.html";
Enumeration<URL> urls = ClassLoader.getSystemResources(template);
if (urls != null) {
if (!urls.hasMoreElements()) {
urls = Thread.currentThread().getContextClassLoader().getResources(template);
}
}
if (urls != null) {
while (urls.hasMoreElements()) {
URL url = urls.nextElement();
if (url.getPath().contains(".jar")) {
configuration.setClassLoaderForTemplateLoading(Thread.currentThread().getContextClassLoader(), "");
} else {
File file = new File(url.getPath());
configuration.setDirectoryForTemplateLoading(file.getParentFile());
}
configuration.setDefaultEncoding("UTF-8");
this.template = configuration.getTemplate(template);
break;
}
} else {
log.error("email.html not found!");
throw new ExceptionInInitializerError("email.html not found!");
}
}

@Override
public void alert(Application application, FlinkAppState appState) {
log.info("Email Alert:{} is {}", application.getJobName(), appState.name());
if (this.senderEmail == null) {
this.senderEmail = settingService.getSenderEmail();
}
if (this.senderEmail != null) {
//发送邮件
if (Utils.notEmpty(application.getAlertEmail())) {
try {
HtmlEmail htmlEmail = new HtmlEmail();
htmlEmail.setCharset("UTF-8");
htmlEmail.setHostName(this.senderEmail.getSmtpHost());
htmlEmail.setAuthentication(this.senderEmail.getEmail(), this.senderEmail.getPassword());
htmlEmail.setFrom(this.senderEmail.getEmail());
if (this.senderEmail.isSsl()) {
htmlEmail.setSSLOnConnect(true);
htmlEmail.setSslSmtpPort(this.senderEmail.getSmtpPort().toString());
}
htmlEmail.setSubject("StreamX Alert: [ " + application.getJobName() + " ] is " + appState.name());
String html = getHtmlMessage(application, appState);
htmlEmail.setHtmlMsg(html);
htmlEmail.addTo(application.getAlertEmail().split(","));
htmlEmail.send();
} catch (Exception e) {
e.printStackTrace();
}
}
}
}

private String getHtmlMessage(Application application, FlinkAppState appState) throws Exception {
String content = "Job [" + application.getJobName() + "] is " + appState.name() + "<br>" +
"Start Time: " + DateUtils.format(application.getStartTime(), DateUtils.fullFormat(), TimeZone.getDefault()) + "<br>" +
"End Time: " + DateUtils.format(application.getEndTime(), DateUtils.fullFormat(), TimeZone.getDefault()) + "<br>" +
"Duration: " + DateUtils.toRichTimeDuration(application.getEndTime().getTime() - application.getStartTime().getTime()) + "<br><br>" +
"please check it,Thank you for using StreamX<br><br>" +
"Best Wishes!!";

Map<String, String> root = new HashMap<>();
root.put("title", "Notify :" + application.getJobName().concat(" is ").concat(appState.name()));
root.put("message", content);
String format = "%s/proxy/%s/";
String url = String.format(format, HadoopUtils.rmHttpAddress(false), application.getAppId());
root.put("link", url);

StringWriter writer = new StringWriter();
template.process(root, writer);
return writer.toString();
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -423,6 +423,7 @@ public boolean update(Application appParam) {
application.setDescription(appParam.getDescription());
application.setResolveOrder(appParam.getResolveOrder());
application.setExecutionMode(appParam.getExecutionMode());
application.setAlertEmail(appParam.getAlertEmail());
// Flink Sql job...
if (application.isFlinkSqlJob()) {
updateFlinkSqlJob(application, appParam);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
import com.streamxhub.streamx.console.base.utils.SortUtil;
import com.streamxhub.streamx.console.core.dao.SavePointMapper;
import com.streamxhub.streamx.console.core.entity.SavePoint;
import com.streamxhub.streamx.console.core.service.ApplicationService;
import com.streamxhub.streamx.console.core.service.SavePointService;
import com.streamxhub.streamx.console.core.service.SettingService;
import lombok.extern.slf4j.Slf4j;
Expand All @@ -55,7 +56,7 @@ public class SavePointServiceImpl extends ServiceImpl<SavePointMapper, SavePoint


@Autowired
private SettingService settingService;
private ApplicationService settingService;

@Override
public void obsolete(Long appId) {
Expand All @@ -70,8 +71,8 @@ public boolean save(SavePoint entity) {
}

private void expire(SavePoint entity) {
Integer threshold = settingService.getCheckpointThreshold();
LambdaQueryWrapper<SavePoint> queryWrapper = new QueryWrapper<SavePoint>().lambda();
Integer threshold = entity.getCpThreshold();
queryWrapper.select(SavePoint::getTriggerTime)
.eq(SavePoint::getAppId, entity.getAppId())
.orderByDesc(SavePoint::getTriggerTime)
Expand Down
Loading

0 comments on commit b95440e

Please sign in to comment.