From 3145160563afd81dae371e048f166138fa212dd2 Mon Sep 17 00:00:00 2001 From: ideal Date: Thu, 30 Sep 2021 12:22:39 +0800 Subject: [PATCH 1/3] support jdk 17 update flink version to 1.14.3 update spark version to 3.2.0 --- .travis.yml | 3 + README.md | 19 +- build.gradle | 104 ++--- gradle.properties | 2 +- gradle/wrapper/gradle-wrapper.properties | 2 +- json-reader/build.gradle | 8 + .../sylph/json/ByteCodeClassLoader.java | 16 +- .../harbby/sylph/json/JsonPathReader.java | 37 +- .../sylph/json/JsonReadCodeGenerator.java | 109 +++++ .../github/harbby/sylph/json/KafkaRecord.java | 19 +- ...ile-java11.gradle => profile-java17.gradle | 13 +- profile-java8.gradle | 7 - profile-runtime.gradle | 3 + settings.gradle | 4 +- src/checkstyle/facebook_checks.xml | 103 +++-- .../intellij-java-facebook-style.xml | 29 +- sylph-api/build.gradle | 3 +- .../github/harbby/sylph/api}/Field.java | 7 +- .../github/harbby/sylph/api}/Operator.java | 2 +- .../github/harbby/sylph/api}/Plugin.java | 2 +- .../harbby/sylph/api}/PluginConfig.java | 2 +- .../harbby/sylph}/api/RealTimePipeline.java | 8 +- .../harbby/sylph}/api/RealTimeSink.java | 11 +- .../github/harbby/sylph/api}/Record.java | 6 +- .../github/harbby/sylph/api}/Schema.java | 15 +- .../github/harbby/sylph}/api/Sink.java | 11 +- .../github/harbby/sylph}/api/Source.java | 6 +- .../harbby/sylph/api}/TableContext.java | 4 +- .../sylph/api}/annotation/Description.java | 2 +- .../harbby/sylph/api}/annotation/Name.java | 2 +- .../harbby/sylph/api}/annotation/Version.java | 2 +- .../main/java/ideal/sylph/etl/Collector.java | 35 -- .../ideal/sylph/etl/api/JoinOperator.java | 22 -- .../sylph/etl/api/RealTimeTransForm.java | 38 -- .../ideal/sylph/etl/join/SelectField.java | 105 ----- .../java/ideal/sylph/etl/api/SinkTest.java | 1 + sylph-connectors/build.gradle | 11 +- sylph-connectors/flink-kafka/build.gradle | 13 +- .../plugins/kafka/flink/KafkaBaseSource.java | 205 ---------- .../plugins/kafka/flink/KafkaSource.java | 85 ++-- .../kafka/flink/KafkaSourceConfig.java | 8 +- .../sylph/plugins/kafka/flink/Plugin.java | 10 +- .../runtime/JsonDeserializationSchema.java | 140 +++++++ .../plugins/kafka/flink/KafkaSourceTest.java | 66 ---- sylph-connectors/spark-kafka/build.gradle | 1 + .../sylph/plugins/kafka/spark/JsonSchema.java | 111 ------ .../plugins/kafka/spark/KafkaSource.java | 193 ++++++--- .../kafka/spark/KafkaSourceConfig.java | 6 +- .../sylph/plugins/kafka/spark/Plugin.java | 16 +- .../plugins/kafka/spark/SocketSource.java | 47 +-- .../kafka/spark/StructuredKafkaSource.java | 186 ++++++--- .../spark/structured/KafkaSourceUtil.java | 18 +- .../plugins/kafka/spark/util/DStreamUtil.java | 79 ---- .../ideal/sylph/plugins/example/Plugin.java | 11 +- .../sylph/plugins/example/PrintSink.java | 8 +- .../sylph/plugins/example/TestTrans.java | 48 --- sylph-connectors/sylph-hdfs/build.gradle | 9 - .../ideal/sylph/plugins/hdfs/HdfsSink.java | 149 ------- .../java/ideal/sylph/plugins/hdfs/Plugin.java | 33 -- .../plugins/hdfs/factory/HDFSFactorys.java | 97 ----- .../plugins/hdfs/factory/TimeParser.java | 77 ---- .../plugins/hdfs/txt/TextFileFactory.java | 197 --------- .../plugins/hdfs/txt/TextTimeParser.java | 53 --- .../sylph/plugins/hdfs/utils/CommonUtil.java | 64 --- .../sylph/plugins/hdfs/utils/MemoryUtil.java | 91 ----- .../sylph/plugins/hdfs/utils/ParquetUtil.java | 74 ---- .../plugins/hdfs/utils/ParquetUtilTest.java | 73 ---- sylph-connectors/sylph-kudu/build.gradle | 6 - .../ideal/sylph/plugins/kudu/KuduSink.java | 22 +- .../java/ideal/sylph/plugins/kudu/Plugin.java | 10 +- .../sylph/plugins/kudu/KuduSinkTest.java | 116 ------ sylph-connectors/sylph-mysql/build.gradle | 1 - .../sylph/plugins/jdbc/JdbcAsyncJoin.java | 235 ----------- .../sylph/plugins/jdbc/JdbcRealTimeSink.java | 24 +- .../ideal/sylph/plugins/jdbc/JdbcUtils.java | 55 --- .../sylph/plugins/mysql/MysqlAsyncJoin.java | 41 -- .../ideal/sylph/plugins/mysql/MysqlSink.java | 4 +- .../ideal/sylph/plugins/mysql/Plugin.java | 11 +- .../sylph/plugins/mysql/MysqlSinkTest.java | 53 --- sylph-dist/build.gradle | 21 +- sylph-dist/src/bin/sylph | 2 +- sylph-dist/src/bin/sylph.bat | 2 +- sylph-dist/src/data/data.db | Bin 77824 -> 77824 bytes sylph-dist/src/etc/sylph/sylph.properties | 6 +- sylph-dist/src/webapp/asset-manifest.json | 17 + sylph-dist/src/webapp/favicon.ico | Bin 0 -> 270398 bytes sylph-dist/src/webapp/index.html | 1 + sylph-dist/src/webapp/manifest.json | 15 + ...nifest.8d18d443a962454a50ea0284d745a6c5.js | 26 ++ ...nifest.d88bc1619cb00da833b4a341043e357f.js | 26 ++ sylph-dist/src/webapp/service-worker.js | 39 ++ .../webapp/static/css/2.174b1a74.chunk.css | 12 + .../static/css/2.174b1a74.chunk.css.map | 1 + .../webapp/static/css/2.bda4aaa2.chunk.css | 12 + .../static/css/2.bda4aaa2.chunk.css.map | 1 + .../webapp/static/css/main.8b0312c3.chunk.css | 2 + .../static/css/main.8b0312c3.chunk.css.map | 1 + .../src/webapp/static/js/2.4fd4a0fa.chunk.js | 2 + .../webapp/static/js/2.4fd4a0fa.chunk.js.map | 1 + .../src/webapp/static/js/2.77393c61.chunk.js | 2 + .../webapp/static/js/2.77393c61.chunk.js.map | 1 + .../webapp/static/js/main.02c07421.chunk.js | 2 + .../static/js/main.02c07421.chunk.js.map | 1 + .../webapp/static/js/main.b695cdd2.chunk.js | 2 + .../static/js/main.b695cdd2.chunk.js.map | 1 + .../webapp/static/js/runtime~main.a8a9905a.js | 2 + .../static/js/runtime~main.a8a9905a.js.map | 1 + sylph-main/build.gradle | 27 +- .../harbby/sylph/main/SylphException.java | 24 +- .../harbby}/sylph/main/SylphMaster.java | 66 ++-- .../harbby/sylph/main/dao/JobRepository.java | 34 ++ .../sylph/main/dao/JobRepositoryImpl.java | 105 +++++ .../sylph/main/dao/StatusRepository.java | 48 +++ .../sylph/main/dao/StatusRepositoryImpl.java | 114 ++++++ .../sylph/main/server/ServerMainConfig.java | 26 +- .../harbby}/sylph/main/server/SylphBean.java | 29 +- .../sylph/main/server/SylphContextImpl.java | 57 ++- .../sylph/main/service/JobCompiler.java | 118 ++++++ .../sylph/main/service/JobEngineManager.java | 109 +++++ .../sylph/main/service/JobEngineWrapper.java | 59 +++ .../harbby/sylph/main/service/JobManager.java | 292 ++++++++++++++ .../sylph/main/service/OperatorManager.java | 107 ++--- .../sylph/main/util/PropertiesUtil.java | 2 +- .../sylph/main/service/JobEngineImpl.java | 137 ------- .../sylph/main/service/JobEngineManager.java | 190 --------- .../ideal/sylph/main/service/JobManager.java | 249 ------------ .../sylph/main/service/SqliteDbJobStore.java | 154 -------- sylph-main/src/test/java/JansiTest.java | 35 -- sylph-parser/build.gradle | 33 +- .../harbby}/sylph/parser/antlr4/SqlBase.g4 | 2 +- .../harbby/sylph/parser}/AntlrSqlParser.java | 10 +- .../harbby/sylph/parser}/AstBuilder.java | 84 ++-- .../sylph/parser}/CaseInsensitiveStream.java | 2 +- .../sylph/parser}/ParsingException.java | 4 +- .../sylph/parser/SqlParserException.java | 2 +- .../sylph/parser}/StatementSplitter.java | 2 +- .../sylph/parser}/tree/AllowedLateness.java | 7 +- .../sylph/parser}/tree/BooleanLiteral.java | 8 +- .../sylph/parser}/tree/ColumnDefinition.java | 21 +- .../sylph/parser}/tree/CreateFunction.java | 10 +- .../parser}/tree/CreateStreamAsSelect.java | 24 +- .../sylph/parser}/tree/CreateTable.java | 29 +- .../sylph/parser}/tree/DoubleLiteral.java | 10 +- .../harbby/sylph/parser}/tree/Expression.java | 6 +- .../harbby/sylph/parser}/tree/Identifier.java | 14 +- .../harbby/sylph/parser}/tree/InsertInto.java | 11 +- .../harbby/sylph/parser}/tree/Literal.java | 5 +- .../sylph/parser}/tree/LongLiteral.java | 10 +- .../harbby/sylph/parser}/tree/Node.java | 14 +- .../sylph/parser}/tree/NodeLocation.java | 5 +- .../harbby/sylph/parser}/tree/Proctime.java | 8 +- .../harbby/sylph/parser}/tree/Property.java | 10 +- .../sylph/parser}/tree/QualifiedName.java | 5 +- .../sylph/parser}/tree/SelectQuery.java | 29 +- .../harbby/sylph/parser}/tree/Statement.java | 6 +- .../sylph/parser}/tree/StringLiteral.java | 8 +- .../sylph/parser}/tree/TableElement.java | 6 +- .../harbby/sylph/parser}/tree/WaterMark.java | 5 +- .../sylph/parser}/tree/WindowTrigger.java | 7 +- .../parser/calcite/CalciteSqlParser.java | 238 ----------- .../ideal/sylph/parser/calcite/JoinInfo.java | 125 ------ .../ideal/sylph/parser/calcite/TableName.java | 56 --- .../sylph/parser}/tree/StringLiteralTest.java | 2 +- .../src/test/java/test/TriggerSql.java | 68 ---- sylph-runners/build.gradle | 19 +- sylph-runners/flink/build.gradle | 25 +- .../harbby}/sylph/runner/flink/FlinkBean.java | 2 +- .../sylph/runner/flink/FlinkJobClient.java | 244 ++++++++++++ .../sylph/runner/flink/FlinkJobConfig.java | 50 +-- .../runner/flink/FlinkOperatorFactory.java | 70 ++++ .../sylph/runner/flink/FlinkRunner.java | 150 +++++++ .../runner/flink/engines/FlinkEnvFactory.java | 66 ++++ .../flink/engines/FlinkMainClassEngine.java | 94 +---- .../flink/engines/FlinkStreamSqlEngine.java | 71 ++++ .../flink/engines/StreamSqlBuilder.java | 222 +++++++++++ .../runner/flink/engines/StreamSqlUtil.java | 72 +--- .../runner/flink/local/MiniExecutor.java | 6 +- .../runner/flink/plugins/TestSource.java | 20 +- .../sylph/runner/flink/runtime}/ArrayAgg.java | 13 +- .../runner/flink/runtime}/FlinkRecord.java | 37 +- .../runner/flink/runtime}/FlinkSink.java | 4 +- .../sylph/runner/flink/runtime}/UDFJson.java | 18 +- .../runner/flink/FlinkContainerFactory.java | 216 ---------- .../ideal/sylph/runner/flink/FlinkRunner.java | 92 ----- .../flink/SylphFsCheckpointStorage.java | 195 --------- .../runner/flink/engines/FlinkEnvFactory.java | 98 ----- .../flink/engines/FlinkStreamEtlEngine.java | 130 ------ .../flink/engines/FlinkStreamSqlEngine.java | 178 --------- .../flink/engines/StreamSqlBuilder.java | 270 ------------- .../runner/flink/etl/FlinkNodeLoader.java | 215 ---------- .../runner/flink/etl/FlinkTransFrom.java | 77 ---- .../runner/flink/sql/AsyncFunctionHelper.java | 112 ------ .../runner/flink/sql/FlinkSqlParser.java | 373 ------------------ .../runner/flink/sql/JoinContextImpl.java | 109 ----- .../runner/flink/sql/TriggerWindowHelper.java | 109 ----- .../runner/flink/table/SylphTableSink.java | 79 ---- .../runner/flink/table/SylphTableSource.java | 78 ---- .../sylph/runner/flink/udf/JsonParser.java | 72 ---- .../ideal/sylph/runner/flink/udf/RowGet.java | 43 -- .../sylph/runner/flink/udf/TimeUtil.java | 91 ----- .../flink/yarn/FlinkYarnJobLauncher.java | 162 -------- .../runner/flink/yarn/YarnJobDescriptor.java | 100 ----- .../engines/FlinkStreamSqlActuatorTest.java | 2 +- .../flink/engines/StreamSqlBuilderTest.java | 5 +- .../flink/resource/SylphTestSource.java | 8 +- .../sylph/runner/flink/sqlTest/JoinTest.java | 238 ----------- .../runner/flink/sqlTest/JsonPathUdfTest.java | 2 +- sylph-runners/spark/build.gradle | 34 +- .../harbby}/sylph/runner/spark/SQLHepler.java | 40 +- .../sylph/runner/spark/SparkJobConfig.java | 23 +- .../runner/spark/SparkMainClassEngine.java | 72 ++++ .../sylph/runner/spark/SparkRecord.java | 8 +- .../sylph/runner/spark/SparkRunner.java | 131 ++++++ .../spark/SparkStreamingSqlAnalyse.java | 89 ++--- .../runner/spark/SparkStreamingSqlEngine.java | 90 +++++ .../harbby/sylph/runner/spark/SparkUtil.java | 37 ++ .../sylph/runner/spark/SqlAnalyse.java | 15 +- .../spark/StructuredStreamingSqlAnalyse.java | 84 ++-- .../spark/StructuredStreamingSqlEngine.java | 101 +++++ .../runner/spark/kafka/SylphKafkaOffset.java | 2 +- .../runner/spark/runtime}/SparkAppMain.java | 16 +- .../spark/sparkstreaming/DStreamUtil.java | 5 +- .../SparkStreamingOperatorFactory.java | 140 +++++++ .../structured/StructuredNodeLoader.java | 139 +++++++ .../runner/spark/yarn/SparkJobClient.java | 181 +++++++++ .../spark/yarn/SylphSparkYarnClient.java | 7 +- .../ideal/sylph/runner/spark/JobHelper.java | 140 ------- .../runner/spark/SparkContainerFactory.java | 123 ------ .../runner/spark/SparkMainClassEngine.java | 113 ------ .../ideal/sylph/runner/spark/SparkRunner.java | 83 ---- .../runner/spark/SparkStreamingSqlEngine.java | 176 --------- .../sylph/runner/spark/Stream2EtlEngine.java | 65 --- .../sylph/runner/spark/StreamEtlEngine.java | 67 ---- .../spark/StructuredStreamingSqlEngine.java | 127 ------ .../sparkstreaming/StreamNodeLoader.java | 223 ----------- .../structured/StructuredNodeLoader.java | 207 ---------- .../sylph/runner/spark/udf/TimeUtil.java | 89 ----- .../runner/spark/yarn/SparkAppLauncher.java | 140 ------- .../runner/spark/SparkJobConfigTest.java | 3 + sylph-spi/build.gradle | 20 +- .../harbby/sylph/spi/CompileJobException.java | 32 ++ .../github/harbby/sylph/spi/CompileTask.java | 45 +++ .../github/harbby/sylph/spi/JobClient.java} | 31 +- .../harbby/sylph/spi/OperatorFactory.java | 29 +- .../harbby/sylph/spi}/OperatorInfo.java | 86 ++-- .../harbby/sylph/spi}/OperatorType.java | 13 +- .../com/github/harbby/sylph/spi/Runner.java | 65 +++ .../harbby}/sylph/spi/SylphContext.java | 24 +- .../com/github/harbby/sylph/spi/dao/Job.java | 97 +++++ .../github/harbby/sylph/spi/dao}/JobInfo.java | 10 +- .../harbby/sylph/spi/dao/JobRunState.java | 106 +++++ .../harbby/sylph/spi/job/DeployResponse.java | 19 +- .../harbby}/sylph/spi/job/JobConfig.java | 5 +- .../github/harbby/sylph/spi/job/JobDag.java | 64 +++ .../harbby/sylph/spi/job/JobEngine.java} | 41 +- .../harbby/sylph/spi/job/JobParser.java | 73 ++++ .../sylph/spi/job/MainClassJobParser.java | 32 +- .../harbby/sylph/spi/job/SqlJobParser.java | 127 ++++++ .../sylph/spi/utils/PluginFactory.java} | 79 +++- .../java/ideal/sylph/spi/GraphAppUtil.java | 66 ---- .../main/java/ideal/sylph/spi/NodeLoader.java | 56 --- .../ideal/sylph/spi/OperatorMetaData.java | 89 ----- .../src/main/java/ideal/sylph/spi/Runner.java | 77 ---- .../java/ideal/sylph/spi/RunnerContext.java | 23 -- .../ideal/sylph/spi/exception/ErrorCode.java | 90 ----- .../ideal/sylph/spi/exception/ErrorType.java | 24 -- .../spi/exception/StandardErrorCode.java | 58 --- .../sylph/spi/exception/SylphException.java | 62 --- .../java/ideal/sylph/spi/job/EtlFlow.java | 100 ----- .../sylph/spi/job/EtlJobEngineHandle.java | 68 ---- .../main/java/ideal/sylph/spi/job/Flow.java | 25 -- .../main/java/ideal/sylph/spi/job/Job.java | 91 ----- .../ideal/sylph/spi/job/JobContainer.java | 74 ---- .../ideal/sylph/spi/job/JobContainerAbs.java | 78 ---- .../java/ideal/sylph/spi/job/JobStore.java | 67 ---- .../java/ideal/sylph/spi/job/SqlFlow.java | 66 ---- .../java/ideal/sylph/spi/model/EdgeInfo.java | 71 ---- .../java/ideal/sylph/spi/model/NodeInfo.java | 144 ------- .../sylph/spi/utils/GenericTypeReference.java | 44 --- .../ideal/sylph/spi/utils/JsonTextUtil.java | 71 ---- .../ideal/sylph/spi/GraphAppUtilTest.java | 127 ------ .../java/ideal/sylph/spi/NodeLoaderTest.java | 133 ------- .../spi/{model => }/OperatorManagerTest.java | 15 +- .../sylph/spi/PluginConfigFactoryTest.java | 11 +- .../java/ideal/sylph/spi/TestConfigs.java | 12 +- .../java/ideal/sylph/spi/job/SqlFlowTest.java | 40 -- .../sylph/spi/utils/JsonTextUtilTest.java | 48 --- sylph-web/build.gradle | 43 +- .../colltroller}/AppExceptionMapper.java | 16 +- .../harbby/sylph/colltroller}/AuthAspect.java | 17 +- .../sylph/colltroller}/ControllerApp.java | 10 +- .../sylph/colltroller}/JettyServer.java | 36 +- .../sylph/colltroller}/LogAppender.java | 2 +- .../sylph/colltroller}/ServerConfig.java | 2 +- .../sylph/colltroller}/WebApplication.java | 4 +- .../action/JobManagerResource.java | 61 ++- .../colltroller}/action/LoginController.java | 29 +- .../action/PluginManagerResource.java | 27 +- .../action/ServerLogResource.java | 23 +- .../selvet/WebAppProxyServlet.java | 51 +-- .../colltroller}/utils/JsonFormatUtil.java | 10 +- .../sylph/colltroller}/utils/ProxyUtil.java | 16 +- .../sylph/controller/action/EtlResource.java | 84 ---- .../controller/selvet/ProxyAllHttpServer.java | 90 ----- sylph-web/src/main/webapp/src/JobList.js | 2 +- sylph-web/src/main/webapp/yarn.lock | 8 +- .../utils/JsonFormatUtilTest.java | 2 +- sylph-yarn/build.gradle | 5 +- .../harbby/sylph/yarn/YarnClientFactory.java | 59 +++ .../harbby/sylph/yarn/YarnDeployResponse.java | 51 +++ .../sylph/runtime/local/LocalContainer.java | 132 ------- .../sylph/runtime/yarn/YarnJobContainer.java | 224 ----------- .../ideal/sylph/runtime/yarn/YarnModule.java | 96 ----- 313 files changed, 5257 insertions(+), 12264 deletions(-) create mode 100644 json-reader/build.gradle rename sylph-spi/src/main/java/ideal/sylph/spi/job/ContainerFactory.java => json-reader/src/main/java/com/github/harbby/sylph/json/ByteCodeClassLoader.java (64%) rename sylph-connectors/flink-kafka/src/main/java/ideal/sylph/plugins/kafka/flink/JsonDeserializer.java => json-reader/src/main/java/com/github/harbby/sylph/json/JsonPathReader.java (63%) create mode 100644 json-reader/src/main/java/com/github/harbby/sylph/json/JsonReadCodeGenerator.java rename sylph-connectors/sylph-hdfs/src/main/java/ideal/sylph/plugins/hdfs/OutputFormat.java => json-reader/src/main/java/com/github/harbby/sylph/json/KafkaRecord.java (70%) rename profile-java11.gradle => profile-java17.gradle (69%) delete mode 100644 profile-java8.gradle create mode 100644 profile-runtime.gradle rename sylph-api/src/main/java/{ideal/sylph/etl => com/github/harbby/sylph/api}/Field.java (93%) rename sylph-api/src/main/java/{ideal/sylph/etl => com/github/harbby/sylph/api}/Operator.java (94%) rename sylph-api/src/main/java/{ideal/sylph/etl => com/github/harbby/sylph/api}/Plugin.java (95%) rename sylph-api/src/main/java/{ideal/sylph/etl => com/github/harbby/sylph/api}/PluginConfig.java (97%) rename sylph-api/src/main/java/{ideal/sylph/etl => com/github/harbby/sylph}/api/RealTimePipeline.java (84%) rename sylph-api/src/main/java/{ideal/sylph/etl => com/github/harbby/sylph}/api/RealTimeSink.java (79%) rename sylph-api/src/main/java/{ideal/sylph/etl => com/github/harbby/sylph/api}/Record.java (93%) rename sylph-api/src/main/java/{ideal/sylph/etl => com/github/harbby/sylph/api}/Schema.java (81%) rename sylph-api/src/main/java/{ideal/sylph/etl => com/github/harbby/sylph}/api/Sink.java (78%) rename sylph-api/src/main/java/{ideal/sylph/etl => com/github/harbby/sylph}/api/Source.java (88%) rename sylph-api/src/main/java/{ideal/sylph => com/github/harbby/sylph/api}/TableContext.java (94%) rename sylph-api/src/main/java/{ideal/sylph => com/github/harbby/sylph/api}/annotation/Description.java (94%) rename sylph-api/src/main/java/{ideal/sylph => com/github/harbby/sylph/api}/annotation/Name.java (94%) rename sylph-api/src/main/java/{ideal/sylph => com/github/harbby/sylph/api}/annotation/Version.java (94%) delete mode 100644 sylph-api/src/main/java/ideal/sylph/etl/Collector.java delete mode 100644 sylph-api/src/main/java/ideal/sylph/etl/api/JoinOperator.java delete mode 100644 sylph-api/src/main/java/ideal/sylph/etl/api/RealTimeTransForm.java delete mode 100644 sylph-api/src/main/java/ideal/sylph/etl/join/SelectField.java delete mode 100644 sylph-connectors/flink-kafka/src/main/java/ideal/sylph/plugins/kafka/flink/KafkaBaseSource.java create mode 100644 sylph-connectors/flink-kafka/src/main/java/ideal/sylph/plugins/kafka/flink/runtime/JsonDeserializationSchema.java delete mode 100644 sylph-connectors/flink-kafka/src/test/java/ideal/sylph/plugins/kafka/flink/KafkaSourceTest.java delete mode 100644 sylph-connectors/spark-kafka/src/main/java/ideal/sylph/plugins/kafka/spark/JsonSchema.java delete mode 100644 sylph-connectors/spark-kafka/src/main/java/ideal/sylph/plugins/kafka/spark/util/DStreamUtil.java delete mode 100644 sylph-connectors/sylph-example/src/main/java/ideal/sylph/plugins/example/TestTrans.java delete mode 100644 sylph-connectors/sylph-hdfs/build.gradle delete mode 100644 sylph-connectors/sylph-hdfs/src/main/java/ideal/sylph/plugins/hdfs/HdfsSink.java delete mode 100644 sylph-connectors/sylph-hdfs/src/main/java/ideal/sylph/plugins/hdfs/Plugin.java delete mode 100644 sylph-connectors/sylph-hdfs/src/main/java/ideal/sylph/plugins/hdfs/factory/HDFSFactorys.java delete mode 100644 sylph-connectors/sylph-hdfs/src/main/java/ideal/sylph/plugins/hdfs/factory/TimeParser.java delete mode 100644 sylph-connectors/sylph-hdfs/src/main/java/ideal/sylph/plugins/hdfs/txt/TextFileFactory.java delete mode 100644 sylph-connectors/sylph-hdfs/src/main/java/ideal/sylph/plugins/hdfs/txt/TextTimeParser.java delete mode 100644 sylph-connectors/sylph-hdfs/src/main/java/ideal/sylph/plugins/hdfs/utils/CommonUtil.java delete mode 100755 sylph-connectors/sylph-hdfs/src/main/java/ideal/sylph/plugins/hdfs/utils/MemoryUtil.java delete mode 100644 sylph-connectors/sylph-hdfs/src/main/java/ideal/sylph/plugins/hdfs/utils/ParquetUtil.java delete mode 100644 sylph-connectors/sylph-hdfs/src/test/java/ideal/sylph/plugins/hdfs/utils/ParquetUtilTest.java delete mode 100644 sylph-connectors/sylph-kudu/src/test/java/ideal/sylph/plugins/kudu/KuduSinkTest.java delete mode 100644 sylph-connectors/sylph-mysql/src/main/java/ideal/sylph/plugins/jdbc/JdbcAsyncJoin.java delete mode 100644 sylph-connectors/sylph-mysql/src/main/java/ideal/sylph/plugins/jdbc/JdbcUtils.java delete mode 100644 sylph-connectors/sylph-mysql/src/main/java/ideal/sylph/plugins/mysql/MysqlAsyncJoin.java delete mode 100644 sylph-connectors/sylph-mysql/src/test/java/ideal/sylph/plugins/mysql/MysqlSinkTest.java create mode 100644 sylph-dist/src/webapp/asset-manifest.json create mode 100644 sylph-dist/src/webapp/favicon.ico create mode 100644 sylph-dist/src/webapp/index.html create mode 100644 sylph-dist/src/webapp/manifest.json create mode 100644 sylph-dist/src/webapp/precache-manifest.8d18d443a962454a50ea0284d745a6c5.js create mode 100644 sylph-dist/src/webapp/precache-manifest.d88bc1619cb00da833b4a341043e357f.js create mode 100644 sylph-dist/src/webapp/service-worker.js create mode 100644 sylph-dist/src/webapp/static/css/2.174b1a74.chunk.css create mode 100644 sylph-dist/src/webapp/static/css/2.174b1a74.chunk.css.map create mode 100644 sylph-dist/src/webapp/static/css/2.bda4aaa2.chunk.css create mode 100644 sylph-dist/src/webapp/static/css/2.bda4aaa2.chunk.css.map create mode 100644 sylph-dist/src/webapp/static/css/main.8b0312c3.chunk.css create mode 100644 sylph-dist/src/webapp/static/css/main.8b0312c3.chunk.css.map create mode 100644 sylph-dist/src/webapp/static/js/2.4fd4a0fa.chunk.js create mode 100644 sylph-dist/src/webapp/static/js/2.4fd4a0fa.chunk.js.map create mode 100644 sylph-dist/src/webapp/static/js/2.77393c61.chunk.js create mode 100644 sylph-dist/src/webapp/static/js/2.77393c61.chunk.js.map create mode 100644 sylph-dist/src/webapp/static/js/main.02c07421.chunk.js create mode 100644 sylph-dist/src/webapp/static/js/main.02c07421.chunk.js.map create mode 100644 sylph-dist/src/webapp/static/js/main.b695cdd2.chunk.js create mode 100644 sylph-dist/src/webapp/static/js/main.b695cdd2.chunk.js.map create mode 100644 sylph-dist/src/webapp/static/js/runtime~main.a8a9905a.js create mode 100644 sylph-dist/src/webapp/static/js/runtime~main.a8a9905a.js.map rename sylph-yarn/src/main/java/ideal/sylph/runtime/local/Kernel32.java => sylph-main/src/main/java/com/github/harbby/sylph/main/SylphException.java (63%) rename sylph-main/src/main/java/{ideal => com/github/harbby}/sylph/main/SylphMaster.java (55%) create mode 100644 sylph-main/src/main/java/com/github/harbby/sylph/main/dao/JobRepository.java create mode 100644 sylph-main/src/main/java/com/github/harbby/sylph/main/dao/JobRepositoryImpl.java create mode 100644 sylph-main/src/main/java/com/github/harbby/sylph/main/dao/StatusRepository.java create mode 100644 sylph-main/src/main/java/com/github/harbby/sylph/main/dao/StatusRepositoryImpl.java rename sylph-main/src/main/java/{ideal => com/github/harbby}/sylph/main/server/ServerMainConfig.java (64%) rename sylph-main/src/main/java/{ideal => com/github/harbby}/sylph/main/server/SylphBean.java (64%) rename sylph-main/src/main/java/{ideal => com/github/harbby}/sylph/main/server/SylphContextImpl.java (66%) create mode 100644 sylph-main/src/main/java/com/github/harbby/sylph/main/service/JobCompiler.java create mode 100644 sylph-main/src/main/java/com/github/harbby/sylph/main/service/JobEngineManager.java create mode 100644 sylph-main/src/main/java/com/github/harbby/sylph/main/service/JobEngineWrapper.java create mode 100644 sylph-main/src/main/java/com/github/harbby/sylph/main/service/JobManager.java rename sylph-main/src/main/java/{ideal => com/github/harbby}/sylph/main/service/OperatorManager.java (68%) rename sylph-main/src/main/java/{ideal => com/github/harbby}/sylph/main/util/PropertiesUtil.java (97%) mode change 100755 => 100644 delete mode 100644 sylph-main/src/main/java/ideal/sylph/main/service/JobEngineImpl.java delete mode 100644 sylph-main/src/main/java/ideal/sylph/main/service/JobEngineManager.java delete mode 100644 sylph-main/src/main/java/ideal/sylph/main/service/JobManager.java delete mode 100644 sylph-main/src/main/java/ideal/sylph/main/service/SqliteDbJobStore.java delete mode 100644 sylph-main/src/test/java/JansiTest.java rename sylph-parser/src/main/antlr/{ideal => com/github/harbby}/sylph/parser/antlr4/SqlBase.g4 (99%) mode change 100755 => 100644 rename sylph-parser/src/main/java/{ideal/sylph/parser/antlr => com/github/harbby/sylph/parser}/AntlrSqlParser.java (92%) rename sylph-parser/src/main/java/{ideal/sylph/parser/antlr => com/github/harbby/sylph/parser}/AstBuilder.java (86%) rename sylph-parser/src/main/java/{ideal/sylph/parser/antlr => com/github/harbby/sylph/parser}/CaseInsensitiveStream.java (97%) mode change 100755 => 100644 rename sylph-parser/src/main/java/{ideal/sylph/parser/antlr => com/github/harbby/sylph/parser}/ParsingException.java (94%) mode change 100755 => 100644 rename sylph-parser/src/main/java/{ideal => com/github/harbby}/sylph/parser/SqlParserException.java (95%) rename sylph-parser/src/main/java/{ideal/sylph/parser/antlr => com/github/harbby/sylph/parser}/StatementSplitter.java (98%) rename sylph-parser/src/main/java/{ideal/sylph/parser/antlr => com/github/harbby/sylph/parser}/tree/AllowedLateness.java (87%) rename sylph-parser/src/main/java/{ideal/sylph/parser/antlr => com/github/harbby/sylph/parser}/tree/BooleanLiteral.java (90%) rename sylph-parser/src/main/java/{ideal/sylph/parser/antlr => com/github/harbby/sylph/parser}/tree/ColumnDefinition.java (80%) mode change 100755 => 100644 rename sylph-parser/src/main/java/{ideal/sylph/parser/antlr => com/github/harbby/sylph/parser}/tree/CreateFunction.java (90%) rename sylph-parser/src/main/java/{ideal/sylph/parser/antlr => com/github/harbby/sylph/parser}/tree/CreateStreamAsSelect.java (79%) rename sylph-parser/src/main/java/{ideal/sylph/parser/antlr => com/github/harbby/sylph/parser}/tree/CreateTable.java (84%) rename sylph-parser/src/main/java/{ideal/sylph/parser/antlr => com/github/harbby/sylph/parser}/tree/DoubleLiteral.java (87%) rename sylph-parser/src/main/java/{ideal/sylph/parser/antlr => com/github/harbby/sylph/parser}/tree/Expression.java (93%) mode change 100755 => 100644 rename sylph-parser/src/main/java/{ideal/sylph/parser/antlr => com/github/harbby/sylph/parser}/tree/Identifier.java (86%) mode change 100755 => 100644 rename sylph-parser/src/main/java/{ideal/sylph/parser/antlr => com/github/harbby/sylph/parser}/tree/InsertInto.java (85%) rename sylph-parser/src/main/java/{ideal/sylph/parser/antlr => com/github/harbby/sylph/parser}/tree/Literal.java (87%) mode change 100755 => 100644 rename sylph-parser/src/main/java/{ideal/sylph/parser/antlr => com/github/harbby/sylph/parser}/tree/LongLiteral.java (87%) rename sylph-parser/src/main/java/{ideal/sylph/parser/antlr => com/github/harbby/sylph/parser}/tree/Node.java (78%) mode change 100755 => 100644 rename sylph-parser/src/main/java/{ideal/sylph/parser/antlr => com/github/harbby/sylph/parser}/tree/NodeLocation.java (90%) mode change 100755 => 100644 rename sylph-parser/src/main/java/{ideal/sylph/parser/antlr => com/github/harbby/sylph/parser}/tree/Proctime.java (89%) rename sylph-parser/src/main/java/{ideal/sylph/parser/antlr => com/github/harbby/sylph/parser}/tree/Property.java (89%) mode change 100755 => 100644 rename sylph-parser/src/main/java/{ideal/sylph/parser/antlr => com/github/harbby/sylph/parser}/tree/QualifiedName.java (96%) mode change 100755 => 100644 rename sylph-parser/src/main/java/{ideal/sylph/parser/antlr => com/github/harbby/sylph/parser}/tree/SelectQuery.java (78%) rename sylph-parser/src/main/java/{ideal/sylph/parser/antlr => com/github/harbby/sylph/parser}/tree/Statement.java (85%) mode change 100755 => 100644 rename sylph-parser/src/main/java/{ideal/sylph/parser/antlr => com/github/harbby/sylph/parser}/tree/StringLiteral.java (94%) mode change 100755 => 100644 rename sylph-parser/src/main/java/{ideal/sylph/parser/antlr => com/github/harbby/sylph/parser}/tree/TableElement.java (85%) mode change 100755 => 100644 rename sylph-parser/src/main/java/{ideal/sylph/parser/antlr => com/github/harbby/sylph/parser}/tree/WaterMark.java (95%) rename sylph-parser/src/main/java/{ideal/sylph/parser/antlr => com/github/harbby/sylph/parser}/tree/WindowTrigger.java (88%) delete mode 100644 sylph-parser/src/main/java/ideal/sylph/parser/calcite/CalciteSqlParser.java delete mode 100644 sylph-parser/src/main/java/ideal/sylph/parser/calcite/JoinInfo.java delete mode 100644 sylph-parser/src/main/java/ideal/sylph/parser/calcite/TableName.java rename sylph-parser/src/test/java/{ideal/sylph/parser/antlr => com/github/harbby/sylph/parser}/tree/StringLiteralTest.java (97%) delete mode 100644 sylph-parser/src/test/java/test/TriggerSql.java rename sylph-runners/flink/src/main/java/{ideal => com/github/harbby}/sylph/runner/flink/FlinkBean.java (96%) create mode 100644 sylph-runners/flink/src/main/java/com/github/harbby/sylph/runner/flink/FlinkJobClient.java rename sylph-runners/flink/src/main/java/{ideal => com/github/harbby}/sylph/runner/flink/FlinkJobConfig.java (80%) create mode 100644 sylph-runners/flink/src/main/java/com/github/harbby/sylph/runner/flink/FlinkOperatorFactory.java create mode 100644 sylph-runners/flink/src/main/java/com/github/harbby/sylph/runner/flink/FlinkRunner.java create mode 100644 sylph-runners/flink/src/main/java/com/github/harbby/sylph/runner/flink/engines/FlinkEnvFactory.java rename sylph-runners/flink/src/main/java/{ideal => com/github/harbby}/sylph/runner/flink/engines/FlinkMainClassEngine.java (50%) create mode 100644 sylph-runners/flink/src/main/java/com/github/harbby/sylph/runner/flink/engines/FlinkStreamSqlEngine.java create mode 100644 sylph-runners/flink/src/main/java/com/github/harbby/sylph/runner/flink/engines/StreamSqlBuilder.java rename sylph-runners/flink/src/main/java/{ideal => com/github/harbby}/sylph/runner/flink/engines/StreamSqlUtil.java (61%) rename sylph-runners/flink/src/main/java/{ideal => com/github/harbby}/sylph/runner/flink/local/MiniExecutor.java (96%) rename sylph-runners/flink/src/main/java/{ideal => com/github/harbby}/sylph/runner/flink/plugins/TestSource.java (86%) rename sylph-runners/flink/src/main/java/{ideal/sylph/runner/flink/udf => com/github/harbby/sylph/runner/flink/runtime}/ArrayAgg.java (89%) rename sylph-runners/flink/src/main/java/{ideal/sylph/runner/flink/etl => com/github/harbby/sylph/runner/flink/runtime}/FlinkRecord.java (65%) rename sylph-runners/flink/src/main/java/{ideal/sylph/runner/flink/etl => com/github/harbby/sylph/runner/flink/runtime}/FlinkSink.java (96%) rename sylph-runners/flink/src/main/java/{ideal/sylph/runner/flink/udf => com/github/harbby/sylph/runner/flink/runtime}/UDFJson.java (94%) mode change 100755 => 100644 delete mode 100644 sylph-runners/flink/src/main/java/ideal/sylph/runner/flink/FlinkContainerFactory.java delete mode 100644 sylph-runners/flink/src/main/java/ideal/sylph/runner/flink/FlinkRunner.java delete mode 100644 sylph-runners/flink/src/main/java/ideal/sylph/runner/flink/SylphFsCheckpointStorage.java delete mode 100644 sylph-runners/flink/src/main/java/ideal/sylph/runner/flink/engines/FlinkEnvFactory.java delete mode 100644 sylph-runners/flink/src/main/java/ideal/sylph/runner/flink/engines/FlinkStreamEtlEngine.java delete mode 100644 sylph-runners/flink/src/main/java/ideal/sylph/runner/flink/engines/FlinkStreamSqlEngine.java delete mode 100644 sylph-runners/flink/src/main/java/ideal/sylph/runner/flink/engines/StreamSqlBuilder.java delete mode 100644 sylph-runners/flink/src/main/java/ideal/sylph/runner/flink/etl/FlinkNodeLoader.java delete mode 100644 sylph-runners/flink/src/main/java/ideal/sylph/runner/flink/etl/FlinkTransFrom.java delete mode 100644 sylph-runners/flink/src/main/java/ideal/sylph/runner/flink/sql/AsyncFunctionHelper.java delete mode 100644 sylph-runners/flink/src/main/java/ideal/sylph/runner/flink/sql/FlinkSqlParser.java delete mode 100644 sylph-runners/flink/src/main/java/ideal/sylph/runner/flink/sql/JoinContextImpl.java delete mode 100644 sylph-runners/flink/src/main/java/ideal/sylph/runner/flink/sql/TriggerWindowHelper.java delete mode 100644 sylph-runners/flink/src/main/java/ideal/sylph/runner/flink/table/SylphTableSink.java delete mode 100644 sylph-runners/flink/src/main/java/ideal/sylph/runner/flink/table/SylphTableSource.java delete mode 100644 sylph-runners/flink/src/main/java/ideal/sylph/runner/flink/udf/JsonParser.java delete mode 100644 sylph-runners/flink/src/main/java/ideal/sylph/runner/flink/udf/RowGet.java delete mode 100644 sylph-runners/flink/src/main/java/ideal/sylph/runner/flink/udf/TimeUtil.java delete mode 100644 sylph-runners/flink/src/main/java/ideal/sylph/runner/flink/yarn/FlinkYarnJobLauncher.java delete mode 100644 sylph-runners/flink/src/main/java/ideal/sylph/runner/flink/yarn/YarnJobDescriptor.java rename sylph-runners/flink/src/test/java/{ideal => com/github/harbby}/sylph/runner/flink/engines/FlinkStreamSqlActuatorTest.java (95%) rename sylph-runners/flink/src/test/java/{ideal => com/github/harbby}/sylph/runner/flink/engines/StreamSqlBuilderTest.java (97%) delete mode 100644 sylph-runners/flink/src/test/java/ideal/sylph/runner/flink/sqlTest/JoinTest.java rename sylph-runners/spark/src/main/java/{ideal => com/github/harbby}/sylph/runner/spark/SQLHepler.java (89%) rename sylph-runners/spark/src/main/java/{ideal => com/github/harbby}/sylph/runner/spark/SparkJobConfig.java (88%) create mode 100644 sylph-runners/spark/src/main/java/com/github/harbby/sylph/runner/spark/SparkMainClassEngine.java rename sylph-runners/spark/src/main/java/{ideal => com/github/harbby}/sylph/runner/spark/SparkRecord.java (92%) create mode 100644 sylph-runners/spark/src/main/java/com/github/harbby/sylph/runner/spark/SparkRunner.java rename sylph-runners/spark/src/main/java/{ideal => com/github/harbby}/sylph/runner/spark/SparkStreamingSqlAnalyse.java (78%) create mode 100644 sylph-runners/spark/src/main/java/com/github/harbby/sylph/runner/spark/SparkStreamingSqlEngine.java create mode 100644 sylph-runners/spark/src/main/java/com/github/harbby/sylph/runner/spark/SparkUtil.java rename sylph-runners/spark/src/main/java/{ideal => com/github/harbby}/sylph/runner/spark/SqlAnalyse.java (69%) rename sylph-runners/spark/src/main/java/{ideal => com/github/harbby}/sylph/runner/spark/StructuredStreamingSqlAnalyse.java (70%) create mode 100644 sylph-runners/spark/src/main/java/com/github/harbby/sylph/runner/spark/StructuredStreamingSqlEngine.java rename sylph-runners/spark/src/main/java/{ideal => com/github/harbby}/sylph/runner/spark/kafka/SylphKafkaOffset.java (97%) rename sylph-runners/spark/src/main/java/{ideal/sylph/runner/spark => com/github/harbby/sylph/runner/spark/runtime}/SparkAppMain.java (82%) rename sylph-runners/spark/src/main/java/{ideal => com/github/harbby}/sylph/runner/spark/sparkstreaming/DStreamUtil.java (94%) create mode 100644 sylph-runners/spark/src/main/java/com/github/harbby/sylph/runner/spark/sparkstreaming/SparkStreamingOperatorFactory.java create mode 100644 sylph-runners/spark/src/main/java/com/github/harbby/sylph/runner/spark/structured/StructuredNodeLoader.java create mode 100644 sylph-runners/spark/src/main/java/com/github/harbby/sylph/runner/spark/yarn/SparkJobClient.java rename sylph-runners/spark/src/main/java/{ideal => com/github/harbby}/sylph/runner/spark/yarn/SylphSparkYarnClient.java (90%) delete mode 100644 sylph-runners/spark/src/main/java/ideal/sylph/runner/spark/JobHelper.java delete mode 100644 sylph-runners/spark/src/main/java/ideal/sylph/runner/spark/SparkContainerFactory.java delete mode 100644 sylph-runners/spark/src/main/java/ideal/sylph/runner/spark/SparkMainClassEngine.java delete mode 100644 sylph-runners/spark/src/main/java/ideal/sylph/runner/spark/SparkRunner.java delete mode 100644 sylph-runners/spark/src/main/java/ideal/sylph/runner/spark/SparkStreamingSqlEngine.java delete mode 100644 sylph-runners/spark/src/main/java/ideal/sylph/runner/spark/Stream2EtlEngine.java delete mode 100644 sylph-runners/spark/src/main/java/ideal/sylph/runner/spark/StreamEtlEngine.java delete mode 100644 sylph-runners/spark/src/main/java/ideal/sylph/runner/spark/StructuredStreamingSqlEngine.java delete mode 100644 sylph-runners/spark/src/main/java/ideal/sylph/runner/spark/sparkstreaming/StreamNodeLoader.java delete mode 100644 sylph-runners/spark/src/main/java/ideal/sylph/runner/spark/structured/StructuredNodeLoader.java delete mode 100644 sylph-runners/spark/src/main/java/ideal/sylph/runner/spark/udf/TimeUtil.java delete mode 100644 sylph-runners/spark/src/main/java/ideal/sylph/runner/spark/yarn/SparkAppLauncher.java create mode 100644 sylph-spi/src/main/java/com/github/harbby/sylph/spi/CompileJobException.java create mode 100644 sylph-spi/src/main/java/com/github/harbby/sylph/spi/CompileTask.java rename sylph-spi/src/main/java/{ideal/sylph/spi/job/JobEngine.java => com/github/harbby/sylph/spi/JobClient.java} (54%) rename sylph-api/src/main/java/ideal/sylph/etl/join/JoinContext.java => sylph-spi/src/main/java/com/github/harbby/sylph/spi/OperatorFactory.java (59%) rename sylph-spi/src/main/java/{ideal/sylph/spi/model => com/github/harbby/sylph/spi}/OperatorInfo.java (76%) rename {sylph-api/src/main/java/ideal/sylph/etl => sylph-spi/src/main/java/com/github/harbby/sylph/spi}/OperatorType.java (79%) create mode 100644 sylph-spi/src/main/java/com/github/harbby/sylph/spi/Runner.java rename sylph-spi/src/main/java/{ideal => com/github/harbby}/sylph/spi/SylphContext.java (71%) create mode 100644 sylph-spi/src/main/java/com/github/harbby/sylph/spi/dao/Job.java rename sylph-spi/src/main/java/{ideal/sylph/spi/model => com/github/harbby/sylph/spi/dao}/JobInfo.java (91%) create mode 100644 sylph-spi/src/main/java/com/github/harbby/sylph/spi/dao/JobRunState.java rename sylph-api/src/main/java/ideal/sylph/etl/api/TransForm.java => sylph-spi/src/main/java/com/github/harbby/sylph/spi/job/DeployResponse.java (72%) rename sylph-spi/src/main/java/{ideal => com/github/harbby}/sylph/spi/job/JobConfig.java (86%) create mode 100644 sylph-spi/src/main/java/com/github/harbby/sylph/spi/job/JobDag.java rename sylph-spi/src/main/java/{ideal/sylph/spi/job/JobEngineHandle.java => com/github/harbby/sylph/spi/job/JobEngine.java} (51%) create mode 100644 sylph-spi/src/main/java/com/github/harbby/sylph/spi/job/JobParser.java rename sylph-api/src/main/java/ideal/sylph/etl/impl/ListCollector.java => sylph-spi/src/main/java/com/github/harbby/sylph/spi/job/MainClassJobParser.java (58%) create mode 100644 sylph-spi/src/main/java/com/github/harbby/sylph/spi/job/SqlJobParser.java rename sylph-spi/src/main/java/{ideal/sylph/spi/PluginConfigFactory.java => com/github/harbby/sylph/spi/utils/PluginFactory.java} (61%) delete mode 100644 sylph-spi/src/main/java/ideal/sylph/spi/GraphAppUtil.java delete mode 100644 sylph-spi/src/main/java/ideal/sylph/spi/NodeLoader.java delete mode 100644 sylph-spi/src/main/java/ideal/sylph/spi/OperatorMetaData.java delete mode 100644 sylph-spi/src/main/java/ideal/sylph/spi/Runner.java delete mode 100644 sylph-spi/src/main/java/ideal/sylph/spi/RunnerContext.java delete mode 100755 sylph-spi/src/main/java/ideal/sylph/spi/exception/ErrorCode.java delete mode 100755 sylph-spi/src/main/java/ideal/sylph/spi/exception/ErrorType.java delete mode 100644 sylph-spi/src/main/java/ideal/sylph/spi/exception/StandardErrorCode.java delete mode 100644 sylph-spi/src/main/java/ideal/sylph/spi/exception/SylphException.java delete mode 100644 sylph-spi/src/main/java/ideal/sylph/spi/job/EtlFlow.java delete mode 100644 sylph-spi/src/main/java/ideal/sylph/spi/job/EtlJobEngineHandle.java delete mode 100644 sylph-spi/src/main/java/ideal/sylph/spi/job/Flow.java delete mode 100644 sylph-spi/src/main/java/ideal/sylph/spi/job/Job.java delete mode 100644 sylph-spi/src/main/java/ideal/sylph/spi/job/JobContainer.java delete mode 100644 sylph-spi/src/main/java/ideal/sylph/spi/job/JobContainerAbs.java delete mode 100644 sylph-spi/src/main/java/ideal/sylph/spi/job/JobStore.java delete mode 100644 sylph-spi/src/main/java/ideal/sylph/spi/job/SqlFlow.java delete mode 100644 sylph-spi/src/main/java/ideal/sylph/spi/model/EdgeInfo.java delete mode 100644 sylph-spi/src/main/java/ideal/sylph/spi/model/NodeInfo.java delete mode 100644 sylph-spi/src/main/java/ideal/sylph/spi/utils/GenericTypeReference.java delete mode 100644 sylph-spi/src/main/java/ideal/sylph/spi/utils/JsonTextUtil.java delete mode 100644 sylph-spi/src/test/java/ideal/sylph/spi/GraphAppUtilTest.java delete mode 100644 sylph-spi/src/test/java/ideal/sylph/spi/NodeLoaderTest.java rename sylph-spi/src/test/java/ideal/sylph/spi/{model => }/OperatorManagerTest.java (88%) delete mode 100644 sylph-spi/src/test/java/ideal/sylph/spi/job/SqlFlowTest.java delete mode 100644 sylph-spi/src/test/java/ideal/sylph/spi/utils/JsonTextUtilTest.java rename sylph-web/src/main/java/{ideal/sylph/controller => com/github/harbby/sylph/colltroller}/AppExceptionMapper.java (81%) rename sylph-web/src/main/java/{ideal/sylph/controller => com/github/harbby/sylph/colltroller}/AuthAspect.java (75%) rename sylph-web/src/main/java/{ideal/sylph/controller => com/github/harbby/sylph/colltroller}/ControllerApp.java (84%) rename sylph-web/src/main/java/{ideal/sylph/controller => com/github/harbby/sylph/colltroller}/JettyServer.java (84%) rename sylph-web/src/main/java/{ideal/sylph/controller => com/github/harbby/sylph/colltroller}/LogAppender.java (98%) rename sylph-web/src/main/java/{ideal/sylph/controller => com/github/harbby/sylph/colltroller}/ServerConfig.java (96%) rename sylph-web/src/main/java/{ideal/sylph/controller => com/github/harbby/sylph/colltroller}/WebApplication.java (91%) rename sylph-web/src/main/java/{ideal/sylph/controller => com/github/harbby/sylph/colltroller}/action/JobManagerResource.java (62%) rename sylph-web/src/main/java/{ideal/sylph/controller => com/github/harbby/sylph/colltroller}/action/LoginController.java (83%) rename sylph-web/src/main/java/{ideal/sylph/controller => com/github/harbby/sylph/colltroller}/action/PluginManagerResource.java (90%) rename sylph-web/src/main/java/{ideal/sylph/controller => com/github/harbby/sylph/colltroller}/action/ServerLogResource.java (73%) rename sylph-web/src/main/java/{ideal/sylph/controller => com/github/harbby/sylph/colltroller}/selvet/WebAppProxyServlet.java (64%) rename sylph-web/src/main/java/{ideal/sylph/controller => com/github/harbby/sylph/colltroller}/utils/JsonFormatUtil.java (93%) rename sylph-web/src/main/java/{ideal/sylph/controller => com/github/harbby/sylph/colltroller}/utils/ProxyUtil.java (93%) delete mode 100644 sylph-web/src/main/java/ideal/sylph/controller/action/EtlResource.java delete mode 100644 sylph-web/src/main/java/ideal/sylph/controller/selvet/ProxyAllHttpServer.java rename sylph-web/src/test/java/{ideal/sylph/controller => com/github/harbby/sylph/colltroller}/utils/JsonFormatUtilTest.java (97%) create mode 100644 sylph-yarn/src/main/java/com/github/harbby/sylph/yarn/YarnClientFactory.java create mode 100644 sylph-yarn/src/main/java/com/github/harbby/sylph/yarn/YarnDeployResponse.java delete mode 100644 sylph-yarn/src/main/java/ideal/sylph/runtime/local/LocalContainer.java delete mode 100644 sylph-yarn/src/main/java/ideal/sylph/runtime/yarn/YarnJobContainer.java delete mode 100644 sylph-yarn/src/main/java/ideal/sylph/runtime/yarn/YarnModule.java diff --git a/.travis.yml b/.travis.yml index 4ca3c9f6b..fa4266dc7 100755 --- a/.travis.yml +++ b/.travis.yml @@ -21,6 +21,9 @@ jobs: - jdk: openjdk11 dist: focal script: ./gradlew clean assemble --no-daemon -Pjdk=java11 + - jdk: openjdk17 + dist: focal + script: ./gradlew clean assemble --no-daemon -Pjdk=java17 env: - CI=false diff --git a/README.md b/README.md index 9b5f44a08..2fc0b67e5 100755 --- a/README.md +++ b/README.md @@ -1,4 +1,7 @@ # Sylph [![Build Status](http://img.shields.io/travis/harbby/sylph.svg?style=flat&branch=master)](https://travis-ci.org/harbby/sylph) +[![license](https://img.shields.io/badge/license-apache_v2-groon.svg)]() +[![language](https://img.shields.io/badge/language-java_17-green.svg)]() +[![os](https://img.shields.io/badge/os-Linux_macOS-blue.svg)]() Welcome to Sylph ! @@ -33,14 +36,16 @@ limitations under the License. ## StreamingSql ```sql -create function get_json_object as 'ideal.sylph.runner.flink.udf.UDFJson'; +create function get_json_object as 'com.github.harbby.sylph.runner.flink.runtime.UDFJson'; create source table topic1( _topic varchar, _key varchar, _partition integer, _offset bigint, - _message varchar + _message varchar, + ip varchar extend '$.conntent.ip', -- json path + event_time varchar extend '$.conntent.event_time' -- json path ) with ( type = 'kafka08', kafka_topic = 'event_topic', @@ -71,15 +76,9 @@ from topic1 ## UDF UDAF UDTF The registration of the custom function is consistent with the hive ```sql -create function get_json_object as 'ideal.sylph.runner.flink.udf.UDFJson'; +create function get_json_object as 'com.github.harbby.sylph.runner.flink.runtime.UDFJson'; ``` -## StreamETL -Support `flink-stream` `spark-streaming` `spark-structured-streaming(spark2.2x)` - -[![loading...](https://raw.githubusercontent.com/harbby/harbby.github.io/master/logo/sylph/job_flow.png)](https://travis-ci.org/harbby/sylph) - - ## Building sylph builds use Gradle and requires Java 8. Also if you want read a chinese deploy docs,[中文部署文档](sylph-docs/src/main/docs/source/zh-cn/docs/intro/deploy.md) @@ -100,7 +99,7 @@ After opening the project in IntelliJ, double check that the Java SDK is properl Sylph comes with sample configuration that should work out-of-the-box for development. Use the following options to create a run configuration: -* Main Class: ideal.sylph.main.SylphMaster +* Main Class: com.github.harbby.sylph.main.SylphMaster * VM Options: -Dconfig=etc/sylph/sylph.properties -Dlogging.config=etc/sylph/logback.xml * ENV Options: FLINK_HOME= HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop diff --git a/build.gradle b/build.gradle index 77ff3b77c..445ef1241 100644 --- a/build.gradle +++ b/build.gradle @@ -2,40 +2,36 @@ plugins { id "com.github.hierynomus.license" version "0.16.1" id "com.github.harbby.gradle.serviceloader" version "1.1.8" id "jacoco" + id "idea" } allprojects { - group 'com.github.harbby' - version '0.8.0-SNAPSHOT' //SNAPSHOT - ext.deps = [ - flink : '1.13.1', - jetty : "9.4.6.v20170531", //8.1.17.v20150415 "9.4.6.v20170531" hadoop : "3.2.1", - hbase : '1.1.2', - spark : "3.1.2", - scala : '2.12.8', + spark : "3.2.0", + flink : '1.14.3', + jetty : "11.0.6", + scala : '2.12.15', joda_time : '2.9.3', - slf4j : '1.7.25', - guice : '4.2.1', - gadtry : '1.9.4', - guava : '27.0-jre', - jackson : '2.9.8', - jersey : '2.28', - scala_binary_version: '2.12' + slf4j : '2.0.0-alpha1', + gadtry : '1.9.9', + guava : '31.0.1-jre', + jackson : '2.12.2', + jersey : '3.0.3', ] } subprojects { + apply plugin: 'java' + group 'com.github.harbby' + version '1.0.0-SNAPSHOT' //SNAPSHOT + apply plugin: 'java' apply plugin: 'java-library' apply plugin: 'maven-publish' apply plugin: 'checkstyle' apply plugin: 'jacoco' - - def jdk = project.hasProperty('jdk') ? project.jdk : 'java8' //or java11 use -Pjdk=java8 - //def jdk = System.getProperty("jdk") ?: "java8" //or java11 use -Djdk=java8 - apply from: "$rootProject.projectDir/profile-${jdk}.gradle" + apply plugin: 'com.github.hierynomus.license' tasks.withType(JavaCompile) { options.encoding = 'UTF-8' @@ -44,12 +40,16 @@ subprojects { options.encoding = 'UTF-8' } - if (project != rootProject) { - apply plugin: 'com.github.hierynomus.license' + apply plugin: 'idea' + idea { + module { + downloadJavadoc = false + downloadSources = false + } } configurations { - testCompile.extendsFrom compileOnly + testImplementation.extendsFrom compileOnly } repositories { @@ -62,8 +62,8 @@ subprojects { } dependencies { - testCompile group: 'junit', name: 'junit', version: '4.12' - testCompile group: 'com.github.harbby', name: 'gadtry', version: deps.gadtry + testImplementation group: 'junit', name: 'junit', version: '4.13.1' + testImplementation group: 'com.github.harbby', name: 'gadtry', version: deps.gadtry } task clearOutDir(type: Delete) { @@ -80,7 +80,7 @@ subprojects { check.dependsOn jacocoTestReport checkstyle { - toolVersion '8.12' + toolVersion '9.2.1' showViolations true } @@ -134,9 +134,9 @@ subprojects { javadoc.failOnError = false } - artifacts { - archives sourcesJar, javadocJar - } +// artifacts { +// archives sourcesJar, javadocJar +// } javadoc { options { @@ -144,7 +144,7 @@ subprojects { charSet 'UTF-8' author true version true - links "https://harbby.github.io/project/sylph/en/docs/intro/" + links "https://github.com/harbby/sylph" title "sylph" } } @@ -160,50 +160,4 @@ subprojects { required { shouldSign } sign configurations.archives } - -// uploadArchives { -// repositories { -// mavenDeployer { -// beforeDeployment { MavenDeployment deployment -> signing.signPom(deployment) } -// -// repository(url: "https://oss.sonatype.org/service/local/staging/deploy/maven2/") { -// authentication(userName: mavenUsername, password: mavenPassword) -// } -// -// snapshotRepository(url: "https://oss.sonatype.org/content/repositories/snapshots/") { -// authentication(userName: mavenUsername, password: mavenPassword) -// } -// -// pom.project { -// name project.name -// packaging 'jar' -// // optionally artifactId can be defined here -// description 'A lightweight API test framework' -// url 'https://github.com/harbby/sylph' -// -// scm { -// connection 'https://github.com/harbby/sylph.git' -// developerConnection 'git@github.com:harbby/sylph.git' -// url 'https://github.com/harbby/sylph' -// } -// -// licenses { -// license { -// name 'The Apache Software License, Version 2.0' -// url 'http://www.apache.org/licenses/LICENSE-2.0.txt' -// distribution 'repo' -// } -// } -// -// developers { -// developer { -// id 'harbby' -// name 'harbby' -// email 'yezhixinghai@gmail.com' -// } -// } -// } -// } -// } -// } } diff --git a/gradle.properties b/gradle.properties index f89471a84..ebaa889d9 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1 +1 @@ -jdk=java8 \ No newline at end of file +jdk=java17 \ No newline at end of file diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 72b4fd049..12a5c9ab4 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -3,4 +3,4 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.2-bin.zip \ No newline at end of file +distributionUrl=https\://services.gradle.org/distributions/gradle-7.3.3-bin.zip \ No newline at end of file diff --git a/json-reader/build.gradle b/json-reader/build.gradle new file mode 100644 index 000000000..60a05e7af --- /dev/null +++ b/json-reader/build.gradle @@ -0,0 +1,8 @@ + +apply from: "$rootDir/profile-runtime.gradle" + +dependencies { + compileOnly project(":sylph-api") + implementation group: 'com.jayway.jsonpath', name: 'json-path', version: '2.6.0' + compileOnly group: 'com.github.harbby', name: 'gadtry', version: deps.gadtry +} \ No newline at end of file diff --git a/sylph-spi/src/main/java/ideal/sylph/spi/job/ContainerFactory.java b/json-reader/src/main/java/com/github/harbby/sylph/json/ByteCodeClassLoader.java similarity index 64% rename from sylph-spi/src/main/java/ideal/sylph/spi/job/ContainerFactory.java rename to json-reader/src/main/java/com/github/harbby/sylph/json/ByteCodeClassLoader.java index c2f02a3a4..108caec82 100644 --- a/sylph-spi/src/main/java/ideal/sylph/spi/job/ContainerFactory.java +++ b/json-reader/src/main/java/com/github/harbby/sylph/json/ByteCodeClassLoader.java @@ -13,16 +13,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package ideal.sylph.spi.job; +package com.github.harbby.sylph.json; -public interface ContainerFactory +public class ByteCodeClassLoader + extends ClassLoader { - JobContainer createYarnContainer(Job job, String lastRunid); - - JobContainer createLocalContainer(Job job, String lastRunid); + public ByteCodeClassLoader(ClassLoader parent) + { + super(parent); + } - default JobContainer createK8sContainer(Job job, String lastRunid) + public Class defineClass(String className, byte[] byteCode) { - throw new UnsupportedOperationException("this k8s have't support!"); + return this.defineClass(className, byteCode, 0, byteCode.length); } } diff --git a/sylph-connectors/flink-kafka/src/main/java/ideal/sylph/plugins/kafka/flink/JsonDeserializer.java b/json-reader/src/main/java/com/github/harbby/sylph/json/JsonPathReader.java similarity index 63% rename from sylph-connectors/flink-kafka/src/main/java/ideal/sylph/plugins/kafka/flink/JsonDeserializer.java rename to json-reader/src/main/java/com/github/harbby/sylph/json/JsonPathReader.java index 52ab5ae53..caff2c972 100644 --- a/sylph-connectors/flink-kafka/src/main/java/ideal/sylph/plugins/kafka/flink/JsonDeserializer.java +++ b/json-reader/src/main/java/com/github/harbby/sylph/json/JsonPathReader.java @@ -13,51 +13,48 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package ideal.sylph.plugins.kafka.flink; +package com.github.harbby.sylph.json; -import com.github.harbby.gadtry.base.Lazys; import com.jayway.jsonpath.Configuration; import com.jayway.jsonpath.JsonPath; import com.jayway.jsonpath.Option; import com.jayway.jsonpath.ReadContext; import com.jayway.jsonpath.spi.json.JacksonJsonProvider; import com.jayway.jsonpath.spi.mapper.JacksonMappingProvider; -import ideal.sylph.etl.Field; -import java.io.ByteArrayInputStream; import java.io.Serializable; -import java.util.function.Supplier; +import java.nio.charset.StandardCharsets; -public class JsonDeserializer +public abstract class JsonPathReader implements Serializable { private static final Configuration jsonConfig = Configuration.builder() .jsonProvider(new JacksonJsonProvider()) .mappingProvider(new JacksonMappingProvider()) + .options(Option.SUPPRESS_EXCEPTIONS) .options(Option.DEFAULT_PATH_LEAF_TO_NULL) //path 不存在时返回null //.options(Option.SUPPRESS_EXCEPTIONS) //path 不存在时返回null .build(); + private static final long serialVersionUID = 6208262176869833572L; - private Supplier jsonContext; + private byte[] message; + private ReadContext readContext; public void initNewMessage(byte[] message) { - this.jsonContext = Lazys.goLazy(() -> JsonPath.using(jsonConfig) - .parse(new ByteArrayInputStream(message))); + this.message = message; + this.readContext = null; } - public Object deserialize(Field field) + public abstract void deserialize(KafkaRecord record, Object[] values); + + protected Object read(String keyPath) { - String keyPath = field.getExtend().orElse(field.getName()); - if (!keyPath.startsWith("$")) { - keyPath = "$." + keyPath; - } - Object value = jsonContext.get().read(keyPath); - if (value == null) { - return null; - } - else { - return value; + if (readContext == null) { + readContext = JsonPath.using(jsonConfig).parse(new String(message, StandardCharsets.UTF_8)); } + //Type type = field.getJavaType(); + //value = MAPPER.convertValue(value, typeFactory.constructType(type)); + return readContext.read(keyPath); } } diff --git a/json-reader/src/main/java/com/github/harbby/sylph/json/JsonReadCodeGenerator.java b/json-reader/src/main/java/com/github/harbby/sylph/json/JsonReadCodeGenerator.java new file mode 100644 index 000000000..a04dec91d --- /dev/null +++ b/json-reader/src/main/java/com/github/harbby/sylph/json/JsonReadCodeGenerator.java @@ -0,0 +1,109 @@ +/* + * Copyright (C) 2018 The Sylph Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.harbby.sylph.json; + +import com.github.harbby.gadtry.compiler.JavaClassCompiler; +import com.github.harbby.gadtry.compiler.JavaSourceObject; +import com.github.harbby.sylph.api.Field; +import com.github.harbby.sylph.api.Schema; + +import java.net.URL; +import java.net.URLClassLoader; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; + +import static java.util.Objects.requireNonNull; + +public class JsonReadCodeGenerator +{ + private static final List KAFKA_COLUMNS = Arrays.asList("_topic", "_key", "_message", "_partition", "_offset"); + private String fullName; + private byte[] byteCode; + private final String className; + private String code; + + public JsonReadCodeGenerator(String className) + { + this.className = className; + } + + public void doCodeGen(Schema schema) + { + StringBuilder coder = new StringBuilder("package " + JsonPathReader.class.getPackage().getName() + ";\n" + + "public class " + className + " extends " + JsonPathReader.class.getName() + " {\n" + + " @Override\n" + + " public void deserialize(" + KafkaRecord.class.getName() + " record, Object[] values) {\n" + + " int i =0;\n"); + for (Field field : schema.getFields()) { + coder.append("values[i++] = "); + if (KAFKA_COLUMNS.contains(field.getName())) { + switch (field.getName()) { + case "_topic": + coder.append("record.topic();"); + break; + case "_message": + coder.append("new String(record.value(), java.nio.charset.StandardCharsets.UTF_8);"); + break; + case "_key": + coder.append("record.key() == null ? null : new String(record.key(), java.nio.charset.StandardCharsets.UTF_8);"); + break; + case "_partition": + coder.append("record.partition();"); + break; + case "_offset": + coder.append("record.offset();"); + break; + default: + throw new UnsupportedOperationException(); + } + } + else { + Optional extend = Optional.ofNullable(field.getExtend()); + String jsonPath = extend.orElse("$." + field.getName()); + coder.append(String.format("this.read(\"%s\");", jsonPath)); + } + coder.append("\n"); + } + coder.append("}}"); + System.out.println(coder); + ClassLoader classLoader = new URLClassLoader(new URL[0]); + JavaClassCompiler compiler = new JavaClassCompiler(classLoader); + JavaSourceObject target = compiler.doCompile(className, coder.toString(), Arrays.asList( + "-source", "1.8", + "-target", "1.8", + "-cp", this.getClass().getProtectionDomain().getCodeSource().getLocation().toString())); + + this.fullName = JsonPathReader.class.getPackage().getName() + "." + className; + this.byteCode = requireNonNull(target.getClassByteCodes().get(fullName), "byte code is null"); + this.code = coder.toString(); + } + + public byte[] getByteCode() + { + return byteCode; + } + + public String getFullName() + { + return fullName; + } + + public String getCode() + { + return code; + } +} diff --git a/sylph-connectors/sylph-hdfs/src/main/java/ideal/sylph/plugins/hdfs/OutputFormat.java b/json-reader/src/main/java/com/github/harbby/sylph/json/KafkaRecord.java similarity index 70% rename from sylph-connectors/sylph-hdfs/src/main/java/ideal/sylph/plugins/hdfs/OutputFormat.java rename to json-reader/src/main/java/com/github/harbby/sylph/json/KafkaRecord.java index b016cc399..d62adf492 100644 --- a/sylph-connectors/sylph-hdfs/src/main/java/ideal/sylph/plugins/hdfs/OutputFormat.java +++ b/json-reader/src/main/java/com/github/harbby/sylph/json/KafkaRecord.java @@ -13,16 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package ideal.sylph.plugins.hdfs; +package com.github.harbby.sylph.json; -import ideal.sylph.etl.Record; +public abstract class KafkaRecord +{ + public abstract String topic(); -import java.io.Closeable; -import java.io.IOException; + public abstract Integer partition(); -public interface OutputFormat - extends Closeable -{ - public void writeLine(Record record) - throws IOException; + public abstract K key(); + + public abstract V value(); + + public abstract Long offset(); } diff --git a/profile-java11.gradle b/profile-java17.gradle similarity index 69% rename from profile-java11.gradle rename to profile-java17.gradle index 3a840aba5..1ea4c63b1 100644 --- a/profile-java11.gradle +++ b/profile-java17.gradle @@ -1,21 +1,18 @@ -//java 11 +//java 17 -sourceCompatibility = 11 -targetCompatibility = 11 +sourceCompatibility = 17 +targetCompatibility = 17 if (project == project(':sylph-web')) { project.dependencies { - implementation group: 'javax.xml.bind', name: 'jaxb-api', version: '2.3.0' - implementation group: 'javax.activation', name: 'activation', version: '1.1.1' + implementation group: 'javax.xml.bind', name: 'jaxb-api', version: '2.3.1' + implementation 'org.glassfish.jaxb:jaxb-runtime:3.0.2' } } tasks.withType(JavaCompile) { options.encoding = 'UTF-8' options.compilerArgs << "--add-exports=java.base/jdk.internal.ref=ALL-UNNAMED" - options.compilerArgs << "--add-exports=java.base/sun.reflect.generics.reflectiveObjects=ALL-UNNAMED" - options.compilerArgs << "--add-exports=java.base/sun.reflect.generics.tree=ALL-UNNAMED" - options.compilerArgs << "--add-exports=java.base/sun.reflect.generics.repository=ALL-UNNAMED" options.compilerArgs << "--add-exports=java.base/sun.nio.cs=ALL-UNNAMED" options.compilerArgs << "--add-exports=java.base/jdk.internal.vm.annotation=ALL-UNNAMED" } diff --git a/profile-java8.gradle b/profile-java8.gradle deleted file mode 100644 index 0cd1d6d77..000000000 --- a/profile-java8.gradle +++ /dev/null @@ -1,7 +0,0 @@ - -sourceCompatibility = 1.8 -targetCompatibility = 1.8 - -tasks.withType(JavaCompile) { - options.encoding = 'UTF-8' -} \ No newline at end of file diff --git a/profile-runtime.gradle b/profile-runtime.gradle new file mode 100644 index 000000000..a183a877f --- /dev/null +++ b/profile-runtime.gradle @@ -0,0 +1,3 @@ + +sourceCompatibility = 1.8 +targetCompatibility = 1.8 diff --git a/settings.gradle b/settings.gradle index 4e7914eec..d5418425c 100644 --- a/settings.gradle +++ b/settings.gradle @@ -16,7 +16,6 @@ include 'sylph-runners' include 'sylph-runners:flink' project(':sylph-runners:flink').name = 'sylph-flink' include ':sylph-runners:spark' -//project(":sql-13").projectDir = new File(settingsDir, "spark/sql-13") project(':sylph-runners:spark').name = 'sylph-spark' //---- @@ -25,7 +24,6 @@ include 'sylph-connectors' include 'sylph-connectors:sylph-example' include 'sylph-connectors:flink-kafka' include 'sylph-connectors:sylph-mysql' -include 'sylph-connectors:sylph-hdfs' include 'sylph-connectors:sylph-kudu' include 'sylph-connectors:spark-kafka' @@ -34,3 +32,5 @@ include 'sylph-dist' include 'sylph-parser' include 'sylph-yarn' //include 'sylph-cli' +include 'json-reader' + diff --git a/src/checkstyle/facebook_checks.xml b/src/checkstyle/facebook_checks.xml index a8b568efd..1b207626b 100755 --- a/src/checkstyle/facebook_checks.xml +++ b/src/checkstyle/facebook_checks.xml @@ -1,7 +1,7 @@ + "http://checkstyle.sourceforge.net/dtds/configuration_1_3.dtd"> @@ -15,6 +15,15 @@ + + + + + + + + + @@ -23,17 +32,9 @@ - - - - - - - - - - + + @@ -43,50 +44,59 @@ + - - + + - - + + - - + + - - + + + - - + + - - - + + + - + - + + + + + + - + - + + + + + @@ -155,6 +167,10 @@ + + + + @@ -172,34 +188,35 @@ + + - - + + + + + + + + - + + - + \ No newline at end of file diff --git a/src/checkstyle/intellij-java-facebook-style.xml b/src/checkstyle/intellij-java-facebook-style.xml index ae89268dd..fbe1ff0ec 100755 --- a/src/checkstyle/intellij-java-facebook-style.xml +++ b/src/checkstyle/intellij-java-facebook-style.xml @@ -51,6 +51,26 @@