Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add gcp support sql bugfix #197

Closed
wants to merge 12 commits into from
Closed

Add gcp support sql bugfix #197

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
56d4b62
initial commit to add support for GCP
anighanta May 26, 2022
b096a20
WIP : gcp support
anighanta Jun 10, 2022
8e3349b
merge master
anighanta Jun 10, 2022
1766e7a
working on gcp
anighanta Jun 16, 2022
2d260f1
Merge branch 'master' into add-GCP-suuport
anighanta Jun 16, 2022
fed91c3
WIP : set up gcp support with pub sub notifications
anighanta Jun 21, 2022
321e970
WIP pom fixes
anighanta Jun 21, 2022
52f25e7
use new entry points for aws and gcp
anighanta Jun 21, 2022
bec3021
use sdk-full instead of sdk-jdk7 artifact from hedera
anighanta Jun 22, 2022
9408bf6
update to latest sdk
Aug 31, 2022
4a199ae
add-GCP-support-SQL-bugfix
avetswirldslabs Feb 9, 2023
53dfdcf
add-GCP-support-SQL-bugfix
avetswirldslabs Feb 12, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
122 changes: 89 additions & 33 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,18 +40,18 @@

<properties>
<hamcrest-all.version>1.3</hamcrest-all.version>
<mockito.version>3.12.4</mockito.version>
<junit.version>5.7.2</junit.version>
<junit.platform.version>1.8.0-RC1</junit.platform.version>
<junit.vintage.version>5.8.0-RC1</junit.vintage.version>
<jmockit.version>1.48</jmockit.version>
<mockito.version>4.5.1</mockito.version>
<junit.version>5.8.2</junit.version>
<junit.platform.version>1.8.2</junit.platform.version>
<junit.vintage.version>5.8.2</junit.vintage.version>
<jmockit.version>1.49</jmockit.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<log4j.version>2.17.1</log4j.version>
<amazonaws-s3.version>1.12.73</amazonaws-s3.version>
<log4j.version>2.17.2</log4j.version>
<amazonaws-s3.version>1.12.237</amazonaws-s3.version>
<amazonaws-lambda.version>1.2.1</amazonaws-lambda.version>
<amazonaws-log4j.version>1.2.0</amazonaws-log4j.version>
<amazonaws-sns.version>1.12.73</amazonaws-sns.version>
<hedera-java-sdk.version>2.14.0-beta.1</hedera-java-sdk.version>
<amazonaws-log4j.version>1.5.1</amazonaws-log4j.version>
<amazonaws-sns.version>1.12.237</amazonaws-sns.version>
<hedera-java-sdk.version>2.17.3</hedera-java-sdk.version>
</properties>


Expand All @@ -71,7 +71,7 @@
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>2.12.6.1</version>
<version>2.13.3</version>
</dependency>
<dependency>
<groupId>org.jmockit</groupId>
Expand All @@ -93,7 +93,7 @@
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-math3</artifactId>
<version>3.5</version>
<version>3.6.1</version>
</dependency>

<dependency>
Expand Down Expand Up @@ -152,13 +152,16 @@
<version>${mockito.version}</version>
<scope>test</scope>
</dependency>


<!-- https://mvnrepository.com/artifact/com.hedera.hashgraph/sdk -->
<dependency>
<groupId>com.hedera.hashgraph</groupId>
<artifactId>sdk-jdk7</artifactId>
<version>${hedera-java-sdk.version}</version>
<artifactId>sdk-full</artifactId>
<version>2.16.1</version>
</dependency>
<!-- netty transport (for server or desktop applications) -->
<dependency>
<groupId>io.grpc</groupId>
<artifactId>grpc-netty-shaded</artifactId>
<version>1.46.0</version>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
Expand All @@ -183,34 +186,48 @@
<dependency>
<groupId>javax.xml.bind</groupId>
<artifactId>jaxb-api</artifactId>
<version>2.3.0</version>
</dependency>
<!-- https://mvnrepository.com/artifact/com.google.cloud/google-cloud-spanner -->
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-spanner</artifactId>
<version>1.33.0</version>
</dependency>
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-storage</artifactId>
<version>1.88.0</version>
<version>2.3.1</version>
</dependency>
<dependency>
<groupId>org.flywaydb</groupId>
<artifactId>flyway-core</artifactId>
<version>7.4.0</version>
<version>8.5.11</version>
</dependency>
<dependency>
<groupId>org.bouncycastle</groupId>
<artifactId>bcprov-jdk15on</artifactId>
<version>1.67</version>
<version>1.70</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.postgresql/postgresql -->
<dependency>
<groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId>
<version>42.3.3</version>
<version>42.3.4</version>
</dependency>
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-storage</artifactId>
<version>2.8.1</version>
</dependency>
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-pubsub</artifactId>
<version>1.119.1</version>
</dependency>
<dependency>
<groupId>com.google.cloud.functions</groupId>
<artifactId>functions-framework-api</artifactId>
<version>1.0.4</version>
</dependency>
<dependency>
<groupId>com.google.cloud.sql</groupId>
<artifactId>postgres-socket-factory</artifactId>
<version>1.6.1</version>
</dependency>
<dependency>
<groupId>com.zaxxer</groupId>
<artifactId>HikariCP</artifactId>
<version>5.0.1</version>
</dependency>
</dependencies>

Expand Down Expand Up @@ -238,7 +255,23 @@
<build>
<finalName>${project.artifactId}</finalName>
<plugins>
<plugin>
<!--
Google Cloud Functions Framework Maven plugin

This plugin allows you to run Cloud Functions Java code
locally. Use the following terminal command to run a
given function locally:

mvn function:run -Drun.functionTarget=your.package.yourFunction
-->
<groupId>com.google.cloud.functions</groupId>
<artifactId>function-maven-plugin</artifactId>
<version>0.10.0</version>
<configuration>
<functionTarget>com.hedera.exchange.ERTGcp</functionTarget>
</configuration>
</plugin>
<plugin>
<!-- To re-generate the LICENSE.txt and copyright headers, execute the Maven goals
"license:update-project-license" and "license:update-file-header". -->
Expand Down Expand Up @@ -297,7 +330,30 @@
</argLine>
</configuration>
</plugin>

<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<executions>
<execution>
<phase>package</phase>
<goals><goal>shade</goal></goals>
<configuration>
<outputFile>${project.build.directory}/deployment/${project.build.finalName}.jar</outputFile>
<transformers>
<!-- This may be needed if you need to shade a signed JAR -->
<transformer implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
<resource>.SF</resource>
<resource>.DSA</resource>
<resource>.RSA</resource>
</transformer>
<!-- This is needed if you have dependencies that use Service Loader. Most Google Cloud client libraries does. -->
<transformer implementation=
"org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
</transformers>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
Expand Down
62 changes: 62 additions & 0 deletions src/main/java/com/hedera/exchange/ERTAws.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
package com.hedera.exchange;

/*-
* ‌
* Hedera Exchange Rate Tool
* ​
* Copyright (C) 2019 - 2020 Hedera Hashgraph, LLC
* ​
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
*
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* ‍
*
* Copyright (c) 2007-present, Stephen Colebourne & Michael Nascimento Santos.
*
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
*
* * Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* * Neither the name of JSR-310 nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
* EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
* PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
* NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/

import static com.hedera.exchange.ExchangeRateTool.AWS_TAG;

public class ERTAws {
public static void main(final String ... args) {
ExchangeRateTool ert = new ExchangeRateTool();
ert.run(AWS_TAG);
}
}
26 changes: 10 additions & 16 deletions ...hedera/exchange/database/AWSDBParams.java → ...main/java/com/hedera/exchange/ERTGcp.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package com.hedera.exchange.database;
package com.hedera.exchange;

/*-
* ‌
Expand Down Expand Up @@ -52,22 +52,16 @@
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/

import com.hedera.exchange.ERTUtils;
import com.google.cloud.functions.HttpFunction;
import com.google.cloud.functions.HttpRequest;
import com.google.cloud.functions.HttpResponse;

public class AWSDBParams {
public String getEndpoint() {
return ERTUtils.getDecryptedEnvironmentVariableFromAWS("ENDPOINT") + getDatabaseName();
}

public String getUsername() {
return ERTUtils.getDecryptedEnvironmentVariableFromAWS("USERNAME");
}

public String getPassword() {
return ERTUtils.getDecryptedEnvironmentVariableFromAWS("PASSWORD");
}
import static com.hedera.exchange.ExchangeRateTool.GCP_TAG;

public String getDatabaseName() {
return ERTUtils.getDecryptedEnvironmentVariableFromAWS("DATABASE");
public class ERTGcp implements HttpFunction {
@Override
public void service(final HttpRequest httpRequest, final HttpResponse httpResponse) throws Exception {
ExchangeRateTool ert = new ExchangeRateTool();
ert.run(GCP_TAG);
}
}
77 changes: 77 additions & 0 deletions src/main/java/com/hedera/exchange/ERTNotificationHelper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,10 +57,23 @@
import com.amazonaws.services.sns.AmazonSNSClient;
import com.amazonaws.services.sns.AmazonSNSClientBuilder;
import com.amazonaws.services.sns.model.AmazonSNSException;
import com.google.api.core.ApiFuture;
import com.google.api.core.ApiFutureCallback;
import com.google.api.core.ApiFutures;
import com.google.api.gax.rpc.ApiException;
import com.google.cloud.pubsub.v1.Publisher;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.protobuf.ByteString;
import com.google.pubsub.v1.PubsubMessage;
import com.google.pubsub.v1.TopicName;
import com.hedera.exchange.exchanges.Exchange;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

import java.io.IOException;
import java.util.List;
import java.util.concurrent.TimeUnit;

import static com.hedera.exchange.ERTUtils.getDecryptedEnvironmentVariableFromAWS;

public final class ERTNotificationHelper {
Expand All @@ -78,6 +91,70 @@ private ERTNotificationHelper() {
* Content of the Email
*/
public static void publishMessage(final String subject, final String message, final String region) {
if (ExchangeRateTool.env == Environment.AWS) {
publishMessageUsingSNS(subject, message, region);
} else {
try {
publishMessageUsingPubSub(List.of(subject, message));
} catch (InterruptedException ex) {
LOGGER.error(Exchange.EXCHANGE_FILTER, "failed to terminate PubSub publisher");
}
}
}

private static void publishMessageUsingPubSub(List<String> messages) throws InterruptedException {
final var projectId = System.getenv("PROJECT_ID");
final var topicId = System.getenv("TOPIC_ID");
LOGGER.info(Exchange.EXCHANGE_FILTER, "projectId is : {}, topicId is {}", projectId, topicId);
TopicName topicName = TopicName.of(projectId, topicId);
Publisher publisher = null;

try {
publisher = Publisher.newBuilder(topicName).build();

for (var message : messages) {
ByteString data = ByteString.copyFromUtf8(message);
PubsubMessage pubsubMessage = PubsubMessage.newBuilder().setData(data).build();

// Once published, returns a server-assigned message id (unique within the topic)
ApiFuture<String> future = publisher.publish(pubsubMessage);

// Add an asynchronous callback to handle success / failure
ApiFutures.addCallback(
future,
new ApiFutureCallback<>() {

@Override
public void onFailure(Throwable throwable) {
if (throwable instanceof ApiException) {
ApiException apiException = ((ApiException) throwable);
// details on the API exception
System.out.println(apiException.getStatusCode().getCode());
System.out.println(apiException.isRetryable());
}
System.out.println("Error publishing message : " + message);
}

@Override
public void onSuccess(String messageId) {
// Once published, returns server-assigned message ids (unique within the topic)
System.out.println("Published message ID: " + messageId);
}
},
MoreExecutors.directExecutor());
}
} catch (IOException ex) {
LOGGER.error(Exchange.EXCHANGE_FILTER, "Failed to publish message");
} finally {
if (publisher != null) {
// When finished with the publisher, shutdown to free up resources.
publisher.shutdown();
publisher.awaitTermination(1, TimeUnit.MINUTES);
}
}
}

private static void publishMessageUsingSNS(final String subject, final String message, final String region) {
try {
final AmazonSNSClient SNS_CLIENT = (AmazonSNSClient) AmazonSNSClientBuilder.standard()
.withRegion(getValidRegion(region))
Expand Down
Loading