Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add Data model Info to the Mlayer All Datasets API Response #368

Open
wants to merge 12 commits into
base: 5.5.0
Choose a base branch
from
Open

Add Data model Info to the Mlayer All Datasets API Response #368

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
9c28ca5
Api call to voc service to fetch data model info
Kranthi-Guribilli Jun 14, 2024
25d0af3
update dbservice's mlayer all datasets to pass config as param
Kranthi-Guribilli Jun 14, 2024
539cd6b
update mlayer dataset to include domain field in the resp
Kranthi-Guribilli Jun 14, 2024
f5c41c3
Revert "update dbservice's mlayer all datasets to pass config as param"
Kranthi-Guribilli Jun 14, 2024
1ffad13
revert changes
Kranthi-Guribilli Jun 14, 2024
0e349e7
Done requested changes
Kranthi-Guribilli Jun 14, 2024
b497108
resolve checkstyle warning
Kranthi-Guribilli Jun 14, 2024
8cd4dc0
Rename variables
Kranthi-Guribilli Jun 18, 2024
80060cf
Update MlayerDataset.java
Kranthi-Guribilli Jun 18, 2024
f12e8d4
Update DataModel.java
Kranthi-Guribilli Jun 18, 2024
d121870
Improve checkstyle warnings
Kranthi-Guribilli Jun 18, 2024
36ec3b9
Update junits
Kranthi-Guribilli Jun 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions src/main/java/iudx/catalogue/server/database/Constants.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,9 @@ public class Constants {
+ "[\"type\",\"id\",\"label\",\"accessPolicy\",\"tags\",\"instance\","
+ "\"provider\", \"resourceServerRegURL\",\"description\" ,\"cosURL\", "
+ "\"cos\", \"resourceGroup\", \"resourceType\"]},\"size\": 10000}";
public static final String GET_ALL_DATASETS_BY_RS_GRP =
"{\"size\":10000,\"query\":{\"bool\":{\"must\":[{\"bool\":{\"should\":[{\"match\":"
+ "{\"type.keyword\":\"iudx:ResourceGroup\"}}]}}]}}}";
public static final String GET_ALL_DATASETS_BY_FIELDS =
"{\"query\":{\"bool\":{\"should\":[{\"bool\":{\"must\":[{\"match\":"
+ "{\"type.keyword\":\"iudx:ResourceGroup\"}}";
Expand Down
157 changes: 90 additions & 67 deletions src/main/java/iudx/catalogue/server/database/mlayer/MlayerDataset.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import iudx.catalogue.server.database.ElasticClient;
import iudx.catalogue.server.database.RespBuilder;
import iudx.catalogue.server.database.Util;
import iudx.catalogue.server.mlayer.vocabulary.DataModel;
import java.util.HashMap;
import java.util.Map;
import org.apache.logging.log4j.LogManager;
Expand Down Expand Up @@ -136,7 +137,9 @@ public void getMlayerDataset(JsonObject requestData, Handler<AsyncResult<JsonObj
}

public void getMlayerAllDatasets(
JsonObject requestParam, String query, Handler<AsyncResult<JsonObject>> handler) {
JsonObject requestParam,
String query,
Handler<AsyncResult<JsonObject>> handler) {

LOGGER.debug("Getting all the resource group items");
Promise<JsonObject> datasetResult = Promise.promise();
Expand All @@ -151,74 +154,94 @@ public void getMlayerAllDatasets(
.onComplete(
ar -> {
if (ar.succeeded()) {
JsonObject instanceList = ar.result().resultAt(0);
JsonObject resourceGroupList = ar.result().resultAt(1);
JsonObject resourceAndPolicyCount = ar.result().resultAt(2);
JsonArray resourceGroupArray = new JsonArray();
LOGGER.debug("getMlayerDatasets resourceGroupList iteration started");
for (int i = 0; i < resourceGroupList.getInteger("resourceGroupCount"); i++) {
JsonObject record =
resourceGroupList.getJsonArray("resourceGroup").getJsonObject(i);
record.put(
"icon",
record.containsKey(INSTANCE)
? instanceList.getString(record.getString(INSTANCE))
: "");
record.put(
"totalResources",
resourceAndPolicyCount
.getJsonObject("resourceItemCount")
.containsKey(record.getString(ID))
? resourceAndPolicyCount
.getJsonObject("resourceItemCount")
.getInteger(record.getString(ID))
: 0);
if (resourceAndPolicyCount
.getJsonObject("resourceAccessPolicy")
.containsKey(record.getString(ID))) {
record.put(
ACCESS_POLICY,
resourceAndPolicyCount
.getJsonObject("resourceAccessPolicy")
.getJsonObject(record.getString(ID)));
} else {
record.put(
ACCESS_POLICY,
new JsonObject().put("PII", 0).put("SECURE", 0).put("OPEN", 0));
}
record.remove(TYPE);
resourceGroupArray.add(record);
}
JsonArray pagedResourceGroups = new JsonArray();
int endIndex =
requestParam.getInteger(LIMIT) + requestParam.getInteger(OFFSET);
if (endIndex >= resourceGroupArray.size()) {
if (requestParam.getInteger(OFFSET) >= resourceGroupArray.size()) {
LOGGER.debug("Offset value has exceeded total hits");
RespBuilder respBuilder =
new RespBuilder()
.withType(TYPE_SUCCESS)
.withTitle(SUCCESS)
.withTotalHits(resourceGroupList.getInteger("resourceGroupCount"));
handler.handle(Future.succeededFuture(respBuilder.getJsonResponse()));
} else {
endIndex = resourceGroupArray.size();
}
}
for (int i = requestParam.getInteger(OFFSET); i < endIndex; i++) {
pagedResourceGroups.add(resourceGroupArray.getJsonObject(i));
}
DataModel dataModel = new DataModel(client, docIndex);
dataModel
.getDataModelInfo()
.onComplete(
domainInfoResult -> {
if (domainInfoResult.succeeded()) {
JsonObject domains = domainInfoResult.result();

LOGGER.debug("getMlayerDatasets resourceGroupList interation succeeded");
RespBuilder respBuilder =
new RespBuilder()
.withType(TYPE_SUCCESS)
.withTitle(SUCCESS)
.withTotalHits(resourceGroupList.getInteger("resourceGroupCount"))
.withResult(pagedResourceGroups);
LOGGER.debug("getMlayerDatasets succeeded");
handler.handle(Future.succeededFuture(respBuilder.getJsonResponse()));
JsonObject instanceList = ar.result().resultAt(0);
JsonObject resourceGroupList = ar.result().resultAt(1);
JsonObject resourceAndPolicyCount = ar.result().resultAt(2);
JsonArray resourceGroupArray = new JsonArray();
LOGGER.debug("getMlayerDatasets resourceGroupList iteration started");
for (int i = 0;
i < resourceGroupList.getInteger("resourceGroupCount");
i++) {
JsonObject record =
resourceGroupList.getJsonArray("resourceGroup").getJsonObject(i);
record.put(
"icon",
record.containsKey(INSTANCE)
? instanceList.getString(record.getString(INSTANCE))
: "");
record.put(
"totalResources",
resourceAndPolicyCount
.getJsonObject("resourceItemCount")
.containsKey(record.getString(ID))
? resourceAndPolicyCount
.getJsonObject("resourceItemCount")
.getInteger(record.getString(ID))
: 0);
if (resourceAndPolicyCount
.getJsonObject("resourceAccessPolicy")
.containsKey(record.getString(ID))) {
record.put(
ACCESS_POLICY,
resourceAndPolicyCount
.getJsonObject("resourceAccessPolicy")
.getJsonObject(record.getString(ID)));
} else {
record.put(
ACCESS_POLICY,
new JsonObject().put("PII", 0).put("SECURE", 0).put("OPEN", 0));
}
if (domains.getString(record.getString("id")) != null) {
record.put("domain", domains.getString(record.getString("id")));
}
record.remove(TYPE);
resourceGroupArray.add(record);
}
JsonArray pagedResourceGroups = new JsonArray();
int endIndex =
requestParam.getInteger(LIMIT) + requestParam.getInteger(OFFSET);
if (endIndex >= resourceGroupArray.size()) {
if (requestParam.getInteger(OFFSET) >= resourceGroupArray.size()) {
LOGGER.debug("Offset value has exceeded total hits");
RespBuilder respBuilder =
new RespBuilder()
.withType(TYPE_SUCCESS)
.withTitle(SUCCESS)
.withTotalHits(
resourceGroupList.getInteger("resourceGroupCount"));
handler.handle(
Future.succeededFuture(respBuilder.getJsonResponse()));
} else {
endIndex = resourceGroupArray.size();
}
}
for (int i = requestParam.getInteger(OFFSET); i < endIndex; i++) {
pagedResourceGroups.add(resourceGroupArray.getJsonObject(i));
}

LOGGER.debug("getMlayerDatasets resourceGroupList iteration succeeded");
RespBuilder respBuilder =
new RespBuilder()
.withType(TYPE_SUCCESS)
.withTitle(SUCCESS)
.withTotalHits(
resourceGroupList.getInteger("resourceGroupCount"))
.withResult(pagedResourceGroups);
LOGGER.debug("getMlayerDatasets succeeded");
handler.handle(Future.succeededFuture(respBuilder.getJsonResponse()));
} else {
LOGGER.error("Fail: failed DataModel request");
handler.handle(Future.failedFuture(internalErrorResp));
}
});
} else {
LOGGER.error("Fail: failed DB request");
handler.handle(Future.failedFuture(internalErrorResp));
Expand Down
14 changes: 4 additions & 10 deletions src/main/java/iudx/catalogue/server/mlayer/MlayerServiceImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,7 @@ public class MlayerServiceImpl implements MlayerService {
private JsonArray excludedIdsJson;

MlayerServiceImpl(
DatabaseService databaseService,
PostgresService postgresService,
JsonObject config) {
DatabaseService databaseService, PostgresService postgresService, JsonObject config) {
this.databaseService = databaseService;
this.postgresService = postgresService;
this.configJson = config;
Expand Down Expand Up @@ -281,13 +279,10 @@ public MlayerService getMlayerDataset(
|| requestData.containsKey("providers")
|| requestData.containsKey("domains"))
&& (!requestData.containsKey(ID) || requestData.getString(ID).isBlank())) {
if (requestData.containsKey("domains")
&& !requestData.getJsonArray("domains").isEmpty()) {
if (requestData.containsKey("domains") && !requestData.getJsonArray("domains").isEmpty()) {
JsonArray domainsArray = requestData.getJsonArray("domains");
JsonArray tagsArray =
requestData.containsKey("tags")
? requestData.getJsonArray("tags")
: new JsonArray();
requestData.containsKey("tags") ? requestData.getJsonArray("tags") : new JsonArray();

tagsArray.addAll(domainsArray);
requestData.put("tags", tagsArray);
Expand All @@ -314,8 +309,7 @@ public MlayerService getMlayerDataset(
",{\"match\":{\"instance.keyword\":\"$1\"}}"
.replace("$1", requestData.getString(INSTANCE).toLowerCase()));
}
if (requestData.containsKey(PROVIDERS)
&& !requestData.getJsonArray(PROVIDERS).isEmpty()) {
if (requestData.containsKey(PROVIDERS) && !requestData.getJsonArray(PROVIDERS).isEmpty()) {
query =
query.concat(
",{\"terms\":{\"provider.keyword\":$1}}"
Expand Down
Loading