From b00716153500e6c852e478052d054457fbd4db70 Mon Sep 17 00:00:00 2001 From: hilary egesa Date: Wed, 28 Aug 2024 14:18:01 +0300 Subject: [PATCH 1/8] make updating fts search tables optional - This updates can take a long time during initial sync - The implementing application can directly query the original tables with the that have columns that are indexed at a similar or better speeds --- README.md | 5 +++-- gradle.properties | 2 +- gradle/wrapper/gradle-wrapper.properties | 2 +- .../src/main/java/org/smartregister/AllConstants.java | 1 + .../java/org/smartregister/sync/ClientProcessorForJava.java | 3 ++- 5 files changed, 8 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index b1f2e8da5..34bb0a109 100644 --- a/README.md +++ b/README.md @@ -307,11 +307,12 @@ By placing a file named `app.properties` in your implementation assets folder (S ### Configurable Settings | Configuration | Type | Default | Description | -| ---------------------------------------- | ------- | ------- | --------------------------------------------------------------------------------------------------------------------------------------------------- | +| ---------------------------------------- | ------- | ------- |-----------------------------------------------------------------------------------------------------------------------------------------------------| | `system.toaster.centered` | Boolean | false | Position toaster(s) at the center of the view(s) | | `disable.location.picker.view` | Boolean | false | Disables LocationPicker View | | `location.picker.tag.shown` | Boolean | false | Hides/Shows the location tag in the location picker tree view | | `encrypt.shared.preferences` | Boolean | false | Enable/disables encrypting SharedPreferences | | `allow.offline.login.with.invalid.token` | Boolean | false | Allow offline login when token is no longer valid after a successful login when online and user is forcefully logged out | | `enable.search.button` | Boolean | false | Enable/Disable search to be triggered only after clicking the search icon in `org.smartregister.view.fragment.BaseRegisterFragment` or its subclass | -| `feature.profile.images.disabled` | Boolean | false | Disable profile image capturing and rendering | \ No newline at end of file +| `feature.profile.images.disabled` | Boolean | false | Disable profile image capturing and rendering | +| `"disable.fts.search` | Boolean | false | Disable fts search which can slow down sync process | \ No newline at end of file diff --git a/gradle.properties b/gradle.properties index c869817a4..e27e4ee3a 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ -VERSION_NAME=6.1.4-SNAPSHOT +VERSION_NAME=6.1.5-SNAPSHOT VERSION_CODE=1 GROUP=org.smartregister POM_SETTING_DESCRIPTION=OpenSRP Client Core Application diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 7546b80ac..bc537e0d2 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -7,4 +7,4 @@ org.gradle.daemon=true org.gradle.parallel=true zipStoreBase=GRADLE_USER_HOME org.gradle.configureondemand=true -org.gradle.jvmargs=-Xmx2048m +org.gradle.jvmargs=-Xmx2048m --add-opens java.base/java.io=ALL-UNNAMED --add-exports jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED diff --git a/opensrp-core/src/main/java/org/smartregister/AllConstants.java b/opensrp-core/src/main/java/org/smartregister/AllConstants.java index 4d7c0a6ce..8f1391868 100644 --- a/opensrp-core/src/main/java/org/smartregister/AllConstants.java +++ b/opensrp-core/src/main/java/org/smartregister/AllConstants.java @@ -352,6 +352,7 @@ public static class PROPERTY { public static final String IGNORE_LOCATION_DELETION = "ignore.location.deletion"; public static final String ENABLE_SEARCH_BUTTON = "enable.search.button"; public static final String DISABLE_PROFILE_IMAGES_FEATURE = "feature.profile.images.disabled"; + public static final String DISABLE_FTS_SEARCH_DISABLE="disable.fts.search"; } public static class HTTP_REQUEST_HEADERS { diff --git a/opensrp-core/src/main/java/org/smartregister/sync/ClientProcessorForJava.java b/opensrp-core/src/main/java/org/smartregister/sync/ClientProcessorForJava.java index 79e01dd63..8848e6435 100644 --- a/opensrp-core/src/main/java/org/smartregister/sync/ClientProcessorForJava.java +++ b/opensrp-core/src/main/java/org/smartregister/sync/ClientProcessorForJava.java @@ -364,7 +364,8 @@ public Boolean processCaseModel(Event event, Client client, List creates String entityId = contentValues.getAsString(CommonRepository.BASE_ENTITY_ID_COLUMN); String clientType = client.getClientType() != null ? client.getClientType() : (client.getRelationships() != null ? AllConstants.ECClientType.CHILD : null); - updateFTSsearch(tableName, clientType, entityId, contentValues); + if (!CoreLibrary.getInstance().context().getAppProperties().isTrue(AllConstants.PROPERTY.DISABLE_PROFILE_IMAGES_FEATURE)) + updateFTSsearch(tableName, clientType, entityId, contentValues); Long timestamp = getEventDate(event.getEventDate()); addContentValuesToDetailsTable(contentValues, timestamp); updateClientDetailsTable(event, client); From 5724c5640c2fea4ec92956bf55522b125675e7b6 Mon Sep 17 00:00:00 2001 From: Hilary Baraka Egesa Date: Wed, 28 Aug 2024 14:23:27 +0300 Subject: [PATCH 2/8] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 34bb0a109..1e1e39d87 100644 --- a/README.md +++ b/README.md @@ -315,4 +315,4 @@ By placing a file named `app.properties` in your implementation assets folder (S | `allow.offline.login.with.invalid.token` | Boolean | false | Allow offline login when token is no longer valid after a successful login when online and user is forcefully logged out | | `enable.search.button` | Boolean | false | Enable/Disable search to be triggered only after clicking the search icon in `org.smartregister.view.fragment.BaseRegisterFragment` or its subclass | | `feature.profile.images.disabled` | Boolean | false | Disable profile image capturing and rendering | -| `"disable.fts.search` | Boolean | false | Disable fts search which can slow down sync process | \ No newline at end of file +| `disable.fts.search` | Boolean | false | Disable fts search which can slow down sync process | From e7e227b4f2ba62b92f29c37bf33430935b616c7e Mon Sep 17 00:00:00 2001 From: hilary egesa Date: Thu, 5 Sep 2024 12:11:56 +0300 Subject: [PATCH 3/8] additional instrumentation --- gradle.properties | 2 +- .../repository/DetailsRepository.java | 6 +++ .../sync/ClientProcessorForJava.java | 50 +++++++++++++++++-- 3 files changed, 52 insertions(+), 6 deletions(-) diff --git a/gradle.properties b/gradle.properties index e27e4ee3a..4006d8b03 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ -VERSION_NAME=6.1.5-SNAPSHOT +VERSION_NAME=6.1.5-ALPHA24-SNAPSHOT VERSION_CODE=1 GROUP=org.smartregister POM_SETTING_DESCRIPTION=OpenSRP Client Core Application diff --git a/opensrp-core/src/main/java/org/smartregister/repository/DetailsRepository.java b/opensrp-core/src/main/java/org/smartregister/repository/DetailsRepository.java index f98b84c83..0d6b9c3a7 100644 --- a/opensrp-core/src/main/java/org/smartregister/repository/DetailsRepository.java +++ b/opensrp-core/src/main/java/org/smartregister/repository/DetailsRepository.java @@ -34,7 +34,9 @@ protected void onCreate(SQLiteDatabase database) { public void add(String baseEntityId, String key, String value, Long timestamp) { SQLiteDatabase database = masterRepository().getWritableDatabase(); +// long start = System.currentTimeMillis(); Boolean exists = getIdForDetailsIfExists(baseEntityId, key, value); +// Timber.d("check if details exist's took %s, ", System.currentTimeMillis() - start); if (exists == null) { // Value has not changed, no need to update return; } @@ -46,12 +48,16 @@ public void add(String baseEntityId, String key, String value, Long timestamp) { values.put(EVENT_DATE_COLUMN, timestamp); if (exists) { + long startUpdate = System.currentTimeMillis(); int updated = database.update(TABLE_NAME, values, BASE_ENTITY_ID_COLUMN + " = ? AND " + KEY_COLUMN + " MATCH ? ", new String[]{baseEntityId, key}); +// Timber.d("updating details for %S took %s, ", TABLE_NAME, System.currentTimeMillis() - startUpdate); //Log.i(getClass().getName(), "Detail Row Updated: " + String.valueOf(updated)); } else { + long insertStart = System.currentTimeMillis(); long rowId = database.insert(TABLE_NAME, null, values); +// Timber.d("insert into details %s table took %s, ", TABLE_NAME, System.currentTimeMillis() - insertStart); //Log.i(getClass().getName(), "Details Row Inserted : " + String.valueOf(rowId)); } } diff --git a/opensrp-core/src/main/java/org/smartregister/sync/ClientProcessorForJava.java b/opensrp-core/src/main/java/org/smartregister/sync/ClientProcessorForJava.java index 8848e6435..b2434840c 100644 --- a/opensrp-core/src/main/java/org/smartregister/sync/ClientProcessorForJava.java +++ b/opensrp-core/src/main/java/org/smartregister/sync/ClientProcessorForJava.java @@ -84,7 +84,7 @@ public synchronized void processClient(List eventClientList) throws } public synchronized void processClient(List eventClientList, boolean localSubmission) throws Exception { - + long startProcessingClient = System.currentTimeMillis(); final String EC_CLIENT_CLASSIFICATION = "ec_client_classification.json"; ClientClassification clientClassification = assetJsonToJava(EC_CLIENT_CLASSIFICATION, ClientClassification.class); if (clientClassification == null) { @@ -92,6 +92,8 @@ public synchronized void processClient(List eventClientList, boolea } if (!eventClientList.isEmpty()) { + long startLooping = System.currentTimeMillis(); + Timber.d("number of eventClientList %s, ", eventClientList.size()); for (EventClient eventClient : eventClientList) { // Iterate through the events Client client = eventClient.getClient(); @@ -101,20 +103,28 @@ public synchronized void processClient(List eventClientList, boolea if (processorMap.containsKey(eventType)) { try { + long startMiniProcessor = System.currentTimeMillis(); processEventUsingMiniProcessor(clientClassification, eventClient, eventType); + Timber.d("processEventUsingMiniProcessor took, %s ", System.currentTimeMillis()-startMiniProcessor); } catch (Exception ex) { Timber.e(ex); } } else { + long startProcessEvent = System.currentTimeMillis(); processEvent(event, client, clientClassification); + Timber.d("processEvent Inside processClient method took, %s ", System.currentTimeMillis() - startProcessEvent); } } if (localSubmission && CoreLibrary.getInstance().getSyncConfiguration().runPlanEvaluationOnClientProcessing()) { + long startProcessPlan = System.currentTimeMillis(); processPlanEvaluation(eventClient); + Timber.d("process planEvaluation inside processClient took, %s ", System.currentTimeMillis()-startProcessingClient); } } + Timber.d("looping through eventClientList took %s, ", System.currentTimeMillis() - startLooping); } + Timber.d("processing client took, %s", System.currentTimeMillis()-startProcessingClient); } /** @@ -182,7 +192,9 @@ public Boolean processEvent(Event event, Client client, ClientClassification cli } for (ClassificationRule clientClass : clientClasses) { + long start = System.currentTimeMillis(); processClientClass(clientClass, event, client); + Timber.d("process client class took, %s", System.currentTimeMillis()-start); } // Incase the details have not been updated @@ -214,10 +226,13 @@ public Boolean processClientClass(ClassificationRule clientClass, Event event, C Rule rule = clientClass.rule; List fields = rule.fields; - + long startProcessingFields = System.currentTimeMillis(); for (org.smartregister.domain.jsonmapping.Field field : fields) { + long start = System.currentTimeMillis(); processField(field, event, client); + Timber.d("processing %s field took %s, ", field.field, System.currentTimeMillis() - start); } + Timber.i("processing %s fields for %s took %s, ", rule.fields.size(), rule.type, System.currentTimeMillis() - startProcessingFields); return true; } catch (Exception e) { Timber.e(e); @@ -277,8 +292,12 @@ public Boolean processField(org.smartregister.domain.jsonmapping.Field field, Ev .disjoint(responseValues, docSegmentResponseValues))) { // this is the event obs we're interested in put it in the respective // bucket specified by type variable + long processCaseStart = System.currentTimeMillis(); processCaseModel(event, client, createsCase); + long processCaseEnd = System.currentTimeMillis(); + Timber.w("==processCaseModel if dataSegment is instance of list took %s, ", processCaseEnd - processCaseStart); closeCase(client, closesCase); + Timber.w("==closeCase took if dataSegment is instance of list took %s, ", System.currentTimeMillis() - processCaseEnd); } } @@ -291,8 +310,12 @@ public Boolean processField(org.smartregister.domain.jsonmapping.Field field, Ev if (objectValue != null && objectValue instanceof String) { String docSegmentFieldValue = objectValue.toString(); if (docSegmentFieldValue.equalsIgnoreCase(fieldValue)) { + long startProcessCaseModel = System.currentTimeMillis(); processCaseModel(event, client, createsCase); + long endCaseModelProcessing = System.currentTimeMillis(); + Timber.w("**processCaseModel if dataSegment is a map took, %s ",endCaseModelProcessing - startProcessCaseModel); closeCase(client, closesCase); + Timber.w("**closecase if dataSegment is a map, %s ", System.currentTimeMillis() - endCaseModelProcessing); } } } @@ -304,8 +327,12 @@ public Boolean processField(org.smartregister.domain.jsonmapping.Field field, Ev Object value = getValue(event, fieldName); String docSegmentFieldValue = value != null ? value.toString() : ""; if (docSegmentFieldValue.equalsIgnoreCase(fieldValue)) { + long startProcessCaseModel = System.currentTimeMillis(); processCaseModel(event, client, createsCase); + long endProccessCaseModel = System.currentTimeMillis(); + Timber.w("&&&&processCaseModel if dataSegment is null, took %s ", endProccessCaseModel-startProcessCaseModel); closeCase(client, closesCase); + Timber.w("&&&&&&closeCase if dataSegment is null, took %s ", System.currentTimeMillis() - endProccessCaseModel); } } return true; @@ -351,24 +378,35 @@ public Boolean processCaseModel(Event event, Client client, List creates //Add the base_entity_id contentValues.put(CommonRepository.BASE_ENTITY_ID_COLUMN, baseEntityId); contentValues.put(CommonRepository.IS_CLOSED_COLUMN, 0); - + long iterateOverColumns = System.currentTimeMillis(); for (Column colObject : columns) { processCaseModel(event, client, colObject, contentValues); } + Timber.w("processingCaseModel for %s columns took, %s", columns.size(), System.currentTimeMillis() - iterateOverColumns); // Modify openmrs generated identifier, Remove hyphen if it exists updateIdentifier(contentValues); // save the values to db + long execInsertStart = System.currentTimeMillis(); executeInsertStatement(contentValues, tableName); + Timber.i("executeInsertStatement took %s, ", System.currentTimeMillis()-execInsertStart); String entityId = contentValues.getAsString(CommonRepository.BASE_ENTITY_ID_COLUMN); String clientType = client.getClientType() != null ? client.getClientType() : (client.getRelationships() != null ? AllConstants.ECClientType.CHILD : null); - if (!CoreLibrary.getInstance().context().getAppProperties().isTrue(AllConstants.PROPERTY.DISABLE_PROFILE_IMAGES_FEATURE)) + if (!CoreLibrary.getInstance().context().getAppProperties().isTrue(AllConstants.PROPERTY.DISABLE_PROFILE_IMAGES_FEATURE)) { + long ftsstart = System.currentTimeMillis(); updateFTSsearch(tableName, clientType, entityId, contentValues); + Timber.w("Fts search update took %s, ", System.currentTimeMillis() - ftsstart); + } Long timestamp = getEventDate(event.getEventDate()); + long addContent = System.currentTimeMillis(); addContentValuesToDetailsTable(contentValues, timestamp); + long endAdd = System.currentTimeMillis(); + Timber.i("addContentValuesToDetailsTable took %s, ", endAdd - addContent); updateClientDetailsTable(event, client); + Timber.i("updateClientDetailsTable took %s, ", System.currentTimeMillis() - endAdd); + } return true; @@ -632,6 +670,7 @@ protected void addContentValuesToDetailsTable(ContentValues values, Long eventDa public void updateClientDetailsTable(Event event, Client client) { try { Timber.d("Started updateClientDetailsTable"); + long updateClientDetailsStart = System.currentTimeMillis(); if (CoreLibrary.getInstance().getSyncConfiguration().updateClientDetailsTable()) { String baseEntityId = client.getBaseEntityId(); @@ -649,10 +688,11 @@ public void updateClientDetailsTable(Event event, Client client) { Map obs = getObsFromEvent(event); saveClientDetails(baseEntityId, obs, timestamp); } - + Timber.i("Updating client details took, %s ", System.currentTimeMillis()-updateClientDetailsStart); event.addDetails(detailsUpdated, Boolean.TRUE.toString()); Timber.d("Finished updateClientDetailsTable"); + // save the other misc, client info date of birth... } catch (Exception e) { Timber.e(e); From 7d5426de35bd569538a0e42e17aaf026b9b0015d Mon Sep 17 00:00:00 2001 From: hilary egesa Date: Sun, 15 Sep 2024 06:27:07 +0300 Subject: [PATCH 4/8] add some batch inserts --- gradle.properties | 2 +- .../repository/DetailsRepository.java | 51 +++++++++++++++---- .../org/smartregister/service/HTTPAgent.java | 4 -- .../sync/ClientProcessorForJava.java | 25 ++++++--- 4 files changed, 61 insertions(+), 21 deletions(-) diff --git a/gradle.properties b/gradle.properties index 4006d8b03..0eb048829 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ -VERSION_NAME=6.1.5-ALPHA24-SNAPSHOT +VERSION_NAME=6.1.5-ALPHA29-SNAPSHOT VERSION_CODE=1 GROUP=org.smartregister POM_SETTING_DESCRIPTION=OpenSRP Client Core Application diff --git a/opensrp-core/src/main/java/org/smartregister/repository/DetailsRepository.java b/opensrp-core/src/main/java/org/smartregister/repository/DetailsRepository.java index 0d6b9c3a7..fd2fa4a72 100644 --- a/opensrp-core/src/main/java/org/smartregister/repository/DetailsRepository.java +++ b/opensrp-core/src/main/java/org/smartregister/repository/DetailsRepository.java @@ -9,6 +9,7 @@ import org.smartregister.commonregistry.CommonPersonObjectClient; import java.util.HashMap; +import java.util.List; import java.util.Map; import timber.log.Timber; @@ -47,21 +48,53 @@ public void add(String baseEntityId, String key, String value, Long timestamp) { values.put(VALUE_COLUMN, value); values.put(EVENT_DATE_COLUMN, timestamp); - if (exists) { - long startUpdate = System.currentTimeMillis(); - int updated = database.update(TABLE_NAME, values, - BASE_ENTITY_ID_COLUMN + " = ? AND " + KEY_COLUMN + " MATCH ? ", - new String[]{baseEntityId, key}); + if (exists) { + long startUpdate = System.currentTimeMillis(); + int updated = database.update(TABLE_NAME, values, + BASE_ENTITY_ID_COLUMN + " = ? AND " + KEY_COLUMN + " MATCH ? ", + new String[]{baseEntityId, key}); // Timber.d("updating details for %S took %s, ", TABLE_NAME, System.currentTimeMillis() - startUpdate); - //Log.i(getClass().getName(), "Detail Row Updated: " + String.valueOf(updated)); - } else { - long insertStart = System.currentTimeMillis(); - long rowId = database.insert(TABLE_NAME, null, values); + //Log.i(getClass().getName(), "Detail Row Updated: " + String.valueOf(updated)); + } else { + long insertStart = System.currentTimeMillis(); + long rowId = database.insert(TABLE_NAME, null, values); // Timber.d("insert into details %s table took %s, ", TABLE_NAME, System.currentTimeMillis() - insertStart); //Log.i(getClass().getName(), "Details Row Inserted : " + String.valueOf(rowId)); } } + public void batchInsertDetails(Map values, long timestamp){ + SQLiteDatabase database = null; + try { + database = masterRepository().getReadableDatabase(); + String baseEntityId = values.get(BASE_ENTITY_ID_COLUMN); + + for (String key : values.keySet()) { + String val = values.get(key); + Boolean exists = getIdForDetailsIfExists(baseEntityId, key, val); + if (exists == null) { // Value has not changed, no need to update + continue; + } + ContentValues insertValues = new ContentValues(); + insertValues.put(BASE_ENTITY_ID_COLUMN, baseEntityId); + insertValues.put(KEY_COLUMN, key); + insertValues.put(VALUE_COLUMN, val); + insertValues.put(EVENT_DATE_COLUMN, timestamp); + if (exists) { + long startUpdate = System.currentTimeMillis(); + int updated = database.update(TABLE_NAME, insertValues, + BASE_ENTITY_ID_COLUMN + " = ? AND " + KEY_COLUMN + " MATCH ? ", + new String[]{baseEntityId, key}); + + } else { + long rowId = database.insert(TABLE_NAME, null, insertValues); + } + + } + } catch (Exception e) { + Timber.e(e); + } + } private Boolean getIdForDetailsIfExists(String baseEntityId, String key, String value) { Cursor mCursor = null; try { diff --git a/opensrp-core/src/main/java/org/smartregister/service/HTTPAgent.java b/opensrp-core/src/main/java/org/smartregister/service/HTTPAgent.java index 4c03c092c..9cfd4b9a0 100644 --- a/opensrp-core/src/main/java/org/smartregister/service/HTTPAgent.java +++ b/opensrp-core/src/main/java/org/smartregister/service/HTTPAgent.java @@ -267,8 +267,6 @@ public LoginResponse urlCanBeAccessWithGivenCredentials(String requestURL, Strin if (inputStream != null) responseString = IOUtils.toString(inputStream); if (statusCode == HttpURLConnection.HTTP_OK) { - - Timber.d("response String: %s using request url %s", responseString, url); LoginResponseData responseData = getResponseBody(responseString); loginResponse = retrieveResponse(responseData); } else if (statusCode == HttpURLConnection.HTTP_UNAUTHORIZED) { @@ -355,8 +353,6 @@ private Response processResponse(HttpURLConnection urlConnection) { totalRecords = urlConnection.getHeaderField(AllConstants.SyncProgressConstants.TOTAL_RECORDS); - Timber.d("response string: %s using url %s", responseString, urlConnection.getURL()); - } catch (MalformedURLException exception) { Timber.e(exception, "%s %s", MALFORMED_URL, exception.toString()); ResponseStatus.failure.setDisplayValue(ResponseErrorStatus.malformed_url.name()); diff --git a/opensrp-core/src/main/java/org/smartregister/sync/ClientProcessorForJava.java b/opensrp-core/src/main/java/org/smartregister/sync/ClientProcessorForJava.java index b2434840c..b1e551575 100644 --- a/opensrp-core/src/main/java/org/smartregister/sync/ClientProcessorForJava.java +++ b/opensrp-core/src/main/java/org/smartregister/sync/ClientProcessorForJava.java @@ -4,6 +4,7 @@ import android.content.ContentValues; import android.content.Context; +import android.provider.SyncStateContract; import androidx.annotation.NonNull; @@ -390,7 +391,7 @@ public Boolean processCaseModel(Event event, Client client, List creates // save the values to db long execInsertStart = System.currentTimeMillis(); executeInsertStatement(contentValues, tableName); - Timber.i("executeInsertStatement took %s, ", System.currentTimeMillis()-execInsertStart); + Timber.i("executeInsertStatement took, %s ", System.currentTimeMillis()-execInsertStart); String entityId = contentValues.getAsString(CommonRepository.BASE_ENTITY_ID_COLUMN); String clientType = client.getClientType() != null ? client.getClientType() : (client.getRelationships() != null ? AllConstants.ECClientType.CHILD : null); @@ -647,12 +648,13 @@ protected void addContentValuesToDetailsTable(ContentValues values, Long eventDa return; try { - String baseEntityId = values.getAsString("base_entity_id"); + Map valsMap = new HashMap<>(); for (String key : values.keySet()) { String value = values.getAsString(key); - saveClientDetails(baseEntityId, key, value, eventDate); + valsMap.put(key, value); } + batchSaveClientDetails(valsMap, eventDate); } catch (Exception e) { Timber.e(e); } @@ -677,16 +679,19 @@ public void updateClientDetailsTable(Event event, Client client) { Long timestamp = getEventDate(event.getEventDate()); Map genderInfo = getGender(client); - saveClientDetails(baseEntityId, genderInfo, timestamp); Map addressInfo = getClientAddressAsMap(client); - saveClientDetails(baseEntityId, addressInfo, timestamp); Map attributes = getClientAttributes(client); - saveClientDetails(baseEntityId, attributes, timestamp); Map obs = getObsFromEvent(event); - saveClientDetails(baseEntityId, obs, timestamp); + Map clientDetails = new HashMap<>(); + clientDetails.putAll(genderInfo); + clientDetails.putAll(addressInfo); + clientDetails.putAll(attributes); + clientDetails.putAll(obs); + clientDetails.put("base_entity_id", baseEntityId); + batchSaveClientDetails(clientDetails, timestamp); } Timber.i("Updating client details took, %s ", System.currentTimeMillis()-updateClientDetailsStart); event.addDetails(detailsUpdated, Boolean.TRUE.toString()); @@ -784,6 +789,12 @@ private void saveClientDetails(String baseEntityId, String key, String value, Lo detailsRepository.add(baseEntityId, key, value, timestamp); } + private void batchSaveClientDetails(Map values, Long timestamp) { + DetailsRepository detailsRepository = org.smartregister.CoreLibrary.getInstance().context(). + detailsRepository(); + detailsRepository.batchInsertDetails(values, timestamp); + } + /** * Get human readable values from the json doc humanreadablevalues key if the key is empty From 11e08e77daec5781d7d55029a6c548fb5755480c Mon Sep 17 00:00:00 2001 From: hilary egesa Date: Wed, 25 Sep 2024 18:15:48 +0300 Subject: [PATCH 5/8] replace contentvalues with reusable SqliteStatement --- gradle.properties | 2 +- .../repository/DetailsRepository.java | 65 ++++++++++++++----- .../sync/ClientProcessorForJava.java | 4 +- 3 files changed, 52 insertions(+), 19 deletions(-) diff --git a/gradle.properties b/gradle.properties index 0eb048829..19d4dc6da 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ -VERSION_NAME=6.1.5-ALPHA29-SNAPSHOT +VERSION_NAME=6.1.5-ALPHA41-SNAPSHOT VERSION_CODE=1 GROUP=org.smartregister POM_SETTING_DESCRIPTION=OpenSRP Client Core Application diff --git a/opensrp-core/src/main/java/org/smartregister/repository/DetailsRepository.java b/opensrp-core/src/main/java/org/smartregister/repository/DetailsRepository.java index fd2fa4a72..79bc3471e 100644 --- a/opensrp-core/src/main/java/org/smartregister/repository/DetailsRepository.java +++ b/opensrp-core/src/main/java/org/smartregister/repository/DetailsRepository.java @@ -4,12 +4,12 @@ import android.database.Cursor; import net.sqlcipher.database.SQLiteDatabase; +import net.sqlcipher.database.SQLiteStatement; import org.smartregister.commonregistry.CommonPersonObject; import org.smartregister.commonregistry.CommonPersonObjectClient; import java.util.HashMap; -import java.util.List; import java.util.Map; import timber.log.Timber; @@ -34,7 +34,7 @@ protected void onCreate(SQLiteDatabase database) { } public void add(String baseEntityId, String key, String value, Long timestamp) { - SQLiteDatabase database = masterRepository().getWritableDatabase(); + SQLiteDatabase database = masterRepository().getReadableDatabase(); // long start = System.currentTimeMillis(); Boolean exists = getIdForDetailsIfExists(baseEntityId, key, value); // Timber.d("check if details exist's took %s, ", System.currentTimeMillis() - start); @@ -63,38 +63,71 @@ public void add(String baseEntityId, String key, String value, Long timestamp) { } } - public void batchInsertDetails(Map values, long timestamp){ + public void batchInsertDetails(Map values, long timestamp) { SQLiteDatabase database = null; + SQLiteStatement insertStatement = null; + SQLiteStatement updateStatement = null; + try { - database = masterRepository().getReadableDatabase(); + database = masterRepository().getWritableDatabase(); + + // Prepare the SQL for inserts and updates + String insertSQL = "INSERT INTO " + TABLE_NAME + " (" + + BASE_ENTITY_ID_COLUMN + ", " + KEY_COLUMN + ", " + VALUE_COLUMN + ", " + EVENT_DATE_COLUMN + + ") VALUES (?, ?, ?, ?)"; + + String updateSQL = "UPDATE " + TABLE_NAME + " SET " + VALUE_COLUMN + " = ?, " + + EVENT_DATE_COLUMN + " = ? WHERE " + BASE_ENTITY_ID_COLUMN + " = ? AND " + KEY_COLUMN + " = ?"; + + insertStatement = database.compileStatement(insertSQL); + updateStatement = database.compileStatement(updateSQL); + String baseEntityId = values.get(BASE_ENTITY_ID_COLUMN); for (String key : values.keySet()) { String val = values.get(key); + if(val == null ) continue; Boolean exists = getIdForDetailsIfExists(baseEntityId, key, val); + if (exists == null) { // Value has not changed, no need to update continue; } - ContentValues insertValues = new ContentValues(); - insertValues.put(BASE_ENTITY_ID_COLUMN, baseEntityId); - insertValues.put(KEY_COLUMN, key); - insertValues.put(VALUE_COLUMN, val); - insertValues.put(EVENT_DATE_COLUMN, timestamp); - if (exists) { - long startUpdate = System.currentTimeMillis(); - int updated = database.update(TABLE_NAME, insertValues, - BASE_ENTITY_ID_COLUMN + " = ? AND " + KEY_COLUMN + " MATCH ? ", - new String[]{baseEntityId, key}); + if (exists) { + // Bind values for update + updateStatement.bindString(1, val); // Bind VALUE_COLUMN + updateStatement.bindLong(2, timestamp); // Bind EVENT_DATE_COLUMN + updateStatement.bindString(3, baseEntityId); // Bind BASE_ENTITY_ID_COLUMN + updateStatement.bindString(4, key); // Bind KEY_COLUMN + + // Execute the update + updateStatement.execute(); + updateStatement.clearBindings(); } else { - long rowId = database.insert(TABLE_NAME, null, insertValues); + // Bind values for insert + insertStatement.bindString(1, baseEntityId); // Bind BASE_ENTITY_ID_COLUMN + insertStatement.bindString(2, key); // Bind KEY_COLUMN + insertStatement.bindString(3, val); // Bind VALUE_COLUMN + insertStatement.bindLong(4, timestamp); // Bind EVENT_DATE_COLUMN + + // Execute the insert + insertStatement.executeInsert(); + insertStatement.clearBindings(); } - } } catch (Exception e) { Timber.e(e); + } finally { + // Close the prepared statements + if (insertStatement != null) { + insertStatement.close(); + } + if (updateStatement != null) { + updateStatement.close(); + } } } + private Boolean getIdForDetailsIfExists(String baseEntityId, String key, String value) { Cursor mCursor = null; try { diff --git a/opensrp-core/src/main/java/org/smartregister/sync/ClientProcessorForJava.java b/opensrp-core/src/main/java/org/smartregister/sync/ClientProcessorForJava.java index b1e551575..2ebbcf242 100644 --- a/opensrp-core/src/main/java/org/smartregister/sync/ClientProcessorForJava.java +++ b/opensrp-core/src/main/java/org/smartregister/sync/ClientProcessorForJava.java @@ -404,9 +404,9 @@ public Boolean processCaseModel(Event event, Client client, List creates long addContent = System.currentTimeMillis(); addContentValuesToDetailsTable(contentValues, timestamp); long endAdd = System.currentTimeMillis(); - Timber.i("addContentValuesToDetailsTable took %s, ", endAdd - addContent); + Timber.i("addContentValuesToDetailsTable took, %s", endAdd - addContent); updateClientDetailsTable(event, client); - Timber.i("updateClientDetailsTable took %s, ", System.currentTimeMillis() - endAdd); + Timber.i("updateClientDetailsTable took, %s", System.currentTimeMillis() - endAdd); } From 6adb17798401313f8f32174570580626be2fa7a6 Mon Sep 17 00:00:00 2001 From: hilary egesa Date: Thu, 26 Sep 2024 13:17:48 +0300 Subject: [PATCH 6/8] remove fetchretry recurssive call --- gradle.properties | 2 +- .../java/org/smartregister/sync/intent/SyncIntentService.java | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/gradle.properties b/gradle.properties index 19d4dc6da..570dde5ba 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ -VERSION_NAME=6.1.5-ALPHA41-SNAPSHOT +VERSION_NAME=6.1.5-ALPHA42-SNAPSHOT VERSION_CODE=1 GROUP=org.smartregister POM_SETTING_DESCRIPTION=OpenSRP Client Core Application diff --git a/opensrp-core/src/main/java/org/smartregister/sync/intent/SyncIntentService.java b/opensrp-core/src/main/java/org/smartregister/sync/intent/SyncIntentService.java index 4ed1366ae..404ea4e0c 100644 --- a/opensrp-core/src/main/java/org/smartregister/sync/intent/SyncIntentService.java +++ b/opensrp-core/src/main/java/org/smartregister/sync/intent/SyncIntentService.java @@ -37,6 +37,7 @@ import org.smartregister.domain.SyncEntity; import org.smartregister.domain.SyncProgress; import org.smartregister.domain.db.EventClient; +import org.smartregister.job.SyncServiceJob; import org.smartregister.receiver.SyncStatusBroadcastReceiver; import org.smartregister.repository.AllSharedPreferences; import org.smartregister.repository.EventClientRepository; @@ -271,7 +272,8 @@ private void processFetchedEvents(Response resp, ECSyncHelper ecSyncUpdater, fin ecSyncUpdater.updateLastSyncTimeStamp(lastServerVersion); } sendSyncProgressBroadcast(eCount); - fetchRetry(0, true); + SyncServiceJob.scheduleJobImmediately(SyncServiceJob.TAG); + } } From 8e99d5c461e6c8c511a33d3e6269b6b9f81481b8 Mon Sep 17 00:00:00 2001 From: hilary egesa Date: Thu, 17 Oct 2024 18:08:58 +0300 Subject: [PATCH 7/8] wrap client details db insertion in transaction --- gradle.properties | 2 +- .../repository/DetailsRepository.java | 8 ++++ .../sync/ClientProcessorForJava.java | 37 ++++++++++--------- .../sync/helper/LocationServiceHelper.java | 15 ++++---- .../sync/intent/SyncIntentService.java | 3 +- 5 files changed, 37 insertions(+), 28 deletions(-) diff --git a/gradle.properties b/gradle.properties index 570dde5ba..ae23624b9 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ -VERSION_NAME=6.1.5-ALPHA42-SNAPSHOT +VERSION_NAME=6.1.5-ALPHA48-SNAPSHOT VERSION_CODE=1 GROUP=org.smartregister POM_SETTING_DESCRIPTION=OpenSRP Client Core Application diff --git a/opensrp-core/src/main/java/org/smartregister/repository/DetailsRepository.java b/opensrp-core/src/main/java/org/smartregister/repository/DetailsRepository.java index 79bc3471e..5b71bbd25 100644 --- a/opensrp-core/src/main/java/org/smartregister/repository/DetailsRepository.java +++ b/opensrp-core/src/main/java/org/smartregister/repository/DetailsRepository.java @@ -70,6 +70,8 @@ public void batchInsertDetails(Map values, long timestamp) { try { database = masterRepository().getWritableDatabase(); + // Start transaction + database.beginTransaction(); // Prepare the SQL for inserts and updates String insertSQL = "INSERT INTO " + TABLE_NAME + " (" + @@ -84,6 +86,7 @@ public void batchInsertDetails(Map values, long timestamp) { String baseEntityId = values.get(BASE_ENTITY_ID_COLUMN); + for (String key : values.keySet()) { String val = values.get(key); if(val == null ) continue; @@ -115,9 +118,14 @@ public void batchInsertDetails(Map values, long timestamp) { insertStatement.clearBindings(); } } + database.setTransactionSuccessful(); } catch (Exception e) { Timber.e(e); } finally { + // End the transaction + if (database != null) { + database.endTransaction(); + } // Close the prepared statements if (insertStatement != null) { insertStatement.close(); diff --git a/opensrp-core/src/main/java/org/smartregister/sync/ClientProcessorForJava.java b/opensrp-core/src/main/java/org/smartregister/sync/ClientProcessorForJava.java index 2ebbcf242..fee4bdfb6 100644 --- a/opensrp-core/src/main/java/org/smartregister/sync/ClientProcessorForJava.java +++ b/opensrp-core/src/main/java/org/smartregister/sync/ClientProcessorForJava.java @@ -15,6 +15,7 @@ import org.json.JSONArray; import org.smartregister.AllConstants; import org.smartregister.CoreLibrary; +import org.smartregister.SyncConfiguration; import org.smartregister.commonregistry.AllCommonsRepository; import org.smartregister.commonregistry.CommonRepository; import org.smartregister.converters.ClientConverter; @@ -66,9 +67,13 @@ public class ClientProcessorForJava { private AppExecutors appExecutors; + private Boolean updatesDetailsTable; + + public ClientProcessorForJava(Context context) { mContext = context; appExecutors = new AppExecutors(); + updatesDetailsTable = CoreLibrary.getInstance().getSyncConfiguration().updateClientDetailsTable(); } public static ClientProcessorForJava getInstance(Context context) { @@ -644,7 +649,7 @@ protected String getFormattedValue(Column column, String columnValue) { * @param eventDate */ protected void addContentValuesToDetailsTable(ContentValues values, Long eventDate) { - if (!CoreLibrary.getInstance().getSyncConfiguration().updateClientDetailsTable()) + if (!updatesDetailsTable) return; try { @@ -674,26 +679,24 @@ public void updateClientDetailsTable(Event event, Client client) { Timber.d("Started updateClientDetailsTable"); long updateClientDetailsStart = System.currentTimeMillis(); - if (CoreLibrary.getInstance().getSyncConfiguration().updateClientDetailsTable()) { + if (updatesDetailsTable) { String baseEntityId = client.getBaseEntityId(); Long timestamp = getEventDate(event.getEventDate()); - Map genderInfo = getGender(client); - - Map addressInfo = getClientAddressAsMap(client); - - Map attributes = getClientAttributes(client); - - Map obs = getObsFromEvent(event); - Map clientDetails = new HashMap<>(); - clientDetails.putAll(genderInfo); - clientDetails.putAll(addressInfo); - clientDetails.putAll(attributes); - clientDetails.putAll(obs); + // Retrieve necessary client information in a single step + Map clientDetails = new HashMap<>(getGender(client)); + clientDetails.putAll(getClientAddressAsMap(client)); + clientDetails.putAll(getClientAttributes(client)); + clientDetails.putAll(getObsFromEvent(event)); clientDetails.put("base_entity_id", baseEntityId); + + // Save client details in a batch process batchSaveClientDetails(clientDetails, timestamp); + + // Clear the map to free memory + clientDetails.clear(); } - Timber.i("Updating client details took, %s ", System.currentTimeMillis()-updateClientDetailsStart); + Timber.i("Updating client details took, %d ", System.currentTimeMillis()-updateClientDetailsStart); event.addDetails(detailsUpdated, Boolean.TRUE.toString()); Timber.d("Finished updateClientDetailsTable"); @@ -850,9 +853,7 @@ public Map getClientAddressAsMap(Client client) { Address address = addressList.get(0); Map addressFieldMap = address.getAddressFields(); if (addressFieldMap != null) { - for (Map.Entry entry : addressFieldMap.entrySet()) { - addressMap.put(entry.getKey(), entry.getValue()); - } + addressMap.putAll(addressFieldMap); } List fields = getFields(address.getClass()); diff --git a/opensrp-core/src/main/java/org/smartregister/sync/helper/LocationServiceHelper.java b/opensrp-core/src/main/java/org/smartregister/sync/helper/LocationServiceHelper.java index cf15111c2..bdfca92be 100644 --- a/opensrp-core/src/main/java/org/smartregister/sync/helper/LocationServiceHelper.java +++ b/opensrp-core/src/main/java/org/smartregister/sync/helper/LocationServiceHelper.java @@ -472,13 +472,14 @@ public void fetchAllLocations(int recordCount) { location.setSyncStatus(BaseRepository.TYPE_Synced); locationRepository.addOrUpdate(location); - - for (LocationTag tag : location.getLocationTags()) { - LocationTag locationTag = new LocationTag(); - locationTag.setLocationId(location.getId()); - locationTag.setName(tag.getName()); - - locationTagRepository.addOrUpdate(locationTag); + if(location.getLocationTags() != null){ + for (LocationTag tag : location.getLocationTags()) { + LocationTag locationTag = new LocationTag(); + locationTag.setLocationId(location.getId()); + locationTag.setName(tag.getName()); + + locationTagRepository.addOrUpdate(locationTag); + } } } catch (Exception e) { Timber.e(e, "EXCEPTION %s", e.toString()); diff --git a/opensrp-core/src/main/java/org/smartregister/sync/intent/SyncIntentService.java b/opensrp-core/src/main/java/org/smartregister/sync/intent/SyncIntentService.java index 404ea4e0c..6bacaf7c4 100644 --- a/opensrp-core/src/main/java/org/smartregister/sync/intent/SyncIntentService.java +++ b/opensrp-core/src/main/java/org/smartregister/sync/intent/SyncIntentService.java @@ -272,8 +272,7 @@ private void processFetchedEvents(Response resp, ECSyncHelper ecSyncUpdater, fin ecSyncUpdater.updateLastSyncTimeStamp(lastServerVersion); } sendSyncProgressBroadcast(eCount); - SyncServiceJob.scheduleJobImmediately(SyncServiceJob.TAG); - + fetchRetry(0, true); } } From 4cb69955a6b1d219b71cc6dd7e6caa782787a9fd Mon Sep 17 00:00:00 2001 From: hilary egesa Date: Tue, 22 Oct 2024 11:01:02 +0300 Subject: [PATCH 8/8] update version --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index ae23624b9..e27e4ee3a 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ -VERSION_NAME=6.1.5-ALPHA48-SNAPSHOT +VERSION_NAME=6.1.5-SNAPSHOT VERSION_CODE=1 GROUP=org.smartregister POM_SETTING_DESCRIPTION=OpenSRP Client Core Application