diff --git a/importer/README.md b/importer/README.md index 03f816d5..0fffc8f2 100644 --- a/importer/README.md +++ b/importer/README.md @@ -175,7 +175,18 @@ The coverage report `coverage.html` will be at the working directory - A separate List resource with references to all the Group and List resources generated is also created - You can pass in a `list_resource_id` to be used as the identifier for the (reference) List resource, or you can leave it empty and a random uuid will be generated -### 12. Import JSON resources from file +### 12. Import flags from openSRP 1 +- Run `python3 main.py --csv_file csv/import/flags.csv --setup flags --encounter_id 123 --practitioner_id 456 --visit_location_id 789 --log_level info` +- See example csv [here](/importer/csv/import/flags.csv) +- `encounter_id` (Required) is the id of the visit encounter that will be linked to all the resources that are imported when the command is run +- `practitioner_id` (Required) is the id of the practitioner that is linked to all the resources created during the import +- `visit_location_id` (Required) is the location linked to the visit encounter, ideally should be the root location +- This function creates a Visit Encounter using the provided encounter id (if one does not already exist) +- It then creates a Flag resource, an Observation resource and an encounter resource for every row on the csv, depending on its type +- It checks to confirm that the location provided in the csv exists, if it does not, it skips that row +- If the entityType is product, it also checks to confirm that the Group id exists in the server and skips the row if it does not + +### 13. Import JSON resources from file - Run `python3 main.py --bulk_import True --json_file tests/json/sample.json --chunk_size 500000 --sync sort --resources_count 100 --log_level info` - This takes in a file with a JSON array, reads the resources from the array in the file and posts them to the FHIR server - `bulk_import` (Required) must be set to True diff --git a/importer/csv/import/flags.csv b/importer/csv/import/flags.csv new file mode 100644 index 00000000..42d7bb67 --- /dev/null +++ b/importer/csv/import/flags.csv @@ -0,0 +1,5 @@ +event_id,event_date,id,entitytype,locationid,locationname,productid,productname,product_flag_problem,product_not_there,product_not_good,product_not_good_specify_other,product_not_there_specify_other,product_misuse,product_issue_details,product_service_point_good_order,product_service_point_not_good_order_reason,product_consult_beneficiaries_flag,product_consult_beneficiaries_issues_raised,product_required_action,product_required_action_yes +100,2022-11-06 21:00:00,5f513cfd-90fa-40a9-8d84-63eed687063d,service_point,661604af-d9f9-4be2-a500-4d127c4bfbed,Nakuru,,,,,,,,,,"[""no""]","[""Lack of infrastructure""]",,,, +101,2023-05-17 21:00:00,b0038380-7800-4771-b8e3-b97c413f8177,service_point,66cb98a3-6f0b-439d-95c1-7f562996f070,Isiolo,,,,,,,,,,,,"[""yes""]","[""Infrequent power""]",, +102,2024-10-11 21:00:00,f9a1bd74-23e8-433b-84b3-bbec68f70036,service_point,f0b8cfc6-eea7-4a54-8eec-057a14a4cc95,Kisumu,,,,,,,,,,,,,,"[""yes""]","[""Cleaning""]" +103,2024-10-13 21:00:00,128053da-639f-40ab-becc-374713fe60e2,product,1523b35a-1a23-4fc9-af8f-6fb5905236de,Malindi,46d7c50b-bfee-4715-a61b-141eb80a4a9d,Wheelbarrow,"[""Product is not there""]","[""never_received""]",,,,,,,,,,, \ No newline at end of file diff --git a/importer/importer/builder.py b/importer/importer/builder.py index e4d7eaf8..dbdb811f 100644 --- a/importer/importer/builder.py +++ b/importer/importer/builder.py @@ -679,7 +679,7 @@ def get_valid_resource_type(resource_type): # This function gets the current resource version from the API def get_resource(resource_id, resource_type): - if resource_type not in ["List", "Group"]: + if resource_type not in ["List", "Group", "Encounter", "Location"]: resource_type = get_valid_resource_type(resource_type) resource_url = "/".join([fhir_base_url, resource_type, resource_id]) response = handle_request("GET", "", resource_url) @@ -1150,3 +1150,192 @@ def build_report(csv_file, response, error_details, fail_count, fail_all): logging.info(string_report) logging.info("============================================================") logging.info("============================================================") + + +def build_single_resource( + resource_type, resource_id, practitioner_id, period_start, location_id, form_encounter, + visit_encounter, subject, value_string="", note="", +): + template_map = { + "visit": "visit_encounter_payload.json", + "flag": "flag_payload.json", + "encounter": "flag_encounter_payload.json", + "observation": "flag_observation_payload.json", + } + json_template = next( + (template for key, template in template_map.items() if key in resource_type), + None, + ) + + boolean_code = boolean_value = "" + if "product" in resource_type: + code = "PRODCHECK" + display = text = "Product Check" + c_code = "issue_details" + c_display = c_text = value_string + elif "service_point_check" in resource_type: + code = "SPCHECK" + display = text = "Service Point Check" + c_code = "34657579" + c_display = c_text = "Service Point Good Order Check" + boolean_code = "373067005" + boolean_value = "No (qualifier value)" + elif "consult_beneficiaries" in resource_type: + code = "CNBEN" + display = text = "Consult Beneficiaries Visit" + c_code = "77223346" + c_display = c_text = "Consult Beneficiaries" + boolean_code = "373066001" + boolean_value = "Yes (qualifier value)" + elif "warehouse_check" in resource_type: + code = "WHCHECK" + display = text = "Warehouse check Visit" + c_code = "68561322" + c_display = c_text = "Required action" + boolean_code = "373066001" + boolean_value = "Yes (qualifier value)" + else: + code = display = text = c_code = c_display = c_text = "" + + with open(json_path + json_template) as json_file: + resource_payload = json_file.read() + + visit_encounter_vars = { + "$id": resource_id, + "$version": "1", + "$category_code": code, + "$category_display": display, + "$category_text": text, + "$code_code": c_code, + "$code_display": c_display, + "$code_text": c_text, + "$practitioner_id": practitioner_id, + "$start": period_start.replace(" ", "T"), + "$end": period_start.replace(" ", "T"), + "$subject": subject, + "$location": location_id, + "$form_encounter": form_encounter, + "$visit_encounter": visit_encounter, + "$value_string": value_string, + "$boolean_code": boolean_code, + "$boolean_value": boolean_value, + "$note": note, + } + for var, value in visit_encounter_vars.items(): + resource_payload = resource_payload.replace(var, value) + + obj = json.loads(resource_payload) + if "product_observation" in resource_type: + del obj["resource"]["valueCodeableConcept"] + del obj["resource"]["note"] + if ( + "service_point_check_encounter" in resource_type + or "consult_beneficiaries_encounter" in resource_type + ): + del obj["resource"]["subject"] + if ( + "service_point_check_observation" in resource_type + or "consult_beneficiaries_observation" in resource_type + ): + del obj["resource"]["focus"] + del obj["resource"]["valueString"] + resource_payload = json.dumps(obj, indent=4) + + return resource_payload + + +def build_resources( + resource_type, encounter_id, flag_id, observation_id, practitioner_id, period, location, + visit_encounter, subject, value_string="", note="", + ): + encounter = build_single_resource( + resource_type + "_encounter", encounter_id, practitioner_id, period, location, "", + visit_encounter, subject, + ) + flag = build_single_resource( + resource_type + "_flag", flag_id, practitioner_id, period, location, encounter_id, + "", subject, + ) + observation = build_single_resource( + resource_type + "_observation", observation_id, practitioner_id, period, location, encounter_id, + "", subject, value_string, note, + ) + + resources = encounter + "," + flag + "," + observation + "," + return resources + + +def check_location(location_id, locations_list): + if location_id in locations_list: + return locations_list[location_id] + + check = get_resource(location_id, "Location") + if check != "0": + locations_list[location_id] = True + return True + else: + locations_list[location_id] = False + logging.info("-- Skipping location, it does NOT EXIST " + location_id) + return False + + +def build_flag_payload(resources, practitioner_id, visit_encounter): + initial_string = """{"resourceType": "Bundle","type": "transaction","entry": [ """ + final_string = "" + locations = {} + for resource in resources: + flag_id = str(uuid.uuid5(uuid.NAMESPACE_DNS, "flag" + resource[2])) + observation_id = str(uuid.uuid5(uuid.NAMESPACE_DNS, "observation" + resource[2])) + + sub_list = [] + valid_location = check_location(resource[4], locations) + if valid_location: + if resource[3] == "service_point": + if "no" in resource[15]: + note = ( + resource[16].replace('"', "").replace("[", "").replace("]", "") + ) + sub_list = build_resources( + "service_point_check", resource[2], flag_id, observation_id, practitioner_id, + resource[1], resource[4], visit_encounter, "Location/" + resource[4], "", note, + ) + if "yes" in resource[17]: + note = ( + resource[18].replace('"', "").replace("[", "").replace("]", "") + ) + sub_list = build_resources( + "consult_beneficiaries", resource[2], flag_id, observation_id, practitioner_id, + resource[1], resource[4], visit_encounter, "Location/" + resource[4], "", note, + ) + if "yes" in resource[19]: + note = ( + resource[20].replace('"', "").replace("[", "").replace("]", "") + ) + sub_list = build_resources( + "warehouse", resource[2], flag_id, observation_id, practitioner_id, resource[1], + resource[4], visit_encounter, "Location/" + resource[4], "", note, + ) + + elif resource[3] == "product": + if resource[8]: + product_info = [ + resource[8], resource[9], resource[10], resource[11], resource[12], resource[13], resource[14], + ] + value_string = " | ".join(filter(None, product_info)) + value_string = value_string.replace('"', "") + if resource[6]: + sub_list = build_resources( + "product", resource[2], flag_id, observation_id, practitioner_id, + resource[1], resource[4], visit_encounter, "Group/" + resource[6], value_string, + ) + else: + logging.info("-- Missing Group, skipping resource: " + str(resource)) + else: + logging.info("-- This entityType is not supported") + + if len(sub_list) < 1: + sub_list = "" + + final_string = final_string + sub_list + final_string = initial_string + final_string[:-1] + " ] } " + return final_string diff --git a/importer/importer/services/fhir_keycloak_api.py b/importer/importer/services/fhir_keycloak_api.py index e64a8e0a..49844daa 100644 --- a/importer/importer/services/fhir_keycloak_api.py +++ b/importer/importer/services/fhir_keycloak_api.py @@ -99,7 +99,7 @@ def request(self, **kwargs): # TODO - spread headers into kwargs. headers = {"content-type": "application/json", "accept": "application/json"} response = self.api_service.oauth.request(headers=headers, **kwargs) - if response.status_code == 401 or '' in response.text: + if response.status_code == 401 or 'login' in response.text: self.api_service.refresh_token() return self.api_service.oauth.request(headers=headers, **kwargs) return response diff --git a/importer/json_payloads/flag_encounter_payload.json b/importer/json_payloads/flag_encounter_payload.json new file mode 100644 index 00000000..ee13b77d --- /dev/null +++ b/importer/json_payloads/flag_encounter_payload.json @@ -0,0 +1,82 @@ +{ + "request": { + "method": "PUT", + "url": "Encounter/$id", + "ifMatch": "$version" + }, + "resource": { + "resourceType": "Encounter", + "id": "$id", + "identifier": [ + { + "use": "usual", + "value": "$id" + } + ], + "status": "finished", + "class": { + "system": "http://terminology.hl7.org/CodeSystem/v3-ActCode", + "code": "OBSENC", + "display": "Observation Encounter" + }, + "type": [ + { + "coding": [ + { + "system": "http://smartregister.org/", + "code": "$category_code", + "display": "$category_display" + } + ], + "text": "$category_text" + } + ], + "priority": { + "coding": [ + { + "system": "http://terminology.hl7.org/ValueSet/v3-ActPriority", + "code": "EL", + "display": "elective" + } + ], + "text": "elective" + }, + "subject": { + "reference": "$subject" + }, + "participant": [ + { + "individual": { + "reference": "Practitioner/$practitioner_id" + } + } + ], + "period": { + "start": "$start", + "end": "$end" + }, + "reasonCode": [ + { + "coding": [ + { + "system": "http://smartregister.org/", + "code": "$category_code", + "display": "$category_display" + } + ], + "text": "$category_text" + } + ], + "location": [ + { + "location": { + "reference": "Location/$location" + }, + "status": "active" + } + ], + "partOf": { + "reference": "Encounter/$visit_encounter" + } + } +} diff --git a/importer/json_payloads/flag_observation_payload.json b/importer/json_payloads/flag_observation_payload.json new file mode 100644 index 00000000..6f8452fa --- /dev/null +++ b/importer/json_payloads/flag_observation_payload.json @@ -0,0 +1,77 @@ +{ + "request": { + "method": "PUT", + "url": "Observation/$id", + "ifMatch": "$version" + }, + "resource": { + "resourceType": "Observation", + "id": "$id", + "identifier": [ + { + "use": "usual", + "value": "$id" + } + ], + "status": "final", + "category": [ + { + "coding": [ + { + "system": "http://smartregister.org/", + "code": "$category_code", + "display": "$category_display" + } + ], + "text": "$category_text" + } + ], + "code": { + "coding": [ + { + "system": "http://snomed.info/sct", + "code": "$code_code", + "display": "$code_display" + } + ], + "text": "$code_text" + }, + "subject": { + "reference": "$subject" + }, + "focus": [ + { + "reference": "Location/$location" + } + ], + "encounter": { + "reference": "Encounter/$form_encounter" + }, + "effectivePeriod": { + "start": "$start", + "end": "$end" + }, + "performer": [ + { + "reference": "Practitioner/$practitioner_id" + } + ], + "valueCodeableConcept": { + "coding": [ + { + "system": "http://snomed.info/sct", + "code": "$boolean_code", + "display": "$boolean_value" + } + ], + "text": "$boolean_value" + }, + "note": [ + { + "time": "$start", + "text": "$note" + } + ], + "valueString": "$value_string" + } +} diff --git a/importer/json_payloads/flag_payload.json b/importer/json_payloads/flag_payload.json new file mode 100644 index 00000000..01a0c8ba --- /dev/null +++ b/importer/json_payloads/flag_payload.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "PUT", + "url": "Flag/$id", + "ifMatch": "$version" + }, + "resource": { + "resourceType": "Flag", + "id": "$id", + "identifier": [ + { + "use": "usual", + "value": "$id" + } + ], + "status": "active", + "category": [ + { + "coding": [ + { + "system": "http://smartregister.org/", + "code": "$category_code", + "display": "$category_display" + } + ], + "text": "$category_text" + } + ], + "code": { + "coding": [ + { + "system": "http://smartregister.org/", + "code": "65347579", + "display": "Vist Flag" + } + ], + "text": "Vist Flag" + }, + "subject": { + "reference": "$subject" + }, + "period": { + "start": "$start", + "end": "$end" + }, + "encounter": { + "reference": "Encounter/$form_encounter" + } + } +} diff --git a/importer/json_payloads/visit_encounter_payload.json b/importer/json_payloads/visit_encounter_payload.json new file mode 100644 index 00000000..653dc672 --- /dev/null +++ b/importer/json_payloads/visit_encounter_payload.json @@ -0,0 +1,90 @@ +{ + "request": { + "method": "PUT", + "url": "Encounter/$id", + "ifMatch": "$version" + }, + "resource": { + "resourceType": "Encounter", + "id": "$id", + "status": "in-progress", + "class": { + "system": "http://terminology.hl7.org/CodeSystem/v3-ActCode", + "code": "OBSENC", + "display": "Observation Encounter" + }, + "type": [ + { + "coding": [ + { + "system": "http://smartregister.org/", + "code": "SVISIT", + "display": "Service Point Visit" + } + ], + "text": "Service Point Visit" + }, + { + "coding": [ + { + "system": "http://smartregister.org/CodeSystem/visit", + "code": "SVIST_IMPORT", + "display": "Service Point Visit Import" + } + ], + "text": "Service Point Visit Import" + } + ], + "priority": { + "coding": [ + { + "system": "http://terminology.hl7.org/ValueSet/v3-ActPriority", + "code": "EL", + "display": "elective" + } + ], + "text": "elective" + }, + "participant": [ + { + "individual": { + "reference": "Practitioner/$practitioner_id" + } + } + ], + "period": { + "start": "$start", + "end": "$end" + }, + "reasonCode": [ + { + "coding": [ + { + "system": "http://smartregister.org/", + "code": "SVISIT", + "display": "Service Point Visit" + } + ], + "text": "Service Point Visit" + }, + { + "coding": [ + { + "system": "http://smartregister.org/CodeSystem/visit", + "code": "SVIST_IMPORT", + "display": "Service Point Visit Import" + } + ], + "text": "Service Point Visit Import" + } + ], + "location": [ + { + "location": { + "reference": "Location/$location" + }, + "status": "active" + } + ] + } +} diff --git a/importer/main.py b/importer/main.py index 2f41859a..1d551664 100644 --- a/importer/main.py +++ b/importer/main.py @@ -5,9 +5,11 @@ import click -from importer.builder import (build_assign_payload, build_group_list_resource, - build_org_affiliation, build_payload, - build_report, extract_matches, extract_resources, +from importer.builder import (build_assign_payload, build_flag_payload, + build_group_list_resource, build_org_affiliation, + build_payload, build_report, + build_single_resource, extract_matches, + extract_resources, get_resource, link_to_location) from importer.config.settings import fhir_base_url from importer.request import handle_request @@ -40,6 +42,9 @@ @click.option("--chunk_size", required=False, default=1000000) @click.option("--resources_count", required=False, default=100) @click.option("--list_resource_id", required=False) +@click.option("--encounter_id", required=False) +@click.option("--practitioner_id", required=False) +@click.option("--visit_location_id", required=False) @click.option( "--log_level", type=click.Choice(["DEBUG", "INFO", "ERROR"], case_sensitive=False) ) @@ -74,6 +79,9 @@ def main( chunk_size, resources_count, list_resource_id, + encounter_id, + practitioner_id, + visit_location_id, sync, location_type_coding_system, ): @@ -287,6 +295,39 @@ def main( final_response = handle_request("POST", "", fhir_base_url, list_payload) logging.info(final_response.text) logging.info("Processing complete!") + elif setup == "flags": + logging.info("Importing flags from OpenSRP1") + visit_encounter_exists = get_resource(encounter_id, "Encounter") + creation_success = True + if visit_encounter_exists == "0": + logging.info("Encounter does not exist, proceed to create") + period_start = datetime.now().strftime("%Y-%m-%dT%H:%M:%S") + encounter_payload = build_single_resource( + "service_point_visit_encounter", encounter_id, practitioner_id, period_start, + visit_location_id, "", "", "", "", "", + ) + initial_string = ( + """{"resourceType": "Bundle","type": "transaction","entry": [ """ + ) + encounter_payload = initial_string + encounter_payload + " ] } " + encounter_response = handle_request( + "POST", encounter_payload, fhir_base_url + ) + if encounter_response.status_code > 201: + logging.error("Error creating the visit encounter...") + logging.error(encounter_response.text) + issues.append( + str(encounter_response.status_code) + ": " + encounter_response.text + ) + creation_success = False + + if creation_success: + json_payload = build_flag_payload( + resource_list, practitioner_id, encounter_id + ) + final_response = handle_request("POST", json_payload, fhir_base_url) + logging.info(final_response.text) + logging.info("Processing complete!") else: message = "Unsupported request!" fail_all = True