Skip to content

Commit

Permalink
Initial commit for flag import functionality
Browse files Browse the repository at this point in the history
  • Loading branch information
Wambere committed Feb 4, 2025
1 parent d3cd13e commit 40aea24
Show file tree
Hide file tree
Showing 9 changed files with 551 additions and 6 deletions.
13 changes: 12 additions & 1 deletion importer/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -175,7 +175,18 @@ The coverage report `coverage.html` will be at the working directory
- A separate List resource with references to all the Group and List resources generated is also created
- You can pass in a `list_resource_id` to be used as the identifier for the (reference) List resource, or you can leave it empty and a random uuid will be generated

### 12. Import JSON resources from file
### 12. Import flags from openSRP 1
- Run `python3 main.py --csv_file csv/import/flags.csv --setup flags --encounter_id 123 --practitioner_id 456 --visit_location_id 789 --log_level info`
- See example csv [here](/importer/csv/import/flags.csv)
- `encounter_id` (Required) is the id of the visit encounter that will be linked to all the resources that are imported when the command is run
- `practitioner_id` (Required) is the id of the practitioner that is linked to all the resources created during the import
- `visit_location_id` (Required) is the location linked to the visit encounter, ideally should be the root location
- This function creates a Visit Encounter using the provided encounter id (if one does not already exist)
- It then creates a Flag resource, an Observation resource and an encounter resource for every row on the csv, depending on its type
- It checks to confirm that the location provided in the csv exists, if it does not, it skips that row
- If the entityType is product, it also checks to confirm that the Group id exists in the server and skips the row if it does not

### 13. Import JSON resources from file
- Run `python3 main.py --bulk_import True --json_file tests/json/sample.json --chunk_size 500000 --sync sort --resources_count 100 --log_level info`
- This takes in a file with a JSON array, reads the resources from the array in the file and posts them to the FHIR server
- `bulk_import` (Required) must be set to True
Expand Down
5 changes: 5 additions & 0 deletions importer/csv/import/flags.csv
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
event_id,event_date,id,entitytype,locationid,locationname,productid,productname,product_flag_problem,product_not_there,product_not_good,product_not_good_specify_other,product_not_there_specify_other,product_misuse,product_issue_details,product_service_point_good_order,product_service_point_not_good_order_reason,product_consult_beneficiaries_flag,product_consult_beneficiaries_issues_raised,product_required_action,product_required_action_yes
100,2022-11-06 21:00:00,5f513cfd-90fa-40a9-8d84-63eed687063d,service_point,661604af-d9f9-4be2-a500-4d127c4bfbed,Nakuru,,,,,,,,,,"[""no""]","[""Lack of infrastructure""]",,,,
101,2023-05-17 21:00:00,b0038380-7800-4771-b8e3-b97c413f8177,service_point,66cb98a3-6f0b-439d-95c1-7f562996f070,Isiolo,,,,,,,,,,,,"[""yes""]","[""Infrequent power""]",,
102,2024-10-11 21:00:00,f9a1bd74-23e8-433b-84b3-bbec68f70036,service_point,f0b8cfc6-eea7-4a54-8eec-057a14a4cc95,Kisumu,,,,,,,,,,,,,,"[""yes""]","[""Cleaning""]"
103,2024-10-13 21:00:00,128053da-639f-40ab-becc-374713fe60e2,product,1523b35a-1a23-4fc9-af8f-6fb5905236de,Malindi,46d7c50b-bfee-4715-a61b-141eb80a4a9d,Wheelbarrow,"[""Product is not there""]","[""never_received""]",,,,,,,,,,,
191 changes: 190 additions & 1 deletion importer/importer/builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -679,7 +679,7 @@ def get_valid_resource_type(resource_type):

# This function gets the current resource version from the API
def get_resource(resource_id, resource_type):
if resource_type not in ["List", "Group"]:
if resource_type not in ["List", "Group", "Encounter", "Location"]:
resource_type = get_valid_resource_type(resource_type)
resource_url = "/".join([fhir_base_url, resource_type, resource_id])
response = handle_request("GET", "", resource_url)
Expand Down Expand Up @@ -1150,3 +1150,192 @@ def build_report(csv_file, response, error_details, fail_count, fail_all):
logging.info(string_report)
logging.info("============================================================")
logging.info("============================================================")


def build_single_resource(
resource_type, resource_id, practitioner_id, period_start, location_id, form_encounter,
visit_encounter, subject, value_string="", note="",
):
template_map = {
"visit": "visit_encounter_payload.json",
"flag": "flag_payload.json",
"encounter": "flag_encounter_payload.json",
"observation": "flag_observation_payload.json",
}
json_template = next(
(template for key, template in template_map.items() if key in resource_type),
None,
)

boolean_code = boolean_value = ""
if "product" in resource_type:
code = "PRODCHECK"
display = text = "Product Check"
c_code = "issue_details"
c_display = c_text = value_string
elif "service_point_check" in resource_type:
code = "SPCHECK"
display = text = "Service Point Check"
c_code = "34657579"
c_display = c_text = "Service Point Good Order Check"
boolean_code = "373067005"
boolean_value = "No (qualifier value)"
elif "consult_beneficiaries" in resource_type:
code = "CNBEN"
display = text = "Consult Beneficiaries Visit"
c_code = "77223346"
c_display = c_text = "Consult Beneficiaries"
boolean_code = "373066001"
boolean_value = "Yes (qualifier value)"
elif "warehouse_check" in resource_type:
code = "WHCHECK"
display = text = "Warehouse check Visit"
c_code = "68561322"
c_display = c_text = "Required action"
boolean_code = "373066001"
boolean_value = "Yes (qualifier value)"
else:
code = display = text = c_code = c_display = c_text = ""

with open(json_path + json_template) as json_file:
resource_payload = json_file.read()

visit_encounter_vars = {
"$id": resource_id,
"$version": "1",
"$category_code": code,
"$category_display": display,
"$category_text": text,
"$code_code": c_code,
"$code_display": c_display,
"$code_text": c_text,
"$practitioner_id": practitioner_id,
"$start": period_start.replace(" ", "T"),
"$end": period_start.replace(" ", "T"),
"$subject": subject,
"$location": location_id,
"$form_encounter": form_encounter,
"$visit_encounter": visit_encounter,
"$value_string": value_string,
"$boolean_code": boolean_code,
"$boolean_value": boolean_value,
"$note": note,
}
for var, value in visit_encounter_vars.items():
resource_payload = resource_payload.replace(var, value)

obj = json.loads(resource_payload)
if "product_observation" in resource_type:
del obj["resource"]["valueCodeableConcept"]
del obj["resource"]["note"]
if (
"service_point_check_encounter" in resource_type
or "consult_beneficiaries_encounter" in resource_type
):
del obj["resource"]["subject"]
if (
"service_point_check_observation" in resource_type
or "consult_beneficiaries_observation" in resource_type
):
del obj["resource"]["focus"]
del obj["resource"]["valueString"]
resource_payload = json.dumps(obj, indent=4)

return resource_payload


def build_resources(
resource_type, encounter_id, flag_id, observation_id, practitioner_id, period, location,
visit_encounter, subject, value_string="", note="",
):
encounter = build_single_resource(
resource_type + "_encounter", encounter_id, practitioner_id, period, location, "",
visit_encounter, subject,
)
flag = build_single_resource(
resource_type + "_flag", flag_id, practitioner_id, period, location, encounter_id,
"", subject,
)
observation = build_single_resource(
resource_type + "_observation", observation_id, practitioner_id, period, location, encounter_id,
"", subject, value_string, note,
)

resources = encounter + "," + flag + "," + observation + ","
return resources


def check_location(location_id, locations_list):
if location_id in locations_list:
return locations_list[location_id]

check = get_resource(location_id, "Location")
if check != "0":
locations_list[location_id] = True
return True
else:
locations_list[location_id] = False
logging.info("-- Skipping location, it does NOT EXIST " + location_id)
return False


def build_flag_payload(resources, practitioner_id, visit_encounter):
initial_string = """{"resourceType": "Bundle","type": "transaction","entry": [ """
final_string = ""
locations = {}
for resource in resources:
flag_id = str(uuid.uuid5(uuid.NAMESPACE_DNS, "flag" + resource[2]))
observation_id = str(uuid.uuid5(uuid.NAMESPACE_DNS, "observation" + resource[2]))

sub_list = []
valid_location = check_location(resource[4], locations)
if valid_location:
if resource[3] == "service_point":
if "no" in resource[15]:
note = (
resource[16].replace('"', "").replace("[", "").replace("]", "")
)
sub_list = build_resources(
"service_point_check", resource[2], flag_id, observation_id, practitioner_id,
resource[1], resource[4], visit_encounter, "Location/" + resource[4], "", note,
)
if "yes" in resource[17]:
note = (
resource[18].replace('"', "").replace("[", "").replace("]", "")
)
sub_list = build_resources(
"consult_beneficiaries", resource[2], flag_id, observation_id, practitioner_id,
resource[1], resource[4], visit_encounter, "Location/" + resource[4], "", note,
)
if "yes" in resource[19]:
note = (
resource[20].replace('"', "").replace("[", "").replace("]", "")
)
sub_list = build_resources(
"warehouse", resource[2], flag_id, observation_id, practitioner_id, resource[1],
resource[4], visit_encounter, "Location/" + resource[4], "", note,
)

elif resource[3] == "product":
if resource[8]:
product_info = [
resource[8], resource[9], resource[10], resource[11], resource[12], resource[13], resource[14],
]
value_string = " | ".join(filter(None, product_info))
value_string = value_string.replace('"', "")
if resource[6]:
sub_list = build_resources(
"product", resource[2], flag_id, observation_id, practitioner_id,
resource[1], resource[4], visit_encounter, "Group/" + resource[6], value_string,
)
else:
logging.info("-- Missing Group, skipping resource: " + str(resource))
else:
logging.info("-- This entityType is not supported")

if len(sub_list) < 1:
sub_list = ""

final_string = final_string + sub_list
final_string = initial_string + final_string[:-1] + " ] } "
return final_string
2 changes: 1 addition & 1 deletion importer/importer/services/fhir_keycloak_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ def request(self, **kwargs):
# TODO - spread headers into kwargs.
headers = {"content-type": "application/json", "accept": "application/json"}
response = self.api_service.oauth.request(headers=headers, **kwargs)
if response.status_code == 401 or '<html class="login-pf">' in response.text:
if response.status_code == 401 or 'login' in response.text:
self.api_service.refresh_token()
return self.api_service.oauth.request(headers=headers, **kwargs)
return response
Expand Down
82 changes: 82 additions & 0 deletions importer/json_payloads/flag_encounter_payload.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
{
"request": {
"method": "PUT",
"url": "Encounter/$id",
"ifMatch": "$version"
},
"resource": {
"resourceType": "Encounter",
"id": "$id",
"identifier": [
{
"use": "usual",
"value": "$id"
}
],
"status": "finished",
"class": {
"system": "http://terminology.hl7.org/CodeSystem/v3-ActCode",
"code": "OBSENC",
"display": "Observation Encounter"
},
"type": [
{
"coding": [
{
"system": "http://smartregister.org/",
"code": "$category_code",
"display": "$category_display"
}
],
"text": "$category_text"
}
],
"priority": {
"coding": [
{
"system": "http://terminology.hl7.org/ValueSet/v3-ActPriority",
"code": "EL",
"display": "elective"
}
],
"text": "elective"
},
"subject": {
"reference": "$subject"
},
"participant": [
{
"individual": {
"reference": "Practitioner/$practitioner_id"
}
}
],
"period": {
"start": "$start",
"end": "$end"
},
"reasonCode": [
{
"coding": [
{
"system": "http://smartregister.org/",
"code": "$category_code",
"display": "$category_display"
}
],
"text": "$category_text"
}
],
"location": [
{
"location": {
"reference": "Location/$location"
},
"status": "active"
}
],
"partOf": {
"reference": "Encounter/$visit_encounter"
}
}
}
77 changes: 77 additions & 0 deletions importer/json_payloads/flag_observation_payload.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
{
"request": {
"method": "PUT",
"url": "Observation/$id",
"ifMatch": "$version"
},
"resource": {
"resourceType": "Observation",
"id": "$id",
"identifier": [
{
"use": "usual",
"value": "$id"
}
],
"status": "final",
"category": [
{
"coding": [
{
"system": "http://smartregister.org/",
"code": "$category_code",
"display": "$category_display"
}
],
"text": "$category_text"
}
],
"code": {
"coding": [
{
"system": "http://snomed.info/sct",
"code": "$code_code",
"display": "$code_display"
}
],
"text": "$code_text"
},
"subject": {
"reference": "$subject"
},
"focus": [
{
"reference": "Location/$location"
}
],
"encounter": {
"reference": "Encounter/$form_encounter"
},
"effectivePeriod": {
"start": "$start",
"end": "$end"
},
"performer": [
{
"reference": "Practitioner/$practitioner_id"
}
],
"valueCodeableConcept": {
"coding": [
{
"system": "http://snomed.info/sct",
"code": "$boolean_code",
"display": "$boolean_value"
}
],
"text": "$boolean_value"
},
"note": [
{
"time": "$start",
"text": "$note"
}
],
"valueString": "$value_string"
}
}
Loading

0 comments on commit 40aea24

Please sign in to comment.