Beds history no equip (#109)

* late nite imports, eat sleep code repeat

* added api

* remove odd eval

* store typeName and no longer store empty observations (no value there)

* fix missing serializer, view from faulty rebase

* set name properly

* set user agent and default timemout for geopy

* add migration

* set a default url for client

* add a merge migration

* dont fail silently

README.md

@@ -23,18 +23,21 @@ docker-compose exec api python manage.py migrate # only when you need to run mig
 #import data using bonobo
 docker-compose exec api python manage.py historicaldata
 docker-compose exec api python manage.py importDetailedHospitals
+docker-compose exec api python manage.py importbedshistory
 docker-compose exec api python manage.py importpopulation
 docker-compose exec api python manage.py importpopulationdetailed
 docker-compose exec api python manage.py importdepression
 docker-compose exec api python manage.py importcancer
 ```
 
 To access the apis :
-http://localhost:8000/api/hospitals
-http://localhost:8000/api/population
-http://localhost:8000/api/populationdetailed
-http://localhost:8000/api/depression
-http://localhost:8000/api/cancer
+* http://localhost:8000/api/hospitals
+* http://localhost:8000/api/population
+* http://localhost:8000/api/populationdetailed
+* http://localhost:8000/api/depression
+* http://localhost:8000/api/cancer
+* http://localhost:8000/api/hospital-networks
+* http://localhost:8000/api/hospital-networks/[id]/beds (optional query params: type, year)
 
 depression_data.csv comes from : https://hisia.wiv-isp.be/SitePages/Home.aspx
 cancer.csv comes from : http://www.kankerregister.org/default.aspx?url=Statistiques_tableaux%20annuelle

backend/api/management/commands/importDetailedHospitals.py

@@ -5,12 +5,15 @@
 import xlrd
 
 from geopy.geocoders import Nominatim
+import geopy
 from bonobo.contrib.django import ETLCommand
-from api.models import Hospital
+from api.models import Hospital, HospitalNetwork
 from django.conf import settings
 from datetime import date
 from api.utils import geocode
 
+geopy.geocoders.options.default_user_agent = 'healthstory-data-import'
+geopy.geocoders.options.default_timeout = 30
 def isInt(value):
     try:
         int(eval(str(value)))
@@ -37,6 +40,14 @@ def parse_hospital_data():
     for row_number in range(2,sh.nrows):
         yield dict(zip(headers, sh.row_values(row_number)))
 
+def get_network(id,name):
+    try:
+        network = HospitalNetwork.objects.get(pk=id)
+    except:
+        network = HospitalNetwork(name=name,id=id)
+        network.save()
+    return network
+
 def transform_hospital_data(row):
     geolocator = Nominatim()
     try:
@@ -52,18 +63,35 @@ def transform_hospital_data(row):
     else:
         beds = int(eval(str(row["TOTAAL BEDDEN"])))
     try:
-        p = Hospital(name=row['ZIEKENHUIS '], latitude=lat, longitude=long, nbBeds=beds,
-            siteNbr=int(eval(str(row["VESTIGINGSNR"]))), address=row["ADRES"], postalCode=int(eval(str(row["POST"]))),
-            town=row["GEMEENTE "], website=row["WEBSITE"], telephone=row["TELEFOON"], province=row["PROVINCIE "], type=row["SOORT ZIEKENHUIS"])
-        yield p
+        if isInt(row['ERK']):
+            network = get_network(int(row['ERK']),row['ZIEKENHUIS '])
+            cleaned_name = row['CAMPUS'].strip()
+            name = cleaned_name if cleaned_name else row['ZIEKENHUIS '].strip()
+            p = Hospital(
+                network=network,
+                latitude=lat,
+                longitude=long,
+                nbBeds=beds,
+                siteNbr=row["VESTIGINGSNR"],
+                address=row["ADRES"],
+                postalCode=int(row["POST"]),
+                town=row["GEMEENTE "],
+                website=row["WEBSITE"],
+                telephone=row["TELEFOON"],
+                province=row["PROVINCIE "],
+                name=name,
+                type=row["SOORT ZIEKENHUIS"]
+            )
+            yield p
     except:
+        print('failed for ')
+        print(row)
+        print(sys.exc_info()[0])
         pass
 
 def load_hospital_data(hospital):
-    try:
-        hospital.save() #todo: find a way to ignore duplicate VESTIGINGSNR
-    except:
-        pass
+    hospital.save() #todo: find a way to ignore duplicate VESTIGINGSNR
+
 # https://fair-acc.healthdata.be/api/3/action/group_package_show?id=469baf11-ddd1-4c30-9ad3-a21a7d0f7397
 class Command(ETLCommand):
     def get_graph(self, **options):

backend/api/management/commands/importbedshistory.py

@@ -0,0 +1,102 @@
+import bonobo
+import os
+import csv
+import xlrd
+import re
+
+from bonobo.contrib.django import ETLCommand
+from api.models import Bed, Hospital, HospitalNetwork
+from django.conf import settings
+
+def isInt(value):
+    try:
+        int(value)
+        return True
+    except:
+        return False
+
+def find_excels():
+    dataDir = os.path.join(settings.BASE_DIR, 'api', 'source-data', 'hospitals')
+    # Ziekenhuisbedden%2001_02_2011
+    for root, dirs, files in os.walk(dataDir):
+        for name in files:
+            match = re.match(r'Ziekenhuisbedden%20\d{2}_(\d{2})_(\d{4}).*.xlsx', name)
+            if match is not None and int(match.group(2)) > 2008:
+                yield (name, match.group(1), match.group(2))
+
+def bed_type_for_name(type):
+    names = [
+'Neuropsychiatric department for observation and treatment',
+'Day nursing in an A department',
+'Night nursing in an A department',
+'Department for surgical diagnosis and treatment',
+'Mixed inpatient department C + D',
+'Department for medical diagnosis and treatment',
+'Paediatric medicine department',
+'Exclusive Medicine for Older People department',
+'Department for intensive treatment of psychiatric patients “Adult SGA (severely disturbed and aggressive)”',
+'Neuropsychiatric department for children',
+'Day nursing in K department',
+'Night nursing in K department',
+'Infectious diseases department',
+'Maternity',
+'Neonatal intensive care department',
+'Specialist department for cardio-pulmonary conditions',
+'Specialist department for locomotor conditions',
+'Specialist department for neurological conditions',
+'Specialist department for palliative care',
+'Specialist department for chronic diseases',
+'Specialist Older Persons Mental Health department',
+'Neuropsychiatric department for treatment',
+'Day nursing in T department',
+'Night nursing in T department',
+'Inpatient placement with family',
+'Placement in family context',
+'Day and night nursing for older patients requiring neuropsychiatric treatment',
+'Total Result',
+'Total Result'
+]
+    types =  ['A','A1','A2','C','CD','D','E','G','I1','K','K1','K2','L','M','NIC','S1','S2','S3','S4','S5','S6','T','T1','T2','TFB','TFP','TG', 'Total Result','Eindtotaal']
+    map = dict(zip(types,names))
+    return map[type]
+
+def transform_excels_to_beds_history(excel, month, year):
+    filename = os.path.join(settings.BASE_DIR, 'api', 'source-data', 'hospitals', excel)
+    wb = xlrd.open_workbook(filename, 'wb')
+    sh = wb.sheet_by_index(0)
+    headers = sh.row_values(3)
+    print('parsing excel %s' % excel)
+    for row_number in range(4,sh.nrows):
+        row = dict(zip(headers,sh.row_values(row_number)))
+        try:
+            network = HospitalNetwork.objects.get(pk=int(row['Erkenningsnummer Ziekenhuis']))
+        except:
+            network = None
+        if network is not None:
+            for type in ['A','A1','A2','C','CD','D','E','G','I1','K','K1','K2','L','M','NIC','S1','S2','S3','S4','S5','S6','T','T1','T2','TFB','TFP','TG', 'Total Result','Eindtotaal']:
+                if row.get(type) is not None:
+                    value = int(row[type]) if isInt(row[type]) else -1
+                    if (value > -1):
+                        yield Bed(
+                            network=network,
+                            year=year,
+                            month=month,
+                            amount=value,
+                            type=type,
+                            typeName=bed_type_for_name(type)
+                        )
+        else:
+            print('network not found for ERK: %s, excel: %s' % (row['Erkenningsnummer Ziekenhuis'], excel))
+
+def save_beds(bed):
+    bed.save()
+
+class Command(ETLCommand):
+    def get_graph(self, **options):
+        graph = bonobo.Graph()
+        graph.add_chain(
+            find_excels,
+            transform_excels_to_beds_history,
+            save_beds
+        )
+        return graph

backend/api/migrations/0005_merge_20180719_1809.py

@@ -0,0 +1,14 @@
+# Generated by Django 2.0.7 on 2018-07-19 18:09
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0004_populationdetailed'),
+        ('api', '0002_auto_20180717_1254'),
+    ]
+
+    operations = [
+    ]

backend/api/migrations/0006_auto_20180719_1809.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.0.7 on 2018-07-19 18:09
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0005_merge_20180719_1809'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='bed',
+            name='type',
+            field=models.CharField(max_length=5, null=True),
+        ),
+    ]

backend/api/migrations/0007_auto_20180719_1829.py

@@ -0,0 +1,24 @@
+# Generated by Django 2.0.7 on 2018-07-19 18:29
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0006_auto_20180719_1809'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='bed',
+            name='hospital_siteNbr',
+            field=models.ForeignKey(db_column='hospital_siteNbr', on_delete=django.db.models.deletion.CASCADE, to='api.Hospital'),
+        ),
+        migrations.AlterField(
+            model_name='hospital',
+            name='siteNbr',
+            field=models.IntegerField(null=True),
+        ),
+    ]

backend/api/migrations/0008_auto_20180719_2012.py

@@ -0,0 +1,32 @@
+# Generated by Django 2.0.7 on 2018-07-19 20:12
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0007_auto_20180719_1829'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='HospitalNetwork',
+            fields=[
+                ('name', models.CharField(max_length=500)),
+                ('id', models.IntegerField(primary_key=True, serialize=False)),
+            ],
+        ),
+        migrations.AlterField(
+            model_name='hospital',
+            name='siteNbr',
+            field=models.IntegerField(null=True, unique=True),
+        ),
+        migrations.AddField(
+            model_name='hospital',
+            name='network',
+            field=models.ForeignKey(db_column='hospital_network_id', default='', on_delete=django.db.models.deletion.CASCADE, to='api.HospitalNetwork'),
+            preserve_default=False,
+        ),
+    ]

backend/api/migrations/0009_auto_20180719_2014.py

@@ -0,0 +1,24 @@
+# Generated by Django 2.0.7 on 2018-07-19 20:14
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0008_auto_20180719_2012'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='bed',
+            name='hospital_siteNbr',
+        ),
+        migrations.AddField(
+            model_name='bed',
+            name='network',
+            field=models.ForeignKey(db_column='network_id', default='', on_delete=django.db.models.deletion.CASCADE, to='api.HospitalNetwork'),
+            preserve_default=False,
+        ),
+    ]

backend/api/migrations/0010_auto_20180719_2044.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.0.7 on 2018-07-19 20:44
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0009_auto_20180719_2014'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='hospital',
+            name='siteNbr',
+            field=models.CharField(max_length=10, null=True, unique=True),
+        ),
+    ]

backend/api/migrations/0011_bed_typename.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.0.7 on 2018-07-19 21:48
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0010_auto_20180719_2044'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='bed',
+            name='typeName',
+            field=models.CharField(max_length=500, null=True),
+        ),
+    ]

backend/api/migrations/0012_auto_20180719_2154.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.0.7 on 2018-07-19 21:54
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0011_bed_typename'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='bed',
+            name='type',
+            field=models.CharField(max_length=20, null=True),
+        ),
+    ]

backend/api/migrations/0013_merge_20180723_0744.py

@@ -0,0 +1,14 @@
+# Generated by Django 2.0.7 on 2018-07-23 07:44
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0008_merge_20180723_0613'),
+        ('api', '0012_auto_20180719_2154'),
+    ]
+
+    operations = [
+    ]