173 support on device training

app/build.gradle

@@ -67,6 +67,12 @@ android {
     configurations.all {
         resolutionStrategy.force 'com.google.code.findbugs:jsr305:3.0.2'
     }
+    sourceSets {
+        androidTest {
+            assets.srcDirs = ['src/main/assets', 'src/androidTest/assets', 'src/debug/assets/']
+            java.srcDirs = ['src/main/java', 'src/androidTest/java', 'src/debug/java']
+        }
+    }
 }
 
 dependencies {
@@ -83,11 +89,11 @@ dependencies {
     implementation 'androidx.swiperefreshlayout:swiperefreshlayout:1.1.0'
     implementation 'androidx.preference:preference-ktx:1.2.0'
     implementation 'androidx.preference:preference-ktx:1.2.0'
-    implementation 'org.tensorflow:tensorflow-lite:2.7.0'
-    implementation 'org.tensorflow:tensorflow-lite-support:0.3.1'
-    implementation 'org.tensorflow:tensorflow-lite-metadata:0.3.1'
-    implementation 'org.tensorflow:tensorflow-lite-gpu:2.7.0'
-    implementation 'org.tensorflow:tensorflow-lite-select-tf-ops:2.6.0'
+    implementation 'org.tensorflow:tensorflow-lite:2.8.0'
+    implementation 'org.tensorflow:tensorflow-lite-support:0.4.0'
+    implementation 'org.tensorflow:tensorflow-lite-metadata:0.4.0'
+    implementation 'org.tensorflow:tensorflow-lite-gpu:2.8.0'
+    implementation 'org.tensorflow:tensorflow-lite-select-tf-ops:2.8.0'
 
     implementation 'androidx.camera:camera-lifecycle:1.0.2'
     implementation 'com.google.mlkit:pose-detection-common:17.0.0'

app/src/androidTest/java/sensors_in_paradise/sonar/InMemoryWindowTest.kt

@@ -0,0 +1,80 @@
+package sensors_in_paradise.sonar
+
+import com.xsens.dot.android.sdk.events.XsensDotData
+import org.junit.Assert
+import org.junit.Test
+import sensors_in_paradise.sonar.machine_learning.InMemoryWindow
+
+class InMemoryWindowTest {
+    @Test
+    fun initializationTest() {
+        val features = arrayOf("Quat_Z_LF", "dq_W_LF", "dv[1]_LF").map { it.uppercase() }.toTypedArray()
+        val window = InMemoryWindow(features, 2)
+        assert(features.contentEquals(window.keys.toTypedArray()))
+    }
+    @Test
+    fun addDataTest() {
+        val features = arrayOf("Quat_Z_LF", "dq_W_RW", "dv[1]_LF")
+        val window = InMemoryWindow(features, 2)
+        for (feature in features) {
+            assert(window.needsFeature(feature))
+        }
+        val data = XsensDotData().apply {
+            quat = floatArrayOf(0f, 0f, 1f, 0f)
+            dq = doubleArrayOf(0.0, 0.0, 0.2, 23.0)
+            dv = doubleArrayOf(23.0, 0.0, 0.0)
+        }
+
+        window.appendSensorData("LF", data)
+        Assert.assertFalse(window.hasEnoughDataToCompileWindow())
+        window.appendSensorData("RW", data)
+        Assert.assertFalse(window.hasEnoughDataToCompileWindow())
+        data.sampleTimeFine += 1L
+
+        window.appendSensorData("LF", data)
+        Assert.assertFalse(window.hasEnoughDataToCompileWindow())
+        data.sampleTimeFine += 1L
+
+        window.appendSensorData("RW", data)
+        Assert.assertTrue(window.hasEnoughDataToCompileWindow())
+
+        window.compileWindow()
+
+    }
+    @Test
+    fun forwardFillDataTest(){
+        val features = arrayOf("Quat_W_LF", "Quat_X_LF", "Quat_Y_LF", "Quat_Z_LF")
+        val window = InMemoryWindow(features, 3)
+
+        val data0 = XsensDotData().apply {
+            sampleTimeFine = 0L
+            quat = floatArrayOf(0f, Float.NaN, 1f, 0f)
+            dq = doubleArrayOf(0.0, 0.0, 0.2, 23.0)
+            dv = doubleArrayOf(23.0, 0.0, Double.NaN)
+            quat = floatArrayOf(0f, Float.NaN, 1f, 1f)
+        }
+        val data1 = XsensDotData().apply {
+            sampleTimeFine = 1L
+            quat = floatArrayOf(0f, 1f, Float.NaN, 2f)
+        }
+        val data2 = XsensDotData().apply {
+            sampleTimeFine = 2L
+            quat = floatArrayOf(0f, Float.NaN, 0f, 0f)
+        }
+
+        window.appendSensorData("LF", data0)
+        Assert.assertFalse(windowHasNan(window))
+        window.appendSensorData("LF", data1)
+        Assert.assertFalse(windowHasNan(window))
+        window.appendSensorData("LF", data2)
+        Assert.assertFalse(windowHasNan(window))
+        Assert.assertTrue(window["QUAT_W_LF"]!! == arrayListOf(Pair(0L, 0f), Pair(1L, 0f), Pair(2L, 0f)))
+        Assert.assertTrue(window["QUAT_X_LF"]!! == arrayListOf(Pair(0L, 0f), Pair(1L, 1f), Pair(2L, 1f)))
+        Assert.assertTrue(window["QUAT_Y_LF"]!! == arrayListOf(Pair(0L, 1f), Pair(1L, 1f), Pair(2L, 0f)))
+        Assert.assertTrue(window["QUAT_Z_LF"]!! == arrayListOf(Pair(0L, 1f), Pair(1L, 2f), Pair(2L, 0f)))
+    }
+
+    private fun windowHasNan(window: InMemoryWindow): Boolean {
+        return window.values.any { arrayList -> arrayList.any { it.second.isNaN() } }
+    }
+}

app/src/androidTest/java/sensors_in_paradise/sonar/RecordingDataFileTest.kt

@@ -0,0 +1,123 @@
+package sensors_in_paradise.sonar
+
+import android.util.Log
+import androidx.test.platform.app.InstrumentationRegistry
+import org.junit.Before
+import org.junit.Test
+import sensors_in_paradise.sonar.machine_learning.DataSet
+import sensors_in_paradise.sonar.machine_learning.TFLiteModel
+import sensors_in_paradise.sonar.screen_recording.RecordingDataFile
+import sensors_in_paradise.sonar.use_cases.UseCase
+import java.io.File
+
+class RecordingDataFileTest {
+    private val assetContext = InstrumentationRegistry.getInstrumentation().context
+    private val appContext = InstrumentationRegistry.getInstrumentation().targetContext
+    private val recordingFile = File(appContext.cacheDir, "recordingData.csv")
+    private val modelFile = File(appContext.cacheDir, "model_test.tflite")
+
+    @Before
+    fun init() {
+        UseCase.extractFileFromAssets(assetContext, "0_orhan_1652085453257.csv", recordingFile)
+        UseCase.extractFileFromAssets(assetContext, "resnet_model_22_06_2022.tflite", modelFile)
+    }
+
+    @Test
+    fun windowizeTest() {
+        val features =
+            arrayOf("Quat_Z_LF", "dq_W_LF", "dv[1]_LF").map { it.uppercase() }.toTypedArray()
+        val windowSize = 90
+        val data = RecordingDataFile(recordingFile)
+        val startIndexes = data.getWindowStartIndexes(windowSize)
+
+        assert(startIndexes.size > 0)
+        var i = 1
+        for (startIndex in startIndexes) {
+            Log.d(
+                "RecordingDataFileTest-windowizeTest",
+                "Working on window $i of ${startIndexes.size}"
+            )
+            val (window, activity) = data.getWindowAtIndex(startIndex, windowSize, features)
+            assert(window.size == features.size)
+
+            // Test if compiling the window into a float buffer runs through
+            window.compileWindow()
+            i++
+        }
+    }
+
+    @Test
+    fun predictionPipelineTest() {
+        val model = TFLiteModel(modelFile)
+
+        val features = model.getFeaturesToPredict().map { it.uppercase() }.toTypedArray()
+
+        val data = RecordingDataFile(recordingFile)
+        val startIndexes = data.getWindowStartIndexes(model.windowSize)
+        var correctPredictions = 0
+        assert(startIndexes.size > 0)
+
+        var i = 1
+        for (startIndex in startIndexes) {
+            val (window, activity) = data.getWindowAtIndex(startIndex, model.windowSize, features)
+            assert(window.size == features.size)
+
+            // Test if compiling the window into a float buffer runs through
+            val input = window.compileWindowToArray()
+            val prediction = model.infer(arrayOf(input))
+            val predictedLabel = model.convertPredictionToLabel(prediction[0])
+            if (predictedLabel == activity) {
+                correctPredictions++
+            }
+            Log.d(
+                "RecordingDataFileTest-predictionPipelineTest",
+                "Working on window $i of ${startIndexes.size}"
+            )
+            i++
+        }
+        val accuracy = (correctPredictions * 100 / startIndexes.size)
+        Log.d("RecordingDataFileTest", "Prediction accuracy on the example recording: $accuracy%")
+    }
+
+    @Test
+    fun trainingPipelineTest() {
+        val model = TFLiteModel(modelFile)
+        val data = RecordingDataFile(recordingFile)
+        val dataSet = DataSet().apply { add(data) }
+
+        val batches = dataSet.convertToBatches(7, model.windowSize, progressCallback = { progress ->
+            Log.d(
+                "RecordingDataFileTest-trainingPipelineTest",
+                "Batching dataset: $progress%"
+            )
+        })
+
+        val accuracyBeforeTraining = model.evaluate(batches) { batch, window ->
+            Log.d(
+                "RecordingDataFileTest",
+                "Evaluating model before training. Batch: $batch%, Window Progress: $window%"
+            )
+        }
+        val losses = model.train(batches, 5) { epoch, batch, window ->
+            Log.d(
+                "RecordingDataFileTest",
+                "Training model. Epoch $epoch%, batch $batch%, window $window%"
+            )
+        }
+        Log.d(
+            "RecordingDataFileTest",
+            "Losses during training epochs: ${losses.joinToString()}"
+        )
+        val accuracyAfterTraining = model.evaluate(batches) { batch, window ->
+            Log.d(
+                "RecordingDataFileTest",
+                "Evaluating model after training. Batch: $batch%, Window Progress: $window%"
+            )
+        }
+        Log.d(
+            "RecordingDataFileTest",
+            "Prediction accuracy on the example recording before training: " +
+                    "$accuracyBeforeTraining and after training: $accuracyAfterTraining%"
+        )
+    }
+}

app/src/main/java/sensors_in_paradise/sonar/GlobalValues.kt

@@ -3,8 +3,13 @@ package sensors_in_paradise.sonar
 import android.Manifest
 import android.content.Context
 import android.os.Build
+import android.util.Log
 import com.xsens.dot.android.sdk.models.XsensDotPayload
+import java.io.BufferedReader
 import java.io.File
+import java.io.FileReader
+import java.net.NetworkInterface
+import java.net.SocketException
 import java.util.*
 import kotlin.collections.ArrayList
 
@@ -17,10 +22,6 @@ class GlobalValues private constructor() {
         const val METADATA_JSON_FILENAME = "metadata.json"
         const val MEASUREMENT_MODE = XsensDotPayload.PAYLOAD_TYPE_CUSTOM_MODE_4
 
-        fun getSensorRecordingsBaseDir(context: Context): File {
-            return context.getExternalFilesDir(null) ?: context.dataDir
-        }
-
         fun getSensorRecordingsTempDir(context: Context): File {
             return context.dataDir.resolve("temp")
         }
@@ -52,8 +53,6 @@ class GlobalValues private constructor() {
             return result
         }
 
-        val sensorTagPrefixes = listOf("LF", "LW", "ST", "RW", "RF")
-
         fun formatTag(tagPrefix: String, deviceSetKey: String): String {
             return "$tagPrefix-$deviceSetKey"
         }
@@ -66,6 +65,32 @@ class GlobalValues private constructor() {
 
             return minutes.toString().padStart(2, '0') + ":" + seconds.toString().padStart(2, '0')
         }
+
+        fun getMacAddress(): String {
+            try {
+                val all = Collections.list(NetworkInterface.getNetworkInterfaces())
+                for (nif in all) {
+                    if (!nif.name.equals("wlan0", ignoreCase=true)) continue
+
+                    val macBytes = nif.hardwareAddress ?: return ""
+
+                    val res1 = StringBuilder()
+                    for (b in macBytes) {
+                        res1.append(String.format(Locale.US, "%02X:", b))
+                    }
+
+                    if (res1.isNotEmpty()) {
+                        res1.deleteCharAt(res1.length - 1)
+                    }
+                    return res1.toString()
+                }
+            } catch (ex: SocketException) {
+                ex.message?.let { Log.e("GlobalValues", it) }
+            }
+
+            return "02:00:00:00:00:00"
+        }
+
         private val fileEmojiMap = mapOf(
             "mp4" to "\uD83C\uDF9E️",
             "json" to "\uD83D\uDCD8",
@@ -80,5 +105,14 @@ class GlobalValues private constructor() {
             val extension = name.substring(name.lastIndexOf(".") + 1)
             return fileEmojiMap[extension] ?: "\uD83D\uDCC4"
         }
+        @Throws(NumberFormatException::class)
+        fun getCSVHeaderAwareFileReader(inputFile: File): BufferedReader {
+            val fileReader = BufferedReader(FileReader(inputFile))
+            var line = fileReader.readLine()
+            while (line != "") {
+                line = fileReader.readLine()
+            }
+            return fileReader
+        }
     }
 }